├── .gitignore ├── JAVA8 ├── LICENSE ├── LICENSE.txt ├── LICENSE_GPL ├── NOTICE ├── README.md ├── pom.xml └── src ├── main ├── antlr4 │ ├── UberScriptQuerySql.g4 │ └── UberScriptQueryTemplate.g4 ├── java │ ├── com │ │ └── uber │ │ │ └── uberscriptquery │ │ │ ├── antlr4 │ │ │ ├── generated │ │ │ │ ├── UberScriptQuerySql.tokens │ │ │ │ ├── UberScriptQuerySqlBaseListener.java │ │ │ │ ├── UberScriptQuerySqlBaseVisitor.java │ │ │ │ ├── UberScriptQuerySqlLexer.java │ │ │ │ ├── UberScriptQuerySqlLexer.tokens │ │ │ │ ├── UberScriptQuerySqlListener.java │ │ │ │ ├── UberScriptQuerySqlParser.java │ │ │ │ ├── UberScriptQuerySqlVisitor.java │ │ │ │ ├── UberScriptQueryTemplate.tokens │ │ │ │ ├── UberScriptQueryTemplateBaseListener.java │ │ │ │ ├── UberScriptQueryTemplateBaseVisitor.java │ │ │ │ ├── UberScriptQueryTemplateLexer.java │ │ │ │ ├── UberScriptQueryTemplateLexer.tokens │ │ │ │ ├── UberScriptQueryTemplateListener.java │ │ │ │ ├── UberScriptQueryTemplateParser.java │ │ │ │ └── UberScriptQueryTemplateVisitor.java │ │ │ └── parsing │ │ │ │ ├── ActionParamValue.java │ │ │ │ ├── ActionStatement.java │ │ │ │ ├── FileAssignment.java │ │ │ │ ├── ParserUtils.java │ │ │ │ ├── QuerySqlParseResult.java │ │ │ │ ├── QuerySqlParser.java │ │ │ │ ├── QueryTemplateParseResult.java │ │ │ │ ├── QueryTemplateParser.java │ │ │ │ ├── RootStatement.java │ │ │ │ ├── StatementAssignment.java │ │ │ │ └── ValueType.java │ │ │ ├── examples │ │ │ ├── QueryExampleJob.java │ │ │ └── QueryExecutionExample.java │ │ │ ├── execution │ │ │ ├── ActionStatementExecutor.java │ │ │ ├── CredentialEntry.java │ │ │ ├── JdbcSqlInputStatementExecutor.java │ │ │ ├── JsonInputStatementExecutor.java │ │ │ ├── PrintTableActionStatementExecutor.java │ │ │ ├── QueryActionEngine.java │ │ │ ├── QueryEngine.java │ │ │ ├── SendMailGunEmailActionStatementExecutor.java │ │ │ ├── SqlInputStatementExecutor.java │ │ │ ├── WriteCsvFileActionStatementExecutor.java │ │ │ ├── WriteJdbcActionStatementExecutor.java │ │ │ ├── WriteJsonFileActionStatementExecutor.java │ │ │ └── WriteParquetFileActionStatementExecutor.java │ │ │ ├── jdbc │ │ │ ├── DataSetResult.java │ │ │ ├── JdbcConnectionProvider.java │ │ │ ├── JdbcUtils.java │ │ │ └── SingleTableJdbcWriter.java │ │ │ └── util │ │ │ ├── BufferedCredentialProvider.java │ │ │ ├── CounterMap.java │ │ │ ├── CredentialProvider.java │ │ │ ├── DateTimeUtils.java │ │ │ ├── DummyCredentialProvider.java │ │ │ ├── ExponentialBackoffRetryPolicy.java │ │ │ ├── FileUtils.java │ │ │ ├── HttpUtils.java │ │ │ ├── JsonFileCredentialProvider.java │ │ │ ├── JsonPathUtils.java │ │ │ ├── JsonUtils.java │ │ │ ├── NetworkUtils.java │ │ │ ├── ResourceUtils.java │ │ │ ├── RetryPolicy.java │ │ │ ├── SparkUtils.java │ │ │ ├── SqlUtils.java │ │ │ └── TemplateUtils.java │ └── org │ │ └── apache │ │ └── spark │ │ └── sql │ │ └── DriverRegistryWrapper.java ├── resources │ └── example.txt └── scala │ └── org │ └── apache │ └── spark │ └── sql │ ├── DataFrameJdbcWriter.scala │ └── execution │ └── datasources │ └── jdbc │ └── JdbcWriterUtils.scala └── test ├── java └── com │ └── uber │ └── uberscriptquery │ ├── antlr4 │ └── parsing │ │ ├── QueryFile001.txt │ │ ├── QuerySqlParserTest.java │ │ └── QueryTemplateParserTest.java │ ├── execution │ ├── QueryActionEngineTest.java │ └── QueryEngineTest.java │ ├── providers │ └── JsonFileCredentialProviderTest.java │ └── util │ ├── SparkUtilsTest.java │ └── SqlUtilsTest.java └── scala └── org └── apache └── spark └── sql └── DataFrameJdbcWriterTestSpec.scala /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | target/ 3 | *.iml 4 | 5 | dependency-reduced-pom.xml 6 | 7 | -------------------------------------------------------------------------------- /JAVA8: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/uber/uberscriptquery/4d876a2a8d4083629fcc473577f141eff7071dde/JAVA8 -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2015 [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | UberScriptQuery 2 | Copyright (c) 2017 Uber Technologies, Inc. 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | UberScriptQuery depends on MySQL Connector/J 17 | Copyright (c) 1998, 2017, Oracle and/or its affiliates 18 | 19 | The MySQL Connector/J is licensed under the terms of the GPLv2 20 | , like most MySQL Connectors. 21 | There are special exceptions to the terms and conditions of the GPLv2 as it is applied to 22 | this software, see the FOSS License Exception 23 | . 24 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | SparkScript 2 | Copyright (c) 2017 Uber Technologies, Inc. 3 | 4 | I. Included Software 5 | 6 | This product includes software developed at 7 | The Apache Software Foundation (http://www.apache.org/). 8 | Licensed under the Apache License 2.0. 9 | 10 | This product includes software developed at 11 | Spark (http://spark.apache.org/). 12 | Licensed under the Apache License 2.0. 13 | 14 | This product includes software developed at 15 | Hadoop (http://hadoop.apache.org/). 16 | Licensed under the Apache License 2.0. 17 | 18 | This product includes software developed at 19 | scala (http://www.scala-lang.org/). 20 | Licensed under the BSD 3-Clause License. 21 | 22 | This product includes software developed at 23 | freemarker (http://freemarker.org/). 24 | Licensed under the Apache License 2.0. 25 | 26 | This product includes software developed at 27 | jayway/JsonPath (https://github.com/jayway/JsonPath). 28 | Licensed under the Apache License 2.0. 29 | 30 | This product includes software developed at 31 | mysql connector (https://dev.mysql.com/downloads/connector/j/). 32 | Licensed under the GPL License. 33 | 34 | This software contains unmodified binary redistributions for 35 | H2 database engine (http://www.h2database.com/), 36 | which is dual licensed and available under the MPL 2.0 37 | (Mozilla Public License) or under the EPL 1.0 (Eclipse Public License). 38 | An original copy of the license agreement can be found at: 39 | http://www.h2database.com/html/license.html 40 | 41 | 42 | II. License Summary 43 | - Apache License 2.0 -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | UberScriptQuery 2 | ============ 3 | 4 | UberScriptQuery is a script query wrapper to run Spark SQL jobs. 5 | 6 | Why did we build this? Apache Spark is a great tool to do data processing, yet people usually end up writing many similar Spark jobs. There is substantial development cost to write and maintain all these jobs. Additionally, Spark is still mostly for developers, and other people such as data analysts or data scientists may still feel that Spark has a steep learning curve. 7 | 8 | To make Spark easier, we define a high level SQL-like DSL (Domain Specific Language) on top of Spark. People can use that DSL to write Spark jobs without worrying about Spark internal details. Another benefit to define such a DSL is to break up complicated logic or SQL query to a declarative script, which is easy to review and maintain. Our result? UberScriptQuery, a SQL-like DSL to make writing Spark jobs super easy. 9 | 10 | DSL Example 11 | ============ 12 | 13 | The following is a quick example for the UberScriptQuery DSL. It queries data from a MySQL database and Hadoop file, joins them together, and saves the result to another MySQL data table. 14 | 15 | ``` 16 | -- Define variables 17 | datePath = '2017/01/10'; 18 | 19 | -- Load data from mysql 20 | clients = sql jdbc set connectionString='jdbc:mysql://server:3306/database'; 21 | select clientId, clientName from dim_client; 22 | 23 | -- Load data from hadoop 24 | orders = file json hdfs:///dir/to/files/${datePath}/*; 25 | 26 | -- Join data from two tables 27 | result = select clientName, productName, totalCount, orderId, orderDescription 28 | from orders 29 | join clients on clients.clientId = orders.clientId; 30 | 31 | -- Write result to mysql output table 32 | -- The output table will be auto created if not exist, and will have: 33 | -- Primary Key Columns: clientName,productName,orderId 34 | -- Index Columns: clientName,orderId 35 | -- Text Columns: orderDescription 36 | 37 | writeJdbc('jdbc:mysql://server:3306/database', 38 | 'jdbcTable', 39 | 'clientName,productName,orderId', 40 | 'clientName,orderId', 41 | 'orderDescription', 42 | 'Append', 43 | result); 44 | 45 | -- Send email via MailGun 46 | sendMailGunEmail('https://api.mailgun.net/v3/sandbox549566ecba1d49fab0d7b53d4cfb01a4.mailgun.org/messages', 47 | 'MailGun_ApiKey', 48 | 'from_email@example.com', 49 | 'to_email@example.com', 50 | 'Email Title - Job Done', 51 | 'Email Content - Successfully queried data on ${datePath}'); 52 | 53 | ``` 54 | 55 | DSL Features 56 | ============ 57 | 58 | 1. **Flexible Input/Output**: it supports multiple input/output data sources with different formats, including database and Hadoop. It is also possible to add other data sources like Cassandra and Elasticsearch. 59 | 60 | 2. **Multiple SQL Statements**: it allows executing multiple SQL Statements, storing temporary results in another table, and referencing them in other SQL statements. This avoids a huge complicated single SQL statement, and makes the logic very clear and easy to maintain. 61 | 62 | 3. **Variable Substitution**: it allows defining variables with names/values, and substitute these variable in the script body. It also allows variable overwriting from outside of the script, so people can run the same script with different variable bindings. 63 | 64 | 4. **Custom Action**: it supports Actions like writeJdbc() in the previous DSL example. It also allows users to write their own Actions and plug into the script. 65 | 66 | 5. **Upsert Result to Database**: it implements an "upsert" based JDBC writer, and can insert/update database records in a single operation. This makes it easy to provide "Exactly Once" semantic support for Spark Jobs. 67 | 68 | Quick Start 69 | ============ 70 | 71 | Build this project with Maven with Java 1.8: 72 | ``` 73 | mvn package -DskipTests 74 | ``` 75 | 76 | Run the following command to execute your first UberScriptQuery job: 77 | ``` 78 | java -cp target/UberScriptQuery-1.1.01.jar com.uber.uberscriptquery.examples.QueryExampleJob \ 79 | -query "result = select cast(unix_timestamp() as timestamp) as time, 'Hello World' as message; printTable(result);" 80 | ``` 81 | 82 | The following is another example to run with variable overwriting (note we use '\${message}' in following command because of escaping $ in bash command, in programming code, it should be like '${message}'): 83 | ``` 84 | java -cp target/UberScriptQuery-1.1.01.jar com.uber.uberscriptquery.examples.QueryExampleJob \ 85 | -query "message = 'Hello World'; result = select cast(unix_timestamp() as timestamp) as time, '\${message}' as message; printTable(result);" \ 86 | -queryOverwrite "message = 'Hello New World';" 87 | ``` 88 | 89 | You could also integrate the UberScriptQuery Engine into your own code, and run the script in your own job: 90 | ``` 91 | QueryEngine engine = new QueryEngine(); 92 | engine.executeScript(script, sparkSession); 93 | ``` 94 | 95 | There are more detailed sample codes in this class: 96 | 97 | ``` 98 | com.uber.uberscriptquery.examples.QueryExecutionExample 99 | ``` 100 | 101 | Future Work 102 | ============ 103 | 104 | 1. Support more data sources, e.g. Cassandra and Elasticsearch. 105 | 106 | 2. Support "upsert" into more databases like PostgreSQL. Now it only supports MySQL and H2 database. 107 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/generated/UberScriptQuerySql.tokens: -------------------------------------------------------------------------------- 1 | T__0=1 2 | T__1=2 3 | T__2=3 4 | T__3=4 5 | T__4=5 6 | T__5=6 7 | T__6=7 8 | T__7=8 9 | T__8=9 10 | T__9=10 11 | T__10=11 12 | T__11=12 13 | T__12=13 14 | URL_STRING=14 15 | JSON=15 16 | TEXT=16 17 | CSV=17 18 | PARQUET=18 19 | SELECT=19 20 | FROM=20 21 | ADD=21 22 | AS=22 23 | ALL=23 24 | DISTINCT=24 25 | WHERE=25 26 | GROUP=26 27 | BY=27 28 | GROUPING=28 29 | SETS=29 30 | CUBE=30 31 | ROLLUP=31 32 | ORDER=32 33 | HAVING=33 34 | LIMIT=34 35 | AT=35 36 | OR=36 37 | AND=37 38 | IN=38 39 | NOT=39 40 | NO=40 41 | EXISTS=41 42 | BETWEEN=42 43 | LIKE=43 44 | RLIKE=44 45 | IS=45 46 | NULL=46 47 | TRUE=47 48 | FALSE=48 49 | NULLS=49 50 | ASC=50 51 | DESC=51 52 | FOR=52 53 | INTERVAL=53 54 | CASE=54 55 | WHEN=55 56 | THEN=56 57 | ELSE=57 58 | END=58 59 | JOIN=59 60 | CROSS=60 61 | OUTER=61 62 | INNER=62 63 | LEFT=63 64 | SEMI=64 65 | RIGHT=65 66 | FULL=66 67 | NATURAL=67 68 | ON=68 69 | LATERAL=69 70 | WINDOW=70 71 | OVER=71 72 | PARTITION=72 73 | RANGE=73 74 | ROWS=74 75 | UNBOUNDED=75 76 | PRECEDING=76 77 | FOLLOWING=77 78 | CURRENT=78 79 | ROW=79 80 | WITH=80 81 | VALUES=81 82 | CREATE=82 83 | TABLE=83 84 | VIEW=84 85 | REPLACE=85 86 | INSERT=86 87 | DELETE=87 88 | INTO=88 89 | DESCRIBE=89 90 | EXPLAIN=90 91 | FORMAT=91 92 | LOGICAL=92 93 | CODEGEN=93 94 | CAST=94 95 | SHOW=95 96 | TABLES=96 97 | COLUMNS=97 98 | COLUMN=98 99 | USE=99 100 | PARTITIONS=100 101 | FUNCTIONS=101 102 | DROP=102 103 | UNION=103 104 | EXCEPT=104 105 | INTERSECT=105 106 | TO=106 107 | TABLESAMPLE=107 108 | STRATIFY=108 109 | ALTER=109 110 | RENAME=110 111 | ARRAY=111 112 | MAP=112 113 | STRUCT=113 114 | COMMENT=114 115 | SET=115 116 | RESET=116 117 | DATA=117 118 | START=118 119 | TRANSACTION=119 120 | COMMIT=120 121 | ROLLBACK=121 122 | MACRO=122 123 | IF=123 124 | EQ=124 125 | NSEQ=125 126 | NEQ=126 127 | NEQJ=127 128 | LT=128 129 | LTE=129 130 | GT=130 131 | GTE=131 132 | PLUS=132 133 | MINUS=133 134 | ASTERISK=134 135 | SLASH=135 136 | PERCENT=136 137 | DIV=137 138 | TILDE=138 139 | AMPERSAND=139 140 | PIPE=140 141 | HAT=141 142 | PERCENTLIT=142 143 | BUCKET=143 144 | OUT=144 145 | OF=145 146 | SORT=146 147 | CLUSTER=147 148 | DISTRIBUTE=148 149 | OVERWRITE=149 150 | TRANSFORM=150 151 | REDUCE=151 152 | USING=152 153 | SERDE=153 154 | SERDEPROPERTIES=154 155 | RECORDREADER=155 156 | RECORDWRITER=156 157 | DELIMITED=157 158 | FIELDS=158 159 | TERMINATED=159 160 | COLLECTION=160 161 | ITEMS=161 162 | KEYS=162 163 | ESCAPED=163 164 | LINES=164 165 | SEPARATED=165 166 | FUNCTION=166 167 | EXTENDED=167 168 | REFRESH=168 169 | CLEAR=169 170 | CACHE=170 171 | UNCACHE=171 172 | LAZY=172 173 | FORMATTED=173 174 | TEMPORARY=174 175 | OPTIONS=175 176 | UNSET=176 177 | TBLPROPERTIES=177 178 | DBPROPERTIES=178 179 | BUCKETS=179 180 | SKEWED=180 181 | STORED=181 182 | DIRECTORIES=182 183 | LOCATION=183 184 | EXCHANGE=184 185 | ARCHIVE=185 186 | UNARCHIVE=186 187 | FILEFORMAT=187 188 | TOUCH=188 189 | COMPACT=189 190 | CONCATENATE=190 191 | CHANGE=191 192 | CASCADE=192 193 | RESTRICT=193 194 | CLUSTERED=194 195 | SORTED=195 196 | PURGE=196 197 | INPUTFORMAT=197 198 | OUTPUTFORMAT=198 199 | DATABASE=199 200 | DATABASES=200 201 | DFS=201 202 | TRUNCATE=202 203 | ANALYZE=203 204 | COMPUTE=204 205 | LIST=205 206 | STATISTICS=206 207 | PARTITIONED=207 208 | EXTERNAL=208 209 | DEFINED=209 210 | REVOKE=210 211 | GRANT=211 212 | LOCK=212 213 | UNLOCK=213 214 | MSCK=214 215 | REPAIR=215 216 | EXPORT=216 217 | IMPORT=217 218 | LOAD=218 219 | ROLE=219 220 | ROLES=220 221 | COMPACTIONS=221 222 | PRINCIPALS=222 223 | TRANSACTIONS=223 224 | INDEX=224 225 | INDEXES=225 226 | LOCKS=226 227 | OPTION=227 228 | ANTI=228 229 | LOCAL=229 230 | INPATH=230 231 | STRING=231 232 | BIGINT_LITERAL=232 233 | SMALLINT_LITERAL=233 234 | TINYINT_LITERAL=234 235 | BYTELENGTH_LITERAL=235 236 | INTEGER_VALUE=236 237 | DECIMAL_VALUE=237 238 | SCIENTIFIC_DECIMAL_VALUE=238 239 | DOUBLE_LITERAL=239 240 | IDENTIFIER=240 241 | BACKQUOTED_IDENTIFIER=241 242 | SIMPLE_COMMENT=242 243 | BRACKETED_COMMENT=243 244 | WS=244 245 | UNRECOGNIZED=245 246 | DELIMITER=246 247 | ';'=1 248 | 'SQL'=2 249 | 'DATAGEN'=3 250 | 'FILE'=4 251 | '('=5 252 | ')'=6 253 | ','=7 254 | '{'=8 255 | '}'=9 256 | '['=10 257 | ']'=11 258 | ':'=12 259 | '.'=13 260 | 'JSON'=15 261 | 'TEXT'=16 262 | 'CSV'=17 263 | 'PARQUET'=18 264 | 'SELECT'=19 265 | 'FROM'=20 266 | 'ADD'=21 267 | 'AS'=22 268 | 'ALL'=23 269 | 'DISTINCT'=24 270 | 'WHERE'=25 271 | 'GROUP'=26 272 | 'BY'=27 273 | 'GROUPING'=28 274 | 'SETS'=29 275 | 'CUBE'=30 276 | 'ROLLUP'=31 277 | 'ORDER'=32 278 | 'HAVING'=33 279 | 'LIMIT'=34 280 | 'AT'=35 281 | 'OR'=36 282 | 'AND'=37 283 | 'IN'=38 284 | 'NO'=40 285 | 'EXISTS'=41 286 | 'BETWEEN'=42 287 | 'LIKE'=43 288 | 'IS'=45 289 | 'NULL'=46 290 | 'TRUE'=47 291 | 'FALSE'=48 292 | 'NULLS'=49 293 | 'ASC'=50 294 | 'DESC'=51 295 | 'FOR'=52 296 | 'INTERVAL'=53 297 | 'CASE'=54 298 | 'WHEN'=55 299 | 'THEN'=56 300 | 'ELSE'=57 301 | 'END'=58 302 | 'JOIN'=59 303 | 'CROSS'=60 304 | 'OUTER'=61 305 | 'INNER'=62 306 | 'LEFT'=63 307 | 'SEMI'=64 308 | 'RIGHT'=65 309 | 'FULL'=66 310 | 'NATURAL'=67 311 | 'ON'=68 312 | 'LATERAL'=69 313 | 'WINDOW'=70 314 | 'OVER'=71 315 | 'PARTITION'=72 316 | 'RANGE'=73 317 | 'ROWS'=74 318 | 'UNBOUNDED'=75 319 | 'PRECEDING'=76 320 | 'FOLLOWING'=77 321 | 'CURRENT'=78 322 | 'ROW'=79 323 | 'WITH'=80 324 | 'VALUES'=81 325 | 'CREATE'=82 326 | 'TABLE'=83 327 | 'VIEW'=84 328 | 'REPLACE'=85 329 | 'INSERT'=86 330 | 'DELETE'=87 331 | 'INTO'=88 332 | 'DESCRIBE'=89 333 | 'EXPLAIN'=90 334 | 'FORMAT'=91 335 | 'LOGICAL'=92 336 | 'CODEGEN'=93 337 | 'CAST'=94 338 | 'SHOW'=95 339 | 'TABLES'=96 340 | 'COLUMNS'=97 341 | 'COLUMN'=98 342 | 'USE'=99 343 | 'PARTITIONS'=100 344 | 'FUNCTIONS'=101 345 | 'DROP'=102 346 | 'UNION'=103 347 | 'EXCEPT'=104 348 | 'INTERSECT'=105 349 | 'TO'=106 350 | 'TABLESAMPLE'=107 351 | 'STRATIFY'=108 352 | 'ALTER'=109 353 | 'RENAME'=110 354 | 'ARRAY'=111 355 | 'MAP'=112 356 | 'STRUCT'=113 357 | 'COMMENT'=114 358 | 'SET'=115 359 | 'RESET'=116 360 | 'DATA'=117 361 | 'START'=118 362 | 'TRANSACTION'=119 363 | 'COMMIT'=120 364 | 'ROLLBACK'=121 365 | 'MACRO'=122 366 | 'IF'=123 367 | '<=>'=125 368 | '<>'=126 369 | '!='=127 370 | '<'=128 371 | '>'=130 372 | '+'=132 373 | '-'=133 374 | '*'=134 375 | '/'=135 376 | '%'=136 377 | 'DIV'=137 378 | '~'=138 379 | '&'=139 380 | '|'=140 381 | '^'=141 382 | 'PERCENT'=142 383 | 'BUCKET'=143 384 | 'OUT'=144 385 | 'OF'=145 386 | 'SORT'=146 387 | 'CLUSTER'=147 388 | 'DISTRIBUTE'=148 389 | 'OVERWRITE'=149 390 | 'TRANSFORM'=150 391 | 'REDUCE'=151 392 | 'USING'=152 393 | 'SERDE'=153 394 | 'SERDEPROPERTIES'=154 395 | 'RECORDREADER'=155 396 | 'RECORDWRITER'=156 397 | 'DELIMITED'=157 398 | 'FIELDS'=158 399 | 'TERMINATED'=159 400 | 'COLLECTION'=160 401 | 'ITEMS'=161 402 | 'KEYS'=162 403 | 'ESCAPED'=163 404 | 'LINES'=164 405 | 'SEPARATED'=165 406 | 'FUNCTION'=166 407 | 'EXTENDED'=167 408 | 'REFRESH'=168 409 | 'CLEAR'=169 410 | 'CACHE'=170 411 | 'UNCACHE'=171 412 | 'LAZY'=172 413 | 'FORMATTED'=173 414 | 'OPTIONS'=175 415 | 'UNSET'=176 416 | 'TBLPROPERTIES'=177 417 | 'DBPROPERTIES'=178 418 | 'BUCKETS'=179 419 | 'SKEWED'=180 420 | 'STORED'=181 421 | 'DIRECTORIES'=182 422 | 'LOCATION'=183 423 | 'EXCHANGE'=184 424 | 'ARCHIVE'=185 425 | 'UNARCHIVE'=186 426 | 'FILEFORMAT'=187 427 | 'TOUCH'=188 428 | 'COMPACT'=189 429 | 'CONCATENATE'=190 430 | 'CHANGE'=191 431 | 'CASCADE'=192 432 | 'RESTRICT'=193 433 | 'CLUSTERED'=194 434 | 'SORTED'=195 435 | 'PURGE'=196 436 | 'INPUTFORMAT'=197 437 | 'OUTPUTFORMAT'=198 438 | 'DFS'=201 439 | 'TRUNCATE'=202 440 | 'ANALYZE'=203 441 | 'COMPUTE'=204 442 | 'LIST'=205 443 | 'STATISTICS'=206 444 | 'PARTITIONED'=207 445 | 'EXTERNAL'=208 446 | 'DEFINED'=209 447 | 'REVOKE'=210 448 | 'GRANT'=211 449 | 'LOCK'=212 450 | 'UNLOCK'=213 451 | 'MSCK'=214 452 | 'REPAIR'=215 453 | 'EXPORT'=216 454 | 'IMPORT'=217 455 | 'LOAD'=218 456 | 'ROLE'=219 457 | 'ROLES'=220 458 | 'COMPACTIONS'=221 459 | 'PRINCIPALS'=222 460 | 'TRANSACTIONS'=223 461 | 'INDEX'=224 462 | 'INDEXES'=225 463 | 'LOCKS'=226 464 | 'OPTION'=227 465 | 'ANTI'=228 466 | 'LOCAL'=229 467 | 'INPATH'=230 468 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/generated/UberScriptQuerySqlLexer.tokens: -------------------------------------------------------------------------------- 1 | T__0=1 2 | T__1=2 3 | T__2=3 4 | T__3=4 5 | T__4=5 6 | T__5=6 7 | T__6=7 8 | T__7=8 9 | T__8=9 10 | T__9=10 11 | T__10=11 12 | T__11=12 13 | T__12=13 14 | URL_STRING=14 15 | JSON=15 16 | TEXT=16 17 | CSV=17 18 | PARQUET=18 19 | SELECT=19 20 | FROM=20 21 | ADD=21 22 | AS=22 23 | ALL=23 24 | DISTINCT=24 25 | WHERE=25 26 | GROUP=26 27 | BY=27 28 | GROUPING=28 29 | SETS=29 30 | CUBE=30 31 | ROLLUP=31 32 | ORDER=32 33 | HAVING=33 34 | LIMIT=34 35 | AT=35 36 | OR=36 37 | AND=37 38 | IN=38 39 | NOT=39 40 | NO=40 41 | EXISTS=41 42 | BETWEEN=42 43 | LIKE=43 44 | RLIKE=44 45 | IS=45 46 | NULL=46 47 | TRUE=47 48 | FALSE=48 49 | NULLS=49 50 | ASC=50 51 | DESC=51 52 | FOR=52 53 | INTERVAL=53 54 | CASE=54 55 | WHEN=55 56 | THEN=56 57 | ELSE=57 58 | END=58 59 | JOIN=59 60 | CROSS=60 61 | OUTER=61 62 | INNER=62 63 | LEFT=63 64 | SEMI=64 65 | RIGHT=65 66 | FULL=66 67 | NATURAL=67 68 | ON=68 69 | LATERAL=69 70 | WINDOW=70 71 | OVER=71 72 | PARTITION=72 73 | RANGE=73 74 | ROWS=74 75 | UNBOUNDED=75 76 | PRECEDING=76 77 | FOLLOWING=77 78 | CURRENT=78 79 | ROW=79 80 | WITH=80 81 | VALUES=81 82 | CREATE=82 83 | TABLE=83 84 | VIEW=84 85 | REPLACE=85 86 | INSERT=86 87 | DELETE=87 88 | INTO=88 89 | DESCRIBE=89 90 | EXPLAIN=90 91 | FORMAT=91 92 | LOGICAL=92 93 | CODEGEN=93 94 | CAST=94 95 | SHOW=95 96 | TABLES=96 97 | COLUMNS=97 98 | COLUMN=98 99 | USE=99 100 | PARTITIONS=100 101 | FUNCTIONS=101 102 | DROP=102 103 | UNION=103 104 | EXCEPT=104 105 | INTERSECT=105 106 | TO=106 107 | TABLESAMPLE=107 108 | STRATIFY=108 109 | ALTER=109 110 | RENAME=110 111 | ARRAY=111 112 | MAP=112 113 | STRUCT=113 114 | COMMENT=114 115 | SET=115 116 | RESET=116 117 | DATA=117 118 | START=118 119 | TRANSACTION=119 120 | COMMIT=120 121 | ROLLBACK=121 122 | MACRO=122 123 | IF=123 124 | EQ=124 125 | NSEQ=125 126 | NEQ=126 127 | NEQJ=127 128 | LT=128 129 | LTE=129 130 | GT=130 131 | GTE=131 132 | PLUS=132 133 | MINUS=133 134 | ASTERISK=134 135 | SLASH=135 136 | PERCENT=136 137 | DIV=137 138 | TILDE=138 139 | AMPERSAND=139 140 | PIPE=140 141 | HAT=141 142 | PERCENTLIT=142 143 | BUCKET=143 144 | OUT=144 145 | OF=145 146 | SORT=146 147 | CLUSTER=147 148 | DISTRIBUTE=148 149 | OVERWRITE=149 150 | TRANSFORM=150 151 | REDUCE=151 152 | USING=152 153 | SERDE=153 154 | SERDEPROPERTIES=154 155 | RECORDREADER=155 156 | RECORDWRITER=156 157 | DELIMITED=157 158 | FIELDS=158 159 | TERMINATED=159 160 | COLLECTION=160 161 | ITEMS=161 162 | KEYS=162 163 | ESCAPED=163 164 | LINES=164 165 | SEPARATED=165 166 | FUNCTION=166 167 | EXTENDED=167 168 | REFRESH=168 169 | CLEAR=169 170 | CACHE=170 171 | UNCACHE=171 172 | LAZY=172 173 | FORMATTED=173 174 | TEMPORARY=174 175 | OPTIONS=175 176 | UNSET=176 177 | TBLPROPERTIES=177 178 | DBPROPERTIES=178 179 | BUCKETS=179 180 | SKEWED=180 181 | STORED=181 182 | DIRECTORIES=182 183 | LOCATION=183 184 | EXCHANGE=184 185 | ARCHIVE=185 186 | UNARCHIVE=186 187 | FILEFORMAT=187 188 | TOUCH=188 189 | COMPACT=189 190 | CONCATENATE=190 191 | CHANGE=191 192 | CASCADE=192 193 | RESTRICT=193 194 | CLUSTERED=194 195 | SORTED=195 196 | PURGE=196 197 | INPUTFORMAT=197 198 | OUTPUTFORMAT=198 199 | DATABASE=199 200 | DATABASES=200 201 | DFS=201 202 | TRUNCATE=202 203 | ANALYZE=203 204 | COMPUTE=204 205 | LIST=205 206 | STATISTICS=206 207 | PARTITIONED=207 208 | EXTERNAL=208 209 | DEFINED=209 210 | REVOKE=210 211 | GRANT=211 212 | LOCK=212 213 | UNLOCK=213 214 | MSCK=214 215 | REPAIR=215 216 | EXPORT=216 217 | IMPORT=217 218 | LOAD=218 219 | ROLE=219 220 | ROLES=220 221 | COMPACTIONS=221 222 | PRINCIPALS=222 223 | TRANSACTIONS=223 224 | INDEX=224 225 | INDEXES=225 226 | LOCKS=226 227 | OPTION=227 228 | ANTI=228 229 | LOCAL=229 230 | INPATH=230 231 | STRING=231 232 | BIGINT_LITERAL=232 233 | SMALLINT_LITERAL=233 234 | TINYINT_LITERAL=234 235 | BYTELENGTH_LITERAL=235 236 | INTEGER_VALUE=236 237 | DECIMAL_VALUE=237 238 | SCIENTIFIC_DECIMAL_VALUE=238 239 | DOUBLE_LITERAL=239 240 | IDENTIFIER=240 241 | BACKQUOTED_IDENTIFIER=241 242 | SIMPLE_COMMENT=242 243 | BRACKETED_COMMENT=243 244 | WS=244 245 | UNRECOGNIZED=245 246 | ';'=1 247 | 'SQL'=2 248 | 'DATAGEN'=3 249 | 'FILE'=4 250 | '('=5 251 | ')'=6 252 | ','=7 253 | '{'=8 254 | '}'=9 255 | '['=10 256 | ']'=11 257 | ':'=12 258 | '.'=13 259 | 'JSON'=15 260 | 'TEXT'=16 261 | 'CSV'=17 262 | 'PARQUET'=18 263 | 'SELECT'=19 264 | 'FROM'=20 265 | 'ADD'=21 266 | 'AS'=22 267 | 'ALL'=23 268 | 'DISTINCT'=24 269 | 'WHERE'=25 270 | 'GROUP'=26 271 | 'BY'=27 272 | 'GROUPING'=28 273 | 'SETS'=29 274 | 'CUBE'=30 275 | 'ROLLUP'=31 276 | 'ORDER'=32 277 | 'HAVING'=33 278 | 'LIMIT'=34 279 | 'AT'=35 280 | 'OR'=36 281 | 'AND'=37 282 | 'IN'=38 283 | 'NO'=40 284 | 'EXISTS'=41 285 | 'BETWEEN'=42 286 | 'LIKE'=43 287 | 'IS'=45 288 | 'NULL'=46 289 | 'TRUE'=47 290 | 'FALSE'=48 291 | 'NULLS'=49 292 | 'ASC'=50 293 | 'DESC'=51 294 | 'FOR'=52 295 | 'INTERVAL'=53 296 | 'CASE'=54 297 | 'WHEN'=55 298 | 'THEN'=56 299 | 'ELSE'=57 300 | 'END'=58 301 | 'JOIN'=59 302 | 'CROSS'=60 303 | 'OUTER'=61 304 | 'INNER'=62 305 | 'LEFT'=63 306 | 'SEMI'=64 307 | 'RIGHT'=65 308 | 'FULL'=66 309 | 'NATURAL'=67 310 | 'ON'=68 311 | 'LATERAL'=69 312 | 'WINDOW'=70 313 | 'OVER'=71 314 | 'PARTITION'=72 315 | 'RANGE'=73 316 | 'ROWS'=74 317 | 'UNBOUNDED'=75 318 | 'PRECEDING'=76 319 | 'FOLLOWING'=77 320 | 'CURRENT'=78 321 | 'ROW'=79 322 | 'WITH'=80 323 | 'VALUES'=81 324 | 'CREATE'=82 325 | 'TABLE'=83 326 | 'VIEW'=84 327 | 'REPLACE'=85 328 | 'INSERT'=86 329 | 'DELETE'=87 330 | 'INTO'=88 331 | 'DESCRIBE'=89 332 | 'EXPLAIN'=90 333 | 'FORMAT'=91 334 | 'LOGICAL'=92 335 | 'CODEGEN'=93 336 | 'CAST'=94 337 | 'SHOW'=95 338 | 'TABLES'=96 339 | 'COLUMNS'=97 340 | 'COLUMN'=98 341 | 'USE'=99 342 | 'PARTITIONS'=100 343 | 'FUNCTIONS'=101 344 | 'DROP'=102 345 | 'UNION'=103 346 | 'EXCEPT'=104 347 | 'INTERSECT'=105 348 | 'TO'=106 349 | 'TABLESAMPLE'=107 350 | 'STRATIFY'=108 351 | 'ALTER'=109 352 | 'RENAME'=110 353 | 'ARRAY'=111 354 | 'MAP'=112 355 | 'STRUCT'=113 356 | 'COMMENT'=114 357 | 'SET'=115 358 | 'RESET'=116 359 | 'DATA'=117 360 | 'START'=118 361 | 'TRANSACTION'=119 362 | 'COMMIT'=120 363 | 'ROLLBACK'=121 364 | 'MACRO'=122 365 | 'IF'=123 366 | '<=>'=125 367 | '<>'=126 368 | '!='=127 369 | '<'=128 370 | '>'=130 371 | '+'=132 372 | '-'=133 373 | '*'=134 374 | '/'=135 375 | '%'=136 376 | 'DIV'=137 377 | '~'=138 378 | '&'=139 379 | '|'=140 380 | '^'=141 381 | 'PERCENT'=142 382 | 'BUCKET'=143 383 | 'OUT'=144 384 | 'OF'=145 385 | 'SORT'=146 386 | 'CLUSTER'=147 387 | 'DISTRIBUTE'=148 388 | 'OVERWRITE'=149 389 | 'TRANSFORM'=150 390 | 'REDUCE'=151 391 | 'USING'=152 392 | 'SERDE'=153 393 | 'SERDEPROPERTIES'=154 394 | 'RECORDREADER'=155 395 | 'RECORDWRITER'=156 396 | 'DELIMITED'=157 397 | 'FIELDS'=158 398 | 'TERMINATED'=159 399 | 'COLLECTION'=160 400 | 'ITEMS'=161 401 | 'KEYS'=162 402 | 'ESCAPED'=163 403 | 'LINES'=164 404 | 'SEPARATED'=165 405 | 'FUNCTION'=166 406 | 'EXTENDED'=167 407 | 'REFRESH'=168 408 | 'CLEAR'=169 409 | 'CACHE'=170 410 | 'UNCACHE'=171 411 | 'LAZY'=172 412 | 'FORMATTED'=173 413 | 'OPTIONS'=175 414 | 'UNSET'=176 415 | 'TBLPROPERTIES'=177 416 | 'DBPROPERTIES'=178 417 | 'BUCKETS'=179 418 | 'SKEWED'=180 419 | 'STORED'=181 420 | 'DIRECTORIES'=182 421 | 'LOCATION'=183 422 | 'EXCHANGE'=184 423 | 'ARCHIVE'=185 424 | 'UNARCHIVE'=186 425 | 'FILEFORMAT'=187 426 | 'TOUCH'=188 427 | 'COMPACT'=189 428 | 'CONCATENATE'=190 429 | 'CHANGE'=191 430 | 'CASCADE'=192 431 | 'RESTRICT'=193 432 | 'CLUSTERED'=194 433 | 'SORTED'=195 434 | 'PURGE'=196 435 | 'INPUTFORMAT'=197 436 | 'OUTPUTFORMAT'=198 437 | 'DFS'=201 438 | 'TRUNCATE'=202 439 | 'ANALYZE'=203 440 | 'COMPUTE'=204 441 | 'LIST'=205 442 | 'STATISTICS'=206 443 | 'PARTITIONED'=207 444 | 'EXTERNAL'=208 445 | 'DEFINED'=209 446 | 'REVOKE'=210 447 | 'GRANT'=211 448 | 'LOCK'=212 449 | 'UNLOCK'=213 450 | 'MSCK'=214 451 | 'REPAIR'=215 452 | 'EXPORT'=216 453 | 'IMPORT'=217 454 | 'LOAD'=218 455 | 'ROLE'=219 456 | 'ROLES'=220 457 | 'COMPACTIONS'=221 458 | 'PRINCIPALS'=222 459 | 'TRANSACTIONS'=223 460 | 'INDEX'=224 461 | 'INDEXES'=225 462 | 'LOCKS'=226 463 | 'OPTION'=227 464 | 'ANTI'=228 465 | 'LOCAL'=229 466 | 'INPATH'=230 467 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/generated/UberScriptQueryTemplate.tokens: -------------------------------------------------------------------------------- 1 | T__0=1 2 | T__1=2 3 | T__2=3 4 | T__3=4 5 | T__4=5 6 | T__5=6 7 | T__6=7 8 | T__7=8 9 | SELECT=9 10 | FROM=10 11 | ADD=11 12 | AS=12 13 | ALL=13 14 | DISTINCT=14 15 | WHERE=15 16 | GROUP=16 17 | BY=17 18 | GROUPING=18 19 | SETS=19 20 | CUBE=20 21 | ROLLUP=21 22 | ORDER=22 23 | HAVING=23 24 | LIMIT=24 25 | AT=25 26 | OR=26 27 | AND=27 28 | IN=28 29 | NOT=29 30 | NO=30 31 | EXISTS=31 32 | BETWEEN=32 33 | LIKE=33 34 | RLIKE=34 35 | IS=35 36 | NULL=36 37 | TRUE=37 38 | FALSE=38 39 | NULLS=39 40 | ASC=40 41 | DESC=41 42 | FOR=42 43 | INTERVAL=43 44 | CASE=44 45 | WHEN=45 46 | THEN=46 47 | ELSE=47 48 | END=48 49 | JOIN=49 50 | CROSS=50 51 | OUTER=51 52 | INNER=52 53 | LEFT=53 54 | SEMI=54 55 | RIGHT=55 56 | FULL=56 57 | NATURAL=57 58 | ON=58 59 | LATERAL=59 60 | WINDOW=60 61 | OVER=61 62 | PARTITION=62 63 | RANGE=63 64 | ROWS=64 65 | UNBOUNDED=65 66 | PRECEDING=66 67 | FOLLOWING=67 68 | CURRENT=68 69 | ROW=69 70 | WITH=70 71 | VALUES=71 72 | CREATE=72 73 | TABLE=73 74 | VIEW=74 75 | REPLACE=75 76 | INSERT=76 77 | DELETE=77 78 | INTO=78 79 | DESCRIBE=79 80 | EXPLAIN=80 81 | FORMAT=81 82 | LOGICAL=82 83 | CODEGEN=83 84 | CAST=84 85 | SHOW=85 86 | TABLES=86 87 | COLUMNS=87 88 | COLUMN=88 89 | USE=89 90 | PARTITIONS=90 91 | FUNCTIONS=91 92 | DROP=92 93 | UNION=93 94 | EXCEPT=94 95 | INTERSECT=95 96 | TO=96 97 | TABLESAMPLE=97 98 | STRATIFY=98 99 | ALTER=99 100 | RENAME=100 101 | ARRAY=101 102 | MAP=102 103 | STRUCT=103 104 | COMMENT=104 105 | SET=105 106 | RESET=106 107 | DATA=107 108 | START=108 109 | TRANSACTION=109 110 | COMMIT=110 111 | ROLLBACK=111 112 | MACRO=112 113 | IF=113 114 | EQ=114 115 | NSEQ=115 116 | NEQ=116 117 | NEQJ=117 118 | LT=118 119 | LTE=119 120 | GT=120 121 | GTE=121 122 | PLUS=122 123 | MINUS=123 124 | ASTERISK=124 125 | SLASH=125 126 | PERCENT=126 127 | DIV=127 128 | TILDE=128 129 | AMPERSAND=129 130 | PIPE=130 131 | HAT=131 132 | PERCENTLIT=132 133 | BUCKET=133 134 | OUT=134 135 | OF=135 136 | SORT=136 137 | CLUSTER=137 138 | DISTRIBUTE=138 139 | OVERWRITE=139 140 | TRANSFORM=140 141 | REDUCE=141 142 | USING=142 143 | SERDE=143 144 | SERDEPROPERTIES=144 145 | RECORDREADER=145 146 | RECORDWRITER=146 147 | DELIMITED=147 148 | FIELDS=148 149 | TERMINATED=149 150 | COLLECTION=150 151 | ITEMS=151 152 | KEYS=152 153 | ESCAPED=153 154 | LINES=154 155 | SEPARATED=155 156 | FUNCTION=156 157 | EXTENDED=157 158 | REFRESH=158 159 | CLEAR=159 160 | CACHE=160 161 | UNCACHE=161 162 | LAZY=162 163 | FORMATTED=163 164 | TEMPORARY=164 165 | OPTIONS=165 166 | UNSET=166 167 | TBLPROPERTIES=167 168 | DBPROPERTIES=168 169 | BUCKETS=169 170 | SKEWED=170 171 | STORED=171 172 | DIRECTORIES=172 173 | LOCATION=173 174 | EXCHANGE=174 175 | ARCHIVE=175 176 | UNARCHIVE=176 177 | FILEFORMAT=177 178 | TOUCH=178 179 | COMPACT=179 180 | CONCATENATE=180 181 | CHANGE=181 182 | CASCADE=182 183 | RESTRICT=183 184 | CLUSTERED=184 185 | SORTED=185 186 | PURGE=186 187 | INPUTFORMAT=187 188 | OUTPUTFORMAT=188 189 | DATABASE=189 190 | DATABASES=190 191 | DFS=191 192 | TRUNCATE=192 193 | ANALYZE=193 194 | COMPUTE=194 195 | LIST=195 196 | STATISTICS=196 197 | PARTITIONED=197 198 | EXTERNAL=198 199 | DEFINED=199 200 | REVOKE=200 201 | GRANT=201 202 | LOCK=202 203 | UNLOCK=203 204 | MSCK=204 205 | REPAIR=205 206 | EXPORT=206 207 | IMPORT=207 208 | LOAD=208 209 | ROLE=209 210 | ROLES=210 211 | COMPACTIONS=211 212 | PRINCIPALS=212 213 | TRANSACTIONS=213 214 | INDEX=214 215 | INDEXES=215 216 | LOCKS=216 217 | OPTION=217 218 | ANTI=218 219 | LOCAL=219 220 | INPATH=220 221 | STRING=221 222 | BIGINT_LITERAL=222 223 | SMALLINT_LITERAL=223 224 | TINYINT_LITERAL=224 225 | BYTELENGTH_LITERAL=225 226 | INTEGER_VALUE=226 227 | DECIMAL_VALUE=227 228 | SCIENTIFIC_DECIMAL_VALUE=228 229 | DOUBLE_LITERAL=229 230 | IDENTIFIER=230 231 | BACKQUOTED_IDENTIFIER=231 232 | SIMPLE_COMMENT=232 233 | BRACKETED_COMMENT=233 234 | WS=234 235 | UNRECOGNIZED=235 236 | DELIMITER=236 237 | ';'=1 238 | '('=2 239 | ')'=3 240 | ','=4 241 | '.'=5 242 | '['=6 243 | ']'=7 244 | ':'=8 245 | 'SELECT'=9 246 | 'FROM'=10 247 | 'ADD'=11 248 | 'AS'=12 249 | 'ALL'=13 250 | 'DISTINCT'=14 251 | 'WHERE'=15 252 | 'GROUP'=16 253 | 'BY'=17 254 | 'GROUPING'=18 255 | 'SETS'=19 256 | 'CUBE'=20 257 | 'ROLLUP'=21 258 | 'ORDER'=22 259 | 'HAVING'=23 260 | 'LIMIT'=24 261 | 'AT'=25 262 | 'OR'=26 263 | 'AND'=27 264 | 'IN'=28 265 | 'NO'=30 266 | 'EXISTS'=31 267 | 'BETWEEN'=32 268 | 'LIKE'=33 269 | 'IS'=35 270 | 'NULL'=36 271 | 'TRUE'=37 272 | 'FALSE'=38 273 | 'NULLS'=39 274 | 'ASC'=40 275 | 'DESC'=41 276 | 'FOR'=42 277 | 'INTERVAL'=43 278 | 'CASE'=44 279 | 'WHEN'=45 280 | 'THEN'=46 281 | 'ELSE'=47 282 | 'END'=48 283 | 'JOIN'=49 284 | 'CROSS'=50 285 | 'OUTER'=51 286 | 'INNER'=52 287 | 'LEFT'=53 288 | 'SEMI'=54 289 | 'RIGHT'=55 290 | 'FULL'=56 291 | 'NATURAL'=57 292 | 'ON'=58 293 | 'LATERAL'=59 294 | 'WINDOW'=60 295 | 'OVER'=61 296 | 'PARTITION'=62 297 | 'RANGE'=63 298 | 'ROWS'=64 299 | 'UNBOUNDED'=65 300 | 'PRECEDING'=66 301 | 'FOLLOWING'=67 302 | 'CURRENT'=68 303 | 'ROW'=69 304 | 'WITH'=70 305 | 'VALUES'=71 306 | 'CREATE'=72 307 | 'TABLE'=73 308 | 'VIEW'=74 309 | 'REPLACE'=75 310 | 'INSERT'=76 311 | 'DELETE'=77 312 | 'INTO'=78 313 | 'DESCRIBE'=79 314 | 'EXPLAIN'=80 315 | 'FORMAT'=81 316 | 'LOGICAL'=82 317 | 'CODEGEN'=83 318 | 'CAST'=84 319 | 'SHOW'=85 320 | 'TABLES'=86 321 | 'COLUMNS'=87 322 | 'COLUMN'=88 323 | 'USE'=89 324 | 'PARTITIONS'=90 325 | 'FUNCTIONS'=91 326 | 'DROP'=92 327 | 'UNION'=93 328 | 'EXCEPT'=94 329 | 'INTERSECT'=95 330 | 'TO'=96 331 | 'TABLESAMPLE'=97 332 | 'STRATIFY'=98 333 | 'ALTER'=99 334 | 'RENAME'=100 335 | 'ARRAY'=101 336 | 'MAP'=102 337 | 'STRUCT'=103 338 | 'COMMENT'=104 339 | 'SET'=105 340 | 'RESET'=106 341 | 'DATA'=107 342 | 'START'=108 343 | 'TRANSACTION'=109 344 | 'COMMIT'=110 345 | 'ROLLBACK'=111 346 | 'MACRO'=112 347 | 'IF'=113 348 | '<=>'=115 349 | '<>'=116 350 | '!='=117 351 | '<'=118 352 | '>'=120 353 | '+'=122 354 | '-'=123 355 | '*'=124 356 | '/'=125 357 | '%'=126 358 | 'DIV'=127 359 | '~'=128 360 | '&'=129 361 | '|'=130 362 | '^'=131 363 | 'PERCENT'=132 364 | 'BUCKET'=133 365 | 'OUT'=134 366 | 'OF'=135 367 | 'SORT'=136 368 | 'CLUSTER'=137 369 | 'DISTRIBUTE'=138 370 | 'OVERWRITE'=139 371 | 'TRANSFORM'=140 372 | 'REDUCE'=141 373 | 'USING'=142 374 | 'SERDE'=143 375 | 'SERDEPROPERTIES'=144 376 | 'RECORDREADER'=145 377 | 'RECORDWRITER'=146 378 | 'DELIMITED'=147 379 | 'FIELDS'=148 380 | 'TERMINATED'=149 381 | 'COLLECTION'=150 382 | 'ITEMS'=151 383 | 'KEYS'=152 384 | 'ESCAPED'=153 385 | 'LINES'=154 386 | 'SEPARATED'=155 387 | 'FUNCTION'=156 388 | 'EXTENDED'=157 389 | 'REFRESH'=158 390 | 'CLEAR'=159 391 | 'CACHE'=160 392 | 'UNCACHE'=161 393 | 'LAZY'=162 394 | 'FORMATTED'=163 395 | 'OPTIONS'=165 396 | 'UNSET'=166 397 | 'TBLPROPERTIES'=167 398 | 'DBPROPERTIES'=168 399 | 'BUCKETS'=169 400 | 'SKEWED'=170 401 | 'STORED'=171 402 | 'DIRECTORIES'=172 403 | 'LOCATION'=173 404 | 'EXCHANGE'=174 405 | 'ARCHIVE'=175 406 | 'UNARCHIVE'=176 407 | 'FILEFORMAT'=177 408 | 'TOUCH'=178 409 | 'COMPACT'=179 410 | 'CONCATENATE'=180 411 | 'CHANGE'=181 412 | 'CASCADE'=182 413 | 'RESTRICT'=183 414 | 'CLUSTERED'=184 415 | 'SORTED'=185 416 | 'PURGE'=186 417 | 'INPUTFORMAT'=187 418 | 'OUTPUTFORMAT'=188 419 | 'DFS'=191 420 | 'TRUNCATE'=192 421 | 'ANALYZE'=193 422 | 'COMPUTE'=194 423 | 'LIST'=195 424 | 'STATISTICS'=196 425 | 'PARTITIONED'=197 426 | 'EXTERNAL'=198 427 | 'DEFINED'=199 428 | 'REVOKE'=200 429 | 'GRANT'=201 430 | 'LOCK'=202 431 | 'UNLOCK'=203 432 | 'MSCK'=204 433 | 'REPAIR'=205 434 | 'EXPORT'=206 435 | 'IMPORT'=207 436 | 'LOAD'=208 437 | 'ROLE'=209 438 | 'ROLES'=210 439 | 'COMPACTIONS'=211 440 | 'PRINCIPALS'=212 441 | 'TRANSACTIONS'=213 442 | 'INDEX'=214 443 | 'INDEXES'=215 444 | 'LOCKS'=216 445 | 'OPTION'=217 446 | 'ANTI'=218 447 | 'LOCAL'=219 448 | 'INPATH'=220 449 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/generated/UberScriptQueryTemplateLexer.tokens: -------------------------------------------------------------------------------- 1 | T__0=1 2 | T__1=2 3 | T__2=3 4 | T__3=4 5 | T__4=5 6 | T__5=6 7 | T__6=7 8 | T__7=8 9 | SELECT=9 10 | FROM=10 11 | ADD=11 12 | AS=12 13 | ALL=13 14 | DISTINCT=14 15 | WHERE=15 16 | GROUP=16 17 | BY=17 18 | GROUPING=18 19 | SETS=19 20 | CUBE=20 21 | ROLLUP=21 22 | ORDER=22 23 | HAVING=23 24 | LIMIT=24 25 | AT=25 26 | OR=26 27 | AND=27 28 | IN=28 29 | NOT=29 30 | NO=30 31 | EXISTS=31 32 | BETWEEN=32 33 | LIKE=33 34 | RLIKE=34 35 | IS=35 36 | NULL=36 37 | TRUE=37 38 | FALSE=38 39 | NULLS=39 40 | ASC=40 41 | DESC=41 42 | FOR=42 43 | INTERVAL=43 44 | CASE=44 45 | WHEN=45 46 | THEN=46 47 | ELSE=47 48 | END=48 49 | JOIN=49 50 | CROSS=50 51 | OUTER=51 52 | INNER=52 53 | LEFT=53 54 | SEMI=54 55 | RIGHT=55 56 | FULL=56 57 | NATURAL=57 58 | ON=58 59 | LATERAL=59 60 | WINDOW=60 61 | OVER=61 62 | PARTITION=62 63 | RANGE=63 64 | ROWS=64 65 | UNBOUNDED=65 66 | PRECEDING=66 67 | FOLLOWING=67 68 | CURRENT=68 69 | ROW=69 70 | WITH=70 71 | VALUES=71 72 | CREATE=72 73 | TABLE=73 74 | VIEW=74 75 | REPLACE=75 76 | INSERT=76 77 | DELETE=77 78 | INTO=78 79 | DESCRIBE=79 80 | EXPLAIN=80 81 | FORMAT=81 82 | LOGICAL=82 83 | CODEGEN=83 84 | CAST=84 85 | SHOW=85 86 | TABLES=86 87 | COLUMNS=87 88 | COLUMN=88 89 | USE=89 90 | PARTITIONS=90 91 | FUNCTIONS=91 92 | DROP=92 93 | UNION=93 94 | EXCEPT=94 95 | INTERSECT=95 96 | TO=96 97 | TABLESAMPLE=97 98 | STRATIFY=98 99 | ALTER=99 100 | RENAME=100 101 | ARRAY=101 102 | MAP=102 103 | STRUCT=103 104 | COMMENT=104 105 | SET=105 106 | RESET=106 107 | DATA=107 108 | START=108 109 | TRANSACTION=109 110 | COMMIT=110 111 | ROLLBACK=111 112 | MACRO=112 113 | IF=113 114 | EQ=114 115 | NSEQ=115 116 | NEQ=116 117 | NEQJ=117 118 | LT=118 119 | LTE=119 120 | GT=120 121 | GTE=121 122 | PLUS=122 123 | MINUS=123 124 | ASTERISK=124 125 | SLASH=125 126 | PERCENT=126 127 | DIV=127 128 | TILDE=128 129 | AMPERSAND=129 130 | PIPE=130 131 | HAT=131 132 | PERCENTLIT=132 133 | BUCKET=133 134 | OUT=134 135 | OF=135 136 | SORT=136 137 | CLUSTER=137 138 | DISTRIBUTE=138 139 | OVERWRITE=139 140 | TRANSFORM=140 141 | REDUCE=141 142 | USING=142 143 | SERDE=143 144 | SERDEPROPERTIES=144 145 | RECORDREADER=145 146 | RECORDWRITER=146 147 | DELIMITED=147 148 | FIELDS=148 149 | TERMINATED=149 150 | COLLECTION=150 151 | ITEMS=151 152 | KEYS=152 153 | ESCAPED=153 154 | LINES=154 155 | SEPARATED=155 156 | FUNCTION=156 157 | EXTENDED=157 158 | REFRESH=158 159 | CLEAR=159 160 | CACHE=160 161 | UNCACHE=161 162 | LAZY=162 163 | FORMATTED=163 164 | TEMPORARY=164 165 | OPTIONS=165 166 | UNSET=166 167 | TBLPROPERTIES=167 168 | DBPROPERTIES=168 169 | BUCKETS=169 170 | SKEWED=170 171 | STORED=171 172 | DIRECTORIES=172 173 | LOCATION=173 174 | EXCHANGE=174 175 | ARCHIVE=175 176 | UNARCHIVE=176 177 | FILEFORMAT=177 178 | TOUCH=178 179 | COMPACT=179 180 | CONCATENATE=180 181 | CHANGE=181 182 | CASCADE=182 183 | RESTRICT=183 184 | CLUSTERED=184 185 | SORTED=185 186 | PURGE=186 187 | INPUTFORMAT=187 188 | OUTPUTFORMAT=188 189 | DATABASE=189 190 | DATABASES=190 191 | DFS=191 192 | TRUNCATE=192 193 | ANALYZE=193 194 | COMPUTE=194 195 | LIST=195 196 | STATISTICS=196 197 | PARTITIONED=197 198 | EXTERNAL=198 199 | DEFINED=199 200 | REVOKE=200 201 | GRANT=201 202 | LOCK=202 203 | UNLOCK=203 204 | MSCK=204 205 | REPAIR=205 206 | EXPORT=206 207 | IMPORT=207 208 | LOAD=208 209 | ROLE=209 210 | ROLES=210 211 | COMPACTIONS=211 212 | PRINCIPALS=212 213 | TRANSACTIONS=213 214 | INDEX=214 215 | INDEXES=215 216 | LOCKS=216 217 | OPTION=217 218 | ANTI=218 219 | LOCAL=219 220 | INPATH=220 221 | STRING=221 222 | BIGINT_LITERAL=222 223 | SMALLINT_LITERAL=223 224 | TINYINT_LITERAL=224 225 | BYTELENGTH_LITERAL=225 226 | INTEGER_VALUE=226 227 | DECIMAL_VALUE=227 228 | SCIENTIFIC_DECIMAL_VALUE=228 229 | DOUBLE_LITERAL=229 230 | IDENTIFIER=230 231 | BACKQUOTED_IDENTIFIER=231 232 | SIMPLE_COMMENT=232 233 | BRACKETED_COMMENT=233 234 | WS=234 235 | UNRECOGNIZED=235 236 | ';'=1 237 | '('=2 238 | ')'=3 239 | ','=4 240 | '.'=5 241 | '['=6 242 | ']'=7 243 | ':'=8 244 | 'SELECT'=9 245 | 'FROM'=10 246 | 'ADD'=11 247 | 'AS'=12 248 | 'ALL'=13 249 | 'DISTINCT'=14 250 | 'WHERE'=15 251 | 'GROUP'=16 252 | 'BY'=17 253 | 'GROUPING'=18 254 | 'SETS'=19 255 | 'CUBE'=20 256 | 'ROLLUP'=21 257 | 'ORDER'=22 258 | 'HAVING'=23 259 | 'LIMIT'=24 260 | 'AT'=25 261 | 'OR'=26 262 | 'AND'=27 263 | 'IN'=28 264 | 'NO'=30 265 | 'EXISTS'=31 266 | 'BETWEEN'=32 267 | 'LIKE'=33 268 | 'IS'=35 269 | 'NULL'=36 270 | 'TRUE'=37 271 | 'FALSE'=38 272 | 'NULLS'=39 273 | 'ASC'=40 274 | 'DESC'=41 275 | 'FOR'=42 276 | 'INTERVAL'=43 277 | 'CASE'=44 278 | 'WHEN'=45 279 | 'THEN'=46 280 | 'ELSE'=47 281 | 'END'=48 282 | 'JOIN'=49 283 | 'CROSS'=50 284 | 'OUTER'=51 285 | 'INNER'=52 286 | 'LEFT'=53 287 | 'SEMI'=54 288 | 'RIGHT'=55 289 | 'FULL'=56 290 | 'NATURAL'=57 291 | 'ON'=58 292 | 'LATERAL'=59 293 | 'WINDOW'=60 294 | 'OVER'=61 295 | 'PARTITION'=62 296 | 'RANGE'=63 297 | 'ROWS'=64 298 | 'UNBOUNDED'=65 299 | 'PRECEDING'=66 300 | 'FOLLOWING'=67 301 | 'CURRENT'=68 302 | 'ROW'=69 303 | 'WITH'=70 304 | 'VALUES'=71 305 | 'CREATE'=72 306 | 'TABLE'=73 307 | 'VIEW'=74 308 | 'REPLACE'=75 309 | 'INSERT'=76 310 | 'DELETE'=77 311 | 'INTO'=78 312 | 'DESCRIBE'=79 313 | 'EXPLAIN'=80 314 | 'FORMAT'=81 315 | 'LOGICAL'=82 316 | 'CODEGEN'=83 317 | 'CAST'=84 318 | 'SHOW'=85 319 | 'TABLES'=86 320 | 'COLUMNS'=87 321 | 'COLUMN'=88 322 | 'USE'=89 323 | 'PARTITIONS'=90 324 | 'FUNCTIONS'=91 325 | 'DROP'=92 326 | 'UNION'=93 327 | 'EXCEPT'=94 328 | 'INTERSECT'=95 329 | 'TO'=96 330 | 'TABLESAMPLE'=97 331 | 'STRATIFY'=98 332 | 'ALTER'=99 333 | 'RENAME'=100 334 | 'ARRAY'=101 335 | 'MAP'=102 336 | 'STRUCT'=103 337 | 'COMMENT'=104 338 | 'SET'=105 339 | 'RESET'=106 340 | 'DATA'=107 341 | 'START'=108 342 | 'TRANSACTION'=109 343 | 'COMMIT'=110 344 | 'ROLLBACK'=111 345 | 'MACRO'=112 346 | 'IF'=113 347 | '<=>'=115 348 | '<>'=116 349 | '!='=117 350 | '<'=118 351 | '>'=120 352 | '+'=122 353 | '-'=123 354 | '*'=124 355 | '/'=125 356 | '%'=126 357 | 'DIV'=127 358 | '~'=128 359 | '&'=129 360 | '|'=130 361 | '^'=131 362 | 'PERCENT'=132 363 | 'BUCKET'=133 364 | 'OUT'=134 365 | 'OF'=135 366 | 'SORT'=136 367 | 'CLUSTER'=137 368 | 'DISTRIBUTE'=138 369 | 'OVERWRITE'=139 370 | 'TRANSFORM'=140 371 | 'REDUCE'=141 372 | 'USING'=142 373 | 'SERDE'=143 374 | 'SERDEPROPERTIES'=144 375 | 'RECORDREADER'=145 376 | 'RECORDWRITER'=146 377 | 'DELIMITED'=147 378 | 'FIELDS'=148 379 | 'TERMINATED'=149 380 | 'COLLECTION'=150 381 | 'ITEMS'=151 382 | 'KEYS'=152 383 | 'ESCAPED'=153 384 | 'LINES'=154 385 | 'SEPARATED'=155 386 | 'FUNCTION'=156 387 | 'EXTENDED'=157 388 | 'REFRESH'=158 389 | 'CLEAR'=159 390 | 'CACHE'=160 391 | 'UNCACHE'=161 392 | 'LAZY'=162 393 | 'FORMATTED'=163 394 | 'OPTIONS'=165 395 | 'UNSET'=166 396 | 'TBLPROPERTIES'=167 397 | 'DBPROPERTIES'=168 398 | 'BUCKETS'=169 399 | 'SKEWED'=170 400 | 'STORED'=171 401 | 'DIRECTORIES'=172 402 | 'LOCATION'=173 403 | 'EXCHANGE'=174 404 | 'ARCHIVE'=175 405 | 'UNARCHIVE'=176 406 | 'FILEFORMAT'=177 407 | 'TOUCH'=178 408 | 'COMPACT'=179 409 | 'CONCATENATE'=180 410 | 'CHANGE'=181 411 | 'CASCADE'=182 412 | 'RESTRICT'=183 413 | 'CLUSTERED'=184 414 | 'SORTED'=185 415 | 'PURGE'=186 416 | 'INPUTFORMAT'=187 417 | 'OUTPUTFORMAT'=188 418 | 'DFS'=191 419 | 'TRUNCATE'=192 420 | 'ANALYZE'=193 421 | 'COMPUTE'=194 422 | 'LIST'=195 423 | 'STATISTICS'=196 424 | 'PARTITIONED'=197 425 | 'EXTERNAL'=198 426 | 'DEFINED'=199 427 | 'REVOKE'=200 428 | 'GRANT'=201 429 | 'LOCK'=202 430 | 'UNLOCK'=203 431 | 'MSCK'=204 432 | 'REPAIR'=205 433 | 'EXPORT'=206 434 | 'IMPORT'=207 435 | 'LOAD'=208 436 | 'ROLE'=209 437 | 'ROLES'=210 438 | 'COMPACTIONS'=211 439 | 'PRINCIPALS'=212 440 | 'TRANSACTIONS'=213 441 | 'INDEX'=214 442 | 'INDEXES'=215 443 | 'LOCKS'=216 444 | 'OPTION'=217 445 | 'ANTI'=218 446 | 'LOCAL'=219 447 | 'INPATH'=220 448 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/ActionParamValue.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | public class ActionParamValue { 20 | private ValueType valueType; 21 | private Object value; 22 | 23 | public ActionParamValue() { 24 | } 25 | 26 | public ActionParamValue(ValueType valueType, Object value) { 27 | this.valueType = valueType; 28 | this.value = value; 29 | } 30 | 31 | public ValueType getValueType() { 32 | return valueType; 33 | } 34 | 35 | public void setValueType(ValueType valueType) { 36 | this.valueType = valueType; 37 | } 38 | 39 | public Object getValue() { 40 | return value; 41 | } 42 | 43 | public void setValue(Object value) { 44 | this.value = value; 45 | } 46 | 47 | @Override 48 | public String toString() { 49 | return "ActionParamValue{" + 50 | "valueType=" + valueType + 51 | ", value=" + value + 52 | '}'; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/ActionStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | import java.util.ArrayList; 20 | import java.util.List; 21 | 22 | public class ActionStatement { 23 | private String functionName; 24 | private List paramValues = new ArrayList<>(); 25 | 26 | public String getFunctionName() { 27 | return functionName; 28 | } 29 | 30 | public void setFunctionName(String functionName) { 31 | this.functionName = functionName; 32 | } 33 | 34 | public List getParamValues() { 35 | return paramValues; 36 | } 37 | 38 | public void setParamValues(List paramValues) { 39 | this.paramValues = paramValues; 40 | } 41 | 42 | @Override 43 | public String toString() { 44 | return "ActionStatement{" + 45 | "functionName='" + functionName + '\'' + 46 | ", paramValues=" + paramValues + 47 | '}'; 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/FileAssignment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | public class FileAssignment implements java.io.Serializable { 20 | private String tableAlias; 21 | private String fileType; 22 | private String fileLocation; 23 | 24 | public String getTableAlias() { 25 | return tableAlias; 26 | } 27 | 28 | public void setTableAlias(String tableAlias) { 29 | this.tableAlias = tableAlias; 30 | } 31 | 32 | public String getFileType() { 33 | return fileType; 34 | } 35 | 36 | public void setFileType(String fileType) { 37 | this.fileType = fileType; 38 | } 39 | 40 | public String getFileLocation() { 41 | return fileLocation; 42 | } 43 | 44 | public void setFileLocation(String fileLocation) { 45 | this.fileLocation = fileLocation; 46 | } 47 | 48 | @Override 49 | public String toString() { 50 | return "FileAssignment{" + 51 | "tableAlias='" + tableAlias + '\'' + 52 | ", fileType='" + fileType + '\'' + 53 | ", fileLocation='" + fileLocation + '\'' + 54 | '}'; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/ParserUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | public class ParserUtils { 20 | public static String getStringStatementInnerValue(String variableValue) { 21 | // remove quote from value, and handle character escape 22 | // TODO handle more character escapes rather than \' and \" 23 | if (variableValue.startsWith("'")) { 24 | variableValue = variableValue.substring(1, variableValue.length() - 1); 25 | variableValue = variableValue.replace("\\'", "'"); 26 | } else if (variableValue.startsWith("\"")) { 27 | variableValue = variableValue.substring(1, variableValue.length() - 1); 28 | variableValue = variableValue.replace("\\\"", "\""); 29 | } 30 | return variableValue; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/QuerySqlParseResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | public class QuerySqlParseResult { 20 | private RootStatement rootStatement; 21 | 22 | public RootStatement getRootStatement() { 23 | return rootStatement; 24 | } 25 | 26 | public void setRootStatement(RootStatement rootStatement) { 27 | this.rootStatement = rootStatement; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/QueryTemplateParseResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | public class QueryTemplateParseResult implements java.io.Serializable { 23 | 24 | private Map variableMap = new HashMap<>(); 25 | 26 | private String templateBody; 27 | 28 | public Map getVariableMap() { 29 | return variableMap; 30 | } 31 | 32 | public void setVariableMap(Map variableMap) { 33 | this.variableMap = variableMap; 34 | } 35 | 36 | public String getTemplateBody() { 37 | return templateBody; 38 | } 39 | 40 | public void setTemplateBody(String templateBody) { 41 | this.templateBody = templateBody; 42 | } 43 | 44 | @Override 45 | public String toString() { 46 | return "QueryTemplateParseResult{" + 47 | "variableMap=" + variableMap + 48 | ", templateBody='" + templateBody + '\'' + 49 | '}'; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/QueryTemplateParser.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | import com.uber.uberscriptquery.antlr4.generated.UberScriptQueryTemplateBaseListener; 20 | import com.uber.uberscriptquery.antlr4.generated.UberScriptQueryTemplateLexer; 21 | import com.uber.uberscriptquery.util.TemplateUtils; 22 | import org.antlr.v4.runtime.*; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | import java.util.concurrent.atomic.AtomicInteger; 29 | 30 | public class QueryTemplateParser { 31 | private static final Logger logger = LoggerFactory.getLogger(QueryTemplateParser.class); 32 | 33 | public QueryTemplateParseResult parse(String text) { 34 | return parse(text, null); 35 | } 36 | 37 | public QueryTemplateParseResult parse(String text, Map variableOverwrite) { 38 | final char[] charArray = text.toCharArray(); 39 | 40 | final char[] charArrayUpperCase = text.toUpperCase().toCharArray(); 41 | 42 | final QueryTemplateParseResult parsedResult = new QueryTemplateParseResult(); 43 | 44 | UberScriptQueryTemplateLexer l = new UberScriptQueryTemplateLexer(new ANTLRInputStream(charArrayUpperCase, charArrayUpperCase.length)); 45 | com.uber.uberscriptquery.antlr4.generated.UberScriptQueryTemplateParser p = new com.uber.uberscriptquery.antlr4.generated.UberScriptQueryTemplateParser(new CommonTokenStream(l)); 46 | p.addErrorListener(new BaseErrorListener() { 47 | @Override 48 | public void syntaxError(Recognizer recognizer, Object offendingSymbol, int line, int charPositionInLine, String msg, RecognitionException e) { 49 | throw new IllegalStateException( 50 | String.format("Failed to parse at line %s char %s due to %s", line, charPositionInLine, msg), 51 | e); 52 | } 53 | }); 54 | 55 | final AtomicInteger lastOffset = new AtomicInteger(-1); 56 | 57 | final Map variableMap = new HashMap<>(); 58 | 59 | if (variableOverwrite != null && !variableOverwrite.isEmpty()) { 60 | variableMap.putAll(variableOverwrite); 61 | } 62 | 63 | p.addParseListener( 64 | new UberScriptQueryTemplateBaseListener() { 65 | @Override 66 | public void exitVariableAssignment(com.uber.uberscriptquery.antlr4.generated.UberScriptQueryTemplateParser.VariableAssignmentContext ctx) { 67 | lastOffset.set(ctx.getStop().getStopIndex()); 68 | 69 | String variableName = getMatchedString(ctx.variableAssignment_variableName(), charArray); 70 | 71 | if (variableOverwrite != null && variableOverwrite.containsKey(variableName)) { 72 | logger.info(String.format("Ignored variable %s because of variable overwrite with value %s", variableName, variableOverwrite.get(variableName))); 73 | return; 74 | } 75 | 76 | String variableValue = getMatchedString(ctx.STRING().getSymbol(), charArray); 77 | 78 | if (variableValue == null) { 79 | return; 80 | } 81 | variableValue = ParserUtils.getStringStatementInnerValue(variableValue); 82 | 83 | variableValue = TemplateUtils.transform(variableValue, variableMap); 84 | 85 | variableMap.put(variableName, variableValue); 86 | } 87 | 88 | @Override 89 | public void exitAssignmentWithSemicolon(com.uber.uberscriptquery.antlr4.generated.UberScriptQueryTemplateParser.AssignmentWithSemicolonContext ctx) { 90 | lastOffset.set(ctx.getStop().getStopIndex()); 91 | } 92 | }); 93 | 94 | p.root(); 95 | 96 | parsedResult.setVariableMap(variableMap); 97 | 98 | String templateBody = new String(charArray, lastOffset.get() + 1, charArray.length - lastOffset.get() - 1); 99 | templateBody = TemplateUtils.transform(templateBody, variableMap); 100 | parsedResult.setTemplateBody(templateBody); 101 | 102 | return parsedResult; 103 | } 104 | 105 | private String getMatchedString(ParserRuleContext ctx, char[] charArray) { 106 | if (ctx == null || ctx.getStart() == null) { 107 | return null; 108 | } 109 | 110 | int startIndex = ctx.getStart().getStartIndex(); 111 | int endIndex = ctx.getStop().getStopIndex(); 112 | return new String(charArray, startIndex, endIndex + 1 - startIndex); 113 | } 114 | 115 | private String getMatchedString(org.antlr.v4.runtime.Token ctx, char[] charArray) { 116 | if (ctx == null) { 117 | return null; 118 | } 119 | 120 | int startIndex = ctx.getStartIndex(); 121 | int endIndex = ctx.getStopIndex(); 122 | return new String(charArray, startIndex, endIndex + 1 - startIndex); 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/RootStatement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | import com.fasterxml.jackson.annotation.JsonIgnore; 20 | import com.uber.uberscriptquery.util.CounterMap; 21 | 22 | import java.util.ArrayList; 23 | import java.util.List; 24 | 25 | public class RootStatement implements java.io.Serializable { 26 | private List fileAssignments = new ArrayList<>(); 27 | private List jsonQueryStatementAssignments = new ArrayList<>(); 28 | private List statementAssignments = new ArrayList<>(); 29 | private List actionStatements = new ArrayList<>(); 30 | 31 | private CounterMap tableReferenceCount = new CounterMap(); 32 | 33 | public List getFileAssignments() { 34 | return fileAssignments; 35 | } 36 | 37 | public void setFileAssignments(List fileAssignments) { 38 | this.fileAssignments = fileAssignments; 39 | } 40 | 41 | public List getJsonQueryStatementAssignments() { 42 | return jsonQueryStatementAssignments; 43 | } 44 | 45 | public void setJsonQueryStatementAssignments(List jsonQueryStatementAssignments) { 46 | this.jsonQueryStatementAssignments = jsonQueryStatementAssignments; 47 | } 48 | 49 | public List getStatementAssignments() { 50 | return statementAssignments; 51 | } 52 | 53 | public void setStatementAssignments(List statementAssignments) { 54 | this.statementAssignments = statementAssignments; 55 | } 56 | 57 | public List getActionStatements() { 58 | return actionStatements; 59 | } 60 | 61 | public void setActionStatements(List actionStatements) { 62 | this.actionStatements = actionStatements; 63 | } 64 | 65 | 66 | @JsonIgnore 67 | public CounterMap getTableReferenceCount() { 68 | return tableReferenceCount; 69 | } 70 | 71 | public void setTableReferenceCount(CounterMap tableReferenceCount) { 72 | this.tableReferenceCount = tableReferenceCount; 73 | } 74 | 75 | @Override 76 | public String toString() { 77 | return "RootStatement{" + 78 | "fileAssignments=" + fileAssignments + 79 | ", jsonQueryStatementAssignments=" + jsonQueryStatementAssignments + 80 | ", statementAssignments=" + statementAssignments + 81 | ", actionStatements=" + actionStatements + 82 | ", tableReferenceCount=" + tableReferenceCount + 83 | '}'; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/StatementAssignment.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | public class StatementAssignment implements java.io.Serializable { 23 | public static final String QUERY_CONFIG_CONNECTION_STRING = "connectionString"; 24 | public static final String QUERY_CONFIG_PASSWORD_FILE = "passwordFile"; 25 | public static final String QUERY_CONFIG_PASSWORD_ENTRY = "passwordEntry"; 26 | 27 | private String tableAlias; 28 | private String queryType; 29 | private String queryEngine; 30 | private String queryText; 31 | 32 | private Map queryConfig = new HashMap<>(); 33 | private String queryStatement; 34 | 35 | public String getTableAlias() { 36 | return tableAlias; 37 | } 38 | 39 | public void setTableAlias(String tableAlias) { 40 | this.tableAlias = tableAlias; 41 | } 42 | 43 | public String getQueryType() { 44 | return queryType; 45 | } 46 | 47 | public void setQueryType(String queryType) { 48 | this.queryType = queryType; 49 | } 50 | 51 | public String getQueryEngine() { 52 | return queryEngine; 53 | } 54 | 55 | public void setQueryEngine(String queryEngine) { 56 | this.queryEngine = queryEngine; 57 | } 58 | 59 | public String getQueryText() { 60 | return queryText; 61 | } 62 | 63 | public void setQueryText(String queryText) { 64 | this.queryText = queryText; 65 | } 66 | 67 | public Map getQueryConfig() { 68 | return queryConfig; 69 | } 70 | 71 | public void setQueryConfig(Map queryConfig) { 72 | this.queryConfig = queryConfig; 73 | } 74 | 75 | public String getQueryStatement() { 76 | return queryStatement; 77 | } 78 | 79 | public void setQueryStatement(String queryStatement) { 80 | this.queryStatement = queryStatement; 81 | } 82 | 83 | @Override 84 | public String toString() { 85 | return "StatementAssignment{" + 86 | "tableAlias='" + tableAlias + '\'' + 87 | ", queryType='" + queryType + '\'' + 88 | ", queryEngine='" + queryEngine + '\'' + 89 | ", queryText='" + queryText + '\'' + 90 | ", queryConfig=" + queryConfig + 91 | ", queryStatement='" + queryStatement + '\'' + 92 | '}'; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/antlr4/parsing/ValueType.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | public enum ValueType { 20 | String, Table 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/examples/QueryExampleJob.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.examples; 18 | 19 | import com.uber.uberscriptquery.execution.QueryEngine; 20 | import com.uber.uberscriptquery.util.HttpUtils; 21 | import org.apache.spark.SparkConf; 22 | import org.apache.spark.sql.SparkSession; 23 | import scala.Tuple2; 24 | 25 | /** 26 | * This spark job takes a query and run it. 27 | * It takes a parameter "-query" to specify the query content, which could be a text, http url, or hdfs url. 28 | * It also takes a parameter "-queryOverwrite" which could overwrite the variable values in the query. 29 | */ 30 | public class QueryExampleJob { 31 | 32 | public static void main(String[] args) { 33 | String query = null; 34 | String queryOverwrite = null; 35 | 36 | // Read program arguments 37 | 38 | for (int i = 0; i < args.length; i++) { 39 | if (args[i].equalsIgnoreCase("-query")) { 40 | query = args[i + 1]; 41 | } else if (args[i].equalsIgnoreCase("-queryOverwrite")) { 42 | queryOverwrite = args[i + 1]; 43 | } 44 | } 45 | 46 | if (query == null) { 47 | query = "message = 'Hello World'; \n" 48 | + "result = select cast(unix_timestamp() as timestamp) as time, '${message}' as message; \n" 49 | + "printTable(result);"; 50 | } else if (query.toLowerCase().startsWith("http://") || query.toLowerCase().startsWith("https://")) { 51 | System.out.println("Fetching query from http: " + query); 52 | query = HttpUtils.get(query, null).getBody(); 53 | } 54 | 55 | System.out.println("Query: " + query); 56 | System.out.println("Query Overwrite: " + queryOverwrite); 57 | 58 | // Start Spark Session 59 | 60 | String master = "local[1]"; 61 | String appName = "QueryExampleJob"; 62 | 63 | SparkConf sparkConf = new SparkConf() 64 | .setMaster(master) 65 | .setAppName(appName); 66 | 67 | SparkSession sparkSession = SparkSession 68 | .builder() 69 | .config(sparkConf).getOrCreate(); 70 | 71 | // Load query from hdfs if it is hdfs url 72 | if (query.toLowerCase().startsWith("hdfs://")) { 73 | System.out.println("Fetching query from hdfs: " + query); 74 | Tuple2[] tuples = (Tuple2[]) sparkSession.sparkContext().wholeTextFiles(query, 1).collect(); 75 | query = tuples[0]._2(); 76 | System.out.println("Query: " + query); 77 | } 78 | 79 | // Run query 80 | QueryEngine engine = new QueryEngine(); 81 | engine.executeScript(query, queryOverwrite, sparkSession, false); 82 | 83 | sparkSession.stop(); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/examples/QueryExecutionExample.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.examples; 18 | 19 | import com.uber.uberscriptquery.execution.QueryEngine; 20 | import com.uber.uberscriptquery.jdbc.DataSetResult; 21 | import com.uber.uberscriptquery.util.JsonUtils; 22 | import com.uber.uberscriptquery.util.SqlUtils; 23 | import org.apache.spark.SparkConf; 24 | import org.apache.spark.sql.SparkSession; 25 | 26 | import java.io.File; 27 | import java.io.FileOutputStream; 28 | import java.io.IOException; 29 | import java.nio.charset.StandardCharsets; 30 | import java.util.HashMap; 31 | import java.util.Map; 32 | import java.util.Random; 33 | 34 | /** 35 | * This class contains example codes to use UberScriptQuery to run Spark SQL job. 36 | */ 37 | public class QueryExecutionExample { 38 | 39 | public static void main(String[] args) { 40 | // Prepare data as the input for the script. 41 | // This is just for demonstration. In real scenario, you will have your own data in database, hadoop, or other places. 42 | String connectionString = prepareJdbcData(); 43 | String jsonFile = prepareJsonFile(); 44 | 45 | // Start Spark Session 46 | 47 | String master = "local[1]"; 48 | String appName = "QueryExecutionExample"; 49 | 50 | SparkConf sparkConf = new SparkConf() 51 | .setMaster(master) 52 | .setAppName(appName); 53 | 54 | SparkSession sparkSession = SparkSession 55 | .builder() 56 | .config(sparkConf).getOrCreate(); 57 | 58 | // Set log level to avoid too many outputs 59 | sparkSession.sparkContext().setLogLevel("ERROR"); 60 | 61 | // Script to run 62 | String script = 63 | String.format("clients = sql jdbc set connectionString='%s'; select clientId, clientName from dim_client; \n", connectionString) 64 | + String.format("orders = file json file://%s; \n", jsonFile) 65 | + "result = select clientName,productName,totalCount from orders join clients on clients.clientId = orders.clientId; \n" 66 | + String.format("writeJdbc('%s', 'jdbcTable', 'clientName,productName', 'clientName,productName', '', 'Append', result);", connectionString); 67 | 68 | System.out.println("-----------------------------------------------"); 69 | System.out.println("Running script"); 70 | System.out.println("-----------------------------------------------"); 71 | System.out.println(script); 72 | System.out.println("-----------------------------------------------"); 73 | 74 | // Run Script 75 | QueryEngine engine = new QueryEngine(); 76 | engine.executeScript(script, sparkSession); 77 | 78 | // The data should be written into the database by writeJdbc action. 79 | // Let's query it and print it out. 80 | DataSetResult dataResult = SqlUtils.executeJdbcQuery(connectionString, "select * from jdbcTable"); 81 | System.out.println("-----------------------------------------------"); 82 | System.out.println("Data in jdbc table after executing script"); 83 | System.out.println("-----------------------------------------------"); 84 | dataResult.print(); 85 | System.out.println("-----------------------------------------------"); 86 | 87 | sparkSession.stop(); 88 | } 89 | 90 | 91 | private static String prepareJdbcData() { 92 | File file; 93 | try { 94 | file = File.createTempFile("h2dbfile", ".db"); 95 | } catch (IOException e) { 96 | throw new RuntimeException("Failed to create temp file", e); 97 | } 98 | file.deleteOnExit(); 99 | 100 | String connectionString = String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath()); 101 | 102 | SqlUtils.executeJdbcUpdate(connectionString, "create table dim_client(clientId bigint, clientName varchar(15))"); 103 | SqlUtils.executeJdbcUpdate(connectionString, "insert into dim_client(clientId, clientName) values (1, 'client1')"); 104 | SqlUtils.executeJdbcUpdate(connectionString, "insert into dim_client(clientId, clientName) values (2, 'client2')"); 105 | SqlUtils.executeJdbcUpdate(connectionString, "insert into dim_client(clientId, clientName) values (3, 'client3')"); 106 | 107 | return connectionString; 108 | } 109 | 110 | private static String prepareJsonFile() { 111 | File file; 112 | try { 113 | file = File.createTempFile("test_file", ".json"); 114 | } catch (IOException e) { 115 | throw new RuntimeException("Failed to create temp file", e); 116 | } 117 | file.deleteOnExit(); 118 | 119 | try (FileOutputStream outputStream = new FileOutputStream(file)) { 120 | Random random = new Random(); 121 | for (int i = 1; i <= 100; i++) { 122 | Map jsonRecord = new HashMap<>(); 123 | jsonRecord.put("clientId", i); 124 | jsonRecord.put("productName", "product_" + random.nextInt(100)); 125 | jsonRecord.put("totalCount", random.nextInt(100)); 126 | 127 | String jsonString = JsonUtils.serialize(jsonRecord) + "\n"; 128 | outputStream.write(jsonString.getBytes(StandardCharsets.UTF_8)); 129 | } 130 | outputStream.flush(); 131 | outputStream.close(); 132 | } catch (IOException e) { 133 | throw new RuntimeException("Failed to write to file " + file.getAbsolutePath(), e); 134 | } 135 | 136 | return file.getAbsolutePath(); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/ActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 20 | import com.uber.uberscriptquery.util.CredentialProvider; 21 | import org.apache.spark.sql.SparkSession; 22 | 23 | public interface ActionStatementExecutor { 24 | Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager); 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/CredentialEntry.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | public class CredentialEntry implements java.io.Serializable { 20 | private String passwordFile; 21 | private String passwordEntry; 22 | private String passwordValue; 23 | 24 | public String getPasswordFile() { 25 | return passwordFile; 26 | } 27 | 28 | public void setPasswordFile(String passwordFile) { 29 | this.passwordFile = passwordFile; 30 | } 31 | 32 | public String getPasswordEntry() { 33 | return passwordEntry; 34 | } 35 | 36 | public void setPasswordEntry(String passwordEntry) { 37 | this.passwordEntry = passwordEntry; 38 | } 39 | 40 | public String getPasswordValue() { 41 | return passwordValue; 42 | } 43 | 44 | public void setPasswordValue(String passwordValue) { 45 | this.passwordValue = passwordValue; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/JdbcSqlInputStatementExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.StatementAssignment; 20 | import com.uber.uberscriptquery.util.CredentialProvider; 21 | import com.uber.uberscriptquery.util.SparkUtils; 22 | import org.apache.spark.sql.Dataset; 23 | import org.apache.spark.sql.Row; 24 | import org.apache.spark.sql.SparkSession; 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | import java.util.Map; 29 | 30 | public class JdbcSqlInputStatementExecutor implements SqlInputStatementExecutor { 31 | private static final Logger logger = LoggerFactory.getLogger(JdbcSqlInputStatementExecutor.class); 32 | 33 | @Override 34 | public Dataset execute(SparkSession sparkSession, StatementAssignment statementAssignment, CredentialProvider credentialManager) { 35 | logger.info("Running query by sql jdbc: " + statementAssignment); 36 | Map queryConfig = statementAssignment.getQueryConfig(); 37 | String connectionString = queryConfig.get(StatementAssignment.QUERY_CONFIG_CONNECTION_STRING); 38 | String passwordFile = queryConfig.get(StatementAssignment.QUERY_CONFIG_PASSWORD_FILE); 39 | String passwordEntry = queryConfig.get(StatementAssignment.QUERY_CONFIG_PASSWORD_ENTRY); 40 | String password = credentialManager.getPassword(passwordFile, passwordEntry); 41 | if (password != null) { 42 | connectionString = connectionString.replace("[password]", password); 43 | } 44 | return SparkUtils.readJdbc(connectionString, statementAssignment.getQueryStatement(), sparkSession); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/JsonInputStatementExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.StatementAssignment; 20 | import com.uber.uberscriptquery.util.CredentialProvider; 21 | import org.apache.spark.sql.Dataset; 22 | import org.apache.spark.sql.Row; 23 | import org.apache.spark.sql.SparkSession; 24 | 25 | public interface JsonInputStatementExecutor { 26 | Dataset execute(SparkSession sparkSession, StatementAssignment statementAssignment, CredentialProvider credentialManager); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/PrintTableActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 20 | import com.uber.uberscriptquery.jdbc.DataSetResult; 21 | import com.uber.uberscriptquery.util.CredentialProvider; 22 | import com.uber.uberscriptquery.util.SparkUtils; 23 | import org.apache.spark.sql.SparkSession; 24 | 25 | public class PrintTableActionStatementExecutor implements ActionStatementExecutor { 26 | 27 | public static final String ACTION_NAME = "printTable"; 28 | 29 | @Override 30 | public Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager) { 31 | String tableName = actionStatement.getParamValues().get(0).getValue().toString(); 32 | 33 | DataSetResult dataSetResult = SparkUtils.getTableData(sparkSession, tableName); 34 | System.out.println("------------------------------"); 35 | System.out.println("Table: " + tableName); 36 | dataSetResult.print(); 37 | System.out.println("------------------------------"); 38 | 39 | return null; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/QueryActionEngine.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 20 | import com.uber.uberscriptquery.util.CredentialProvider; 21 | import com.uber.uberscriptquery.util.DummyCredentialProvider; 22 | import org.apache.spark.sql.SparkSession; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | 26 | import java.util.HashMap; 27 | import java.util.Map; 28 | 29 | public class QueryActionEngine { 30 | private static final Logger logger = LoggerFactory.getLogger(QueryActionEngine.class); 31 | 32 | private CredentialProvider credentialManager = new DummyCredentialProvider(); 33 | 34 | private Map actionStatementExecutors = new HashMap<>(); 35 | 36 | public void setCredentialProvider(CredentialProvider credentialManager) { 37 | this.credentialManager = credentialManager; 38 | } 39 | 40 | public void addActionStatementExecutor(String name, ActionStatementExecutor executor) { 41 | actionStatementExecutors.put(name, executor); 42 | logger.info(String.format("Added action executor %s for %s", executor.getClass().getName(), name)); 43 | } 44 | 45 | public Object execute(ActionStatement actionStatement, SparkSession sparkSession) { 46 | ActionStatementExecutor actionStatementExecutor = actionStatementExecutors.get(actionStatement.getFunctionName()); 47 | if (actionStatementExecutor != null) { 48 | return actionStatementExecutor.execute(sparkSession, actionStatement, this.credentialManager); 49 | } else { 50 | throw new RuntimeException(String.format("Unsupported action %s", actionStatement)); 51 | } 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/SendMailGunEmailActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package com.uber.uberscriptquery.execution; 2 | 3 | import com.uber.uberscriptquery.antlr4.parsing.ActionParamValue; 4 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 5 | import com.uber.uberscriptquery.antlr4.parsing.ValueType; 6 | import com.uber.uberscriptquery.jdbc.DataSetResult; 7 | import com.uber.uberscriptquery.util.CredentialProvider; 8 | import com.uber.uberscriptquery.util.SparkUtils; 9 | import org.apache.commons.io.IOUtils; 10 | import org.apache.http.NameValuePair; 11 | import org.apache.http.client.entity.UrlEncodedFormEntity; 12 | import org.apache.http.client.methods.CloseableHttpResponse; 13 | import org.apache.http.client.methods.HttpPost; 14 | import org.apache.http.impl.client.BasicCredentialsProvider; 15 | import org.apache.http.impl.client.CloseableHttpClient; 16 | import org.apache.http.impl.client.HttpClientBuilder; 17 | import org.apache.http.impl.client.HttpClients; 18 | import org.apache.http.message.BasicNameValuePair; 19 | import org.apache.spark.sql.SparkSession; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.io.BufferedReader; 24 | import java.io.IOException; 25 | import java.io.InputStreamReader; 26 | import java.io.UnsupportedEncodingException; 27 | import java.util.ArrayList; 28 | import java.util.List; 29 | 30 | public class SendMailGunEmailActionStatementExecutor implements ActionStatementExecutor { 31 | public static final String ACTION_NAME = "sendMailGunEmail"; 32 | private static final Logger logger = LoggerFactory.getLogger(SendMailGunEmailActionStatementExecutor.class); 33 | 34 | @Override 35 | public Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager) { 36 | 37 | String mailGunUrl = actionStatement.getParamValues().get(0).getValue().toString(); 38 | String apiKey = actionStatement.getParamValues().get(1).getValue().toString(); 39 | String from = actionStatement.getParamValues().get(2).getValue().toString(); 40 | String to = actionStatement.getParamValues().get(3).getValue().toString(); 41 | ActionParamValue subjectParamValue = actionStatement.getParamValues().get(4); 42 | ActionParamValue htmlParamValue = actionStatement.getParamValues().get(5); 43 | 44 | String subject; 45 | if (subjectParamValue.getValueType() == ValueType.String) { 46 | subject = subjectParamValue.getValue().toString(); 47 | } else { 48 | DataSetResult dataSetResult = SparkUtils.getTableData(sparkSession, subjectParamValue.getValue().toString()); 49 | subject = mergeFirstColumnData(dataSetResult); 50 | } 51 | 52 | String html; 53 | if (htmlParamValue.getValueType() == ValueType.String) { 54 | html = htmlParamValue.getValue().toString(); 55 | } else { 56 | DataSetResult dataSetResult = SparkUtils.getTableData(sparkSession, htmlParamValue.getValue().toString()); 57 | html = mergeFirstColumnData(dataSetResult); 58 | } 59 | 60 | HttpClientBuilder httpClientBuilder = HttpClients.custom().useSystemProperties(); 61 | BasicCredentialsProvider credentialsProvider = new BasicCredentialsProvider(); 62 | credentialsProvider.setCredentials(org.apache.http.auth.AuthScope.ANY, 63 | new org.apache.http.auth.UsernamePasswordCredentials("api", apiKey)); 64 | httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); 65 | 66 | CloseableHttpClient httpClient = httpClientBuilder.build(); 67 | 68 | HttpPost httpPost = new HttpPost(mailGunUrl); 69 | 70 | List urlParameters = new ArrayList<>(); 71 | urlParameters.add(new BasicNameValuePair("from", from)); 72 | urlParameters.add(new BasicNameValuePair("to", to)); 73 | urlParameters.add(new BasicNameValuePair("subject", subject.toString())); 74 | urlParameters.add(new BasicNameValuePair("html", html.toString())); 75 | 76 | UrlEncodedFormEntity formEntity; 77 | try { 78 | formEntity = new UrlEncodedFormEntity(urlParameters); 79 | } catch (UnsupportedEncodingException e) { 80 | throw new RuntimeException("Failed to send MailGun email", e); 81 | } 82 | 83 | httpPost.setEntity(formEntity); 84 | 85 | try (CloseableHttpResponse httpResponse = httpClient.execute(httpPost)) { 86 | int statusCode = httpResponse.getStatusLine().getStatusCode(); 87 | try (BufferedReader reader = new BufferedReader(new InputStreamReader(httpResponse.getEntity().getContent()))) { 88 | String responseBody = IOUtils.toString(reader); 89 | if (statusCode != 200) { 90 | throw new RuntimeException(String.format("Failed to send MailGun email, got http response: %s, %s", statusCode, responseBody)); 91 | } 92 | } 93 | } catch (IOException e) { 94 | throw new RuntimeException("Failed to send MailGun email", e); 95 | } 96 | 97 | return null; 98 | } 99 | 100 | private String mergeFirstColumnData(DataSetResult dataSetResult) { 101 | StringBuilder sb = new StringBuilder(); 102 | for (List row : dataSetResult.getRows()) { 103 | String str = String.valueOf(row.get(0)); 104 | sb.append(str); 105 | } 106 | return sb.toString(); 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/SqlInputStatementExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.StatementAssignment; 20 | import com.uber.uberscriptquery.util.CredentialProvider; 21 | import org.apache.spark.sql.Dataset; 22 | import org.apache.spark.sql.Row; 23 | import org.apache.spark.sql.SparkSession; 24 | 25 | public interface SqlInputStatementExecutor { 26 | Dataset execute(SparkSession sparkSession, StatementAssignment statementAssignment, CredentialProvider credentialManager); 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/WriteCsvFileActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package com.uber.uberscriptquery.execution; 2 | 3 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 4 | import com.uber.uberscriptquery.util.CredentialProvider; 5 | import org.apache.spark.sql.Dataset; 6 | import org.apache.spark.sql.Row; 7 | import org.apache.spark.sql.SaveMode; 8 | import org.apache.spark.sql.SparkSession; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | public class WriteCsvFileActionStatementExecutor implements ActionStatementExecutor { 13 | public static final String ACTION_NAME = "writeCsvFile"; 14 | private static final Logger logger = LoggerFactory.getLogger(WriteCsvFileActionStatementExecutor.class); 15 | 16 | @Override 17 | public Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager) { 18 | 19 | String filePath = actionStatement.getParamValues().get(0).getValue().toString(); 20 | String saveModeStr = actionStatement.getParamValues().get(1).getValue().toString(); 21 | String dfTableName = actionStatement.getParamValues().get(2).getValue().toString(); 22 | 23 | SaveMode saveMode = SaveMode.valueOf(saveModeStr); 24 | 25 | String sql = String.format("select * from %s", dfTableName); 26 | logger.info(String.format("Running sql [%s] to get data and then save it", sql)); 27 | Dataset df = sparkSession.sql(sql); 28 | 29 | logger.info(String.format("Saving to csv %s, saveMode: %s", filePath, saveMode)); 30 | df.coalesce(1).write().mode(saveMode).option("header", "false").csv(filePath); 31 | logger.info(String.format("Saved to csv %s, saveMode: %s", filePath, saveMode)); 32 | return null; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/WriteJdbcActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 20 | import com.uber.uberscriptquery.util.CredentialProvider; 21 | import com.uber.uberscriptquery.util.SparkUtils; 22 | import org.apache.spark.sql.Dataset; 23 | import org.apache.spark.sql.Row; 24 | import org.apache.spark.sql.SaveMode; 25 | import org.apache.spark.sql.SparkSession; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.util.ArrayList; 30 | import java.util.Arrays; 31 | import java.util.List; 32 | 33 | public class WriteJdbcActionStatementExecutor implements ActionStatementExecutor { 34 | public static final String ACTION_NAME = "writeJdbc"; 35 | private static final Logger logger = LoggerFactory.getLogger(WriteJdbcActionStatementExecutor.class); 36 | 37 | @Override 38 | public Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager) { 39 | String connectionString = actionStatement.getParamValues().get(0).getValue().toString(); 40 | String outputTableName = actionStatement.getParamValues().get(1).getValue().toString(); 41 | String primaryKeysStr = actionStatement.getParamValues().get(2).getValue().toString(); 42 | String indexColumnsStr = actionStatement.getParamValues().get(3).getValue().toString(); 43 | String textColumnsStr = actionStatement.getParamValues().get(4).getValue().toString(); 44 | String saveModeStr = actionStatement.getParamValues().get(5).getValue().toString(); 45 | String dfTableName = actionStatement.getParamValues().get(6).getValue().toString(); 46 | 47 | int dfPartitionCount = 1; 48 | if (actionStatement.getParamValues().size() > 7) { 49 | String str = actionStatement.getParamValues().get(7).getValue().toString(); 50 | dfPartitionCount = Integer.parseInt(str); 51 | } 52 | 53 | double writesPerSecond = 100000; 54 | if (actionStatement.getParamValues().size() > 8) { 55 | writesPerSecond = Double.parseDouble(actionStatement.getParamValues().get(8).getValue().toString()); 56 | logger.info(String.format("Use writesPerSecond: %s for db table %s", writesPerSecond, outputTableName)); 57 | } 58 | 59 | String postWriteSql = null; 60 | if (actionStatement.getParamValues().size() > 9) { 61 | postWriteSql = actionStatement.getParamValues().get(9).getValue().toString(); 62 | } 63 | 64 | List primaryKeys = primaryKeysStr.isEmpty() ? new ArrayList<>() : Arrays.asList(primaryKeysStr.split(",")); 65 | List indexColumns = indexColumnsStr.isEmpty() ? new ArrayList<>() : Arrays.asList(indexColumnsStr.split(",")); 66 | List textColumns = textColumnsStr.isEmpty() ? new ArrayList<>() : Arrays.asList(textColumnsStr.split(",")); 67 | 68 | SaveMode saveMode = SaveMode.valueOf(saveModeStr); 69 | 70 | String sql = String.format("select * from %s", dfTableName); 71 | logger.info(String.format("Running sql [%s] to get data and then save it", sql)); 72 | Dataset df = sparkSession.sql(sql).coalesce(dfPartitionCount); 73 | SparkUtils.writeJdbc(df, connectionString, outputTableName, primaryKeys, indexColumns, textColumns, saveMode, postWriteSql, writesPerSecond); 74 | logger.info(String.format("Saved data [%s] to %s, %s", sql, outputTableName, saveMode)); 75 | 76 | return null; 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/WriteJsonFileActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package com.uber.uberscriptquery.execution; 2 | 3 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 4 | import com.uber.uberscriptquery.util.CredentialProvider; 5 | import org.apache.spark.sql.Dataset; 6 | import org.apache.spark.sql.Row; 7 | import org.apache.spark.sql.SaveMode; 8 | import org.apache.spark.sql.SparkSession; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | public class WriteJsonFileActionStatementExecutor implements ActionStatementExecutor { 13 | public static final String ACTION_NAME = "writeJsonFile"; 14 | private static final Logger logger = LoggerFactory.getLogger(WriteJsonFileActionStatementExecutor.class); 15 | 16 | @Override 17 | public Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager) { 18 | 19 | String filePath = actionStatement.getParamValues().get(0).getValue().toString(); 20 | String saveModeStr = actionStatement.getParamValues().get(1).getValue().toString(); 21 | String dfTableName = actionStatement.getParamValues().get(2).getValue().toString(); 22 | 23 | SaveMode saveMode = SaveMode.valueOf(saveModeStr); 24 | 25 | String sql = String.format("select * from %s", dfTableName); 26 | logger.info(String.format("Running sql [%s] to get data and then save it", sql)); 27 | Dataset df = sparkSession.sql(sql); 28 | 29 | logger.info(String.format("Saving to json %s, saveMode: %s", filePath, saveMode)); 30 | df.coalesce(1).write().mode(saveMode).json(filePath); 31 | logger.info(String.format("Saved to json %s, saveMode: %s", filePath, saveMode)); 32 | return null; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/execution/WriteParquetFileActionStatementExecutor.java: -------------------------------------------------------------------------------- 1 | package com.uber.uberscriptquery.execution; 2 | 3 | import com.uber.uberscriptquery.antlr4.parsing.ActionStatement; 4 | import com.uber.uberscriptquery.util.CredentialProvider; 5 | import org.apache.spark.sql.Dataset; 6 | import org.apache.spark.sql.Row; 7 | import org.apache.spark.sql.SaveMode; 8 | import org.apache.spark.sql.SparkSession; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | 12 | public class WriteParquetFileActionStatementExecutor implements ActionStatementExecutor { 13 | public static final String ACTION_NAME = "writeParquetFile"; 14 | private static final Logger logger = LoggerFactory.getLogger(WriteParquetFileActionStatementExecutor.class); 15 | 16 | @Override 17 | public Object execute(SparkSession sparkSession, ActionStatement actionStatement, CredentialProvider credentialManager) { 18 | 19 | String filePath = actionStatement.getParamValues().get(0).getValue().toString(); 20 | String saveModeStr = actionStatement.getParamValues().get(1).getValue().toString(); 21 | String dfTableName = actionStatement.getParamValues().get(2).getValue().toString(); 22 | 23 | SaveMode saveMode = SaveMode.valueOf(saveModeStr); 24 | 25 | String sql = String.format("select * from %s", dfTableName); 26 | logger.info(String.format("Running sql [%s] to get data and then save it", sql)); 27 | Dataset df = sparkSession.sql(sql); 28 | 29 | logger.info(String.format("Saving to parquet %s, saveMode: %s", filePath, saveMode)); 30 | df.coalesce(1).write().mode(saveMode).parquet(filePath); 31 | logger.info(String.format("Saved to parquet %s, saveMode: %s", filePath, saveMode)); 32 | return null; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/jdbc/DataSetResult.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.jdbc; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | public class DataSetResult { 25 | private List columnNames = new ArrayList<>(); 26 | private List> rows = new ArrayList<>(); 27 | 28 | public List getColumnNames() { 29 | return columnNames; 30 | } 31 | 32 | public void setColumnNames(List columnNames) { 33 | this.columnNames = columnNames; 34 | } 35 | 36 | public List> getRows() { 37 | return rows; 38 | } 39 | 40 | public void setRows(List> rows) { 41 | this.rows = rows; 42 | } 43 | 44 | public String getSingleStringValue() { 45 | if (rows == null || rows.isEmpty()) { 46 | return ""; 47 | } 48 | List values = rows.get(0); 49 | if (values == null || values.isEmpty()) { 50 | return ""; 51 | } 52 | return String.valueOf(values.get(0)); 53 | } 54 | 55 | public void print() { 56 | System.out.println(StringUtils.join(columnNames, ", ")); 57 | for (List row : rows) { 58 | System.out.println(StringUtils.join(row, ", ")); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/jdbc/JdbcConnectionProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.jdbc; 18 | 19 | import com.uber.uberscriptquery.util.SqlUtils; 20 | import org.apache.log4j.Logger; 21 | 22 | import java.sql.Connection; 23 | import java.sql.DriverManager; 24 | import java.sql.SQLException; 25 | 26 | public class JdbcConnectionProvider implements AutoCloseable { 27 | private static final Logger LOG = Logger.getLogger(JdbcConnectionProvider.class); 28 | 29 | private String jdbcConnectionString; 30 | 31 | private Connection connection; 32 | 33 | public JdbcConnectionProvider(String jdbcConnectionString) { 34 | this.jdbcConnectionString = jdbcConnectionString; 35 | } 36 | 37 | public String getJdbcConnectionString() { 38 | return jdbcConnectionString; 39 | } 40 | 41 | public synchronized Connection getConnection() { 42 | if (connection != null) { 43 | try { 44 | if (!connection.isClosed()) { 45 | return connection; 46 | } 47 | } catch (SQLException e) { 48 | LOG.warn(String.format("Failed to check whether jdbc connection is close: %s", jdbcConnectionString), e); 49 | } 50 | } 51 | 52 | SqlUtils.loadJdbcDriverClass(jdbcConnectionString); 53 | 54 | try { 55 | connection = DriverManager.getConnection(jdbcConnectionString); 56 | return connection; 57 | } catch (SQLException e) { 58 | throw new RuntimeException("Failed to create jdbc connection: " + jdbcConnectionString, e); 59 | } 60 | } 61 | 62 | @Override 63 | public synchronized void close() { 64 | if (connection != null) { 65 | try { 66 | connection.close(); 67 | connection = null; 68 | } catch (SQLException e) { 69 | LOG.warn("Failed to close jdbc connection: " + jdbcConnectionString, e); 70 | } 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/jdbc/JdbcUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.jdbc; 18 | 19 | import org.apache.commons.lang3.StringUtils; 20 | import org.apache.log4j.Logger; 21 | import org.apache.spark.sql.types.DataTypes; 22 | 23 | import java.sql.*; 24 | import java.util.*; 25 | import java.util.Date; 26 | 27 | public class JdbcUtils { 28 | private static final Logger LOG = Logger.getLogger(JdbcUtils.class); 29 | 30 | public static String getCreateTableSql(List columnNames, List columnTypes, String tableName, List primaryKeys, List indexColumns, List textColumns) { 31 | if (columnNames.size() != columnTypes.size()) { 32 | throw new RuntimeException(String.format("Number of column names (%s) is different from number of column types (%s)", columnNames.size(), columnTypes.size())); 33 | } 34 | 35 | if (indexColumns == null) { 36 | indexColumns = new ArrayList<>(); 37 | } 38 | 39 | if (textColumns == null) { 40 | textColumns = new ArrayList<>(); 41 | } 42 | 43 | LOG.info(String.format("Getting sql to create table, column name: %s, column types: %s, table name: %s, primary keys: %s, index columns: %s, text columns: %s", 44 | StringUtils.join(columnNames, ','), 45 | StringUtils.join(columnTypes, ','), 46 | tableName, 47 | StringUtils.join(primaryKeys, ','), 48 | StringUtils.join(indexColumns, ','), 49 | StringUtils.join(textColumns, ','))); 50 | 51 | Set primaryKeysLowerCase = new HashSet<>(); 52 | for (String entry : primaryKeys) { 53 | primaryKeysLowerCase.add(entry.toLowerCase()); 54 | } 55 | 56 | Set indexColumnsLowerCase = new HashSet<>(); 57 | for (String entry : indexColumns) { 58 | indexColumnsLowerCase.add(entry.toLowerCase()); 59 | } 60 | 61 | Set textColumnsLowerCase = new HashSet<>(); 62 | for (String entry : textColumns) { 63 | textColumnsLowerCase.add(entry.toLowerCase()); 64 | } 65 | 66 | StringBuilder sb = new StringBuilder(); 67 | sb.append(String.format("CREATE TABLE IF NOT EXISTS %s (", tableName)); 68 | 69 | List primaryKeyTypes = new ArrayList<>(); 70 | List matchedPrimaryKeys = new ArrayList<>(); 71 | 72 | List indexColumnTypes = new ArrayList<>(); 73 | List matchedIndexColumns = new ArrayList<>(); 74 | 75 | List otherColumnNames = new ArrayList<>(); 76 | List otherColumnTypes = new ArrayList<>(); 77 | 78 | for (int i = 0; i < columnNames.size(); i++) { 79 | String columnName = columnNames.get(i); 80 | org.apache.spark.sql.types.DataType columnType = columnTypes.get(i); 81 | if (primaryKeysLowerCase.contains(columnName.toLowerCase())) { 82 | primaryKeyTypes.add(columnType); 83 | matchedPrimaryKeys.add(columnName); 84 | } 85 | if (indexColumnsLowerCase.contains(columnName.toLowerCase())) { 86 | indexColumnTypes.add(columnType); 87 | matchedIndexColumns.add(columnName); 88 | } 89 | if (!primaryKeysLowerCase.contains(columnName.toLowerCase()) 90 | && !indexColumnsLowerCase.contains(columnName.toLowerCase())) { 91 | otherColumnNames.add(columnName); 92 | otherColumnTypes.add(columnType); 93 | } 94 | } 95 | 96 | if (primaryKeyTypes.size() != primaryKeys.size()) { 97 | throw new RuntimeException(String.format("Invalid primary keys. There are %s in matched primary keys (%s), but %s in given primary keys (%s)", 98 | primaryKeyTypes.size(), 99 | StringUtils.join(matchedPrimaryKeys, ','), 100 | primaryKeys.size(), 101 | StringUtils.join(primaryKeys, ','))); 102 | } 103 | 104 | if (indexColumnTypes.size() != indexColumns.size()) { 105 | throw new RuntimeException(String.format("Invalid index fields. There are %s matched index columns (%s), but %s in given index columns (%s)", 106 | indexColumnTypes.size(), 107 | StringUtils.join(matchedIndexColumns, ','), 108 | indexColumns.size(), 109 | StringUtils.join(indexColumns, ','))); 110 | } 111 | 112 | // Order columns in sequence: Primary Keys, Index Columns (exclude primary keys), Other Columns 113 | List reorderedColumnNames = new ArrayList<>(); 114 | List reorderedColumnTypes = new ArrayList<>(); 115 | 116 | 117 | reorderedColumnNames.addAll(matchedPrimaryKeys); 118 | reorderedColumnTypes.addAll(primaryKeyTypes); 119 | 120 | for (int i = 0; i < matchedIndexColumns.size(); i++) { 121 | String columnName = matchedIndexColumns.get(i); 122 | if (primaryKeysLowerCase.contains(columnName.toLowerCase())) { 123 | continue; 124 | } 125 | reorderedColumnNames.add(columnName); 126 | reorderedColumnTypes.add(indexColumnTypes.get(i)); 127 | } 128 | 129 | reorderedColumnNames.addAll(otherColumnNames); 130 | reorderedColumnTypes.addAll(otherColumnTypes); 131 | 132 | if (reorderedColumnNames.size() != columnNames.size()) { 133 | throw new RuntimeException(String.format("Invalid columns after re-ordering: %s (%s) v.s. %s (%s)", 134 | reorderedColumnNames.size(), 135 | StringUtils.join(reorderedColumnNames, ','), 136 | columnNames.size(), 137 | StringUtils.join(columnNames, ','))); 138 | } 139 | 140 | 141 | sb.append(reorderedColumnNames.get(0)); 142 | sb.append(" "); 143 | sb.append(getJdbcTypeString( 144 | reorderedColumnTypes.get(0), 145 | primaryKeysLowerCase.contains(reorderedColumnNames.get(0).toLowerCase()) || indexColumnsLowerCase.contains(reorderedColumnNames.get(0).toLowerCase()), 146 | textColumnsLowerCase.contains(reorderedColumnNames.get(0).toLowerCase()))); 147 | 148 | for (int i = 1; i < reorderedColumnNames.size(); i++) { 149 | sb.append(", "); 150 | sb.append(reorderedColumnNames.get(i)); 151 | sb.append(" "); 152 | sb.append(getJdbcTypeString( 153 | reorderedColumnTypes.get(i), 154 | primaryKeysLowerCase.contains(reorderedColumnNames.get(i).toLowerCase()) || indexColumnsLowerCase.contains(reorderedColumnNames.get(i).toLowerCase()), 155 | textColumnsLowerCase.contains(reorderedColumnNames.get(i).toLowerCase()))); 156 | } 157 | 158 | if (!primaryKeyTypes.isEmpty()) { 159 | sb.append(", PRIMARY KEY("); 160 | sb.append(primaryKeys.get(0)); 161 | for (int i = 1; i < primaryKeyTypes.size(); i++) { 162 | sb.append(", "); 163 | sb.append(primaryKeys.get(i)); 164 | } 165 | sb.append(")"); 166 | } 167 | 168 | for (int i = 0; i < indexColumnTypes.size(); i++) { 169 | sb.append(String.format(", INDEX index_%s (%s)", 170 | indexColumns.get(i), 171 | indexColumns.get(i))); 172 | } 173 | 174 | sb.append(")"); 175 | return sb.toString(); 176 | } 177 | 178 | public static String getJdbcTypeString(Class valueClass, boolean isPrimaryKeyOrUniqueKey, boolean isDatetime, boolean isText) { 179 | int maxVarcharLength = isPrimaryKeyOrUniqueKey ? 150 : 250; 180 | String sqlTypeForString = isText ? "TEXT" : String.format("VARCHAR(%s)", maxVarcharLength); 181 | if (isDatetime || valueClass.equals(Date.class)) { 182 | return "DATETIME"; 183 | } else if (valueClass.equals(String.class)) { 184 | return sqlTypeForString; 185 | } else if (valueClass.equals(Integer.class)) { 186 | return "INT"; 187 | } else if (valueClass.equals(Long.class)) { 188 | return "BIGINT"; 189 | } else if (valueClass.equals(Float.class)) { 190 | return "FLOAT"; 191 | } else if (valueClass.equals(Double.class)) { 192 | return "DOUBLE"; 193 | } else if (valueClass.equals(Boolean.class)) { 194 | return "TINYINT"; 195 | } else { 196 | throw new RuntimeException(String.format("Unsupported property type for JDBC: %s", valueClass)); 197 | } 198 | } 199 | 200 | public static String getJdbcTypeString(org.apache.spark.sql.types.DataType dataType, boolean isPrimaryKeyOrIndexKey, boolean isText) { 201 | int maxVarcharLength = isPrimaryKeyOrIndexKey ? 150 : 250; 202 | String sqlTypeForString = isText ? "TEXT" : String.format("VARCHAR(%s)", maxVarcharLength); 203 | if (dataType == DataTypes.TimestampType || dataType == DataTypes.DateType) { 204 | return "DATETIME"; 205 | } else if (dataType == DataTypes.StringType) { 206 | return sqlTypeForString; 207 | } else if (dataType == DataTypes.IntegerType) { 208 | return "INT"; 209 | } else if (dataType == DataTypes.LongType) { 210 | return "BIGINT"; 211 | } else if (dataType == DataTypes.FloatType) { 212 | return "FLOAT"; 213 | } else if (dataType == DataTypes.DoubleType) { 214 | return "DOUBLE"; 215 | } else if (dataType == DataTypes.BooleanType) { 216 | return "TINYINT"; 217 | } else if (dataType == DataTypes.ByteType) { 218 | return "SMALLINT"; 219 | } else if (dataType instanceof org.apache.spark.sql.types.DecimalType) { 220 | org.apache.spark.sql.types.DecimalType decimalType = (org.apache.spark.sql.types.DecimalType) dataType; 221 | return String.format("DECIMAL(%d,%d)", decimalType.precision(), decimalType.scale()); 222 | } else { 223 | throw new RuntimeException(String.format("Unsupported property type for JDBC: %s", dataType)); 224 | } 225 | } 226 | 227 | public static List> executeQuery(String jdbcConnectionString, String sql) { 228 | LOG.info("Running sql: " + sql); 229 | try (Connection connection = getConnection(jdbcConnectionString)) { 230 | try (PreparedStatement stmt = connection.prepareStatement(sql)) { 231 | ResultSet resultSet = stmt.executeQuery(); 232 | List> result = new ArrayList>(); 233 | while (resultSet.next()) { 234 | List row = new ArrayList<>(); 235 | for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { 236 | row.add(resultSet.getObject(i + 1)); 237 | } 238 | result.add(row); 239 | } 240 | LOG.info("Finished sql: " + sql); 241 | return result; 242 | } 243 | } catch (SQLException e) { 244 | throw new RuntimeException("Failed to execute sql: " + sql, e); 245 | } 246 | } 247 | 248 | public static Connection getConnection(String jdbcConnectionString) { 249 | try { 250 | return DriverManager.getConnection(jdbcConnectionString); 251 | } catch (SQLException e) { 252 | throw new RuntimeException("Failed to create jdbc connection: " + jdbcConnectionString, e); 253 | } 254 | } 255 | } 256 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/BufferedCredentialProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import com.uber.uberscriptquery.execution.CredentialEntry; 20 | import org.apache.commons.lang3.StringUtils; 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | import java.util.ArrayList; 25 | import java.util.List; 26 | 27 | public class BufferedCredentialProvider implements java.io.Serializable, CredentialProvider { 28 | private static final Logger logger = LoggerFactory.getLogger(BufferedCredentialProvider.class); 29 | 30 | private List credentialEntries = new ArrayList<>(); 31 | 32 | public void addPassword(String passwordFile, String passwordEntry, String passwordValue) { 33 | CredentialEntry credentialEntry = new CredentialEntry(); 34 | credentialEntry.setPasswordFile(passwordFile); 35 | credentialEntry.setPasswordEntry(passwordEntry); 36 | credentialEntry.setPasswordValue(passwordValue); 37 | 38 | credentialEntries.add(credentialEntry); 39 | } 40 | 41 | @Override 42 | public String getPassword(String passwordFile, String passwordEntry) { 43 | for (CredentialEntry entry : credentialEntries) { 44 | if (StringUtils.equals(entry.getPasswordFile(), passwordFile) 45 | && StringUtils.equals(entry.getPasswordEntry(), passwordEntry)) { 46 | return entry.getPasswordValue(); 47 | } 48 | } 49 | return null; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/CounterMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import java.util.HashMap; 20 | import java.util.Map; 21 | 22 | public class CounterMap implements java.io.Serializable { 23 | private Map map = new HashMap<>(); 24 | 25 | public synchronized void increase(String name) { 26 | Long oldValue = map.get(name); 27 | if (oldValue == null) { 28 | map.put(name, 1L); 29 | } else { 30 | map.put(name, oldValue + 1); 31 | } 32 | } 33 | 34 | public synchronized long getCount(String name) { 35 | Long oldValue = map.get(name); 36 | if (oldValue == null) { 37 | return 0L; 38 | } else { 39 | return oldValue; 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/CredentialProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import java.io.Serializable; 20 | 21 | public interface CredentialProvider extends Serializable { 22 | String getPassword(String passwordFile, String passwordEntry); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/DateTimeUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import java.util.Calendar; 20 | import java.util.Date; 21 | import java.util.TimeZone; 22 | 23 | public class DateTimeUtils { 24 | 25 | public static long getMillisSmart(double value) { 26 | Date dt = new Date((long) value); 27 | Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 28 | cal.setTime(dt); 29 | int year = cal.get(Calendar.YEAR); 30 | if (year >= 2000 && year < 3000) { 31 | return (long) value; 32 | } 33 | return (long) (value * 1000); 34 | } 35 | 36 | public static long getMillisSmart(long value) { 37 | Date dt = new Date(value); 38 | Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC")); 39 | cal.setTime(dt); 40 | int year = cal.get(Calendar.YEAR); 41 | if (year >= 2000 && year < 3000) { 42 | return value; 43 | } 44 | return value * 1000; 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/DummyCredentialProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | public class DummyCredentialProvider implements CredentialProvider { 20 | public String getPassword(String passwordFile, String passwordEntry) { 21 | return ""; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/ExponentialBackoffRetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import com.google.common.util.concurrent.Uninterruptibles; 20 | import org.apache.log4j.Logger; 21 | 22 | import java.util.concurrent.Callable; 23 | import java.util.concurrent.ThreadLocalRandom; 24 | import java.util.concurrent.TimeUnit; 25 | 26 | public class ExponentialBackoffRetryPolicy implements RetryPolicy, java.io.Serializable { 27 | private static final Logger LOG = Logger.getLogger(ExponentialBackoffRetryPolicy.class); 28 | 29 | private final int maximumAttemptCount; 30 | private final long minimumMilliseconds; 31 | private final float retryScaleFactor; 32 | 33 | public ExponentialBackoffRetryPolicy(int maximumAttemptCount, long minimumMilliseconds) { 34 | this(maximumAttemptCount, minimumMilliseconds, 2.0f); 35 | } 36 | 37 | public ExponentialBackoffRetryPolicy(int maximumAttemptCount, long minimumMilliseconds, float retryScaleFactor) { 38 | this.maximumAttemptCount = maximumAttemptCount; 39 | this.minimumMilliseconds = minimumMilliseconds; 40 | this.retryScaleFactor = retryScaleFactor; 41 | } 42 | 43 | @Override 44 | public T attempt(Callable callable) { 45 | ThreadLocalRandom random = ThreadLocalRandom.current(); 46 | int remainingAttempts = maximumAttemptCount - 1; 47 | long minimumSleepTime = minimumMilliseconds; 48 | long maximumSleepTime = (long) (minimumSleepTime * retryScaleFactor); 49 | 50 | Throwable previousException; 51 | 52 | try { 53 | return callable.call(); 54 | } catch (Throwable ex) { 55 | if (remainingAttempts <= 0) { 56 | throw new RuntimeException("Failed on first try and there is no remaining retry", ex); 57 | } 58 | previousException = ex; 59 | } 60 | 61 | while (remainingAttempts > 0) { 62 | long sleepTime = random.nextLong(minimumSleepTime, maximumSleepTime); 63 | LOG.info(String.format("Sleeping %s milliseconds and retrying on exception: %s", sleepTime, previousException)); 64 | Uninterruptibles.sleepUninterruptibly(sleepTime, TimeUnit.MILLISECONDS); 65 | try { 66 | return callable.call(); 67 | } catch (Throwable ex) { 68 | previousException = ex; 69 | } 70 | remainingAttempts--; 71 | minimumSleepTime *= retryScaleFactor; 72 | maximumSleepTime *= retryScaleFactor; 73 | } 74 | 75 | String msg = String.format("Failed after tried %s times", maximumAttemptCount); 76 | throw new RuntimeException(msg, previousException); 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/FileUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import java.io.IOException; 20 | import java.nio.file.Files; 21 | import java.nio.file.Path; 22 | import java.nio.file.Paths; 23 | 24 | public class FileUtils { 25 | 26 | public static String getSymbolicLinkTargetFile(String path) { 27 | Path result = getSymbolicLinkTargetFile(Paths.get(path)); 28 | return result.toString(); 29 | } 30 | 31 | public static Path getSymbolicLinkTargetFile(Path path) { 32 | if (!Files.isSymbolicLink(path)) { 33 | return path; 34 | } 35 | 36 | try { 37 | Path targetPath = Files.readSymbolicLink(path); 38 | return getSymbolicLinkTargetFile(targetPath); 39 | } catch (IOException e) { 40 | throw new RuntimeException("Failed to read symbolic link: " + path, e); 41 | } 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/HttpUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import org.apache.commons.io.IOUtils; 20 | import org.apache.http.client.methods.CloseableHttpResponse; 21 | import org.apache.http.client.methods.HttpGet; 22 | import org.apache.http.client.methods.HttpPost; 23 | import org.apache.http.entity.StringEntity; 24 | import org.apache.http.impl.client.CloseableHttpClient; 25 | import org.apache.http.impl.client.HttpClients; 26 | import org.slf4j.Logger; 27 | import org.slf4j.LoggerFactory; 28 | 29 | import java.io.BufferedReader; 30 | import java.io.InputStreamReader; 31 | import java.util.Map; 32 | 33 | public class HttpUtils { 34 | public static final int OK_CODE = 200; 35 | private static final Logger logger = LoggerFactory.getLogger(HttpUtils.class); 36 | 37 | public static HttpResponse get(String url, Map header) { 38 | int statusCode = 0; 39 | String responseBody = ""; 40 | 41 | try { 42 | logger.info(String.format("Getting url: %s", url)); 43 | try (CloseableHttpClient httpClient = HttpClients.createDefault()) { 44 | HttpGet httpGet = new HttpGet(url); 45 | if (header != null && !header.isEmpty()) { 46 | for (Map.Entry entry : header.entrySet()) { 47 | httpGet.addHeader(entry.getKey(), entry.getValue()); 48 | } 49 | } 50 | try (CloseableHttpResponse httpResponse = httpClient.execute(httpGet)) { 51 | statusCode = httpResponse.getStatusLine().getStatusCode(); 52 | try (BufferedReader reader = new BufferedReader(new InputStreamReader(httpResponse.getEntity().getContent()))) { 53 | responseBody = IOUtils.toString(reader); 54 | logger.info(String.format("Finished getting url: %s, Http status: %s, Response body: %s", 55 | url, statusCode, truncateLogString(responseBody))); 56 | } 57 | } 58 | } 59 | return new HttpResponse(statusCode, responseBody); 60 | } catch (Exception ex) { 61 | String msg = String.format("Failed getting url: %s", url); 62 | logger.warn(msg, ex); 63 | throw new RuntimeException(msg, ex); 64 | } 65 | } 66 | 67 | public static HttpResponse post(String url, Map header, String body) { 68 | int statusCode = 0; 69 | String responseBody = ""; 70 | 71 | try { 72 | logger.info(String.format("Getting url: %s", url)); 73 | try (CloseableHttpClient httpClient = HttpClients.createDefault()) { 74 | HttpPost httpPost = new HttpPost(url); 75 | if (header != null && !header.isEmpty()) { 76 | for (Map.Entry entry : header.entrySet()) { 77 | httpPost.addHeader(entry.getKey(), entry.getValue()); 78 | } 79 | } 80 | httpPost.setEntity(new StringEntity(body)); 81 | try (CloseableHttpResponse httpResponse = httpClient.execute(httpPost)) { 82 | statusCode = httpResponse.getStatusLine().getStatusCode(); 83 | try (BufferedReader reader = new BufferedReader(new InputStreamReader(httpResponse.getEntity().getContent()))) { 84 | responseBody = IOUtils.toString(reader); 85 | logger.info(String.format("Finished getting url: %s, Http status: %s, Response body: %s", 86 | url, statusCode, truncateLogString(responseBody))); 87 | } 88 | } 89 | } 90 | return new HttpResponse(statusCode, responseBody); 91 | } catch (Exception ex) { 92 | String msg = String.format("Failed getting url: %s", url); 93 | logger.warn(msg, ex); 94 | throw new RuntimeException(msg, ex); 95 | } 96 | } 97 | 98 | private static String truncateLogString(String str) { 99 | int maxSize = 1000; 100 | if (str == null || str.length() <= maxSize) { 101 | return str; 102 | } 103 | 104 | return str.substring(0, maxSize) + "..."; 105 | } 106 | 107 | public static class HttpResponse { 108 | 109 | private int code; 110 | private String body; 111 | 112 | public HttpResponse(int code, String body) { 113 | super(); 114 | this.code = code; 115 | this.body = body; 116 | } 117 | 118 | public int getCode() { 119 | return code; 120 | } 121 | 122 | public String getBody() { 123 | return body; 124 | } 125 | 126 | @Override 127 | public String toString() { 128 | return "HttpResponse [code=" + code + ", body=" + body + "]"; 129 | } 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/JsonFileCredentialProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import org.apache.commons.io.IOUtils; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import java.io.FileReader; 24 | import java.io.IOException; 25 | 26 | public class JsonFileCredentialProvider implements java.io.Serializable, CredentialProvider { 27 | private static final Logger logger = LoggerFactory.getLogger(JsonFileCredentialProvider.class); 28 | 29 | @Override 30 | public String getPassword(String passwordFile, String passwordEntry) { 31 | if (passwordFile == null || passwordFile.isEmpty()) { 32 | throw new RuntimeException("passwordFile is null or empty"); 33 | } 34 | 35 | String passwordFileOriginalValue = passwordFile; 36 | passwordFile = com.uber.uberscriptquery.util.FileUtils.getSymbolicLinkTargetFile(passwordFile); 37 | logger.info(String.format("Reading password file: %s (%s)", passwordFileOriginalValue, passwordFile)); 38 | String password; 39 | final String fileContent; 40 | try (FileReader fileReader = new FileReader(passwordFile)) { 41 | fileContent = IOUtils.toString(fileReader); 42 | } catch (IOException e) { 43 | throw new RuntimeException(String.format("Failed to read password file %s on current machine %s", passwordFile, NetworkUtils.getLocalHostName()), e); 44 | } 45 | if (passwordEntry == null || passwordEntry.isEmpty()) { 46 | password = fileContent; 47 | } else { 48 | password = JsonPathUtils.getJsonPathAsString(fileContent, passwordEntry); 49 | } 50 | 51 | return password; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/JsonPathUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import com.jayway.jsonpath.Configuration; 20 | import com.jayway.jsonpath.JsonPath; 21 | import com.jayway.jsonpath.Option; 22 | 23 | public class JsonPathUtils { 24 | 25 | public static String getJsonPathAsString(String json, String path) { 26 | Configuration conf = Configuration.defaultConfiguration().addOptions(Option.DEFAULT_PATH_LEAF_TO_NULL).addOptions(Option.SUPPRESS_EXCEPTIONS); 27 | Object document = conf.jsonProvider().parse(json); 28 | Object jsonObj = JsonPath.read(document, path); 29 | return convertJsonPathValueToString(jsonObj); 30 | } 31 | 32 | private static String convertJsonPathValueToString(Object jsonPathValue) { 33 | if (jsonPathValue == null) { 34 | return null; 35 | } 36 | 37 | if (jsonPathValue instanceof String) { 38 | return (String) jsonPathValue; 39 | } 40 | 41 | return JsonUtils.serialize(jsonPathValue); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/JsonUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | 20 | import com.fasterxml.jackson.core.JsonParser.Feature; 21 | import com.fasterxml.jackson.core.JsonProcessingException; 22 | import com.fasterxml.jackson.core.type.TypeReference; 23 | import com.fasterxml.jackson.databind.ObjectMapper; 24 | 25 | import java.io.IOException; 26 | import java.util.List; 27 | 28 | public class JsonUtils { 29 | protected static ObjectMapper mapper = new ObjectMapper(); 30 | 31 | static { 32 | mapper.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); 33 | mapper.configure(Feature.ALLOW_SINGLE_QUOTES, true); 34 | mapper.configure(Feature.ALLOW_COMMENTS, true); 35 | } 36 | 37 | public static String serialize(Object obj) { 38 | if (obj == null) { 39 | return ""; 40 | } else { 41 | try { 42 | return mapper.writeValueAsString(obj); 43 | } catch (JsonProcessingException ex) { 44 | throw new RuntimeException(String.format("Failed to serialize %s (%s)", obj, obj.getClass()), ex); 45 | } 46 | } 47 | } 48 | 49 | public static T deserialize(String content, Class valueType) { 50 | try { 51 | return mapper.readValue(content, valueType); 52 | } catch (IOException ex) { 53 | throw new RuntimeException(String.format("Failed to deserialize %s from json %s", valueType, content), ex); 54 | } 55 | } 56 | 57 | public static List deserializeList(String content, Class valueType) { 58 | try { 59 | return mapper.readValue(content, new TypeReference>() { 60 | }); 61 | } catch (IOException ex) { 62 | throw new RuntimeException(String.format("Failed to deserialize %s from json %s", valueType, content), ex); 63 | } 64 | } 65 | } -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/NetworkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import java.net.InetAddress; 20 | import java.util.Map; 21 | 22 | public class NetworkUtils { 23 | public static String getLocalHostName() { 24 | try { 25 | Map env = System.getenv(); 26 | if (env.containsKey("COMPUTERNAME")) 27 | return env.get("COMPUTERNAME"); 28 | else if (env.containsKey("HOSTNAME")) 29 | return env.get("HOSTNAME"); 30 | else 31 | return InetAddress.getLocalHost().getHostName(); 32 | } catch (Throwable e) { 33 | return "unknown_localhost_name"; 34 | } 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/ResourceUtils.java: -------------------------------------------------------------------------------- 1 | package com.uber.uberscriptquery.util; 2 | 3 | 4 | /* 5 | * Copyright (c) 2017 Uber Technologies, Inc. 6 | * 7 | * Licensed under the Apache License, Version 2.0 (the "License"); 8 | * you may not use this file except in compliance with the License. 9 | * You may obtain a copy of the License at 10 | * 11 | * http://www.apache.org/licenses/LICENSE-2.0 12 | * 13 | * Unless required by applicable law or agreed to in writing, software 14 | * distributed under the License is distributed on an "AS IS" BASIS, 15 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 16 | * See the License for the specific language governing permissions and 17 | * limitations under the License. 18 | */ 19 | 20 | import org.apache.commons.io.IOUtils; 21 | 22 | import java.io.InputStream; 23 | import java.util.Arrays; 24 | import java.util.List; 25 | 26 | public class ResourceUtils { 27 | public static String readResource(Class clazz, String resourceName) { 28 | try (InputStream in = clazz.getResourceAsStream(resourceName)) { 29 | return IOUtils.toString(in); 30 | } catch (Throwable ex) { 31 | throw new RuntimeException(ex); 32 | } 33 | } 34 | 35 | public static String readResource(String resourceName) { 36 | try (InputStream in = ResourceUtils.class.getClassLoader().getResourceAsStream(resourceName)) { 37 | return IOUtils.toString(in); 38 | } catch (Throwable ex) { 39 | throw new RuntimeException(ex); 40 | } 41 | } 42 | 43 | public static List readLinesFromResource(Class clazz, String resourceName) { 44 | String text = readResource(clazz, resourceName); 45 | return Arrays.asList(text.split("\n")); 46 | } 47 | 48 | } -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/RetryPolicy.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import java.util.concurrent.Callable; 20 | 21 | public interface RetryPolicy { 22 | T attempt(Callable callable); 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/SparkUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import com.uber.uberscriptquery.jdbc.DataSetResult; 20 | import org.apache.spark.api.java.JavaRDD; 21 | import org.apache.spark.api.java.JavaSparkContext; 22 | import org.apache.spark.api.java.function.FlatMapFunction; 23 | import org.apache.spark.api.java.function.Function; 24 | import org.apache.spark.sql.*; 25 | import org.apache.spark.sql.types.DataTypes; 26 | import org.apache.spark.sql.types.Metadata; 27 | import org.apache.spark.sql.types.StructField; 28 | import org.apache.spark.sql.types.StructType; 29 | import org.apache.spark.storage.StorageLevel; 30 | import org.slf4j.Logger; 31 | import org.slf4j.LoggerFactory; 32 | import scala.Tuple2; 33 | 34 | import java.sql.*; 35 | import java.util.*; 36 | 37 | public class SparkUtils { 38 | public final static String HDFS_PREFIX_LOWERCASE = "hdfs://"; 39 | private static final Logger logger = LoggerFactory.getLogger(SparkUtils.class); 40 | 41 | public static Dataset loadFile(String inputFormat, String inputPath, SparkSession spark) { 42 | if (inputFormat == null || inputFormat.isEmpty() || inputFormat.equalsIgnoreCase("text")) { 43 | return spark.read().text(inputPath); 44 | } else if (inputFormat.equalsIgnoreCase("parquet")) { 45 | return spark.read().parquet(inputPath); 46 | } else if (inputFormat.equalsIgnoreCase("csv")) { 47 | return spark.read().option("header", "false").csv(inputPath); 48 | } else if (inputFormat.equalsIgnoreCase("csv_with_header")) { 49 | return spark.read().option("header", "true").csv(inputPath); 50 | } else if (inputFormat.equalsIgnoreCase("json")) { 51 | return spark.read().json(inputPath); 52 | } else { 53 | throw new RuntimeException(String.format("Unsupported inputFormat: %s, %s", inputFormat, inputPath)); 54 | } 55 | } 56 | 57 | public static void writeParquet(Dataset df, String outputPath, SaveMode saveMode, int numPartitions) { 58 | logger.info(String.format("Saving parquet file %s, saveMode: %s, numPartitions: %s", outputPath, saveMode, numPartitions)); 59 | String hdfsOutputPath = outputPath; 60 | if (hdfsOutputPath.toLowerCase().startsWith(HDFS_PREFIX_LOWERCASE)) { 61 | hdfsOutputPath = hdfsOutputPath.substring(HDFS_PREFIX_LOWERCASE.length()); 62 | } 63 | df.coalesce(numPartitions).write().mode(saveMode).parquet(hdfsOutputPath); 64 | logger.info(String.format("Saved parquet file %s, saveMode: %s, numPartitions: %s", outputPath, saveMode, numPartitions)); 65 | } 66 | 67 | // There is always issue like following when use spark.read().format("jdbc").options(jdbcMap).load(), so we write our own to load data from jdbc. 68 | // 16/09/20 00:16:19 ERROR yarn.ApplicationMaster: User class threw exception: com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near '...' at line 1 69 | // com.mysql.jdbc.exceptions.jdbc4.MySQLSyntaxErrorException: You have an error in your SQL syntax; check the manual that corresponds to your MySQL server version for the right syntax to use near 'SELECT city_name FROM dataquerier.fifo_supplies_01 WHERE 1=0' at line 1 70 | public static Dataset readJdbc(String jdbcUrlWithPassword, String sql, SparkSession spark) { 71 | JavaRDD, StructType>> javaRdd1 = JavaSparkContext.fromSparkContext(spark.sparkContext()) 72 | .parallelize(Arrays.asList(0)) 73 | .map(new Function, StructType>>() { 74 | @Override 75 | public Tuple2, StructType> call(Integer v1) throws Exception { 76 | Tuple2, StructType> tuple = new ExponentialBackoffRetryPolicy, StructType>>(3, 100) 77 | .attempt(() -> readJdbcAndReturnRowsAndSchema(jdbcUrlWithPassword, sql)); 78 | return tuple; 79 | } 80 | }); 81 | 82 | javaRdd1 = javaRdd1.persist(StorageLevel.DISK_ONLY_2()); 83 | 84 | StructType structType = javaRdd1.map(new Function, StructType>, StructType>() { 85 | @Override 86 | public StructType call(Tuple2, StructType> v1) throws Exception { 87 | return v1._2(); 88 | } 89 | }).collect().get(0); 90 | 91 | JavaRDD javaRdd2 = javaRdd1.flatMap(new FlatMapFunction, StructType>, Row>() { 92 | @Override 93 | public Iterator call(Tuple2, StructType> listStructTypeTuple2) throws Exception { 94 | return listStructTypeTuple2._1().iterator(); 95 | } 96 | }); 97 | 98 | return spark.createDataFrame(javaRdd2, structType); 99 | 100 | //SqlUtils.loadJdbcDriverClass(jdbcUrlWithPassword); 101 | //DriverRegistryWrapper.register(com.mysql.jdbc.Driver.class.getName()); 102 | //ExponentialBackoffRetryPolicy> retryPolicy = new ExponentialBackoffRetryPolicy<>(3, 100); 103 | //return retryPolicy.attempt(() -> readJdbcWithoutRetry(jdbcUrlWithPassword, sql, spark)); 104 | } 105 | 106 | //private static Dataset readJdbcWithoutRetry(String jdbcUrlWithPassword, String sql, SparkSession spark) { 107 | // Tuple2, StructType> tuple2 = readJdbcAndReturnRowsAndSchema(jdbcUrlWithPassword, sql); 108 | // Dataset df = spark.createDataFrame(tuple2._1(), tuple2._2()); 109 | // return df; 110 | //} 111 | 112 | private static Tuple2, StructType> readJdbcAndReturnRowsAndSchema(String jdbcUrlWithPassword, String sql) { 113 | SqlUtils.loadJdbcDriverClass(jdbcUrlWithPassword); 114 | logger.info("Running jdbc sql: " + sql); 115 | try (Connection con = DriverManager.getConnection(jdbcUrlWithPassword)) { 116 | PreparedStatement stmt = con.prepareStatement(sql); 117 | ResultSet resultSet = stmt.executeQuery(); 118 | logger.info(String.format("Finished jdbc sql: %s", sql)); 119 | 120 | ResultSetMetaData resultSetMetaData = resultSet.getMetaData(); 121 | int[] sqlTypes = new int[resultSetMetaData.getColumnCount()]; 122 | StructField[] structFields = new StructField[resultSetMetaData.getColumnCount()]; 123 | for (int i = 0; i < structFields.length; i++) { 124 | int sqlType = resultSetMetaData.getColumnType(i + 1); 125 | sqlTypes[i] = sqlType; 126 | org.apache.spark.sql.types.DataType dataType = convertSqlTypeToSparkSqlDataType(sqlType); 127 | String column = resultSetMetaData.getColumnName(i + 1); 128 | StructField structField = new StructField(column, dataType, true, Metadata.empty()); 129 | structFields[i] = structField; 130 | } 131 | 132 | StructType structType = new StructType(structFields); 133 | 134 | List rows = new ArrayList<>(); 135 | 136 | while (resultSet.next()) { 137 | Object[] objects = new Object[structFields.length]; 138 | for (int i = 0; i < structFields.length; i++) { 139 | Object obj = resultSet.getObject(i + 1); 140 | objects[i] = obj; 141 | } 142 | Row row = RowFactory.create(objects); 143 | rows.add(row); 144 | 145 | if (rows.size() == 1 || rows.size() % 5000 == 0) { 146 | logger.info(String.format("Read %s rows from sql result", rows.size())); 147 | } 148 | } 149 | 150 | logger.info(String.format("Got %s rows from sql result: %s", rows.size(), sql)); 151 | 152 | return new Tuple2<>(rows, structType); 153 | } catch (SQLException e) { 154 | throw new RuntimeException("Failed to execute jdbc sql: " + sql, e); 155 | } 156 | } 157 | 158 | public static org.apache.spark.sql.types.DataType convertSqlTypeToSparkSqlDataType(int sqlType) { 159 | if (sqlType == java.sql.Types.BOOLEAN) { 160 | return DataTypes.BooleanType; 161 | } else if (sqlType == Types.TINYINT) { 162 | return DataTypes.ByteType; 163 | } else if (sqlType == Types.SMALLINT) { 164 | return DataTypes.ShortType; 165 | } else if (sqlType == java.sql.Types.INTEGER) { 166 | return DataTypes.IntegerType; 167 | } else if (sqlType == java.sql.Types.BIGINT) { 168 | return DataTypes.LongType; 169 | } else if (sqlType == Types.DECIMAL) { 170 | return DataTypes.createDecimalType(); 171 | } else if (sqlType == java.sql.Types.FLOAT) { 172 | return DataTypes.FloatType; 173 | } else if (sqlType == java.sql.Types.DOUBLE) { 174 | return DataTypes.DoubleType; 175 | } else if (sqlType == Types.DATE) { 176 | return DataTypes.DateType; 177 | } else if (sqlType == Types.TIME) { 178 | return DataTypes.TimestampType; 179 | } else if (sqlType == Types.TIMESTAMP) { 180 | return DataTypes.TimestampType; 181 | } else if (sqlType == java.sql.Types.VARCHAR) { 182 | return DataTypes.StringType; 183 | } else { 184 | logger.warn(String.format("Using string for unsupported sql type %s", sqlType)); 185 | return DataTypes.StringType; 186 | } 187 | } 188 | 189 | public static DataSetResult getDataSetResult(Dataset df) { 190 | DataSetResult result = new DataSetResult(); 191 | 192 | String[] fieldNames = df.schema().fieldNames(); 193 | 194 | result.getColumnNames().addAll(Arrays.asList(fieldNames)); 195 | 196 | Row[] rows = (Row[]) df.collect(); 197 | for (Row row : rows) { 198 | List values = new ArrayList<>(); 199 | for (int i = 0; i < fieldNames.length; i++) { 200 | Object obj = row.get(i); 201 | values.add(obj); 202 | } 203 | result.getRows().add(values); 204 | } 205 | 206 | return result; 207 | } 208 | 209 | public static void writeJdbc(Dataset df, String jdbcUrl, String tableName, List primaryKeys, List indexColumns, List textColumns, SaveMode saveMode, String postWriteSql, double writesPerSecond) { 210 | logger.info(String.format("Saving to jdbc table %s, saveMode: %s", tableName, saveMode)); 211 | 212 | String jdbcDriver = SqlUtils.loadJdbcDriverClass(jdbcUrl); 213 | 214 | try { 215 | DriverManager.getConnection(jdbcUrl).close(); 216 | } catch (SQLException e) { 217 | throw new RuntimeException("Failed to open jdbc connection", e); 218 | } 219 | 220 | org.apache.spark.sql.execution.datasources.jdbc.DriverRegistry.register(com.mysql.jdbc.Driver.class.getName()); 221 | Properties properties = new Properties(); 222 | properties.put("driver", jdbcDriver); 223 | (new DataFrameJdbcWriter(df)).mode(saveMode).jdbc(jdbcUrl, tableName, primaryKeys, indexColumns, textColumns, postWriteSql, writesPerSecond, properties); 224 | logger.info(String.format("Saved to jdbc table %s, saveMode: %s", tableName, saveMode)); 225 | } 226 | 227 | public static DataSetResult getTableData(SparkSession sparkSession, String tableName) { 228 | String sql = String.format("select * from %s", tableName); 229 | logger.info(String.format("Running sql [%s] to get data", sql)); 230 | Dataset df = sparkSession.sql(sql); 231 | DataSetResult dataSetResult = SparkUtils.getDataSetResult(df); 232 | logger.info(String.format("Finished sql [%s] to get data", sql)); 233 | return dataSetResult; 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/SqlUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import com.uber.uberscriptquery.jdbc.DataSetResult; 20 | import org.apache.log4j.Logger; 21 | 22 | import java.sql.*; 23 | import java.util.ArrayList; 24 | import java.util.List; 25 | import java.util.regex.Matcher; 26 | import java.util.regex.Pattern; 27 | 28 | public class SqlUtils { 29 | private static final Logger logger = Logger.getLogger(SqlUtils.class); 30 | 31 | public static boolean isTableNotExistExceptionMessage(String msg, String table) { 32 | { 33 | // Table 'dataquerier.airport_eta_ata_summary' doesn't exist 34 | Pattern p = Pattern.compile(String.format("^[Tt]able .*%s.* doesn't exist$$", table)); 35 | Matcher m = p.matcher(msg); 36 | if (m.matches()) { 37 | return true; 38 | } 39 | } 40 | { 41 | // Table xxx not exist 42 | Pattern p = Pattern.compile(String.format("^[Tt]able .*%s.* not exist$$", table)); 43 | Matcher m = p.matcher(msg); 44 | if (m.matches()) { 45 | return true; 46 | } 47 | } 48 | { 49 | // Table xxx not found 50 | Pattern p = Pattern.compile(String.format("^Table .*%s.* not found.*", table), Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE | Pattern.DOTALL); 51 | Matcher m = p.matcher(msg); 52 | if (m.matches()) { 53 | return true; 54 | } 55 | } 56 | return false; 57 | } 58 | 59 | public static String loadJdbcDriverClass(String jdbcUrl) { 60 | final String jdbcDriver; 61 | if (jdbcUrl.toLowerCase().startsWith("jdbc:h2:")) { 62 | jdbcDriver = "org.h2.Driver"; 63 | } else if (jdbcUrl.toLowerCase().startsWith("jdbc:mysql:")) { 64 | jdbcDriver = com.mysql.jdbc.Driver.class.getName(); 65 | } else { 66 | throw new RuntimeException("Unsupported jdbc url: " + jdbcUrl); 67 | } 68 | 69 | try { 70 | Class.forName(jdbcDriver); 71 | } catch (ClassNotFoundException e) { 72 | throw new RuntimeException("Failed to load jdbc driver", e); 73 | } 74 | 75 | return jdbcDriver; 76 | } 77 | 78 | public static int executeJdbcUpdate(String jdbcUrl, String sql) { 79 | logger.info(String.format("Updating jdbc data by sql: %s", sql)); 80 | 81 | loadJdbcDriverClass(jdbcUrl); 82 | 83 | try (Connection con = DriverManager.getConnection(jdbcUrl)) { 84 | PreparedStatement stmt = con.prepareStatement(sql); 85 | int rowCount = stmt.executeUpdate(); 86 | logger.info(String.format("Finished db update sql: %s, row count: %s", sql, rowCount)); 87 | return rowCount; 88 | } catch (SQLException e) { 89 | throw new RuntimeException("Failed to execute jdbc sql: " + sql, e); 90 | } 91 | } 92 | 93 | public static DataSetResult executeJdbcQuery(String jdbcUrl, String sql) { 94 | logger.info(String.format("Querying jdbc data by sql: %s", sql)); 95 | 96 | loadJdbcDriverClass(jdbcUrl); 97 | 98 | DataSetResult dataSetResult = new DataSetResult(); 99 | List columnNames = new ArrayList<>(); 100 | List> rows = new ArrayList<>(); 101 | try (Connection con = DriverManager.getConnection(jdbcUrl)) { 102 | logger.info("Running sql: " + sql); 103 | try (Statement stmt = con.createStatement()) { 104 | try (ResultSet resultSet = stmt.executeQuery(sql)) { 105 | for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { 106 | columnNames.add(resultSet.getMetaData().getColumnName(i + 1)); 107 | } 108 | while (resultSet.next()) { 109 | List values = new ArrayList<>(); 110 | for (int i = 0; i < resultSet.getMetaData().getColumnCount(); i++) { 111 | Object value = resultSet.getObject(i + 1); 112 | values.add(value); 113 | } 114 | rows.add(values); 115 | } 116 | logger.info("Finished sql: " + sql); 117 | dataSetResult.setColumnNames(columnNames); 118 | dataSetResult.setRows(rows); 119 | return dataSetResult; 120 | } 121 | } 122 | } catch (SQLException e) { 123 | throw new RuntimeException(String.format("Failed to execute query sql: %s, %s", sql, e.getMessage()), e); 124 | } 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /src/main/java/com/uber/uberscriptquery/util/TemplateUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import freemarker.cache.StringTemplateLoader; 20 | import freemarker.template.Configuration; 21 | import freemarker.template.Template; 22 | import freemarker.template.TemplateException; 23 | import freemarker.template.TemplateExceptionHandler; 24 | import org.apache.commons.lang3.StringUtils; 25 | 26 | import java.io.BufferedWriter; 27 | import java.io.ByteArrayOutputStream; 28 | import java.io.IOException; 29 | import java.io.OutputStreamWriter; 30 | import java.nio.charset.StandardCharsets; 31 | import java.util.ArrayList; 32 | import java.util.List; 33 | import java.util.Map; 34 | 35 | public class TemplateUtils { 36 | public static String transform(String text, Map map) { 37 | List supportedVariables = new ArrayList<>(); 38 | for (String entry : map.keySet()) { 39 | supportedVariables.add(String.format("${%s}", entry)); 40 | } 41 | 42 | String supportedVariablesStr = StringUtils.join(supportedVariables, ", "); 43 | 44 | final String templateName = "query"; 45 | Configuration cfg = new Configuration(Configuration.VERSION_2_3_24); 46 | cfg.setDefaultEncoding("UTF-8"); 47 | cfg.setTemplateExceptionHandler(TemplateExceptionHandler.RETHROW_HANDLER); 48 | 49 | StringTemplateLoader ctl = new StringTemplateLoader(); 50 | ctl.putTemplate(templateName, text); 51 | cfg.setTemplateLoader(ctl); 52 | 53 | Template template; 54 | try { 55 | template = cfg.getTemplate(templateName); 56 | } catch (IOException e) { 57 | throw new RuntimeException("Failed to get template: " + templateName, e); 58 | } 59 | 60 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 61 | BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(byteArrayOutputStream)); 62 | 63 | try { 64 | template.process(map, writer); 65 | } catch (TemplateException | IOException e) { 66 | throw new RuntimeException(String.format("Failed to process text template: %s. Supported variables: %s", text, supportedVariablesStr), e); 67 | } 68 | 69 | try { 70 | writer.flush(); 71 | } catch (IOException e) { 72 | throw new RuntimeException(String.format("Failed to flush writer for text template: %s. Supported variables: %s", text, supportedVariablesStr), e); 73 | } 74 | 75 | byte[] bytes = byteArrayOutputStream.toByteArray(); 76 | String result = new String(bytes, StandardCharsets.UTF_8); 77 | return result; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/org/apache/spark/sql/DriverRegistryWrapper.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql; 18 | 19 | // See http://techblog.applift.com/upgrading-spark 20 | 21 | import org.apache.spark.sql.execution.datasources.jdbc.DriverRegistry; 22 | 23 | public class DriverRegistryWrapper { 24 | public static void register(String className) { 25 | DriverRegistry.register(className); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/main/resources/example.txt: -------------------------------------------------------------------------------- 1 | This is an example. -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/sql/DataFrameJdbcWriter.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | * Modified by Uber by copying from original file DataFrameWriter.scala. 18 | * Modifications copyright (C) 2017 Uber 19 | * 20 | */ 21 | 22 | package org.apache.spark.sql 23 | 24 | import java.util.Properties 25 | 26 | import org.apache.spark.sql.execution.datasources.jdbc.{JdbcUtils, JdbcWriterUtils} 27 | 28 | import scala.collection.JavaConverters._ 29 | 30 | /** 31 | * Class used to write a [[Dataset]] to JDBC. This file is to mimic Spark DataFrameWriter, and provide 32 | * enhanced capabilities, for example, supporting primary keys and doing upsert based on the primary keys 33 | * to make JDBC write idempotent. 34 | * 35 | */ 36 | final class DataFrameJdbcWriter[T](ds: Dataset[T]) { 37 | 38 | private val df = ds.toDF() 39 | private var mode: SaveMode = SaveMode.ErrorIfExists 40 | 41 | /** 42 | * Specifies the behavior when data or table already exists. Options include: 43 | * - `SaveMode.Overwrite`: overwrite the existing data. 44 | * - `SaveMode.Append`: append the data. 45 | * - `SaveMode.Ignore`: ignore the operation (i.e. no-op). 46 | * - `SaveMode.ErrorIfExists`: default option, throw an exception at runtime. 47 | */ 48 | def mode(saveMode: SaveMode): DataFrameJdbcWriter[T] = { 49 | this.mode = saveMode 50 | this 51 | } 52 | 53 | /** 54 | * Specifies the behavior when data or table already exists. Options include: 55 | * - `overwrite`: overwrite the existing data. 56 | * - `append`: append the data. 57 | * - `ignore`: ignore the operation (i.e. no-op). 58 | * - `error`: default option, throw an exception at runtime. 59 | */ 60 | def mode(saveMode: String): DataFrameJdbcWriter[T] = { 61 | this.mode = saveMode.toLowerCase match { 62 | case "overwrite" => SaveMode.Overwrite 63 | case "append" => SaveMode.Append 64 | case "ignore" => SaveMode.Ignore 65 | case "error" | "default" => SaveMode.ErrorIfExists 66 | case _ => throw new IllegalArgumentException(s"Unknown save mode: $saveMode. " + 67 | "Accepted save modes are 'overwrite', 'append', 'ignore', 'error'.") 68 | } 69 | this 70 | } 71 | 72 | def jdbc(url: String, table: String, primaryKeys: java.util.List[String], indexColumns: java.util.List[String], textColumns: java.util.List[String], postWriteSql: String, writesPerSecond: Double, connectionProperties: Properties): Unit = { 73 | jdbc(url, table, primaryKeys.asScala.toSeq, indexColumns.asScala.toSeq, textColumns.asScala.toSeq, postWriteSql, writesPerSecond, connectionProperties) 74 | } 75 | 76 | /////////////////////////////////////////////////////////////////////////////////////// 77 | // Builder pattern config options 78 | /////////////////////////////////////////////////////////////////////////////////////// 79 | 80 | def jdbc(url: String, table: String, primaryKeys: Seq[String], indexColumns: Seq[String], textColumns: Seq[String], postWriteSql: String, writesPerSecond: Double, connectionProperties: Properties): Unit = { 81 | val props = new Properties() 82 | 83 | // connectionProperties should override settings in extraOptions 84 | props.putAll(connectionProperties) 85 | val conn = JdbcUtils.createConnectionFactory(url, props)() 86 | 87 | try { 88 | var tableExists = JdbcUtils.tableExists(conn, url, table) 89 | 90 | if (mode == SaveMode.Ignore && tableExists) { 91 | return 92 | } 93 | 94 | if (mode == SaveMode.ErrorIfExists && tableExists) { 95 | sys.error(s"Table $table already exists.") 96 | } 97 | 98 | if (mode == SaveMode.Overwrite && tableExists) { 99 | JdbcUtils.dropTable(conn, table) 100 | tableExists = false 101 | } 102 | 103 | // Create the table if the table didn't exist. 104 | if (!tableExists) { 105 | //val schema = JdbcUtils.schemaString(df, url) 106 | //val sql = s"CREATE TABLE $table ($schema)" 107 | val columnNames = df.schema.fields.map(_.name).toList.asJava 108 | val columnTypes = df.schema.fields.map(_.dataType).toList.asJava 109 | val sql = com.uber.uberscriptquery.jdbc.JdbcUtils.getCreateTableSql( 110 | columnNames, columnTypes, table, primaryKeys.toList.asJava, indexColumns.toList.asJava, textColumns.toList.asJava) 111 | val statement = conn.createStatement 112 | try { 113 | System.out.println(String.format("Creating jdbc table: %s", sql)) 114 | statement.executeUpdate(sql) 115 | System.out.println(String.format("Created jdbc table: %s", sql)) 116 | } finally { 117 | statement.close() 118 | } 119 | } 120 | } finally { 121 | conn.close() 122 | } 123 | 124 | val retry = if (primaryKeys.isEmpty) { 125 | false 126 | } else { 127 | true 128 | } 129 | System.out.println(String.format("Save to table %s with retry %s", table, String.valueOf(retry))) 130 | JdbcWriterUtils.saveTable(df, url, table, textColumns, postWriteSql, retry, writesPerSecond, props) 131 | } 132 | 133 | } 134 | -------------------------------------------------------------------------------- /src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JdbcWriterUtils.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Licensed to the Apache Software Foundation (ASF) under one or more 3 | * contributor license agreements. See the NOTICE file distributed with 4 | * this work for additional information regarding copyright ownership. 5 | * The ASF licenses this file to You under the Apache License, Version 2.0 6 | * (the "License"); you may not use this file except in compliance with 7 | * the License. You may obtain a copy of the License at 8 | * 9 | * http://www.apache.org/licenses/LICENSE-2.0 10 | * 11 | * Unless required by applicable law or agreed to in writing, software 12 | * distributed under the License is distributed on an "AS IS" BASIS, 13 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | * See the License for the specific language governing permissions and 15 | * limitations under the License. 16 | * 17 | * Modified by Uber by copying from original file JdbcUtils.scala. 18 | * Modifications copyright (C) 2017 Uber 19 | * 20 | */ 21 | 22 | package org.apache.spark.sql.execution.datasources.jdbc 23 | 24 | import java.sql.{Connection, Driver, DriverManager} 25 | import java.util.Properties 26 | import java.util.concurrent.Callable 27 | import java.util.concurrent.atomic.AtomicLong 28 | 29 | import com.uber.uberscriptquery.jdbc.SingleTableJdbcWriter 30 | import com.uber.uberscriptquery.util.{ExponentialBackoffRetryPolicy, SqlUtils} 31 | import org.apache.spark.internal.Logging 32 | import org.apache.spark.sql.DataFrame 33 | import org.apache.spark.sql.jdbc.{JdbcDialect, JdbcDialects, JdbcType} 34 | import org.apache.spark.sql.types._ 35 | import org.spark_project.guava.util.concurrent.RateLimiter 36 | 37 | import scala.collection.JavaConverters._ 38 | 39 | object JdbcWriterUtils extends Logging { 40 | 41 | def saveTable( 42 | df: DataFrame, 43 | url: String, 44 | table: String, 45 | textColumns: Seq[String], 46 | postWriteSql: String, 47 | retry: Boolean, 48 | writesPerSecond: Double, 49 | properties: Properties) { 50 | val dialect = JdbcDialects.get(url) 51 | val nullTypes: Array[Int] = df.schema.fields.map { field => 52 | getJdbcType(field.dataType, dialect).jdbcNullType 53 | } 54 | 55 | val rddSchema = df.schema 56 | val columnNames = rddSchema.fields.map(_.name).toSeq.asJava 57 | 58 | val mapPartitionsResult = df.rdd.mapPartitions { iterator => 59 | logInfo(s"UberJdbcUtils.saveTable: $table") 60 | val rateLimiter = RateLimiter.create(writesPerSecond); 61 | try { 62 | val singleTableJdbcWriter = new SingleTableJdbcWriter(url, table, null, textColumns.asJava); 63 | 64 | val atomicLong = new AtomicLong() 65 | 66 | iterator.foreach(row => { 67 | atomicLong.incrementAndGet() 68 | val columnValues = row.toSeq.map(_.asInstanceOf[AnyRef]).asJava 69 | if (retry) { 70 | val retryPolicy: ExponentialBackoffRetryPolicy[String] = new ExponentialBackoffRetryPolicy[String](3, 100L) 71 | retryPolicy.attempt(new Callable[String] { 72 | override def call(): String = { 73 | rateLimiter.acquire() 74 | singleTableJdbcWriter.writeColumns(columnNames, columnValues) 75 | return null 76 | } 77 | }) 78 | } else { 79 | singleTableJdbcWriter.writeColumns(columnNames, columnValues) 80 | } 81 | }) 82 | 83 | Seq(atomicLong.longValue()).iterator 84 | } catch { 85 | case e: Throwable => { 86 | throw e 87 | } 88 | } 89 | } 90 | 91 | val rowCounts = mapPartitionsResult.collect() 92 | val totalSavedRows = rowCounts.sum 93 | logInfo(s"Saved $totalSavedRows rows to jdbc table $table") 94 | 95 | if (postWriteSql != null && !postWriteSql.isEmpty) { 96 | logInfo(s"Running post save sql: $postWriteSql") 97 | SqlUtils.executeJdbcUpdate(url, postWriteSql) 98 | } 99 | } 100 | 101 | private def getJdbcType(dt: DataType, dialect: JdbcDialect): JdbcType = { 102 | dialect.getJDBCType(dt).orElse(getCommonJDBCType(dt)).getOrElse( 103 | throw new IllegalArgumentException(s"Can't get JDBC type for ${dt.simpleString}")) 104 | } 105 | 106 | private def getCommonJDBCType(dt: DataType): Option[JdbcType] = { 107 | dt match { 108 | case IntegerType => Option(JdbcType("INTEGER", java.sql.Types.INTEGER)) 109 | case LongType => Option(JdbcType("BIGINT", java.sql.Types.BIGINT)) 110 | case DoubleType => Option(JdbcType("DOUBLE PRECISION", java.sql.Types.DOUBLE)) 111 | case FloatType => Option(JdbcType("REAL", java.sql.Types.FLOAT)) 112 | case ShortType => Option(JdbcType("INTEGER", java.sql.Types.SMALLINT)) 113 | case ByteType => Option(JdbcType("BYTE", java.sql.Types.TINYINT)) 114 | case BooleanType => Option(JdbcType("BIT(1)", java.sql.Types.BIT)) 115 | case StringType => Option(JdbcType("TEXT", java.sql.Types.CLOB)) 116 | case BinaryType => Option(JdbcType("BLOB", java.sql.Types.BLOB)) 117 | case TimestampType => Option(JdbcType("TIMESTAMP", java.sql.Types.TIMESTAMP)) 118 | case DateType => Option(JdbcType("DATE", java.sql.Types.DATE)) 119 | case t: DecimalType => Option( 120 | JdbcType(s"DECIMAL(${t.precision},${t.scale})", java.sql.Types.DECIMAL)) 121 | case _ => None 122 | } 123 | } 124 | 125 | private def createConnectionFactory(url: String, properties: Properties): () => Connection = { 126 | val userSpecifiedDriverClass = Option(properties.getProperty("driver")) 127 | userSpecifiedDriverClass.foreach(DriverRegistry.register) 128 | // Performing this part of the logic on the driver guards against the corner-case where the 129 | // driver returned for a URL is different on the driver and executors due to classpath 130 | // differences. 131 | val driverClass: String = userSpecifiedDriverClass.getOrElse { 132 | DriverManager.getDriver(url).getClass.getCanonicalName 133 | } 134 | () => { 135 | userSpecifiedDriverClass.foreach(DriverRegistry.register) 136 | val driver: Driver = DriverManager.getDrivers.asScala.collectFirst { 137 | case d: DriverWrapper if d.wrapped.getClass.getCanonicalName == driverClass => d 138 | case d if d.getClass.getCanonicalName == driverClass => d 139 | }.getOrElse { 140 | throw new IllegalStateException( 141 | s"Did not find registered driver with class $driverClass") 142 | } 143 | driver.connect(url, properties) 144 | } 145 | } 146 | 147 | } 148 | -------------------------------------------------------------------------------- /src/test/java/com/uber/uberscriptquery/antlr4/parsing/QueryFile001.txt: -------------------------------------------------------------------------------- 1 | var_current_date = 'current_date()'; 2 | 3 | t1 = select ${var_current_date} as date; 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /src/test/java/com/uber/uberscriptquery/antlr4/parsing/QueryTemplateParserTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.antlr4.parsing; 18 | 19 | import junit.framework.Assert; 20 | import org.junit.Test; 21 | 22 | public class QueryTemplateParserTest { 23 | 24 | @Test 25 | public void test_parse() { 26 | { 27 | String query = "v1 = 'abc'; v2 = \"def\";"; 28 | 29 | QueryTemplateParser parser = new QueryTemplateParser(); 30 | QueryTemplateParseResult root = parser.parse(query); 31 | Assert.assertEquals(2, root.getVariableMap().size()); 32 | Assert.assertEquals("abc", root.getVariableMap().get("v1")); 33 | Assert.assertEquals("def", root.getVariableMap().get("v2")); 34 | Assert.assertEquals("", root.getTemplateBody()); 35 | } 36 | { 37 | String query = "v1 = 'abc'; v2 = \"def\"; body"; 38 | 39 | QueryTemplateParser parser = new QueryTemplateParser(); 40 | QueryTemplateParseResult root = parser.parse(query); 41 | Assert.assertEquals(2, root.getVariableMap().size()); 42 | Assert.assertEquals("abc", root.getVariableMap().get("v1")); 43 | Assert.assertEquals("def", root.getVariableMap().get("v2")); 44 | Assert.assertEquals(" body", root.getTemplateBody()); 45 | } 46 | { 47 | String query = "v1 = 'abc'; v2 = \"def\"; ~!@# $%^&*()_+ '\" = == abc='def' v=\"ttt\""; 48 | 49 | QueryTemplateParser parser = new QueryTemplateParser(); 50 | QueryTemplateParseResult root = parser.parse(query); 51 | Assert.assertEquals(2, root.getVariableMap().size()); 52 | Assert.assertEquals("abc", root.getVariableMap().get("v1")); 53 | Assert.assertEquals("def", root.getVariableMap().get("v2")); 54 | Assert.assertEquals(" ~!@# $%^&*()_+ '\" = == abc='def' v=\"ttt\"", root.getTemplateBody()); 55 | } 56 | 57 | { 58 | String query = "v1 = abc; v2 = \"def\"; ~!@# $%^&*()_+ '\" = == abc='def' v=\"ttt\""; 59 | 60 | QueryTemplateParser parser = new QueryTemplateParser(); 61 | QueryTemplateParseResult root = parser.parse(query); 62 | Assert.assertEquals(0, root.getVariableMap().size()); 63 | Assert.assertEquals("v1 = abc; v2 = \"def\"; ~!@# $%^&*()_+ '\" = == abc='def' v=\"ttt\"", root.getTemplateBody()); 64 | } 65 | } 66 | 67 | @Test 68 | public void test_parse_table_assignement() { 69 | { 70 | String query = "v1 = select a,b, c from table1; v2 = select c, d,e from table2;"; 71 | 72 | QueryTemplateParser parser = new QueryTemplateParser(); 73 | QueryTemplateParseResult root = parser.parse(query); 74 | Assert.assertEquals(0, root.getVariableMap().size()); 75 | Assert.assertEquals("v1 = select a,b, c from table1; v2 = select c, d,e from table2;", root.getTemplateBody()); 76 | } 77 | { 78 | String query = "v1 = select a,b, c from table1; v2 = select c, d,e from table2; select a,b,d,e from v1 v1 join (select * from v2 v2) on v1.c = v2.c"; 79 | 80 | QueryTemplateParser parser = new QueryTemplateParser(); 81 | QueryTemplateParseResult root = parser.parse(query); 82 | Assert.assertEquals(0, root.getVariableMap().size()); 83 | Assert.assertEquals("v1 = select a,b, c from table1; v2 = select c, d,e from table2; select a,b,d,e from v1 v1 join (select * from v2 v2) on v1.c = v2.c", root.getTemplateBody()); 84 | } 85 | } 86 | 87 | @Test 88 | public void test_parse_reference_variable_inside_variable() { 89 | { 90 | String query = "v1 = 'abc'; v2 = \"d${v1}f\"; body"; 91 | 92 | QueryTemplateParser parser = new QueryTemplateParser(); 93 | QueryTemplateParseResult root = parser.parse(query); 94 | Assert.assertEquals(2, root.getVariableMap().size()); 95 | Assert.assertEquals("abc", root.getVariableMap().get("v1")); 96 | Assert.assertEquals("dabcf", root.getVariableMap().get("v2")); 97 | Assert.assertEquals(" body", root.getTemplateBody()); 98 | } 99 | { 100 | String query = "v1 = 'abc'; v2 = \"d${v1}f\"; v3 = '1${v2}2'; body -- ${v1}, ${v2}, ${v3} --"; 101 | 102 | QueryTemplateParser parser = new QueryTemplateParser(); 103 | QueryTemplateParseResult root = parser.parse(query); 104 | Assert.assertEquals(3, root.getVariableMap().size()); 105 | Assert.assertEquals("abc", root.getVariableMap().get("v1")); 106 | Assert.assertEquals("dabcf", root.getVariableMap().get("v2")); 107 | Assert.assertEquals("1dabcf2", root.getVariableMap().get("v3")); 108 | Assert.assertEquals(" body -- abc, dabcf, 1dabcf2 --", root.getTemplateBody()); 109 | } 110 | } 111 | 112 | @Test 113 | public void test_parse_special_characters() { 114 | { 115 | String query = "v1 = '\"abc\"';body"; 116 | 117 | QueryTemplateParser parser = new QueryTemplateParser(); 118 | QueryTemplateParseResult root = parser.parse(query); 119 | Assert.assertEquals(1, root.getVariableMap().size()); 120 | Assert.assertEquals("\"abc\"", root.getVariableMap().get("v1")); 121 | Assert.assertEquals("body", root.getTemplateBody()); 122 | } 123 | { 124 | String query = "v1 = \"'abc'\";body"; 125 | 126 | QueryTemplateParser parser = new QueryTemplateParser(); 127 | QueryTemplateParseResult root = parser.parse(query); 128 | Assert.assertEquals(1, root.getVariableMap().size()); 129 | Assert.assertEquals("'abc'", root.getVariableMap().get("v1")); 130 | Assert.assertEquals("body", root.getTemplateBody()); 131 | } 132 | { 133 | String query = "v1 = '\\'abc\\'';body"; 134 | 135 | QueryTemplateParser parser = new QueryTemplateParser(); 136 | QueryTemplateParseResult root = parser.parse(query); 137 | Assert.assertEquals(1, root.getVariableMap().size()); 138 | Assert.assertEquals("'abc'", root.getVariableMap().get("v1")); 139 | Assert.assertEquals("body", root.getTemplateBody()); 140 | } 141 | { 142 | String query = "v1 = \"\\\"abc\\\"\";body"; 143 | 144 | QueryTemplateParser parser = new QueryTemplateParser(); 145 | QueryTemplateParseResult root = parser.parse(query); 146 | Assert.assertEquals(1, root.getVariableMap().size()); 147 | Assert.assertEquals("\"abc\"", root.getVariableMap().get("v1")); 148 | Assert.assertEquals("body", root.getTemplateBody()); 149 | } 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /src/test/java/com/uber/uberscriptquery/execution/QueryEngineTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.execution; 18 | 19 | import com.uber.uberscriptquery.util.JsonFileCredentialProvider; 20 | import com.uber.uberscriptquery.util.SqlUtils; 21 | import junit.framework.Assert; 22 | import org.apache.commons.io.FileUtils; 23 | import org.apache.spark.SparkConf; 24 | import org.apache.spark.sql.Dataset; 25 | import org.apache.spark.sql.Row; 26 | import org.apache.spark.sql.SparkSession; 27 | import org.junit.After; 28 | import org.junit.Before; 29 | import org.junit.Test; 30 | 31 | import java.io.File; 32 | import java.io.IOException; 33 | 34 | public class QueryEngineTest { 35 | private String master = "local[1]"; 36 | private String appName = "spark_unit_test"; 37 | 38 | private SparkConf sparkConf; 39 | private SparkSession sparkSession; 40 | 41 | @Before 42 | public void before() { 43 | sparkConf = new SparkConf() 44 | .setMaster(master) 45 | .setAppName(appName) 46 | .set("spark.driver.allowMultipleContexts", "true"); 47 | 48 | sparkSession = SparkSession 49 | .builder() 50 | .config(sparkConf).getOrCreate(); 51 | } 52 | 53 | @After 54 | public void after() { 55 | sparkSession.stop(); 56 | } 57 | 58 | @Test 59 | public void test_executeScript_read_jdbc() throws IOException { 60 | File file = File.createTempFile("h2dbfile", ".db"); 61 | file.deleteOnExit(); 62 | 63 | String connectionString = String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath()); 64 | 65 | SqlUtils.executeJdbcUpdate(connectionString, "create table table1(intColumn bigint, strColumn varchar(15))"); 66 | SqlUtils.executeJdbcUpdate(connectionString, "insert into table1(intColumn, strColumn) values (11, 'str1')"); 67 | 68 | File passwordFile = File.createTempFile("password", ".json"); 69 | passwordFile.deleteOnExit(); 70 | 71 | FileUtils.write(passwordFile, "{'jdbc': {'pwd': 'password1'}}"); 72 | 73 | QueryEngine engine = new QueryEngine(); 74 | engine.setCredentialProvider(new JsonFileCredentialProvider()); 75 | String query = String.format( 76 | "source1 = SQL jdbc set connectionString='%s'; set passwordFile='%s'; set passwordEntry='$.jdbc.pwd'; select intColumn, strColumn from table1;", 77 | connectionString, 78 | passwordFile.getAbsolutePath()); 79 | engine.executeScript(query, sparkSession); 80 | Dataset df = sparkSession.sql("select * from source1"); 81 | Row[] rows = (Row[]) df.collect(); 82 | Assert.assertEquals(1, rows.length); 83 | Assert.assertEquals(2, rows[0].size()); 84 | Assert.assertEquals(new Long(11), rows[0].get(0)); 85 | Assert.assertEquals("str1", rows[0].get(1)); 86 | } 87 | 88 | @Test 89 | public void test_executeScript_read_jdbc_empty_passwordFile() throws IOException { 90 | File file = File.createTempFile("h2dbfile", ".db"); 91 | file.deleteOnExit(); 92 | 93 | String connectionString = String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath()); 94 | 95 | SqlUtils.executeJdbcUpdate(connectionString, "create table table1(intColumn bigint, strColumn varchar(15))"); 96 | SqlUtils.executeJdbcUpdate(connectionString, "insert into table1(intColumn, strColumn) values (11, 'str1')"); 97 | 98 | File passwordFile = File.createTempFile("password", ".json"); 99 | passwordFile.deleteOnExit(); 100 | 101 | FileUtils.write(passwordFile, "{'jdbc': {'pwd': 'password1'}}"); 102 | 103 | QueryEngine engine = new QueryEngine(); 104 | String query = String.format( 105 | "source1 = SQL jdbc set connectionString='%s'; set passwordFile=''; set passwordEntry=''; select intColumn, strColumn from table1;", 106 | connectionString); 107 | engine.executeScript(query, sparkSession); 108 | Dataset df = sparkSession.sql("select * from source1"); 109 | Row[] rows = (Row[]) df.collect(); 110 | Assert.assertEquals(1, rows.length); 111 | Assert.assertEquals(2, rows[0].size()); 112 | Assert.assertEquals(new Long(11), rows[0].get(0)); 113 | Assert.assertEquals("str1", rows[0].get(1)); 114 | } 115 | 116 | @Test 117 | public void test_executeBatchQuery_week_timepoints_by_10_minutes() { 118 | QueryEngine engine = new QueryEngine(); 119 | String query = "source1 = datagen week_timepoints_by_10_minutes select timepoint from week_timepoints_by_10_minutes;result=select * from source1;"; 120 | engine.executeScript(query, sparkSession); 121 | Dataset df = sparkSession.sql("select * from result"); 122 | Row[] rows = (Row[]) df.collect(); 123 | Assert.assertEquals(1008, rows.length); 124 | Assert.assertEquals(new Integer(0), rows[0].get(0)); 125 | Assert.assertEquals(new Integer(1), rows[1].get(0)); 126 | Assert.assertEquals(new Integer(1007), rows[1007].get(0)); 127 | } 128 | 129 | @Test 130 | public void test_executeBatchQuery_numbers_1k() { 131 | QueryEngine engine = new QueryEngine(); 132 | String query = "source1 = datagen numbers_1k select number from numbers_1k;result = select * from source1;"; 133 | engine.executeScript(query, sparkSession); 134 | Dataset df = sparkSession.sql("select * from result"); 135 | Row[] rows = (Row[]) df.collect(); 136 | Assert.assertEquals(1001, rows.length); 137 | Assert.assertEquals(new Integer(0), rows[0].get(0)); 138 | Assert.assertEquals(new Integer(1), rows[1].get(0)); 139 | Assert.assertEquals(new Integer(1000), rows[1000].get(0)); 140 | } 141 | 142 | 143 | @Test 144 | public void test_unix_timestamp() { 145 | QueryEngine engine = new QueryEngine(); 146 | String query = "result=select unix_timestamp(split('2016-12-03T10:38:11.760000+00:00', '\\\\.')[0], \"yyyy-MM-dd'T'HH:mm:ss\") as timestamp;"; 147 | engine.executeScript(query, sparkSession); 148 | Dataset df = sparkSession.sql("select * from result"); 149 | Row[] rows = (Row[]) df.collect(); 150 | Assert.assertEquals(1, rows.length); 151 | System.out.println("RESULT: " + rows[0].get(0)); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/test/java/com/uber/uberscriptquery/providers/JsonFileCredentialProviderTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.providers; 18 | 19 | import com.uber.uberscriptquery.util.JsonFileCredentialProvider; 20 | import junit.framework.Assert; 21 | import org.apache.commons.io.FileUtils; 22 | import org.junit.Test; 23 | 24 | import java.io.File; 25 | import java.io.IOException; 26 | 27 | public class JsonFileCredentialProviderTest { 28 | @Test 29 | public void test_addPassword() throws IOException { 30 | File file = File.createTempFile("test_password_file", ".json"); 31 | file.deleteOnExit(); 32 | 33 | FileUtils.write(file, "{'aa':{'bb':'cc'}}"); 34 | 35 | JsonFileCredentialProvider credentialManager = new JsonFileCredentialProvider(); 36 | Assert.assertEquals("cc", credentialManager.getPassword(file.getAbsolutePath(), "$.aa.bb")); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/com/uber/uberscriptquery/util/SparkUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import com.uber.uberscriptquery.jdbc.DataSetResult; 20 | import junit.framework.Assert; 21 | import org.apache.spark.SparkConf; 22 | import org.apache.spark.sql.Dataset; 23 | import org.apache.spark.sql.Row; 24 | import org.apache.spark.sql.RowFactory; 25 | import org.apache.spark.sql.SparkSession; 26 | import org.apache.spark.sql.types.DataTypes; 27 | import org.apache.spark.sql.types.Metadata; 28 | import org.apache.spark.sql.types.StructField; 29 | import org.apache.spark.sql.types.StructType; 30 | import org.junit.After; 31 | import org.junit.Before; 32 | import org.junit.Test; 33 | 34 | import java.util.ArrayList; 35 | import java.util.List; 36 | 37 | public class SparkUtilsTest { 38 | private String master = "local[1]"; 39 | private String appName = "spark_unit_test"; 40 | 41 | private SparkConf sparkConf; 42 | private SparkSession sparkSession; 43 | 44 | @Before 45 | public void before() { 46 | sparkConf = new SparkConf() 47 | .setMaster(master) 48 | .setAppName(appName) 49 | .set("spark.driver.allowMultipleContexts", "true"); 50 | 51 | sparkSession = SparkSession 52 | .builder() 53 | .config(sparkConf).getOrCreate(); 54 | } 55 | 56 | @After 57 | public void after() { 58 | sparkSession.stop(); 59 | } 60 | 61 | @Test 62 | public void test_getDataSetResult() { 63 | 64 | StructField[] structFields = new StructField[]{ 65 | new StructField("intColumn", DataTypes.IntegerType, true, Metadata.empty()), 66 | new StructField("stringColumn", DataTypes.StringType, true, Metadata.empty()) 67 | }; 68 | 69 | StructType structType = new StructType(structFields); 70 | 71 | List rows = new ArrayList<>(); 72 | rows.add(RowFactory.create(1, "v1")); 73 | rows.add(RowFactory.create(2, "v2")); 74 | 75 | Dataset df = sparkSession.createDataFrame(rows, structType); 76 | 77 | DataSetResult dataSetResult = SparkUtils.getDataSetResult(df); 78 | Assert.assertEquals(2, dataSetResult.getColumnNames().size()); 79 | Assert.assertEquals(2, dataSetResult.getRows().size()); 80 | Assert.assertEquals(new Integer(1), dataSetResult.getRows().get(0).get(0)); 81 | Assert.assertEquals("v1", dataSetResult.getRows().get(0).get(1)); 82 | Assert.assertEquals(new Integer(2), dataSetResult.getRows().get(1).get(0)); 83 | Assert.assertEquals("v2", dataSetResult.getRows().get(1).get(1)); 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/com/uber/uberscriptquery/util/SqlUtilsTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.uber.uberscriptquery.util; 18 | 19 | import junit.framework.Assert; 20 | import org.junit.Test; 21 | 22 | public class SqlUtilsTest { 23 | @Test 24 | public void test_isTableNotExistExceptionMessage() { 25 | Assert.assertEquals(true, SqlUtils.isTableNotExistExceptionMessage("Table 'dataquerier.airport_eta_ata_summary' doesn't exist", "airport_eta_ata_summary")); 26 | Assert.assertEquals(true, SqlUtils.isTableNotExistExceptionMessage("Table airport_eta_ata_summary doesn't exist", "airport_eta_ata_summary")); 27 | 28 | Assert.assertEquals(true, SqlUtils.isTableNotExistExceptionMessage("Table 'dataquerier.airport_eta_ata_summary' not exist", "airport_eta_ata_summary")); 29 | Assert.assertEquals(true, SqlUtils.isTableNotExistExceptionMessage("Table airport_eta_ata_summary not exist", "airport_eta_ata_summary")); 30 | 31 | Assert.assertEquals(false, SqlUtils.isTableNotExistExceptionMessage("Table 'dataquerier.airport_eta_ata_summary' exist", "airport_eta_ata_summary")); 32 | Assert.assertEquals(false, SqlUtils.isTableNotExistExceptionMessage("Table 'dataquerier.abc' doesn't exist", "airport_eta_ata_summary")); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/test/scala/org/apache/spark/sql/DataFrameJdbcWriterTestSpec.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright (c) 2017 Uber Technologies, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package org.apache.spark.sql 18 | 19 | import java.io.File 20 | import java.util.Properties 21 | 22 | import org.apache.spark.SparkConf 23 | import org.apache.spark.sql.types.{DataTypes, Metadata, StructField, StructType} 24 | import org.scalatest.{FunSpec, Matchers} 25 | 26 | class DataFrameJdbcWriterTestSpec extends FunSpec with Matchers { 27 | 28 | describe("jdbc - with primary key") { 29 | it("should create jdb table and write data") { 30 | val master: String = "local[1]" 31 | val appName: String = "spark_unit_test" 32 | 33 | val sparkConf = new SparkConf().setMaster(master).setAppName(appName) 34 | .set("spark.driver.allowMultipleContexts", "true") 35 | 36 | val sparkSession = SparkSession.builder.config(sparkConf).getOrCreate 37 | 38 | val structFields: Array[StructField] = Array[StructField]( 39 | new StructField("intColumn", DataTypes.IntegerType, true, Metadata.empty), 40 | new StructField("stringColumn", DataTypes.StringType, true, Metadata.empty), 41 | new StructField("textColumn", DataTypes.StringType, true, Metadata.empty)) 42 | 43 | val structType: StructType = new StructType(structFields) 44 | 45 | val rows: java.util.List[Row] = new java.util.ArrayList[Row] 46 | rows.add(RowFactory.create(new Integer(1), "v1", "text1")) 47 | rows.add(RowFactory.create(new Integer(2), "v2", "text2")) 48 | 49 | val df: Dataset[Row] = sparkSession.createDataFrame(rows, structType) 50 | 51 | val dataFrameWriter = new DataFrameJdbcWriter[Row](df).mode(SaveMode.Append) 52 | 53 | 54 | val file: File = File.createTempFile("h2dbfile", ".db") 55 | file.deleteOnExit 56 | 57 | val connectionString: String = String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath) 58 | 59 | val properties: Properties = new Properties 60 | properties.put("driver", classOf[org.h2.Driver].getName) 61 | 62 | dataFrameWriter.jdbc(connectionString, "table1", Seq("intColumn"), Seq("stringColumn"), Seq("textColumn"), null, 100000, properties) 63 | 64 | var queryResult = com.uber.uberscriptquery.jdbc.JdbcUtils.executeQuery(connectionString, "select * from table1") 65 | assert(queryResult.size() === 2) 66 | assert(queryResult.get(0).size() === 3) 67 | 68 | dataFrameWriter.jdbc(connectionString, "table1", Seq("intColumn"), Seq("stringColumn"), Seq("textColumn"), null, 100000, properties) 69 | 70 | queryResult = com.uber.uberscriptquery.jdbc.JdbcUtils.executeQuery(connectionString, "select * from table1") 71 | assert(queryResult.size() === 2) 72 | assert(queryResult.get(0).size() === 3) 73 | 74 | dataFrameWriter.jdbc(connectionString, "table1", Seq("intColumn"), Seq("stringColumn"), Seq("textColumn"), "delete from table1", 100000, properties) 75 | 76 | queryResult = com.uber.uberscriptquery.jdbc.JdbcUtils.executeQuery(connectionString, "select * from table1") 77 | assert(queryResult.size() === 0) 78 | 79 | sparkSession.stop() 80 | } 81 | } 82 | 83 | describe("jdbc - with primary key and null values for non primary keys") { 84 | it("should create jdb table and write data") { 85 | val master: String = "local[1]" 86 | val appName: String = "spark_unit_test" 87 | 88 | val sparkConf = new SparkConf().setMaster(master).setAppName(appName) 89 | .set("spark.driver.allowMultipleContexts", "true") 90 | 91 | val sparkSession = SparkSession.builder.config(sparkConf).getOrCreate 92 | 93 | val structFields: Array[StructField] = Array[StructField]( 94 | new StructField("intColumn", DataTypes.IntegerType, true, Metadata.empty), 95 | new StructField("stringColumn", DataTypes.StringType, true, Metadata.empty), 96 | new StructField("textColumn", DataTypes.StringType, true, Metadata.empty)) 97 | 98 | val structType: StructType = new StructType(structFields) 99 | 100 | val rows: java.util.List[Row] = new java.util.ArrayList[Row] 101 | rows.add(RowFactory.create(new Integer(1), "v1", "text1")) 102 | rows.add(RowFactory.create(new Integer(2), "v2", null)) 103 | rows.add(RowFactory.create(new Integer(3), "v3", "text3")) 104 | 105 | val df: Dataset[Row] = sparkSession.createDataFrame(rows, structType) 106 | 107 | val dataFrameWriter = new DataFrameJdbcWriter[Row](df) 108 | 109 | 110 | val file: File = File.createTempFile("h2dbfile", ".db") 111 | file.deleteOnExit 112 | 113 | val connectionString: String = String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath) 114 | 115 | val properties: Properties = new Properties 116 | properties.put("driver", classOf[org.h2.Driver].getName) 117 | 118 | dataFrameWriter.jdbc(connectionString, "table1", Seq("intColumn", "stringColumn"), Seq("intColumn", "stringColumn"), Seq(), null, 100000, properties) 119 | 120 | val queryResult = com.uber.uberscriptquery.jdbc.JdbcUtils.executeQuery(connectionString, "select * from table1") 121 | assert(queryResult.size() === 3) 122 | assert(queryResult.get(0).size() === 3) 123 | 124 | sparkSession.stop() 125 | } 126 | } 127 | 128 | describe("jdbc - without primary key") { 129 | it("should create jdb table and write data") { 130 | val master: String = "local[1]" 131 | val appName: String = "spark_unit_test" 132 | 133 | val sparkConf = new SparkConf().setMaster(master).setAppName(appName) 134 | .set("spark.driver.allowMultipleContexts", "true") 135 | 136 | val sparkSession = SparkSession.builder.config(sparkConf).getOrCreate 137 | 138 | val structFields: Array[StructField] = Array[StructField]( 139 | new StructField("intColumn", DataTypes.IntegerType, true, Metadata.empty), 140 | new StructField("stringColumn", DataTypes.StringType, true, Metadata.empty), 141 | new StructField("textColumn", DataTypes.StringType, true, Metadata.empty)) 142 | 143 | val structType: StructType = new StructType(structFields) 144 | 145 | val rows: java.util.List[Row] = new java.util.ArrayList[Row] 146 | rows.add(RowFactory.create(new Integer(1), "v1", "text1")) 147 | rows.add(RowFactory.create(new Integer(2), "v2", "text2")) 148 | 149 | val df: Dataset[Row] = sparkSession.createDataFrame(rows, structType) 150 | 151 | val dataFrameWriter = new DataFrameJdbcWriter[Row](df) 152 | 153 | 154 | val file: File = File.createTempFile("h2dbfile", ".db") 155 | file.deleteOnExit 156 | 157 | val connectionString: String = String.format("jdbc:h2:%s;DB_CLOSE_DELAY=-1;MODE=MySQL", file.getAbsolutePath) 158 | 159 | val properties: Properties = new Properties 160 | properties.put("driver", classOf[org.h2.Driver].getName) 161 | 162 | dataFrameWriter.jdbc(connectionString, "table1", Seq[String](), Seq[String](), Seq("textColumn"), null, 100000, properties) 163 | 164 | val queryResult = com.uber.uberscriptquery.jdbc.JdbcUtils.executeQuery(connectionString, "select * from table1") 165 | assert(queryResult.size() === 2) 166 | assert(queryResult.get(0).size() === 3) 167 | 168 | sparkSession.stop() 169 | } 170 | } 171 | } 172 | --------------------------------------------------------------------------------