├── .gitignore
├── data
    └── records.json
├── src
    ├── main
    │   └── scala
    │   │   └── MainApp.scala
    └── test
    │   └── java
    │       └── TestThriftClient.java
├── README.md
├── dependency-reduced-pom.xml
└── pom.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | .idea
2 | target
3 | *.iml
4 | *.log
5 | metastore_db
6 | 


--------------------------------------------------------------------------------
/data/records.json:
--------------------------------------------------------------------------------
1 | {"id":"305901156","language":"English","content":"Demo SparkSql as JDBC source via Spark ThriftServer"}


--------------------------------------------------------------------------------
/src/main/scala/MainApp.scala:
--------------------------------------------------------------------------------
 1 | import org.apache.spark.sql.SparkSession
 2 | 
 3 | /**
 4 |   * Created by surthi on 28/06/17.
 5 |   */
 6 | object MainApp {
 7 | 
 8 |   def main(args: Array[String]): Unit = {
 9 | 
10 |     println("Hi")
11 | 
12 |     // change master from local[2] accordingly (change it to "yarn" if we are running it in yarn cluster..)
13 |     val spark = SparkSession.builder()
14 |       .master("local[2]")
15 |       .appName("SparkSqlExample")
16 |       .config("spark.sql.warehouse.dir", "/beeline/spark-sql-warehouse")
17 |       .config("hive.server2.thrift.port", "10000")
18 |       .config("spark.sql.hive.thriftServer.singleSession", true)
19 |       .enableHiveSupport()
20 |       .getOrCreate()
21 | 
22 |     import spark.implicits._
23 | 
24 |     // For simplicity, Iam loading data from one file for now..
25 |     // But, in reality, this records dataset can be some real-time continuous streaming data
26 |     val records = spark.read.format("json").load("data/records.json")
27 |     records.show()
28 |     records.write.saveAsTable("records")
29 | 
30 |     // This loop will keep the session alive
31 |     while (true) {
32 |       Thread.`yield`()
33 |     }
34 |   }
35 | }
36 | 
37 | // Cloud-based SQL engine using SPARK made available for access as a JDBC/ODBC data source via the Spark thrift server.


--------------------------------------------------------------------------------
/src/test/java/TestThriftClient.java:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Created by surthi on 30/06/17.
 3 |  */
 4 | import java.sql.SQLException;
 5 | import java.sql.Connection;
 6 | import java.sql.ResultSet;
 7 | import java.sql.Statement;
 8 | import java.sql.DriverManager;
 9 | 
10 | /**
11 |  * This is a test case that connects to sprak's thrift server and asserts that the data is present.
12 |  * In this example, 3 asserts are done: "show tables", "describe table <table-name>" and "select * from <table-name>".
13 |  */
14 | public class TestThriftClient {
15 |     private static String driverName = "org.apache.hive.jdbc.HiveDriver";
16 | 
17 |     public static void main(String[] args) throws SQLException {
18 | 
19 |         // Assert that the driver is present in classpath
20 |         try {
21 |             Class.forName(driverName);
22 |         } catch (ClassNotFoundException e) {
23 |             e.printStackTrace();
24 |             System.exit(1);
25 |         }
26 | 
27 |         // Change the connection-url, username and password accordingly
28 |         String connUrl = "jdbc:hive2://localhost:10000/default";
29 |         String username = "";
30 |         String password = "";
31 | 
32 |         Connection con = DriverManager.getConnection(connUrl, username, password);
33 |         Statement stmt = con.createStatement();
34 | 
35 |         // show tables
36 |         String sql = "show tables";
37 |         System.out.println("Running: " + sql);
38 |         ResultSet res = stmt.executeQuery(sql);
39 |         if (res.next()) {
40 |             System.out.println(res.getString(1));
41 |         }
42 | 
43 |         // describe table
44 |         String tableName = "records";
45 |         sql = "describe " + tableName;
46 |         System.out.println("Running: " + sql);
47 |         res = stmt.executeQuery(sql);
48 |         while (res.next()) {
49 |             System.out.println(res.getString(1) + "\t" + res.getString(2));
50 |         }
51 | 
52 |         // select * query
53 |         sql = "select * from " + tableName;
54 |         System.out.println("Running: " + sql);
55 |         res = stmt.executeQuery(sql);
56 |         while (res.next()) {
57 |             System.out.println(String.valueOf(res.getString(1)) + "\t" + res.getString(2));
58 |         }
59 |     }
60 | }


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Spark As CLoudBased SQL Engine
 2 | This project shows how to use SPARK as Cloud-based SQL Engine and expose your big-data as a JDBC/ODBC data source via the Spark thrift server. 
 3 | 
 4 | ## 1. Central Idea
 5 | Traditional relational Database engines like SQL had scalability problems and so evolved couple of SQL-on-Hadoop frameworks like Hive, Cloudier Impala, Presto etc. These frameworks are essentially cloud-based solutions and they all come with their own advantages and limitations. This project will demo how SparkSQL comes across as one more SQL-on-Hadoop framework.
 6 | 
 7 | ## 2. Architecture
 8 | Following picture illustrates how ApacheSpark can be used as SQL-on-Hadoop framework to serve your big-data as a JDBC/ODBC data source via the Spark thrift server.:
 9 | <img src="https://user-images.githubusercontent.com/22542670/27733176-54b684c2-5db2-11e7-946b-5b5ef5595e43.png" width="600" />
10 | 
11 | - Data from multiple sources can be pushed into Spark and then exposed as SQLtable
12 | - These tables are then made accessible as a JDBC/ODBC data source via the Spark thrift server.
13 | - Multiple clients like ```Beeline CLI```, ```JDBC```, ```ODBC``` or ```BI tools like Tableau``` connect to Spark thrift server.
14 | - Once the connection is established, ThriftServer will contact ```SparkSQL engine to access Hive or Spark temp tables and run the sql queries on ApacheSpark framework```.
15 | - Spark Thrift basically works similar to HiveServer2 thrift where HiveServer2 submits the sql queries as Hive MapReduce job vs Spark thrift server will use Spark SQL engine which underline uses full spark capabilities.
16 | 
17 | #### To know more about this topic, please refer to my blog [here](https://spoddutur.github.io/spark-notes/spark-as-cloud-based-sql-engine-via-thrift-server) where I briefed the concept in detail.
18 | 
19 | ## 3. Structure of the project:
20 | - **data:** Contains input json used in MainApp to register sample data with SparkSql.
21 | - **src/main/java/MainApp.scala:** Spark 2.1 implementation where it starts SparkSession and registers data from input.json with SparkSQL. (To keep the spark-session alive, there's a continuous while-loop in there).
22 | - **src/test/java/TestThriftClient.java:** Java class to demo how to connect to thrift server as JDBC source and query the registered data
23 | 
24 | ## 4. How to run this project?
25 | This project does demo 2 things:
26 | - 4.1. How to register data with SparkSql
27 | - 4.2. How to query registered data via Spark ThriftServer - using **Beeline** and **JDBC**
28 | 
29 | ### 4.1 How to register data with SparkSql
30 | - Download this project.
31 | - Build it: `mvn clean install` and
32 | - Run MainApp: `spark-submit --class MainApp cloud-based-sql-engine-using-spark.jar`. Tht's it! 
33 | - It'll register some sample data in `records` table with SparkSQL.
34 | 
35 | ### 4.2 How to query registered data via Spark Thrift Server using Beeline and JDBC?
36 | For this, first connect to Spark ThriftServer. Once the connection is established, just like HiveServer2, access Hive or Spark temp tables to run the sql queries on ApacheSpark framework. I'll show 2 ways to do this:
37 | 
38 | 1. **Beeline:** Perhaps, the simplest is to use beeline command-line tool provided in Spark's bin folder. 
39 | ```markdown
40 | `$> beeline`
41 | Beeline version 2.1.1-amzn-0 by Apache Hive
42 | 
43 | // Connect to spark thrift server..
44 | `beeline> !connect jdbc:hive2://localhost:10000`
45 | Connecting to jdbc:hive2://localhost:10000
46 | Enter username for jdbc:hive2://localhost:10000:
47 | Enter password for jdbc:hive2://localhost:10000:
48 | 
49 | // run your sql queries and access data..
50 | `jdbc:hive2://localhost:10000> show tables;,`
51 | ```
52 | 2. **Java JDBC:** Please refer to this project's test folder where I've shared a java example `TestThriftClient.java` to demo the same.
53 | 
54 | ## 5. Requirements
55 | - Spark 2.1.0, Java 1.8 and Scala 2.11
56 | 
57 | ## 6. References:
58 | - Complete guide and references to this project are briefed in my blog [here](https://spoddutur.github.io/spark-notes/spark-as-cloud-based-sql-engine-via-thrift-server).
59 | 


--------------------------------------------------------------------------------
/dependency-reduced-pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  3 |   <modelVersion>4.0.0</modelVersion>
  4 |   <groupId>com.spoddutur</groupId>
  5 |   <artifactId>cloud-based-sql-engine-using-spark</artifactId>
  6 |   <version>1.0-SNAPSHOT</version>
  7 |   <build>
  8 |     <resources>
  9 |       <resource>
 10 |         <directory>src/main/resources/${env}</directory>
 11 |       </resource>
 12 |       <resource>
 13 |         <directory>src/main/resources/common</directory>
 14 |       </resource>
 15 |     </resources>
 16 |     <plugins>
 17 |       <plugin>
 18 |         <groupId>net.alchim31.maven</groupId>
 19 |         <artifactId>scala-maven-plugin</artifactId>
 20 |         <version>3.2.0</version>
 21 |         <executions>
 22 |           <execution>
 23 |             <phase>process-sources</phase>
 24 |             <goals>
 25 |               <goal>compile</goal>
 26 |               <goal>testCompile</goal>
 27 |             </goals>
 28 |             <configuration />
 29 |           </execution>
 30 |         </executions>
 31 |       </plugin>
 32 |       <plugin>
 33 |         <artifactId>maven-compiler-plugin</artifactId>
 34 |         <version>3.3</version>
 35 |         <configuration>
 36 |           <source>${java.version}</source>
 37 |           <target>${java.version}</target>
 38 |         </configuration>
 39 |       </plugin>
 40 |       <plugin>
 41 |         <artifactId>maven-shade-plugin</artifactId>
 42 |         <executions>
 43 |           <execution>
 44 |             <phase>package</phase>
 45 |             <goals>
 46 |               <goal>shade</goal>
 47 |             </goals>
 48 |             <configuration>
 49 |               <filters>
 50 |                 <filter>
 51 |                   <artifact>*:*</artifact>
 52 |                   <excludes>
 53 |                     <exclude>META-INF/*.SF</exclude>
 54 |                     <exclude>META-INF/*.DSA</exclude>
 55 |                     <exclude>META-INF/*.RSA</exclude>
 56 |                   </excludes>
 57 |                 </filter>
 58 |               </filters>
 59 |               <transformers>
 60 |                 <transformer>
 61 |                   <mainClass>at.seresunit.lecturemanager_connector.App</mainClass>
 62 |                 </transformer>
 63 |                 <transformer>
 64 |                   <resource>META-INF/spring.handlers</resource>
 65 |                 </transformer>
 66 |                 <transformer>
 67 |                   <resource>META-INF/spring.schemas</resource>
 68 |                 </transformer>
 69 |               </transformers>
 70 |             </configuration>
 71 |           </execution>
 72 |         </executions>
 73 |       </plugin>
 74 |     </plugins>
 75 |   </build>
 76 |   <dependencies>
 77 |     <dependency>
 78 |       <groupId>junit</groupId>
 79 |       <artifactId>junit</artifactId>
 80 |       <version>4.11</version>
 81 |       <scope>test</scope>
 82 |       <exclusions>
 83 |         <exclusion>
 84 |           <artifactId>hamcrest-core</artifactId>
 85 |           <groupId>org.hamcrest</groupId>
 86 |         </exclusion>
 87 |       </exclusions>
 88 |     </dependency>
 89 |     <dependency>
 90 |       <groupId>org.hamcrest</groupId>
 91 |       <artifactId>hamcrest-all</artifactId>
 92 |       <version>1.3</version>
 93 |       <scope>test</scope>
 94 |     </dependency>
 95 |     <dependency>
 96 |       <groupId>org.scalacheck</groupId>
 97 |       <artifactId>scalacheck_2.11</artifactId>
 98 |       <version>1.11.4</version>
 99 |       <scope>test</scope>
100 |       <exclusions>
101 |         <exclusion>
102 |           <artifactId>test-interface</artifactId>
103 |           <groupId>org.scala-sbt</groupId>
104 |         </exclusion>
105 |       </exclusions>
106 |     </dependency>
107 |     <dependency>
108 |       <groupId>org.scalatest</groupId>
109 |       <artifactId>scalatest_2.11</artifactId>
110 |       <version>2.2.0</version>
111 |       <scope>test</scope>
112 |     </dependency>
113 |   </dependencies>
114 |   <properties>
115 |     <scala.version>2.11.8</scala.version>
116 |     <java.version>1.8</java.version>
117 |     <scala.binary.version>2.11</scala.binary.version>
118 |     <aws.version>1.10.50</aws.version>
119 |     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
120 |     <spark.version>2.1.0</spark.version>
121 |   </properties>
122 | </project>
123 | 
124 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <project xmlns="http://maven.apache.org/POM/4.0.0"
  3 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  4 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  5 |     <modelVersion>4.0.0</modelVersion>
  6 | 
  7 |     <groupId>com.spoddutur</groupId>
  8 |     <artifactId>cloud-based-sql-engine-using-spark</artifactId>
  9 |     <version>1.0-SNAPSHOT</version>
 10 | 
 11 |     <properties>
 12 |         <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
 13 |         <java.version>1.8</java.version>
 14 |         <scala.version>2.11.8</scala.version>
 15 |         <scala.binary.version>2.11</scala.binary.version>
 16 |         <spark.version>2.1.0</spark.version>
 17 |         <aws.version>1.10.50</aws.version>
 18 |     </properties>
 19 | 
 20 |     <build>
 21 |         <plugins>
 22 |             <plugin>
 23 |                 <groupId>net.alchim31.maven</groupId>
 24 |                 <artifactId>scala-maven-plugin</artifactId>
 25 |                 <version>3.2.0</version>
 26 |                 <executions>
 27 |                     <execution>
 28 |                         <!-- this is so we don't end with a compile error in maven-compiler-plugin -->
 29 |                         <phase>process-sources</phase>
 30 |                         <goals>
 31 |                             <goal>compile</goal>
 32 |                             <goal>testCompile</goal>
 33 |                         </goals>
 34 |                         <configuration>
 35 |                         </configuration>
 36 |                     </execution>
 37 |                 </executions>
 38 |             </plugin>
 39 |             <plugin>
 40 |                 <groupId>org.apache.maven.plugins</groupId>
 41 |                 <artifactId>maven-compiler-plugin</artifactId>
 42 |                 <version>3.3</version>
 43 |                 <configuration>
 44 |                     <source>${java.version}</source>
 45 |                     <target>${java.version}</target>
 46 |                 </configuration>
 47 |             </plugin>
 48 |             <plugin>
 49 |                 <groupId>org.apache.maven.plugins</groupId>
 50 |                 <artifactId>maven-shade-plugin</artifactId>
 51 |                 <executions>
 52 |                     <execution>
 53 |                         <phase>package</phase>
 54 |                         <goals>
 55 |                             <goal>shade</goal>
 56 |                         </goals>
 57 |                         <configuration>
 58 |                             <filters>
 59 |                                 <filter>
 60 |                                     <artifact>*:*</artifact>
 61 |                                     <excludes>
 62 |                                         <exclude>META-INF/*.SF</exclude>
 63 |                                         <exclude>META-INF/*.DSA</exclude>
 64 |                                         <exclude>META-INF/*.RSA</exclude>
 65 |                                     </excludes>
 66 |                                 </filter>
 67 |                             </filters>
 68 |                             <transformers>
 69 |                                 <transformer
 70 |                                         implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
 71 |                                     <mainClass>at.seresunit.lecturemanager_connector.App</mainClass>
 72 |                                 </transformer>
 73 |                                 <transformer
 74 |                                         implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
 75 |                                     <resource>META-INF/spring.handlers</resource>
 76 |                                 </transformer>
 77 |                                 <transformer
 78 |                                         implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
 79 |                                     <resource>META-INF/spring.schemas</resource>
 80 |                                 </transformer>
 81 |                             </transformers>
 82 |                         </configuration>
 83 |                     </execution>
 84 |                 </executions>
 85 |             </plugin>
 86 |         </plugins>
 87 |         <resources>
 88 |             <resource>
 89 |                 <directory>src/main/resources/${env}</directory>
 90 |             </resource>
 91 |             <resource>
 92 |                 <directory>src/main/resources/common</directory>
 93 |             </resource>
 94 |         </resources>
 95 |     </build>
 96 | 
 97 |     <dependencies>
 98 |         <dependency>
 99 |             <groupId>com.typesafe</groupId>
100 |             <artifactId>config</artifactId>
101 |             <version>1.3.1</version>
102 |         </dependency>
103 |         <dependency>
104 |             <groupId>com.fasterxml.jackson.core</groupId>
105 |             <artifactId>jackson-core</artifactId>
106 |             <version>2.6.5</version>
107 |         </dependency>
108 |         <dependency>
109 |             <groupId>com.fasterxml.jackson.core</groupId>
110 |             <artifactId>jackson-databind</artifactId>
111 |             <version>2.6.5</version>
112 |         </dependency>
113 |         <dependency>
114 |             <groupId>org.scala-lang</groupId>
115 |             <artifactId>scala-library</artifactId>
116 |             <version>${scala.version}</version>
117 |         </dependency>
118 |         <dependency>
119 |             <groupId>junit</groupId>
120 |             <artifactId>junit</artifactId>
121 |             <version>4.11</version>
122 |             <scope>test</scope>
123 |         </dependency>
124 |         <dependency>
125 |             <groupId>org.hamcrest</groupId>
126 |             <artifactId>hamcrest-all</artifactId>
127 |             <version>1.3</version>
128 |             <scope>test</scope>
129 |         </dependency>
130 |         <dependency>
131 |             <groupId>org.scalacheck</groupId>
132 |             <artifactId>scalacheck_${scala.binary.version}</artifactId>
133 |             <version>1.11.4</version>
134 |             <scope>test</scope>
135 |         </dependency>
136 |         <dependency>
137 |             <groupId>org.scalatest</groupId>
138 |             <artifactId>scalatest_${scala.binary.version}</artifactId>
139 |             <version>2.2.0</version>
140 |             <scope>test</scope>
141 |         </dependency>
142 |         <dependency>
143 |             <groupId>org.apache.spark</groupId>
144 |             <artifactId>spark-core_${scala.binary.version}</artifactId>
145 |             <version>${spark.version}</version>
146 |         </dependency>
147 |         <dependency>
148 |             <groupId>org.apache.spark</groupId>
149 |             <artifactId>spark-sql_${scala.binary.version}</artifactId>
150 |             <version>${spark.version}</version>
151 |         </dependency>
152 |         <dependency>
153 |             <groupId>org.apache.spark</groupId>
154 |             <artifactId>spark-streaming_${scala.binary.version}</artifactId>
155 |             <version>${spark.version}</version>
156 |         </dependency>
157 |         <!--HIVE-->
158 |         <dependency>
159 |             <groupId>org.apache.spark</groupId>
160 |             <artifactId>spark-hive_${scala.binary.version}</artifactId>
161 |             <version>${spark.version}</version>
162 |         </dependency>
163 |         <dependency>
164 |             <groupId>org.apache.calcite</groupId>
165 |             <artifactId>calcite-avatica</artifactId>
166 |             <version>1.6.0</version>
167 |         </dependency>
168 | 
169 |         <!-- https://mvnrepository.com/artifact/org.apache.calcite/calcite-core -->
170 |         <dependency>
171 |             <groupId>org.apache.calcite</groupId>
172 |             <artifactId>calcite-core</artifactId>
173 |             <version>1.12.0</version>
174 |         </dependency>
175 | 
176 |         <!-- https://mvnrepository.com/artifact/org.spark-project.hive/hive-exec -->
177 |         <dependency>
178 |             <groupId>org.spark-project.hive</groupId>
179 |             <artifactId>hive-exec</artifactId>
180 |             <version>1.2.1.spark2</version>
181 |         </dependency>
182 | 
183 |         <!-- https://mvnrepository.com/artifact/org.spark-project.hive/hive-metastore -->
184 |         <dependency>
185 |             <groupId>org.spark-project.hive</groupId>
186 |             <artifactId>hive-metastore</artifactId>
187 |             <version>1.2.1.spark2</version>
188 |         </dependency>
189 | 
190 |         <!-- https://mvnrepository.com/artifact/org.codehaus.jackson/jackson-mapper-asl -->
191 |         <dependency>
192 |             <groupId>org.codehaus.jackson</groupId>
193 |             <artifactId>jackson-mapper-asl</artifactId>
194 |             <version>1.9.13</version>
195 |         </dependency>
196 | 
197 | 
198 |         <!--HIVE-->
199 |         <dependency>
200 |             <groupId>com.amazonaws</groupId>
201 |             <artifactId>aws-java-sdk-core</artifactId>
202 |             <version>${aws.version}</version>
203 |         </dependency>
204 |         <dependency>
205 |             <groupId>com.amazonaws</groupId>
206 |             <artifactId>aws-java-sdk-s3</artifactId>
207 |             <version>${aws.version}</version>
208 |         </dependency>
209 |         <dependency>
210 |             <groupId>org.slf4j</groupId>
211 |             <artifactId>slf4j-log4j12</artifactId>
212 |             <version>1.7.13</version>
213 |         </dependency>
214 |         <dependency>
215 |             <groupId>log4j</groupId>
216 |             <artifactId>log4j</artifactId>
217 |             <version>1.2.17</version>
218 |         </dependency>
219 |         <dependency>
220 |             <groupId>org.slf4j</groupId>
221 |             <artifactId>slf4j-api</artifactId>
222 |             <version>1.7.13</version>
223 |         </dependency>
224 |         <dependency>
225 |             <groupId>com.google.code.gson</groupId>
226 |             <artifactId>gson</artifactId>
227 |             <version>2.2.4</version>
228 |         </dependency>
229 | 
230 |        <dependency>
231 |             <groupId>org.apache.hive</groupId>
232 |             <artifactId>hive-common</artifactId>
233 |             <version>2.1.1</version>
234 |         </dependency>
235 | 
236 |         <dependency>
237 |             <groupId>org.apache.hive</groupId>
238 |             <artifactId>hive-jdbc</artifactId>
239 |             <version>2.1.1</version>
240 |         </dependency>
241 | 
242 |         <dependency>
243 |             <groupId>org.apache.hive</groupId>
244 |             <artifactId>hive-exec</artifactId>
245 |             <version>2.1.1</version>
246 |         </dependency>
247 | 
248 |         <dependency>
249 |             <groupId>org.apache.hive</groupId>
250 |             <artifactId>hive-service</artifactId>
251 |             <version>2.1.1</version>
252 |         </dependency>
253 |     </dependencies>
254 | 
255 | </project>


--------------------------------------------------------------------------------