BytesToGreyImg(28, 28) -> GreyImgNormalizer(trainMean, trainStd) -> GreyImgToBatch(batchSize)
66 | val optimizer = Optimizer(model = model, dataset = trainSet, criterion = ClassNLLCriterion[Float]())
67 | val testMean = 0.13251460696903547
68 | val testStd = 0.31048024
69 | val maxEpoch = 2
70 | val validationSet = DataSet.array(load(validationData, validationLabel), sc) -> BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToBatch(batchSize)
71 | optimizer.setEndWhen(Trigger.maxEpoch(2))
72 | optimizer.setState(T("learningRate" -> 0.05, "learningRateDecay" -> 0.0))
73 | optimizer.setCheckpoint("/Users/aurobindosarkar/Downloads/mnist/checkpoint", Trigger.severalIteration(500))
74 | optimizer.setValidation(trigger = Trigger.everyEpoch, dataset = validationSet, vMethods = Array(new Top1Accuracy, new Top5Accuracy[Float], new Loss[Float]))
75 | optimizer.optimize()
76 | model.save("/Users/aurobindosarkar/Downloads/mnist/model")
77 | val model = Module.load[Float]("/Users/aurobindosarkar/Downloads/mnist/model")
78 | val partitionNum = 2
79 | val rddData = sc.parallelize(load(validationData, validationLabel), partitionNum)
80 | val transformer = BytesToGreyImg(28, 28) -> GreyImgNormalizer(testMean, testStd) -> GreyImgToSample()
81 | val evaluationSet = transformer(rddData)
82 | val result = model.evaluate(evaluationSet, Array(new Top1Accuracy[Float]), Some(batchSize))
83 | result.foreach(r => println(s"${r._2} is ${r._1}"))
84 |
85 |
--------------------------------------------------------------------------------
/Chapter11/Chapter11.scala:
--------------------------------------------------------------------------------
1 | //Code for Chapter 11 to be executed in Spark shell
2 |
3 | //Code for Optimizing data serialization
4 | import org.apache.spark.sql._
5 | import org.apache.spark.sql.types._
6 | import org.apache.spark.sql.functions._
7 | import org.apache.spark.sql.streaming._
8 | import spark.implicits._
9 | import spark.sessionState.conf
10 | import org.apache.spark.sql.internal.SQLConf.SHUFFLE_PARTITIONS
11 | import org.apache.spark.sql.Encoders
12 | import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
13 | import org.apache.spark.util.SizeEstimator
14 |
15 | //Code for Optimizing data serialization section
16 | case class Bid(bidid: String, timestamp: String, ipinyouid: String, useragent: String, IP: String, region: Integer, cityID: Integer, adexchange: String, domain: String, turl: String, urlid: String, slotid: String, slotwidth: String, slotheight: String, slotvisibility: String, slotformat: String, slotprice: String, creative: String, bidprice: String)
17 | val bidEncoder = Encoders.product[Bid]
18 | bidEncoder.schema
19 | val bidExprEncoder = bidEncoder.asInstanceOf[ExpressionEncoder[Bid]]
20 | bidExprEncoder.serializer
21 | bidExprEncoder.namedExpressions
22 | val bidsDF = spark.read.format("csv").schema(bidEncoder.schema).option("sep", "\t").option("header", false).load("file:///Users/aurobindosarkar/Downloads/make-ipinyou-data-master/original-data/ipinyou.contest.dataset/bidfiles")
23 | bidsDF.take(1).foreach(println)
24 | val bid = Bid("e3d962536ef3ac7096b31fdd1c1c24b0","20130311172101557","37a6259cc0c1dae299a7866489dff0bd","Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; QQDownload 734; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; eSobiSubscriber 2.0.4.16; MAAR),gzip(gfe),gzip(gfe)","219.232.120.*",1,1,"2","","DF9blS9bQqsIFYB4uA5R,b6c5272dfc63032f659be9b786c5f8da",null,"2006366309","728","90","1","0","5","5aca4c5f29e59e425c7ea657fdaac91e","300")
25 | val row = bidExprEncoder.toRow(bid)
26 | import org.apache.spark.sql.catalyst.dsl.expressions._
27 | val attrs = Seq(DslSymbol('bidid).string, DslSymbol('timestamp).string, DslSymbol('ipinyouid).string, DslSymbol('useragent).string, DslSymbol('IP).string, DslSymbol('region).int, DslSymbol('cityID).int, DslSymbol('adexchange).string, DslSymbol('domain).string, DslSymbol('turl).string, DslSymbol('urlid).string, DslSymbol('slotid).string, DslSymbol('slotwidth).string, DslSymbol('slotheight).string, DslSymbol('slotvisibility).string, DslSymbol('slotformat).string, DslSymbol('slotprice).string, DslSymbol('creative).string, DslSymbol('bidprice).string)
28 | val getBackBid = bidExprEncoder.resolveAndBind(attrs).fromRow(row)
29 | bid == getBackBid
30 |
31 | //Code for Understanding the Dataset/DataFrame API section
32 | val t1 = spark.range(7)
33 | val t2 = spark.range(13)
34 | val t3 = spark.range(19)
35 | t1.explain()
36 | t1.explain(extended=true)
37 | t1.filter("id != 0").filter("id != 2").explain(true)
38 | spark.sessionState.analyzer
39 | t1.filter("id != 0").filter("id != 2")
40 |
41 | //Code for Understanding Catalyst transformations sections
42 | val t0 = spark.range(0, 10000000)
43 | val df1 = t0.withColumn("uniform", rand(seed=10))
44 | val df2 = t0.withColumn("normal", randn(seed=27))
45 | df1.createOrReplaceTempView("t1")
46 | df2.createOrReplaceTempView("t2")
47 | spark.sql("SELECT sum(v) FROM (SELECT t1.id, 1 + t1.normal AS v FROM t1 JOIN t2 WHERE t1.id = t2.id AND t2.id > 5000000) tmp").explain(true)
48 |
49 | //Code for Visualizing Spark application execution section
50 | val t1 = spark.range(7)
51 | val t2 = spark.range(13)
52 | val t3 = spark.range(19)
53 | val t4 = spark.range(1e8.toLong)
54 | val t5 = spark.range(1e8.toLong)
55 | val t6 = spark.range(1e3.toLong)
56 |
57 | val query = t1.join(t2).where(t1("id") === t2("id")).join(t3).where(t3("id") === t1("id")).explain()
58 | val query = t1.join(t2).where(t1("id") === t2("id")).join(t3).where(t3("id") === t1("id")).count()
59 | val query = t4.join(t5).where(t4("id") === t5("id")).join(t6).where(t4("id") === t6("id")).explain()
60 |
61 | //Code for Understanding the CBO statistics collection section
62 |
63 | sql("DESCRIBE EXTENDED customers").collect.foreach(println)
64 |
65 | //Code for Build side selection section
66 | spark.sql("DROP TABLE IF EXISTS t1")
67 | spark.sql("DROP TABLE IF EXISTS t2")
68 | spark.sql("CREATE TABLE IF NOT EXISTS t1(id long, value long) USING parquet")
69 | spark.sql("CREATE TABLE IF NOT EXISTS t2(id long, value string) USING parquet")
70 |
71 | spark.range(5E8.toLong).select('id, (rand(17) * 1E6) cast "long").write.mode("overwrite").insertInto("t1")
72 | spark.range(1E8.toLong).select('id, 'id cast "string").write.mode("overwrite").insertInto("t2")
73 |
74 | sql("SELECT t1.id FROM t1, t2 WHERE t1.id = t2.id AND t1.value = 100").explain()
75 |
76 | //Code for Understanding multi-way JOIN ordering optimization section
77 | sql("CREATE TABLE IF NOT EXISTS customers(id long, name string) USING parquet")
78 | sql("CREATE TABLE IF NOT EXISTS goods(id long, price long) USING parquet")
79 | sql("CREATE TABLE IF NOT EXISTS orders(customer_id long, good_id long) USING parquet")
80 |
81 | import org.apache.spark.sql.functions.rand
82 |
83 | spark.sql("CREATE TABLE IF NOT EXISTS customers(id long, name string) USING parquet")
84 | spark.sql("CREATE TABLE IF NOT EXISTS goods(id long, price long) USING parquet")
85 | spark.sql("CREATE TABLE IF NOT EXISTS orders(customer_id long, good_id long) USING parquet")
86 |
87 | spark.range(2E8.toLong).select('id, 'id cast "string").write.mode("overwrite").insertInto("customers")
88 |
89 | spark.range(1E8.toLong).select('id, (rand(17) * 1E6 + 2) cast "long").write.mode("overwrite").insertInto("goods")
90 | spark.range(1E7.toLong).select(rand(3) * 2E8 cast "long", (rand(5) * 1E8) cast "long").write.mode("overwrite").insertInto("orders")
91 |
92 | def benchmark(name: String)(f: => Unit) {
93 | val startTime = System.nanoTime
94 | f
95 | val endTime = System.nanoTime
96 | println(s"Time taken with $name: " + (endTime -
97 | startTime).toDouble / 1000000000 + " seconds")
98 | }
99 |
100 | val conf = spark.sessionState.conf
101 |
102 | spark.conf.set("spark.sql.cbo.enabled", false)
103 |
104 | conf.cboEnabled
105 | conf.joinReorderEnabled
106 | benchmark("CBO OFF & JOIN REORDER DISABLED"){ sql("SELECT name FROM customers, orders, goods WHERE customers.id = orders.customer_id AND orders.good_id = goods.id AND goods.price > 1000000").show() }
107 |
108 | spark.conf.set("spark.sql.cbo.enabled", true)
109 | conf.cboEnabled
110 | conf.joinReorderEnabled
111 |
112 | benchmark("CBO ON & JOIN REORDER DIABLED"){ sql("SELECT name FROM customers, orders, goods WHERE customers.id = orders.customer_id AND orders.good_id = goods.id AND goods.price > 1000000").show()}
113 |
114 | spark.conf.set("spark.sql.cbo.enabled", true)
115 | spark.conf.set("spark.sql.cbo.joinReorder.enabled", true)
116 | conf.cboEnabled
117 | conf.joinReorderEnabled
118 |
119 | scala> benchmark("CBO ON & JOIN REORDER ENABLED"){ sql("SELECT name FROM customers, orders, goods WHERE customers.id = orders.customer_id AND orders.good_id = goods.id AND goods.price > 1000000").show()}
120 |
121 | //Code for Understanding performance improvements using whole-stage code generation section
122 | scala> sql("select count(*) from orders where customer_id = 26333955").explain()
123 | long count = 0;
124 | for (customer_id in orders) {
125 | if (customer_id == 26333955) {
126 | count += 1;
127 | }
128 | }
129 |
130 | sql("EXPLAIN CODEGEN SELECT name FROM customers, orders, goods WHERE customers.id = orders.customer_id AND orders.good_id = goods.id AND goods.price > 1000000").take(1).foreach(println)
131 |
132 | spark.conf.set("spark.sql.codegen.wholeStage", false)
133 |
134 | conf.wholeStageEnabled
135 |
136 | val N = 20 << 20
137 | val M = 1 << 16
138 |
139 | val dim = broadcast(spark.range(M).selectExpr("id as k", "cast(id as string) as v"))
140 |
141 | benchmark("Join w long") {
142 | spark.range(N).join(dim, (col("id") % M) === col("k")).count()
143 | }
144 |
145 | spark.conf.set("spark.sql.codegen.wholeStage", true)
146 |
147 | conf.wholeStageEnabled
148 | val dim = broadcast(spark.range(M).selectExpr("id as k", "cast(id as string) as v"))
149 |
150 | benchmark("Join w long") {
151 | spark.range(N).join(dim, (col("id") % M) === col("k")).count()
152 | }
153 |
154 | val dim = broadcast(spark.range(M).selectExpr("id as k", "cast(id as string) as v"))
155 | benchmark("Join w long duplicated") {
156 | val dim = broadcast(spark.range(M).selectExpr("cast(id/10 as long) as k"))
157 | spark.range(N).join(dim, (col("id") % M) === col("k")).count()
158 | }
159 |
160 | val dim3 = broadcast(spark.range(M).selectExpr("id as k1", "id as k2", "cast(id as string) as v"))
161 | benchmark("Join w 2 longs") {
162 | spark.range(N).join(dim3, (col("id") % M) === col("k1") && (col("id") % M) === col("k2")).count()
163 | }
164 |
165 | val dim4 = broadcast(spark.range(M).selectExpr("cast(id/10 as long) as k1", "cast(id/10 as long) as k2"))
166 | benchmark("Join w 2 longs duplicated") {
167 | spark.range(N).join(dim4, (col("id") bitwiseAND M) === col("k1") && (col("id") bitwiseAND M) === col("k2")).count()
168 | }
169 |
170 | val dim = broadcast(spark.range(M).selectExpr("id as k", "cast(id as string) as v"))
171 | benchmark("outer join w long") {
172 | spark.range(N).join(dim, (col("id") % M) === col("k"), "left").count()
173 | }
174 |
175 | val dim = broadcast(spark.range(M).selectExpr("id as k", "cast(id as string) as v"))
176 | benchmark("semi join w long") {
177 | spark.range(N).join(dim, (col("id") % M) === col("k"), "leftsemi").count()
178 | }
179 |
180 | val N = 2 << 20
181 | benchmark("merge join") {
182 | val df1 = spark.range(N).selectExpr(s"id * 2 as k1")
183 | val df2 = spark.range(N).selectExpr(s"id * 3 as k2")
184 | df1.join(df2, col("k1") === col("k2")).count()
185 | }
186 |
187 | val N = 2 << 20
188 | benchmark("sort merge join") {
189 | val df1 = spark.range(N).selectExpr(s"(id * 15485863) % ${N*10} as k1")
190 | val df2 = spark.range(N).selectExpr(s"(id * 15485867) % ${N*10} as k2")
191 | df1.join(df2, col("k1") === col("k2")).count()
192 | }
193 |
194 | scala> conf.getAllConfs.foreach(println)
195 | scala> conf.getAllDefinedConfs.foreach(println)
196 |
--------------------------------------------------------------------------------
/Chapter12/Chapter12.scala:
--------------------------------------------------------------------------------
1 | //Code for Chapter 12 to be executed in Spark root_shell
2 |
3 | //Code for Transforming data in ETL pipelines section
4 | //This example requires Kafka commands to be executed at the appropriate places, please follow the instructions and sequence in the chapter.
5 | val jsonDF = spark.read.json("file:///Users/aurobindosarkar/Downloads/cache-0-json")
6 | jsonDF.printSchema()
7 | val rawTweetsSchema = jsonDF.schema
8 | val jsonString = rawTweetsSchema.json
9 | val schema = DataType.fromJson(jsonString).asInstanceOf[StructType]
10 | val rawTweets = spark.readStream.format("kafka").option("kafka.bootstrap.servers", "localhost:9092").option("subscribe", "tweetsa").load()
11 | val parsedTweets = rawTweets.selectExpr("cast (value as string) as json").select(from_json($"json", schema).as("data")).select("data.*")
12 |
13 | //************* Kafka commands here ************************//
14 |
15 | val selectFields = parsedTweets.select("place.country").where($"place.country".isNotNull)
16 | val s5 = selectFields.writeStream.outputMode("append").format("console").start()
17 | val selectFields = parsedTweets.select("place.*").where($"place.country".isNotNull)
18 | val selectFields = parsedTweets.select(struct("place.country_code", "place.name") as 'locationInfo).where($"locationInfo.country_code".isNotNull)
19 | val selectFields = parsedTweets.select($"entities.hashtags" as 'tags).select('tags.getItem(0) as 'x).select($"x.indices" as 'y).select($"y".getItem(0) as 'z).where($"z".isNotNull)
20 | val selectFields = parsedTweets.select($"entities.hashtags" as 'tags).select('tags.getItem(0) as 'x).select($"x.text" as 'y).where($"y".isNotNull)
21 | val selectFields = parsedTweets.select($"entities.hashtags.indices" as 'tags).select(explode('tags))
22 | val selectFields = parsedTweets.select($"entities.hashtags.indices".getItem(0) as 'tags).select(explode('tags))
23 | val selectFields = parsedTweets.select(struct($"entities.media.type" as 'x, $"entities.media.url" as 'y) as 'z).where($"z.x".isNotNull).select(to_json('z) as 'c)
24 |
25 |
26 | //Code for Addressing errors in ETL pipelines section
27 | import spark.implicits._
28 | import org.apache.spark.sql.types._
29 | spark.read.json("file:///Users/aurobindosarkar/Downloads/test1.json").printSchema()
30 | val schema = new StructType().add("a", "int").add("b", "double")
31 | spark.read.option("header", true).schema(schema).csv("file:///Users/aurobindosarkar/Downloads/test1.csv").show()
32 | spark.read.option("mode", "PERMISSIVE").option("columnNameOfCorruptRecord", "_corrupt_record").json("file:///Users/aurobindosarkar/Downloads/test1.json").show()
33 | spark.read.option("mode", "DROPMALFORMED").json("file:///Users/aurobindosarkar/Downloads/test1.json").show()
34 | spark.read.option("mode", "FAILFAST").json("file:///Users/aurobindosarkar/Downloads/test1.json").show()
35 | spark.read.option("wholeFile",true).option("mode", "PERMISSIVE").option("columnNameOfCorruptRecord", "_corrupt_record").json("file:///Users/aurobindosarkar/Downloads/testMultiLine.json")
36 | spark.read.option("multiLine",true).option("mode", "PERMISSIVE").option("columnNameOfCorruptRecord", "_corrupt_record").json("file:///Users/aurobindosarkar/Downloads/testMultiLine.json")
37 | spark.read.option("multiLine",true).option("header", true).csv("file:///Users/aurobindosarkar/Downloads/test1.csv").show()
38 |
39 | //Code for Implementing a scalable monitoring solution section
40 | //Uses Kafka so please follow the instructions in the book for executing the code in this section
41 | import org.apache.spark.sql.types._
42 | import org.apache.spark.sql.functions._
43 | import spark.implicits._
44 | import org.apache.spark.sql.streaming._
45 |
46 | val schema = new StructType().add("clientIpAddress", "string").add("rfc1413ClientIdentity", "string").add("remoteUser", "string").add("dateTime", "string").add("zone", "string").add("request","string").add("httpStatusCode", "string").add("bytesSent", "string").add("referer", "string").add("userAgent", "string")
47 | val rawRecords = spark.readStream.option("header", false).schema(schema).option("sep", " ").format("csv").load("file:///Users/aurobindosarkar/Downloads/NASA")
48 | val ts = unix_timestamp(concat($"dateTime", lit(" "), $"zone"), "[dd/MMM/yyyy:HH:mm:ss Z]").cast("timestamp")
49 | val logEvents = rawRecords.withColumn("ts", ts).withColumn("date", ts.cast(DateType)).select($"ts", $"date", $"clientIpAddress", concat($"dateTime", lit(" "), $"zone").as("original_dateTime"), $"request", $"httpStatusCode", $"bytesSent")
50 | val query = logEvents.writeStream.outputMode("append").format("console").start()
51 | val streamingETLQuery = logEvents.writeStream.trigger(Trigger.ProcessingTime("2 minutes")).format("parquet").partitionBy("date").option("path", "file:///Users/aurobindosarkar/Downloads/NASALogs").option("checkpointLocation", "file:///Users/aurobindosarkar/Downloads/NASALogs/checkpoint/").start()
52 | val rawCSV = spark.readStream.schema(schema).option("latestFirst", "true").option("maxFilesPerTrigger", "5").option("header", false).option("sep", " ").format("csv").load("file:///Users/aurobindosarkar/Downloads/NASA")
53 | val streamingETLQuery = logEvents.writeStream.trigger(Trigger.ProcessingTime("2 minutes")).format("json").partitionBy("date").option("path", "file:///Users/aurobindosarkar/Downloads/NASALogs").option("checkpointLocation", "file:///Users/aurobindosarkar/Downloads/NASALogs/checkpoint/").start()
54 | val kafkaQuery = logEvents.selectExpr("CAST(ts AS STRING) AS key", "to_json(struct(*)) AS value").writeStream.format("kafka").option("kafka.bootstrap.servers", "localhost:9092").option("topic", "topica").option("checkpointLocation", "file:///Users/aurobindosarkar/Downloads/NASALogs/kafkacheckpoint/").start()
55 | val kafkaDF = spark.readStream.format("kafka").option("kafka.bootstrap.servers", "localhost:9092").option("subscribe", "topica").option("startingOffsets", "earliest").load()
56 | kafkaDF.printSchema()
57 | val kafkaSchema = new StructType().add("ts", "timestamp").add("date", "string").add("clientIpAddress", "string").add("rfc1413ClientIdentity", "string").add("remoteUser", "string").add("original_dateTime", "string").add("request", "string").add("httpStatusCode", "string").add("bytesSent", "string")
58 | val kafkaDF1 = kafkaDF.select(col("key").cast("string"), from_json(col("value").cast("string"), kafkaSchema).as("data")).select("data.*")
59 | val kafkaQuery1 = kafkaDF1.select($"ts", $"date", $"clientIpAddress", $"original_dateTime", $"request", $"httpStatusCode", $"bytesSent").writeStream.outputMode("append").format("console").start()
60 | val kafkaDF2 = spark.read.format("kafka").option("kafka.bootstrap.servers","localhost:9092").option("subscribe", "topica").load().selectExpr("CAST(value AS STRING) as myvalue")
61 | kafkaDF2.registerTempTable("topicData3")
62 | spark.sql("select myvalue from topicData3").take(3).foreach(println)
63 | val parsed = spark.readStream.format("kafka").option("kafka.bootstrap.servers", "localhost:9092").option("subscribe", "topica").option("startingOffsets", "earliest").load().select(from_json(col("value").cast("string"), kafkaSchema).alias("parsed_value"))
64 | val query = parsed.writeStream.outputMode("append").format("console").start()
65 | val selectAllParsed = parsed.select("parsed_value.*")
66 | val selectFieldsParsed = selectAllParsed.select("ts", "clientIpAddress", "request", "httpStatusCode")
67 | val s1 = selectFieldsParsed.groupBy(window($"ts", "10 minutes", "5 minutes"), $"httpStatusCode").count().writeStream.outputMode("complete").format("console").start()
68 | val s2 = selectFieldsParsed.groupBy(window($"ts", "10 minutes", "5 minutes"), $"request").count().writeStream.outputMode("complete").format("console").start()
69 | val s4 = selectFieldsParsed.withWatermark("ts", "10 minutes").groupBy(window($"ts", "10 minutes", "5 minutes"), $"request").count().writeStream.outputMode("complete").format("console").start()
70 |
71 | //Code for Deploying Spark machine learning pipelines section
72 | //Follow the book for the sample pyspark session
73 | import org.apache.spark.ml.{Pipeline, PipelineModel}
74 | import org.apache.spark.ml.classification.LogisticRegression
75 | import org.apache.spark.ml.feature.{HashingTF, Tokenizer}
76 | import org.apache.spark.ml.linalg.Vector
77 | import org.apache.spark.sql.Row
78 |
79 | val sameModel = PipelineModel.load("file:///Users/aurobindosarkar/Downloads/spark-logistic-regression-model")
80 | val test = spark.createDataFrame(Seq(
81 | (4L, "spark i j k"),
82 | (5L, "l m n"),
83 | (6L, "spark hadoop spark"),
84 | (7L, "apache hadoop")
85 | )).toDF("id", "text")
86 |
87 | sameModel.transform(test).select("id", "text", "probability", "prediction").collect().foreach { case Row(id: Long, text: String, prob: Vector, prediction: Double) => println(s"($id, $text) --> prob=$prob, prediction=$prediction")}
88 | val df = spark.read.parquet("file:///Users/aurobindosarkar/Downloads/spark-logistic-regression-model/stages/2_LogisticRegression_4abda37bdde1ddf65ea0/data/part-00000-415bf215-207a-4a49-985e-190eaf7253a7-c000.snappy.parquet")
89 | df.show()
90 |
91 | df.collect.foreach(println)
92 |
93 | //Code for Understanding types of model scoring architectures section
94 | //Follow the instructions in book to complete the non-Spark Scala parts
95 | spark.read.parquet("file:///Users/aurobindosarkar/Downloads/spark-logistic-regression-model/stages/2_LogisticRegression_4abda37bdde1ddf65ea0/data/part-00000-415bf215-207a-4a49-985e-190eaf7253a7-c000.snappy.parquet").write.mode("overwrite").json("file:///Users/aurobindosarkar/Downloads/lr-model-json")
96 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2017 Packt
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | # Learning Spark SQL
5 |
6 | This is the code repository for [Learning Spark SQL](https://www.packtpub.com/big-data-and-business-intelligence/learning-spark-sql?utm_source=github&utm_medium=repository&utm_campaign=9781785888359), published by [Packt](https://www.packtpub.com/?utm_source=github). It contains all the supporting project files necessary to work through the book from start to finish.
7 |
8 | ## About the Book
9 |
10 | In the past year, Apache Spark has been increasingly adopted for the development of distributed applications. Spark SQL APIs provide an optimized interface that helps developers build such applications quickly and easily. However, designing web-scale production applications using Spark SQL APIs can be a complex task. Hence, understanding the design and implementation best practices before you start your project will help you avoid these problems.
11 |
12 | This book gives an insight into the engineering practices used to design and build real-world, Spark-based applications. The book's hands-on examples will give you the required confidence to work on any future projects you encounter in Spark SQL.
13 |
14 | It starts by familiarizing you with data exploration and data munging tasks using Spark SQL and Scala. Extensive code examples will help you understand the methods used to implement typical use-cases for various types of applications. You will get a walkthrough of the key concepts and terms that are common to streaming, machine learning, and graph applications. You will also learn how such systems are architected and deployed for a successful delivery of your project. Finally, you will move on to performance tuning, where you will learn practical tips and tricks to resolve performance issues.
15 |
16 | ## Instructions and Navigation
17 |
18 | All of the code is organized into folders. Each folder starts with a number followed by the application name. For example, Chapter02.
19 |
20 |
21 |
22 | The code will look like the following:
23 | ```
24 | scala> val inDiaDataDF = spark.read.option("header", true).csv("file:///Users/aurobindosarkar/Downloads/dataset_diabetes/diabetic_data.csv").cache()
25 | ```
26 |
27 |
28 |
29 | ## Related Products
30 |
31 | * [Getting Started with NoSQL](https://www.packtpub.com/big-data-and-business-intelligence/getting-started-nosql?utm_source=github&utm_medium=repository&utm_campaign=9781849694988)
32 |
33 | * [Learning PySpark](https://www.packtpub.com/big-data-and-business-intelligence/learning-pyspark?utm_source=github&utm_medium=repository&utm_campaign=9781786463708)
34 |
35 | * [PostgreSQL Development Essentials](https://www.packtpub.com/big-data-and-business-intelligence/postgresql-development-essentials?utm_source=github&utm_medium=repository&utm_campaign=9781783989003)
36 |
37 |
38 | ### Download a free PDF
39 |
40 | If you have already purchased a print or Kindle version of this book, you can get a DRM-free PDF version at no cost.
Simply click on the link to claim your free PDF.
41 | https://packt.link/free-ebook/9781785888359
--------------------------------------------------------------------------------