├── example_data
    └── testData.txt
├── README.md
├── src
    └── main
    │   └── scala
    │       └── cloudera
    │           └── fun
    │               └── sparklambda
    │                   ├── common
    │                       └── ErrorCount.scala
    │                   ├── etl
    │                       └── BatchErrorCount.scala
    │                   └── streaming
    │                       └── StreamingErrorCount.scala
└── pom.xml


/example_data/testData.txt:
--------------------------------------------------------------------------------
 1 | 14/08/20 21:40:52 INFO BlockFetcherIterator$BasicBlockFetcherIterator: Getting 1 non-empty blocks out of 1 blocks
 2 | 14/08/20 21:40:52 INFO BlockFetcherIterator$BasicBlockFetcherIterator: Started 0 remote fetches in 3 ms
 3 | 14/08/20 21:40:52 INFO FileOutputCommitter: Saved output of task 'attempt_201408202140_0000_m_000000_1' to file:/Users/gshapira/resData2/_temporary/0/task_201408202140_0000_m_000000
 4 | 14/08/20 21:40:52 INFO SparkHadoopWriter: attempt_201408202140_0000_m_000000_1: Committed
 5 | 14/08/20 21:40:52 INFO Executor: Serialized size of result for 1 is 825
 6 | 14/08/20 21:40:52 ERROR Executor: Sending result for 1 directly to driver
 7 | 14/08/20 21:40:52 INFO Executor: Finished task ID 1
 8 | 14/08/20 21:40:52 INFO DAGScheduler: Completed ResultTask(0, 0)
 9 | 14/08/20 21:40:52 INFO TaskSetManager: Finished TID 1 in 98 ms on localhost (progress: 1/1)
10 | 14/08/20 21:40:52 ERROR TaskSchedulerImpl: Removed TaskSet 0.0, whose tasks have all completed, from pool
11 | 14/08/20 21:40:52 INFO DAGScheduler: Stage 0 (saveAsTextFile at BatchErrorCount.scala:41) finished in 0.099 s
12 | 14/08/20 21:40:52 ERROR SparkContext: Job finished: saveAsTextFile at BatchErrorCount.scala:41, took 0.345168 s
13 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | # Spark Lambda Architecture
 2 | ## Overview
 3 | This project is intended to show an example of how Spark can be used to efficiently implement a Lambda Architecture
 4 | 
 5 | Lambda Architectures typically share some of the business logic between the batch layer and speed layer.
 6 | When each layer is implemented in a different language or framework, this leads to code duplication, painful
 7 | maintenance and errors.
 8 | 
 9 | On the other hand, if Spark is used to implement the batch layer and SparkStreaming for the speed layer, they can share
10 | common functions, reducing code duplication and the associated maintenance overhead.
11 | 
12 | This project is intended as an example of how this can be done.
13 | It contains two packages for counting errors in logs. One for batch use and the other for streaming.
14 | 
15 | ##Build
16 | mvn clean package
17 | 
18 | ##Usage
19 | 
20 | ###ETL example:
21 | 
22 | java -cp SparkStreamingLambda-1.0-SNAPSHOT.jar:<spark dir>/lib/spark-assembly-1.0.2-hadoop2.2.0.jar cloudera.fun.sparklambda.etl.BatchErrorCount <master> <input> <output>
23 | 
24 | ###Streaming example:
25 | 
26 | java -cp SparkStreamingLambda-1.0-SNAPSHOT.jar:<spark dir>/lib/spark-assembly-1.0.2-hadoop2.2.0.jar cloudera.fun.sparklambda.streaming.StreamingErrorCount <master> localhost <port>
27 | 
28 | to send data to the streaming example, use:
29 | nc -lk <port>
30 | 
31 | 


--------------------------------------------------------------------------------
/src/main/scala/cloudera/fun/sparklambda/common/ErrorCount.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  *
 9 |  *      http://www.apache.org/licenses/LICENSE-2.0
10 |  *
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 |  */
17 | 
18 | package cloudera.fun.sparklambda.common
19 | 
20 | import org.apache.spark.rdd.RDD
21 | import org.apache.spark.SparkContext._
22 | 
23 | 
24 | object ErrorCount {
25 |   def countErrors(rdd: RDD[String]): RDD[(String, Int)] = {
26 |     rdd
27 |       .filter(_.contains("ERROR")) // Keep "ERROR" lines
28 |       .map( s => (s.split(" ")(0), 1) ) // Return tuple with date & count
29 |       .reduceByKey(_+_) // Sum counts for each date
30 |   }
31 | }
32 | 


--------------------------------------------------------------------------------
/src/main/scala/cloudera/fun/sparklambda/etl/BatchErrorCount.scala:
--------------------------------------------------------------------------------
 1 | /*
 2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
 3 |  * contributor license agreements.  See the NOTICE file distributed with
 4 |  * this work for additional information regarding copyright ownership.
 5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
 6 |  * (the "License"); you may not use this file except in compliance with
 7 |  * the License.  You may obtain a copy of the License at
 8 |  *
 9 |  *      http://www.apache.org/licenses/LICENSE-2.0
10 |  *
11 |  * Unless required by applicable law or agreed to in writing, software
12 |  * distributed under the License is distributed on an "AS IS" BASIS,
13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 |  * See the License for the specific language governing permissions and
15 |  * limitations under the License.
16 |  */
17 | 
18 | package cloudera.fun.sparklambda.etl
19 | 
20 | import cloudera.fun.sparklambda.common.ErrorCount
21 | import org.apache.spark.{SparkConf, SparkContext}
22 | 
23 | 
24 | object BatchErrorCount {
25 |   def main(args: Array[String]): Unit = {
26 |     if (args.length<3) {
27 |       System.err.println("Usage: BatchErrorCount <master> <inputfile> <outputfile>")
28 |       System.exit(1)
29 |     }
30 |     val conf = new SparkConf()
31 |       .setMaster(args(0))
32 |       .setAppName(this.getClass.getCanonicalName)
33 |       .setJars(Seq(SparkContext.jarOfClass(this.getClass).get))
34 | 
35 |     val sc = new SparkContext(conf)
36 | 
37 |     val lines = sc.textFile(args(1))
38 | 
39 |     val errCount = ErrorCount.countErrors(lines)
40 | 
41 |     errCount.saveAsTextFile(args(2))
42 |   }
43 | }
44 | 


--------------------------------------------------------------------------------
/src/main/scala/cloudera/fun/sparklambda/streaming/StreamingErrorCount.scala:
--------------------------------------------------------------------------------
  1 | /*
  2 |  * Licensed to the Apache Software Foundation (ASF) under one or more
  3 |  * contributor license agreements.  See the NOTICE file distributed with
  4 |  * this work for additional information regarding copyright ownership.
  5 |  * The ASF licenses this file to You under the Apache License, Version 2.0
  6 |  * (the "License"); you may not use this file except in compliance with
  7 |  * the License.  You may obtain a copy of the License at
  8 |  *
  9 |  *      http://www.apache.org/licenses/LICENSE-2.0
 10 |  *
 11 |  * Unless required by applicable law or agreed to in writing, software
 12 |  * distributed under the License is distributed on an "AS IS" BASIS,
 13 |  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 14 |  * See the License for the specific language governing permissions and
 15 |  * limitations under the License.
 16 |  */
 17 | 
 18 | package cloudera.fun.sparklambda.streaming
 19 | 
 20 | import cloudera.fun.sparklambda.common.ErrorCount
 21 | import org.apache.spark.rdd.RDD
 22 | import org.apache.spark.{SparkConf, Logging}
 23 | import org.apache.spark.storage.StorageLevel
 24 | import org.apache.spark.streaming.{Seconds, StreamingContext}
 25 | import org.apache.spark.streaming.StreamingContext._
 26 | import org.apache.log4j.{Level, Logger}
 27 | 
 28 | 
 29 | /**
 30 |  * Created by gshapira on 8/20/14.
 31 |  */
 32 | object StreamingErrorCount extends Logging{
 33 | 
 34 |   def main(args: Array[String]): Unit = {
 35 | 
 36 | 
 37 |    if (args.length < 3) {
 38 |       System.err.println("Usage: StreamingErrorCount <master> <hostname> <port>")
 39 |       System.exit(1)
 40 |     }
 41 | 
 42 |     //Configure the Streaming Context
 43 | 
 44 |     val sparkConf = new SparkConf()
 45 |       .setMaster(args(0))
 46 |       .setAppName(this.getClass.getCanonicalName)
 47 | 
 48 |     setStreamingLogLevels()
 49 | 
 50 | 
 51 |     val ssc = new StreamingContext(sparkConf, Seconds(10))
 52 |     ssc.checkpoint(".")
 53 | 
 54 |     // Create the DStream from data sent over the network
 55 |     val dStream = ssc.socketTextStream(args(1), args(2).toInt, StorageLevel.MEMORY_AND_DISK_SER)
 56 | 
 57 |     // Counting the errors in each RDD in the stream
 58 |     val errCountStream = dStream.transform(rdd => ErrorCount.countErrors(rdd))
 59 | 
 60 | 
 61 |     // printing out the current error count
 62 |     errCountStream.foreachRDD(rdd => {
 63 |       System.out.println("Errors this minute:%d".format(rdd.first()._2))
 64 |     })
 65 | 
 66 |     // creating a stream with running error count
 67 |     val stateStream = errCountStream.updateStateByKey[Int](updateFunc)
 68 | 
 69 |     // printing the running error count
 70 |     stateStream.foreachRDD(rdd => {
 71 |       System.out.println("Errors today:%d".format(rdd.first()._2))
 72 |     })
 73 | 
 74 |     // starting the action
 75 |     ssc.start()
 76 |     ssc.awaitTermination()
 77 |   }
 78 | 
 79 | 
 80 |   def setStreamingLogLevels() {
 81 |     val log4jInitialized = Logger.getRootLogger.getAllAppenders.hasMoreElements
 82 |     if (!log4jInitialized) {
 83 |       // We first log something to initialize Spark's default logging, then we override the
 84 |       // logging level.
 85 |       logInfo("Setting log level to [WARN] for streaming example." +
 86 |         " To override add a custom log4j.properties to the classpath.")
 87 |       Logger.getRootLogger.setLevel(Level.WARN)
 88 |     }
 89 |   }
 90 | 
 91 |   val updateFunc = (values: Seq[Int], state: Option[Int]) => {
 92 |     val currentCount = values.foldLeft(0)(_ + _)
 93 | 
 94 |     val previousCount = state.getOrElse(0)
 95 | 
 96 |     Some(currentCount + previousCount)
 97 |   }
 98 | 
 99 | }
100 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <?xml version="1.0" encoding="UTF-8"?>
  2 | <!--
  3 |   Licensed to the Apache Software Foundation (ASF) under one or more
  4 |   contributor license agreements.  See the NOTICE file distributed with
  5 |   this work for additional information regarding copyright ownership.
  6 |   The ASF licenses this file to You under the Apache License, Version 2.0
  7 |   (the "License"); you may not use this file except in compliance with
  8 |   the License.  You may obtain a copy of the License at
  9 | 
 10 |        http://www.apache.org/licenses/LICENSE-2.0
 11 | 
 12 |   Unless required by applicable law or agreed to in writing, software
 13 |   distributed under the License is distributed on an "AS IS" BASIS,
 14 |   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 15 |   See the License for the specific language governing permissions and
 16 |   limitations under the License.
 17 |   -->
 18 | 
 19 | <project xmlns="http://maven.apache.org/POM/4.0.0"
 20 |          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
 21 |          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
 22 |   <modelVersion>4.0.0</modelVersion>
 23 | 
 24 |   <groupId>demo</groupId>
 25 |   <artifactId>SparkStreamingLambda</artifactId>
 26 |   <version>1.0-SNAPSHOT</version>
 27 | 
 28 |   <repositories>
 29 | 
 30 |     <repository>
 31 |       <id>scala-tools.org</id>
 32 |       <name>Scala-tools Maven2 Repository</name>
 33 |       <url>http://scala-tools.org/repo-releases</url>
 34 |     </repository>
 35 | 
 36 |   <repository>
 37 |     <id>cloudera-repos</id>
 38 |     <name>Cloudera Repos</name>
 39 |     <url>https://repository.cloudera.com/artifactory/cloudera-repos/</url>
 40 |   </repository>
 41 | 
 42 |   </repositories>
 43 | 
 44 |   <dependencies>
 45 |     <dependency>
 46 |       <groupId>org.apache.spark</groupId>
 47 |       <artifactId>spark-core_2.10</artifactId>
 48 |       <version>1.0.0-cdh5.1.0</version>
 49 |       <scope>provided</scope>
 50 |     </dependency>
 51 | 
 52 |     <dependency>
 53 |       <groupId>org.apache.spark</groupId>
 54 |       <artifactId>spark-streaming_2.10</artifactId>
 55 |       <version>1.0.0-cdh5.1.0</version>
 56 |       <scope>provided</scope>
 57 |     </dependency>
 58 | 
 59 |   </dependencies>
 60 | 
 61 |   <pluginRepositories>
 62 |     <pluginRepository>
 63 |       <id>scala-tools.org</id>
 64 |       <name>Scala-tools Maven2 Repository</name>
 65 |       <url>http://scala-tools.org/repo-releases</url>
 66 |     </pluginRepository>
 67 |   </pluginRepositories>
 68 | 
 69 |   <build>
 70 |     <sourceDirectory>src/main/scala</sourceDirectory>
 71 |     <testSourceDirectory>src/test/scala</testSourceDirectory>
 72 |     <plugins>
 73 |       <plugin>
 74 |         <groupId>org.scala-tools</groupId>
 75 |         <artifactId>maven-scala-plugin</artifactId>
 76 |         <version>2.15.2</version>
 77 |         <executions>
 78 |           <execution>
 79 |             <goals>
 80 |               <goal>compile</goal>
 81 |               <goal>testCompile</goal>
 82 |             </goals>
 83 |           </execution>
 84 |         </executions>
 85 |         <configuration>
 86 |           <scalaVersion>2.10</scalaVersion>
 87 |         </configuration>
 88 |       </plugin>
 89 |       <plugin>
 90 |         <artifactId>maven-assembly-plugin</artifactId>
 91 |         <version>2.3</version>
 92 |         <configuration>
 93 |           <descriptorRefs>
 94 |             <descriptorRef>jar-with-dependencies</descriptorRef>
 95 |           </descriptorRefs>
 96 |         </configuration>
 97 |         <executions>
 98 |           <execution>
 99 |             <id>make-assembly</id> <!-- this is used for inheritance merges -->
100 |             <phase>package</phase> <!-- bind to the packaging phase -->
101 |             <goals>
102 |               <goal>single</goal>
103 |             </goals>
104 |           </execution>
105 |         </executions>
106 |       </plugin>
107 |     </plugins>
108 |   </build>
109 | 
110 | </project>


--------------------------------------------------------------------------------