├── .gitignore
├── README.md
├── src
    └── main
    │   └── scala
    │       └── com
    │           └── examples
    │               └── MainExample.scala
└── pom.xml


/.gitignore:
--------------------------------------------------------------------------------
1 | .cache
2 | target/
3 | target/*
4 | ./.settings/
5 | ./.settings/*
6 | 
7 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ### Instructions:
 2 | 
 3 | [Follow this article to find more detailed instructions.](https://nosqlnocry.wordpress.com/2015/02/27/how-to-build-a-spark-fat-jar-in-scala-and-submit-a-job/)
 4 | 
 5 | Modify the class "MainExample.scala" writing your Spark code, then compile the project with the command:
 6 | 
 7 | ```mvn clean package```
 8 | 
 9 | Inside the ```/target``` folder you will find the result fat jar called ```spark-scala-maven-project-0.0.1-SNAPSHOT-jar-with-depencencies.jar```. In order to launch the Spark job use this command in a shell with a configured Spark environment:
10 | 
11 |     spark-submit --class com.examples.MainExample \
12 |       --master yarn-cluster \
13 |       spark-scala-maven-project-0.0.1-SNAPSHOT-jar-with-depencencies.jar \
14 |       inputhdfspath \
15 |       outputhdfspath
16 | 
17 | The parameters ```inputhdfspath``` and ```outputhdfspath``` don't have to present the form ```hdfs://path/to/your/file``` but directly ```/path/to/your/files/``` because submitting a job the default file system is HDFS. To retrieve the result locally:
18 | 
19 |     hadoop fs -getmerge outputhdfspath resultSavedLocally
20 | 


--------------------------------------------------------------------------------
/src/main/scala/com/examples/MainExample.scala:
--------------------------------------------------------------------------------
 1 | package com.examples
 2 | 
 3 | import org.apache.spark.SparkContext
 4 | import org.apache.spark.SparkContext._
 5 | import org.apache.spark.SparkConf
 6 | import org.apache.log4j.Logger
 7 | 
 8 | object MainExample {
 9 | 
10 |   def main(arg: Array[String]) {
11 | 
12 |     var logger = Logger.getLogger(this.getClass())
13 | 
14 |     if (arg.length < 2) {
15 |       logger.error("=> wrong parameters number")
16 |       System.err.println("Usage: MainExample <path-to-files> <output-path>")
17 |       System.exit(1)
18 |     }
19 | 
20 |     val jobName = "MainExample"
21 | 
22 |     val conf = new SparkConf().setAppName(jobName)
23 |     val sc = new SparkContext(conf)
24 | 
25 |     val pathToFiles = arg(0)
26 |     val outputPath = arg(1)
27 | 
28 |     logger.info("=> jobName \"" + jobName + "\"")
29 |     logger.info("=> pathToFiles \"" + pathToFiles + "\"")
30 | 
31 |     val files = sc.textFile(pathToFiles)
32 | 
33 |     // do your work here
34 |     val rowsWithoutSpaces = files.map(_.replaceAll(" ", ","))
35 | 
36 |     // and save the result
37 |     rowsWithoutSpaces.saveAsTextFile(outputPath)
38 | 
39 |   }
40 | }
41 | 


--------------------------------------------------------------------------------
/pom.xml:
--------------------------------------------------------------------------------
  1 | <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2 | 	xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
  3 | 	<modelVersion>4.0.0</modelVersion>
  4 | 	<groupId>spark-scala-maven-project</groupId>
  5 | 	<artifactId>spark-scala-maven-project</artifactId>
  6 | 	<version>0.0.1-SNAPSHOT</version>
  7 | 	<name>${project.artifactId}</name>
  8 | 	<description>This is a boilerplate maven project to start using Spark in Scala</description>
  9 | 	<inceptionYear>2010</inceptionYear>
 10 | 
 11 | 	<properties>
 12 | 		<maven.compiler.source>1.6</maven.compiler.source>
 13 | 		<maven.compiler.target>1.6</maven.compiler.target>
 14 | 		<encoding>UTF-8</encoding>
 15 | 		<scala.tools.version>2.10</scala.tools.version>
 16 | 		<!-- Put the Scala version of the cluster -->
 17 | 		<scala.version>2.10.4</scala.version>
 18 | 	</properties>
 19 | 	
 20 | 	<!-- repository to add org.apache.spark -->
 21 | 	<repositories>
 22 | 		<repository>
 23 | 			<id>cloudera-repo-releases</id>
 24 | 			<url>https://repository.cloudera.com/artifactory/repo/</url>
 25 | 		</repository>
 26 | 	</repositories>
 27 | 
 28 | 	<build>
 29 | 		<sourceDirectory>src/main/scala</sourceDirectory>
 30 | 		<testSourceDirectory>src/test/scala</testSourceDirectory>
 31 | 		<plugins>
 32 | 			<plugin>
 33 | 				<!-- see http://davidb.github.com/scala-maven-plugin -->
 34 | 				<groupId>net.alchim31.maven</groupId>
 35 | 				<artifactId>scala-maven-plugin</artifactId>
 36 | 				<version>3.1.3</version>
 37 | 				<executions>
 38 | 					<execution>
 39 | 						<goals>
 40 | 							<goal>compile</goal>
 41 | 							<goal>testCompile</goal>
 42 | 						</goals>
 43 | 						<configuration>
 44 | 							<args>
 45 | 								<arg>-make:transitive</arg>
 46 | 								<arg>-dependencyfile</arg>
 47 | 								<arg>${project.build.directory}/.scala_dependencies</arg>
 48 | 							</args>
 49 | 						</configuration>
 50 | 					</execution>
 51 | 				</executions>
 52 | 			</plugin>
 53 | 			<plugin>
 54 | 				<groupId>org.apache.maven.plugins</groupId>
 55 | 				<artifactId>maven-surefire-plugin</artifactId>
 56 | 				<version>2.13</version>
 57 | 				<configuration>
 58 | 					<useFile>false</useFile>
 59 | 					<disableXmlReport>true</disableXmlReport>
 60 | 					<!-- If you have classpath issue like NoDefClassError,... -->
 61 | 					<!-- useManifestOnlyJar>false</useManifestOnlyJar -->
 62 | 					<includes>
 63 | 						<include>**/*Test.*</include>
 64 | 						<include>**/*Suite.*</include>
 65 | 					</includes>
 66 | 				</configuration>
 67 | 			</plugin>
 68 | 
 69 | 			<!-- "package" command plugin -->
 70 | 			<plugin>
 71 | 				<artifactId>maven-assembly-plugin</artifactId>
 72 | 				<version>2.4.1</version>
 73 | 				<configuration>
 74 | 					<descriptorRefs>
 75 | 						<descriptorRef>jar-with-dependencies</descriptorRef>
 76 | 					</descriptorRefs>
 77 | 				</configuration>
 78 | 				<executions>
 79 | 					<execution>
 80 | 						<id>make-assembly</id>
 81 | 						<phase>package</phase>
 82 | 						<goals>
 83 | 							<goal>single</goal>
 84 | 						</goals>
 85 | 					</execution>
 86 | 				</executions>
 87 | 			</plugin>
 88 | 		</plugins>
 89 | 	</build>
 90 | 
 91 | 	<dependencies>
 92 | 		<!-- Scala and Spark dependencies -->
 93 | 		<dependency>
 94 | 			<groupId>org.scala-lang</groupId>
 95 | 			<artifactId>scala-library</artifactId>
 96 | 			<version>${scala.version}</version>
 97 | 		</dependency>
 98 | 		<dependency>
 99 | 			<groupId>org.apache.spark</groupId>
100 | 			<artifactId>spark-core_2.10</artifactId>
101 | 			<version>1.2.0-cdh5.3.1</version>
102 | 		</dependency>
103 | 
104 | 	</dependencies>
105 | </project>
106 | 
107 | 


--------------------------------------------------------------------------------