├── .gitignore ├── CHANGES.md ├── LICENSE.md ├── README.md ├── archetype-catalog.xml ├── org └── sia │ └── spark-archetype-scala │ ├── 0.9 │ ├── _remote.repositories │ ├── spark-archetype-scala-0.9.jar │ └── spark-archetype-scala-0.9.pom │ └── maven-metadata-local.xml ├── pom.xml └── src └── main └── resources ├── META-INF └── maven │ ├── archetype-metadata.xml │ └── archetype.xml └── archetype-resources ├── .gitignore ├── pom.xml └── src ├── main └── scala │ └── App.scala └── test └── scala └── samples ├── junit.scala ├── scalatest.scala └── specs.scala /.gitignore: -------------------------------------------------------------------------------- 1 | # use glob syntax. 2 | syntax: glob 3 | *.ser 4 | *.class 5 | *~ 6 | *.bak 7 | *.off 8 | *.old 9 | 10 | # eclipse conf file 11 | .settings 12 | .classpath 13 | .project 14 | .manager 15 | 16 | # idea conf 17 | .idea/ 18 | *.iml 19 | 20 | # building 21 | target 22 | build 23 | null 24 | tmp* 25 | temp* 26 | dist 27 | test-output 28 | 29 | # other scm 30 | .svn 31 | .CVS 32 | .hg* 33 | 34 | # switch to regexp syntax. 35 | # syntax: regexp 36 | # ^\.pc/ 37 | 38 | 39 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | 2 | ### Changes: 3 | 4 | ### 0.8 5 | * upgrade to Spark-1.3.1 6 | * active version only the latest from now on 7 | 8 | ### 0.7 9 | * add spark.version as a parameter in pom 10 | * improve README (add IntelliJ instructions) 11 | 12 | ### 0.6 13 | * fix problem with IntelliJ (uses the same param for arch. catalog and repository) 14 | 15 | ### 0.5 16 | * test multiple versions of archetype 17 | 18 | ### 0.4 19 | * add back unit, integration and BDD-style test examples 20 | 21 | ### 0.3 22 | * improve README with more detailed information (step 6) 23 | * improve README formatting and style 24 | 25 | ### 0.2 26 | * improve README with more info 27 | * add book pitch to README :) 28 | 29 | ### 0.1 30 | * adapt README instructions to Spark 31 | * change java to 1.7 32 | * make all other changes in pom to reflect SiA requirements 33 | 34 | ---- 35 | 36 | ### Changes in the original repo (davidB's `scala-archetype-simple`): 37 | ### 1.5 38 | 39 | * upgrade of scala 2.10.0 40 | * upgrade version of Specs(2), ScalaTest, Surefire, scala-maven-plugin 41 | 42 | ### 1.4 43 | 44 | * move to sonatype for hosting 45 | * change groupId to net.alchim31.maven 46 | * upgrade version of scala to 2.9.2, version of JUnit, Specs(2) and ScalaTest 47 | 48 | ### 1.3 49 | 50 | * upgrade to scala 2.8.0 (as default) 51 | * upgrade version of Specs and ScalaTest 52 | * provide sample of Specs and ScalaTest runnable from maven and eclipse (at least) 53 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | [License: MIT](http://mbo.mit-license.org) 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ### Maven archetype to scaffold a new Spark Scala project 2 | 3 | ### For the latest version please go to the [book's repo](https://github.com/spark-in-action/scala-archetype-sparkinaction). 4 | This repo was just use while the archetype was in development, but all subsequent changes and bug fixes went to the official book's repo. 5 | 6 | Not up to date: 7 | ### To generate a new project 8 | Since **IntelliJ IDEA** doesn't distinguish between _archetype catalog_ and _archetype repository_ when you try to add remote archetype, you should first use interactive or batch mode (see below) in terminal to generate a new project then import it in Idea as existing maven project. 9 | 10 | * In **Eclipse** (requires [ScalaIDE](http://scala-ide.org/download/current.html) and [m2e-scala](https://github.com/sonatype/m2eclipse-scala)) 11 | 1. `File > New > Project... > Maven > Maven Project` 12 | 2. Click `Next` on the first screen of the _New project_ wizard 13 | 3. Select `Configure... > Add Remote Catalog...` 14 | 4. Enter the following URL in the `Catalog file` field: https://github.com/mbonaci/spark-archetype-scala/raw/master/archetype-catalog.xml 15 | Enter `Spark Scala Archetype` in the `Description` field 16 | 5. After you close the dialog, choose the `Spark Scala Archetype` catalog in the `Catalog` dropdown list 17 | 6. In the next dialog enter you project details and confirm with `Finish` 18 | 7. Once the new projects generates change the Scala version to 2.10.5 by right-clicking on the generated project's root and selecting: 19 | `Scala > Set the Scala Installation > Fixed Scala Installation 2.10.5.(bundled)` 20 | 21 | 22 | 23 | * **From the terminal** (interactive mode): 24 | 25 | Select the only possible option (1) and answer subsequent questions. 26 | 27 | 28 | ```sh 29 | mvn archetype:generate \ 30 | -DarchetypeCatalog=https://github.com/mbonaci/spark-archetype-scala/raw/master/archetype-catalog.xml \ 31 | -DarchetypeRepository=https://github.com/mbonaci/spark-archetype-scala/raw/master 32 | ``` 33 | 34 | 35 | * **From the terminal or a shell script** (batch mode): 36 | 37 | Don't forget to change the parameter values in the last line. 38 | 39 | 40 | ```sh 41 | mvn archetype:generate -B \ 42 | -DarchetypeCatalog=https://github.com/mbonaci/spark-archetype-scala/raw/master/archetype-catalog.xml \ 43 | -DarchetypeRepository=https://github.com/mbonaci/spark-archetype-scala/raw/master \ 44 | -DarchetypeGroupId=org.sia \ 45 | -DarchetypeArtifactId=spark-archetype-scala \ 46 | -DarchetypeVersion=0.9 \ 47 | -DgroupId=com.company -DartifactId=project -Dversion=0.1-SNAPSHOT -Dpackage=com.company 48 | ``` 49 | For your convenience, here's the copy/paste friendly version (customize your new project by changing the last 4 params): 50 | 51 | ``` 52 | mvn archetype:generate -B -DarchetypeCatalog=https://github.com/mbonaci/spark-archetype-scala/raw/master/archetype-catalog.xml -DarchetypeRepository=https://github.com/mbonaci/spark-archetype-scala/raw/master -DarchetypeGroupId=org.sia -DarchetypeArtifactId=spark-archetype-scala -DarchetypeVersion=0.9 -DgroupId=org.sijaset -DartifactId=sija -Dversion=0.1 -Dpackage=org.sijaset 53 | ``` 54 | 55 | * ***Generated project example usage*** *(run* `mvn scala:help` *for the full list of commands):* 56 | 57 | You can run the generated project by simply doing `Shift+Alt+x s` (while positioned in `App.scala`), which is the shortcut for `Run As > Scala Application`. 58 | 59 | You can also run it from the command line or from Eclipse Maven _Run configuration_ with these _goals_: 60 | 61 | ```sh 62 | mvn scala:compile 63 | mvn scala:run -DmainClass=com.company.App 64 | ``` 65 | 66 | Have fun :) 67 | -------------------------------------------------------------------------------- /archetype-catalog.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | org.sia 8 | spark-archetype-scala 9 | 0.9 10 | Maven archetype used to bootstrap a new Spark/Scala project. 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /org/sia/spark-archetype-scala/0.9/_remote.repositories: -------------------------------------------------------------------------------- 1 | #NOTE: This is an Aether internal implementation file, its format can be changed without prior notice. 2 | #Sat Apr 25 01:14:39 CEST 2015 3 | spark-archetype-scala-0.9.jar>= 4 | spark-archetype-scala-0.9.pom>= 5 | -------------------------------------------------------------------------------- /org/sia/spark-archetype-scala/0.9/spark-archetype-scala-0.9.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mbonaci/spark-archetype-scala/4f2acce34266de14fcfc31a8ba2f45cc2a8f53b3/org/sia/spark-archetype-scala/0.9/spark-archetype-scala-0.9.jar -------------------------------------------------------------------------------- /org/sia/spark-archetype-scala/0.9/spark-archetype-scala-0.9.pom: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | org.sonatype.oss 6 | oss-parent 7 | 7 8 | 9 | 10 | org.sia 11 | spark-archetype-scala 12 | 0.9 13 | maven-archetype 14 | 15 | Scala Spark Maven archetype 16 | http://github.com/mbonaci/${project.artifactId} 17 | 2015 18 | Archetype used to scaffold a new Spark Scala project. 19 | 20 | 21 | 22 | MIT License 23 | http://www.opensource.org/licenses/mit-license.php 24 | 25 | 26 | 27 | 28 | scm:git:git://github.com/mbonaci/${project.artifactId}.git 29 | scm:git:git@github.com:mbonaci/${project.artifactId}.git 30 | http://github.com/mbonaci/${project.artifactId}/ 31 | 32 | 33 | 34 | github 35 | http://github.com/mbonaci/${project.artifactId}/issues#issue/ 36 | 37 | 38 | 39 | 40 | Spark in Action Manning forum 41 | https://forums.manning.com/forums/spark-in-action 42 | https://forums.manning.com/forums/spark-in-action 43 | 44 | 45 | 46 | 47 | 48 | Marko Bonaci 49 | Europe/Zagreb 50 | 51 | 52 | 53 | 54 | true 55 | 3.0 56 | 1.7 57 | 1.7 58 | UTF-8 59 | ${encoding} 60 | 61 | 62 | 63 | ${maven.version} 64 | 65 | 66 | 67 | 68 | 69 | org.apache.maven.archetype 70 | archetype-packaging 71 | 2.3 72 | 73 | 74 | 75 | 76 | 77 | 78 | maven-archetype-plugin 79 | 2.3 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /org/sia/spark-archetype-scala/maven-metadata-local.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | org.sia 4 | spark-archetype-scala 5 | 6 | 0.9 7 | 8 | 0.9 9 | 10 | 20140329153455 11 | 12 | 13 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | org.sonatype.oss 6 | oss-parent 7 | 7 8 | 9 | 10 | org.sia 11 | spark-archetype-scala 12 | 0.9 13 | maven-archetype 14 | 15 | Scala Spark Maven archetype 16 | http://github.com/mbonaci/${project.artifactId} 17 | 2015 18 | Archetype used to scaffold a new Spark Scala project. 19 | 20 | 21 | 22 | MIT License 23 | http://www.opensource.org/licenses/mit-license.php 24 | 25 | 26 | 27 | 28 | scm:git:git://github.com/mbonaci/${project.artifactId}.git 29 | scm:git:git@github.com:mbonaci/${project.artifactId}.git 30 | http://github.com/mbonaci/${project.artifactId}/ 31 | 32 | 33 | 34 | github 35 | http://github.com/mbonaci/${project.artifactId}/issues#issue/ 36 | 37 | 38 | 39 | 40 | Spark in Action Manning forum 41 | https://forums.manning.com/forums/spark-in-action 42 | https://forums.manning.com/forums/spark-in-action 43 | 44 | 45 | 46 | 47 | 48 | Marko Bonaci 49 | Europe/Zagreb 50 | 51 | 52 | 53 | 54 | true 55 | 3.0 56 | 1.7 57 | 1.7 58 | UTF-8 59 | ${encoding} 60 | 61 | 62 | 63 | ${maven.version} 64 | 65 | 66 | 67 | 68 | 69 | org.apache.maven.archetype 70 | archetype-packaging 71 | 2.3 72 | 73 | 74 | 75 | 76 | 77 | 78 | maven-archetype-plugin 79 | 2.3 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/maven/archetype-metadata.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | src/main/scala 9 | 10 | **/*.scala 11 | 12 | 13 | 14 | 15 | src/test/scala 16 | 17 | **/*.scala 18 | 19 | 20 | 21 | 22 | 23 | 24 | .gitignore 25 | 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/maven/archetype.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | spark-archetype-scala 4 | 5 | src/main/scala/App.scala 6 | 7 | 8 | src/test/scala/AppTest.scala 9 | src/test/scala/MySpec.scala 10 | src/test/scala/ScalaTestExamples.scala 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/main/resources/archetype-resources/.gitignore: -------------------------------------------------------------------------------- 1 | # use glob syntax. 2 | syntax: glob 3 | *.ser 4 | *.class 5 | *~ 6 | *.bak 7 | #*.off 8 | *.old 9 | 10 | # eclipse conf file 11 | .settings 12 | .classpath 13 | .project 14 | .manager 15 | .scala_dependencies 16 | 17 | # idea 18 | .idea 19 | *.iml 20 | 21 | # building 22 | target 23 | build 24 | null 25 | tmp* 26 | temp* 27 | dist 28 | test-output 29 | build.log 30 | 31 | # other scm 32 | .svn 33 | .CVS 34 | .hg* 35 | 36 | # switch to regexp syntax. 37 | # syntax: regexp 38 | # ^\.pc/ 39 | 40 | #SHITTY output not in target directory 41 | build.log 42 | -------------------------------------------------------------------------------- /src/main/resources/archetype-resources/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | ${groupId} 4 | ${artifactId} 5 | ${version} 6 | ${artifactId} 7 | 8 | 2015 9 | 10 | 11 | Choose License 12 | http://.... 13 | repo 14 | 15 | 16 | 25 | 26 | 27 | 1.7 28 | 1.7 29 | UTF-8 30 | 2.10 31 | 2.10.4 32 | 1.3.1 33 | 34 | 35 | 36 | 37 | org.scala-lang 38 | scala-library 39 | ${scala.version} 40 | 41 | 42 | org.apache.spark 43 | spark-core_${scala.tools.version} 44 | ${spark.version} 45 | provided 46 | 47 | 48 | org.apache.spark 49 | spark-sql_${scala.tools.version} 50 | ${spark.version} 51 | provided 52 | 53 | 54 | 55 | 56 | 57 | junit 58 | junit 59 | 4.11 60 | test 61 | 62 | 63 | org.specs2 64 | specs2_${scala.tools.version} 65 | 1.13 66 | test 67 | 68 | 69 | org.scalatest 70 | scalatest_${scala.tools.version} 71 | 2.0.M6-SNAP8 72 | test 73 | 74 | 75 | 76 | 77 | 78 | src/main/scala 79 | src/test/scala 80 | 81 | 82 | 83 | 84 | net.alchim31.maven 85 | scala-maven-plugin 86 | 3.2.0 87 | 88 | 89 | 90 | compile 91 | testCompile 92 | 93 | 94 | 95 | -dependencyfile 96 | ${project.build.directory}/.scala_dependencies 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | org.apache.maven.plugins 105 | maven-surefire-plugin 106 | 2.18.1 107 | 108 | false 109 | true 110 | 111 | 112 | 113 | **/*Test.* 114 | **/*Suite.* 115 | 116 | 117 | 118 | 119 | 120 | org.apache.maven.plugins 121 | maven-shade-plugin 122 | 2.3 123 | 124 | 125 | package 126 | 127 | shade 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | -------------------------------------------------------------------------------- /src/main/resources/archetype-resources/src/main/scala/App.scala: -------------------------------------------------------------------------------- 1 | package ${package} 2 | 3 | import org.apache.spark.SparkContext 4 | import org.apache.spark.SparkContext._ 5 | import org.apache.spark.SparkConf 6 | 7 | /** 8 | * @author ${user.name} 9 | */ 10 | object App { 11 | 12 | def main(args : Array[String]) { 13 | val conf = new SparkConf() 14 | .setAppName("The swankiest Spark app ever") 15 | .setMaster("local[2]") 16 | 17 | val sc = new SparkContext(conf) 18 | 19 | val col = sc.parallelize(0 to 100 by 5) 20 | val smp = col.sample(true, 4) 21 | val colCount = col.count 22 | val smpCount = smp.count 23 | 24 | println("orig count = " + colCount) 25 | println("sampled count = " + smpCount) 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/resources/archetype-resources/src/test/scala/samples/junit.scala: -------------------------------------------------------------------------------- 1 | package samples 2 | 3 | import org.junit._ 4 | import Assert._ 5 | 6 | @Test 7 | class AppTest { 8 | 9 | @Test 10 | def testOK() = assertTrue(true) 11 | 12 | // @Test 13 | // def testKO() = assertTrue(false) 14 | 15 | } 16 | 17 | 18 | -------------------------------------------------------------------------------- /src/main/resources/archetype-resources/src/test/scala/samples/scalatest.scala: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2001-2009 Artima, Inc. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package samples 17 | 18 | /* 19 | ScalaTest facilitates different styles of testing by providing traits you can mix 20 | together to get the behavior and syntax you prefer. A few examples are 21 | included here. For more information, visit: 22 | 23 | http://www.scalatest.org/ 24 | 25 | One way to use ScalaTest is to help make JUnit or TestNG tests more 26 | clear and concise. Here's an example: 27 | */ 28 | import scala.collection.mutable.Stack 29 | import org.scalatest.Assertions 30 | import org.junit.Test 31 | 32 | class StackSuite extends Assertions { 33 | 34 | @Test def stackShouldPopValuesIinLastInFirstOutOrder() { 35 | val stack = new Stack[Int] 36 | stack.push(1) 37 | stack.push(2) 38 | assert(stack.pop() === 2) 39 | assert(stack.pop() === 1) 40 | } 41 | 42 | @Test def stackShouldThrowNoSuchElementExceptionIfAnEmptyStackIsPopped() { 43 | val emptyStack = new Stack[String] 44 | intercept[NoSuchElementException] { 45 | emptyStack.pop() 46 | } 47 | } 48 | } 49 | 50 | /* 51 | Here's an example of a FunSuite with ShouldMatchers mixed in: 52 | */ 53 | import org.scalatest.FunSuite 54 | import org.scalatest.matchers.ShouldMatchers 55 | 56 | import org.junit.runner.RunWith 57 | import org.scalatest.junit.JUnitRunner 58 | @RunWith(classOf[JUnitRunner]) 59 | class ListSuite extends FunSuite with ShouldMatchers { 60 | 61 | test("An empty list should be empty") { 62 | List() should be ('empty) 63 | Nil should be ('empty) 64 | } 65 | 66 | test("A non-empty list should not be empty") { 67 | List(1, 2, 3) should not be ('empty) 68 | List("fee", "fie", "foe", "fum") should not be ('empty) 69 | } 70 | 71 | test("A list's length should equal the number of elements it contains") { 72 | List() should have length (0) 73 | List(1, 2) should have length (2) 74 | List("fee", "fie", "foe", "fum") should have length (4) 75 | } 76 | } 77 | 78 | /* 79 | ScalaTest also supports the behavior-driven development style, in which you 80 | combine tests with text that specifies the behavior being tested. Here's 81 | an example whose text output when run looks like: 82 | 83 | A Map 84 | - should only contain keys and values that were added to it 85 | - should report its size as the number of key/value pairs it contains 86 | */ 87 | import org.scalatest.FunSpec 88 | import scala.collection.mutable.Stack 89 | 90 | class ExampleSpec extends FunSpec { 91 | 92 | describe("A Stack") { 93 | 94 | it("should pop values in last-in-first-out order") { 95 | val stack = new Stack[Int] 96 | stack.push(1) 97 | stack.push(2) 98 | assert(stack.pop() === 2) 99 | assert(stack.pop() === 1) 100 | } 101 | 102 | it("should throw NoSuchElementException if an empty stack is popped") { 103 | val emptyStack = new Stack[Int] 104 | intercept[NoSuchElementException] { 105 | emptyStack.pop() 106 | } 107 | } 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/main/resources/archetype-resources/src/test/scala/samples/specs.scala: -------------------------------------------------------------------------------- 1 | package samples 2 | 3 | import org.junit.runner.RunWith 4 | import org.specs2.mutable._ 5 | import org.specs2.runner._ 6 | 7 | 8 | /** 9 | * Sample specification. 10 | * 11 | * This specification can be executed with: scala -cp ${package}.SpecsTest 12 | * Or using maven: mvn test 13 | * 14 | * For more information on how to write or run specifications, please visit: 15 | * http://etorreborre.github.com/specs2/guide/org.specs2.guide.Runners.html 16 | * 17 | */ 18 | @RunWith(classOf[JUnitRunner]) 19 | class MySpecTest extends Specification { 20 | "The 'Hello world' string" should { 21 | "contain 11 characters" in { 22 | "Hello world" must have size(11) 23 | } 24 | "start with 'Hello'" in { 25 | "Hello world" must startWith("Hello") 26 | } 27 | "end with 'world'" in { 28 | "Hello world" must endWith("world") 29 | } 30 | } 31 | } 32 | --------------------------------------------------------------------------------