├── project ├── build.properties ├── plugins.sbt └── ApplicationBuild.scala ├── .gitignore ├── src ├── main │ ├── resources │ │ ├── data │ │ │ ├── departments.txt │ │ │ ├── employees.txt │ │ │ └── words.txt │ │ ├── logback.xml │ │ └── log4j.properties │ └── scala │ │ └── example │ │ ├── Department.scala │ │ ├── Employee.scala │ │ ├── package.scala │ │ ├── DepartmentDao.scala │ │ ├── SparkExample.scala │ │ ├── SparkStreamingExample.scala │ │ ├── EmployeeDao.scala │ │ ├── WordCount.scala │ │ └── SparkSqlExample.scala └── test │ └── scala │ ├── org │ ├── apache │ │ └── spark │ │ │ └── ClockWrapper.scala │ └── mkuthan │ │ └── spark │ │ ├── SparkSqlSpec.scala │ │ ├── SparkSpec.scala │ │ └── SparkStreamingSpec.scala │ └── example │ ├── SparkExampleSpec.scala │ ├── SparkStreamingExampleSpec.scala │ └── SparkSqlExampleSpec.scala ├── .travis.yml ├── README.md ├── scalastyle-config.xml ├── LICENSE.md └── sbt /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.11 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | derby.log 4 | 5 | target/ 6 | project/project/ 7 | -------------------------------------------------------------------------------- /src/main/resources/data/departments.txt: -------------------------------------------------------------------------------- 1 | 14,IT,65000 2 | 37,Accounting,15000 3 | 59,Human Resources,240000 4 | 77,Research,55000 5 | -------------------------------------------------------------------------------- /src/main/resources/data/employees.txt: -------------------------------------------------------------------------------- 1 | 123234877,Michael,Rogers,14 2 | 152934485,Anand,Manikutty,14 3 | 222364883,Carol,Smith,37 4 | 326587417,Joe,Stevens,37 5 | 332154719,Mary-Anne,Foster,14 6 | 332569843,George,ODonnell,77 7 | 546523478,John,Doe,59 8 | 631231482,David,Smith,77 9 | 654873219,Zacary,Efron,59 10 | 745685214,Eric,Goldsmith,59 11 | 845657245,Elizabeth,Doe,14 12 | 845657246,Kumar,Swamy,14 -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | 3 | cache: 4 | directories: 5 | - $HOME/.ivy2/cache 6 | - $HOME/.sbt/boot/ 7 | 8 | language: scala 9 | 10 | scala: 11 | - 2.11.7 12 | 13 | jdk: 14 | - oraclejdk8 15 | 16 | script: 17 | - sbt ++$TRAVIS_SCALA_VERSION clean coverage test 18 | 19 | # Trick to avoid unnecessary cache updates 20 | - find $HOME/.sbt -name "*.lock" | xargs rm 21 | - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm 22 | 23 | after_success: 24 | - sbt coveralls 25 | -------------------------------------------------------------------------------- /src/main/scala/example/Department.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | case class Department(code: Int, name: String, budget: Int) 20 | -------------------------------------------------------------------------------- /src/main/scala/example/Employee.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | case class Employee(ssn: String, name: String, lastName: String, department: Int) 20 | -------------------------------------------------------------------------------- /src/main/scala/example/package.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package object example { 18 | 19 | implicit class StringUtils(val value: String) { 20 | def strip(stripChars: String): String = value.stripPrefix(stripChars).stripSuffix(stripChars) 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0") 18 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.3") 19 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.3") 20 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.1") 21 | -------------------------------------------------------------------------------- /src/test/scala/org/apache/spark/ClockWrapper.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package org.apache.spark.streaming 18 | 19 | import org.apache.spark.util.ManualClock 20 | 21 | /** Ugly hack to access Spark private ManualClock class. */ 22 | object ClockWrapper { 23 | def advance(ssc: StreamingContext, timeToAdd: Duration): Unit = { 24 | val manualClock = ssc.scheduler.clock.asInstanceOf[ManualClock] 25 | manualClock.advance(timeToAdd.milliseconds) 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/test/scala/org/mkuthan/spark/SparkSqlSpec.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package org.mkuthan.spark 18 | 19 | import org.apache.spark.sql.hive.HiveContext 20 | import org.scalatest.Suite 21 | 22 | trait SparkSqlSpec extends SparkSpec { 23 | this: Suite => 24 | 25 | private var _sqlc: HiveContext = _ 26 | 27 | def sqlc: HiveContext = _sqlc 28 | 29 | override def beforeAll(): Unit = { 30 | super.beforeAll() 31 | 32 | _sqlc = new HiveContext(sc) 33 | } 34 | 35 | override def afterAll(): Unit = { 36 | _sqlc = null 37 | 38 | super.afterAll() 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Set everything to be logged to the console 2 | log4j.rootCategory=INFO, console 3 | log4j.appender.console=org.apache.log4j.ConsoleAppender 4 | log4j.appender.console.target=System.err 5 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 6 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n 7 | 8 | # Set the default spark-shell log level to WARN. When running the spark-shell, the 9 | # log level for this class is used to overwrite the root logger's log level, so that 10 | # the user can have different defaults for the shell and regular Spark apps. 11 | log4j.logger.org.apache.spark.repl.Main=WARN 12 | 13 | # Settings to quiet third party logs that are too verbose 14 | log4j.logger.org.spark-project.jetty=WARN 15 | log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR 16 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO 17 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO 18 | log4j.logger.org.apache.parquet=ERROR 19 | log4j.logger.parquet=ERROR 20 | 21 | # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support 22 | log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL 23 | log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR 24 | 25 | # Example 26 | log4j.logger.example=DEBUG 27 | -------------------------------------------------------------------------------- /src/main/scala/example/DepartmentDao.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import org.apache.spark.rdd.RDD 20 | import org.apache.spark.sql.SQLContext 21 | 22 | /** Notice: Variable binding is vulnerable for SQL injection. */ 23 | class DepartmentDao(sqlc: SQLContext) { 24 | 25 | def sumBudgets(): Long = 26 | sqlc 27 | .sql("SELECT SUM(budget) FROM departments") 28 | .map(row => row.getLong(0)).first() 29 | 30 | def numberOfEmployees(): RDD[(Int, Long)] = 31 | sqlc 32 | .sql("SELECT department, COUNT(*) FROM employees GROUP BY department") 33 | .map(row => (row.getInt(0), row.getLong(1))) 34 | 35 | } 36 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Spark, Spark Streaming and Spark SQL unit testing strategies 2 | 3 | [![Build Status](https://travis-ci.org/mkuthan/example-spark.svg?branch=master)](https://travis-ci.org/mkuthan/example-spark) [![Coverage Status](https://img.shields.io/coveralls/mkuthan/example-spark.svg)](https://coveralls.io/r/mkuthan/example-spark?branch=master) 4 | 5 | Features 6 | ======== 7 | 8 | * [ClockWrapper](src/test/scala/org/apache/spark/ClockWrapper.scala) for efficient clock management in Spark Streaming jobs. 9 | * Base traits for testing [Spark](src/test/scala/org/mkuthan/spark/SparkSpec.scala), [Spark Streaming](src/test/scala/org/mkuthan/spark/SparkStreamingSpec.scala) and [Spark SQL](src/test/scala/org/mkuthan/spark/SparkSqlSpec.scala) to eliminate boilerplate code. 10 | * Sample applications to show how to make your code testable. 11 | * All tests can be run or debugged directly from IDE, or using SBT. 12 | * All test fixtures are prepared as in-memory data structures. 13 | * SBT is configured to avoid problems with multiple Spark contexts in the same JVM [SPARK-2243](https://issues.apache.org/jira/browse/SPARK-2243). 14 | * SBT is configured to prepare project assembly for deployment on the cluster. 15 | 16 | References 17 | ========== 18 | 19 | * [http://mkuthan.github.io/blog/2015/03/01/spark-unit-testing/](http://mkuthan.github.io/blog/2015/03/01/spark-unit-testing/) 20 | * [https://github.com/holdenk/spark-testing-base](https://github.com/holdenk/spark-testing-base) 21 | -------------------------------------------------------------------------------- /src/main/scala/example/SparkExample.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import com.typesafe.scalalogging.LazyLogging 20 | import org.apache.spark.{SparkConf, SparkContext} 21 | 22 | object SparkExample extends LazyLogging { 23 | 24 | private val master = "local[2]" 25 | private val appName = "example-spark" 26 | private val stopWords = Set("a", "an", "the") 27 | 28 | def main(args: Array[String]): Unit = { 29 | 30 | val conf = new SparkConf() 31 | .setMaster(master) 32 | .setAppName(appName) 33 | 34 | val sc = new SparkContext(conf) 35 | 36 | val lines = sc.textFile("src/main/resources/data/words.txt") 37 | val wordsCount = WordCount.count(sc, lines, stopWords) 38 | 39 | val counts = wordsCount.collect().mkString("[", ", ", "]") 40 | logger.info(counts) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/test/scala/org/mkuthan/spark/SparkSpec.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package org.mkuthan.spark 18 | 19 | import org.apache.spark.{SparkConf, SparkContext} 20 | import org.scalatest.{BeforeAndAfterAll, Suite} 21 | 22 | trait SparkSpec extends BeforeAndAfterAll { 23 | this: Suite => 24 | 25 | private var _sc: SparkContext = _ 26 | 27 | override def beforeAll(): Unit = { 28 | super.beforeAll() 29 | 30 | val conf = new SparkConf() 31 | .setMaster("local[*]") 32 | .setAppName(this.getClass.getSimpleName) 33 | 34 | sparkConfig.foreach { case (k, v) => conf.setIfMissing(k, v) } 35 | 36 | _sc = new SparkContext(conf) 37 | } 38 | 39 | def sparkConfig: Map[String, String] = Map.empty 40 | 41 | override def afterAll(): Unit = { 42 | if (_sc != null) { 43 | _sc.stop() 44 | _sc = null 45 | } 46 | super.afterAll() 47 | } 48 | 49 | def sc: SparkContext = _sc 50 | 51 | } 52 | -------------------------------------------------------------------------------- /src/test/scala/example/SparkExampleSpec.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import org.mkuthan.spark.SparkSpec 20 | import org.scalatest.{FlatSpec, GivenWhenThen, Matchers} 21 | 22 | class SparkExampleSpec extends FlatSpec with SparkSpec with GivenWhenThen with Matchers { 23 | 24 | "Empty set" should "be counted" in { 25 | Given("empty set") 26 | val lines = Array("") 27 | 28 | When("count words") 29 | val wordCounts = WordCount.count(sc, sc.parallelize(lines)).collect() 30 | 31 | Then("empty count") 32 | wordCounts shouldBe empty 33 | } 34 | 35 | "Shakespeare most famous quote" should "be counted" in { 36 | Given("quote") 37 | val lines = Array("To be or not to be.", "That is the question.") 38 | 39 | Given("stop words") 40 | val stopWords = Set("the") 41 | 42 | When("count words") 43 | val wordCounts = WordCount.count(sc, sc.parallelize(lines), stopWords).collect() 44 | 45 | Then("words counted") 46 | wordCounts should equal(Array( 47 | WordCount("be", 2), 48 | WordCount("is", 1), 49 | WordCount("not", 1), 50 | WordCount("or", 1), 51 | WordCount("question", 1), 52 | WordCount("that", 1), 53 | WordCount("to", 2))) 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/test/scala/org/mkuthan/spark/SparkStreamingSpec.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package org.mkuthan.spark 18 | 19 | import org.apache.spark.streaming.{ClockWrapper, Duration, Seconds, StreamingContext} 20 | import org.scalatest.Suite 21 | 22 | trait SparkStreamingSpec extends SparkSpec { 23 | this: Suite => 24 | 25 | import java.nio.file.Files 26 | 27 | private var _ssc: StreamingContext = _ 28 | 29 | override def beforeAll(): Unit = { 30 | super.beforeAll() 31 | 32 | _ssc = new StreamingContext(sc, batchDuration) 33 | _ssc.checkpoint(checkpointDir) 34 | } 35 | 36 | def batchDuration: Duration = Seconds(1) 37 | 38 | def checkpointDir: String = Files.createTempDirectory(this.getClass.getSimpleName).toUri.toString 39 | 40 | override def afterAll(): Unit = { 41 | if (_ssc != null) { 42 | _ssc.stop(stopSparkContext = false, stopGracefully = false) 43 | _ssc = null 44 | } 45 | 46 | super.afterAll() 47 | } 48 | 49 | override def sparkConfig: Map[String, String] = { 50 | super.sparkConfig + ("spark.streaming.clock" -> "org.apache.spark.streaming.util.ManualClock") 51 | } 52 | 53 | def ssc: StreamingContext = _ssc 54 | 55 | def advanceClock(timeToAdd: Duration): Unit = { 56 | ClockWrapper.advance(_ssc, timeToAdd) 57 | } 58 | 59 | def advanceClockOneBatch(): Unit = { 60 | advanceClock(Duration(batchDuration.milliseconds)) 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/main/scala/example/SparkStreamingExample.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import java.nio.file.Files 20 | 21 | import com.typesafe.scalalogging.LazyLogging 22 | import org.apache.spark.SparkConf 23 | import org.apache.spark.rdd.RDD 24 | import org.apache.spark.streaming.{Seconds, StreamingContext, Time} 25 | 26 | /** To run this example, run Netcat server first: nc -lk 9999. */ 27 | object SparkStreamingExample extends LazyLogging { 28 | 29 | private val master = "local[2]" 30 | private val appName = "example-spark-streaming" 31 | private val stopWords = Set("a", "an", "the") 32 | 33 | private val batchDuration = Seconds(1) 34 | private val checkpointDir = Files.createTempDirectory(appName).toString 35 | private val windowDuration = Seconds(30) 36 | private val slideDuration = Seconds(3) 37 | 38 | def main(args: Array[String]): Unit = { 39 | val conf = new SparkConf() 40 | .setMaster(master) 41 | .setAppName(appName) 42 | 43 | val ssc = new StreamingContext(conf, batchDuration) 44 | ssc.checkpoint(checkpointDir) 45 | 46 | val lines = ssc.socketTextStream("localhost", 9999) 47 | WordCount.count(ssc, lines, windowDuration, slideDuration, stopWords) { (wordsCount: RDD[WordCount], time: Time) => 48 | val counts = time + ": " + wordsCount.collect().mkString("[", ", ", "]") 49 | logger.info(counts) 50 | } 51 | 52 | ssc.start() 53 | ssc.awaitTermination() 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/main/scala/example/EmployeeDao.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import org.apache.spark.rdd.RDD 20 | import org.apache.spark.sql.{Row, SQLContext} 21 | 22 | /** Notice: Variable binding is vulnerable for SQL injection. */ 23 | class EmployeeDao(sqlc: SQLContext) { 24 | 25 | import example.EmployeeDao._ 26 | 27 | def lastNames(): RDD[String] = 28 | sqlc 29 | .sql("SELECT lastName FROM employees") 30 | .map(row => row.getString(0)) 31 | 32 | def distinctLastNames(): RDD[String] = 33 | sqlc 34 | .sql("SELECT DISTINCT lastName FROM employees") 35 | .map(row => row.getString(0)) 36 | 37 | def byLastName(lastNames: String*): RDD[Employee] = 38 | sqlc 39 | .sql(s"SELECT * FROM employees WHERE lastName IN(${lastNames.mkString("'", "', '", "'")})") 40 | .map(toEmployee) 41 | 42 | def byLastNameLike(lastName: String): RDD[Employee] = 43 | sqlc 44 | .sql(s"SELECT * FROM employees WHERE lastName LIKE '$lastName%'") 45 | .map(toEmployee) 46 | 47 | def withDepartment(): RDD[(String, String, String, String, Int)] = { 48 | val sql = 49 | """ 50 | |SELECT ssn, e.name AS name_e, lastName, d.name AS name_d, budget 51 | | FROM employees e INNER JOIN departments d 52 | | ON e.department = d.code 53 | """.stripMargin 54 | 55 | sqlc 56 | .sql(sql) 57 | .map(row => (row.getString(0), row.getString(1), row.getString(2), row.getString(3), row.getInt(4))) 58 | } 59 | } 60 | 61 | object EmployeeDao { 62 | private def toEmployee(row: Row): Employee = 63 | Employee(row.getString(0), row.getString(1), row.getString(2), row.getInt(3)) 64 | } 65 | -------------------------------------------------------------------------------- /src/main/scala/example/WordCount.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import org.apache.spark.SparkContext 20 | import org.apache.spark.broadcast.Broadcast 21 | import org.apache.spark.rdd.RDD 22 | import org.apache.spark.streaming.dstream.DStream 23 | import org.apache.spark.streaming.{Duration, StreamingContext, Time} 24 | 25 | case class WordCount(word: String, count: Int) 26 | 27 | object WordCount { 28 | 29 | type WordHandler = (RDD[WordCount], Time) => Unit 30 | 31 | def count(sc: SparkContext, lines: RDD[String]): RDD[WordCount] = count(sc, lines, Set()) 32 | 33 | def count(sc: SparkContext, lines: RDD[String], stopWords: Set[String]): RDD[WordCount] = { 34 | val stopWordsVar = sc.broadcast(stopWords) 35 | 36 | val words = prepareWords(lines, stopWordsVar) 37 | 38 | val wordCounts = words.map(word => (word, 1)).reduceByKey(_ + _).map { 39 | case (word: String, count: Int) => WordCount(word, count) 40 | } 41 | 42 | val sortedWordCounts = wordCounts.sortBy(_.word) 43 | 44 | sortedWordCounts 45 | } 46 | 47 | def count(ssc: StreamingContext, 48 | lines: DStream[String], 49 | windowDuration: Duration, 50 | slideDuration: Duration) 51 | (handler: WordHandler): Unit = count(ssc, lines, windowDuration, slideDuration, Set())(handler) 52 | 53 | def count(ssc: StreamingContext, 54 | lines: DStream[String], 55 | windowDuration: Duration, 56 | slideDuration: Duration, 57 | stopWords: Set[String]) 58 | (handler: WordHandler): Unit = { 59 | 60 | val sc = ssc.sparkContext 61 | val stopWordsVar = sc.broadcast(stopWords) 62 | 63 | val words = lines.transform(prepareWords(_, stopWordsVar)) 64 | 65 | val wordCounts = words.map(x => (x, 1)).reduceByKeyAndWindow(_ + _, _ - _, windowDuration, slideDuration).map { 66 | case (word: String, count: Int) => WordCount(word, count) 67 | } 68 | 69 | wordCounts.foreachRDD((rdd: RDD[WordCount], time: Time) => { 70 | handler(rdd.sortBy(_.word), time) 71 | }) 72 | } 73 | 74 | private def prepareWords(lines: RDD[String], stopWords: Broadcast[Set[String]]): RDD[String] = { 75 | lines.flatMap(_.split("\\s")) 76 | .map(_.strip(",").strip(".").toLowerCase) 77 | .filter(!stopWords.value.contains(_)).filter(!_.isEmpty) 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/test/scala/example/SparkStreamingExampleSpec.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import org.apache.spark.rdd.RDD 20 | import org.apache.spark.streaming.{Seconds, Time} 21 | import org.mkuthan.spark.SparkStreamingSpec 22 | import org.scalatest.concurrent.Eventually 23 | import org.scalatest.time.{Millis, Span} 24 | import org.scalatest.{FlatSpec, GivenWhenThen, Matchers} 25 | 26 | import scala.collection.mutable 27 | import scala.collection.mutable.ListBuffer 28 | 29 | class SparkStreamingExampleSpec extends FlatSpec 30 | with SparkStreamingSpec with GivenWhenThen with Matchers with Eventually { 31 | 32 | // default timeout for eventually trait 33 | implicit override val patienceConfig = 34 | PatienceConfig(timeout = scaled(Span(5000, Millis))) 35 | private val windowDuration = Seconds(4) 36 | private val slideDuration = Seconds(2) 37 | 38 | "Sample set" should "be counted" in { 39 | Given("streaming context is initialized") 40 | val lines = mutable.Queue[RDD[String]]() 41 | 42 | var results = ListBuffer.empty[Array[WordCount]] 43 | 44 | WordCount.count(ssc, 45 | ssc.queueStream(lines), 46 | windowDuration, 47 | slideDuration) { (wordsCount: RDD[WordCount], time: Time) => 48 | results += wordsCount.collect() 49 | } 50 | 51 | ssc.start() 52 | 53 | When("first set of words queued") 54 | lines += sc.makeRDD(Seq("a", "b")) 55 | 56 | Then("words counted after first slide") 57 | advanceClock(slideDuration) 58 | eventually { 59 | results.last should equal(Array( 60 | WordCount("a", 1), 61 | WordCount("b", 1))) 62 | } 63 | 64 | When("second set of words queued") 65 | lines += sc.makeRDD(Seq("b", "c")) 66 | 67 | Then("words counted after second slide") 68 | advanceClock(slideDuration) 69 | eventually { 70 | results.last should equal(Array( 71 | WordCount("a", 1), 72 | WordCount("b", 2), 73 | WordCount("c", 1))) 74 | } 75 | 76 | When("nothing more queued") 77 | 78 | Then("word counted after third slide") 79 | advanceClock(slideDuration) 80 | eventually { 81 | results.last should equal(Array( 82 | WordCount("a", 0), 83 | WordCount("b", 1), 84 | WordCount("c", 1))) 85 | } 86 | 87 | When("nothing more queued") 88 | 89 | Then("word counted after fourth slide") 90 | advanceClock(slideDuration) 91 | eventually { 92 | results.last should equal(Array( 93 | WordCount("a", 0), 94 | WordCount("b", 0), 95 | WordCount("c", 0))) 96 | } 97 | } 98 | 99 | } 100 | -------------------------------------------------------------------------------- /src/main/scala/example/SparkSqlExample.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import com.typesafe.scalalogging.LazyLogging 20 | import org.apache.spark.sql.hive.HiveContext 21 | import org.apache.spark.{SparkConf, SparkContext} 22 | 23 | /** Based on http://en.wikibooks.org/wiki/SQL_Exercises/Employee_management. */ 24 | object SparkSqlExample extends LazyLogging { 25 | 26 | private val master = "local[2]" 27 | private val appName = "example-spark" 28 | 29 | def main(args: Array[String]): Unit = { 30 | 31 | val conf = new SparkConf() 32 | .setMaster(master) 33 | .setAppName(appName) 34 | 35 | val sc = new SparkContext(conf) 36 | val sqlc = new HiveContext(sc) 37 | 38 | val employeeDao = new EmployeeDao(sqlc) 39 | val departmentDao = new DepartmentDao(sqlc) 40 | 41 | import sqlc.implicits._ 42 | 43 | val employees = sc.textFile("src/main/resources/data/employees.txt") 44 | .map(_.split(",")) 45 | .map(fields => Employee(fields(0), fields(1), fields(2), fields(3).trim.toInt)) 46 | .toDF() 47 | employees.registerTempTable("employees") 48 | 49 | val departments = sc.textFile("src/main/resources/data/departments.txt") 50 | .map(_.split(",")) 51 | .map(fields => Department(fields(0).trim.toInt, fields(1), fields(2).trim.toInt)) 52 | .toDF() 53 | departments.registerTempTable("departments") 54 | 55 | logger.info("Select the last name of all employees") 56 | employeeDao.lastNames().collect().foreach(logger.info(_)) 57 | 58 | logger.info("Select the last name of all employees, without duplicates.") 59 | employeeDao.distinctLastNames().collect().foreach(logger.info(_)) 60 | 61 | logger.info("Select all the data of employees whose last name is \"Smith\".") 62 | employeeDao.byLastName("Smith").collect().map(_.toString) foreach (logger.info(_)) 63 | 64 | logger.info("Select all the data of employees whose last name is \"Smith\" or \"Doe\".") 65 | employeeDao.byLastName("Smith", "Doe").collect().map(_.toString).foreach(logger.info(_)) 66 | 67 | logger.info("Select all the data of employees whose last name begins with an \"S\".") 68 | employeeDao.byLastNameLike("S").collect().map(_.toString).foreach(logger.info(_)) 69 | 70 | logger.info("Select the sum of all the departments' budgets.") 71 | logger.info(departmentDao.sumBudgets().toString) 72 | 73 | logger.info("Select the number of employees in each department.") 74 | departmentDao.numberOfEmployees().collect().map(_.toString()).foreach(logger.info(_)) 75 | 76 | logger.info("Select all the data of employees, including each employee's department's data.") 77 | val employeesWithDepartments = employeeDao.withDepartment() 78 | employeesWithDepartments.collect().map(_.toString).foreach(logger.info(_)) 79 | 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /project/ApplicationBuild.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | import sbt.Keys._ 18 | import sbt._ 19 | 20 | object ApplicationBuild extends Build { 21 | 22 | object Versions { 23 | val spark = "1.6.1" 24 | } 25 | 26 | val projectName = "example-spark" 27 | 28 | val common = Seq( 29 | version := "1.0", 30 | organization := "http://mkuthan.github.io/", 31 | scalaVersion := "2.11.7" 32 | ) 33 | 34 | val customScalacOptions = Seq( 35 | "-deprecation", 36 | "-encoding", "UTF-8", 37 | "-feature", 38 | "-language:existentials", 39 | "-language:higherKinds", 40 | "-language:implicitConversions", 41 | "-unchecked", 42 | "-Xfatal-warnings", 43 | "-Xfuture", 44 | "-Xlint", 45 | "-Yno-adapted-args", 46 | "-Ywarn-dead-code", 47 | "-Ywarn-numeric-widen", 48 | "-Ywarn-unused-import" 49 | ) 50 | 51 | val customJavaInRuntimeOptions = Seq( 52 | "-Xmx512m" 53 | ) 54 | 55 | val customJavaInTestOptions = Seq( 56 | "-Xmx512m" 57 | ) 58 | 59 | val customLibraryDependencies = Seq( 60 | "org.apache.spark" %% "spark-core" % Versions.spark % "provided", 61 | "org.apache.spark" %% "spark-sql" % Versions.spark % "provided", 62 | "org.apache.spark" %% "spark-hive" % Versions.spark % "provided", 63 | "org.apache.spark" %% "spark-streaming" % Versions.spark % "provided", 64 | 65 | "org.apache.spark" %% "spark-streaming-kafka" % Versions.spark 66 | exclude("log4j", "log4j") 67 | exclude("org.spark-project.spark", "unused"), 68 | 69 | "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0", 70 | 71 | "org.slf4j" % "slf4j-api" % "1.7.10", 72 | 73 | "org.slf4j" % "slf4j-log4j12" % "1.7.10" 74 | exclude("log4j", "log4j"), 75 | 76 | "log4j" % "log4j" % "1.2.17" % "provided", 77 | 78 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 79 | ) 80 | 81 | lazy val testScalastyle = taskKey[Unit]("testScalastyle") 82 | lazy val compileScalastyle = taskKey[Unit]("compileScalastyle") 83 | 84 | lazy val main = Project(projectName, base = file(".")) 85 | .settings(common) 86 | .settings(javaOptions in Runtime ++= customJavaInRuntimeOptions) 87 | .settings(javaOptions in Test ++= customJavaInTestOptions) 88 | .settings(scalacOptions ++= customScalacOptions) 89 | .settings(libraryDependencies ++= customLibraryDependencies) 90 | // spark does not support parallel tests and requires JVM fork 91 | .settings(parallelExecution in Test := false) 92 | .settings(fork in Test := true) 93 | // add provided dependencies to the classpath for run and run-main tasks 94 | .settings(run in Compile <<= Defaults.runTask(fullClasspath in Compile, mainClass in(Compile, run), runner in(Compile, run))) 95 | .settings(runMain in Compile <<= Defaults.runMainTask(fullClasspath in Compile, runner in(Compile, run))) 96 | // enable scalastyle checks during compilation 97 | .settings(compileScalastyle := org.scalastyle.sbt.ScalastylePlugin.scalastyle.in(Compile).toTask("").value) 98 | .settings(compile in Compile <<= (compile in Compile) dependsOn compileScalastyle) 99 | // enable scalastyle checks during tests compilation 100 | .settings(testScalastyle := org.scalastyle.sbt.ScalastylePlugin.scalastyle.in(Test).toTask("").value) 101 | .settings(test in Test <<= (test in Test) dependsOn testScalastyle) 102 | } 103 | 104 | -------------------------------------------------------------------------------- /src/test/scala/example/SparkSqlExampleSpec.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example 18 | 19 | import org.mkuthan.spark.SparkSqlSpec 20 | import org.scalatest.{FlatSpec, GivenWhenThen, Matchers} 21 | 22 | class SparkSqlExampleSpec extends FlatSpec with SparkSqlSpec with GivenWhenThen with Matchers { 23 | 24 | private val employees = Array( 25 | Employee("123234877", "Michael", "Rogers", 14), 26 | Employee("152934485", "Anand", "Manikutty", 14), 27 | Employee("222364883", "Carol", "Smith", 37), 28 | Employee("326587417", "Joe", "Stevens", 37), 29 | Employee("332154719", "Mary-Anne", "Foster", 14), 30 | Employee("332569843", "George", "ODonnell", 77), 31 | Employee("546523478", "John", "Doe", 59), 32 | Employee("631231482", "David", "Smith", 77), 33 | Employee("654873219", "Zacary", "Efron", 59), 34 | Employee("745685214", "Eric", "Goldsmith", 59), 35 | Employee("845657245", "Elizabeth", "Doe", 14), 36 | Employee("845657246", "Kumar", "Swamy", 14) 37 | ) 38 | private val departments = Array( 39 | Department(14, "IT", 65000), 40 | Department(37, "Accounting", 15000), 41 | Department(59, "Human Resources", 240000), 42 | Department(77, "Research", 55000) 43 | ) 44 | private var employeeDao: EmployeeDao = _ 45 | private var departmentDao: DepartmentDao = _ 46 | 47 | override def beforeAll(): Unit = { 48 | super.beforeAll() 49 | 50 | val _sqlc = sqlc 51 | 52 | import _sqlc.implicits._ 53 | 54 | sc.parallelize(employees).toDF().registerTempTable("employees") 55 | sc.parallelize(departments).toDF().registerTempTable("departments") 56 | 57 | employeeDao = new EmployeeDao(sqlc) 58 | departmentDao = new DepartmentDao(sqlc) 59 | } 60 | 61 | "The last name of all employees" should "be selected" in { 62 | val lastNames = employeeDao.lastNames().collect() 63 | 64 | lastNames should have length 12 65 | } 66 | 67 | "The last name of all employees" should "be selected without duplicates" in { 68 | val distinctLastNames = employeeDao.distinctLastNames().collect() 69 | 70 | distinctLastNames should have length 10 71 | } 72 | 73 | "The employees whose last name is 'Smith'" should "be selected" in { 74 | val smiths = employeeDao.byLastName("Smith").collect() 75 | 76 | smiths should equal(Array( 77 | Employee("222364883", "Carol", "Smith", 37), 78 | Employee("631231482", "David", "Smith", 77) 79 | )) 80 | } 81 | 82 | "The employees whose last name is 'Smith' or 'Doe'" should "be selected" in { 83 | val smithsOrDoes = employeeDao.byLastName("Smith", "Doe").collect() 84 | 85 | smithsOrDoes should equal(Array( 86 | Employee("222364883", "Carol", "Smith", 37), 87 | Employee("546523478", "John", "Doe", 59), 88 | Employee("631231482", "David", "Smith", 77), 89 | Employee("845657245", "Elizabeth", "Doe", 14) 90 | )) 91 | } 92 | 93 | "The employees whose last name name begins with an 'S'" should "be selected" in { 94 | val smithsOrDoes = employeeDao.byLastNameLike("S").collect() 95 | 96 | smithsOrDoes should equal(Array( 97 | Employee("222364883", "Carol", "Smith", 37), 98 | Employee("326587417", "Joe", "Stevens", 37), 99 | Employee("631231482", "David", "Smith", 77), 100 | Employee("845657246", "Kumar", "Swamy", 14) 101 | )) 102 | } 103 | 104 | "The sum of all the departments' budgets" should "be calculated" in { 105 | val budget = departmentDao.sumBudgets() 106 | 107 | budget should equal(375000) 108 | } 109 | 110 | "The number of all the employees in each department " should "be calculated" in { 111 | val numberOfEmployees = departmentDao.numberOfEmployees().collect() 112 | 113 | numberOfEmployees should equal(Array( 114 | (37, 2), 115 | (59, 3), 116 | (77, 2), 117 | (14, 5) 118 | )) 119 | } 120 | 121 | "All employees including each employee's department's data" should "be selected" in { 122 | val employeesWithDepartment = employeeDao.withDepartment().collect() 123 | 124 | employeesWithDepartment should equal(Array( 125 | ("222364883", "Carol", "Smith", "Accounting", 15000), 126 | ("326587417", "Joe", "Stevens", "Accounting", 15000), 127 | ("546523478", "John", "Doe", "Human Resources", 240000), 128 | ("654873219", "Zacary", "Efron", "Human Resources", 240000), 129 | ("745685214", "Eric", "Goldsmith", "Human Resources", 240000), 130 | ("332569843", "George", "ODonnell", "Research", 55000), 131 | ("631231482", "David", "Smith", "Research", 55000), 132 | ("123234877", "Michael", "Rogers", "IT", 65000), 133 | ("152934485", "Anand", "Manikutty", "IT", 65000), 134 | ("332154719", "Mary-Anne", "Foster", "IT", 65000), 135 | ("845657245", "Elizabeth", "Doe", "IT", 65000), 136 | ("845657246", "Kumar", "Swamy", "IT", 65000) 137 | )) 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /scalastyle-config.xml: -------------------------------------------------------------------------------- 1 | 2 | Scalastyle standard configuration 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 26 | 27 | 28 | 29 | 30 | 2 31 | 2 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | (?m)^\s*$(\r|)\n^\s*$(\r|)\n 93 | false 94 | 95 | No double blank lines 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | java,scala,others 141 | javax?\..+ 142 | scala\..+ 143 | .+ 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | (.*Spec$)|(.*SpecIT$) 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright 2012 The Obvious Corporation and contributors. 2 | 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | 16 | ``` 17 | ------------------------------------------------------------------------- 18 | Apache License 19 | Version 2.0, January 2004 20 | http://www.apache.org/licenses/ 21 | 22 | 23 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 24 | 25 | 1. Definitions. 26 | 27 | "License" shall mean the terms and conditions for use, reproduction, 28 | and distribution as defined by Sections 1 through 9 of this document. 29 | 30 | "Licensor" shall mean the copyright owner or entity authorized by 31 | the copyright owner that is granting the License. 32 | 33 | "Legal Entity" shall mean the union of the acting entity and all 34 | other entities that control, are controlled by, or are under common 35 | control with that entity. For the purposes of this definition, 36 | "control" means (i) the power, direct or indirect, to cause the 37 | direction or management of such entity, whether by contract or 38 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 39 | outstanding shares, or (iii) beneficial ownership of such entity. 40 | 41 | "You" (or "Your") shall mean an individual or Legal Entity 42 | exercising permissions granted by this License. 43 | 44 | "Source" form shall mean the preferred form for making modifications, 45 | including but not limited to software source code, documentation 46 | source, and configuration files. 47 | 48 | "Object" form shall mean any form resulting from mechanical 49 | transformation or translation of a Source form, including but 50 | not limited to compiled object code, generated documentation, 51 | and conversions to other media types. 52 | 53 | "Work" shall mean the work of authorship, whether in Source or 54 | Object form, made available under the License, as indicated by a 55 | copyright notice that is included in or attached to the work 56 | (an example is provided in the Appendix below). 57 | 58 | "Derivative Works" shall mean any work, whether in Source or Object 59 | form, that is based on (or derived from) the Work and for which the 60 | editorial revisions, annotations, elaborations, or other modifications 61 | represent, as a whole, an original work of authorship. For the purposes 62 | of this License, Derivative Works shall not include works that remain 63 | separable from, or merely link (or bind by name) to the interfaces of, 64 | the Work and Derivative Works thereof. 65 | 66 | "Contribution" shall mean any work of authorship, including 67 | the original version of the Work and any modifications or additions 68 | to that Work or Derivative Works thereof, that is intentionally 69 | submitted to Licensor for inclusion in the Work by the copyright owner 70 | or by an individual or Legal Entity authorized to submit on behalf of 71 | the copyright owner. For the purposes of this definition, "submitted" 72 | means any form of electronic, verbal, or written communication sent 73 | to the Licensor or its representatives, including but not limited to 74 | communication on electronic mailing lists, source code control systems, 75 | and issue tracking systems that are managed by, or on behalf of, the 76 | Licensor for the purpose of discussing and improving the Work, but 77 | excluding communication that is conspicuously marked or otherwise 78 | designated in writing by the copyright owner as "Not a Contribution." 79 | 80 | "Contributor" shall mean Licensor and any individual or Legal Entity 81 | on behalf of whom a Contribution has been received by Licensor and 82 | subsequently incorporated within the Work. 83 | 84 | 2. Grant of Copyright License. Subject to the terms and conditions of 85 | this License, each Contributor hereby grants to You a perpetual, 86 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 87 | copyright license to reproduce, prepare Derivative Works of, 88 | publicly display, publicly perform, sublicense, and distribute the 89 | Work and such Derivative Works in Source or Object form. 90 | 91 | 3. Grant of Patent License. Subject to the terms and conditions of 92 | this License, each Contributor hereby grants to You a perpetual, 93 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 94 | (except as stated in this section) patent license to make, have made, 95 | use, offer to sell, sell, import, and otherwise transfer the Work, 96 | where such license applies only to those patent claims licensable 97 | by such Contributor that are necessarily infringed by their 98 | Contribution(s) alone or by combination of their Contribution(s) 99 | with the Work to which such Contribution(s) was submitted. If You 100 | institute patent litigation against any entity (including a 101 | cross-claim or counterclaim in a lawsuit) alleging that the Work 102 | or a Contribution incorporated within the Work constitutes direct 103 | or contributory patent infringement, then any patent licenses 104 | granted to You under this License for that Work shall terminate 105 | as of the date such litigation is filed. 106 | 107 | 4. Redistribution. You may reproduce and distribute copies of the 108 | Work or Derivative Works thereof in any medium, with or without 109 | modifications, and in Source or Object form, provided that You 110 | meet the following conditions: 111 | 112 | (a) You must give any other recipients of the Work or 113 | Derivative Works a copy of this License; and 114 | 115 | (b) You must cause any modified files to carry prominent notices 116 | stating that You changed the files; and 117 | 118 | (c) You must retain, in the Source form of any Derivative Works 119 | that You distribute, all copyright, patent, trademark, and 120 | attribution notices from the Source form of the Work, 121 | excluding those notices that do not pertain to any part of 122 | the Derivative Works; and 123 | 124 | (d) If the Work includes a "NOTICE" text file as part of its 125 | distribution, then any Derivative Works that You distribute must 126 | include a readable copy of the attribution notices contained 127 | within such NOTICE file, excluding those notices that do not 128 | pertain to any part of the Derivative Works, in at least one 129 | of the following places: within a NOTICE text file distributed 130 | as part of the Derivative Works; within the Source form or 131 | documentation, if provided along with the Derivative Works; or, 132 | within a display generated by the Derivative Works, if and 133 | wherever such third-party notices normally appear. The contents 134 | of the NOTICE file are for informational purposes only and 135 | do not modify the License. You may add Your own attribution 136 | notices within Derivative Works that You distribute, alongside 137 | or as an addendum to the NOTICE text from the Work, provided 138 | that such additional attribution notices cannot be construed 139 | as modifying the License. 140 | 141 | You may add Your own copyright statement to Your modifications and 142 | may provide additional or different license terms and conditions 143 | for use, reproduction, or distribution of Your modifications, or 144 | for any such Derivative Works as a whole, provided Your use, 145 | reproduction, and distribution of the Work otherwise complies with 146 | the conditions stated in this License. 147 | 148 | 5. Submission of Contributions. Unless You explicitly state otherwise, 149 | any Contribution intentionally submitted for inclusion in the Work 150 | by You to the Licensor shall be under the terms and conditions of 151 | this License, without any additional terms or conditions. 152 | Notwithstanding the above, nothing herein shall supersede or modify 153 | the terms of any separate license agreement you may have executed 154 | with Licensor regarding such Contributions. 155 | 156 | 6. Trademarks. This License does not grant permission to use the trade 157 | names, trademarks, service marks, or product names of the Licensor, 158 | except as required for reasonable and customary use in describing the 159 | origin of the Work and reproducing the content of the NOTICE file. 160 | 161 | 7. Disclaimer of Warranty. Unless required by applicable law or 162 | agreed to in writing, Licensor provides the Work (and each 163 | Contributor provides its Contributions) on an "AS IS" BASIS, 164 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 165 | implied, including, without limitation, any warranties or conditions 166 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 167 | PARTICULAR PURPOSE. You are solely responsible for determining the 168 | appropriateness of using or redistributing the Work and assume any 169 | risks associated with Your exercise of permissions under this License. 170 | 171 | 8. Limitation of Liability. In no event and under no legal theory, 172 | whether in tort (including negligence), contract, or otherwise, 173 | unless required by applicable law (such as deliberate and grossly 174 | negligent acts) or agreed to in writing, shall any Contributor be 175 | liable to You for damages, including any direct, indirect, special, 176 | incidental, or consequential damages of any character arising as a 177 | result of this License or out of the use or inability to use the 178 | Work (including but not limited to damages for loss of goodwill, 179 | work stoppage, computer failure or malfunction, or any and all 180 | other commercial damages or losses), even if such Contributor 181 | has been advised of the possibility of such damages. 182 | 183 | 9. Accepting Warranty or Additional Liability. While redistributing 184 | the Work or Derivative Works thereof, You may choose to offer, 185 | and charge a fee for, acceptance of support, warranty, indemnity, 186 | or other liability obligations and/or rights consistent with this 187 | License. However, in accepting such obligations, You may act only 188 | on Your own behalf and on Your sole responsibility, not on behalf 189 | of any other Contributor, and only if You agree to indemnify, 190 | defend, and hold each Contributor harmless for any liability 191 | incurred by, or claims asserted against, such Contributor by reason 192 | of your accepting any such warranty or additional liability. 193 | 194 | END OF TERMS AND CONDITIONS 195 | ``` -------------------------------------------------------------------------------- /sbt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # A more capable sbt runner, coincidentally also called sbt. 4 | # Author: Paul Phillips 5 | 6 | set -o pipefail 7 | 8 | # todo - make this dynamic 9 | declare -r sbt_release_version="0.13.10" 10 | declare -r sbt_unreleased_version="0.13.10" 11 | declare -r buildProps="project/build.properties" 12 | 13 | declare sbt_jar sbt_dir sbt_create sbt_version 14 | declare scala_version sbt_explicit_version 15 | declare verbose noshare batch trace_level log_level 16 | declare sbt_saved_stty debugUs 17 | 18 | echoerr () { echo >&2 "$@"; } 19 | vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } 20 | 21 | # spaces are possible, e.g. sbt.version = 0.13.0 22 | build_props_sbt () { 23 | [[ -r "$buildProps" ]] && \ 24 | grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' 25 | } 26 | 27 | update_build_props_sbt () { 28 | local ver="$1" 29 | local old="$(build_props_sbt)" 30 | 31 | [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && { 32 | perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps" 33 | grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps" 34 | 35 | vlog "!!!" 36 | vlog "!!! Updated file $buildProps setting sbt.version to: $ver" 37 | vlog "!!! Previous value was: $old" 38 | vlog "!!!" 39 | } 40 | } 41 | 42 | set_sbt_version () { 43 | sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" 44 | [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version 45 | export sbt_version 46 | } 47 | 48 | # restore stty settings (echo in particular) 49 | onSbtRunnerExit() { 50 | [[ -n "$sbt_saved_stty" ]] || return 51 | vlog "" 52 | vlog "restoring stty: $sbt_saved_stty" 53 | stty "$sbt_saved_stty" 54 | unset sbt_saved_stty 55 | } 56 | 57 | # save stty and trap exit, to ensure echo is reenabled if we are interrupted. 58 | trap onSbtRunnerExit EXIT 59 | sbt_saved_stty="$(stty -g 2>/dev/null)" 60 | vlog "Saved stty: $sbt_saved_stty" 61 | 62 | # this seems to cover the bases on OSX, and someone will 63 | # have to tell me about the others. 64 | get_script_path () { 65 | local path="$1" 66 | [[ -L "$path" ]] || { echo "$path" ; return; } 67 | 68 | local target="$(readlink "$path")" 69 | if [[ "${target:0:1}" == "/" ]]; then 70 | echo "$target" 71 | else 72 | echo "${path%/*}/$target" 73 | fi 74 | } 75 | 76 | die() { 77 | echo "Aborting: $@" 78 | exit 1 79 | } 80 | 81 | url_base () { 82 | local version="$1" 83 | 84 | case "$version" in 85 | 0.7.*) echo "http://simple-build-tool.googlecode.com" ;; 86 | 0.10.* ) echo "$sbt_launch_release_repo" ;; 87 | 0.11.[12]) echo "$sbt_launch_release_repo" ;; 88 | *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss" 89 | echo "$sbt_launch_snapshot_repo" ;; 90 | *) echo "$sbt_launch_release_repo" ;; 91 | esac 92 | } 93 | 94 | make_url () { 95 | local version="$1" 96 | 97 | local base="${sbt_launch_repo:-$(url_base "$version")}" 98 | 99 | case "$version" in 100 | 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;; 101 | 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 102 | 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 103 | *) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; 104 | esac 105 | } 106 | 107 | init_default_option_file () { 108 | local overriding_var="${!1}" 109 | local default_file="$2" 110 | if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then 111 | local envvar_file="${BASH_REMATCH[1]}" 112 | if [[ -r "$envvar_file" ]]; then 113 | default_file="$envvar_file" 114 | fi 115 | fi 116 | echo "$default_file" 117 | } 118 | 119 | declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" 120 | declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation" 121 | declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts" 122 | declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" 123 | declare -r latest_28="2.8.2" 124 | declare -r latest_29="2.9.3" 125 | declare -r latest_210="2.10.6" 126 | declare -r latest_211="2.11.7" 127 | declare -r latest_212="2.12.0-M3" 128 | declare -r sbt_launch_release_repo="http://repo.typesafe.com/typesafe/ivy-releases" 129 | declare -r sbt_launch_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots" 130 | 131 | declare -r script_path="$(get_script_path "$BASH_SOURCE")" 132 | declare -r script_name="${script_path##*/}" 133 | 134 | # some non-read-onlies set with defaults 135 | declare java_cmd="java" 136 | declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" 137 | declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" 138 | declare sbt_launch_dir="$HOME/.sbt/launchers" 139 | 140 | declare sbt_launch_repo 141 | 142 | # pull -J and -D options to give to java. 143 | declare -a residual_args 144 | declare -a java_args 145 | declare -a scalac_args 146 | declare -a sbt_commands 147 | 148 | # args to jvm/sbt via files or environment variables 149 | declare -a extra_jvm_opts extra_sbt_opts 150 | 151 | addJava () { 152 | vlog "[addJava] arg = '$1'" 153 | java_args+=("$1") 154 | } 155 | addSbt () { 156 | vlog "[addSbt] arg = '$1'" 157 | sbt_commands+=("$1") 158 | } 159 | setThisBuild () { 160 | vlog "[addBuild] args = '$@'" 161 | local key="$1" && shift 162 | addSbt "set $key in ThisBuild := $@" 163 | } 164 | addScalac () { 165 | vlog "[addScalac] arg = '$1'" 166 | scalac_args+=("$1") 167 | } 168 | addResidual () { 169 | vlog "[residual] arg = '$1'" 170 | residual_args+=("$1") 171 | } 172 | addResolver () { 173 | addSbt "set resolvers += $1" 174 | } 175 | addDebugger () { 176 | addJava "-Xdebug" 177 | addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1" 178 | } 179 | setScalaVersion () { 180 | [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' 181 | addSbt "++ $1" 182 | } 183 | setJavaHome () { 184 | java_cmd="$1/bin/java" 185 | setThisBuild javaHome "scala.Some(file(\"$1\"))" 186 | export JAVA_HOME="$1" 187 | export JDK_HOME="$1" 188 | export PATH="$JAVA_HOME/bin:$PATH" 189 | } 190 | setJavaHomeQuietly () { 191 | addSbt warn 192 | setJavaHome "$1" 193 | addSbt info 194 | } 195 | 196 | # if set, use JDK_HOME/JAVA_HOME over java found in path 197 | if [[ -e "$JDK_HOME/lib/tools.jar" ]]; then 198 | setJavaHomeQuietly "$JDK_HOME" 199 | elif [[ -e "$JAVA_HOME/bin/java" ]]; then 200 | setJavaHomeQuietly "$JAVA_HOME" 201 | fi 202 | 203 | # directory to store sbt launchers 204 | [[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir" 205 | [[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)" 206 | 207 | java_version () { 208 | local version=$("$java_cmd" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \") 209 | vlog "Detected Java version: $version" 210 | echo "${version:2:1}" 211 | } 212 | 213 | # MaxPermSize critical on pre-8 jvms but incurs noisy warning on 8+ 214 | default_jvm_opts () { 215 | local v="$(java_version)" 216 | if [[ $v -ge 8 ]]; then 217 | echo "$default_jvm_opts_common" 218 | else 219 | echo "-XX:MaxPermSize=384m $default_jvm_opts_common" 220 | fi 221 | } 222 | 223 | build_props_scala () { 224 | if [[ -r "$buildProps" ]]; then 225 | versionLine="$(grep '^build.scala.versions' "$buildProps")" 226 | versionString="${versionLine##build.scala.versions=}" 227 | echo "${versionString%% .*}" 228 | fi 229 | } 230 | 231 | execRunner () { 232 | # print the arguments one to a line, quoting any containing spaces 233 | vlog "# Executing command line:" && { 234 | for arg; do 235 | if [[ -n "$arg" ]]; then 236 | if printf "%s\n" "$arg" | grep -q ' '; then 237 | printf >&2 "\"%s\"\n" "$arg" 238 | else 239 | printf >&2 "%s\n" "$arg" 240 | fi 241 | fi 242 | done 243 | vlog "" 244 | } 245 | 246 | [[ -n "$batch" ]] && exec /dev/null; then 268 | curl --fail --silent --location "$url" --output "$jar" 269 | elif which wget >/dev/null; then 270 | wget --quiet -O "$jar" "$url" 271 | fi 272 | } && [[ -r "$jar" ]] 273 | } 274 | 275 | acquire_sbt_jar () { 276 | local sbt_url="$(jar_url "$sbt_version")" 277 | sbt_jar="$(jar_file "$sbt_version")" 278 | 279 | [[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar" 280 | } 281 | 282 | usage () { 283 | set_sbt_version 284 | cat < display stack traces with a max of frames (default: -1, traces suppressed) 303 | -debug-inc enable debugging log for the incremental compiler 304 | -no-colors disable ANSI color codes 305 | -sbt-create start sbt even if current directory contains no sbt project 306 | -sbt-dir path to global settings/plugins directory (default: ~/.sbt/) 307 | -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11+) 308 | -ivy path to local Ivy repository (default: ~/.ivy2) 309 | -no-share use all local caches; no sharing 310 | -offline put sbt in offline mode 311 | -jvm-debug Turn on JVM debugging, open at the given port. 312 | -batch Disable interactive mode 313 | -prompt Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted 314 | 315 | # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) 316 | -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version 317 | -sbt-version use the specified version of sbt (default: $sbt_release_version) 318 | -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version 319 | -sbt-jar use the specified jar as the sbt launcher 320 | -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir) 321 | -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version")) 322 | 323 | # scala version (default: as chosen by sbt) 324 | -28 use $latest_28 325 | -29 use $latest_29 326 | -210 use $latest_210 327 | -211 use $latest_211 328 | -212 use $latest_212 329 | -scala-home use the scala build at the specified directory 330 | -scala-version use the specified version of scala 331 | -binary-version use the specified scala version when searching for dependencies 332 | 333 | # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) 334 | -java-home alternate JAVA_HOME 335 | 336 | # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution 337 | # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found 338 | $(default_jvm_opts) 339 | JVM_OPTS environment variable holding either the jvm args directly, or 340 | the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') 341 | Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. 342 | -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) 343 | -Dkey=val pass -Dkey=val directly to the jvm 344 | -J-X pass option -X directly to the jvm (-J is stripped) 345 | 346 | # passing options to sbt, OR to this runner 347 | SBT_OPTS environment variable holding either the sbt args directly, or 348 | the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') 349 | Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. 350 | -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) 351 | -S-X add -X to sbt's scalacOptions (-S is stripped) 352 | EOM 353 | } 354 | 355 | process_args () { 356 | require_arg () { 357 | local type="$1" 358 | local opt="$2" 359 | local arg="$3" 360 | 361 | if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then 362 | die "$opt requires <$type> argument" 363 | fi 364 | } 365 | while [[ $# -gt 0 ]]; do 366 | case "$1" in 367 | -h|-help) usage; exit 1 ;; 368 | -v) verbose=true && shift ;; 369 | -d) addSbt "--debug" && addSbt debug && shift ;; 370 | -w) addSbt "--warn" && addSbt warn && shift ;; 371 | -q) addSbt "--error" && addSbt error && shift ;; 372 | -x) debugUs=true && shift ;; 373 | -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; 374 | -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; 375 | -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; 376 | -no-share) noshare=true && shift ;; 377 | -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; 378 | -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; 379 | -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; 380 | -offline) addSbt "set offline := true" && shift ;; 381 | -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; 382 | -batch) batch=true && shift ;; 383 | -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; 384 | 385 | -sbt-create) sbt_create=true && shift ;; 386 | -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; 387 | -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; 388 | -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; 389 | -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; 390 | -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; 391 | -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; 392 | -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; 393 | -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; 394 | -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "scala.Some(file(\"$2\"))" && shift 2 ;; 395 | -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; 396 | -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; 397 | -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; 398 | 399 | -D*) addJava "$1" && shift ;; 400 | -J*) addJava "${1:2}" && shift ;; 401 | -S*) addScalac "${1:2}" && shift ;; 402 | -28) setScalaVersion "$latest_28" && shift ;; 403 | -29) setScalaVersion "$latest_29" && shift ;; 404 | -210) setScalaVersion "$latest_210" && shift ;; 405 | -211) setScalaVersion "$latest_211" && shift ;; 406 | -212) setScalaVersion "$latest_212" && shift ;; 407 | 408 | --debug) addSbt debug && addResidual "$1" && shift ;; 409 | --warn) addSbt warn && addResidual "$1" && shift ;; 410 | --error) addSbt error && addResidual "$1" && shift ;; 411 | *) addResidual "$1" && shift ;; 412 | esac 413 | done 414 | } 415 | 416 | # process the direct command line arguments 417 | process_args "$@" 418 | 419 | # skip #-styled comments and blank lines 420 | readConfigFile() { 421 | local end=false 422 | until $end; do 423 | read || end=true 424 | [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY" 425 | done < "$1" 426 | } 427 | 428 | # if there are file/environment sbt_opts, process again so we 429 | # can supply args to this runner 430 | if [[ -r "$sbt_opts_file" ]]; then 431 | vlog "Using sbt options defined in file $sbt_opts_file" 432 | while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") 433 | elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then 434 | vlog "Using sbt options defined in variable \$SBT_OPTS" 435 | extra_sbt_opts=( $SBT_OPTS ) 436 | else 437 | vlog "No extra sbt options have been defined" 438 | fi 439 | 440 | [[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}" 441 | 442 | # reset "$@" to the residual args 443 | set -- "${residual_args[@]}" 444 | argumentCount=$# 445 | 446 | # set sbt version 447 | set_sbt_version 448 | 449 | # only exists in 0.12+ 450 | setTraceLevel() { 451 | case "$sbt_version" in 452 | "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; 453 | *) setThisBuild traceLevel $trace_level ;; 454 | esac 455 | } 456 | 457 | # set scalacOptions if we were given any -S opts 458 | [[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\"" 459 | 460 | # Update build.properties on disk to set explicit version - sbt gives us no choice 461 | [[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version" 462 | vlog "Detected sbt version $sbt_version" 463 | 464 | [[ -n "$scala_version" ]] && vlog "Overriding scala version to $scala_version" 465 | 466 | # no args - alert them there's stuff in here 467 | (( argumentCount > 0 )) || { 468 | vlog "Starting $script_name: invoke with -help for other options" 469 | residual_args=( shell ) 470 | } 471 | 472 | # verify this is an sbt dir or -create was given 473 | [[ -r ./build.sbt || -d ./project || -n "$sbt_create" ]] || { 474 | cat <