├── project
├── build.properties
├── plugins.sbt
└── ApplicationBuild.scala
├── .gitignore
├── src
├── main
│ ├── resources
│ │ ├── data
│ │ │ ├── departments.txt
│ │ │ ├── employees.txt
│ │ │ └── words.txt
│ │ ├── logback.xml
│ │ └── log4j.properties
│ └── scala
│ │ └── example
│ │ ├── Department.scala
│ │ ├── Employee.scala
│ │ ├── package.scala
│ │ ├── DepartmentDao.scala
│ │ ├── SparkExample.scala
│ │ ├── SparkStreamingExample.scala
│ │ ├── EmployeeDao.scala
│ │ ├── WordCount.scala
│ │ └── SparkSqlExample.scala
└── test
│ └── scala
│ ├── org
│ ├── apache
│ │ └── spark
│ │ │ └── ClockWrapper.scala
│ └── mkuthan
│ │ └── spark
│ │ ├── SparkSqlSpec.scala
│ │ ├── SparkSpec.scala
│ │ └── SparkStreamingSpec.scala
│ └── example
│ ├── SparkExampleSpec.scala
│ ├── SparkStreamingExampleSpec.scala
│ └── SparkSqlExampleSpec.scala
├── .travis.yml
├── README.md
├── scalastyle-config.xml
├── LICENSE.md
└── sbt
/project/build.properties:
--------------------------------------------------------------------------------
1 | sbt.version=0.13.11
2 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | *.iml
3 | derby.log
4 |
5 | target/
6 | project/project/
7 |
--------------------------------------------------------------------------------
/src/main/resources/data/departments.txt:
--------------------------------------------------------------------------------
1 | 14,IT,65000
2 | 37,Accounting,15000
3 | 59,Human Resources,240000
4 | 77,Research,55000
5 |
--------------------------------------------------------------------------------
/src/main/resources/data/employees.txt:
--------------------------------------------------------------------------------
1 | 123234877,Michael,Rogers,14
2 | 152934485,Anand,Manikutty,14
3 | 222364883,Carol,Smith,37
4 | 326587417,Joe,Stevens,37
5 | 332154719,Mary-Anne,Foster,14
6 | 332569843,George,ODonnell,77
7 | 546523478,John,Doe,59
8 | 631231482,David,Smith,77
9 | 654873219,Zacary,Efron,59
10 | 745685214,Eric,Goldsmith,59
11 | 845657245,Elizabeth,Doe,14
12 | 845657246,Kumar,Swamy,14
--------------------------------------------------------------------------------
/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | sudo: false
2 |
3 | cache:
4 | directories:
5 | - $HOME/.ivy2/cache
6 | - $HOME/.sbt/boot/
7 |
8 | language: scala
9 |
10 | scala:
11 | - 2.11.7
12 |
13 | jdk:
14 | - oraclejdk8
15 |
16 | script:
17 | - sbt ++$TRAVIS_SCALA_VERSION clean coverage test
18 |
19 | # Trick to avoid unnecessary cache updates
20 | - find $HOME/.sbt -name "*.lock" | xargs rm
21 | - find $HOME/.ivy2 -name "ivydata-*.properties" | xargs rm
22 |
23 | after_success:
24 | - sbt coveralls
25 |
--------------------------------------------------------------------------------
/src/main/scala/example/Department.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | case class Department(code: Int, name: String, budget: Int)
20 |
--------------------------------------------------------------------------------
/src/main/scala/example/Employee.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | case class Employee(ssn: String, name: String, lastName: String, department: Int)
20 |
--------------------------------------------------------------------------------
/src/main/scala/example/package.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package object example {
18 |
19 | implicit class StringUtils(val value: String) {
20 | def strip(stripChars: String): String = value.stripPrefix(stripChars).stripSuffix(stripChars)
21 | }
22 |
23 | }
24 |
--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.8.0")
18 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.3.3")
19 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.3")
20 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.14.1")
21 |
--------------------------------------------------------------------------------
/src/test/scala/org/apache/spark/ClockWrapper.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package org.apache.spark.streaming
18 |
19 | import org.apache.spark.util.ManualClock
20 |
21 | /** Ugly hack to access Spark private ManualClock class. */
22 | object ClockWrapper {
23 | def advance(ssc: StreamingContext, timeToAdd: Duration): Unit = {
24 | val manualClock = ssc.scheduler.clock.asInstanceOf[ManualClock]
25 | manualClock.advance(timeToAdd.milliseconds)
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/test/scala/org/mkuthan/spark/SparkSqlSpec.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package org.mkuthan.spark
18 |
19 | import org.apache.spark.sql.hive.HiveContext
20 | import org.scalatest.Suite
21 |
22 | trait SparkSqlSpec extends SparkSpec {
23 | this: Suite =>
24 |
25 | private var _sqlc: HiveContext = _
26 |
27 | def sqlc: HiveContext = _sqlc
28 |
29 | override def beforeAll(): Unit = {
30 | super.beforeAll()
31 |
32 | _sqlc = new HiveContext(sc)
33 | }
34 |
35 | override def afterAll(): Unit = {
36 | _sqlc = null
37 |
38 | super.afterAll()
39 | }
40 |
41 | }
42 |
--------------------------------------------------------------------------------
/src/main/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | # Set everything to be logged to the console
2 | log4j.rootCategory=INFO, console
3 | log4j.appender.console=org.apache.log4j.ConsoleAppender
4 | log4j.appender.console.target=System.err
5 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
6 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{1}: %m%n
7 |
8 | # Set the default spark-shell log level to WARN. When running the spark-shell, the
9 | # log level for this class is used to overwrite the root logger's log level, so that
10 | # the user can have different defaults for the shell and regular Spark apps.
11 | log4j.logger.org.apache.spark.repl.Main=WARN
12 |
13 | # Settings to quiet third party logs that are too verbose
14 | log4j.logger.org.spark-project.jetty=WARN
15 | log4j.logger.org.spark-project.jetty.util.component.AbstractLifeCycle=ERROR
16 | log4j.logger.org.apache.spark.repl.SparkIMain$exprTyper=INFO
17 | log4j.logger.org.apache.spark.repl.SparkILoop$SparkILoopInterpreter=INFO
18 | log4j.logger.org.apache.parquet=ERROR
19 | log4j.logger.parquet=ERROR
20 |
21 | # SPARK-9183: Settings to avoid annoying messages when looking up nonexistent UDFs in SparkSQL with Hive support
22 | log4j.logger.org.apache.hadoop.hive.metastore.RetryingHMSHandler=FATAL
23 | log4j.logger.org.apache.hadoop.hive.ql.exec.FunctionRegistry=ERROR
24 |
25 | # Example
26 | log4j.logger.example=DEBUG
27 |
--------------------------------------------------------------------------------
/src/main/scala/example/DepartmentDao.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import org.apache.spark.rdd.RDD
20 | import org.apache.spark.sql.SQLContext
21 |
22 | /** Notice: Variable binding is vulnerable for SQL injection. */
23 | class DepartmentDao(sqlc: SQLContext) {
24 |
25 | def sumBudgets(): Long =
26 | sqlc
27 | .sql("SELECT SUM(budget) FROM departments")
28 | .map(row => row.getLong(0)).first()
29 |
30 | def numberOfEmployees(): RDD[(Int, Long)] =
31 | sqlc
32 | .sql("SELECT department, COUNT(*) FROM employees GROUP BY department")
33 | .map(row => (row.getInt(0), row.getLong(1)))
34 |
35 | }
36 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Spark, Spark Streaming and Spark SQL unit testing strategies
2 |
3 | [](https://travis-ci.org/mkuthan/example-spark) [](https://coveralls.io/r/mkuthan/example-spark?branch=master)
4 |
5 | Features
6 | ========
7 |
8 | * [ClockWrapper](src/test/scala/org/apache/spark/ClockWrapper.scala) for efficient clock management in Spark Streaming jobs.
9 | * Base traits for testing [Spark](src/test/scala/org/mkuthan/spark/SparkSpec.scala), [Spark Streaming](src/test/scala/org/mkuthan/spark/SparkStreamingSpec.scala) and [Spark SQL](src/test/scala/org/mkuthan/spark/SparkSqlSpec.scala) to eliminate boilerplate code.
10 | * Sample applications to show how to make your code testable.
11 | * All tests can be run or debugged directly from IDE, or using SBT.
12 | * All test fixtures are prepared as in-memory data structures.
13 | * SBT is configured to avoid problems with multiple Spark contexts in the same JVM [SPARK-2243](https://issues.apache.org/jira/browse/SPARK-2243).
14 | * SBT is configured to prepare project assembly for deployment on the cluster.
15 |
16 | References
17 | ==========
18 |
19 | * [http://mkuthan.github.io/blog/2015/03/01/spark-unit-testing/](http://mkuthan.github.io/blog/2015/03/01/spark-unit-testing/)
20 | * [https://github.com/holdenk/spark-testing-base](https://github.com/holdenk/spark-testing-base)
21 |
--------------------------------------------------------------------------------
/src/main/scala/example/SparkExample.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import com.typesafe.scalalogging.LazyLogging
20 | import org.apache.spark.{SparkConf, SparkContext}
21 |
22 | object SparkExample extends LazyLogging {
23 |
24 | private val master = "local[2]"
25 | private val appName = "example-spark"
26 | private val stopWords = Set("a", "an", "the")
27 |
28 | def main(args: Array[String]): Unit = {
29 |
30 | val conf = new SparkConf()
31 | .setMaster(master)
32 | .setAppName(appName)
33 |
34 | val sc = new SparkContext(conf)
35 |
36 | val lines = sc.textFile("src/main/resources/data/words.txt")
37 | val wordsCount = WordCount.count(sc, lines, stopWords)
38 |
39 | val counts = wordsCount.collect().mkString("[", ", ", "]")
40 | logger.info(counts)
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/src/test/scala/org/mkuthan/spark/SparkSpec.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package org.mkuthan.spark
18 |
19 | import org.apache.spark.{SparkConf, SparkContext}
20 | import org.scalatest.{BeforeAndAfterAll, Suite}
21 |
22 | trait SparkSpec extends BeforeAndAfterAll {
23 | this: Suite =>
24 |
25 | private var _sc: SparkContext = _
26 |
27 | override def beforeAll(): Unit = {
28 | super.beforeAll()
29 |
30 | val conf = new SparkConf()
31 | .setMaster("local[*]")
32 | .setAppName(this.getClass.getSimpleName)
33 |
34 | sparkConfig.foreach { case (k, v) => conf.setIfMissing(k, v) }
35 |
36 | _sc = new SparkContext(conf)
37 | }
38 |
39 | def sparkConfig: Map[String, String] = Map.empty
40 |
41 | override def afterAll(): Unit = {
42 | if (_sc != null) {
43 | _sc.stop()
44 | _sc = null
45 | }
46 | super.afterAll()
47 | }
48 |
49 | def sc: SparkContext = _sc
50 |
51 | }
52 |
--------------------------------------------------------------------------------
/src/test/scala/example/SparkExampleSpec.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import org.mkuthan.spark.SparkSpec
20 | import org.scalatest.{FlatSpec, GivenWhenThen, Matchers}
21 |
22 | class SparkExampleSpec extends FlatSpec with SparkSpec with GivenWhenThen with Matchers {
23 |
24 | "Empty set" should "be counted" in {
25 | Given("empty set")
26 | val lines = Array("")
27 |
28 | When("count words")
29 | val wordCounts = WordCount.count(sc, sc.parallelize(lines)).collect()
30 |
31 | Then("empty count")
32 | wordCounts shouldBe empty
33 | }
34 |
35 | "Shakespeare most famous quote" should "be counted" in {
36 | Given("quote")
37 | val lines = Array("To be or not to be.", "That is the question.")
38 |
39 | Given("stop words")
40 | val stopWords = Set("the")
41 |
42 | When("count words")
43 | val wordCounts = WordCount.count(sc, sc.parallelize(lines), stopWords).collect()
44 |
45 | Then("words counted")
46 | wordCounts should equal(Array(
47 | WordCount("be", 2),
48 | WordCount("is", 1),
49 | WordCount("not", 1),
50 | WordCount("or", 1),
51 | WordCount("question", 1),
52 | WordCount("that", 1),
53 | WordCount("to", 2)))
54 | }
55 |
56 | }
57 |
--------------------------------------------------------------------------------
/src/test/scala/org/mkuthan/spark/SparkStreamingSpec.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package org.mkuthan.spark
18 |
19 | import org.apache.spark.streaming.{ClockWrapper, Duration, Seconds, StreamingContext}
20 | import org.scalatest.Suite
21 |
22 | trait SparkStreamingSpec extends SparkSpec {
23 | this: Suite =>
24 |
25 | import java.nio.file.Files
26 |
27 | private var _ssc: StreamingContext = _
28 |
29 | override def beforeAll(): Unit = {
30 | super.beforeAll()
31 |
32 | _ssc = new StreamingContext(sc, batchDuration)
33 | _ssc.checkpoint(checkpointDir)
34 | }
35 |
36 | def batchDuration: Duration = Seconds(1)
37 |
38 | def checkpointDir: String = Files.createTempDirectory(this.getClass.getSimpleName).toUri.toString
39 |
40 | override def afterAll(): Unit = {
41 | if (_ssc != null) {
42 | _ssc.stop(stopSparkContext = false, stopGracefully = false)
43 | _ssc = null
44 | }
45 |
46 | super.afterAll()
47 | }
48 |
49 | override def sparkConfig: Map[String, String] = {
50 | super.sparkConfig + ("spark.streaming.clock" -> "org.apache.spark.streaming.util.ManualClock")
51 | }
52 |
53 | def ssc: StreamingContext = _ssc
54 |
55 | def advanceClock(timeToAdd: Duration): Unit = {
56 | ClockWrapper.advance(_ssc, timeToAdd)
57 | }
58 |
59 | def advanceClockOneBatch(): Unit = {
60 | advanceClock(Duration(batchDuration.milliseconds))
61 | }
62 |
63 | }
64 |
--------------------------------------------------------------------------------
/src/main/scala/example/SparkStreamingExample.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import java.nio.file.Files
20 |
21 | import com.typesafe.scalalogging.LazyLogging
22 | import org.apache.spark.SparkConf
23 | import org.apache.spark.rdd.RDD
24 | import org.apache.spark.streaming.{Seconds, StreamingContext, Time}
25 |
26 | /** To run this example, run Netcat server first: nc -lk 9999. */
27 | object SparkStreamingExample extends LazyLogging {
28 |
29 | private val master = "local[2]"
30 | private val appName = "example-spark-streaming"
31 | private val stopWords = Set("a", "an", "the")
32 |
33 | private val batchDuration = Seconds(1)
34 | private val checkpointDir = Files.createTempDirectory(appName).toString
35 | private val windowDuration = Seconds(30)
36 | private val slideDuration = Seconds(3)
37 |
38 | def main(args: Array[String]): Unit = {
39 | val conf = new SparkConf()
40 | .setMaster(master)
41 | .setAppName(appName)
42 |
43 | val ssc = new StreamingContext(conf, batchDuration)
44 | ssc.checkpoint(checkpointDir)
45 |
46 | val lines = ssc.socketTextStream("localhost", 9999)
47 | WordCount.count(ssc, lines, windowDuration, slideDuration, stopWords) { (wordsCount: RDD[WordCount], time: Time) =>
48 | val counts = time + ": " + wordsCount.collect().mkString("[", ", ", "]")
49 | logger.info(counts)
50 | }
51 |
52 | ssc.start()
53 | ssc.awaitTermination()
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/src/main/scala/example/EmployeeDao.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import org.apache.spark.rdd.RDD
20 | import org.apache.spark.sql.{Row, SQLContext}
21 |
22 | /** Notice: Variable binding is vulnerable for SQL injection. */
23 | class EmployeeDao(sqlc: SQLContext) {
24 |
25 | import example.EmployeeDao._
26 |
27 | def lastNames(): RDD[String] =
28 | sqlc
29 | .sql("SELECT lastName FROM employees")
30 | .map(row => row.getString(0))
31 |
32 | def distinctLastNames(): RDD[String] =
33 | sqlc
34 | .sql("SELECT DISTINCT lastName FROM employees")
35 | .map(row => row.getString(0))
36 |
37 | def byLastName(lastNames: String*): RDD[Employee] =
38 | sqlc
39 | .sql(s"SELECT * FROM employees WHERE lastName IN(${lastNames.mkString("'", "', '", "'")})")
40 | .map(toEmployee)
41 |
42 | def byLastNameLike(lastName: String): RDD[Employee] =
43 | sqlc
44 | .sql(s"SELECT * FROM employees WHERE lastName LIKE '$lastName%'")
45 | .map(toEmployee)
46 |
47 | def withDepartment(): RDD[(String, String, String, String, Int)] = {
48 | val sql =
49 | """
50 | |SELECT ssn, e.name AS name_e, lastName, d.name AS name_d, budget
51 | | FROM employees e INNER JOIN departments d
52 | | ON e.department = d.code
53 | """.stripMargin
54 |
55 | sqlc
56 | .sql(sql)
57 | .map(row => (row.getString(0), row.getString(1), row.getString(2), row.getString(3), row.getInt(4)))
58 | }
59 | }
60 |
61 | object EmployeeDao {
62 | private def toEmployee(row: Row): Employee =
63 | Employee(row.getString(0), row.getString(1), row.getString(2), row.getInt(3))
64 | }
65 |
--------------------------------------------------------------------------------
/src/main/scala/example/WordCount.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import org.apache.spark.SparkContext
20 | import org.apache.spark.broadcast.Broadcast
21 | import org.apache.spark.rdd.RDD
22 | import org.apache.spark.streaming.dstream.DStream
23 | import org.apache.spark.streaming.{Duration, StreamingContext, Time}
24 |
25 | case class WordCount(word: String, count: Int)
26 |
27 | object WordCount {
28 |
29 | type WordHandler = (RDD[WordCount], Time) => Unit
30 |
31 | def count(sc: SparkContext, lines: RDD[String]): RDD[WordCount] = count(sc, lines, Set())
32 |
33 | def count(sc: SparkContext, lines: RDD[String], stopWords: Set[String]): RDD[WordCount] = {
34 | val stopWordsVar = sc.broadcast(stopWords)
35 |
36 | val words = prepareWords(lines, stopWordsVar)
37 |
38 | val wordCounts = words.map(word => (word, 1)).reduceByKey(_ + _).map {
39 | case (word: String, count: Int) => WordCount(word, count)
40 | }
41 |
42 | val sortedWordCounts = wordCounts.sortBy(_.word)
43 |
44 | sortedWordCounts
45 | }
46 |
47 | def count(ssc: StreamingContext,
48 | lines: DStream[String],
49 | windowDuration: Duration,
50 | slideDuration: Duration)
51 | (handler: WordHandler): Unit = count(ssc, lines, windowDuration, slideDuration, Set())(handler)
52 |
53 | def count(ssc: StreamingContext,
54 | lines: DStream[String],
55 | windowDuration: Duration,
56 | slideDuration: Duration,
57 | stopWords: Set[String])
58 | (handler: WordHandler): Unit = {
59 |
60 | val sc = ssc.sparkContext
61 | val stopWordsVar = sc.broadcast(stopWords)
62 |
63 | val words = lines.transform(prepareWords(_, stopWordsVar))
64 |
65 | val wordCounts = words.map(x => (x, 1)).reduceByKeyAndWindow(_ + _, _ - _, windowDuration, slideDuration).map {
66 | case (word: String, count: Int) => WordCount(word, count)
67 | }
68 |
69 | wordCounts.foreachRDD((rdd: RDD[WordCount], time: Time) => {
70 | handler(rdd.sortBy(_.word), time)
71 | })
72 | }
73 |
74 | private def prepareWords(lines: RDD[String], stopWords: Broadcast[Set[String]]): RDD[String] = {
75 | lines.flatMap(_.split("\\s"))
76 | .map(_.strip(",").strip(".").toLowerCase)
77 | .filter(!stopWords.value.contains(_)).filter(!_.isEmpty)
78 | }
79 |
80 | }
81 |
--------------------------------------------------------------------------------
/src/test/scala/example/SparkStreamingExampleSpec.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import org.apache.spark.rdd.RDD
20 | import org.apache.spark.streaming.{Seconds, Time}
21 | import org.mkuthan.spark.SparkStreamingSpec
22 | import org.scalatest.concurrent.Eventually
23 | import org.scalatest.time.{Millis, Span}
24 | import org.scalatest.{FlatSpec, GivenWhenThen, Matchers}
25 |
26 | import scala.collection.mutable
27 | import scala.collection.mutable.ListBuffer
28 |
29 | class SparkStreamingExampleSpec extends FlatSpec
30 | with SparkStreamingSpec with GivenWhenThen with Matchers with Eventually {
31 |
32 | // default timeout for eventually trait
33 | implicit override val patienceConfig =
34 | PatienceConfig(timeout = scaled(Span(5000, Millis)))
35 | private val windowDuration = Seconds(4)
36 | private val slideDuration = Seconds(2)
37 |
38 | "Sample set" should "be counted" in {
39 | Given("streaming context is initialized")
40 | val lines = mutable.Queue[RDD[String]]()
41 |
42 | var results = ListBuffer.empty[Array[WordCount]]
43 |
44 | WordCount.count(ssc,
45 | ssc.queueStream(lines),
46 | windowDuration,
47 | slideDuration) { (wordsCount: RDD[WordCount], time: Time) =>
48 | results += wordsCount.collect()
49 | }
50 |
51 | ssc.start()
52 |
53 | When("first set of words queued")
54 | lines += sc.makeRDD(Seq("a", "b"))
55 |
56 | Then("words counted after first slide")
57 | advanceClock(slideDuration)
58 | eventually {
59 | results.last should equal(Array(
60 | WordCount("a", 1),
61 | WordCount("b", 1)))
62 | }
63 |
64 | When("second set of words queued")
65 | lines += sc.makeRDD(Seq("b", "c"))
66 |
67 | Then("words counted after second slide")
68 | advanceClock(slideDuration)
69 | eventually {
70 | results.last should equal(Array(
71 | WordCount("a", 1),
72 | WordCount("b", 2),
73 | WordCount("c", 1)))
74 | }
75 |
76 | When("nothing more queued")
77 |
78 | Then("word counted after third slide")
79 | advanceClock(slideDuration)
80 | eventually {
81 | results.last should equal(Array(
82 | WordCount("a", 0),
83 | WordCount("b", 1),
84 | WordCount("c", 1)))
85 | }
86 |
87 | When("nothing more queued")
88 |
89 | Then("word counted after fourth slide")
90 | advanceClock(slideDuration)
91 | eventually {
92 | results.last should equal(Array(
93 | WordCount("a", 0),
94 | WordCount("b", 0),
95 | WordCount("c", 0)))
96 | }
97 | }
98 |
99 | }
100 |
--------------------------------------------------------------------------------
/src/main/scala/example/SparkSqlExample.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import com.typesafe.scalalogging.LazyLogging
20 | import org.apache.spark.sql.hive.HiveContext
21 | import org.apache.spark.{SparkConf, SparkContext}
22 |
23 | /** Based on http://en.wikibooks.org/wiki/SQL_Exercises/Employee_management. */
24 | object SparkSqlExample extends LazyLogging {
25 |
26 | private val master = "local[2]"
27 | private val appName = "example-spark"
28 |
29 | def main(args: Array[String]): Unit = {
30 |
31 | val conf = new SparkConf()
32 | .setMaster(master)
33 | .setAppName(appName)
34 |
35 | val sc = new SparkContext(conf)
36 | val sqlc = new HiveContext(sc)
37 |
38 | val employeeDao = new EmployeeDao(sqlc)
39 | val departmentDao = new DepartmentDao(sqlc)
40 |
41 | import sqlc.implicits._
42 |
43 | val employees = sc.textFile("src/main/resources/data/employees.txt")
44 | .map(_.split(","))
45 | .map(fields => Employee(fields(0), fields(1), fields(2), fields(3).trim.toInt))
46 | .toDF()
47 | employees.registerTempTable("employees")
48 |
49 | val departments = sc.textFile("src/main/resources/data/departments.txt")
50 | .map(_.split(","))
51 | .map(fields => Department(fields(0).trim.toInt, fields(1), fields(2).trim.toInt))
52 | .toDF()
53 | departments.registerTempTable("departments")
54 |
55 | logger.info("Select the last name of all employees")
56 | employeeDao.lastNames().collect().foreach(logger.info(_))
57 |
58 | logger.info("Select the last name of all employees, without duplicates.")
59 | employeeDao.distinctLastNames().collect().foreach(logger.info(_))
60 |
61 | logger.info("Select all the data of employees whose last name is \"Smith\".")
62 | employeeDao.byLastName("Smith").collect().map(_.toString) foreach (logger.info(_))
63 |
64 | logger.info("Select all the data of employees whose last name is \"Smith\" or \"Doe\".")
65 | employeeDao.byLastName("Smith", "Doe").collect().map(_.toString).foreach(logger.info(_))
66 |
67 | logger.info("Select all the data of employees whose last name begins with an \"S\".")
68 | employeeDao.byLastNameLike("S").collect().map(_.toString).foreach(logger.info(_))
69 |
70 | logger.info("Select the sum of all the departments' budgets.")
71 | logger.info(departmentDao.sumBudgets().toString)
72 |
73 | logger.info("Select the number of employees in each department.")
74 | departmentDao.numberOfEmployees().collect().map(_.toString()).foreach(logger.info(_))
75 |
76 | logger.info("Select all the data of employees, including each employee's department's data.")
77 | val employeesWithDepartments = employeeDao.withDepartment()
78 | employeesWithDepartments.collect().map(_.toString).foreach(logger.info(_))
79 |
80 | }
81 |
82 | }
83 |
--------------------------------------------------------------------------------
/project/ApplicationBuild.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | import sbt.Keys._
18 | import sbt._
19 |
20 | object ApplicationBuild extends Build {
21 |
22 | object Versions {
23 | val spark = "1.6.1"
24 | }
25 |
26 | val projectName = "example-spark"
27 |
28 | val common = Seq(
29 | version := "1.0",
30 | organization := "http://mkuthan.github.io/",
31 | scalaVersion := "2.11.7"
32 | )
33 |
34 | val customScalacOptions = Seq(
35 | "-deprecation",
36 | "-encoding", "UTF-8",
37 | "-feature",
38 | "-language:existentials",
39 | "-language:higherKinds",
40 | "-language:implicitConversions",
41 | "-unchecked",
42 | "-Xfatal-warnings",
43 | "-Xfuture",
44 | "-Xlint",
45 | "-Yno-adapted-args",
46 | "-Ywarn-dead-code",
47 | "-Ywarn-numeric-widen",
48 | "-Ywarn-unused-import"
49 | )
50 |
51 | val customJavaInRuntimeOptions = Seq(
52 | "-Xmx512m"
53 | )
54 |
55 | val customJavaInTestOptions = Seq(
56 | "-Xmx512m"
57 | )
58 |
59 | val customLibraryDependencies = Seq(
60 | "org.apache.spark" %% "spark-core" % Versions.spark % "provided",
61 | "org.apache.spark" %% "spark-sql" % Versions.spark % "provided",
62 | "org.apache.spark" %% "spark-hive" % Versions.spark % "provided",
63 | "org.apache.spark" %% "spark-streaming" % Versions.spark % "provided",
64 |
65 | "org.apache.spark" %% "spark-streaming-kafka" % Versions.spark
66 | exclude("log4j", "log4j")
67 | exclude("org.spark-project.spark", "unused"),
68 |
69 | "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0",
70 |
71 | "org.slf4j" % "slf4j-api" % "1.7.10",
72 |
73 | "org.slf4j" % "slf4j-log4j12" % "1.7.10"
74 | exclude("log4j", "log4j"),
75 |
76 | "log4j" % "log4j" % "1.2.17" % "provided",
77 |
78 | "org.scalatest" %% "scalatest" % "2.2.4" % "test"
79 | )
80 |
81 | lazy val testScalastyle = taskKey[Unit]("testScalastyle")
82 | lazy val compileScalastyle = taskKey[Unit]("compileScalastyle")
83 |
84 | lazy val main = Project(projectName, base = file("."))
85 | .settings(common)
86 | .settings(javaOptions in Runtime ++= customJavaInRuntimeOptions)
87 | .settings(javaOptions in Test ++= customJavaInTestOptions)
88 | .settings(scalacOptions ++= customScalacOptions)
89 | .settings(libraryDependencies ++= customLibraryDependencies)
90 | // spark does not support parallel tests and requires JVM fork
91 | .settings(parallelExecution in Test := false)
92 | .settings(fork in Test := true)
93 | // add provided dependencies to the classpath for run and run-main tasks
94 | .settings(run in Compile <<= Defaults.runTask(fullClasspath in Compile, mainClass in(Compile, run), runner in(Compile, run)))
95 | .settings(runMain in Compile <<= Defaults.runMainTask(fullClasspath in Compile, runner in(Compile, run)))
96 | // enable scalastyle checks during compilation
97 | .settings(compileScalastyle := org.scalastyle.sbt.ScalastylePlugin.scalastyle.in(Compile).toTask("").value)
98 | .settings(compile in Compile <<= (compile in Compile) dependsOn compileScalastyle)
99 | // enable scalastyle checks during tests compilation
100 | .settings(testScalastyle := org.scalastyle.sbt.ScalastylePlugin.scalastyle.in(Test).toTask("").value)
101 | .settings(test in Test <<= (test in Test) dependsOn testScalastyle)
102 | }
103 |
104 |
--------------------------------------------------------------------------------
/src/test/scala/example/SparkSqlExampleSpec.scala:
--------------------------------------------------------------------------------
1 | // Copyright (C) 2011-2012 the original author or authors.
2 | // See the LICENCE.txt file distributed with this work for additional
3 | // information regarding copyright ownership.
4 | //
5 | // Licensed under the Apache License, Version 2.0 (the "License");
6 | // you may not use this file except in compliance with the License.
7 | // You may obtain a copy of the License at
8 | //
9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 |
17 | package example
18 |
19 | import org.mkuthan.spark.SparkSqlSpec
20 | import org.scalatest.{FlatSpec, GivenWhenThen, Matchers}
21 |
22 | class SparkSqlExampleSpec extends FlatSpec with SparkSqlSpec with GivenWhenThen with Matchers {
23 |
24 | private val employees = Array(
25 | Employee("123234877", "Michael", "Rogers", 14),
26 | Employee("152934485", "Anand", "Manikutty", 14),
27 | Employee("222364883", "Carol", "Smith", 37),
28 | Employee("326587417", "Joe", "Stevens", 37),
29 | Employee("332154719", "Mary-Anne", "Foster", 14),
30 | Employee("332569843", "George", "ODonnell", 77),
31 | Employee("546523478", "John", "Doe", 59),
32 | Employee("631231482", "David", "Smith", 77),
33 | Employee("654873219", "Zacary", "Efron", 59),
34 | Employee("745685214", "Eric", "Goldsmith", 59),
35 | Employee("845657245", "Elizabeth", "Doe", 14),
36 | Employee("845657246", "Kumar", "Swamy", 14)
37 | )
38 | private val departments = Array(
39 | Department(14, "IT", 65000),
40 | Department(37, "Accounting", 15000),
41 | Department(59, "Human Resources", 240000),
42 | Department(77, "Research", 55000)
43 | )
44 | private var employeeDao: EmployeeDao = _
45 | private var departmentDao: DepartmentDao = _
46 |
47 | override def beforeAll(): Unit = {
48 | super.beforeAll()
49 |
50 | val _sqlc = sqlc
51 |
52 | import _sqlc.implicits._
53 |
54 | sc.parallelize(employees).toDF().registerTempTable("employees")
55 | sc.parallelize(departments).toDF().registerTempTable("departments")
56 |
57 | employeeDao = new EmployeeDao(sqlc)
58 | departmentDao = new DepartmentDao(sqlc)
59 | }
60 |
61 | "The last name of all employees" should "be selected" in {
62 | val lastNames = employeeDao.lastNames().collect()
63 |
64 | lastNames should have length 12
65 | }
66 |
67 | "The last name of all employees" should "be selected without duplicates" in {
68 | val distinctLastNames = employeeDao.distinctLastNames().collect()
69 |
70 | distinctLastNames should have length 10
71 | }
72 |
73 | "The employees whose last name is 'Smith'" should "be selected" in {
74 | val smiths = employeeDao.byLastName("Smith").collect()
75 |
76 | smiths should equal(Array(
77 | Employee("222364883", "Carol", "Smith", 37),
78 | Employee("631231482", "David", "Smith", 77)
79 | ))
80 | }
81 |
82 | "The employees whose last name is 'Smith' or 'Doe'" should "be selected" in {
83 | val smithsOrDoes = employeeDao.byLastName("Smith", "Doe").collect()
84 |
85 | smithsOrDoes should equal(Array(
86 | Employee("222364883", "Carol", "Smith", 37),
87 | Employee("546523478", "John", "Doe", 59),
88 | Employee("631231482", "David", "Smith", 77),
89 | Employee("845657245", "Elizabeth", "Doe", 14)
90 | ))
91 | }
92 |
93 | "The employees whose last name name begins with an 'S'" should "be selected" in {
94 | val smithsOrDoes = employeeDao.byLastNameLike("S").collect()
95 |
96 | smithsOrDoes should equal(Array(
97 | Employee("222364883", "Carol", "Smith", 37),
98 | Employee("326587417", "Joe", "Stevens", 37),
99 | Employee("631231482", "David", "Smith", 77),
100 | Employee("845657246", "Kumar", "Swamy", 14)
101 | ))
102 | }
103 |
104 | "The sum of all the departments' budgets" should "be calculated" in {
105 | val budget = departmentDao.sumBudgets()
106 |
107 | budget should equal(375000)
108 | }
109 |
110 | "The number of all the employees in each department " should "be calculated" in {
111 | val numberOfEmployees = departmentDao.numberOfEmployees().collect()
112 |
113 | numberOfEmployees should equal(Array(
114 | (37, 2),
115 | (59, 3),
116 | (77, 2),
117 | (14, 5)
118 | ))
119 | }
120 |
121 | "All employees including each employee's department's data" should "be selected" in {
122 | val employeesWithDepartment = employeeDao.withDepartment().collect()
123 |
124 | employeesWithDepartment should equal(Array(
125 | ("222364883", "Carol", "Smith", "Accounting", 15000),
126 | ("326587417", "Joe", "Stevens", "Accounting", 15000),
127 | ("546523478", "John", "Doe", "Human Resources", 240000),
128 | ("654873219", "Zacary", "Efron", "Human Resources", 240000),
129 | ("745685214", "Eric", "Goldsmith", "Human Resources", 240000),
130 | ("332569843", "George", "ODonnell", "Research", 55000),
131 | ("631231482", "David", "Smith", "Research", 55000),
132 | ("123234877", "Michael", "Rogers", "IT", 65000),
133 | ("152934485", "Anand", "Manikutty", "IT", 65000),
134 | ("332154719", "Mary-Anne", "Foster", "IT", 65000),
135 | ("845657245", "Elizabeth", "Doe", "IT", 65000),
136 | ("845657246", "Kumar", "Swamy", "IT", 65000)
137 | ))
138 | }
139 |
140 | }
141 |
--------------------------------------------------------------------------------
/scalastyle-config.xml:
--------------------------------------------------------------------------------
1 |
2 | Scalastyle standard configuration
3 |
4 |
5 |
6 |
7 |
8 |
9 |
28 |
29 |
30 | 2
31 | 2
32 |
33 |
34 |
35 |
36 |
37 |
38 |
39 |
40 |
41 |
42 |
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 |
68 |
69 |
70 |
71 |
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
85 |
90 |
91 |
92 | (?m)^\s*$(\r|)\n^\s*$(\r|)\n
93 | false
94 |
95 | No double blank lines
96 |
97 |
98 |
99 |
100 |
101 |
102 |
103 |
104 |
105 |
106 |
107 |
108 |
109 |
110 |
111 |
112 |
113 |
114 |
115 |
116 |
117 |
118 |
119 |
120 |
121 |
122 |
123 |
124 |
125 |
126 |
127 |
128 |
129 |
130 |
131 |
132 |
133 |
134 |
135 |
136 |
137 |
138 |
139 |
140 | java,scala,others
141 | javax?\..+
142 | scala\..+
143 | .+
144 |
145 |
146 |
147 |
148 |
149 |
150 |
151 |
152 | (.*Spec$)|(.*SpecIT$)
153 |
154 |
155 |
156 |
157 |
158 |
159 |
160 |
161 |
162 |
163 |
164 |
--------------------------------------------------------------------------------
/LICENSE.md:
--------------------------------------------------------------------------------
1 | Copyright 2012 The Obvious Corporation and contributors.
2 |
3 |
4 | Licensed under the Apache License, Version 2.0 (the "License");
5 | you may not use this file except in compliance with the License.
6 | You may obtain a copy of the License at
7 |
8 |
9 |
10 | Unless required by applicable law or agreed to in writing, software
11 | distributed under the License is distributed on an "AS IS" BASIS,
12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | See the License for the specific language governing permissions and
14 | limitations under the License.
15 |
16 | ```
17 | -------------------------------------------------------------------------
18 | Apache License
19 | Version 2.0, January 2004
20 | http://www.apache.org/licenses/
21 |
22 |
23 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
24 |
25 | 1. Definitions.
26 |
27 | "License" shall mean the terms and conditions for use, reproduction,
28 | and distribution as defined by Sections 1 through 9 of this document.
29 |
30 | "Licensor" shall mean the copyright owner or entity authorized by
31 | the copyright owner that is granting the License.
32 |
33 | "Legal Entity" shall mean the union of the acting entity and all
34 | other entities that control, are controlled by, or are under common
35 | control with that entity. For the purposes of this definition,
36 | "control" means (i) the power, direct or indirect, to cause the
37 | direction or management of such entity, whether by contract or
38 | otherwise, or (ii) ownership of fifty percent (50%) or more of the
39 | outstanding shares, or (iii) beneficial ownership of such entity.
40 |
41 | "You" (or "Your") shall mean an individual or Legal Entity
42 | exercising permissions granted by this License.
43 |
44 | "Source" form shall mean the preferred form for making modifications,
45 | including but not limited to software source code, documentation
46 | source, and configuration files.
47 |
48 | "Object" form shall mean any form resulting from mechanical
49 | transformation or translation of a Source form, including but
50 | not limited to compiled object code, generated documentation,
51 | and conversions to other media types.
52 |
53 | "Work" shall mean the work of authorship, whether in Source or
54 | Object form, made available under the License, as indicated by a
55 | copyright notice that is included in or attached to the work
56 | (an example is provided in the Appendix below).
57 |
58 | "Derivative Works" shall mean any work, whether in Source or Object
59 | form, that is based on (or derived from) the Work and for which the
60 | editorial revisions, annotations, elaborations, or other modifications
61 | represent, as a whole, an original work of authorship. For the purposes
62 | of this License, Derivative Works shall not include works that remain
63 | separable from, or merely link (or bind by name) to the interfaces of,
64 | the Work and Derivative Works thereof.
65 |
66 | "Contribution" shall mean any work of authorship, including
67 | the original version of the Work and any modifications or additions
68 | to that Work or Derivative Works thereof, that is intentionally
69 | submitted to Licensor for inclusion in the Work by the copyright owner
70 | or by an individual or Legal Entity authorized to submit on behalf of
71 | the copyright owner. For the purposes of this definition, "submitted"
72 | means any form of electronic, verbal, or written communication sent
73 | to the Licensor or its representatives, including but not limited to
74 | communication on electronic mailing lists, source code control systems,
75 | and issue tracking systems that are managed by, or on behalf of, the
76 | Licensor for the purpose of discussing and improving the Work, but
77 | excluding communication that is conspicuously marked or otherwise
78 | designated in writing by the copyright owner as "Not a Contribution."
79 |
80 | "Contributor" shall mean Licensor and any individual or Legal Entity
81 | on behalf of whom a Contribution has been received by Licensor and
82 | subsequently incorporated within the Work.
83 |
84 | 2. Grant of Copyright License. Subject to the terms and conditions of
85 | this License, each Contributor hereby grants to You a perpetual,
86 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
87 | copyright license to reproduce, prepare Derivative Works of,
88 | publicly display, publicly perform, sublicense, and distribute the
89 | Work and such Derivative Works in Source or Object form.
90 |
91 | 3. Grant of Patent License. Subject to the terms and conditions of
92 | this License, each Contributor hereby grants to You a perpetual,
93 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable
94 | (except as stated in this section) patent license to make, have made,
95 | use, offer to sell, sell, import, and otherwise transfer the Work,
96 | where such license applies only to those patent claims licensable
97 | by such Contributor that are necessarily infringed by their
98 | Contribution(s) alone or by combination of their Contribution(s)
99 | with the Work to which such Contribution(s) was submitted. If You
100 | institute patent litigation against any entity (including a
101 | cross-claim or counterclaim in a lawsuit) alleging that the Work
102 | or a Contribution incorporated within the Work constitutes direct
103 | or contributory patent infringement, then any patent licenses
104 | granted to You under this License for that Work shall terminate
105 | as of the date such litigation is filed.
106 |
107 | 4. Redistribution. You may reproduce and distribute copies of the
108 | Work or Derivative Works thereof in any medium, with or without
109 | modifications, and in Source or Object form, provided that You
110 | meet the following conditions:
111 |
112 | (a) You must give any other recipients of the Work or
113 | Derivative Works a copy of this License; and
114 |
115 | (b) You must cause any modified files to carry prominent notices
116 | stating that You changed the files; and
117 |
118 | (c) You must retain, in the Source form of any Derivative Works
119 | that You distribute, all copyright, patent, trademark, and
120 | attribution notices from the Source form of the Work,
121 | excluding those notices that do not pertain to any part of
122 | the Derivative Works; and
123 |
124 | (d) If the Work includes a "NOTICE" text file as part of its
125 | distribution, then any Derivative Works that You distribute must
126 | include a readable copy of the attribution notices contained
127 | within such NOTICE file, excluding those notices that do not
128 | pertain to any part of the Derivative Works, in at least one
129 | of the following places: within a NOTICE text file distributed
130 | as part of the Derivative Works; within the Source form or
131 | documentation, if provided along with the Derivative Works; or,
132 | within a display generated by the Derivative Works, if and
133 | wherever such third-party notices normally appear. The contents
134 | of the NOTICE file are for informational purposes only and
135 | do not modify the License. You may add Your own attribution
136 | notices within Derivative Works that You distribute, alongside
137 | or as an addendum to the NOTICE text from the Work, provided
138 | that such additional attribution notices cannot be construed
139 | as modifying the License.
140 |
141 | You may add Your own copyright statement to Your modifications and
142 | may provide additional or different license terms and conditions
143 | for use, reproduction, or distribution of Your modifications, or
144 | for any such Derivative Works as a whole, provided Your use,
145 | reproduction, and distribution of the Work otherwise complies with
146 | the conditions stated in this License.
147 |
148 | 5. Submission of Contributions. Unless You explicitly state otherwise,
149 | any Contribution intentionally submitted for inclusion in the Work
150 | by You to the Licensor shall be under the terms and conditions of
151 | this License, without any additional terms or conditions.
152 | Notwithstanding the above, nothing herein shall supersede or modify
153 | the terms of any separate license agreement you may have executed
154 | with Licensor regarding such Contributions.
155 |
156 | 6. Trademarks. This License does not grant permission to use the trade
157 | names, trademarks, service marks, or product names of the Licensor,
158 | except as required for reasonable and customary use in describing the
159 | origin of the Work and reproducing the content of the NOTICE file.
160 |
161 | 7. Disclaimer of Warranty. Unless required by applicable law or
162 | agreed to in writing, Licensor provides the Work (and each
163 | Contributor provides its Contributions) on an "AS IS" BASIS,
164 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
165 | implied, including, without limitation, any warranties or conditions
166 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
167 | PARTICULAR PURPOSE. You are solely responsible for determining the
168 | appropriateness of using or redistributing the Work and assume any
169 | risks associated with Your exercise of permissions under this License.
170 |
171 | 8. Limitation of Liability. In no event and under no legal theory,
172 | whether in tort (including negligence), contract, or otherwise,
173 | unless required by applicable law (such as deliberate and grossly
174 | negligent acts) or agreed to in writing, shall any Contributor be
175 | liable to You for damages, including any direct, indirect, special,
176 | incidental, or consequential damages of any character arising as a
177 | result of this License or out of the use or inability to use the
178 | Work (including but not limited to damages for loss of goodwill,
179 | work stoppage, computer failure or malfunction, or any and all
180 | other commercial damages or losses), even if such Contributor
181 | has been advised of the possibility of such damages.
182 |
183 | 9. Accepting Warranty or Additional Liability. While redistributing
184 | the Work or Derivative Works thereof, You may choose to offer,
185 | and charge a fee for, acceptance of support, warranty, indemnity,
186 | or other liability obligations and/or rights consistent with this
187 | License. However, in accepting such obligations, You may act only
188 | on Your own behalf and on Your sole responsibility, not on behalf
189 | of any other Contributor, and only if You agree to indemnify,
190 | defend, and hold each Contributor harmless for any liability
191 | incurred by, or claims asserted against, such Contributor by reason
192 | of your accepting any such warranty or additional liability.
193 |
194 | END OF TERMS AND CONDITIONS
195 | ```
--------------------------------------------------------------------------------
/sbt:
--------------------------------------------------------------------------------
1 | #!/usr/bin/env bash
2 | #
3 | # A more capable sbt runner, coincidentally also called sbt.
4 | # Author: Paul Phillips
5 |
6 | set -o pipefail
7 |
8 | # todo - make this dynamic
9 | declare -r sbt_release_version="0.13.10"
10 | declare -r sbt_unreleased_version="0.13.10"
11 | declare -r buildProps="project/build.properties"
12 |
13 | declare sbt_jar sbt_dir sbt_create sbt_version
14 | declare scala_version sbt_explicit_version
15 | declare verbose noshare batch trace_level log_level
16 | declare sbt_saved_stty debugUs
17 |
18 | echoerr () { echo >&2 "$@"; }
19 | vlog () { [[ -n "$verbose" ]] && echoerr "$@"; }
20 |
21 | # spaces are possible, e.g. sbt.version = 0.13.0
22 | build_props_sbt () {
23 | [[ -r "$buildProps" ]] && \
24 | grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
25 | }
26 |
27 | update_build_props_sbt () {
28 | local ver="$1"
29 | local old="$(build_props_sbt)"
30 |
31 | [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && {
32 | perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps"
33 | grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps"
34 |
35 | vlog "!!!"
36 | vlog "!!! Updated file $buildProps setting sbt.version to: $ver"
37 | vlog "!!! Previous value was: $old"
38 | vlog "!!!"
39 | }
40 | }
41 |
42 | set_sbt_version () {
43 | sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
44 | [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
45 | export sbt_version
46 | }
47 |
48 | # restore stty settings (echo in particular)
49 | onSbtRunnerExit() {
50 | [[ -n "$sbt_saved_stty" ]] || return
51 | vlog ""
52 | vlog "restoring stty: $sbt_saved_stty"
53 | stty "$sbt_saved_stty"
54 | unset sbt_saved_stty
55 | }
56 |
57 | # save stty and trap exit, to ensure echo is reenabled if we are interrupted.
58 | trap onSbtRunnerExit EXIT
59 | sbt_saved_stty="$(stty -g 2>/dev/null)"
60 | vlog "Saved stty: $sbt_saved_stty"
61 |
62 | # this seems to cover the bases on OSX, and someone will
63 | # have to tell me about the others.
64 | get_script_path () {
65 | local path="$1"
66 | [[ -L "$path" ]] || { echo "$path" ; return; }
67 |
68 | local target="$(readlink "$path")"
69 | if [[ "${target:0:1}" == "/" ]]; then
70 | echo "$target"
71 | else
72 | echo "${path%/*}/$target"
73 | fi
74 | }
75 |
76 | die() {
77 | echo "Aborting: $@"
78 | exit 1
79 | }
80 |
81 | url_base () {
82 | local version="$1"
83 |
84 | case "$version" in
85 | 0.7.*) echo "http://simple-build-tool.googlecode.com" ;;
86 | 0.10.* ) echo "$sbt_launch_release_repo" ;;
87 | 0.11.[12]) echo "$sbt_launch_release_repo" ;;
88 | *-[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]-[0-9][0-9][0-9][0-9][0-9][0-9]) # ie "*-yyyymmdd-hhMMss"
89 | echo "$sbt_launch_snapshot_repo" ;;
90 | *) echo "$sbt_launch_release_repo" ;;
91 | esac
92 | }
93 |
94 | make_url () {
95 | local version="$1"
96 |
97 | local base="${sbt_launch_repo:-$(url_base "$version")}"
98 |
99 | case "$version" in
100 | 0.7.*) echo "$base/files/sbt-launch-0.7.7.jar" ;;
101 | 0.10.* ) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
102 | 0.11.[12]) echo "$base/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
103 | *) echo "$base/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
104 | esac
105 | }
106 |
107 | init_default_option_file () {
108 | local overriding_var="${!1}"
109 | local default_file="$2"
110 | if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then
111 | local envvar_file="${BASH_REMATCH[1]}"
112 | if [[ -r "$envvar_file" ]]; then
113 | default_file="$envvar_file"
114 | fi
115 | fi
116 | echo "$default_file"
117 | }
118 |
119 | declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
120 | declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation"
121 | declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts"
122 | declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
123 | declare -r latest_28="2.8.2"
124 | declare -r latest_29="2.9.3"
125 | declare -r latest_210="2.10.6"
126 | declare -r latest_211="2.11.7"
127 | declare -r latest_212="2.12.0-M3"
128 | declare -r sbt_launch_release_repo="http://repo.typesafe.com/typesafe/ivy-releases"
129 | declare -r sbt_launch_snapshot_repo="https://repo.scala-sbt.org/scalasbt/ivy-snapshots"
130 |
131 | declare -r script_path="$(get_script_path "$BASH_SOURCE")"
132 | declare -r script_name="${script_path##*/}"
133 |
134 | # some non-read-onlies set with defaults
135 | declare java_cmd="java"
136 | declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
137 | declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
138 | declare sbt_launch_dir="$HOME/.sbt/launchers"
139 |
140 | declare sbt_launch_repo
141 |
142 | # pull -J and -D options to give to java.
143 | declare -a residual_args
144 | declare -a java_args
145 | declare -a scalac_args
146 | declare -a sbt_commands
147 |
148 | # args to jvm/sbt via files or environment variables
149 | declare -a extra_jvm_opts extra_sbt_opts
150 |
151 | addJava () {
152 | vlog "[addJava] arg = '$1'"
153 | java_args+=("$1")
154 | }
155 | addSbt () {
156 | vlog "[addSbt] arg = '$1'"
157 | sbt_commands+=("$1")
158 | }
159 | setThisBuild () {
160 | vlog "[addBuild] args = '$@'"
161 | local key="$1" && shift
162 | addSbt "set $key in ThisBuild := $@"
163 | }
164 | addScalac () {
165 | vlog "[addScalac] arg = '$1'"
166 | scalac_args+=("$1")
167 | }
168 | addResidual () {
169 | vlog "[residual] arg = '$1'"
170 | residual_args+=("$1")
171 | }
172 | addResolver () {
173 | addSbt "set resolvers += $1"
174 | }
175 | addDebugger () {
176 | addJava "-Xdebug"
177 | addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
178 | }
179 | setScalaVersion () {
180 | [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
181 | addSbt "++ $1"
182 | }
183 | setJavaHome () {
184 | java_cmd="$1/bin/java"
185 | setThisBuild javaHome "scala.Some(file(\"$1\"))"
186 | export JAVA_HOME="$1"
187 | export JDK_HOME="$1"
188 | export PATH="$JAVA_HOME/bin:$PATH"
189 | }
190 | setJavaHomeQuietly () {
191 | addSbt warn
192 | setJavaHome "$1"
193 | addSbt info
194 | }
195 |
196 | # if set, use JDK_HOME/JAVA_HOME over java found in path
197 | if [[ -e "$JDK_HOME/lib/tools.jar" ]]; then
198 | setJavaHomeQuietly "$JDK_HOME"
199 | elif [[ -e "$JAVA_HOME/bin/java" ]]; then
200 | setJavaHomeQuietly "$JAVA_HOME"
201 | fi
202 |
203 | # directory to store sbt launchers
204 | [[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
205 | [[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
206 |
207 | java_version () {
208 | local version=$("$java_cmd" -version 2>&1 | grep -E -e '(java|openjdk) version' | awk '{ print $3 }' | tr -d \")
209 | vlog "Detected Java version: $version"
210 | echo "${version:2:1}"
211 | }
212 |
213 | # MaxPermSize critical on pre-8 jvms but incurs noisy warning on 8+
214 | default_jvm_opts () {
215 | local v="$(java_version)"
216 | if [[ $v -ge 8 ]]; then
217 | echo "$default_jvm_opts_common"
218 | else
219 | echo "-XX:MaxPermSize=384m $default_jvm_opts_common"
220 | fi
221 | }
222 |
223 | build_props_scala () {
224 | if [[ -r "$buildProps" ]]; then
225 | versionLine="$(grep '^build.scala.versions' "$buildProps")"
226 | versionString="${versionLine##build.scala.versions=}"
227 | echo "${versionString%% .*}"
228 | fi
229 | }
230 |
231 | execRunner () {
232 | # print the arguments one to a line, quoting any containing spaces
233 | vlog "# Executing command line:" && {
234 | for arg; do
235 | if [[ -n "$arg" ]]; then
236 | if printf "%s\n" "$arg" | grep -q ' '; then
237 | printf >&2 "\"%s\"\n" "$arg"
238 | else
239 | printf >&2 "%s\n" "$arg"
240 | fi
241 | fi
242 | done
243 | vlog ""
244 | }
245 |
246 | [[ -n "$batch" ]] && exec /dev/null; then
268 | curl --fail --silent --location "$url" --output "$jar"
269 | elif which wget >/dev/null; then
270 | wget --quiet -O "$jar" "$url"
271 | fi
272 | } && [[ -r "$jar" ]]
273 | }
274 |
275 | acquire_sbt_jar () {
276 | local sbt_url="$(jar_url "$sbt_version")"
277 | sbt_jar="$(jar_file "$sbt_version")"
278 |
279 | [[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
280 | }
281 |
282 | usage () {
283 | set_sbt_version
284 | cat < display stack traces with a max of frames (default: -1, traces suppressed)
303 | -debug-inc enable debugging log for the incremental compiler
304 | -no-colors disable ANSI color codes
305 | -sbt-create start sbt even if current directory contains no sbt project
306 | -sbt-dir path to global settings/plugins directory (default: ~/.sbt/)
307 | -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11+)
308 | -ivy path to local Ivy repository (default: ~/.ivy2)
309 | -no-share use all local caches; no sharing
310 | -offline put sbt in offline mode
311 | -jvm-debug Turn on JVM debugging, open at the given port.
312 | -batch Disable interactive mode
313 | -prompt Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
314 |
315 | # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
316 | -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version
317 | -sbt-version use the specified version of sbt (default: $sbt_release_version)
318 | -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version
319 | -sbt-jar use the specified jar as the sbt launcher
320 | -sbt-launch-dir directory to hold sbt launchers (default: $sbt_launch_dir)
321 | -sbt-launch-repo repo url for downloading sbt launcher jar (default: $(url_base "$sbt_version"))
322 |
323 | # scala version (default: as chosen by sbt)
324 | -28 use $latest_28
325 | -29 use $latest_29
326 | -210 use $latest_210
327 | -211 use $latest_211
328 | -212 use $latest_212
329 | -scala-home use the scala build at the specified directory
330 | -scala-version use the specified version of scala
331 | -binary-version use the specified scala version when searching for dependencies
332 |
333 | # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
334 | -java-home alternate JAVA_HOME
335 |
336 | # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution
337 | # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found
338 | $(default_jvm_opts)
339 | JVM_OPTS environment variable holding either the jvm args directly, or
340 | the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')
341 | Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.
342 | -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present)
343 | -Dkey=val pass -Dkey=val directly to the jvm
344 | -J-X pass option -X directly to the jvm (-J is stripped)
345 |
346 | # passing options to sbt, OR to this runner
347 | SBT_OPTS environment variable holding either the sbt args directly, or
348 | the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')
349 | Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.
350 | -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present)
351 | -S-X add -X to sbt's scalacOptions (-S is stripped)
352 | EOM
353 | }
354 |
355 | process_args () {
356 | require_arg () {
357 | local type="$1"
358 | local opt="$2"
359 | local arg="$3"
360 |
361 | if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
362 | die "$opt requires <$type> argument"
363 | fi
364 | }
365 | while [[ $# -gt 0 ]]; do
366 | case "$1" in
367 | -h|-help) usage; exit 1 ;;
368 | -v) verbose=true && shift ;;
369 | -d) addSbt "--debug" && addSbt debug && shift ;;
370 | -w) addSbt "--warn" && addSbt warn && shift ;;
371 | -q) addSbt "--error" && addSbt error && shift ;;
372 | -x) debugUs=true && shift ;;
373 | -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
374 | -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
375 | -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
376 | -no-share) noshare=true && shift ;;
377 | -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
378 | -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
379 | -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
380 | -offline) addSbt "set offline := true" && shift ;;
381 | -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
382 | -batch) batch=true && shift ;;
383 | -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
384 |
385 | -sbt-create) sbt_create=true && shift ;;
386 | -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
387 | -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
388 | -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;;
389 | -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;;
390 | -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
391 | -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
392 | -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
393 | -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;;
394 | -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "scala.Some(file(\"$2\"))" && shift 2 ;;
395 | -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;;
396 | -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
397 | -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
398 |
399 | -D*) addJava "$1" && shift ;;
400 | -J*) addJava "${1:2}" && shift ;;
401 | -S*) addScalac "${1:2}" && shift ;;
402 | -28) setScalaVersion "$latest_28" && shift ;;
403 | -29) setScalaVersion "$latest_29" && shift ;;
404 | -210) setScalaVersion "$latest_210" && shift ;;
405 | -211) setScalaVersion "$latest_211" && shift ;;
406 | -212) setScalaVersion "$latest_212" && shift ;;
407 |
408 | --debug) addSbt debug && addResidual "$1" && shift ;;
409 | --warn) addSbt warn && addResidual "$1" && shift ;;
410 | --error) addSbt error && addResidual "$1" && shift ;;
411 | *) addResidual "$1" && shift ;;
412 | esac
413 | done
414 | }
415 |
416 | # process the direct command line arguments
417 | process_args "$@"
418 |
419 | # skip #-styled comments and blank lines
420 | readConfigFile() {
421 | local end=false
422 | until $end; do
423 | read || end=true
424 | [[ $REPLY =~ ^# ]] || [[ -z $REPLY ]] || echo "$REPLY"
425 | done < "$1"
426 | }
427 |
428 | # if there are file/environment sbt_opts, process again so we
429 | # can supply args to this runner
430 | if [[ -r "$sbt_opts_file" ]]; then
431 | vlog "Using sbt options defined in file $sbt_opts_file"
432 | while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file")
433 | elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then
434 | vlog "Using sbt options defined in variable \$SBT_OPTS"
435 | extra_sbt_opts=( $SBT_OPTS )
436 | else
437 | vlog "No extra sbt options have been defined"
438 | fi
439 |
440 | [[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}"
441 |
442 | # reset "$@" to the residual args
443 | set -- "${residual_args[@]}"
444 | argumentCount=$#
445 |
446 | # set sbt version
447 | set_sbt_version
448 |
449 | # only exists in 0.12+
450 | setTraceLevel() {
451 | case "$sbt_version" in
452 | "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;;
453 | *) setThisBuild traceLevel $trace_level ;;
454 | esac
455 | }
456 |
457 | # set scalacOptions if we were given any -S opts
458 | [[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\""
459 |
460 | # Update build.properties on disk to set explicit version - sbt gives us no choice
461 | [[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
462 | vlog "Detected sbt version $sbt_version"
463 |
464 | [[ -n "$scala_version" ]] && vlog "Overriding scala version to $scala_version"
465 |
466 | # no args - alert them there's stuff in here
467 | (( argumentCount > 0 )) || {
468 | vlog "Starting $script_name: invoke with -help for other options"
469 | residual_args=( shell )
470 | }
471 |
472 | # verify this is an sbt dir or -create was given
473 | [[ -r ./build.sbt || -d ./project || -n "$sbt_create" ]] || {
474 | cat <