├── project
    ├── build.properties
    ├── plugins.sbt
    └── ApplicationBuild.scala
├── .gitignore
├── .travis.yml
├── src
    └── main
    │   ├── resources
    │       └── logback.xml
    │   └── scala
    │       └── example
    │           └── flink
    │               ├── WordCount.scala
    │               └── FlinkExample.scala
├── README.md
├── scalastyle-config.xml
└── sbt


/project/build.properties:
--------------------------------------------------------------------------------
1 | sbt.version=0.13.8
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .idea/
2 | *.iml
3 | 
4 | target/
5 | project/project/
6 | 


--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
 1 | language: scala
 2 | 
 3 | scala:
 4 |   - 2.11.7
 5 | 
 6 | jdk:
 7 |   - oraclejdk8
 8 | 
 9 | script:
10 |   - sbt ++$TRAVIS_SCALA_VERSION clean coverage test scalastyle
11 | 
12 | after_success:
13 |   - sbt coveralls
14 | 


--------------------------------------------------------------------------------
/src/main/resources/logback.xml:
--------------------------------------------------------------------------------
 1 | <configuration>
 2 |   <appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
 3 |     <encoder>
 4 |       <pattern>%d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n</pattern>
 5 |     </encoder>
 6 |   </appender>
 7 | 
 8 |   <logger name="example" level="DEBUG"/>
 9 |   <logger name="org.apache.flink" level="INFO"/>
10 | 
11 |   <root level="WARN">
12 |     <appender-ref ref="STDOUT"/>
13 |   </root>
14 | </configuration>


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | Start ZooKeeper server:
 2 | 
 3 | ```
 4 | ./bin/zookeeper-server-start.sh config/zookeeper.properties
 5 | ```
 6 | 
 7 | Start Kafka server:
 8 | 
 9 | ```
10 | ./bin/kafka-server-start.sh config/server.properties
11 | ```
12 | 
13 | Create input topic:
14 | 
15 | ```
16 | ./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic input
17 | ```
18 | 
19 | Create output topic:
20 | 
21 | ```
22 | ./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic output
23 | ```
24 | 
25 | Start kafka producer:
26 | 
27 | ```
28 | ./bin/kafka-console-producer.sh --broker-list localhost:9092 --topic input
29 | ```
30 | 
31 | Start kafka consumer:
32 | 
33 | ```
34 | ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic output
35 | ```
36 | 


--------------------------------------------------------------------------------
/project/plugins.sbt:
--------------------------------------------------------------------------------
 1 | // Copyright (C) 2011-2012 the original author or authors.
 2 | // See the LICENCE.txt file distributed with this work for additional
 3 | // information regarding copyright ownership.
 4 | //
 5 | // Licensed under the Apache License, Version 2.0 (the "License");
 6 | // you may not use this file except in compliance with the License.
 7 | // You may obtain a copy of the License at
 8 | //
 9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 | 
17 | addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.6.0")
18 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4")
19 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0.BETA1")
20 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0")
21 | 


--------------------------------------------------------------------------------
/src/main/scala/example/flink/WordCount.scala:
--------------------------------------------------------------------------------
 1 | // Copyright (C) 2011-2012 the original author or authors.
 2 | // See the LICENCE.txt file distributed with this work for additional
 3 | // information regarding copyright ownership.
 4 | //
 5 | // Licensed under the Apache License, Version 2.0 (the "License");
 6 | // you may not use this file except in compliance with the License.
 7 | // You may obtain a copy of the License at
 8 | //
 9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 | 
17 | package example.flink
18 | 
19 | import org.apache.flink.api.scala._
20 | import org.apache.flink.streaming.api.scala.DataStream
21 | import org.apache.flink.streaming.api.windowing.time.Time
22 | 
23 | object WordCount {
24 | 
25 |   type WordCount = (String, Int)
26 | 
27 |   def countWords(lines: DataStream[String], stopWords: Set[String], window: Time): DataStream[WordCount] = {
28 |     lines
29 |       .flatMap(line => line.split(" "))
30 |       .filter(word => !word.isEmpty)
31 |       .map(word => word.toLowerCase)
32 |       .filter(word => !stopWords.contains(word))
33 |       .map(word => (word, 1))
34 |       .keyBy(0)
35 |       .timeWindow(window)
36 |       .sum(1)
37 |   }
38 | 
39 | }
40 | 


--------------------------------------------------------------------------------
/src/main/scala/example/flink/FlinkExample.scala:
--------------------------------------------------------------------------------
 1 | // Copyright (C) 2011-2012 the original author or authors.
 2 | // See the LICENCE.txt file distributed with this work for additional
 3 | // information regarding copyright ownership.
 4 | //
 5 | // Licensed under the Apache License, Version 2.0 (the "License");
 6 | // you may not use this file except in compliance with the License.
 7 | // You may obtain a copy of the License at
 8 | //
 9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 | 
17 | package example.flink
18 | 
19 | import java.util.concurrent.TimeUnit
20 | 
21 | import org.apache.flink.api.scala._
22 | import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment
23 | import org.apache.flink.streaming.api.windowing.time.Time
24 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.{FetcherType, OffsetStore}
25 | import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer}
26 | import org.apache.flink.streaming.util.serialization.{DeserializationSchema, SerializationSchema}
27 | 
28 | object FlinkExample {
29 | 
30 |   import WordCount._
31 | 
32 |   val stopWords = Set("a", "an", "the")
33 |   val window = Time.of(10, TimeUnit.SECONDS)
34 | 
35 |   def main(args: Array[String]): Unit = {
36 |     val env = StreamExecutionEnvironment.createLocalEnvironment()
37 | 
38 |     val kafkaConsumerProperties = Map(
39 |       "zookeeper.connect" -> "localhost:2181",
40 |       "group.id" -> "flink",
41 |       "bootstrap.servers" -> "localhost:9092"
42 |     )
43 | 
44 |     val kafkaConsumer = new FlinkKafkaConsumer[String](
45 |       "input",
46 |       KafkaStringSchema,
47 |       kafkaConsumerProperties,
48 |       OffsetStore.FLINK_ZOOKEEPER,
49 |       FetcherType.LEGACY_LOW_LEVEL
50 |     )
51 | 
52 |     val kafkaProducer = new FlinkKafkaProducer[String](
53 |       "localhost:9092",
54 |       "output",
55 |       KafkaStringSchema
56 |     )
57 | 
58 |     val lines = env.addSource(kafkaConsumer)
59 | 
60 |     val wordCounts = countWords(lines, stopWords, window)
61 | 
62 |     wordCounts
63 |       .map(_.toString)
64 |       .addSink(kafkaProducer)
65 | 
66 |     env.execute()
67 |   }
68 | 
69 |   implicit def map2Properties(map: Map[String, String]): java.util.Properties = {
70 |     (new java.util.Properties /: map) { case (props, (k, v)) => props.put(k, v); props }
71 |   }
72 | 
73 |   object KafkaStringSchema extends SerializationSchema[String, Array[Byte]] with DeserializationSchema[String] {
74 | 
75 |     import org.apache.flink.api.common.typeinfo.TypeInformation
76 |     import org.apache.flink.api.java.typeutils.TypeExtractor
77 | 
78 |     override def serialize(t: String): Array[Byte] = t.getBytes("UTF-8")
79 | 
80 |     override def isEndOfStream(t: String): Boolean = false
81 | 
82 |     override def deserialize(bytes: Array[Byte]): String = new String(bytes, "UTF-8")
83 | 
84 |     override def getProducedType: TypeInformation[String] = TypeExtractor.getForClass(classOf[String])
85 |   }
86 | 
87 | }
88 | 


--------------------------------------------------------------------------------
/project/ApplicationBuild.scala:
--------------------------------------------------------------------------------
 1 | // Copyright (C) 2011-2012 the original author or authors.
 2 | // See the LICENCE.txt file distributed with this work for additional
 3 | // information regarding copyright ownership.
 4 | //
 5 | // Licensed under the Apache License, Version 2.0 (the "License");
 6 | // you may not use this file except in compliance with the License.
 7 | // You may obtain a copy of the License at
 8 | //
 9 | // http://www.apache.org/licenses/LICENSE-2.0
10 | //
11 | // Unless required by applicable law or agreed to in writing, software
12 | // distributed under the License is distributed on an "AS IS" BASIS,
13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 | // See the License for the specific language governing permissions and
15 | // limitations under the License.
16 | 
17 | import sbt.Keys._
18 | import sbt._
19 | 
20 | object ApplicationBuild extends Build {
21 | 
22 |   object Versions {
23 |     val flink = "0.10.1"
24 |     val kafka = "0.8.2.1"
25 |   }
26 | 
27 |   val projectName = "example-flink-kafka"
28 | 
29 |   val commonSettings = Seq(
30 |     version := "1.0",
31 |     organization := "http://mkuthan.github.io/",
32 |     scalaVersion := "2.11.7"
33 |   )
34 | 
35 |   val commonScalacOptions = Seq(
36 |     "-deprecation",
37 |     "-encoding", "UTF-8",
38 |     "-feature",
39 |     "-language:existentials",
40 |     "-language:higherKinds",
41 |     "-language:implicitConversions",
42 |     "-unchecked",
43 |     "-Xfatal-warnings",
44 |     "-Xfuture",
45 |     "-Xlint",
46 |     "-Yno-adapted-args",
47 |     "-Ywarn-dead-code",
48 |     "-Ywarn-numeric-widen",
49 |     "-Ywarn-unused-import"
50 |   )
51 | 
52 |   val commonJavaInRuntimeOptions = Seq(
53 |     "-Xmx512m"
54 |   )
55 | 
56 |   val commonJavaInTestOptions = Seq(
57 |     "-Xmx512m"
58 |   )
59 | 
60 |   val commonResolvers = Seq(
61 |     "Apache Snapshots" at "http://repository.apache.org/snapshots/"
62 |   )
63 | 
64 |   val commonLibraryDependencies = Seq(
65 |     "org.apache.flink" %% "flink-core" % Versions.flink,
66 |     "org.apache.flink" %% "flink-scala" % Versions.flink,
67 |     "org.apache.flink" %% "flink-streaming-core" % Versions.flink,
68 |     "org.apache.flink" %% "flink-streaming-scala" % Versions.flink,
69 |     "org.apache.flink" %% "flink-connector-kafka" % Versions.flink,
70 | 
71 |     //"org.apache.kafka" %% "kafka" % Versions.kafka,
72 |     //"org.apache.kafka" % "kafka-clients" % Versions.kafka,
73 | 
74 |     "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0",
75 | 
76 |     "org.slf4j" % "slf4j-api" % "1.7.10",
77 |     "ch.qos.logback" % "logback-classic" % "1.1.2",
78 | 
79 |     "org.scalatest" %% "scalatest" % "2.2.4" % "test"
80 |   )
81 | 
82 |   val commonExcludeDependencies = Seq(
83 |     "org.slf4j" % "slf4j-log4j12"
84 |   )
85 | 
86 |   lazy val main = Project(projectName, base = file("."))
87 |     .settings(commonSettings)
88 |     .settings(javaOptions in Runtime ++= commonJavaInRuntimeOptions)
89 |     .settings(javaOptions in Test ++= commonJavaInTestOptions)
90 |     .settings(scalacOptions ++= commonScalacOptions)
91 |     .settings(resolvers ++= commonResolvers)
92 |     .settings(libraryDependencies ++= commonLibraryDependencies)
93 |     .settings(excludeDependencies ++= commonExcludeDependencies)
94 |     .settings(fork in run := true)
95 | 
96 | }
97 | 
98 | 


--------------------------------------------------------------------------------
/scalastyle-config.xml:
--------------------------------------------------------------------------------
  1 | <scalastyle>
  2 |  <name>Scalastyle standard configuration</name>
  3 |  <check level="warning" class="org.scalastyle.file.FileTabChecker" enabled="true"></check>
  4 |  <check level="warning" class="org.scalastyle.file.FileLengthChecker" enabled="true">
  5 |   <parameters>
  6 |    <parameter name="maxFileLength"><![CDATA[800]]></parameter>
  7 |   </parameters>
  8 |  </check>
  9 |  <check level="warning" class="org.scalastyle.file.HeaderMatchesChecker" enabled="true">
 10 |   <parameters>
 11 |    <parameter name="header"><![CDATA[// Copyright (C) 2011-2012 the original author or authors.
 12 | // See the LICENCE.txt file distributed with this work for additional
 13 | // information regarding copyright ownership.
 14 | //
 15 | // Licensed under the Apache License, Version 2.0 (the "License");
 16 | // you may not use this file except in compliance with the License.
 17 | // You may obtain a copy of the License at
 18 | //
 19 | // http://www.apache.org/licenses/LICENSE-2.0
 20 | //
 21 | // Unless required by applicable law or agreed to in writing, software
 22 | // distributed under the License is distributed on an "AS IS" BASIS,
 23 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 24 | // See the License for the specific language governing permissions and
 25 | // limitations under the License.]]></parameter>
 26 |   </parameters>
 27 |  </check>
 28 |  <check level="warning" class="org.scalastyle.scalariform.SpacesAfterPlusChecker" enabled="true"></check>
 29 |  <check level="warning" class="org.scalastyle.file.WhitespaceEndOfLineChecker" enabled="true"></check>
 30 |  <check level="warning" class="org.scalastyle.scalariform.SpacesBeforePlusChecker" enabled="true"></check>
 31 |  <check level="warning" class="org.scalastyle.file.FileLineLengthChecker" enabled="true">
 32 |   <parameters>
 33 |    <parameter name="maxLineLength"><![CDATA[160]]></parameter>
 34 |    <parameter name="tabSize"><![CDATA[4]]></parameter>
 35 |   </parameters>
 36 |  </check>
 37 |  <check level="warning" class="org.scalastyle.scalariform.ClassNamesChecker" enabled="true">
 38 |   <parameters>
 39 |    <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
 40 |   </parameters>
 41 |  </check>
 42 |  <check level="warning" class="org.scalastyle.scalariform.ObjectNamesChecker" enabled="true">
 43 |   <parameters>
 44 |    <parameter name="regex"><![CDATA[[A-Z][A-Za-z]*]]></parameter>
 45 |   </parameters>
 46 |  </check>
 47 |  <check level="warning" class="org.scalastyle.scalariform.PackageObjectNamesChecker" enabled="true">
 48 |   <parameters>
 49 |    <parameter name="regex"><![CDATA[^[a-z][A-Za-z]*$]]></parameter>
 50 |   </parameters>
 51 |  </check>
 52 |  <check level="warning" class="org.scalastyle.scalariform.EqualsHashCodeChecker" enabled="true"></check>
 53 |  <check level="warning" class="org.scalastyle.scalariform.IllegalImportsChecker" enabled="true">
 54 |   <parameters>
 55 |    <parameter name="illegalImports"><![CDATA[sun._,java.awt._]]></parameter>
 56 |   </parameters>
 57 |  </check>
 58 |  <check level="warning" class="org.scalastyle.scalariform.ParameterNumberChecker" enabled="true">
 59 |   <parameters>
 60 |    <parameter name="maxParameters"><![CDATA[8]]></parameter>
 61 |   </parameters>
 62 |  </check>
 63 |  <check level="warning" class="org.scalastyle.scalariform.MagicNumberChecker" enabled="true">
 64 |   <parameters>
 65 |    <parameter name="ignore"><![CDATA[-1,0,1,2,3]]></parameter>
 66 |   </parameters>
 67 |  </check>
 68 |  <check level="warning" class="org.scalastyle.scalariform.NoWhitespaceBeforeLeftBracketChecker" enabled="true"></check>
 69 |  <check level="warning" class="org.scalastyle.scalariform.NoWhitespaceAfterLeftBracketChecker" enabled="true"></check>
 70 |  <check level="warning" class="org.scalastyle.scalariform.ReturnChecker" enabled="true"></check>
 71 |  <check level="warning" class="org.scalastyle.scalariform.NullChecker" enabled="true"></check>
 72 |  <check level="warning" class="org.scalastyle.scalariform.NoCloneChecker" enabled="true"></check>
 73 |  <check level="warning" class="org.scalastyle.scalariform.NoFinalizeChecker" enabled="true"></check>
 74 |  <check level="warning" class="org.scalastyle.scalariform.CovariantEqualsChecker" enabled="true"></check>
 75 |  <check level="warning" class="org.scalastyle.scalariform.StructuralTypeChecker" enabled="true"></check>
 76 |  <check level="warning" class="org.scalastyle.file.RegexChecker" enabled="true">
 77 |   <parameters>
 78 |    <parameter name="regex"><![CDATA[println]]></parameter>
 79 |   </parameters>
 80 |  </check>
 81 |  <check level="warning" class="org.scalastyle.scalariform.NumberOfTypesChecker" enabled="true">
 82 |   <parameters>
 83 |    <parameter name="maxTypes"><![CDATA[30]]></parameter>
 84 |   </parameters>
 85 |  </check>
 86 |  <check level="warning" class="org.scalastyle.scalariform.CyclomaticComplexityChecker" enabled="true">
 87 |   <parameters>
 88 |    <parameter name="maximum"><![CDATA[10]]></parameter>
 89 |   </parameters>
 90 |  </check>
 91 |  <check level="warning" class="org.scalastyle.scalariform.UppercaseLChecker" enabled="true"></check>
 92 |  <check level="warning" class="org.scalastyle.scalariform.SimplifyBooleanExpressionChecker" enabled="true"></check>
 93 |  <check level="warning" class="org.scalastyle.scalariform.IfBraceChecker" enabled="true">
 94 |   <parameters>
 95 |    <parameter name="singleLineAllowed"><![CDATA[true]]></parameter>
 96 |    <parameter name="doubleLineAllowed"><![CDATA[false]]></parameter>
 97 |   </parameters>
 98 |  </check>
 99 |  <check level="warning" class="org.scalastyle.scalariform.MethodLengthChecker" enabled="true">
100 |   <parameters>
101 |    <parameter name="maxLength"><![CDATA[50]]></parameter>
102 |   </parameters>
103 |  </check>
104 |  <check level="warning" class="org.scalastyle.scalariform.MethodNamesChecker" enabled="true">
105 |   <parameters>
106 |    <parameter name="regex"><![CDATA[^[a-z][A-Za-z0-9]*$]]></parameter>
107 |   </parameters>
108 |  </check>
109 |  <check level="warning" class="org.scalastyle.scalariform.NumberOfMethodsInTypeChecker" enabled="true">
110 |   <parameters>
111 |    <parameter name="maxMethods"><![CDATA[30]]></parameter>
112 |   </parameters>
113 |  </check>
114 |  <check level="warning" class="org.scalastyle.scalariform.PublicMethodsHaveTypeChecker" enabled="true"></check>
115 |  <check level="warning" class="org.scalastyle.file.NewLineAtEofChecker" enabled="true"></check>
116 |  <check level="warning" class="org.scalastyle.file.NoNewLineAtEofChecker" enabled="false"></check>
117 | </scalastyle>


--------------------------------------------------------------------------------
/sbt:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env bash
  2 | #
  3 | # A more capable sbt runner, coincidentally also called sbt.
  4 | # Author: Paul Phillips <paulp@improving.org>
  5 | 
  6 | # todo - make this dynamic
  7 | declare -r sbt_release_version="0.13.8"
  8 | declare -r sbt_unreleased_version="0.13.8"
  9 | declare -r buildProps="project/build.properties"
 10 | 
 11 | declare sbt_jar sbt_dir sbt_create sbt_version
 12 | declare scala_version sbt_explicit_version
 13 | declare verbose noshare batch trace_level log_level
 14 | declare sbt_saved_stty debugUs
 15 | 
 16 | echoerr () { echo >&2 "$@"; }
 17 | vlog ()    { [[ -n "$verbose" ]] && echoerr "$@"; }
 18 | 
 19 | # spaces are possible, e.g. sbt.version = 0.13.0
 20 | build_props_sbt () {
 21 |   [[ -r "$buildProps" ]] && \
 22 |     grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }'
 23 | }
 24 | 
 25 | update_build_props_sbt () {
 26 |   local ver="$1"
 27 |   local old="$(build_props_sbt)"
 28 | 
 29 |   [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && {
 30 |     perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps"
 31 |     grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps"
 32 | 
 33 |     vlog "!!!"
 34 |     vlog "!!! Updated file $buildProps setting sbt.version to: $ver"
 35 |     vlog "!!! Previous value was: $old"
 36 |     vlog "!!!"
 37 |   }
 38 | }
 39 | 
 40 | set_sbt_version () {
 41 |   sbt_version="${sbt_explicit_version:-$(build_props_sbt)}"
 42 |   [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version
 43 |   export sbt_version
 44 | }
 45 | 
 46 | # restore stty settings (echo in particular)
 47 | onSbtRunnerExit() {
 48 |   [[ -n "$sbt_saved_stty" ]] || return
 49 |   vlog ""
 50 |   vlog "restoring stty: $sbt_saved_stty"
 51 |   stty "$sbt_saved_stty"
 52 |   unset sbt_saved_stty
 53 | }
 54 | 
 55 | # save stty and trap exit, to ensure echo is reenabled if we are interrupted.
 56 | trap onSbtRunnerExit EXIT
 57 | sbt_saved_stty="$(stty -g 2>/dev/null)"
 58 | vlog "Saved stty: $sbt_saved_stty"
 59 | 
 60 | # this seems to cover the bases on OSX, and someone will
 61 | # have to tell me about the others.
 62 | get_script_path () {
 63 |   local path="$1"
 64 |   [[ -L "$path" ]] || { echo "$path" ; return; }
 65 | 
 66 |   local target="$(readlink "$path")"
 67 |   if [[ "${target:0:1}" == "/" ]]; then
 68 |     echo "$target"
 69 |   else
 70 |     echo "${path%/*}/$target"
 71 |   fi
 72 | }
 73 | 
 74 | die() {
 75 |   echo "Aborting: $@"
 76 |   exit 1
 77 | }
 78 | 
 79 | make_url () {
 80 |   version="$1"
 81 | 
 82 |   case "$version" in
 83 |         0.7.*) echo "http://simple-build-tool.googlecode.com/files/sbt-launch-0.7.7.jar" ;;
 84 |       0.10.* ) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
 85 |     0.11.[12]) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;;
 86 |             *) echo "$sbt_launch_repo/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;;
 87 |   esac
 88 | }
 89 | 
 90 | init_default_option_file () {
 91 |   local overriding_var="${!1}"
 92 |   local default_file="$2"
 93 |   if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then
 94 |     local envvar_file="${BASH_REMATCH[1]}"
 95 |     if [[ -r "$envvar_file" ]]; then
 96 |       default_file="$envvar_file"
 97 |     fi
 98 |   fi
 99 |   echo "$default_file"
100 | }
101 | 
102 | declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC"
103 | declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation"
104 | declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts"
105 | declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy"
106 | declare -r latest_28="2.8.2"
107 | declare -r latest_29="2.9.3"
108 | declare -r latest_210="2.10.5"
109 | declare -r latest_211="2.11.6"
110 | 
111 | declare -r script_path="$(get_script_path "$BASH_SOURCE")"
112 | declare -r script_name="${script_path##*/}"
113 | 
114 | # some non-read-onlies set with defaults
115 | declare java_cmd="java"
116 | declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)"
117 | declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)"
118 | declare sbt_launch_repo="http://typesafe.artifactoryonline.com/typesafe/ivy-releases"
119 | 
120 | # pull -J and -D options to give to java.
121 | declare -a residual_args
122 | declare -a java_args
123 | declare -a scalac_args
124 | declare -a sbt_commands
125 | 
126 | # args to jvm/sbt via files or environment variables
127 | declare -a extra_jvm_opts extra_sbt_opts
128 | 
129 | addJava () {
130 |   vlog "[addJava] arg = '$1'"
131 |   java_args=( "${java_args[@]}" "$1" )
132 | }
133 | addSbt () {
134 |   vlog "[addSbt] arg = '$1'"
135 |   sbt_commands=( "${sbt_commands[@]}" "$1" )
136 | }
137 | setThisBuild () {
138 |   vlog "[addBuild] args = '$@'"
139 |   local key="$1" && shift
140 |   addSbt "set $key in ThisBuild := $@"
141 | }
142 | addScalac () {
143 |   vlog "[addScalac] arg = '$1'"
144 |   scalac_args=( "${scalac_args[@]}" "$1" )
145 | }
146 | addResidual () {
147 |   vlog "[residual] arg = '$1'"
148 |   residual_args=( "${residual_args[@]}" "$1" )
149 | }
150 | addResolver () {
151 |   addSbt "set resolvers += $1"
152 | }
153 | addDebugger () {
154 |   addJava "-Xdebug"
155 |   addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1"
156 | }
157 | setScalaVersion () {
158 |   [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")'
159 |   addSbt "++ $1"
160 | }
161 | setJavaHome () {
162 |   java_cmd="$1/bin/java"
163 |   setThisBuild javaHome "Some(file(\"$1\"))"
164 |   export JAVA_HOME="$1"
165 |   export JDK_HOME="$1"
166 |   export PATH="$JAVA_HOME/bin:$PATH"
167 | }
168 | setJavaHomeQuietly () {
169 |   java_cmd="$1/bin/java"
170 |   addSbt ";warn ;set javaHome in ThisBuild := Some(file(\"$1\")) ;info"
171 |   export JAVA_HOME="$1"
172 |   export JDK_HOME="$1"
173 |   export PATH="$JAVA_HOME/bin:$PATH"
174 | }
175 | 
176 | # if set, use JDK_HOME/JAVA_HOME over java found in path
177 | if [[ -e "$JDK_HOME/lib/tools.jar" ]]; then
178 |   setJavaHomeQuietly "$JDK_HOME"
179 | elif [[ -e "$JAVA_HOME/bin/java" ]]; then
180 |   setJavaHomeQuietly "$JAVA_HOME"
181 | fi
182 | 
183 | # directory to store sbt launchers
184 | declare sbt_launch_dir="$HOME/.sbt/launchers"
185 | [[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir"
186 | [[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)"
187 | 
188 | java_version () {
189 |   local version=$("$java_cmd" -version 2>&1 | grep -e 'java version' | awk '{ print $3 }' | tr -d \")
190 |   vlog "Detected Java version: $version"
191 |   echo "${version:2:1}"
192 | }
193 | 
194 | # MaxPermSize critical on pre-8 jvms but incurs noisy warning on 8+
195 | default_jvm_opts () {
196 |   local v="$(java_version)"
197 |   if [[ $v -ge 8 ]]; then
198 |     echo "$default_jvm_opts_common"
199 |   else
200 |     echo "-XX:MaxPermSize=384m $default_jvm_opts_common"
201 |   fi
202 | }
203 | 
204 | build_props_scala () {
205 |   if [[ -r "$buildProps" ]]; then
206 |     versionLine="$(grep '^build.scala.versions' "$buildProps")"
207 |     versionString="${versionLine##build.scala.versions=}"
208 |     echo "${versionString%% .*}"
209 |   fi
210 | }
211 | 
212 | execRunner () {
213 |   # print the arguments one to a line, quoting any containing spaces
214 |   vlog "# Executing command line:" && {
215 |     for arg; do
216 |       if [[ -n "$arg" ]]; then
217 |         if printf "%s\n" "$arg" | grep -q ' '; then
218 |           printf >&2 "\"%s\"\n" "$arg"
219 |         else
220 |           printf >&2 "%s\n" "$arg"
221 |         fi
222 |       fi
223 |     done
224 |     vlog ""
225 |   }
226 | 
227 |   [[ -n "$batch" ]] && exec </dev/null
228 |   exec "$@"
229 | }
230 | 
231 | jar_url () {
232 |   make_url "$1"
233 | }
234 | 
235 | jar_file () {
236 |   echo "$sbt_launch_dir/$1/sbt-launch.jar"
237 | }
238 | 
239 | download_url () {
240 |   local url="$1"
241 |   local jar="$2"
242 | 
243 |   echoerr "Downloading sbt launcher for $sbt_version:"
244 |   echoerr "  From  $url"
245 |   echoerr "    To  $jar"
246 | 
247 |   mkdir -p "${jar%/*}" && {
248 |     if which curl >/dev/null; then
249 |       curl --fail --silent "$url" --output "$jar"
250 |     elif which wget >/dev/null; then
251 |       wget --quiet -O "$jar" "$url"
252 |     fi
253 |   } && [[ -r "$jar" ]]
254 | }
255 | 
256 | acquire_sbt_jar () {
257 |   sbt_url="$(jar_url "$sbt_version")"
258 |   sbt_jar="$(jar_file "$sbt_version")"
259 | 
260 |   [[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar"
261 | }
262 | 
263 | usage () {
264 |   cat <<EOM
265 | Usage: $script_name [options]
266 | 
267 | Note that options which are passed along to sbt begin with -- whereas
268 | options to this runner use a single dash. Any sbt command can be scheduled
269 | to run first by prefixing the command with --, so --warn, --error and so on
270 | are not special.
271 | 
272 | Output filtering: if there is a file in the home directory called .sbtignore
273 | and this is not an interactive sbt session, the file is treated as a list of
274 | bash regular expressions. Output lines which match any regex are not echoed.
275 | One can see exactly which lines would have been suppressed by starting this
276 | runner with the -x option.
277 | 
278 |   -h | -help         print this message
279 |   -v                 verbose operation (this runner is chattier)
280 |   -d, -w, -q         aliases for --debug, --warn, --error (q means quiet)
281 |   -x                 debug this script
282 |   -trace <level>     display stack traces with a max of <level> frames (default: -1, traces suppressed)
283 |   -debug-inc         enable debugging log for the incremental compiler
284 |   -no-colors         disable ANSI color codes
285 |   -sbt-create        start sbt even if current directory contains no sbt project
286 |   -sbt-dir   <path>  path to global settings/plugins directory (default: ~/.sbt/<version>)
287 |   -sbt-boot  <path>  path to shared boot directory (default: ~/.sbt/boot in 0.11+)
288 |   -ivy       <path>  path to local Ivy repository (default: ~/.ivy2)
289 |   -no-share          use all local caches; no sharing
290 |   -offline           put sbt in offline mode
291 |   -jvm-debug <port>  Turn on JVM debugging, open at the given port.
292 |   -batch             Disable interactive mode
293 |   -prompt <expr>     Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted
294 | 
295 |   # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version)
296 |   -sbt-force-latest         force the use of the latest release of sbt: $sbt_release_version
297 |   -sbt-version  <version>   use the specified version of sbt (default: $sbt_release_version)
298 |   -sbt-dev                  use the latest pre-release version of sbt: $sbt_unreleased_version
299 |   -sbt-jar      <path>      use the specified jar as the sbt launcher
300 |   -sbt-launch-dir <path>    directory to hold sbt launchers (default: ~/.sbt/launchers)
301 |   -sbt-launch-repo <url>    repo url for downloading sbt launcher jar (default: $sbt_launch_repo)
302 | 
303 |   # scala version (default: as chosen by sbt)
304 |   -28                       use $latest_28
305 |   -29                       use $latest_29
306 |   -210                      use $latest_210
307 |   -211                      use $latest_211
308 |   -scala-home <path>        use the scala build at the specified directory
309 |   -scala-version <version>  use the specified version of scala
310 |   -binary-version <version> use the specified scala version when searching for dependencies
311 | 
312 |   # java version (default: java from PATH, currently $(java -version 2>&1 | grep version))
313 |   -java-home <path>         alternate JAVA_HOME
314 | 
315 |   # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution
316 |   # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found
317 |   <default>        $(default_jvm_opts)
318 |   JVM_OPTS         environment variable holding either the jvm args directly, or
319 |                    the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts')
320 |                    Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument.
321 |   -jvm-opts <path> file containing jvm args (if not given, .jvmopts in project root is used if present)
322 |   -Dkey=val        pass -Dkey=val directly to the jvm
323 |   -J-X             pass option -X directly to the jvm (-J is stripped)
324 | 
325 |   # passing options to sbt, OR to this runner
326 |   SBT_OPTS         environment variable holding either the sbt args directly, or
327 |                    the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts')
328 |                    Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument.
329 |   -sbt-opts <path> file containing sbt args (if not given, .sbtopts in project root is used if present)
330 |   -S-X             add -X to sbt's scalacOptions (-S is stripped)
331 | EOM
332 | }
333 | 
334 | process_args ()
335 | {
336 |   require_arg () {
337 |     local type="$1"
338 |     local opt="$2"
339 |     local arg="$3"
340 | 
341 |     if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then
342 |       die "$opt requires <$type> argument"
343 |     fi
344 |   }
345 |   while [[ $# -gt 0 ]]; do
346 |     case "$1" in
347 |           -h|-help) usage; exit 1 ;;
348 |                 -v) verbose=true && shift ;;
349 |                 -d) addSbt "--debug" && addSbt debug && shift ;;
350 |                 -w) addSbt "--warn"  && addSbt warn  && shift ;;
351 |                 -q) addSbt "--error" && addSbt error && shift ;;
352 |                 -x) debugUs=true && shift ;;
353 |             -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;;
354 |               -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;;
355 |         -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;;
356 |          -no-share) noshare=true && shift ;;
357 |          -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;;
358 |           -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;;
359 |         -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;;
360 |           -offline) addSbt "set offline := true" && shift ;;
361 |         -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;;
362 |             -batch) batch=true && shift ;;
363 |            -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;;
364 | 
365 |        -sbt-create) sbt_create=true && shift ;;
366 |           -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;;
367 |       -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;;
368 |  -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;;
369 |           -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;;
370 |    -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;;
371 |   -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;;
372 |     -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;;
373 |    -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;;
374 |        -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "Some(file(\"$2\"))" && shift 2 ;;
375 |         -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;;
376 |          -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;;
377 |          -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;;
378 | 
379 |                -D*) addJava "$1" && shift ;;
380 |                -J*) addJava "${1:2}" && shift ;;
381 |                -S*) addScalac "${1:2}" && shift ;;
382 |                -28) setScalaVersion "$latest_28" && shift ;;
383 |                -29) setScalaVersion "$latest_29" && shift ;;
384 |               -210) setScalaVersion "$latest_210" && shift ;;
385 |               -211) setScalaVersion "$latest_211" && shift ;;
386 | 
387 |            --debug) addSbt debug && addResidual "$1" && shift ;;
388 |             --warn) addSbt warn  && addResidual "$1" && shift ;;
389 |            --error) addSbt error && addResidual "$1" && shift ;;
390 |                  *) addResidual "$1" && shift ;;
391 |     esac
392 |   done
393 | }
394 | 
395 | # process the direct command line arguments
396 | process_args "$@"
397 | 
398 | # skip #-styled comments and blank lines
399 | readConfigFile() {
400 |   while read line; do
401 |     [[ $line =~ ^# ]] || [[ -z $line ]] || echo "$line"
402 |   done < "$1"
403 | }
404 | 
405 | # if there are file/environment sbt_opts, process again so we
406 | # can supply args to this runner
407 | if [[ -r "$sbt_opts_file" ]]; then
408 |   vlog "Using sbt options defined in file $sbt_opts_file"
409 |   while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file")
410 | elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then
411 |   vlog "Using sbt options defined in variable \$SBT_OPTS"
412 |   extra_sbt_opts=( $SBT_OPTS )
413 | else
414 |   vlog "No extra sbt options have been defined"
415 | fi
416 | 
417 | [[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}"
418 | 
419 | # reset "$@" to the residual args
420 | set -- "${residual_args[@]}"
421 | argumentCount=$#
422 | 
423 | # set sbt version
424 | set_sbt_version
425 | 
426 | # only exists in 0.12+
427 | setTraceLevel() {
428 |   case "$sbt_version" in
429 |     "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;;
430 |                                  *) setThisBuild traceLevel $trace_level ;;
431 |   esac
432 | }
433 | 
434 | # set scalacOptions if we were given any -S opts
435 | [[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\""
436 | 
437 | # Update build.properties on disk to set explicit version - sbt gives us no choice
438 | [[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version"
439 | vlog "Detected sbt version $sbt_version"
440 | 
441 | [[ -n "$scala_version" ]] && vlog "Overriding scala version to $scala_version"
442 | 
443 | # no args - alert them there's stuff in here
444 | (( argumentCount > 0 )) || {
445 |   vlog "Starting $script_name: invoke with -help for other options"
446 |   residual_args=( shell )
447 | }
448 | 
449 | # verify this is an sbt dir or -create was given
450 | [[ -r ./build.sbt || -d ./project || -n "$sbt_create" ]] || {
451 |   cat <<EOM
452 | $(pwd) doesn't appear to be an sbt project.
453 | If you want to start sbt anyway, run:
454 |   $0 -sbt-create
455 | 
456 | EOM
457 |   exit 1
458 | }
459 | 
460 | # pick up completion if present; todo
461 | [[ -r .sbt_completion.sh ]] && source .sbt_completion.sh
462 | 
463 | # no jar? download it.
464 | [[ -r "$sbt_jar" ]] || acquire_sbt_jar || {
465 |   # still no jar? uh-oh.
466 |   echo "Download failed. Obtain the jar manually and place it at $sbt_jar"
467 |   exit 1
468 | }
469 | 
470 | if [[ -n "$noshare" ]]; then
471 |   for opt in ${noshare_opts}; do
472 |     addJava "$opt"
473 |   done
474 | else
475 |   case "$sbt_version" in
476 |     "0.7."* | "0.10."* | "0.11."* | "0.12."* )
477 |       [[ -n "$sbt_dir" ]] || {
478 |         sbt_dir="$HOME/.sbt/$sbt_version"
479 |         vlog "Using $sbt_dir as sbt dir, -sbt-dir to override."
480 |       }
481 |     ;;
482 |   esac
483 | 
484 |   if [[ -n "$sbt_dir" ]]; then
485 |     addJava "-Dsbt.global.base=$sbt_dir"
486 |   fi
487 | fi
488 | 
489 | if [[ -r "$jvm_opts_file" ]]; then
490 |   vlog "Using jvm options defined in file $jvm_opts_file"
491 |   while read opt; do extra_jvm_opts+=("$opt"); done < <(readConfigFile "$jvm_opts_file")
492 | elif [[ -n "$JVM_OPTS" && ! ("$JVM_OPTS" =~ ^@.*) ]]; then
493 |   vlog "Using jvm options defined in \$JVM_OPTS variable"
494 |   extra_jvm_opts=( $JVM_OPTS )
495 | else
496 |   vlog "Using default jvm options"
497 |   extra_jvm_opts=( $(default_jvm_opts) )
498 | fi
499 | 
500 | # traceLevel is 0.12+
501 | [[ -n "$trace_level" ]] && setTraceLevel
502 | 
503 | main () {
504 |   execRunner "$java_cmd" \
505 |     "${extra_jvm_opts[@]}" \
506 |     "${java_args[@]}" \
507 |     -jar "$sbt_jar" \
508 |     "${sbt_commands[@]}" \
509 |     "${residual_args[@]}"
510 | }
511 | 
512 | # sbt inserts this string on certain lines when formatting is enabled:
513 | #   val OverwriteLine = "\r\u001BM\u001B[2K"
514 | # ...in order not to spam the console with a million "Resolving" lines.
515 | # Unfortunately that makes it that much harder to work with when
516 | # we're not going to print those lines anyway. We strip that bit of
517 | # line noise, but leave the other codes to preserve color.
518 | mainFiltered () {
519 |   local ansiOverwrite='\r\x1BM\x1B[2K'
520 |   local excludeRegex=$(egrep -v '^#|^$' ~/.sbtignore | paste -sd'|' -)
521 | 
522 |   echoLine () {
523 |     local line="$1"
524 |     local line1="$(echo "$line" | sed -r 's/\r\x1BM\x1B\[2K//g')"       # This strips the OverwriteLine code.
525 |     local line2="$(echo "$line1" | sed -r 's/\x1B\[[0-9;]*[JKmsu]//g')" # This strips all codes - we test regexes against this.
526 | 
527 |     if [[ $line2 =~ $excludeRegex ]]; then
528 |       [[ -n $debugUs ]] && echo "[X] $line1"
529 |     else
530 |       [[ -n $debugUs ]] && echo "    $line1" || echo "$line1"
531 |     fi
532 |   }
533 | 
534 |   echoLine "Starting sbt with output filtering enabled."
535 |   main | while read -r line; do echoLine "$line"; done
536 | }
537 | 
538 | # Only filter if there's a filter file and we don't see a known interactive command.
539 | # Obviously this is super ad hoc but I don't know how to improve on it. Testing whether
540 | # stdin is a terminal is useless because most of my use cases for this filtering are
541 | # exactly when I'm at a terminal, running sbt non-interactively.
542 | shouldFilter () { [[ -f ~/.sbtignore ]] && ! egrep -q '\b(shell|console|consoleProject)\b' <<<"${residual_args[@]}"; }
543 | 
544 | # run sbt
545 | if shouldFilter; then mainFiltered; else main; fi
546 | 


--------------------------------------------------------------------------------