├── .gitignore ├── .travis.yml ├── README.md ├── project ├── ApplicationBuild.scala ├── build.properties └── plugins.sbt ├── sbt ├── scalastyle-config.xml └── src └── main ├── resources └── logback.xml └── scala └── example └── flink ├── FlinkExample.scala └── WordCount.scala /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.iml 3 | 4 | target/ 5 | project/project/ 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: scala 2 | 3 | scala: 4 | - 2.11.7 5 | 6 | jdk: 7 | - oraclejdk8 8 | 9 | script: 10 | - sbt ++$TRAVIS_SCALA_VERSION clean coverage test scalastyle 11 | 12 | after_success: 13 | - sbt coveralls 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Start ZooKeeper server: 2 | 3 | ``` 4 | ./bin/zookeeper-server-start.sh config/zookeeper.properties 5 | ``` 6 | 7 | Start Kafka server: 8 | 9 | ``` 10 | ./bin/kafka-server-start.sh config/server.properties 11 | ``` 12 | 13 | Create input topic: 14 | 15 | ``` 16 | ./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic input 17 | ``` 18 | 19 | Create output topic: 20 | 21 | ``` 22 | ./bin/kafka-topics.sh --create --zookeeper localhost:2181 --replication-factor 1 --partitions 3 --topic output 23 | ``` 24 | 25 | Start kafka producer: 26 | 27 | ``` 28 | ./bin/kafka-console-producer.sh --broker-list localhost:9092 --topic input 29 | ``` 30 | 31 | Start kafka consumer: 32 | 33 | ``` 34 | ./bin/kafka-console-consumer.sh --zookeeper localhost:2181 --topic output 35 | ``` 36 | -------------------------------------------------------------------------------- /project/ApplicationBuild.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | import sbt.Keys._ 18 | import sbt._ 19 | 20 | object ApplicationBuild extends Build { 21 | 22 | object Versions { 23 | val flink = "0.10.1" 24 | val kafka = "0.8.2.1" 25 | } 26 | 27 | val projectName = "example-flink-kafka" 28 | 29 | val commonSettings = Seq( 30 | version := "1.0", 31 | organization := "http://mkuthan.github.io/", 32 | scalaVersion := "2.11.7" 33 | ) 34 | 35 | val commonScalacOptions = Seq( 36 | "-deprecation", 37 | "-encoding", "UTF-8", 38 | "-feature", 39 | "-language:existentials", 40 | "-language:higherKinds", 41 | "-language:implicitConversions", 42 | "-unchecked", 43 | "-Xfatal-warnings", 44 | "-Xfuture", 45 | "-Xlint", 46 | "-Yno-adapted-args", 47 | "-Ywarn-dead-code", 48 | "-Ywarn-numeric-widen", 49 | "-Ywarn-unused-import" 50 | ) 51 | 52 | val commonJavaInRuntimeOptions = Seq( 53 | "-Xmx512m" 54 | ) 55 | 56 | val commonJavaInTestOptions = Seq( 57 | "-Xmx512m" 58 | ) 59 | 60 | val commonResolvers = Seq( 61 | "Apache Snapshots" at "http://repository.apache.org/snapshots/" 62 | ) 63 | 64 | val commonLibraryDependencies = Seq( 65 | "org.apache.flink" %% "flink-core" % Versions.flink, 66 | "org.apache.flink" %% "flink-scala" % Versions.flink, 67 | "org.apache.flink" %% "flink-streaming-core" % Versions.flink, 68 | "org.apache.flink" %% "flink-streaming-scala" % Versions.flink, 69 | "org.apache.flink" %% "flink-connector-kafka" % Versions.flink, 70 | 71 | //"org.apache.kafka" %% "kafka" % Versions.kafka, 72 | //"org.apache.kafka" % "kafka-clients" % Versions.kafka, 73 | 74 | "com.typesafe.scala-logging" %% "scala-logging" % "3.1.0", 75 | 76 | "org.slf4j" % "slf4j-api" % "1.7.10", 77 | "ch.qos.logback" % "logback-classic" % "1.1.2", 78 | 79 | "org.scalatest" %% "scalatest" % "2.2.4" % "test" 80 | ) 81 | 82 | val commonExcludeDependencies = Seq( 83 | "org.slf4j" % "slf4j-log4j12" 84 | ) 85 | 86 | lazy val main = Project(projectName, base = file(".")) 87 | .settings(commonSettings) 88 | .settings(javaOptions in Runtime ++= commonJavaInRuntimeOptions) 89 | .settings(javaOptions in Test ++= commonJavaInTestOptions) 90 | .settings(scalacOptions ++= commonScalacOptions) 91 | .settings(resolvers ++= commonResolvers) 92 | .settings(libraryDependencies ++= commonLibraryDependencies) 93 | .settings(excludeDependencies ++= commonExcludeDependencies) 94 | .settings(fork in run := true) 95 | 96 | } 97 | 98 | -------------------------------------------------------------------------------- /project/build.properties: -------------------------------------------------------------------------------- 1 | sbt.version=0.13.8 2 | -------------------------------------------------------------------------------- /project/plugins.sbt: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | addSbtPlugin("org.scalastyle" %% "scalastyle-sbt-plugin" % "0.6.0") 18 | addSbtPlugin("org.scoverage" % "sbt-scoverage" % "1.0.4") 19 | addSbtPlugin("org.scoverage" % "sbt-coveralls" % "1.0.0.BETA1") 20 | addSbtPlugin("com.eed3si9n" % "sbt-assembly" % "0.12.0") 21 | -------------------------------------------------------------------------------- /sbt: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # A more capable sbt runner, coincidentally also called sbt. 4 | # Author: Paul Phillips 5 | 6 | # todo - make this dynamic 7 | declare -r sbt_release_version="0.13.8" 8 | declare -r sbt_unreleased_version="0.13.8" 9 | declare -r buildProps="project/build.properties" 10 | 11 | declare sbt_jar sbt_dir sbt_create sbt_version 12 | declare scala_version sbt_explicit_version 13 | declare verbose noshare batch trace_level log_level 14 | declare sbt_saved_stty debugUs 15 | 16 | echoerr () { echo >&2 "$@"; } 17 | vlog () { [[ -n "$verbose" ]] && echoerr "$@"; } 18 | 19 | # spaces are possible, e.g. sbt.version = 0.13.0 20 | build_props_sbt () { 21 | [[ -r "$buildProps" ]] && \ 22 | grep '^sbt\.version' "$buildProps" | tr '=\r' ' ' | awk '{ print $2; }' 23 | } 24 | 25 | update_build_props_sbt () { 26 | local ver="$1" 27 | local old="$(build_props_sbt)" 28 | 29 | [[ -r "$buildProps" ]] && [[ "$ver" != "$old" ]] && { 30 | perl -pi -e "s/^sbt\.version\b.*\$/sbt.version=${ver}/" "$buildProps" 31 | grep -q '^sbt.version[ =]' "$buildProps" || printf "\nsbt.version=%s\n" "$ver" >> "$buildProps" 32 | 33 | vlog "!!!" 34 | vlog "!!! Updated file $buildProps setting sbt.version to: $ver" 35 | vlog "!!! Previous value was: $old" 36 | vlog "!!!" 37 | } 38 | } 39 | 40 | set_sbt_version () { 41 | sbt_version="${sbt_explicit_version:-$(build_props_sbt)}" 42 | [[ -n "$sbt_version" ]] || sbt_version=$sbt_release_version 43 | export sbt_version 44 | } 45 | 46 | # restore stty settings (echo in particular) 47 | onSbtRunnerExit() { 48 | [[ -n "$sbt_saved_stty" ]] || return 49 | vlog "" 50 | vlog "restoring stty: $sbt_saved_stty" 51 | stty "$sbt_saved_stty" 52 | unset sbt_saved_stty 53 | } 54 | 55 | # save stty and trap exit, to ensure echo is reenabled if we are interrupted. 56 | trap onSbtRunnerExit EXIT 57 | sbt_saved_stty="$(stty -g 2>/dev/null)" 58 | vlog "Saved stty: $sbt_saved_stty" 59 | 60 | # this seems to cover the bases on OSX, and someone will 61 | # have to tell me about the others. 62 | get_script_path () { 63 | local path="$1" 64 | [[ -L "$path" ]] || { echo "$path" ; return; } 65 | 66 | local target="$(readlink "$path")" 67 | if [[ "${target:0:1}" == "/" ]]; then 68 | echo "$target" 69 | else 70 | echo "${path%/*}/$target" 71 | fi 72 | } 73 | 74 | die() { 75 | echo "Aborting: $@" 76 | exit 1 77 | } 78 | 79 | make_url () { 80 | version="$1" 81 | 82 | case "$version" in 83 | 0.7.*) echo "http://simple-build-tool.googlecode.com/files/sbt-launch-0.7.7.jar" ;; 84 | 0.10.* ) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 85 | 0.11.[12]) echo "$sbt_launch_repo/org.scala-tools.sbt/sbt-launch/$version/sbt-launch.jar" ;; 86 | *) echo "$sbt_launch_repo/org.scala-sbt/sbt-launch/$version/sbt-launch.jar" ;; 87 | esac 88 | } 89 | 90 | init_default_option_file () { 91 | local overriding_var="${!1}" 92 | local default_file="$2" 93 | if [[ ! -r "$default_file" && "$overriding_var" =~ ^@(.*)$ ]]; then 94 | local envvar_file="${BASH_REMATCH[1]}" 95 | if [[ -r "$envvar_file" ]]; then 96 | default_file="$envvar_file" 97 | fi 98 | fi 99 | echo "$default_file" 100 | } 101 | 102 | declare -r cms_opts="-XX:+CMSClassUnloadingEnabled -XX:+UseConcMarkSweepGC" 103 | declare -r jit_opts="-XX:ReservedCodeCacheSize=256m -XX:+TieredCompilation" 104 | declare -r default_jvm_opts_common="-Xms512m -Xmx1536m -Xss2m $jit_opts $cms_opts" 105 | declare -r noshare_opts="-Dsbt.global.base=project/.sbtboot -Dsbt.boot.directory=project/.boot -Dsbt.ivy.home=project/.ivy" 106 | declare -r latest_28="2.8.2" 107 | declare -r latest_29="2.9.3" 108 | declare -r latest_210="2.10.5" 109 | declare -r latest_211="2.11.6" 110 | 111 | declare -r script_path="$(get_script_path "$BASH_SOURCE")" 112 | declare -r script_name="${script_path##*/}" 113 | 114 | # some non-read-onlies set with defaults 115 | declare java_cmd="java" 116 | declare sbt_opts_file="$(init_default_option_file SBT_OPTS .sbtopts)" 117 | declare jvm_opts_file="$(init_default_option_file JVM_OPTS .jvmopts)" 118 | declare sbt_launch_repo="http://typesafe.artifactoryonline.com/typesafe/ivy-releases" 119 | 120 | # pull -J and -D options to give to java. 121 | declare -a residual_args 122 | declare -a java_args 123 | declare -a scalac_args 124 | declare -a sbt_commands 125 | 126 | # args to jvm/sbt via files or environment variables 127 | declare -a extra_jvm_opts extra_sbt_opts 128 | 129 | addJava () { 130 | vlog "[addJava] arg = '$1'" 131 | java_args=( "${java_args[@]}" "$1" ) 132 | } 133 | addSbt () { 134 | vlog "[addSbt] arg = '$1'" 135 | sbt_commands=( "${sbt_commands[@]}" "$1" ) 136 | } 137 | setThisBuild () { 138 | vlog "[addBuild] args = '$@'" 139 | local key="$1" && shift 140 | addSbt "set $key in ThisBuild := $@" 141 | } 142 | addScalac () { 143 | vlog "[addScalac] arg = '$1'" 144 | scalac_args=( "${scalac_args[@]}" "$1" ) 145 | } 146 | addResidual () { 147 | vlog "[residual] arg = '$1'" 148 | residual_args=( "${residual_args[@]}" "$1" ) 149 | } 150 | addResolver () { 151 | addSbt "set resolvers += $1" 152 | } 153 | addDebugger () { 154 | addJava "-Xdebug" 155 | addJava "-Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=$1" 156 | } 157 | setScalaVersion () { 158 | [[ "$1" == *"-SNAPSHOT" ]] && addResolver 'Resolver.sonatypeRepo("snapshots")' 159 | addSbt "++ $1" 160 | } 161 | setJavaHome () { 162 | java_cmd="$1/bin/java" 163 | setThisBuild javaHome "Some(file(\"$1\"))" 164 | export JAVA_HOME="$1" 165 | export JDK_HOME="$1" 166 | export PATH="$JAVA_HOME/bin:$PATH" 167 | } 168 | setJavaHomeQuietly () { 169 | java_cmd="$1/bin/java" 170 | addSbt ";warn ;set javaHome in ThisBuild := Some(file(\"$1\")) ;info" 171 | export JAVA_HOME="$1" 172 | export JDK_HOME="$1" 173 | export PATH="$JAVA_HOME/bin:$PATH" 174 | } 175 | 176 | # if set, use JDK_HOME/JAVA_HOME over java found in path 177 | if [[ -e "$JDK_HOME/lib/tools.jar" ]]; then 178 | setJavaHomeQuietly "$JDK_HOME" 179 | elif [[ -e "$JAVA_HOME/bin/java" ]]; then 180 | setJavaHomeQuietly "$JAVA_HOME" 181 | fi 182 | 183 | # directory to store sbt launchers 184 | declare sbt_launch_dir="$HOME/.sbt/launchers" 185 | [[ -d "$sbt_launch_dir" ]] || mkdir -p "$sbt_launch_dir" 186 | [[ -w "$sbt_launch_dir" ]] || sbt_launch_dir="$(mktemp -d -t sbt_extras_launchers.XXXXXX)" 187 | 188 | java_version () { 189 | local version=$("$java_cmd" -version 2>&1 | grep -e 'java version' | awk '{ print $3 }' | tr -d \") 190 | vlog "Detected Java version: $version" 191 | echo "${version:2:1}" 192 | } 193 | 194 | # MaxPermSize critical on pre-8 jvms but incurs noisy warning on 8+ 195 | default_jvm_opts () { 196 | local v="$(java_version)" 197 | if [[ $v -ge 8 ]]; then 198 | echo "$default_jvm_opts_common" 199 | else 200 | echo "-XX:MaxPermSize=384m $default_jvm_opts_common" 201 | fi 202 | } 203 | 204 | build_props_scala () { 205 | if [[ -r "$buildProps" ]]; then 206 | versionLine="$(grep '^build.scala.versions' "$buildProps")" 207 | versionString="${versionLine##build.scala.versions=}" 208 | echo "${versionString%% .*}" 209 | fi 210 | } 211 | 212 | execRunner () { 213 | # print the arguments one to a line, quoting any containing spaces 214 | vlog "# Executing command line:" && { 215 | for arg; do 216 | if [[ -n "$arg" ]]; then 217 | if printf "%s\n" "$arg" | grep -q ' '; then 218 | printf >&2 "\"%s\"\n" "$arg" 219 | else 220 | printf >&2 "%s\n" "$arg" 221 | fi 222 | fi 223 | done 224 | vlog "" 225 | } 226 | 227 | [[ -n "$batch" ]] && exec /dev/null; then 249 | curl --fail --silent "$url" --output "$jar" 250 | elif which wget >/dev/null; then 251 | wget --quiet -O "$jar" "$url" 252 | fi 253 | } && [[ -r "$jar" ]] 254 | } 255 | 256 | acquire_sbt_jar () { 257 | sbt_url="$(jar_url "$sbt_version")" 258 | sbt_jar="$(jar_file "$sbt_version")" 259 | 260 | [[ -r "$sbt_jar" ]] || download_url "$sbt_url" "$sbt_jar" 261 | } 262 | 263 | usage () { 264 | cat < display stack traces with a max of frames (default: -1, traces suppressed) 283 | -debug-inc enable debugging log for the incremental compiler 284 | -no-colors disable ANSI color codes 285 | -sbt-create start sbt even if current directory contains no sbt project 286 | -sbt-dir path to global settings/plugins directory (default: ~/.sbt/) 287 | -sbt-boot path to shared boot directory (default: ~/.sbt/boot in 0.11+) 288 | -ivy path to local Ivy repository (default: ~/.ivy2) 289 | -no-share use all local caches; no sharing 290 | -offline put sbt in offline mode 291 | -jvm-debug Turn on JVM debugging, open at the given port. 292 | -batch Disable interactive mode 293 | -prompt Set the sbt prompt; in expr, 's' is the State and 'e' is Extracted 294 | 295 | # sbt version (default: sbt.version from $buildProps if present, otherwise $sbt_release_version) 296 | -sbt-force-latest force the use of the latest release of sbt: $sbt_release_version 297 | -sbt-version use the specified version of sbt (default: $sbt_release_version) 298 | -sbt-dev use the latest pre-release version of sbt: $sbt_unreleased_version 299 | -sbt-jar use the specified jar as the sbt launcher 300 | -sbt-launch-dir directory to hold sbt launchers (default: ~/.sbt/launchers) 301 | -sbt-launch-repo repo url for downloading sbt launcher jar (default: $sbt_launch_repo) 302 | 303 | # scala version (default: as chosen by sbt) 304 | -28 use $latest_28 305 | -29 use $latest_29 306 | -210 use $latest_210 307 | -211 use $latest_211 308 | -scala-home use the scala build at the specified directory 309 | -scala-version use the specified version of scala 310 | -binary-version use the specified scala version when searching for dependencies 311 | 312 | # java version (default: java from PATH, currently $(java -version 2>&1 | grep version)) 313 | -java-home alternate JAVA_HOME 314 | 315 | # passing options to the jvm - note it does NOT use JAVA_OPTS due to pollution 316 | # The default set is used if JVM_OPTS is unset and no -jvm-opts file is found 317 | $(default_jvm_opts) 318 | JVM_OPTS environment variable holding either the jvm args directly, or 319 | the reference to a file containing jvm args if given path is prepended by '@' (e.g. '@/etc/jvmopts') 320 | Note: "@"-file is overridden by local '.jvmopts' or '-jvm-opts' argument. 321 | -jvm-opts file containing jvm args (if not given, .jvmopts in project root is used if present) 322 | -Dkey=val pass -Dkey=val directly to the jvm 323 | -J-X pass option -X directly to the jvm (-J is stripped) 324 | 325 | # passing options to sbt, OR to this runner 326 | SBT_OPTS environment variable holding either the sbt args directly, or 327 | the reference to a file containing sbt args if given path is prepended by '@' (e.g. '@/etc/sbtopts') 328 | Note: "@"-file is overridden by local '.sbtopts' or '-sbt-opts' argument. 329 | -sbt-opts file containing sbt args (if not given, .sbtopts in project root is used if present) 330 | -S-X add -X to sbt's scalacOptions (-S is stripped) 331 | EOM 332 | } 333 | 334 | process_args () 335 | { 336 | require_arg () { 337 | local type="$1" 338 | local opt="$2" 339 | local arg="$3" 340 | 341 | if [[ -z "$arg" ]] || [[ "${arg:0:1}" == "-" ]]; then 342 | die "$opt requires <$type> argument" 343 | fi 344 | } 345 | while [[ $# -gt 0 ]]; do 346 | case "$1" in 347 | -h|-help) usage; exit 1 ;; 348 | -v) verbose=true && shift ;; 349 | -d) addSbt "--debug" && addSbt debug && shift ;; 350 | -w) addSbt "--warn" && addSbt warn && shift ;; 351 | -q) addSbt "--error" && addSbt error && shift ;; 352 | -x) debugUs=true && shift ;; 353 | -trace) require_arg integer "$1" "$2" && trace_level="$2" && shift 2 ;; 354 | -ivy) require_arg path "$1" "$2" && addJava "-Dsbt.ivy.home=$2" && shift 2 ;; 355 | -no-colors) addJava "-Dsbt.log.noformat=true" && shift ;; 356 | -no-share) noshare=true && shift ;; 357 | -sbt-boot) require_arg path "$1" "$2" && addJava "-Dsbt.boot.directory=$2" && shift 2 ;; 358 | -sbt-dir) require_arg path "$1" "$2" && sbt_dir="$2" && shift 2 ;; 359 | -debug-inc) addJava "-Dxsbt.inc.debug=true" && shift ;; 360 | -offline) addSbt "set offline := true" && shift ;; 361 | -jvm-debug) require_arg port "$1" "$2" && addDebugger "$2" && shift 2 ;; 362 | -batch) batch=true && shift ;; 363 | -prompt) require_arg "expr" "$1" "$2" && setThisBuild shellPrompt "(s => { val e = Project.extract(s) ; $2 })" && shift 2 ;; 364 | 365 | -sbt-create) sbt_create=true && shift ;; 366 | -sbt-jar) require_arg path "$1" "$2" && sbt_jar="$2" && shift 2 ;; 367 | -sbt-version) require_arg version "$1" "$2" && sbt_explicit_version="$2" && shift 2 ;; 368 | -sbt-force-latest) sbt_explicit_version="$sbt_release_version" && shift ;; 369 | -sbt-dev) sbt_explicit_version="$sbt_unreleased_version" && shift ;; 370 | -sbt-launch-dir) require_arg path "$1" "$2" && sbt_launch_dir="$2" && shift 2 ;; 371 | -sbt-launch-repo) require_arg path "$1" "$2" && sbt_launch_repo="$2" && shift 2 ;; 372 | -scala-version) require_arg version "$1" "$2" && setScalaVersion "$2" && shift 2 ;; 373 | -binary-version) require_arg version "$1" "$2" && setThisBuild scalaBinaryVersion "\"$2\"" && shift 2 ;; 374 | -scala-home) require_arg path "$1" "$2" && setThisBuild scalaHome "Some(file(\"$2\"))" && shift 2 ;; 375 | -java-home) require_arg path "$1" "$2" && setJavaHome "$2" && shift 2 ;; 376 | -sbt-opts) require_arg path "$1" "$2" && sbt_opts_file="$2" && shift 2 ;; 377 | -jvm-opts) require_arg path "$1" "$2" && jvm_opts_file="$2" && shift 2 ;; 378 | 379 | -D*) addJava "$1" && shift ;; 380 | -J*) addJava "${1:2}" && shift ;; 381 | -S*) addScalac "${1:2}" && shift ;; 382 | -28) setScalaVersion "$latest_28" && shift ;; 383 | -29) setScalaVersion "$latest_29" && shift ;; 384 | -210) setScalaVersion "$latest_210" && shift ;; 385 | -211) setScalaVersion "$latest_211" && shift ;; 386 | 387 | --debug) addSbt debug && addResidual "$1" && shift ;; 388 | --warn) addSbt warn && addResidual "$1" && shift ;; 389 | --error) addSbt error && addResidual "$1" && shift ;; 390 | *) addResidual "$1" && shift ;; 391 | esac 392 | done 393 | } 394 | 395 | # process the direct command line arguments 396 | process_args "$@" 397 | 398 | # skip #-styled comments and blank lines 399 | readConfigFile() { 400 | while read line; do 401 | [[ $line =~ ^# ]] || [[ -z $line ]] || echo "$line" 402 | done < "$1" 403 | } 404 | 405 | # if there are file/environment sbt_opts, process again so we 406 | # can supply args to this runner 407 | if [[ -r "$sbt_opts_file" ]]; then 408 | vlog "Using sbt options defined in file $sbt_opts_file" 409 | while read opt; do extra_sbt_opts+=("$opt"); done < <(readConfigFile "$sbt_opts_file") 410 | elif [[ -n "$SBT_OPTS" && ! ("$SBT_OPTS" =~ ^@.*) ]]; then 411 | vlog "Using sbt options defined in variable \$SBT_OPTS" 412 | extra_sbt_opts=( $SBT_OPTS ) 413 | else 414 | vlog "No extra sbt options have been defined" 415 | fi 416 | 417 | [[ -n "${extra_sbt_opts[*]}" ]] && process_args "${extra_sbt_opts[@]}" 418 | 419 | # reset "$@" to the residual args 420 | set -- "${residual_args[@]}" 421 | argumentCount=$# 422 | 423 | # set sbt version 424 | set_sbt_version 425 | 426 | # only exists in 0.12+ 427 | setTraceLevel() { 428 | case "$sbt_version" in 429 | "0.7."* | "0.10."* | "0.11."* ) echoerr "Cannot set trace level in sbt version $sbt_version" ;; 430 | *) setThisBuild traceLevel $trace_level ;; 431 | esac 432 | } 433 | 434 | # set scalacOptions if we were given any -S opts 435 | [[ ${#scalac_args[@]} -eq 0 ]] || addSbt "set scalacOptions in ThisBuild += \"${scalac_args[@]}\"" 436 | 437 | # Update build.properties on disk to set explicit version - sbt gives us no choice 438 | [[ -n "$sbt_explicit_version" ]] && update_build_props_sbt "$sbt_explicit_version" 439 | vlog "Detected sbt version $sbt_version" 440 | 441 | [[ -n "$scala_version" ]] && vlog "Overriding scala version to $scala_version" 442 | 443 | # no args - alert them there's stuff in here 444 | (( argumentCount > 0 )) || { 445 | vlog "Starting $script_name: invoke with -help for other options" 446 | residual_args=( shell ) 447 | } 448 | 449 | # verify this is an sbt dir or -create was given 450 | [[ -r ./build.sbt || -d ./project || -n "$sbt_create" ]] || { 451 | cat < 2 | Scalastyle standard configuration 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /src/main/scala/example/flink/FlinkExample.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example.flink 18 | 19 | import java.util.concurrent.TimeUnit 20 | 21 | import org.apache.flink.api.scala._ 22 | import org.apache.flink.streaming.api.scala.StreamExecutionEnvironment 23 | import org.apache.flink.streaming.api.windowing.time.Time 24 | import org.apache.flink.streaming.connectors.kafka.FlinkKafkaConsumer.{FetcherType, OffsetStore} 25 | import org.apache.flink.streaming.connectors.kafka.{FlinkKafkaConsumer, FlinkKafkaProducer} 26 | import org.apache.flink.streaming.util.serialization.{DeserializationSchema, SerializationSchema} 27 | 28 | object FlinkExample { 29 | 30 | import WordCount._ 31 | 32 | val stopWords = Set("a", "an", "the") 33 | val window = Time.of(10, TimeUnit.SECONDS) 34 | 35 | def main(args: Array[String]): Unit = { 36 | val env = StreamExecutionEnvironment.createLocalEnvironment() 37 | 38 | val kafkaConsumerProperties = Map( 39 | "zookeeper.connect" -> "localhost:2181", 40 | "group.id" -> "flink", 41 | "bootstrap.servers" -> "localhost:9092" 42 | ) 43 | 44 | val kafkaConsumer = new FlinkKafkaConsumer[String]( 45 | "input", 46 | KafkaStringSchema, 47 | kafkaConsumerProperties, 48 | OffsetStore.FLINK_ZOOKEEPER, 49 | FetcherType.LEGACY_LOW_LEVEL 50 | ) 51 | 52 | val kafkaProducer = new FlinkKafkaProducer[String]( 53 | "localhost:9092", 54 | "output", 55 | KafkaStringSchema 56 | ) 57 | 58 | val lines = env.addSource(kafkaConsumer) 59 | 60 | val wordCounts = countWords(lines, stopWords, window) 61 | 62 | wordCounts 63 | .map(_.toString) 64 | .addSink(kafkaProducer) 65 | 66 | env.execute() 67 | } 68 | 69 | implicit def map2Properties(map: Map[String, String]): java.util.Properties = { 70 | (new java.util.Properties /: map) { case (props, (k, v)) => props.put(k, v); props } 71 | } 72 | 73 | object KafkaStringSchema extends SerializationSchema[String, Array[Byte]] with DeserializationSchema[String] { 74 | 75 | import org.apache.flink.api.common.typeinfo.TypeInformation 76 | import org.apache.flink.api.java.typeutils.TypeExtractor 77 | 78 | override def serialize(t: String): Array[Byte] = t.getBytes("UTF-8") 79 | 80 | override def isEndOfStream(t: String): Boolean = false 81 | 82 | override def deserialize(bytes: Array[Byte]): String = new String(bytes, "UTF-8") 83 | 84 | override def getProducedType: TypeInformation[String] = TypeExtractor.getForClass(classOf[String]) 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /src/main/scala/example/flink/WordCount.scala: -------------------------------------------------------------------------------- 1 | // Copyright (C) 2011-2012 the original author or authors. 2 | // See the LICENCE.txt file distributed with this work for additional 3 | // information regarding copyright ownership. 4 | // 5 | // Licensed under the Apache License, Version 2.0 (the "License"); 6 | // you may not use this file except in compliance with the License. 7 | // You may obtain a copy of the License at 8 | // 9 | // http://www.apache.org/licenses/LICENSE-2.0 10 | // 11 | // Unless required by applicable law or agreed to in writing, software 12 | // distributed under the License is distributed on an "AS IS" BASIS, 13 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | // See the License for the specific language governing permissions and 15 | // limitations under the License. 16 | 17 | package example.flink 18 | 19 | import org.apache.flink.api.scala._ 20 | import org.apache.flink.streaming.api.scala.DataStream 21 | import org.apache.flink.streaming.api.windowing.time.Time 22 | 23 | object WordCount { 24 | 25 | type WordCount = (String, Int) 26 | 27 | def countWords(lines: DataStream[String], stopWords: Set[String], window: Time): DataStream[WordCount] = { 28 | lines 29 | .flatMap(line => line.split(" ")) 30 | .filter(word => !word.isEmpty) 31 | .map(word => word.toLowerCase) 32 | .filter(word => !stopWords.contains(word)) 33 | .map(word => (word, 1)) 34 | .keyBy(0) 35 | .timeWindow(window) 36 | .sum(1) 37 | } 38 | 39 | } 40 | --------------------------------------------------------------------------------