├── analytics ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.properties │ │ └── maven-wrapper.jar ├── .gitignore ├── src │ ├── test │ │ └── java │ │ │ └── com │ │ │ └── example │ │ │ └── analytics │ │ │ └── AnalyticsApplicationTests.java │ └── main │ │ ├── resources │ │ └── application.properties │ │ └── java │ │ └── com │ │ └── example │ │ └── analytics │ │ └── AnalyticsApplication.java ├── pom.xml ├── mvnw.cmd └── mvnw ├── word-count ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.properties │ │ └── maven-wrapper.jar ├── .gitignore ├── src │ ├── test │ │ └── java │ │ │ └── wc │ │ │ └── example │ │ │ └── wordcount │ │ │ └── ConsumerApplicationTests.java │ └── main │ │ ├── resources │ │ ├── application.properties │ │ └── data.txt │ │ └── java │ │ └── wc │ │ └── WordCountApplication.java ├── pom.xml ├── mvnw.cmd └── mvnw ├── README.md ├── .gitignore ├── outline.md └── LICENSE /analytics/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.3/apache-maven-3.5.3-bin.zip 2 | -------------------------------------------------------------------------------- /word-count/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.3/apache-maven-3.5.3-bin.zip 2 | -------------------------------------------------------------------------------- /analytics/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/spring-cloud-stream-kafka-streams/HEAD/analytics/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /word-count/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshlong-attic/spring-cloud-stream-kafka-streams/HEAD/word-count/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spring-cloud-kafka-streams 2 | Hi Spring fans! In this installment of _Spring Tips_ we look at Spring Cloud Stream's support for the Apache Kafka Streams project 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | pom.xml.tag 3 | pom.xml.releaseBackup 4 | pom.xml.versionsBackup 5 | pom.xml.next 6 | release.properties 7 | dependency-reduced-pom.xml 8 | buildNumber.properties 9 | .mvn/timing.properties 10 | 11 | # Avoid ignoring Maven wrapper jar file (.jar files are usually ignored) 12 | !/.mvn/wrapper/maven-wrapper.jar 13 | -------------------------------------------------------------------------------- /analytics/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | .sts4-cache 12 | 13 | ### IntelliJ IDEA ### 14 | .idea 15 | *.iws 16 | *.iml 17 | *.ipr 18 | 19 | ### NetBeans ### 20 | nbproject/private/ 21 | build/ 22 | nbbuild/ 23 | dist/ 24 | nbdist/ 25 | .nb-gradle/ -------------------------------------------------------------------------------- /word-count/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | .sts4-cache 12 | 13 | ### IntelliJ IDEA ### 14 | .idea 15 | *.iws 16 | *.iml 17 | *.ipr 18 | 19 | ### NetBeans ### 20 | nbproject/private/ 21 | build/ 22 | nbbuild/ 23 | dist/ 24 | nbdist/ 25 | .nb-gradle/ -------------------------------------------------------------------------------- /analytics/src/test/java/com/example/analytics/AnalyticsApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example.analytics; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class AnalyticsApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /word-count/src/test/java/wc/example/wordcount/ConsumerApplicationTests.java: -------------------------------------------------------------------------------- 1 | package wc.example.wordcount; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class ConsumerApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /word-count/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000 2 | spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde 3 | spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde 4 | spring.cloud.stream.bindings.wordsInbound.destination=words 5 | spring.cloud.stream.bindings.wordsInbound.consumer.headerMode=raw 6 | spring.cloud.stream.bindings.wordsOutbound.destination=words 7 | spring.cloud.stream.bindings.wordsOutbound.producer.headerMode=raw 8 | server.port=8083 9 | -------------------------------------------------------------------------------- /analytics/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # defaults 3 | spring.cloud.stream.kafka.streams.binder.configuration.commit.interval.ms=1000 4 | spring.cloud.stream.kafka.streams.binder.configuration.default.key.serde=org.apache.kafka.common.serialization.Serdes$StringSerde 5 | spring.cloud.stream.kafka.streams.binder.configuration.default.value.serde=org.apache.kafka.common.serialization.Serdes$StringSerde 6 | # 7 | # page views out 8 | spring.cloud.stream.bindings.pveo.destination=pvs 9 | spring.cloud.stream.bindings.pveo.producer.header-mode=raw 10 | # 11 | # page views in 12 | spring.cloud.stream.bindings.pvei.destination=pvs 13 | spring.cloud.stream.bindings.pvei.consumer.header-mode=raw 14 | # 15 | # page counts out 16 | spring.cloud.stream.bindings.pco.destination=pcs 17 | spring.cloud.stream.bindings.pco.producer.use-native-encoding=true 18 | spring.cloud.stream.kafka.streams.bindings.pco.producer.value-serde=org.apache.kafka.common.serialization.Serdes$LongSerde 19 | spring.cloud.stream.kafka.streams.bindings.pco.producer.key-serde=org.apache.kafka.common.serialization.Serdes$StringSerde 20 | # 21 | # page counts in 22 | spring.cloud.stream.bindings.pci.destination=pcs 23 | spring.cloud.stream.bindings.pci.group=pci 24 | spring.cloud.stream.bindings.pci.consumer.header-mode=raw 25 | spring.cloud.stream.bindings.pci.content-type=application/json 26 | spring.cloud.stream.bindings.pci.consumer.use-native-decoding=true 27 | spring.cloud.stream.kafka.streams.bindings.pci.consumer.key-serde=org.apache.kafka.common.serialization.Serdes$StringSerde 28 | spring.cloud.stream.kafka.streams.bindings.pci.consumer.value-serde=org.apache.kafka.common.serialization.Serdes$LongSerde 29 | -------------------------------------------------------------------------------- /outline.md: -------------------------------------------------------------------------------- 1 | # TOC 2 | - what is kafka https://kafka.apache.org/intro 3 | - some big concepts: 4 | -- kafka supports streams of records stored in categories called topics. 5 | -- each record has a timestamp, a key, and a value 6 | - kakfa has a producer/consumer API, but it also has a streams procesing API and a connector API. what were interested in is the streams API. 7 | - each partition is an ordered, immutable sequence of records that is continually appended to—a structured commit log 8 | - the streams API doesnt require spark cluster 9 | - records can be stored for very long periods of time and performance is fixed/constant. 10 | - consumers keep an offset relative to a given parititon. 11 | - partitions are nice because they allow clients to scale to multiple nodes, and to handle concurrently consuming data. 12 | - publishers have to select which parition to use when writing, though theres a load balancing algo that kicks in 13 | - consumers have group names. if memebers belong to the same consumer group then only one node gets any one message from a topic. load-balanced! 14 | - kafka streams goes beyond tradtional pub/sub kinda messaging. it looks more similar to technologies like Apache Spark and Apache Storm. 15 | - it has some notable differences, though. 16 | - 17 | 18 | - spring for kafka 19 | - boot autoconfig 20 | - kafkatemplate 21 | 22 | - setup an example with a producer that sends writes of `PageViewEvent` usingg spring cloud stream kafka 23 | - then add `spring-cloud-stream-binder-kafka-(streams)` 24 | 25 | 26 | 27 | - Kafka Streams lessons 28 | -- kstream: stream of records 29 | -- latest value for a given key: ktable 30 | -- some ops are statess in KS 31 | -- somme require state stores. this is managed behind the scenes with kafka persisting the records. 32 | -- u can window records, as well, lumping them into buckets 33 | -- 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /word-count/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.example 7 | consumer 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | 12 | org.springframework.boot 13 | spring-boot-starter-parent 14 | 2.0.0.RELEASE 15 | 16 | 17 | 18 | 19 | UTF-8 20 | UTF-8 21 | 1.8 22 | Finchley.M9 23 | 24 | 25 | 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter-web 30 | 31 | 32 | org.springframework.cloud 33 | spring-cloud-stream 34 | 35 | 36 | org.springframework.cloud 37 | spring-cloud-stream-binder-kafka-streams 38 | 39 | 40 | org.springframework.cloud 41 | spring-cloud-stream-binder-kafka 42 | 43 | 44 | org.springframework.kafka 45 | spring-kafka 46 | 47 | 48 | org.projectlombok 49 | lombok 50 | true 51 | 52 | 53 | org.springframework.boot 54 | spring-boot-starter-test 55 | test 56 | 57 | 58 | org.springframework.cloud 59 | spring-cloud-stream-test-support 60 | test 61 | 62 | 63 | 64 | 65 | 66 | 67 | org.springframework.cloud 68 | spring-cloud-dependencies 69 | ${spring-cloud.version} 70 | pom 71 | import 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | org.springframework.boot 80 | spring-boot-maven-plugin 81 | 82 | 83 | 84 | 85 | 86 | 87 | spring-milestones 88 | Spring Milestones 89 | https://repo.spring.io/milestone 90 | 91 | false 92 | 93 | 94 | 95 | 96 | 97 | 98 | -------------------------------------------------------------------------------- /analytics/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.example 7 | analytics 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | analytics 12 | Demo project for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 2.0.1.BUILD-SNAPSHOT 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 1.8 25 | Finchley.BUILD-SNAPSHOT 26 | 27 | 28 | 29 | 30 | org.springframework.boot 31 | spring-boot-starter-web 32 | 33 | 34 | org.springframework.cloud 35 | spring-cloud-stream 36 | 37 | 38 | org.springframework.cloud 39 | spring-cloud-stream-binder-kafka-streams 40 | 41 | 42 | org.springframework.cloud 43 | spring-cloud-stream-binder-kafka 44 | 45 | 46 | org.springframework.kafka 47 | spring-kafka 48 | 49 | 50 | 51 | org.projectlombok 52 | lombok 53 | true 54 | 55 | 56 | org.springframework.boot 57 | spring-boot-starter-test 58 | test 59 | 60 | 61 | org.springframework.cloud 62 | spring-cloud-stream-test-support 63 | test 64 | 65 | 66 | 67 | 68 | 69 | 70 | org.springframework.cloud 71 | spring-cloud-dependencies 72 | ${spring-cloud.version} 73 | pom 74 | import 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | org.springframework.boot 83 | spring-boot-maven-plugin 84 | 85 | 86 | 87 | 88 | 89 | 90 | spring-snapshots 91 | Spring Snapshots 92 | https://repo.spring.io/snapshot 93 | 94 | true 95 | 96 | 97 | 98 | spring-milestones 99 | Spring Milestones 100 | https://repo.spring.io/milestone 101 | 102 | false 103 | 104 | 105 | 106 | 107 | 108 | 109 | spring-snapshots 110 | Spring Snapshots 111 | https://repo.spring.io/snapshot 112 | 113 | true 114 | 115 | 116 | 117 | spring-milestones 118 | Spring Milestones 119 | https://repo.spring.io/milestone 120 | 121 | false 122 | 123 | 124 | 125 | 126 | 127 | 128 | -------------------------------------------------------------------------------- /analytics/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /word-count/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /word-count/src/main/java/wc/WordCountApplication.java: -------------------------------------------------------------------------------- 1 | package wc; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import lombok.extern.java.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.apache.kafka.streams.KeyValue; 9 | import org.apache.kafka.streams.kstream.*; 10 | import org.apache.kafka.streams.state.QueryableStoreTypes; 11 | import org.apache.kafka.streams.state.ReadOnlyWindowStore; 12 | import org.apache.kafka.streams.state.WindowStoreIterator; 13 | import org.springframework.boot.ApplicationArguments; 14 | import org.springframework.boot.ApplicationRunner; 15 | import org.springframework.boot.SpringApplication; 16 | import org.springframework.boot.autoconfigure.SpringBootApplication; 17 | import org.springframework.cloud.stream.annotation.EnableBinding; 18 | import org.springframework.cloud.stream.annotation.Input; 19 | import org.springframework.cloud.stream.annotation.Output; 20 | import org.springframework.cloud.stream.annotation.StreamListener; 21 | import org.springframework.cloud.stream.binder.kafka.streams.QueryableStoreRegistry; 22 | import org.springframework.context.annotation.Configuration; 23 | import org.springframework.messaging.MessageChannel; 24 | import org.springframework.messaging.support.MessageBuilder; 25 | import org.springframework.stereotype.Component; 26 | import org.springframework.web.bind.annotation.GetMapping; 27 | import org.springframework.web.bind.annotation.PathVariable; 28 | import org.springframework.web.bind.annotation.RestController; 29 | 30 | import java.io.BufferedReader; 31 | import java.io.InputStreamReader; 32 | import java.util.Arrays; 33 | import java.util.Date; 34 | import java.util.HashMap; 35 | import java.util.Map; 36 | import java.util.concurrent.Executors; 37 | import java.util.concurrent.ScheduledExecutorService; 38 | import java.util.concurrent.TimeUnit; 39 | 40 | @SpringBootApplication 41 | public class WordCountApplication { 42 | 43 | 44 | @Component 45 | public static class WordsProducer implements ApplicationRunner { 46 | 47 | private final MessageChannel outbound; 48 | 49 | private final ScheduledExecutorService executorService = Executors.newSingleThreadScheduledExecutor(); 50 | 51 | public WordsProducer(WordCountChannels channels) { 52 | this.outbound = channels.wordsOutbound(); 53 | } 54 | 55 | private void produce() { 56 | try (BufferedReader br = new BufferedReader( 57 | new InputStreamReader(WordCountApplication.class.getResourceAsStream("/data.txt")))) { 58 | String line; 59 | while ((line = br.readLine()) != null) { 60 | outbound.send(MessageBuilder.withPayload(line).build()); 61 | } 62 | } 63 | catch (Exception e) { 64 | LogFactory.getLog(getClass()).error(e); 65 | } 66 | } 67 | 68 | @Override 69 | public void run(ApplicationArguments args) { 70 | this.executorService.scheduleWithFixedDelay(this::produce, 1, 10, TimeUnit.SECONDS); 71 | } 72 | } 73 | 74 | @Log 75 | @Configuration 76 | @EnableBinding(WordCountChannels.class) 77 | public static class WordsConsumer { 78 | 79 | @StreamListener 80 | public void process(@Input(WordCountChannels.WORDS_INBOUND) KStream words) { 81 | 82 | KTable, Long> count = words 83 | .flatMapValues(value -> Arrays.asList(value.toLowerCase().split("\\W+"))) 84 | .map((key, value) -> new KeyValue<>(value, value)) 85 | .groupByKey() 86 | .windowedBy(TimeWindows.of(10_000)) 87 | .count(Materialized.as("word-counts")); 88 | 89 | count 90 | .toStream() 91 | .map((key, value) -> new KeyValue<>(null, 92 | new WordCount(key.key(), value, new Date(key.window().start()), new Date(key.window().end())))) 93 | .foreach((key, value) -> log.info("value: " + value)); 94 | } 95 | } 96 | 97 | @RestController 98 | public static class WordsRestController { 99 | 100 | private final QueryableStoreRegistry queryableStoreRegistry; 101 | 102 | public WordsRestController(QueryableStoreRegistry queryableStoreRegistry) { 103 | this.queryableStoreRegistry = queryableStoreRegistry; 104 | } 105 | 106 | @GetMapping("/words/{name}") 107 | Map name(@PathVariable String name) { 108 | ReadOnlyWindowStore store = queryableStoreRegistry.getQueryableStoreType("word-counts", QueryableStoreTypes.windowStore()); 109 | long timeFrom = 0; 110 | long timeTo = System.currentTimeMillis(); 111 | WindowStoreIterator iterator = store.fetch(name, timeFrom, timeTo); 112 | Map map = new HashMap<>(); 113 | while (iterator.hasNext()) { 114 | KeyValue next = iterator.next(); 115 | Long timestamp = next.key; 116 | Long cardinality = next.value; 117 | map.put(new Date(timestamp), cardinality); 118 | } 119 | Map ret = new HashMap<>(); 120 | ret.put("keyword", name); 121 | ret.put("changelog", map); 122 | return ret; 123 | } 124 | } 125 | 126 | public static void main(String[] args) { 127 | SpringApplication.run(WordCountApplication.class, args); 128 | } 129 | } 130 | 131 | @Data 132 | @NoArgsConstructor 133 | @AllArgsConstructor 134 | class WordCount { 135 | 136 | private String word; 137 | private long count; 138 | private Date start, end; 139 | } 140 | 141 | 142 | interface WordCountChannels { 143 | 144 | String WORDS_INBOUND = "wordsInbound"; 145 | String WORDS_OUTBOUND = "wordsOutbound"; 146 | 147 | @Input(WORDS_INBOUND) 148 | KStream wordsInbound(); 149 | 150 | @Output(WORDS_OUTBOUND) 151 | MessageChannel wordsOutbound(); 152 | } 153 | 154 | -------------------------------------------------------------------------------- /analytics/src/main/java/com/example/analytics/AnalyticsApplication.java: -------------------------------------------------------------------------------- 1 | package com.example.analytics; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.apache.kafka.streams.KeyValue; 9 | import org.apache.kafka.streams.kstream.KStream; 10 | import org.apache.kafka.streams.kstream.KTable; 11 | import org.apache.kafka.streams.kstream.Materialized; 12 | import org.apache.kafka.streams.state.KeyValueIterator; 13 | import org.apache.kafka.streams.state.QueryableStoreTypes; 14 | import org.apache.kafka.streams.state.ReadOnlyKeyValueStore; 15 | import org.springframework.boot.ApplicationArguments; 16 | import org.springframework.boot.ApplicationRunner; 17 | import org.springframework.boot.SpringApplication; 18 | import org.springframework.boot.autoconfigure.SpringBootApplication; 19 | import org.springframework.cloud.stream.annotation.EnableBinding; 20 | import org.springframework.cloud.stream.annotation.Input; 21 | import org.springframework.cloud.stream.annotation.Output; 22 | import org.springframework.cloud.stream.annotation.StreamListener; 23 | import org.springframework.cloud.stream.binder.kafka.streams.QueryableStoreRegistry; 24 | import org.springframework.kafka.support.KafkaHeaders; 25 | import org.springframework.messaging.Message; 26 | import org.springframework.messaging.MessageChannel; 27 | import org.springframework.messaging.handler.annotation.SendTo; 28 | import org.springframework.messaging.support.MessageBuilder; 29 | import org.springframework.stereotype.Component; 30 | import org.springframework.web.bind.annotation.GetMapping; 31 | import org.springframework.web.bind.annotation.RestController; 32 | 33 | import java.util.*; 34 | import java.util.concurrent.Executors; 35 | import java.util.concurrent.TimeUnit; 36 | 37 | import static com.example.analytics.AnalyticsBinding.*; 38 | 39 | @EnableBinding(AnalyticsBinding.class) 40 | @SpringBootApplication 41 | public class AnalyticsApplication { 42 | 43 | @Component 44 | public static class PageEventSource implements ApplicationRunner { 45 | 46 | private final Log log = LogFactory.getLog(getClass()); 47 | 48 | private final List pages = Arrays.asList("blog", "initializr", "news", "rss", "sitemap", "about", "colophon"); 49 | private final List users = Arrays.asList("jlong", "jwatters", "dsyer", "pwebb", "mfisher"); 50 | 51 | private final MessageChannel out; 52 | 53 | public PageEventSource(AnalyticsBinding binding) { 54 | this.out = binding.pageViewEventsOut(); 55 | } 56 | 57 | @Override 58 | public void run(ApplicationArguments args) throws Exception { 59 | 60 | Runnable runnable = () -> { 61 | PageViewEvent pageViewEvent = new PageViewEvent(random(this.users), random(this.pages), Math.random() > .5 ? 10 : 1000); 62 | Message message = MessageBuilder 63 | .withPayload(pageViewEvent) 64 | .setHeader(KafkaHeaders.MESSAGE_KEY, pageViewEvent.getUser().getBytes()) 65 | .build(); 66 | try { 67 | this.out.send(message); 68 | this.log.info("sent " + pageViewEvent); 69 | } 70 | catch (Exception e) { 71 | this.log.error(e); 72 | } 73 | }; 74 | Executors.newSingleThreadScheduledExecutor().scheduleAtFixedRate(runnable, 1, 1, TimeUnit.SECONDS); 75 | } 76 | 77 | private static T random(List ts) { 78 | return ts.get(new Random().nextInt(ts.size())); 79 | } 80 | } 81 | 82 | @Component 83 | public static class PageEventSink { 84 | 85 | @StreamListener 86 | @SendTo(PAGE_COUNTS_OUT) 87 | public KStream process(@Input(PAGE_VIEW_IN) KStream events) { 88 | return events 89 | .filter((key, value) -> value.getDurationSpentOnPage() > 10) 90 | .map((key, value) -> new KeyValue<>(value.getPage(), "0")) 91 | .groupByKey() 92 | .count(Materialized.as(PAGE_COUNTS_MV)) 93 | .toStream(); 94 | } 95 | } 96 | 97 | @Component 98 | public static class PageCountSink { 99 | 100 | private final Log log = LogFactory.getLog(getClass()); 101 | 102 | @StreamListener 103 | public void process(@Input(PAGE_COUNTS_IN) KTable counts) { 104 | counts 105 | .toStream() 106 | .foreach((key, value) -> log.info(key + '=' + value)); 107 | } 108 | } 109 | 110 | @RestController 111 | public static class CountsRestController { 112 | 113 | private final QueryableStoreRegistry registry; 114 | 115 | public CountsRestController(QueryableStoreRegistry registry) { 116 | this.registry = registry; 117 | } 118 | 119 | @GetMapping("/counts") 120 | Map counts() { 121 | ReadOnlyKeyValueStore store = registry.getQueryableStoreType(PAGE_COUNTS_MV, QueryableStoreTypes.keyValueStore()); 122 | 123 | Map m = new HashMap<>(); 124 | KeyValueIterator iterator = store.all(); 125 | while (iterator.hasNext()) { 126 | KeyValue next = iterator.next(); 127 | m.put(next.key, next.value); 128 | } 129 | return m; 130 | } 131 | } 132 | 133 | public static void main(String[] args) { 134 | SpringApplication.run(AnalyticsApplication.class, args); 135 | } 136 | } 137 | 138 | interface AnalyticsBinding { 139 | 140 | String PAGE_VIEW_OUT = "pveo"; 141 | String PAGE_VIEW_IN = "pvei"; 142 | 143 | String PAGE_COUNTS_OUT = "pco"; 144 | String PAGE_COUNTS_IN = "pci"; 145 | String PAGE_COUNTS_MV = "pcmview"; 146 | 147 | @Input(PAGE_COUNTS_IN) 148 | KTable pageCountsIn(); 149 | 150 | @Output(PAGE_COUNTS_OUT) 151 | KStream pageCountOut(); 152 | 153 | @Output(PAGE_VIEW_OUT) 154 | MessageChannel pageViewEventsOut(); 155 | 156 | @Input(PAGE_VIEW_IN) 157 | KStream pageViewEventsIn(); 158 | } 159 | 160 | @Data 161 | @AllArgsConstructor 162 | @NoArgsConstructor 163 | class PageViewEvent { 164 | private String user, page; 165 | private long durationSpentOnPage; 166 | } -------------------------------------------------------------------------------- /analytics/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /word-count/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /word-count/src/main/resources/data.txt: -------------------------------------------------------------------------------- 1 | 2 | HOME 3 | INTRODUCTION 4 | QUICKSTART 5 | USE CASES 6 | DOCUMENTATION 7 | Getting Started 8 | APIs 9 | Kafka Streams 10 | Kafka Connect 11 | Configuration 12 | Design 13 | Implementation 14 | Operations 15 | Security 16 | PERFORMANCE 17 | POWERED BY 18 | PROJECT INFO 19 | ECOSYSTEM 20 | CLIENTS 21 | EVENTS 22 | CONTACT US 23 | APACHE 24 | Download 25 | @apachekafka 26 | DOCUMENTATION 27 | STREAMS 28 | Tutorial: Write a Streams Application 29 | INTRODUCTION RUN DEMO APP TUTORIAL: WRITE APP CONCEPTS ARCHITECTURE DEVELOPER GUIDE UPGRADE 30 | In this guide we will start from scratch on setting up your own project to write a stream processing application using Kafka Streams. It is highly recommended to read the quickstart first on how to run a Streams application written in Kafka Streams if you have not done so. 31 | 32 | Setting up a Maven Project 33 | We are going to use a Kafka Streams Maven Archetype for creating a Streams project structure with the following commands: 34 | 35 | 1 36 | 2 37 | 3 38 | 4 39 | 5 40 | 6 41 | 7 42 | 8 43 | mvn archetype:generate \ 44 | -DarchetypeGroupId=org.apache.kafka \ 45 | -DarchetypeArtifactId=streams-quickstart-java \ 46 | -DarchetypeVersion=1.0.1 \ 47 | -DgroupId=streams.examples \ 48 | -DartifactId=streams.examples \ 49 | -Dversion=0.1 \ 50 | -Dpackage=myapps 51 | You can use a different value for groupId, artifactId and package parameters if you like. Assuming the above parameter values are used, this command will create a project structure that looks like this: 52 | 53 | 1 54 | 2 55 | 3 56 | 4 57 | 5 58 | 6 59 | 7 60 | 8 61 | 9 62 | 10 63 | 11 64 | 12 65 | > tree streams.examples 66 | streams-quickstart 67 | |-- pom.xml 68 | |-- src 69 | |-- main 70 | |-- java 71 | | |-- myapps 72 | | |-- LineSplit.java 73 | | |-- Pipe.java 74 | | |-- WordCount.java 75 | |-- resources 76 | |-- log4j.properties 77 | The pom.xml file included in the project already has the Streams dependency defined, and there are already several example programs written with Streams library under src/main/java. Since we are going to start writing such programs from scratch, we can now delete these examples: 78 | 79 | 1 80 | 2 81 | > cd streams-quickstart 82 | > rm src/main/java/myapps/*.java 83 | Writing a first Streams application: Pipe 84 | It's coding time now! Feel free to open your favorite IDE and import this Maven project, or simply open a text editor and create a java file under src/main/java. Let's name it Pipe.java: 85 | 1 86 | 2 87 | 3 88 | 4 89 | 5 90 | 6 91 | 7 92 | 8 93 | package myapps; 94 | 95 | public class Pipe { 96 | 97 | public static void main(String[] args) throws Exception { 98 | 99 | } 100 | } 101 | We are going to fill in the main function to write this pipe program. Note that we will not list the import statements as we go since IDEs can usually add them automatically. However if you are using a text editor you need to manually add the imports, and at the end of this section we'll show the complete code snippet with import statement for you. 102 | 103 | The first step to write a Streams application is to create a java.util.Properties map to specify different Streams execution configuration values as defined in StreamsConfig. A couple of important configuration values you need to set are: StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, which specifies a list of host/port pairs to use for establishing the initial connection to the Kafka cluster, and StreamsConfig.APPLICATION_ID_CONFIG, which gives the unique identifier of your Streams application to distinguish itself with other applications talking to the same Kafka cluster: 104 | 105 | 1 106 | 2 107 | 3 108 | Properties props = new Properties(); 109 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pipe"); 110 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); // assuming that the Kafka broker this application is talking to runs on local machine with port 9092 111 | In addition, you can customize other configurations in the same map, for example, default serialization and deserialization libraries for the record key-value pairs: 112 | 113 | 1 114 | 2 115 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 116 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 117 | For a full list of configurations of Kafka Streams please refer to this table. 118 | 119 | Next we will define the computational logic of our Streams application. In Kafka Streams this computational logic is defined as a topology of connected processor nodes. We can use a topology builder to construct such a topology, 120 | 121 | 1 122 | final StreamsBuilder builder = new StreamsBuilder(); 123 | And then create a source stream from a Kafka topic named streams-plaintext-input using this topology builder: 124 | 125 | 1 126 | KStream source = builder.stream("streams-plaintext-input"); 127 | Now we get a KStream that is continuously generating records from its source Kafka topic streams-plaintext-input. The records are organized as String typed key-value pairs. The simplest thing we can do with this stream is to write it into another Kafka topic, say it's named streams-pipe-output: 128 | 129 | 1 130 | source.to("streams-pipe-output"); 131 | Note that we can also concatenate the above two lines into a single line as: 132 | 133 | 1 134 | builder.stream("streams-plaintext-input").to("streams-pipe-output"); 135 | We can inspect what kind of topology is created from this builder by doing the following: 136 | 137 | 1 138 | final Topology topology = builder.build(); 139 | And print its description to standard output as: 140 | 141 | 1 142 | System.out.println(topology.describe()); 143 | If we just stop here, compile and run the program, it will output the following information: 144 | 145 | 1 146 | 2 147 | 3 148 | 4 149 | 5 150 | 6 151 | 7 152 | 8 153 | > mvn clean package 154 | > mvn exec:java -Dexec.mainClass=myapps.Pipe 155 | Sub-topologies: 156 | Sub-topology: 0 157 | Source: KSTREAM-SOURCE-0000000000(topics: streams-plaintext-input) --> KSTREAM-SINK-0000000001 158 | Sink: KSTREAM-SINK-0000000001(topic: streams-pipe-output) <-- KSTREAM-SOURCE-0000000000 159 | Global Stores: 160 | none 161 | As shown above, it illustrates that the constructed topology has two processor nodes, a source node KSTREAM-SOURCE-0000000000 and a sink node KSTREAM-SINK-0000000001. KSTREAM-SOURCE-0000000000 continuously read records from Kafka topic streams-plaintext-input and pipe them to its downstream node KSTREAM-SINK-0000000001; KSTREAM-SINK-0000000001 will write each of its received record in order to another Kafka topic streams-pipe-output (the --> and <-- arrows dictates the downstream and upstream processor nodes of this node, i.e. "children" and "parents" within the topology graph). It also illustrates that this simple topology has no global state stores associated with it (we will talk about state stores more in the following sections). 162 | 163 | Note that we can always describe the topology as we did above at any given point while we are building it in the code, so as a user you can interactively "try and taste" your computational logic defined in the topology until you are happy with it. Suppose we are already done with this simple topology that just pipes data from one Kafka topic to another in an endless streaming manner, we can now construct the Streams client with the two components we have just constructed above: the configuration map and the topology object (one can also construct a StreamsConfig object from the props map and then pass that object to the constructor, KafkaStreams have overloaded constructor functions to takes either type). 164 | 165 | 1 166 | final KafkaStreams streams = new KafkaStreams(topology, props); 167 | By calling its start() function we can trigger the execution of this client. The execution won't stop until close() is called on this client. We can, for example, add a shutdown hook with a countdown latch to capture a user interrupt and close the client upon terminating this program: 168 | 169 | 1 170 | 2 171 | 3 172 | 4 173 | 5 174 | 6 175 | 7 176 | 8 177 | 9 178 | 10 179 | 11 180 | 12 181 | 13 182 | 14 183 | 15 184 | 16 185 | 17 186 | 18 187 | final CountDownLatch latch = new CountDownLatch(1); 188 | 189 | // attach shutdown handler to catch control-c 190 | Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") { 191 | @Override 192 | public void run() { 193 | streams.close(); 194 | latch.countDown(); 195 | } 196 | }); 197 | 198 | try { 199 | streams.start(); 200 | latch.await(); 201 | } catch (Throwable e) { 202 | System.exit(1); 203 | } 204 | System.exit(0); 205 | The complete code so far looks like this: 206 | 207 | 1 208 | 2 209 | 3 210 | 4 211 | 5 212 | 6 213 | 7 214 | 8 215 | 9 216 | 10 217 | 11 218 | 12 219 | 13 220 | 14 221 | 15 222 | 16 223 | 17 224 | 18 225 | 19 226 | 20 227 | 21 228 | 22 229 | 23 230 | 24 231 | 25 232 | 26 233 | 27 234 | 28 235 | 29 236 | 30 237 | 31 238 | 32 239 | 33 240 | 34 241 | 35 242 | 36 243 | 37 244 | 38 245 | 39 246 | 40 247 | 41 248 | 42 249 | 43 250 | 44 251 | 45 252 | 46 253 | 47 254 | package myapps; 255 | 256 | import org.apache.kafka.common.serialization.Serdes; 257 | import org.apache.kafka.streams.KafkaStreams; 258 | import org.apache.kafka.streams.StreamsBuilder; 259 | import org.apache.kafka.streams.StreamsConfig; 260 | import org.apache.kafka.streams.Topology; 261 | 262 | import java.util.Properties; 263 | import java.util.concurrent.CountDownLatch; 264 | 265 | public class Pipe { 266 | 267 | public static void main(String[] args) throws Exception { 268 | Properties props = new Properties(); 269 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-pipe"); 270 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 271 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 272 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 273 | 274 | final StreamsBuilder builder = new StreamsBuilder(); 275 | 276 | builder.stream("streams-plaintext-input").to("streams-pipe-output"); 277 | 278 | final Topology topology = builder.build(); 279 | 280 | final KafkaStreams streams = new KafkaStreams(topology, props); 281 | final CountDownLatch latch = new CountDownLatch(1); 282 | 283 | // attach shutdown handler to catch control-c 284 | Runtime.getRuntime().addShutdownHook(new Thread("streams-shutdown-hook") { 285 | @Override 286 | public void run() { 287 | streams.close(); 288 | latch.countDown(); 289 | } 290 | }); 291 | 292 | try { 293 | streams.start(); 294 | latch.await(); 295 | } catch (Throwable e) { 296 | System.exit(1); 297 | } 298 | System.exit(0); 299 | } 300 | } 301 | If you already have the Kafka broker up and running at localhost:9092, and the topics streams-plaintext-input and streams-pipe-output created on that broker, you can run this code in your IDE or on the command line, using Maven: 302 | 303 | > mvn clean package 304 | > mvn exec:java -Dexec.mainClass=myapps.Pipe 305 | 306 | For detailed instructions on how to run a Streams application and observe its computing results, please read the Play with a Streams Application section. We will not talk about this in the rest of this section. 307 | 308 | Writing a second Streams application: Line Split 309 | We have learned how to construct a Streams client with its two key components: the StreamsConfig and Topology. Now let's move on to add some real processing logic by augmenting the current topology. We can first create another program by first copy the existing Pipe.java class: 310 | 311 | > cp src/main/java/myapps/Pipe.java src/main/java/myapps/LineSplit.java 312 | 313 | And change its class name as well as the application id config to distinguish with the original program: 314 | 315 | 1 316 | 2 317 | 3 318 | 4 319 | 5 320 | 6 321 | 7 322 | 8 323 | public class LineSplit { 324 | 325 | public static void main(String[] args) throws Exception { 326 | Properties props = new Properties(); 327 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-linesplit"); 328 | // ... 329 | } 330 | } 331 | Since each of the source stream's record is a String typed key-value pair, let's treat the value string as a text line and split it into words with a FlatMapValues operator: 332 | 333 | 1 334 | 2 335 | 3 336 | 4 337 | 5 338 | 6 339 | 7 340 | KStream source = builder.stream("streams-plaintext-input"); 341 | KStream words = source.flatMapValues(new ValueMapper>() { 342 | @Override 343 | public Iterable apply(String value) { 344 | return Arrays.asList(value.split("\\W+")); 345 | } 346 | }); 347 | The operator will take the source stream as its input, and generate a new stream named words by processing each record from its source stream in order and breaking its value string into a list of words, and producing each word as a new record to the output words stream. This is a stateless operator that does not need to keep track of any previously received records or processed results. Note if you are using JDK 8 you can use lambda expression and simplify the above code as: 348 | 349 | 1 350 | 2 351 | KStream source = builder.stream("streams-plaintext-input"); 352 | KStream words = source.flatMapValues(value -> Arrays.asList(value.split("\\W+"))); 353 | And finally we can write the word stream back into another Kafka topic, say streams-linesplit-output. Again, these two steps can be concatenated as the following (assuming lambda expression is used): 354 | 355 | 1 356 | 2 357 | 3 358 | KStream source = builder.stream("streams-plaintext-input"); 359 | source.flatMapValues(value -> Arrays.asList(value.split("\\W+"))) 360 | .to("streams-linesplit-output"); 361 | If we now describe this augmented topology as System.out.println(topology.describe()), we will get the following: 362 | 363 | 1 364 | 2 365 | 3 366 | 4 367 | 5 368 | 6 369 | 7 370 | 8 371 | 9 372 | > mvn clean package 373 | > mvn exec:java -Dexec.mainClass=myapps.LineSplit 374 | Sub-topologies: 375 | Sub-topology: 0 376 | Source: KSTREAM-SOURCE-0000000000(topics: streams-plaintext-input) --> KSTREAM-FLATMAPVALUES-0000000001 377 | Processor: KSTREAM-FLATMAPVALUES-0000000001(stores: []) --> KSTREAM-SINK-0000000002 <-- KSTREAM-SOURCE-0000000000 378 | Sink: KSTREAM-SINK-0000000002(topic: streams-linesplit-output) <-- KSTREAM-FLATMAPVALUES-0000000001 379 | Global Stores: 380 | none 381 | As we can see above, a new processor node KSTREAM-FLATMAPVALUES-0000000001 is injected into the topology between the original source and sink nodes. It takes the source node as its parent and the sink node as its child. In other words, each record fetched by the source node will first traverse to the newly added KSTREAM-FLATMAPVALUES-0000000001 node to be processed, and one or more new records will be generated as a result. They will continue traverse down to the sink node to be written back to Kafka. Note this processor node is "stateless" as it is not associated with any stores (i.e. (stores: [])). 382 | 383 | The complete code looks like this (assuming lambda expression is used): 384 | 385 | 1 386 | 2 387 | 3 388 | 4 389 | 5 390 | 6 391 | 7 392 | 8 393 | 9 394 | 10 395 | 11 396 | 12 397 | 13 398 | 14 399 | 15 400 | 16 401 | 17 402 | 18 403 | 19 404 | 20 405 | 21 406 | 22 407 | 23 408 | 24 409 | 25 410 | 26 411 | 27 412 | 28 413 | 29 414 | 30 415 | 31 416 | 32 417 | 33 418 | 34 419 | 35 420 | package myapps; 421 | 422 | import org.apache.kafka.common.serialization.Serdes; 423 | import org.apache.kafka.streams.KafkaStreams; 424 | import org.apache.kafka.streams.StreamsBuilder; 425 | import org.apache.kafka.streams.StreamsConfig; 426 | import org.apache.kafka.streams.Topology; 427 | import org.apache.kafka.streams.kstream.KStream; 428 | 429 | import java.util.Arrays; 430 | import java.util.Properties; 431 | import java.util.concurrent.CountDownLatch; 432 | 433 | public class LineSplit { 434 | 435 | public static void main(String[] args) throws Exception { 436 | Properties props = new Properties(); 437 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-linesplit"); 438 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 439 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 440 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 441 | 442 | final StreamsBuilder builder = new StreamsBuilder(); 443 | 444 | KStream source = builder.stream("streams-plaintext-input"); 445 | source.flatMapValues(value -> Arrays.asList(value.split("\\W+"))) 446 | .to("streams-linesplit-output"); 447 | 448 | final Topology topology = builder.build(); 449 | final KafkaStreams streams = new KafkaStreams(topology, props); 450 | final CountDownLatch latch = new CountDownLatch(1); 451 | 452 | // ... same as Pipe.java above 453 | } 454 | } 455 | Writing a third Streams application: Wordcount 456 | Let's now take a step further to add some "stateful" computations to the topology by counting the occurrence of the words split from the source text stream. Following similar steps let's create another program based on the LineSplit.java class: 457 | 458 | 1 459 | 2 460 | 3 461 | 4 462 | 5 463 | 6 464 | 7 465 | 8 466 | public class WordCount { 467 | 468 | public static void main(String[] args) throws Exception { 469 | Properties props = new Properties(); 470 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount"); 471 | // ... 472 | } 473 | } 474 | In order to count the words we can first modify the flatMapValues operator to treat all of them as lower case (assuming lambda expression is used): 475 | 476 | 1 477 | 2 478 | 3 479 | 4 480 | 5 481 | 6 482 | source.flatMapValues(new ValueMapper>() { 483 | @Override 484 | public Iterable apply(String value) { 485 | return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+")); 486 | } 487 | }); 488 | In order to do the counting aggregation we have to first specify that we want to key the stream on the value string, i.e. the lower cased word, with a groupBy operator. This operator generate a new grouped stream, which can then be aggregated by a count operator, which generates a running count on each of the grouped keys: 489 | 490 | 1 491 | 2 492 | 3 493 | 4 494 | 5 495 | 6 496 | 7 497 | 8 498 | 9 499 | 10 500 | 11 501 | 12 502 | 13 503 | 14 504 | 15 505 | 16 506 | KTable counts = 507 | source.flatMapValues(new ValueMapper>() { 508 | @Override 509 | public Iterable apply(String value) { 510 | return Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+")); 511 | } 512 | }) 513 | .groupBy(new KeyValueMapper() { 514 | @Override 515 | public String apply(String key, String value) { 516 | return value; 517 | } 518 | }) 519 | // Materialize the result into a KeyValueStore named "counts-store". 520 | // The Materialized store is always of type as this is the format of the inner most store. 521 | .count(Materialized.> as("counts-store")); 522 | Note that the count operator has a Materialized parameter that specifies that the running count should be stored in a state store named counts-store. This Counts store can be queried in real-time, with details described in the Developer Manual. 523 | 524 | We can also write the counts KTable's changelog stream back into another Kafka topic, say streams-wordcount-output. Because the result is a changelog stream, the output topic streams-wordcount-output should be configured with log compaction enabled. Note that this time the value type is no longer String but Long, so the default serialization classes are not viable for writing it to Kafka anymore. We need to provide overridden serialization methods for Long types, otherwise a runtime exception will be thrown: 525 | 526 | 1 527 | counts.toStream().to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long())); 528 | Note that in order to read the changelog stream from topic streams-wordcount-output, one needs to set the value deserialization as org.apache.kafka.common.serialization.LongDeserializer. Details of this can be found in the Play with a Streams Application section. Assuming lambda expression from JDK 8 can be used, the above code can be simplified as: 529 | 530 | 1 531 | 2 532 | 3 533 | 4 534 | 5 535 | 6 536 | KStream source = builder.stream("streams-plaintext-input"); 537 | source.flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"))) 538 | .groupBy((key, value) -> value) 539 | .count(Materialized.>as("counts-store")) 540 | .toStream() 541 | .to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()); 542 | If we again describe this augmented topology as System.out.println(topology.describe()), we will get the following: 543 | 544 | 1 545 | 2 546 | 3 547 | 4 548 | 5 549 | 6 550 | 7 551 | 8 552 | 9 553 | 10 554 | 11 555 | 12 556 | 13 557 | 14 558 | 15 559 | 16 560 | > mvn clean package 561 | > mvn exec:java -Dexec.mainClass=myapps.WordCount 562 | Sub-topologies: 563 | Sub-topology: 0 564 | Source: KSTREAM-SOURCE-0000000000(topics: streams-plaintext-input) --> KSTREAM-FLATMAPVALUES-0000000001 565 | Processor: KSTREAM-FLATMAPVALUES-0000000001(stores: []) --> KSTREAM-KEY-SELECT-0000000002 <-- KSTREAM-SOURCE-0000000000 566 | Processor: KSTREAM-KEY-SELECT-0000000002(stores: []) --> KSTREAM-FILTER-0000000005 <-- KSTREAM-FLATMAPVALUES-0000000001 567 | Processor: KSTREAM-FILTER-0000000005(stores: []) --> KSTREAM-SINK-0000000004 <-- KSTREAM-KEY-SELECT-0000000002 568 | Sink: KSTREAM-SINK-0000000004(topic: Counts-repartition) <-- KSTREAM-FILTER-0000000005 569 | Sub-topology: 1 570 | Source: KSTREAM-SOURCE-0000000006(topics: Counts-repartition) --> KSTREAM-AGGREGATE-0000000003 571 | Processor: KSTREAM-AGGREGATE-0000000003(stores: [Counts]) --> KTABLE-TOSTREAM-0000000007 <-- KSTREAM-SOURCE-0000000006 572 | Processor: KTABLE-TOSTREAM-0000000007(stores: []) --> KSTREAM-SINK-0000000008 <-- KSTREAM-AGGREGATE-0000000003 573 | Sink: KSTREAM-SINK-0000000008(topic: streams-wordcount-output) <-- KTABLE-TOSTREAM-0000000007 574 | Global Stores: 575 | none 576 | As we can see above, the topology now contains two disconnected sub-topologies. The first sub-topology's sink node KSTREAM-SINK-0000000004 will write to a repartition topic Counts-repartition, which will be read by the second sub-topology's source node KSTREAM-SOURCE-0000000006. The repartition topic is used to "shuffle" the source stream by its aggregation key, which is in this case the value string. In addition, inside the first sub-topology a stateless KSTREAM-FILTER-0000000005 node is injected between the grouping KSTREAM-KEY-SELECT-0000000002 node and the sink node to filter out any intermediate record whose aggregate key is empty. 577 | 578 | In the second sub-topology, the aggregation node KSTREAM-AGGREGATE-0000000003 is associated with a state store named Counts (the name is specified by the user in the count operator). Upon receiving each record from its upcoming stream source node, the aggregation processor will first query its associated Counts store to get the current count for that key, augment by one, and then write the new count back to the store. Each updated count for the key will also be piped downstream to the KTABLE-TOSTREAM-0000000007 node, which interpret this update stream as a record stream before further piping to the sink node KSTREAM-SINK-0000000008 for writing back to Kafka. 579 | 580 | The complete code looks like this (assuming lambda expression is used): 581 | 582 | 1 583 | 2 584 | 3 585 | 4 586 | 5 587 | 6 588 | 7 589 | 8 590 | 9 591 | 10 592 | 11 593 | 12 594 | 13 595 | 14 596 | 15 597 | 16 598 | 17 599 | 18 600 | 19 601 | 20 602 | 21 603 | 22 604 | 23 605 | 24 606 | 25 607 | 26 608 | 27 609 | 28 610 | 29 611 | 30 612 | 31 613 | 32 614 | 33 615 | 34 616 | 35 617 | 36 618 | 37 619 | 38 620 | 39 621 | package myapps; 622 | 623 | import org.apache.kafka.common.serialization.Serdes; 624 | import org.apache.kafka.streams.KafkaStreams; 625 | import org.apache.kafka.streams.StreamsBuilder; 626 | import org.apache.kafka.streams.StreamsConfig; 627 | import org.apache.kafka.streams.Topology; 628 | import org.apache.kafka.streams.kstream.KStream; 629 | 630 | import java.util.Arrays; 631 | import java.util.Locale; 632 | import java.util.Properties; 633 | import java.util.concurrent.CountDownLatch; 634 | 635 | public class WordCount { 636 | 637 | public static void main(String[] args) throws Exception { 638 | Properties props = new Properties(); 639 | props.put(StreamsConfig.APPLICATION_ID_CONFIG, "streams-wordcount"); 640 | props.put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:9092"); 641 | props.put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 642 | props.put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Serdes.String().getClass()); 643 | 644 | final StreamsBuilder builder = new StreamsBuilder(); 645 | 646 | KStream source = builder.stream("streams-plaintext-input"); 647 | source.flatMapValues(value -> Arrays.asList(value.toLowerCase(Locale.getDefault()).split("\\W+"))) 648 | .groupBy((key, value) -> value) 649 | .count(Materialized.>as("counts-store")) 650 | .toStream() 651 | .to("streams-wordcount-output", Produced.with(Serdes.String(), Serdes.Long()); 652 | 653 | final Topology topology = builder.build(); 654 | final KafkaStreams streams = new KafkaStreams(topology, props); 655 | final CountDownLatch latch = new CountDownLatch(1); 656 | 657 | // ... same as Pipe.java above 658 | } 659 | } 660 | Previous 661 | Next 662 | The contents of this website are © 2017 Apache Software Foundation under the terms of the Apache License v2. Apache Kafka, Kafka, and the Kafka logo are either registered trademarks or trademarks of The Apache Software Foundation in the United States and other countries. 663 | Apache Feather 664 | --------------------------------------------------------------------------------