├── .classpath ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .project ├── .settings ├── org.eclipse.core.resources.prefs ├── org.eclipse.jdt.core.prefs ├── org.eclipse.m2e.core.prefs └── org.eclipse.wst.common.project.facet.core.xml ├── .springBeans ├── README.md ├── checkpoint ├── .checkpoint-1519452350000.crc ├── .checkpoint-1519452360000.bk.crc ├── .checkpoint-1519452360000.crc ├── .checkpoint-1519452370000.bk.crc ├── .checkpoint-1519452370000.crc ├── .checkpoint-1519452380000.bk.crc ├── .checkpoint-1519452380000.crc ├── .checkpoint-1519452390000.bk.crc ├── .checkpoint-1519452390000.crc ├── .checkpoint-1519452400000.crc ├── checkpoint-1519452350000 ├── checkpoint-1519452360000 ├── checkpoint-1519452360000.bk ├── checkpoint-1519452370000 ├── checkpoint-1519452370000.bk ├── checkpoint-1519452380000 ├── checkpoint-1519452380000.bk ├── checkpoint-1519452390000 ├── checkpoint-1519452390000.bk ├── checkpoint-1519452400000 └── receivedBlockMetadata │ └── log-1519452351120-1519452411120 ├── mvnw ├── mvnw.cmd ├── pom.xml ├── src ├── main │ ├── java │ │ └── com │ │ │ └── swjuyhz │ │ │ └── sample │ │ │ ├── SampleApplication.java │ │ │ ├── config │ │ │ └── SparkConfig.java │ │ │ └── sparkstream │ │ │ ├── ApplicationStartup.java │ │ │ └── executor │ │ │ └── SparkKafkaStreamExecutor.java │ └── resources │ │ └── application.properties └── test │ └── java │ └── com │ └── swjuyhz │ └── sample │ └── SampleApplicationTests.java └── target ├── classes ├── META-INF │ ├── MANIFEST.MF │ └── maven │ │ └── com.swjuyhz │ │ └── spring-boot-spark-streaming-kafka-sample │ │ ├── pom.properties │ │ └── pom.xml ├── application.properties └── com │ └── swjuyhz │ └── sample │ ├── SampleApplication.class │ ├── config │ └── SparkConfig.class │ └── sparkstream │ ├── ApplicationStartup.class │ └── executor │ ├── SparkKafkaStreamExecutor$1.class │ └── SparkKafkaStreamExecutor.class └── test-classes └── com └── swjuyhz └── sample └── SampleApplicationTests.class /.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip 2 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | spring-boot-spark-streaming-kafka-sample 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.wst.common.project.facet.core.builder 10 | 11 | 12 | 13 | 14 | org.eclipse.jdt.core.javabuilder 15 | 16 | 17 | 18 | 19 | org.springframework.ide.eclipse.core.springbuilder 20 | 21 | 22 | 23 | 24 | org.eclipse.m2e.core.maven2Builder 25 | 26 | 27 | 28 | 29 | 30 | org.springframework.ide.eclipse.core.springnature 31 | org.eclipse.jdt.core.javanature 32 | org.eclipse.m2e.core.maven2Nature 33 | org.eclipse.wst.common.project.facet.core.nature 34 | 35 | 36 | -------------------------------------------------------------------------------- /.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/main/java=UTF-8 3 | encoding//src/main/resources=UTF-8 4 | encoding//src/main/resources/application.properties=UTF-8 5 | encoding//src/test/java=UTF-8 6 | encoding/=UTF-8 7 | -------------------------------------------------------------------------------- /.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 3 | org.eclipse.jdt.core.compiler.compliance=1.8 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.source=1.8 6 | -------------------------------------------------------------------------------- /.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles=pom.xml 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /.settings/org.eclipse.wst.common.project.facet.core.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | -------------------------------------------------------------------------------- /.springBeans: -------------------------------------------------------------------------------- 1 | 2 | 3 | 1 4 | 5 | 6 | 7 | 8 | 9 | 10 | java:com.swjuyhz.sample.SampleApplication 11 | 12 | 13 | 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # spring-boot-spark-streaming-kafka-sample 2 | An example, spark-streaming-kafka integrates with spring-boot. 3 | 4 | 博文地址：http://blog.csdn.net/swjtu_yhz/article/details/79361472 5 | -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452350000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452350000.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452360000.bk.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452360000.bk.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452360000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452360000.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452370000.bk.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452370000.bk.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452370000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452370000.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452380000.bk.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452380000.bk.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452380000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452380000.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452390000.bk.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452390000.bk.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452390000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452390000.crc -------------------------------------------------------------------------------- /checkpoint/.checkpoint-1519452400000.crc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/.checkpoint-1519452400000.crc -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452350000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452350000 -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452360000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452360000 -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452360000.bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452360000.bk -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452370000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452370000 -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452370000.bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452370000.bk -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452380000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452380000 -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452380000.bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452380000.bk -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452390000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452390000 -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452390000.bk: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452390000.bk -------------------------------------------------------------------------------- /checkpoint/checkpoint-1519452400000: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/checkpoint-1519452400000 -------------------------------------------------------------------------------- /checkpoint/receivedBlockMetadata/log-1519452351120-1519452411120: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/checkpoint/receivedBlockMetadata/log-1519452351120-1519452411120 -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> $.*$$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '$[^ ]*$'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '$[^ ]*$'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '$.*$/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.swjuyhz 7 | spring-boot-spark-streaming-kafka-sample 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | spring-boot-spark-streaming-kafka-sample 12 | spring boot integrate spark and spark stream kafka 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 1.5.10.RELEASE 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 1.8 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-web 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-tomcat 39 | 40 | 41 | 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-starter-undertow 46 | 47 | 48 | org.springframework.boot 49 | spring-boot-starter-test 50 | test 51 | 52 | 53 | org.apache.spark 54 | spark-mllib_2.11 55 | 2.2.0 56 | provided 57 | 58 | 59 | org.codehaus.janino 60 | commons-compiler 61 | 62 | 63 | 64 | 65 | org.codehaus.janino 66 | commons-compiler 67 | 2.7.8 68 | provided 69 | 70 | 71 | org.scala-lang.modules 72 | scala-xml_2.11 73 | 1.0.6 74 | 75 | 76 | org.apache.spark 77 | spark-streaming-kafka-0-8_2.11 78 | 2.2.0 79 | 80 | 81 | 82 | org.hibernate 83 | hibernate-validator 84 | 5.3.6.Final 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | org.springframework.boot 93 | spring-boot-maven-plugin 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /src/main/java/com/swjuyhz/sample/SampleApplication.java: -------------------------------------------------------------------------------- 1 | package com.swjuyhz.sample; 2 | 3 | 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.context.annotation.Bean; 7 | import com.google.gson.Gson; 8 | import com.swjuyhz.sample.sparkstream.ApplicationStartup; 9 | 10 | @SpringBootApplication 11 | public class SampleApplication { 12 | 13 | public static void main(String[] args) { 14 | SpringApplication springApplication = new SpringApplication(SampleApplication.class); 15 | springApplication.addListeners(new ApplicationStartup()); 16 | springApplication.run(args); 17 | } 18 | 19 | //将Gson划归为spring管理 20 | @Bean 21 | public Gson gson() { 22 | return new Gson(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/swjuyhz/sample/config/SparkConfig.java: -------------------------------------------------------------------------------- 1 | package com.swjuyhz.sample.config; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | 9 | import org.apache.spark.SparkConf; 10 | import org.apache.spark.api.java.JavaSparkContext; 11 | 12 | @Configuration 13 | public class SparkConfig { 14 | @Value("${spring.application.name}") 15 | private String sparkAppName; 16 | @Value("${spark.master}") 17 | private String sparkMasteer; 18 | @Value("${spark.stream.kafka.durations}") 19 | private String streamDurationTime; 20 | @Value("${spark.driver.memory}") 21 | private String sparkDriverMemory; 22 | @Value("${spark.worker.memory}") 23 | private String sparkWorkerMemory; 24 | @Value("${spark.executor.memory}") 25 | private String sparkExecutorMemory; 26 | @Value("${spark.rpc.message.maxSize}") 27 | private String sparkRpcMessageMaxSize; 28 | 29 | @Bean 30 | @ConditionalOnMissingBean(SparkConf.class) 31 | public SparkConf sparkConf() { 32 | SparkConf conf = new SparkConf() 33 | .setAppName(sparkAppName) 34 | .setMaster(sparkMasteer).set("spark.driver.memory",sparkDriverMemory) 35 | .set("spark.worker.memory",sparkWorkerMemory)//"26g".set("spark.shuffle.memoryFraction","0") //默认0.2 36 | .set("spark.executor.memory",sparkExecutorMemory) 37 | .set("spark.rpc.message.maxSize",sparkRpcMessageMaxSize); 38 | // .setMaster("local[*]");//just use in test 39 | return conf; 40 | } 41 | 42 | @Bean 43 | @ConditionalOnMissingBean(JavaSparkContext.class) //默认： JVM 只允许存在一个sparkcontext 44 | public JavaSparkContext javaSparkContext(@Autowired SparkConf sparkConf) { 45 | return new JavaSparkContext(sparkConf); 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/com/swjuyhz/sample/sparkstream/ApplicationStartup.java: -------------------------------------------------------------------------------- 1 | package com.swjuyhz.sample.sparkstream; 2 | 3 | import org.springframework.context.ApplicationContext; 4 | import org.springframework.context.ApplicationListener; 5 | import org.springframework.context.event.ContextRefreshedEvent; 6 | 7 | import com.swjuyhz.sample.sparkstream.executor.SparkKafkaStreamExecutor; 8 | /** 9 | * spring boot 容器加载完成后执行 10 | * 启动kafka数据接收和处理 11 | * @author yonghao.zheng 12 | * 13 | */ 14 | public class ApplicationStartup implements ApplicationListener { 15 | 16 | @Override 17 | public void onApplicationEvent(ContextRefreshedEvent event) { 18 | ApplicationContext ac = event.getApplicationContext(); 19 | SparkKafkaStreamExecutor sparkKafkaStreamExecutor= ac.getBean(SparkKafkaStreamExecutor.class); 20 | Thread thread = new Thread(sparkKafkaStreamExecutor); 21 | thread.start(); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/com/swjuyhz/sample/sparkstream/executor/SparkKafkaStreamExecutor.java: -------------------------------------------------------------------------------- 1 | package com.swjuyhz.sample.sparkstream.executor; 2 | 3 | import java.io.Serializable; 4 | import java.util.Arrays; 5 | import java.util.HashMap; 6 | import java.util.HashSet; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | import org.apache.spark.api.java.JavaSparkContext; 14 | import org.apache.spark.streaming.Durations; 15 | import org.apache.spark.streaming.api.java.JavaPairInputDStream; 16 | import org.apache.spark.streaming.api.java.JavaStreamingContext; 17 | import org.apache.spark.streaming.kafka.KafkaUtils; 18 | import org.springframework.beans.factory.annotation.Autowired; 19 | import org.springframework.beans.factory.annotation.Value; 20 | import org.springframework.stereotype.Component; 21 | 22 | import com.google.gson.Gson; 23 | import com.google.gson.reflect.TypeToken; 24 | 25 | import kafka.serializer.StringDecoder; 26 | 27 | @Component 28 | public class SparkKafkaStreamExecutor implements Serializable,Runnable{ 29 | /** 30 | * 31 | */ 32 | private static final long serialVersionUID = 1L; 33 | private static final Logger log = LoggerFactory.getLogger(SparkKafkaStreamExecutor.class); 34 | 35 | @Value("${spark.stream.kafka.durations}") 36 | private String streamDurationTime; 37 | @Value("${kafka.broker.list}") 38 | private String metadatabrokerlist; 39 | @Value("${spark.kafka.topics}") 40 | private String topicsAll; 41 | @Autowired 42 | private transient Gson gson; 43 | 44 | private transient JavaStreamingContext jsc; 45 | @Autowired 46 | private transient JavaSparkContext javaSparkContext; 47 | 48 | @Override 49 | public void run() { 50 | startStreamTask(); 51 | } 52 | 53 | public void startStreamTask() { 54 | // System.setProperty("hadoop.home.dir", "D:\\hadoop-2.7.5"); 55 | Set topics = new HashSet(Arrays.asList(topicsAll.split(","))); 56 | 57 | Map kafkaParams = new HashMap<>(); 58 | kafkaParams.put("metadata.broker.list", metadatabrokerlist); 59 | 60 | jsc = new JavaStreamingContext(javaSparkContext, 61 | Durations.seconds(Integer.valueOf(streamDurationTime))); 62 | jsc.checkpoint("checkpoint"); //保证元数据恢复，就是Driver端挂了之后数据仍然可以恢复 63 | 64 | // 得到数据流 65 | final JavaPairInputDStream stream = KafkaUtils.createDirectStream(jsc, String.class, 66 | String.class, StringDecoder.class, StringDecoder.class, kafkaParams, topics); 67 | System.out.println("stream started!"); 68 | stream.print(); 69 | stream.foreachRDD(v -> { 70 | //针对单篇文章流式处理 71 | List topicDatas = v.values().collect();//优化点：不收集而是并发节点处理？ 72 | for (String topicData : topicDatas) { 73 | List> list = gson 74 | .fromJson(topicData, new TypeToken>>() {}.getType()); 75 | list.parallelStream().forEach(m->{ 76 | //do something 77 | System.out.println(m); 78 | }); 79 | } 80 | log.info("一批次数据流处理完： {}",topicDatas); 81 | }); 82 | //优化：为每个分区创建一个连接 83 | // stream.foreachRDD(t->{ 84 | // t.foreachPartition(f->{ 85 | // while(f.hasNext()) { 86 | // Map symbolLDAHandlered =LDAModelPpl 87 | // .LDAHandlerOneArticle(sparkSession, SymbolAndNews.symbolHandlerOneArticle(sparkSession, f.next()._2)); 88 | // } 89 | // }); 90 | // }); 91 | jsc.start(); 92 | } 93 | 94 | public void destoryStreamTask() { 95 | if(jsc!=null) { 96 | jsc.stop(); 97 | } 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.application.name=spring-boot-spark-streaming-kafka-sample 2 | # LOGGING 3 | logging.level.root=info 4 | logging.path=/applog/${spring.application.name} 5 | #spark config start 6 | spark.driver.memory=32g 7 | spark.worker.memory=25g 8 | spark.executor.memory=25g 9 | spark.rpc.message.maxSize=1024 10 | #spark master 11 | spark.master = spark://10.222.17.21:7077 12 | #spark topics ','号分割 13 | spark.kafka.topics = news 14 | #kafka集群地址，'，'号分割 15 | kafka.broker.list = 10.222.17.21:19092 16 | #从kafka拉数据的间隔时间，单位 S 17 | spark.stream.kafka.durations=10 18 | #spark config end 19 | -------------------------------------------------------------------------------- /src/test/java/com/swjuyhz/sample/SampleApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.swjuyhz.sample; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class SampleApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /target/classes/META-INF/MANIFEST.MF: -------------------------------------------------------------------------------- 1 | Manifest-Version: 1.0 2 | Implementation-Title: spring-boot-spark-streaming-kafka-sample 3 | Implementation-Version: 0.0.1-SNAPSHOT 4 | Built-By: yonghao.zheng 5 | Implementation-Vendor-Id: com.swjuyhz 6 | Build-Jdk: 1.8.0_144 7 | Implementation-URL: http://projects.spring.io/spring-boot/spring-boot- 8 | spark-streaming-kafka-sample/ 9 | Created-By: Maven Integration for Eclipse 10 | Implementation-Vendor: Pivotal Software, Inc. 11 | 12 | -------------------------------------------------------------------------------- /target/classes/META-INF/maven/com.swjuyhz/spring-boot-spark-streaming-kafka-sample/pom.properties: -------------------------------------------------------------------------------- 1 | #Generated by Maven Integration for Eclipse 2 | #Sat Feb 24 13:52:56 CST 2018 3 | version=0.0.1-SNAPSHOT 4 | groupId=com.swjuyhz 5 | m2e.projectName=spring-boot-spark-streaming-kafka-sample 6 | m2e.projectLocation=E\:\\eclipse-workspace\\spring-boot-spark-streaming-kafka-sample 7 | artifactId=spring-boot-spark-streaming-kafka-sample 8 | -------------------------------------------------------------------------------- /target/classes/META-INF/maven/com.swjuyhz/spring-boot-spark-streaming-kafka-sample/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | com.swjuyhz 7 | spring-boot-spark-streaming-kafka-sample 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | spring-boot-spark-streaming-kafka-sample 12 | spring boot integrate spark and spark stream kafka 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 1.5.10.RELEASE 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 1.8 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-web 35 | 36 | 37 | org.springframework.boot 38 | spring-boot-starter-tomcat 39 | 40 | 41 | 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-starter-undertow 46 | 47 | 48 | org.springframework.boot 49 | spring-boot-starter-test 50 | test 51 | 52 | 53 | org.apache.spark 54 | spark-mllib_2.11 55 | 2.2.0 56 | provided 57 | 58 | 59 | org.codehaus.janino 60 | commons-compiler 61 | 62 | 63 | 64 | 65 | org.codehaus.janino 66 | commons-compiler 67 | 2.7.8 68 | provided 69 | 70 | 71 | org.scala-lang.modules 72 | scala-xml_2.11 73 | 1.0.6 74 | 75 | 76 | org.apache.spark 77 | spark-streaming-kafka-0-8_2.11 78 | 2.2.0 79 | 80 | 81 | 82 | org.hibernate 83 | hibernate-validator 84 | 5.3.6.Final 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | org.springframework.boot 93 | spring-boot-maven-plugin 94 | 95 | 96 | 97 | 98 | 99 | 100 | -------------------------------------------------------------------------------- /target/classes/application.properties: -------------------------------------------------------------------------------- 1 | spring.application.name=spring-boot-spark-streaming-kafka-sample 2 | # LOGGING 3 | logging.level.root=info 4 | logging.path=/applog/${spring.application.name} 5 | #spark config start 6 | spark.driver.memory=32g 7 | spark.worker.memory=25g 8 | spark.executor.memory=25g 9 | spark.rpc.message.maxSize=1024 10 | #spark master 11 | spark.master = spark://10.222.17.21:7077 12 | #spark topics ','号分割 13 | spark.kafka.topics = news 14 | #kafka集群地址，'，'号分割 15 | kafka.broker.list = 10.222.17.21:19092 16 | #从kafka拉数据的间隔时间，单位 S 17 | spark.stream.kafka.durations=10 18 | #spark config end 19 | -------------------------------------------------------------------------------- /target/classes/com/swjuyhz/sample/SampleApplication.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/target/classes/com/swjuyhz/sample/SampleApplication.class -------------------------------------------------------------------------------- /target/classes/com/swjuyhz/sample/config/SparkConfig.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/target/classes/com/swjuyhz/sample/config/SparkConfig.class -------------------------------------------------------------------------------- /target/classes/com/swjuyhz/sample/sparkstream/ApplicationStartup.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/target/classes/com/swjuyhz/sample/sparkstream/ApplicationStartup.class -------------------------------------------------------------------------------- /target/classes/com/swjuyhz/sample/sparkstream/executor/SparkKafkaStreamExecutor$1.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/target/classes/com/swjuyhz/sample/sparkstream/executor/SparkKafkaStreamExecutor$1.class -------------------------------------------------------------------------------- /target/classes/com/swjuyhz/sample/sparkstream/executor/SparkKafkaStreamExecutor.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/target/classes/com/swjuyhz/sample/sparkstream/executor/SparkKafkaStreamExecutor.class -------------------------------------------------------------------------------- /target/test-classes/com/swjuyhz/sample/SampleApplicationTests.class: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swjuyhz/spring-boot-spark-streaming-kafka-sample/e65427c1549c802a3a8f4ca954e00aa7ee8fb193/target/test-classes/com/swjuyhz/sample/SampleApplicationTests.class --------------------------------------------------------------------------------