├── README.md ├── mvnw ├── mvnw.cmd ├── pom.xml ├── sparkjob ├── dependency-reduced-pom.xml ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── hrong │ └── job │ └── etl │ └── DbTableEtl.java └── src ├── main ├── java │ └── com │ │ └── hrong │ │ └── springbootspark │ │ ├── SpringbootSparkApplication.java │ │ ├── controller │ │ └── SparkController.java │ │ ├── entity │ │ └── SparkApplicationParam.java │ │ ├── service │ │ ├── ISparkSubmitService.java │ │ └── impl │ │ │ └── SparkSubmitServiceImpl.java │ │ ├── util │ │ └── HttpUtil.java │ │ └── vo │ │ ├── DataBaseExtractorVo.java │ │ └── Result.java └── resources │ ├── application-spark.properties │ ├── application.properties │ └── logback-spring.xml └── test └── java └── com └── hrong └── springbootspark └── SpringbootSparkApplicationTests.java /README.md: -------------------------------------------------------------------------------- 1 | # springboot-spark 2 | 使用springboot构建rest api远程提交spark任务，博客地址：https://blog.csdn.net/hlp4207/article/details/100831384 3 | 4 | **使用方法：** 5 | 1. 前台调用Restful接口，传入spark任务所需参数 6 | 2. 后台接收到参数，使用SparkLauncher构建远程提交spark任务对象 7 | ``` 8 | SparkLauncher launcher = new SparkLauncher() 9 | .setAppResource(sparkAppParams.getJarPath()) 10 | .setMainClass(sparkAppParams.getMainClass()) 11 | .setMaster(sparkAppParams.getMaster()) 12 | .setDeployMode(sparkAppParams.getDeployMode()) 13 | .setConf("spark.driver.memory", sparkAppParams.getDriverMemory()) 14 | .setConf("spark.executor.memory", sparkAppParams.getExecutorMemory()) 15 | .setConf("spark.executor.cores", sparkAppParams.getExecutorCores()); 16 | ``` 17 | 3. 使用CountDownLatch阻塞线程，当任务结束后，调用spark api获取执行结果 18 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | ########################################################################################## 204 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 205 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 206 | ########################################################################################## 207 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 208 | if [ "$MVNW_VERBOSE" = true ]; then 209 | echo "Found .mvn/wrapper/maven-wrapper.jar" 210 | fi 211 | else 212 | if [ "$MVNW_VERBOSE" = true ]; then 213 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 214 | fi 215 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" 216 | while IFS="=" read key value; do 217 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 218 | esac 219 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 220 | if [ "$MVNW_VERBOSE" = true ]; then 221 | echo "Downloading from: $jarUrl" 222 | fi 223 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 224 | 225 | if command -v wget > /dev/null; then 226 | if [ "$MVNW_VERBOSE" = true ]; then 227 | echo "Found wget ... using wget" 228 | fi 229 | wget "$jarUrl" -O "$wrapperJarPath" 230 | elif command -v curl > /dev/null; then 231 | if [ "$MVNW_VERBOSE" = true ]; then 232 | echo "Found curl ... using curl" 233 | fi 234 | curl -o "$wrapperJarPath" "$jarUrl" 235 | else 236 | if [ "$MVNW_VERBOSE" = true ]; then 237 | echo "Falling back to using Java to download" 238 | fi 239 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 240 | if [ -e "$javaClass" ]; then 241 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 242 | if [ "$MVNW_VERBOSE" = true ]; then 243 | echo " - Compiling MavenWrapperDownloader.java ..." 244 | fi 245 | # Compiling the Java class 246 | ("$JAVA_HOME/bin/javac" "$javaClass") 247 | fi 248 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 249 | # Running the downloader 250 | if [ "$MVNW_VERBOSE" = true ]; then 251 | echo " - Running MavenWrapperDownloader.java ..." 252 | fi 253 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 254 | fi 255 | fi 256 | fi 257 | fi 258 | ########################################################################################## 259 | # End of extension 260 | ########################################################################################## 261 | 262 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 263 | if [ "$MVNW_VERBOSE" = true ]; then 264 | echo $MAVEN_PROJECTBASEDIR 265 | fi 266 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 267 | 268 | # For Cygwin, switch paths to Windows format before running java 269 | if $cygwin; then 270 | [ -n "$M2_HOME" ] && 271 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 272 | [ -n "$JAVA_HOME" ] && 273 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 274 | [ -n "$CLASSPATH" ] && 275 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 276 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 277 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 278 | fi 279 | 280 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 281 | 282 | exec "$JAVACMD" \ 283 | $MAVEN_OPTS \ 284 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 285 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 286 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 287 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar" 124 | FOR /F "tokens=1,2 delims==" %%A IN (%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties) DO ( 125 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 126 | ) 127 | 128 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 129 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 130 | if exist %WRAPPER_JAR% ( 131 | echo Found %WRAPPER_JAR% 132 | ) else ( 133 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 134 | echo Downloading from: %DOWNLOAD_URL% 135 | powershell -Command "(New-Object Net.WebClient).DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')" 136 | echo Finished downloading %WRAPPER_JAR% 137 | ) 138 | @REM End of extension 139 | 140 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 141 | if ERRORLEVEL 1 goto error 142 | goto end 143 | 144 | :error 145 | set ERROR_CODE=1 146 | 147 | :end 148 | @endlocal & set ERROR_CODE=%ERROR_CODE% 149 | 150 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 151 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 152 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 153 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 154 | :skipRcPost 155 | 156 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 157 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 158 | 159 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 160 | 161 | exit /B %ERROR_CODE% 162 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.1.8.RELEASE 9 | 10 | 11 | com.hrong 12 | springboot-spark 13 | 0.0.1-SNAPSHOT 14 | springboot-spark 15 | Demo project for Spring Boot commit spark job 16 | 17 | 18 | 1.8 19 | 2.3.3 20 | 2.11 21 | 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-web 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter 31 | 32 | 33 | mysql 34 | mysql-connector-java 35 | 8.0.16 36 | 37 | 38 | org.apache.spark 39 | spark-launcher_${scala.version} 40 | ${spark.version} 41 | 42 | 43 | org.projectlombok 44 | lombok 45 | 46 | 47 | com.alibaba 48 | fastjson 49 | 1.2.49 50 | 51 | 52 | 53 | org.springframework.boot 54 | spring-boot-starter-test 55 | test 56 | 57 | 58 | 59 | 60 | spark 61 | 62 | 63 | org.springframework.boot 64 | spring-boot-maven-plugin 65 | 66 | com.hrong.springbootspark.SpringbootSparkApplication 67 | 68 | 69 | 70 | 71 | repackage 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | -------------------------------------------------------------------------------- /sparkjob/dependency-reduced-pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | com.hrong 5 | spark-job 6 | 1.0-SNAPSHOT 7 | 8 | spark-job 9 | 10 | 11 | 12 | net.alchim31.maven 13 | scala-maven-plugin 14 | 3.2.2 15 | 16 | 17 | 18 | 19 | 20 | net.alchim31.maven 21 | scala-maven-plugin 22 | 23 | 24 | scala-compile-first 25 | process-resources 26 | 27 | add-source 28 | compile 29 | 30 | 31 | 32 | scala-test-compile 33 | process-test-resources 34 | 35 | testCompile 36 | 37 | 38 | 39 | 40 | 41 | maven-compiler-plugin 42 | 43 | 44 | compile 45 | 46 | compile 47 | 48 | 49 | 50 | 51 | 52 | maven-shade-plugin 53 | 2.4.3 54 | 55 | 56 | package 57 | 58 | shade 59 | 60 | 61 | 62 | 63 | *:* 64 | 65 | META-INF/*.SF 66 | META-INF/*.DSA 67 | META-INF/*.RSA 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | org.apache.spark 80 | spark-core_2.11 81 | 2.3.3 82 | provided 83 | 84 | 85 | avro 86 | org.apache.avro 87 | 88 | 89 | avro-mapred 90 | org.apache.avro 91 | 92 | 93 | chill_2.11 94 | com.twitter 95 | 96 | 97 | chill-java 98 | com.twitter 99 | 100 | 101 | xbean-asm5-shaded 102 | org.apache.xbean 103 | 104 | 105 | hadoop-client 106 | org.apache.hadoop 107 | 108 | 109 | spark-launcher_2.11 110 | org.apache.spark 111 | 112 | 113 | spark-kvstore_2.11 114 | org.apache.spark 115 | 116 | 117 | spark-network-common_2.11 118 | org.apache.spark 119 | 120 | 121 | spark-network-shuffle_2.11 122 | org.apache.spark 123 | 124 | 125 | spark-unsafe_2.11 126 | org.apache.spark 127 | 128 | 129 | jets3t 130 | net.java.dev.jets3t 131 | 132 | 133 | curator-recipes 134 | org.apache.curator 135 | 136 | 137 | javax.servlet-api 138 | javax.servlet 139 | 140 | 141 | commons-lang3 142 | org.apache.commons 143 | 144 | 145 | commons-math3 146 | org.apache.commons 147 | 148 | 149 | jsr305 150 | com.google.code.findbugs 151 | 152 | 153 | slf4j-api 154 | org.slf4j 155 | 156 | 157 | jul-to-slf4j 158 | org.slf4j 159 | 160 | 161 | jcl-over-slf4j 162 | org.slf4j 163 | 164 | 165 | log4j 166 | log4j 167 | 168 | 169 | slf4j-log4j12 170 | org.slf4j 171 | 172 | 173 | compress-lzf 174 | com.ning 175 | 176 | 177 | snappy-java 178 | org.xerial.snappy 179 | 180 | 181 | lz4-java 182 | org.lz4 183 | 184 | 185 | zstd-jni 186 | com.github.luben 187 | 188 | 189 | RoaringBitmap 190 | org.roaringbitmap 191 | 192 | 193 | commons-net 194 | commons-net 195 | 196 | 197 | json4s-jackson_2.11 198 | org.json4s 199 | 200 | 201 | jersey-client 202 | org.glassfish.jersey.core 203 | 204 | 205 | jersey-common 206 | org.glassfish.jersey.core 207 | 208 | 209 | jersey-server 210 | org.glassfish.jersey.core 211 | 212 | 213 | jersey-container-servlet 214 | org.glassfish.jersey.containers 215 | 216 | 217 | jersey-container-servlet-core 218 | org.glassfish.jersey.containers 219 | 220 | 221 | netty-all 222 | io.netty 223 | 224 | 225 | netty 226 | io.netty 227 | 228 | 229 | stream 230 | com.clearspring.analytics 231 | 232 | 233 | metrics-core 234 | io.dropwizard.metrics 235 | 236 | 237 | metrics-jvm 238 | io.dropwizard.metrics 239 | 240 | 241 | metrics-json 242 | io.dropwizard.metrics 243 | 244 | 245 | metrics-graphite 246 | io.dropwizard.metrics 247 | 248 | 249 | jackson-databind 250 | com.fasterxml.jackson.core 251 | 252 | 253 | jackson-module-scala_2.11 254 | com.fasterxml.jackson.module 255 | 256 | 257 | ivy 258 | org.apache.ivy 259 | 260 | 261 | oro 262 | oro 263 | 264 | 265 | pyrolite 266 | net.razorvine 267 | 268 | 269 | py4j 270 | net.sf.py4j 271 | 272 | 273 | spark-tags_2.11 274 | org.apache.spark 275 | 276 | 277 | commons-crypto 278 | org.apache.commons 279 | 280 | 281 | unused 282 | org.spark-project.spark 283 | 284 | 285 | 286 | 287 | org.apache.spark 288 | spark-sql_2.11 289 | 2.3.3 290 | provided 291 | 292 | 293 | univocity-parsers 294 | com.univocity 295 | 296 | 297 | spark-sketch_2.11 298 | org.apache.spark 299 | 300 | 301 | spark-catalyst_2.11 302 | org.apache.spark 303 | 304 | 305 | orc-core 306 | org.apache.orc 307 | 308 | 309 | orc-mapreduce 310 | org.apache.orc 311 | 312 | 313 | parquet-column 314 | org.apache.parquet 315 | 316 | 317 | parquet-hadoop 318 | org.apache.parquet 319 | 320 | 321 | arrow-vector 322 | org.apache.arrow 323 | 324 | 325 | spark-tags_2.11 326 | org.apache.spark 327 | 328 | 329 | jackson-databind 330 | com.fasterxml.jackson.core 331 | 332 | 333 | xbean-asm5-shaded 334 | org.apache.xbean 335 | 336 | 337 | unused 338 | org.spark-project.spark 339 | 340 | 341 | 342 | 343 | org.scala-lang 344 | scala-library 345 | 2.11.8 346 | provided 347 | 348 | 349 | org.apache.hadoop 350 | hadoop-common 351 | 2.8.3 352 | provided 353 | 354 | 355 | hadoop-annotations 356 | org.apache.hadoop 357 | 358 | 359 | guava 360 | com.google.guava 361 | 362 | 363 | commons-cli 364 | commons-cli 365 | 366 | 367 | xmlenc 368 | xmlenc 369 | 370 | 371 | httpclient 372 | org.apache.httpcomponents 373 | 374 | 375 | commons-io 376 | commons-io 377 | 378 | 379 | commons-collections 380 | commons-collections 381 | 382 | 383 | servlet-api 384 | javax.servlet 385 | 386 | 387 | jetty 388 | org.mortbay.jetty 389 | 390 | 391 | jetty-util 392 | org.mortbay.jetty 393 | 394 | 395 | jetty-sslengine 396 | org.mortbay.jetty 397 | 398 | 399 | jsp-api 400 | javax.servlet.jsp 401 | 402 | 403 | jersey-core 404 | com.sun.jersey 405 | 406 | 407 | jersey-json 408 | com.sun.jersey 409 | 410 | 411 | jersey-server 412 | com.sun.jersey 413 | 414 | 415 | commons-logging 416 | commons-logging 417 | 418 | 419 | commons-lang 420 | commons-lang 421 | 422 | 423 | commons-configuration 424 | commons-configuration 425 | 426 | 427 | jackson-core-asl 428 | org.codehaus.jackson 429 | 430 | 431 | jackson-mapper-asl 432 | org.codehaus.jackson 433 | 434 | 435 | protobuf-java 436 | com.google.protobuf 437 | 438 | 439 | gson 440 | com.google.code.gson 441 | 442 | 443 | hadoop-auth 444 | org.apache.hadoop 445 | 446 | 447 | jsch 448 | com.jcraft 449 | 450 | 451 | curator-client 452 | org.apache.curator 453 | 454 | 455 | htrace-core4 456 | org.apache.htrace 457 | 458 | 459 | zookeeper 460 | org.apache.zookeeper 461 | 462 | 463 | commons-compress 464 | org.apache.commons 465 | 466 | 467 | commons-math3 468 | org.apache.commons 469 | 470 | 471 | commons-codec 472 | commons-codec 473 | 474 | 475 | commons-net 476 | commons-net 477 | 478 | 479 | log4j 480 | log4j 481 | 482 | 483 | jets3t 484 | net.java.dev.jets3t 485 | 486 | 487 | slf4j-api 488 | org.slf4j 489 | 490 | 491 | slf4j-log4j12 492 | org.slf4j 493 | 494 | 495 | avro 496 | org.apache.avro 497 | 498 | 499 | curator-recipes 500 | org.apache.curator 501 | 502 | 503 | jsr305 504 | com.google.code.findbugs 505 | 506 | 507 | 508 | 509 | 510 | 3.0.10 511 | 2.11.8 512 | 1.8 513 | UTF-8 514 | 11g 515 | 5.1.46 516 | UTF-8 517 | 2.8.3 518 | 2.11 519 | 2.3.3 520 | 521 | 522 | 523 | -------------------------------------------------------------------------------- /sparkjob/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | com.hrong 8 | spark-job 9 | 1.0-SNAPSHOT 10 | 11 | UTF-8 12 | UTF-8 13 | 1.8 14 | 2.8.3 15 | 2.3.3 16 | 2.11 17 | 2.11.8 18 | 8.0.16 19 | 11g 20 | 3.0.10 21 | 22 | 23 | 24 | mysql 25 | mysql-connector-java 26 | ${mysql.version} 27 | 28 | 29 | 30 | com.oracle.driver 31 | jdbc-driver 32 | ${oracle.version} 33 | 34 | 35 | 36 | org.apache.spark 37 | spark-core_${scala.version} 38 | ${spark.version} 39 | provided 40 | 41 | 42 | org.apache.spark 43 | spark-sql_${scala.version} 44 | ${spark.version} 45 | provided 46 | 47 | 48 | org.codehaus.janino 49 | commons-compiler 50 | ${codehaus.version} 51 | 52 | 53 | org.scala-lang 54 | scala-library 55 | ${scala-library.version} 56 | provided 57 | 58 | 59 | org.apache.hadoop 60 | hadoop-common 61 | ${hadoop.version} 62 | provided 63 | 64 | 65 | 66 | spark-job 67 | 68 | 69 | 70 | net.alchim31.maven 71 | scala-maven-plugin 72 | 3.2.2 73 | 74 | 75 | 76 | 77 | 78 | net.alchim31.maven 79 | scala-maven-plugin 80 | 81 | 82 | scala-compile-first 83 | process-resources 84 | 85 | add-source 86 | compile 87 | 88 | 89 | 90 | scala-test-compile 91 | process-test-resources 92 | 93 | testCompile 94 | 95 | 96 | 97 | 98 | 99 | org.apache.maven.plugins 100 | maven-compiler-plugin 101 | 102 | 103 | compile 104 | 105 | compile 106 | 107 | 108 | 109 | 110 | 111 | org.apache.maven.plugins 112 | maven-shade-plugin 113 | 2.4.3 114 | 115 | 116 | package 117 | 118 | shade 119 | 120 | 121 | 122 | 123 | *:* 124 | 125 | META-INF/*.SF 126 | META-INF/*.DSA 127 | META-INF/*.RSA 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | -------------------------------------------------------------------------------- /sparkjob/src/main/java/com/hrong/job/etl/DbTableEtl.java: -------------------------------------------------------------------------------- 1 | package com.hrong.job.etl; 2 | 3 | import org.apache.spark.sql.Dataset; 4 | import org.apache.spark.sql.Row; 5 | import org.apache.spark.sql.SparkSession; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | /** 10 | * @Author hrong 11 | * @Description 将数据库中的表数据保存到hdfs上 12 | **/ 13 | public class DbTableEtl { 14 | private static Logger log = LoggerFactory.getLogger(DbTableEtl.class); 15 | 16 | public static void main(String[] args) { 17 | SparkSession spark = SparkSession.builder() 18 | .appName(DbTableEtl.class.getSimpleName()) 19 | .getOrCreate(); 20 | String url = args[0]; 21 | String dbtable = args[1]; 22 | String user = args[2]; 23 | String password = args[3]; 24 | String targetFileType = args[4]; 25 | String targetFilePath = args[5]; 26 | Dataset dbData = spark.read() 27 | .format("jdbc") 28 | .option("url", url) 29 | .option("dbtable", dbtable) 30 | .option("user", user) 31 | .option("password", password) 32 | .load(); 33 | log.info("展示部分样例数据，即将开始导入到hdfs"); 34 | dbData.show(20, false); 35 | dbData.write().mode("overwrite").format(targetFileType).save(targetFilePath); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/SpringbootSparkApplication.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | /** 7 | * @author hrong 8 | */ 9 | @SpringBootApplication 10 | public class SpringbootSparkApplication { 11 | 12 | public static void main(String[] args) { 13 | SpringApplication.run(SpringbootSparkApplication.class, args); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/controller/SparkController.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.controller; 2 | 3 | import com.hrong.springbootspark.service.ISparkSubmitService; 4 | import com.hrong.springbootspark.vo.DataBaseExtractorVo; 5 | import com.hrong.springbootspark.vo.Result; 6 | import lombok.extern.slf4j.Slf4j; 7 | import org.springframework.stereotype.Controller; 8 | import org.springframework.web.bind.annotation.PostMapping; 9 | import org.springframework.web.bind.annotation.RequestBody; 10 | import org.springframework.web.bind.annotation.ResponseBody; 11 | 12 | import javax.annotation.Resource; 13 | import java.io.IOException; 14 | 15 | /** 16 | * @Author hrong 17 | **/ 18 | @Slf4j 19 | @Controller 20 | public class SparkController { 21 | @Resource 22 | private ISparkSubmitService iSparkSubmitService; 23 | /** 24 | * 调用service进行远程提交spark任务 25 | * @param vo 页面参数 26 | * @return 执行结果 27 | */ 28 | @ResponseBody 29 | @PostMapping("/extract/database") 30 | public Object dbExtractAndLoad2Hdfs(@RequestBody DataBaseExtractorVo vo){ 31 | try { 32 | return iSparkSubmitService.submitApplication(vo.getSparkApplicationParam(), 33 | vo.getUrl(), 34 | vo.getTable(), 35 | vo.getUserName(), 36 | vo.getPassword(), 37 | vo.getTargetFileType(), 38 | vo.getTargetFilePath()); 39 | } catch (IOException | InterruptedException e) { 40 | e.printStackTrace(); 41 | log.error("执行出错：{}", e.getMessage()); 42 | return Result.err(500, e.getMessage()); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/entity/SparkApplicationParam.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.entity; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | import org.springframework.beans.factory.annotation.Value; 7 | 8 | import java.util.Map; 9 | 10 | /** 11 | * @Author hrong 12 | **/ 13 | @Data 14 | @NoArgsConstructor 15 | @AllArgsConstructor 16 | public class SparkApplicationParam { 17 | /** 18 | * 任务的主类 19 | */ 20 | private String mainClass; 21 | /** 22 | * jar包路径 23 | */ 24 | private String jarPath; 25 | @Value("${spark.master:yarn}") 26 | private String master; 27 | @Value("${spark.deploy.mode:cluster}") 28 | private String deployMode; 29 | @Value("${spark.driver.memory:1g}") 30 | private String driverMemory; 31 | @Value("${spark.executor.memory:1g}") 32 | private String executorMemory; 33 | @Value("${spark.executor.cores:1}") 34 | private String executorCores; 35 | /** 36 | * 其他配置：传递给spark job的参数 37 | */ 38 | private Map otherConfParams; 39 | 40 | /** 41 | * 调用该方法可获取spark任务的设置参数 42 | * @return SparkApplicationParam 43 | */ 44 | public SparkApplicationParam getSparkApplicationParam(){ 45 | return new SparkApplicationParam(mainClass, jarPath, master, deployMode, driverMemory, executorMemory, executorCores, otherConfParams); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/service/ISparkSubmitService.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.service; 2 | 3 | import com.hrong.springbootspark.entity.SparkApplicationParam; 4 | 5 | import java.io.IOException; 6 | 7 | /** 8 | * @Author hrong 9 | * @description spark任务提交service 10 | **/ 11 | public interface ISparkSubmitService { 12 | /** 13 | * 提交spark任务入口 14 | * @param sparkAppParams spark任务运行所需参数 15 | * @param otherParams 单独的job所需参数 16 | * @return 结果 17 | * @throws IOException io 18 | * @throws InterruptedException 线程等待中断异常 19 | */ 20 | String submitApplication(SparkApplicationParam sparkAppParams, String... otherParams) throws IOException, InterruptedException; 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/service/impl/SparkSubmitServiceImpl.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.service.impl; 2 | 3 | import com.alibaba.fastjson.JSONObject; 4 | import com.hrong.springbootspark.entity.SparkApplicationParam; 5 | import com.hrong.springbootspark.service.ISparkSubmitService; 6 | import com.hrong.springbootspark.util.HttpUtil; 7 | import org.apache.spark.launcher.SparkAppHandle; 8 | import org.apache.spark.launcher.SparkLauncher; 9 | import org.slf4j.Logger; 10 | import org.slf4j.LoggerFactory; 11 | import org.springframework.beans.factory.annotation.Value; 12 | import org.springframework.stereotype.Service; 13 | 14 | import java.io.IOException; 15 | import java.util.Arrays; 16 | import java.util.Map; 17 | import java.util.concurrent.CountDownLatch; 18 | 19 | /** 20 | * @Author hrong 21 | **/ 22 | @Service 23 | public class SparkSubmitServiceImpl implements ISparkSubmitService { 24 | 25 | private static Logger log = LoggerFactory.getLogger(SparkSubmitServiceImpl.class); 26 | 27 | @Value("${driver.name:n151}") 28 | private String driverName; 29 | 30 | 31 | @Override 32 | public String submitApplication(SparkApplicationParam sparkAppParams, String... otherParams) throws IOException, InterruptedException { 33 | log.info("spark任务传入参数：{}", sparkAppParams.toString()); 34 | CountDownLatch countDownLatch = new CountDownLatch(1); 35 | Map confParams = sparkAppParams.getOtherConfParams(); 36 | SparkLauncher launcher = new SparkLauncher() 37 | .setAppResource(sparkAppParams.getJarPath()) 38 | .setMainClass(sparkAppParams.getMainClass()) 39 | .setMaster(sparkAppParams.getMaster()) 40 | .setDeployMode(sparkAppParams.getDeployMode()) 41 | .setConf("spark.driver.memory", sparkAppParams.getDriverMemory()) 42 | .setConf("spark.executor.memory", sparkAppParams.getExecutorMemory()) 43 | .setConf("spark.executor.cores", sparkAppParams.getExecutorCores()); 44 | if (confParams != null && confParams.size() != 0) { 45 | log.info("开始设置spark job运行参数:{}", JSONObject.toJSONString(confParams)); 46 | for (Map.Entry conf : confParams.entrySet()) { 47 | log.info("{}:{}", conf.getKey(), conf.getValue()); 48 | launcher.setConf(conf.getKey(), conf.getValue()); 49 | } 50 | } 51 | if (otherParams.length != 0) { 52 | log.info("开始设置spark job参数:{}", Arrays.toString(otherParams)); 53 | launcher.addAppArgs(otherParams); 54 | } 55 | log.info("参数设置完成，开始提交spark任务"); 56 | SparkAppHandle handle = launcher.setVerbose(true).startApplication(new SparkAppHandle.Listener() { 57 | @Override 58 | public void stateChanged(SparkAppHandle sparkAppHandle) { 59 | if (sparkAppHandle.getState().isFinal()) { 60 | countDownLatch.countDown(); 61 | } 62 | log.info("stateChanged:{}", sparkAppHandle.getState().toString()); 63 | } 64 | 65 | @Override 66 | public void infoChanged(SparkAppHandle sparkAppHandle) { 67 | log.info("infoChanged:{}", sparkAppHandle.getState().toString()); 68 | } 69 | }); 70 | log.info("The task is executing, please wait ...."); 71 | //线程等待任务结束 72 | countDownLatch.await(); 73 | log.info("The task is finished!"); 74 | //通过Spark原生的监测api获取执行结果信息，需要在spark-default.xml、spark-env.sh、yarn-site.xml进行相应的配置 75 | String restUrl; 76 | try { 77 | restUrl = "http://"+driverName+":18080/api/v1/applications/" + handle.getAppId(); 78 | log.info("访问application运算结果，url:{}", restUrl); 79 | return HttpUtil.httpGet(restUrl, null); 80 | } catch (Exception e) { 81 | log.info("18080端口异常，请确保spark-history-server服务已开启"); 82 | return "{\"state\":\"error\",\"data\":\"history server is not start\"}"; 83 | } 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/util/HttpUtil.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.util; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.net.HttpURLConnection; 7 | import java.net.URL; 8 | import java.nio.charset.StandardCharsets; 9 | import java.util.List; 10 | 11 | /** 12 | * @Author hrong 13 | **/ 14 | public class HttpUtil { 15 | public static String httpGet(String urlStr, List urlParam) throws IOException, InterruptedException { 16 | // 实例一个URL资源 17 | URL url = new URL(urlStr); 18 | HttpURLConnection connection = null; 19 | int i = 0; 20 | while (connection == null || connection.getResponseCode() != 200) { 21 | connection = (HttpURLConnection) url.openConnection(); 22 | connection.setRequestMethod("GET"); 23 | connection.setRequestProperty("Charset", "UTF-8"); 24 | connection.setRequestProperty("Content-Type", "application/json"); 25 | // 连接超时 单位毫秒 26 | connection.setConnectTimeout(15000); 27 | // 读取超时 单位毫秒 28 | connection.setReadTimeout(15000); 29 | i++; 30 | if (i == 50) { 31 | break; 32 | } 33 | Thread.sleep(500); 34 | } 35 | //将返回的值存入到String中 36 | BufferedReader brd = new BufferedReader(new InputStreamReader(connection.getInputStream(), StandardCharsets.UTF_8)); 37 | StringBuilder sb = new StringBuilder(); 38 | String line; 39 | while ((line = brd.readLine()) != null) { 40 | sb.append(line); 41 | } 42 | brd.close(); 43 | connection.disconnect(); 44 | return sb.toString(); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/vo/DataBaseExtractorVo.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.vo; 2 | 3 | import com.hrong.springbootspark.entity.SparkApplicationParam; 4 | import lombok.AllArgsConstructor; 5 | import lombok.Data; 6 | import lombok.NoArgsConstructor; 7 | 8 | /** 9 | * @Author hrong 10 | **/ 11 | @Data 12 | @NoArgsConstructor 13 | @AllArgsConstructor 14 | public class DataBaseExtractorVo extends SparkApplicationParam { 15 | /** 16 | * 数据库连接地址 17 | */ 18 | private String url; 19 | /** 20 | * 数据库连接账号 21 | */ 22 | private String userName; 23 | /** 24 | * 数据库密码 25 | */ 26 | private String password; 27 | /** 28 | * 指定的表名 29 | */ 30 | private String table; 31 | /** 32 | * 目标文件类型 33 | */ 34 | private String targetFileType; 35 | /** 36 | * 目标文件保存路径 37 | */ 38 | private String targetFilePath; 39 | } -------------------------------------------------------------------------------- /src/main/java/com/hrong/springbootspark/vo/Result.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark.vo; 2 | 3 | /** 4 | * @author huangrong 5 | */ 6 | public class Result { 7 | private Integer code; 8 | 9 | private String message; 10 | 11 | private T data; 12 | 13 | public static Result success(T data) { 14 | Result result = new Result(); 15 | result.setCode(200); 16 | result.setMessage("success"); 17 | result.setData(data); 18 | return result; 19 | } 20 | 21 | public static Result err(Integer code, String message) { 22 | Result result = new Result(); 23 | result.setCode(code); 24 | result.setMessage(message); 25 | return result; 26 | } 27 | 28 | public Integer getCode() { 29 | return code; 30 | } 31 | 32 | public void setCode(Integer code) { 33 | this.code = code; 34 | } 35 | 36 | public String getMessage() { 37 | return message; 38 | } 39 | 40 | public void setMessage(String message) { 41 | this.message = message; 42 | } 43 | 44 | public T getData() { 45 | return data; 46 | } 47 | 48 | public void setData(T data) { 49 | this.data = data; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/main/resources/application-spark.properties: -------------------------------------------------------------------------------- 1 | spark.master=yarn 2 | spark.driver.name=n151 3 | spark.deploy.mode=cluster 4 | spark.driver.memory=1g 5 | spark.executor.memory=1g 6 | spark.executor.instances=1 7 | spark.executor.cores=1 8 | spark.default.parallelism=1 -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.profiles.active=spark 2 | server.port=8079 -------------------------------------------------------------------------------- /src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | %d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger{50} - %msg%n 6 | 7 | 8 | 9 | 10 | true 11 | 12 | logs/bitsgrid-%d{yyyy-MM-dd}.log 13 | 30 14 | 1GB 15 | 16 | 17 | %d{yyyy-MM-dd HH:mm:ss.SSS}[%thread]%msg%n 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | -------------------------------------------------------------------------------- /src/test/java/com/hrong/springbootspark/SpringbootSparkApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.hrong.springbootspark; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | 8 | @RunWith(SpringRunner.class) 9 | @SpringBootTest 10 | public class SpringbootSparkApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | --------------------------------------------------------------------------------