├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── HighPerformanceBatch.pptx ├── README.md ├── mvnw ├── mvnw.cmd ├── partitioned-demo ├── .gitignore ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties ├── mvnw ├── mvnw.cmd ├── pom.xml └── src │ ├── main │ ├── java │ │ └── io │ │ │ └── spring │ │ │ └── batch │ │ │ └── partitiondemo │ │ │ ├── DemoApplication.java │ │ │ ├── configuration │ │ │ └── BatchConfiguration.java │ │ │ └── domain │ │ │ ├── JaxbDateSerializer.java │ │ │ └── Transaction.java │ └── resources │ │ ├── application.properties │ │ ├── data │ │ ├── csv │ │ │ ├── transactions1.csv │ │ │ ├── transactions2.csv │ │ │ └── transactions3.csv │ │ └── xml │ │ │ └── transactions.xml │ │ └── schema.sql │ └── test │ └── resources │ ├── application.properties │ └── h2-schema.sql ├── pom.xml ├── remote-chunking ├── .gitignore ├── .mvn │ └── wrapper │ │ ├── maven-wrapper.jar │ │ └── maven-wrapper.properties ├── mvnw ├── mvnw.cmd ├── pom.xml └── src │ └── main │ ├── java │ └── io │ │ └── spring │ │ └── batch │ │ └── remotechunking │ │ ├── RemoteChunkingApplication.java │ │ ├── configuration │ │ └── BatchConfiguration.java │ │ └── domain │ │ ├── JaxbDateSerializer.java │ │ └── Transaction.java │ └── resources │ ├── application-manager.properties │ ├── application-worker.properties │ ├── data │ └── transactions.csv │ └── schema.sql └── single-jvm-demos ├── .gitignore ├── .mvn └── wrapper │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── mvnw ├── mvnw.cmd ├── pom.xml └── src └── main ├── java └── io │ └── spring │ └── batch │ └── scalingdemos │ ├── asyncprocessor │ └── AsyncProcessorJobApplication.java │ ├── domain │ ├── JaxbDateSerializer.java │ └── Transaction.java │ ├── multithreaded │ └── MultithreadedJobApplication.java │ └── parallel │ └── ParallelStepsJobApplication.java └── resources ├── application.properties ├── data ├── csv │ ├── bigtransactions.csv │ ├── transactions.csv │ ├── transactions2.csv │ └── transactions3.csv └── xml │ ├── bigtransactions.xml │ └── transactions.xml └── schema.sql /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .#* 3 | *# 4 | *.sw* 5 | _site/ 6 | .factorypath 7 | .gradletasknamecache 8 | .DS_Store 9 | /application.yml 10 | /application.properties 11 | asciidoctor.css 12 | atlassian-ide-plugin.xml 13 | bin/ 14 | build/ 15 | dump.rdb 16 | out 17 | spring-shell.log 18 | target/ 19 | test-output 20 | 21 | # Eclipse artifacts, including WTP generated manifests 22 | .classpath 23 | .project 24 | .settings/ 25 | .springBeans 26 | spring-*/src/main/java/META-INF/MANIFEST.MF 27 | 28 | # IDEA artifacts and output dirs 29 | *.iml 30 | *.ipr 31 | *.iws 32 | .idea/* 33 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mminella/scaling-demos/3bc280bd35e2661cb14229102e73c2d5522c3726/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip 2 | -------------------------------------------------------------------------------- /HighPerformanceBatch.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mminella/scaling-demos/3bc280bd35e2661cb14229102e73c2d5522c3726/HighPerformanceBatch.pptx -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Docker MySQL configuration 2 | https://www.datacamp.com/tutorial/set-up-and-configure-mysql-in-docker 3 | 4 | Multithreaded Job 5 | - Run in IDE 6 | 7 | Parallel Steps 8 | - Run in IDE 9 | 10 | Async ItemProcessor/ItemWriter 11 | - Run in IDE 12 | 13 | Partitioning 14 | - `java -jar partitioned-demo-0.0.1-SNAPSHOT.jar` 15 | - `ps -ef | grep partition` 16 | 17 | Remote Chunking 18 | - Worker: `java -jar -Dspring.profiles.active=worker -Dspring.amqp.deserialization.trust.all=true remote-chunking-0.0.1-SNAPSHOT.jar` 19 | - Manager: `java -jar -Dspring.profiles.active=manager -Dspring.amqp.deserialization.trust.all=true remote-chunking-0.0.1-SNAPSHOT.jar` 20 | 21 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /partitioned-demo/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | 12 | ### IntelliJ IDEA ### 13 | .idea 14 | *.iws 15 | *.iml 16 | *.ipr 17 | 18 | ### NetBeans ### 19 | nbproject/private/ 20 | build/ 21 | nbbuild/ 22 | dist/ 23 | nbdist/ 24 | .nb-gradle/ -------------------------------------------------------------------------------- /partitioned-demo/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mminella/scaling-demos/3bc280bd35e2661cb14229102e73c2d5522c3726/partitioned-demo/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /partitioned-demo/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip 2 | -------------------------------------------------------------------------------- /partitioned-demo/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /partitioned-demo/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /partitioned-demo/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring.batch 7 | partitioned-demo 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | demo 12 | Demo project for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 3.3.2 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 21 25 | 5.1.2 26 | 3.1.2 27 | 28 | 29 | 30 | 31 | 32 | org.springframework.cloud 33 | spring-cloud-task-dependencies 34 | 3.1.2 35 | pom 36 | import 37 | 38 | 39 | org.springframework.cloud 40 | spring-cloud-dependencies 41 | Dalston.RELEASE 42 | pom 43 | import 44 | 45 | 46 | org.springframework.cloud 47 | spring-cloud-deployer-local 48 | 2.9.4 49 | 50 | 51 | 52 | 53 | 54 | 55 | org.springframework.boot 56 | spring-boot-starter-batch 57 | 58 | 59 | org.springframework.batch 60 | spring-batch-integration 61 | 62 | 63 | org.springframework.boot 64 | spring-boot-starter-aop 65 | 66 | 67 | org.springframework 68 | spring-web 69 | 70 | 71 | org.springframework.cloud 72 | spring-cloud-starter-task 73 | 74 | 75 | org.springframework.cloud 76 | spring-cloud-deployer-local 77 | 78 | 79 | org.springframework.boot 80 | spring-boot-starter-jdbc 81 | 82 | 83 | com.mysql 84 | mysql-connector-j 85 | runtime 86 | 87 | 88 | jakarta.xml.bind 89 | jakarta.xml.bind-api 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | org.springframework.boot 99 | spring-boot-starter-test 100 | test 101 | 102 | 103 | 104 | 105 | 106 | 107 | org.springframework.boot 108 | spring-boot-maven-plugin 109 | 110 | 111 | 112 | 113 | 114 | 115 | spring-snapshots 116 | Spring Snapshots 117 | https://repo.spring.io/snapshot 118 | 119 | true 120 | 121 | 122 | 123 | spring-milestones 124 | Spring Milestones 125 | https://repo.spring.io/milestone 126 | 127 | false 128 | 129 | 130 | 131 | 132 | 133 | 134 | spring-snapshots 135 | Spring Snapshots 136 | https://repo.spring.io/snapshot 137 | 138 | true 139 | 140 | 141 | 142 | spring-milestones 143 | Spring Milestones 144 | https://repo.spring.io/milestone 145 | 146 | false 147 | 148 | 149 | 150 | 151 | 152 | 153 | -------------------------------------------------------------------------------- /partitioned-demo/src/main/java/io/spring/batch/partitiondemo/DemoApplication.java: -------------------------------------------------------------------------------- 1 | package io.spring.batch.partitiondemo; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.boot.autoconfigure.SpringBootApplication; 9 | import org.springframework.cloud.task.configuration.EnableTask; 10 | 11 | @EnableTask 12 | @SpringBootApplication 13 | public class DemoApplication { 14 | 15 | public static void main(String[] args) { 16 | String [] newArgs = new String[] {"inputFiles=/data/csv/transactions*.csv"}; 17 | 18 | List strings = Arrays.asList(args); 19 | 20 | List finalArgs = new ArrayList<>(strings.size() + 1); 21 | finalArgs.addAll(strings); 22 | finalArgs.add("inputFiles=/data/csv/transactions*.csv"); 23 | finalArgs.add("--spring.profiles.active=manager"); 24 | 25 | SpringApplication.run(DemoApplication.class, finalArgs.toArray(new String[finalArgs.size()])); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /partitioned-demo/src/main/java/io/spring/batch/partitiondemo/configuration/BatchConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.partitiondemo.configuration; 17 | 18 | import java.util.ArrayList; 19 | import java.util.List; 20 | 21 | import javax.sql.DataSource; 22 | 23 | import io.spring.batch.partitiondemo.domain.Transaction; 24 | 25 | import org.springframework.batch.core.Job; 26 | import org.springframework.batch.core.Step; 27 | import org.springframework.batch.core.configuration.annotation.StepScope; 28 | import org.springframework.batch.core.explore.JobExplorer; 29 | import org.springframework.batch.core.job.builder.JobBuilder; 30 | import org.springframework.batch.core.partition.support.MultiResourcePartitioner; 31 | import org.springframework.batch.core.repository.JobRepository; 32 | import org.springframework.batch.core.step.builder.StepBuilder; 33 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 34 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 35 | import org.springframework.batch.item.file.FlatFileItemReader; 36 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 37 | import org.springframework.beans.factory.annotation.Autowired; 38 | import org.springframework.beans.factory.annotation.Value; 39 | import org.springframework.cloud.deployer.spi.task.TaskLauncher; 40 | import org.springframework.cloud.task.batch.partition.DeployerPartitionHandler; 41 | import org.springframework.cloud.task.batch.partition.DeployerStepExecutionHandler; 42 | import org.springframework.cloud.task.batch.partition.PassThroughCommandLineArgsProvider; 43 | import org.springframework.cloud.task.batch.partition.SimpleEnvironmentVariablesProvider; 44 | import org.springframework.cloud.task.repository.TaskRepository; 45 | import org.springframework.context.ApplicationContext; 46 | import org.springframework.context.ConfigurableApplicationContext; 47 | import org.springframework.context.annotation.Bean; 48 | import org.springframework.context.annotation.Configuration; 49 | import org.springframework.context.annotation.Profile; 50 | import org.springframework.core.env.Environment; 51 | import org.springframework.core.io.Resource; 52 | import org.springframework.transaction.PlatformTransactionManager; 53 | 54 | /** 55 | * @author Michael Minella 56 | */ 57 | @Configuration 58 | public class BatchConfiguration { 59 | 60 | @Autowired 61 | private ConfigurableApplicationContext context; 62 | 63 | @Bean 64 | @Profile("manager") 65 | public DeployerPartitionHandler partitionHandler(TaskLauncher taskLauncher, 66 | JobExplorer jobExplorer, 67 | ApplicationContext context, 68 | Environment environment, 69 | TaskRepository taskRepository) { 70 | Resource resource = context.getResource("file:///Users/mminella/Documents/IntelliJWorkspace/scaling-demos/partitioned-demo/target/partitioned-demo-0.0.1-SNAPSHOT.jar"); 71 | 72 | DeployerPartitionHandler partitionHandler = new DeployerPartitionHandler(taskLauncher, jobExplorer, resource, "step1", taskRepository); 73 | 74 | List commandLineArgs = new ArrayList<>(3); 75 | commandLineArgs.add("--spring.profiles.active=worker"); 76 | commandLineArgs.add("--spring.cloud.task.initialize.enable=false"); 77 | commandLineArgs.add("--spring.batch.initializer.enabled=false"); 78 | commandLineArgs.add("--spring.datasource.initialize=false"); 79 | partitionHandler.setCommandLineArgsProvider(new PassThroughCommandLineArgsProvider(commandLineArgs)); 80 | partitionHandler.setEnvironmentVariablesProvider(new SimpleEnvironmentVariablesProvider(environment)); 81 | partitionHandler.setMaxWorkers(3); 82 | partitionHandler.setApplicationName("PartitionedBatchJobTask"); 83 | 84 | return partitionHandler; 85 | } 86 | 87 | // @Bean 88 | // public TaskExecutorPartitionHandler partitionHandler() { 89 | // TaskExecutorPartitionHandler partitionHandler = new TaskExecutorPartitionHandler(); 90 | // 91 | // partitionHandler.setStep(step1()); 92 | // partitionHandler.setTaskExecutor(new SimpleAsyncTaskExecutor()); 93 | // 94 | // return partitionHandler; 95 | // } 96 | 97 | @Bean 98 | @Profile("worker") 99 | public DeployerStepExecutionHandler stepExecutionHandler(JobExplorer jobExplorer, JobRepository jobRepository) { 100 | return new DeployerStepExecutionHandler(this.context, jobExplorer, jobRepository); 101 | } 102 | 103 | @Bean 104 | @StepScope 105 | public MultiResourcePartitioner partitioner(@Value("#{jobParameters['inputFiles']}") Resource[] resources) { 106 | MultiResourcePartitioner partitioner = new MultiResourcePartitioner(); 107 | 108 | partitioner.setKeyName("file"); 109 | partitioner.setResources(resources); 110 | 111 | return partitioner; 112 | } 113 | 114 | @Bean 115 | @StepScope 116 | public FlatFileItemReader fileTransactionReader( 117 | @Value("#{stepExecutionContext['file']}") Resource resource) { 118 | 119 | return new FlatFileItemReaderBuilder() 120 | .name("flatFileTransactionReader") 121 | .resource(resource) 122 | .delimited() 123 | .names(new String[] {"account", "amount", "timestamp"}) 124 | .fieldSetMapper(fieldSet -> { 125 | Transaction transaction = new Transaction(); 126 | 127 | transaction.setAccount(fieldSet.readString("account")); 128 | transaction.setAmount(fieldSet.readBigDecimal("amount")); 129 | transaction.setTimestamp(fieldSet.readDate("timestamp", "yyyy-MM-dd HH:mm:ss")); 130 | 131 | return transaction; 132 | }) 133 | .build(); 134 | } 135 | 136 | @Bean 137 | @StepScope 138 | public JdbcBatchItemWriter writer(DataSource dataSource) { 139 | return new JdbcBatchItemWriterBuilder() 140 | .dataSource(dataSource) 141 | .beanMapped() 142 | .sql("INSERT INTO TRANSACTION (ACCOUNT, AMOUNT, TIMESTAMP) VALUES (:account, :amount, :timestamp)") 143 | .build(); 144 | } 145 | 146 | @Bean 147 | @Profile("manager") 148 | public Step partitionedMaster(JobRepository jobRepository) { 149 | return new StepBuilder("step1", jobRepository) 150 | .partitioner(step1(null, null).getName(), partitioner(null)) 151 | // .step(step1()) 152 | .partitionHandler(partitionHandler(null, null, null, null, null)) 153 | .build(); 154 | } 155 | 156 | @Bean 157 | public Step step1(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 158 | return new StepBuilder("step1", jobRepository) 159 | .chunk(100, transactionManager) 160 | .reader(fileTransactionReader(null)) 161 | .writer(writer(null)) 162 | .build(); 163 | } 164 | 165 | // @Bean 166 | // @StepScope 167 | // public MultiResourceItemReader multiResourceItemReader( 168 | // @Value("#{jobParameters['inputFiles']}") Resource[] resources) { 169 | // 170 | // return new MultiResourceItemReaderBuilder() 171 | // .delegate(delegate()) 172 | // .name("multiresourceReader") 173 | // .resources(resources) 174 | // .build(); 175 | // } 176 | // 177 | // @Bean 178 | // public FlatFileItemReader delegate() { 179 | // return new FlatFileItemReaderBuilder() 180 | // .name("flatFileTransactionReader") 181 | // .delimited() 182 | // .names(new String[] {"account", "amount", "timestamp"}) 183 | // .fieldSetMapper(fieldSet -> { 184 | // Transaction transaction = new Transaction(); 185 | // 186 | // transaction.setAccount(fieldSet.readString("account")); 187 | // transaction.setAmount(fieldSet.readBigDecimal("amount")); 188 | // transaction.setTimestamp(fieldSet.readDate("timestamp", "yyyy-MM-dd HH:mm:ss")); 189 | // 190 | // return transaction; 191 | // }) 192 | // .build(); 193 | // } 194 | 195 | @Bean 196 | @Profile("!worker") 197 | public Job partitionedJob(JobRepository jobRepository) { 198 | return new JobBuilder("partionedJob", jobRepository) 199 | .start(partitionedMaster(null)) 200 | .build(); 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /partitioned-demo/src/main/java/io/spring/batch/partitiondemo/domain/JaxbDateSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.partitiondemo.domain; 17 | 18 | import java.text.SimpleDateFormat; 19 | import java.util.Date; 20 | import jakarta.xml.bind.annotation.adapters.XmlAdapter; 21 | 22 | /** 23 | * @author Michael Minella 24 | */ 25 | public class JaxbDateSerializer extends XmlAdapter { 26 | 27 | private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 28 | 29 | @Override 30 | public String marshal(Date date) throws Exception { 31 | return dateFormat.format(date); 32 | } 33 | 34 | @Override 35 | public Date unmarshal(String date) throws Exception { 36 | return dateFormat.parse(date); 37 | } 38 | } -------------------------------------------------------------------------------- /partitioned-demo/src/main/java/io/spring/batch/partitiondemo/domain/Transaction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.partitiondemo.domain; 17 | 18 | import java.math.BigDecimal; 19 | import java.util.Date; 20 | 21 | import jakarta.xml.bind.annotation.XmlRootElement; 22 | import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; 23 | 24 | /** 25 | * @author Michael Minella 26 | */ 27 | @XmlRootElement(name="transaction") 28 | public class Transaction { 29 | 30 | private String account; 31 | 32 | private Date timestamp; 33 | 34 | private BigDecimal amount; 35 | 36 | @XmlJavaTypeAdapter(JaxbDateSerializer.class) 37 | public Date getTimestamp() { 38 | return timestamp; 39 | } 40 | 41 | public void setTimestamp(Date timestamp) { 42 | this.timestamp = timestamp; 43 | } 44 | 45 | public BigDecimal getAmount() { 46 | return amount; 47 | } 48 | 49 | public void setAmount(BigDecimal amount) { 50 | this.amount = amount; 51 | } 52 | 53 | public String getAccount() { 54 | return account; 55 | } 56 | 57 | public void setAccount(String account) { 58 | this.account = account; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "Transaction{" + 64 | "account='" + account + '\'' + 65 | ", timestamp=" + timestamp + 66 | ", amount=" + amount + 67 | '}'; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /partitioned-demo/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.sql.init.platform=mysql 2 | spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver 3 | spring.datasource.url=jdbc:mysql://localhost:3306/scaling?cachePrepStmts=true&elideSetAutoCommits=true&alwaysSendSetIsolation=false&cacheResultSetMetadata=true&useSSL=false 4 | spring.datasource.username=root 5 | spring.datasource.password=p@ssw0rd 6 | #spring.datasource.schema=schema-mysql.sql 7 | spring.sql.init.mode=always 8 | spring.batch.jdbc.initialize-schema=always 9 | logging.level.org.springframework.cloud.task=DEBUG 10 | -------------------------------------------------------------------------------- /partitioned-demo/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS TRANSACTION; 2 | 3 | CREATE TABLE TRANSACTION ( 4 | ACCOUNT VARCHAR(250) NOT NULL , 5 | TIMESTAMP DATETIME NOT NULL , 6 | AMOUNT DOUBLE NOT NULL 7 | ) ENGINE=InnoDB; 8 | -------------------------------------------------------------------------------- /partitioned-demo/src/test/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.platform=h2 2 | spring.datasource.url=jdbc:h2:mem:batch 3 | spring.datasource.schema=h2-schema.sql 4 | spring.datasource.initialization-mode=always 5 | logging.level.org.springframework.cloud.task=DEBUG -------------------------------------------------------------------------------- /partitioned-demo/src/test/resources/h2-schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS TRANSACTION; 2 | CREATE TABLE TRANSACTION ( 3 | ACCOUNT VARCHAR(250) NOT NULL , 4 | TIMESTAMP DATETIME NOT NULL , 5 | AMOUNT DOUBLE NOT NULL 6 | ); 7 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring.batch 7 | scaling-demos 8 | 0.0.1-SNAPSHOT 9 | pom 10 | 11 | scaling-demos 12 | Samples of scaling Spring Batch 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | single-jvm-demos 29 | partitioned-demo 30 | remote-chunking 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | -------------------------------------------------------------------------------- /remote-chunking/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | 12 | ### IntelliJ IDEA ### 13 | .idea 14 | *.iws 15 | *.iml 16 | *.ipr 17 | 18 | ### NetBeans ### 19 | nbproject/private/ 20 | build/ 21 | nbbuild/ 22 | dist/ 23 | nbdist/ 24 | .nb-gradle/ -------------------------------------------------------------------------------- /remote-chunking/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mminella/scaling-demos/3bc280bd35e2661cb14229102e73c2d5522c3726/remote-chunking/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /remote-chunking/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip 2 | -------------------------------------------------------------------------------- /remote-chunking/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /remote-chunking/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /remote-chunking/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring.batch 7 | remote-chunking 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | remote-chunking 12 | Demo project for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 3.3.2 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 21 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-amqp 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-integration 35 | 36 | 37 | org.springframework.integration 38 | spring-integration-amqp 39 | 6.3.2 40 | 41 | 42 | org.springframework.boot 43 | spring-boot-starter-batch 44 | 45 | 46 | org.springframework.boot 47 | spring-boot-starter-jdbc 48 | 49 | 50 | org.springframework.batch 51 | spring-batch-integration 52 | 53 | 54 | jakarta.xml.bind 55 | jakarta.xml.bind-api 56 | 57 | 58 | com.sun.xml.bind 59 | jaxb-impl 60 | runtime 61 | 62 | 63 | 64 | com.mysql 65 | mysql-connector-j 66 | runtime 67 | 68 | 69 | org.springframework.boot 70 | spring-boot-starter-test 71 | test 72 | 73 | 74 | org.springframework.batch 75 | spring-batch-test 76 | test 77 | 78 | 79 | 80 | 81 | 82 | 83 | org.springframework.boot 84 | spring-boot-maven-plugin 85 | 86 | 87 | 88 | 89 | 90 | 91 | spring-snapshots 92 | Spring Snapshots 93 | https://repo.spring.io/snapshot 94 | 95 | true 96 | 97 | 98 | 99 | spring-milestones 100 | Spring Milestones 101 | https://repo.spring.io/milestone 102 | 103 | false 104 | 105 | 106 | 107 | 108 | 109 | 110 | spring-snapshots 111 | Spring Snapshots 112 | https://repo.spring.io/snapshot 113 | 114 | true 115 | 116 | 117 | 118 | spring-milestones 119 | Spring Milestones 120 | https://repo.spring.io/milestone 121 | 122 | false 123 | 124 | 125 | 126 | 127 | 128 | 129 | -------------------------------------------------------------------------------- /remote-chunking/src/main/java/io/spring/batch/remotechunking/RemoteChunkingApplication.java: -------------------------------------------------------------------------------- 1 | package io.spring.batch.remotechunking; 2 | 3 | import java.util.ArrayList; 4 | import java.util.Arrays; 5 | import java.util.List; 6 | 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.boot.autoconfigure.SpringBootApplication; 9 | 10 | @SpringBootApplication 11 | public class RemoteChunkingApplication { 12 | 13 | public static void main(String[] args) { 14 | List strings = Arrays.asList(args); 15 | 16 | List finalArgs = new ArrayList<>(strings.size() + 1); 17 | finalArgs.addAll(strings); 18 | finalArgs.add("inputFlatFile=/data/transactions.csv"); 19 | 20 | SpringApplication.run(RemoteChunkingApplication.class, 21 | finalArgs.toArray(new String[finalArgs.size()])); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /remote-chunking/src/main/java/io/spring/batch/remotechunking/configuration/BatchConfiguration.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.remotechunking.configuration; 17 | 18 | import javax.sql.DataSource; 19 | 20 | import io.spring.batch.remotechunking.domain.Transaction; 21 | 22 | import org.springframework.amqp.core.AmqpTemplate; 23 | import org.springframework.amqp.core.Binding; 24 | import org.springframework.amqp.core.BindingBuilder; 25 | import org.springframework.amqp.core.Queue; 26 | import org.springframework.amqp.core.TopicExchange; 27 | import org.springframework.amqp.rabbit.connection.ConnectionFactory; 28 | import org.springframework.batch.core.Job; 29 | import org.springframework.batch.core.configuration.annotation.StepScope; 30 | import org.springframework.batch.core.job.builder.JobBuilder; 31 | import org.springframework.batch.core.repository.JobRepository; 32 | import org.springframework.batch.core.step.builder.StepBuilder; 33 | import org.springframework.batch.core.step.item.SimpleChunkProcessor; 34 | import org.springframework.batch.core.step.tasklet.TaskletStep; 35 | import org.springframework.batch.integration.chunk.ChunkMessageChannelItemWriter; 36 | import org.springframework.batch.integration.chunk.ChunkProcessorChunkHandler; 37 | import org.springframework.batch.integration.chunk.RemoteChunkHandlerFactoryBean; 38 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 39 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 40 | import org.springframework.batch.item.file.FlatFileItemReader; 41 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 42 | import org.springframework.beans.factory.annotation.Value; 43 | import org.springframework.context.annotation.Bean; 44 | import org.springframework.context.annotation.Configuration; 45 | import org.springframework.context.annotation.Profile; 46 | import org.springframework.core.io.Resource; 47 | import org.springframework.integration.amqp.dsl.Amqp; 48 | import org.springframework.integration.annotation.ServiceActivator; 49 | import org.springframework.integration.channel.DirectChannel; 50 | import org.springframework.integration.channel.QueueChannel; 51 | import org.springframework.integration.core.MessagingTemplate; 52 | import org.springframework.integration.dsl.IntegrationFlow; 53 | import org.springframework.transaction.PlatformTransactionManager; 54 | 55 | /** 56 | * @author Michael Minella 57 | */ 58 | @Configuration 59 | public class BatchConfiguration { 60 | 61 | @Configuration 62 | @Profile("!worker") 63 | public static class ManagerConfiguration { 64 | 65 | @Bean 66 | public DirectChannel requests() { 67 | return new DirectChannel(); 68 | } 69 | 70 | @Bean 71 | public IntegrationFlow outboundFlow(AmqpTemplate amqpTemplate) { 72 | return IntegrationFlow.from("requests") 73 | .handle(Amqp.outboundAdapter(amqpTemplate) 74 | .routingKey("requests")) 75 | .get(); 76 | } 77 | 78 | @Bean 79 | public MessagingTemplate messagingTemplate() { 80 | MessagingTemplate template = new MessagingTemplate(); 81 | template.setDefaultChannel(requests()); 82 | template.setReceiveTimeout(2000); 83 | return template; 84 | } 85 | 86 | @Bean 87 | @StepScope 88 | public ChunkMessageChannelItemWriter itemWriter() { 89 | ChunkMessageChannelItemWriter chunkMessageChannelItemWriter = 90 | new ChunkMessageChannelItemWriter<>(); 91 | chunkMessageChannelItemWriter.setMessagingOperations(messagingTemplate()); 92 | chunkMessageChannelItemWriter.setReplyChannel(replies()); 93 | return chunkMessageChannelItemWriter; 94 | } 95 | 96 | @Bean 97 | public RemoteChunkHandlerFactoryBean chunkHandler() { 98 | RemoteChunkHandlerFactoryBean remoteChunkHandlerFactoryBean = new RemoteChunkHandlerFactoryBean<>(); 99 | remoteChunkHandlerFactoryBean.setChunkWriter(itemWriter()); 100 | remoteChunkHandlerFactoryBean.setStep(step1(null, null)); 101 | return remoteChunkHandlerFactoryBean; 102 | } 103 | 104 | @Bean 105 | public QueueChannel replies() { 106 | return new QueueChannel(); 107 | } 108 | 109 | @Bean 110 | public IntegrationFlow replyFlow(ConnectionFactory connectionFactory) { 111 | return IntegrationFlow 112 | .from(Amqp.inboundAdapter(connectionFactory, "replies")) 113 | .channel(replies()) 114 | .get(); 115 | } 116 | 117 | @Bean 118 | @StepScope 119 | public FlatFileItemReader fileTransactionReader( 120 | @Value("#{jobParameters['inputFlatFile']}") Resource resource) { 121 | 122 | return new FlatFileItemReaderBuilder() 123 | .saveState(false) 124 | .resource(resource) 125 | .delimited() 126 | .names(new String[] {"account", "amount", "timestamp"}) 127 | .fieldSetMapper(fieldSet -> { 128 | Transaction transaction = new Transaction(); 129 | 130 | transaction.setAccount(fieldSet.readString("account")); 131 | transaction.setAmount(fieldSet.readBigDecimal("amount")); 132 | transaction.setTimestamp(fieldSet.readDate("timestamp", "yyyy-MM-dd HH:mm:ss")); 133 | 134 | return transaction; 135 | }) 136 | .build(); 137 | } 138 | 139 | @Bean 140 | public TaskletStep step1(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 141 | return new StepBuilder("step1", jobRepository) 142 | .chunk(100, transactionManager) 143 | .reader(fileTransactionReader(null)) 144 | .writer(itemWriter()) 145 | .build(); 146 | } 147 | 148 | @Bean 149 | public Job remoteChunkingJob(JobRepository jobRepository) { 150 | return new JobBuilder("remoteChunkingJob", jobRepository) 151 | .start(step1(null, null)) 152 | .build(); 153 | } 154 | } 155 | 156 | @Configuration 157 | @Profile("worker") 158 | public static class WorkerConfiguration { 159 | 160 | @Bean 161 | public Queue requestQueue() { 162 | return new Queue("requests", false); 163 | } 164 | 165 | @Bean 166 | public Queue repliesQueue() { 167 | return new Queue("replies", false); 168 | } 169 | 170 | @Bean 171 | public TopicExchange exchange() { 172 | return new TopicExchange("remote-chunking-exchange"); 173 | } 174 | 175 | @Bean 176 | Binding repliesBinding(TopicExchange exchange) { 177 | return BindingBuilder.bind(repliesQueue()).to(exchange).with("replies"); 178 | } 179 | 180 | @Bean 181 | Binding requestBinding(TopicExchange exchange) { 182 | return BindingBuilder.bind(requestQueue()).to(exchange).with("requests"); 183 | } 184 | 185 | @Bean 186 | public DirectChannel requests() { 187 | return new DirectChannel(); 188 | } 189 | 190 | @Bean 191 | public DirectChannel replies() { 192 | return new DirectChannel(); 193 | } 194 | 195 | @Bean 196 | public IntegrationFlow mesagesIn(ConnectionFactory connectionFactory) { 197 | return IntegrationFlow 198 | .from(Amqp.inboundAdapter(connectionFactory, "requests")) 199 | .channel(requests()) 200 | .get(); 201 | } 202 | 203 | @Bean 204 | public IntegrationFlow outgoingReplies(AmqpTemplate template) { 205 | return IntegrationFlow.from("replies") 206 | .handle(Amqp.outboundAdapter(template) 207 | .routingKey("replies")) 208 | .get(); 209 | } 210 | 211 | @Bean 212 | @ServiceActivator(inputChannel = "requests", outputChannel = "replies", sendTimeout = "10000") 213 | public ChunkProcessorChunkHandler chunkProcessorChunkHandler() { 214 | ChunkProcessorChunkHandler chunkProcessorChunkHandler = new ChunkProcessorChunkHandler<>(); 215 | chunkProcessorChunkHandler.setChunkProcessor( 216 | new SimpleChunkProcessor<>((transaction) -> { 217 | System.out.println(">> processing transaction: " + transaction); 218 | Thread.sleep(5); 219 | return transaction; 220 | }, writer(null))); 221 | 222 | return chunkProcessorChunkHandler; 223 | } 224 | 225 | @Bean 226 | public JdbcBatchItemWriter writer(DataSource dataSource) { 227 | return new JdbcBatchItemWriterBuilder() 228 | .dataSource(dataSource) 229 | .beanMapped() 230 | .sql("INSERT INTO TRANSACTION (ACCOUNT, AMOUNT, TIMESTAMP) VALUES (:account, :amount, :timestamp)") 231 | .build(); 232 | } 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /remote-chunking/src/main/java/io/spring/batch/remotechunking/domain/JaxbDateSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.remotechunking.domain; 17 | 18 | import java.text.SimpleDateFormat; 19 | import java.util.Date; 20 | import jakarta.xml.bind.annotation.adapters.XmlAdapter; 21 | 22 | /** 23 | * @author Michael Minella 24 | */ 25 | public class JaxbDateSerializer extends XmlAdapter { 26 | 27 | private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 28 | 29 | @Override 30 | public String marshal(Date date) throws Exception { 31 | return dateFormat.format(date); 32 | } 33 | 34 | @Override 35 | public Date unmarshal(String date) throws Exception { 36 | return dateFormat.parse(date); 37 | } 38 | } -------------------------------------------------------------------------------- /remote-chunking/src/main/java/io/spring/batch/remotechunking/domain/Transaction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.remotechunking.domain; 17 | 18 | import java.io.Serializable; 19 | import java.math.BigDecimal; 20 | import java.util.Date; 21 | import jakarta.xml.bind.annotation.XmlRootElement; 22 | import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; 23 | 24 | /** 25 | * @author Michael Minella 26 | */ 27 | @XmlRootElement(name="transaction") 28 | public class Transaction implements Serializable { 29 | 30 | private String account; 31 | 32 | private Date timestamp; 33 | 34 | private BigDecimal amount; 35 | 36 | @XmlJavaTypeAdapter(JaxbDateSerializer.class) 37 | public Date getTimestamp() { 38 | return timestamp; 39 | } 40 | 41 | public void setTimestamp(Date timestamp) { 42 | this.timestamp = timestamp; 43 | } 44 | 45 | public BigDecimal getAmount() { 46 | return amount; 47 | } 48 | 49 | public void setAmount(BigDecimal amount) { 50 | this.amount = amount; 51 | } 52 | 53 | public String getAccount() { 54 | return account; 55 | } 56 | 57 | public void setAccount(String account) { 58 | this.account = account; 59 | } 60 | 61 | @Override 62 | public String toString() { 63 | return "Transaction{" + 64 | "account='" + account + '\'' + 65 | ", timestamp=" + timestamp + 66 | ", amount=" + amount + 67 | '}'; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /remote-chunking/src/main/resources/application-manager.properties: -------------------------------------------------------------------------------- 1 | spring.sql.init.platform=mysql 2 | spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver 3 | spring.datasource.url=jdbc:mysql://localhost:3306/scaling?cachePrepStmts=true&elideSetAutoCommits=true&alwaysSendSetIsolation=false&cacheResultSetMetadata=true&useSSL=false 4 | spring.datasource.username=root 5 | spring.datasource.password=p@ssw0rd 6 | #spring.datasource.schema=schema.sql 7 | spring.sql.init.mode=always 8 | spring.batch.jdbc.initialize-schema=always 9 | logging.level.org.springframework.integration=DEBUG 10 | -------------------------------------------------------------------------------- /remote-chunking/src/main/resources/application-worker.properties: -------------------------------------------------------------------------------- 1 | spring.sql.init.platform=mysql 2 | spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver 3 | spring.datasource.url=jdbc:mysql://localhost:3306/scaling?cachePrepStmts=true&elideSetAutoCommits=true&alwaysSendSetIsolation=false&cacheResultSetMetadata=true&useSSL=false 4 | spring.datasource.username=root 5 | spring.datasource.password=p@ssw0rd 6 | #spring.datasource.schema=schema.sql 7 | spring.sql.init.mode=never 8 | spring.batch.jdbc.initialize-schema=never 9 | logging.level.org.springframework.integration=DEBUG 10 | -------------------------------------------------------------------------------- /remote-chunking/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS TRANSACTION; 2 | 3 | CREATE TABLE TRANSACTION ( 4 | ACCOUNT VARCHAR(250) NOT NULL , 5 | TIMESTAMP DATETIME NOT NULL , 6 | AMOUNT DOUBLE NOT NULL 7 | ) ENGINE=InnoDB; 8 | -- 9 | -- CREATE TABLE BATCH_JOB_INSTANCE ( 10 | -- JOB_INSTANCE_ID BIGINT NOT NULL PRIMARY KEY , 11 | -- VERSION BIGINT , 12 | -- JOB_NAME VARCHAR(100) NOT NULL, 13 | -- JOB_KEY VARCHAR(32) NOT NULL, 14 | -- constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY) 15 | -- ) ENGINE=InnoDB; 16 | -- 17 | -- CREATE TABLE BATCH_JOB_EXECUTION ( 18 | -- JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY , 19 | -- VERSION BIGINT , 20 | -- JOB_INSTANCE_ID BIGINT NOT NULL, 21 | -- CREATE_TIME DATETIME NOT NULL, 22 | -- START_TIME DATETIME DEFAULT NULL , 23 | -- END_TIME DATETIME DEFAULT NULL , 24 | -- STATUS VARCHAR(10) , 25 | -- EXIT_CODE VARCHAR(2500) , 26 | -- EXIT_MESSAGE VARCHAR(2500) , 27 | -- LAST_UPDATED DATETIME, 28 | -- JOB_CONFIGURATION_LOCATION VARCHAR(2500) NULL, 29 | -- constraint JOB_INST_EXEC_FK foreign key (JOB_INSTANCE_ID) 30 | -- references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID) 31 | -- ) ENGINE=InnoDB; 32 | -- 33 | -- CREATE TABLE BATCH_JOB_EXECUTION_PARAMS ( 34 | -- JOB_EXECUTION_ID BIGINT NOT NULL , 35 | -- TYPE_CD VARCHAR(6) NOT NULL , 36 | -- KEY_NAME VARCHAR(100) NOT NULL , 37 | -- STRING_VAL VARCHAR(250) , 38 | -- DATE_VAL DATETIME DEFAULT NULL , 39 | -- LONG_VAL BIGINT , 40 | -- DOUBLE_VAL DOUBLE PRECISION , 41 | -- IDENTIFYING CHAR(1) NOT NULL , 42 | -- constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID) 43 | -- references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 44 | -- ) ENGINE=InnoDB; 45 | -- 46 | -- CREATE TABLE BATCH_STEP_EXECUTION ( 47 | -- STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY , 48 | -- VERSION BIGINT NOT NULL, 49 | -- STEP_NAME VARCHAR(100) NOT NULL, 50 | -- JOB_EXECUTION_ID BIGINT NOT NULL, 51 | -- START_TIME DATETIME NOT NULL , 52 | -- END_TIME DATETIME DEFAULT NULL , 53 | -- STATUS VARCHAR(10) , 54 | -- COMMIT_COUNT BIGINT , 55 | -- READ_COUNT BIGINT , 56 | -- FILTER_COUNT BIGINT , 57 | -- WRITE_COUNT BIGINT , 58 | -- READ_SKIP_COUNT BIGINT , 59 | -- WRITE_SKIP_COUNT BIGINT , 60 | -- PROCESS_SKIP_COUNT BIGINT , 61 | -- ROLLBACK_COUNT BIGINT , 62 | -- EXIT_CODE VARCHAR(2500) , 63 | -- EXIT_MESSAGE VARCHAR(2500) , 64 | -- LAST_UPDATED DATETIME, 65 | -- constraint JOB_EXEC_STEP_FK foreign key (JOB_EXECUTION_ID) 66 | -- references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 67 | -- ) ENGINE=InnoDB; 68 | -- 69 | -- CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT ( 70 | -- STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY, 71 | -- SHORT_CONTEXT VARCHAR(2500) NOT NULL, 72 | -- SERIALIZED_CONTEXT TEXT , 73 | -- constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID) 74 | -- references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID) 75 | -- ) ENGINE=InnoDB; 76 | -- 77 | -- CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT ( 78 | -- JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY, 79 | -- SHORT_CONTEXT VARCHAR(2500) NOT NULL, 80 | -- SERIALIZED_CONTEXT TEXT , 81 | -- constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID) 82 | -- references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 83 | -- ) ENGINE=InnoDB; 84 | -- 85 | -- CREATE TABLE BATCH_STEP_EXECUTION_SEQ ( 86 | -- ID BIGINT NOT NULL, 87 | -- UNIQUE_KEY CHAR(1) NOT NULL, 88 | -- constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 89 | -- ) ENGINE=InnoDB; 90 | -- 91 | -- INSERT INTO BATCH_STEP_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_STEP_EXECUTION_SEQ); 92 | -- 93 | -- CREATE TABLE BATCH_JOB_EXECUTION_SEQ ( 94 | -- ID BIGINT NOT NULL, 95 | -- UNIQUE_KEY CHAR(1) NOT NULL, 96 | -- constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 97 | -- ) ENGINE=InnoDB; 98 | -- 99 | -- INSERT INTO BATCH_JOB_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_EXECUTION_SEQ); 100 | -- 101 | -- CREATE TABLE BATCH_JOB_SEQ ( 102 | -- ID BIGINT NOT NULL, 103 | -- UNIQUE_KEY CHAR(1) NOT NULL, 104 | -- constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 105 | -- ) ENGINE=InnoDB; 106 | -- 107 | -- INSERT INTO BATCH_JOB_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_SEQ); 108 | -------------------------------------------------------------------------------- /single-jvm-demos/.gitignore: -------------------------------------------------------------------------------- 1 | target/ 2 | !.mvn/wrapper/maven-wrapper.jar 3 | 4 | ### STS ### 5 | .apt_generated 6 | .classpath 7 | .factorypath 8 | .project 9 | .settings 10 | .springBeans 11 | 12 | ### IntelliJ IDEA ### 13 | .idea 14 | *.iws 15 | *.iml 16 | *.ipr 17 | 18 | ### NetBeans ### 19 | nbproject/private/ 20 | build/ 21 | nbbuild/ 22 | dist/ 23 | nbdist/ 24 | .nb-gradle/ -------------------------------------------------------------------------------- /single-jvm-demos/.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mminella/scaling-demos/3bc280bd35e2661cb14229102e73c2d5522c3726/single-jvm-demos/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /single-jvm-demos/.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo1.maven.org/maven2/org/apache/maven/apache-maven/3.5.2/apache-maven-3.5.2-bin.zip 2 | -------------------------------------------------------------------------------- /single-jvm-demos/mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # http://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven2 Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Migwn, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | # TODO classpath? 118 | fi 119 | 120 | if [ -z "$JAVA_HOME" ]; then 121 | javaExecutable="`which javac`" 122 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 123 | # readlink(1) is not available as standard on Solaris 10. 124 | readLink=`which readlink` 125 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 126 | if $darwin ; then 127 | javaHome="`dirname \"$javaExecutable\"`" 128 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 129 | else 130 | javaExecutable="`readlink -f \"$javaExecutable\"`" 131 | fi 132 | javaHome="`dirname \"$javaExecutable\"`" 133 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 134 | JAVA_HOME="$javaHome" 135 | export JAVA_HOME 136 | fi 137 | fi 138 | fi 139 | 140 | if [ -z "$JAVACMD" ] ; then 141 | if [ -n "$JAVA_HOME" ] ; then 142 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 143 | # IBM's JDK on AIX uses strange locations for the executables 144 | JAVACMD="$JAVA_HOME/jre/sh/java" 145 | else 146 | JAVACMD="$JAVA_HOME/bin/java" 147 | fi 148 | else 149 | JAVACMD="`which java`" 150 | fi 151 | fi 152 | 153 | if [ ! -x "$JAVACMD" ] ; then 154 | echo "Error: JAVA_HOME is not defined correctly." >&2 155 | echo " We cannot execute $JAVACMD" >&2 156 | exit 1 157 | fi 158 | 159 | if [ -z "$JAVA_HOME" ] ; then 160 | echo "Warning: JAVA_HOME environment variable is not set." 161 | fi 162 | 163 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 164 | 165 | # traverses directory structure from process work directory to filesystem root 166 | # first directory with .mvn subdirectory is considered project base directory 167 | find_maven_basedir() { 168 | 169 | if [ -z "$1" ] 170 | then 171 | echo "Path not specified to find_maven_basedir" 172 | return 1 173 | fi 174 | 175 | basedir="$1" 176 | wdir="$1" 177 | while [ "$wdir" != '/' ] ; do 178 | if [ -d "$wdir"/.mvn ] ; then 179 | basedir=$wdir 180 | break 181 | fi 182 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 183 | if [ -d "${wdir}" ]; then 184 | wdir=`cd "$wdir/.."; pwd` 185 | fi 186 | # end of workaround 187 | done 188 | echo "${basedir}" 189 | } 190 | 191 | # concatenates all lines of a file 192 | concat_lines() { 193 | if [ -f "$1" ]; then 194 | echo "$(tr -s '\n' ' ' < "$1")" 195 | fi 196 | } 197 | 198 | BASE_DIR=`find_maven_basedir "$(pwd)"` 199 | if [ -z "$BASE_DIR" ]; then 200 | exit 1; 201 | fi 202 | 203 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 204 | echo $MAVEN_PROJECTBASEDIR 205 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 206 | 207 | # For Cygwin, switch paths to Windows format before running java 208 | if $cygwin; then 209 | [ -n "$M2_HOME" ] && 210 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 211 | [ -n "$JAVA_HOME" ] && 212 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 213 | [ -n "$CLASSPATH" ] && 214 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 215 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 216 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 217 | fi 218 | 219 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 220 | 221 | exec "$JAVACMD" \ 222 | $MAVEN_OPTS \ 223 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 224 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 225 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 226 | -------------------------------------------------------------------------------- /single-jvm-demos/mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM http://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven2 Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a key stroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM enable echoing my setting MAVEN_BATCH_ECHO to 'on' 39 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 40 | 41 | @REM set %HOME% to equivalent of $HOME 42 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 43 | 44 | @REM Execute a user defined script before this one 45 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 46 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 47 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 48 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 49 | :skipRcPre 50 | 51 | @setlocal 52 | 53 | set ERROR_CODE=0 54 | 55 | @REM To isolate internal variables from possible post scripts, we use another setlocal 56 | @setlocal 57 | 58 | @REM ==== START VALIDATION ==== 59 | if not "%JAVA_HOME%" == "" goto OkJHome 60 | 61 | echo. 62 | echo Error: JAVA_HOME not found in your environment. >&2 63 | echo Please set the JAVA_HOME variable in your environment to match the >&2 64 | echo location of your Java installation. >&2 65 | echo. 66 | goto error 67 | 68 | :OkJHome 69 | if exist "%JAVA_HOME%\bin\java.exe" goto init 70 | 71 | echo. 72 | echo Error: JAVA_HOME is set to an invalid directory. >&2 73 | echo JAVA_HOME = "%JAVA_HOME%" >&2 74 | echo Please set the JAVA_HOME variable in your environment to match the >&2 75 | echo location of your Java installation. >&2 76 | echo. 77 | goto error 78 | 79 | @REM ==== END VALIDATION ==== 80 | 81 | :init 82 | 83 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 84 | @REM Fallback to current working directory if not found. 85 | 86 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 87 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 88 | 89 | set EXEC_DIR=%CD% 90 | set WDIR=%EXEC_DIR% 91 | :findBaseDir 92 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 93 | cd .. 94 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 95 | set WDIR=%CD% 96 | goto findBaseDir 97 | 98 | :baseDirFound 99 | set MAVEN_PROJECTBASEDIR=%WDIR% 100 | cd "%EXEC_DIR%" 101 | goto endDetectBaseDir 102 | 103 | :baseDirNotFound 104 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 105 | cd "%EXEC_DIR%" 106 | 107 | :endDetectBaseDir 108 | 109 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 110 | 111 | @setlocal EnableExtensions EnableDelayedExpansion 112 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 113 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 114 | 115 | :endReadAdditionalConfig 116 | 117 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 118 | 119 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 120 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 121 | 122 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 123 | if ERRORLEVEL 1 goto error 124 | goto end 125 | 126 | :error 127 | set ERROR_CODE=1 128 | 129 | :end 130 | @endlocal & set ERROR_CODE=%ERROR_CODE% 131 | 132 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 133 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 134 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 135 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 136 | :skipRcPost 137 | 138 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 139 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 140 | 141 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 142 | 143 | exit /B %ERROR_CODE% 144 | -------------------------------------------------------------------------------- /single-jvm-demos/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | io.spring.batch 7 | single-jvm-demos 8 | 0.0.1-SNAPSHOT 9 | jar 10 | 11 | single-jvm-demos 12 | Demo project for Spring Boot 13 | 14 | 15 | org.springframework.boot 16 | spring-boot-starter-parent 17 | 3.3.2 18 | 19 | 20 | 21 | 22 | UTF-8 23 | UTF-8 24 | 21 25 | 26 | 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-batch 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-jdbc 35 | 36 | 37 | org.springframework.batch 38 | spring-batch-integration 39 | 40 | 41 | org.springframework 42 | spring-oxm 43 | 6.1.12 44 | 45 | 46 | jakarta.xml.bind 47 | jakarta.xml.bind-api 48 | 49 | 50 | com.sun.xml.bind 51 | jaxb-impl 52 | runtime 53 | 54 | 55 | 56 | com.mysql 57 | mysql-connector-j 58 | runtime 59 | 60 | 61 | org.springframework.boot 62 | spring-boot-starter-test 63 | test 64 | 65 | 66 | org.springframework.batch 67 | spring-batch-test 68 | test 69 | 70 | 71 | 72 | 73 | 74 | 75 | org.springframework.boot 76 | spring-boot-maven-plugin 77 | 78 | 79 | repackage 80 | 81 | true 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | spring-snapshots 93 | Spring Snapshots 94 | https://repo.spring.io/snapshot 95 | 96 | true 97 | 98 | 99 | 100 | spring-milestones 101 | Spring Milestones 102 | https://repo.spring.io/milestone 103 | 104 | false 105 | 106 | 107 | 108 | 109 | 110 | 111 | spring-snapshots 112 | Spring Snapshots 113 | https://repo.spring.io/snapshot 114 | 115 | true 116 | 117 | 118 | 119 | spring-milestones 120 | Spring Milestones 121 | https://repo.spring.io/milestone 122 | 123 | false 124 | 125 | 126 | 127 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /single-jvm-demos/src/main/java/io/spring/batch/scalingdemos/asyncprocessor/AsyncProcessorJobApplication.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.scalingdemos.asyncprocessor; 17 | 18 | import java.util.concurrent.Future; 19 | 20 | import javax.sql.DataSource; 21 | 22 | import io.spring.batch.scalingdemos.domain.Transaction; 23 | 24 | import org.springframework.batch.core.Job; 25 | import org.springframework.batch.core.Step; 26 | import org.springframework.batch.core.configuration.annotation.StepScope; 27 | import org.springframework.batch.core.job.builder.JobBuilder; 28 | import org.springframework.batch.core.repository.JobRepository; 29 | import org.springframework.batch.core.step.builder.StepBuilder; 30 | import org.springframework.batch.integration.async.AsyncItemProcessor; 31 | import org.springframework.batch.integration.async.AsyncItemWriter; 32 | import org.springframework.batch.item.ItemProcessor; 33 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 34 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 35 | import org.springframework.batch.item.file.FlatFileItemReader; 36 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 37 | import org.springframework.beans.factory.annotation.Value; 38 | import org.springframework.boot.SpringApplication; 39 | import org.springframework.boot.autoconfigure.SpringBootApplication; 40 | import org.springframework.context.annotation.Bean; 41 | import org.springframework.core.io.Resource; 42 | import org.springframework.core.task.VirtualThreadTaskExecutor; 43 | import org.springframework.transaction.PlatformTransactionManager; 44 | 45 | /** 46 | * @author Michael Minella 47 | */ 48 | @SpringBootApplication 49 | public class AsyncProcessorJobApplication { 50 | 51 | @Bean 52 | @StepScope 53 | public FlatFileItemReader fileTransactionReader( 54 | @Value("#{jobParameters['inputFlatFile']}") Resource resource) { 55 | 56 | return new FlatFileItemReaderBuilder() 57 | .saveState(false) 58 | .resource(resource) 59 | .delimited() 60 | .names(new String[] {"account", "amount", "timestamp"}) 61 | .fieldSetMapper(fieldSet -> { 62 | Transaction transaction = new Transaction(); 63 | 64 | transaction.setAccount(fieldSet.readString("account")); 65 | transaction.setAmount(fieldSet.readBigDecimal("amount")); 66 | transaction.setTimestamp(fieldSet.readDate("timestamp", "yyyy-MM-dd HH:mm:ss")); 67 | 68 | return transaction; 69 | }) 70 | .build(); 71 | } 72 | 73 | @Bean 74 | public JdbcBatchItemWriter writer(DataSource dataSource) { 75 | return new JdbcBatchItemWriterBuilder() 76 | .dataSource(dataSource) 77 | .beanMapped() 78 | .sql("INSERT INTO TRANSACTION (ACCOUNT, AMOUNT, TIMESTAMP) VALUES (:account, :amount, :timestamp)") 79 | .build(); 80 | } 81 | 82 | @Bean 83 | public AsyncItemProcessor asyncItemProcessor() { 84 | AsyncItemProcessor processor = new AsyncItemProcessor<>(); 85 | 86 | processor.setDelegate(processor()); 87 | processor.setTaskExecutor(new VirtualThreadTaskExecutor()); 88 | 89 | return processor; 90 | } 91 | 92 | @Bean 93 | public AsyncItemWriter asyncItemWriter() { 94 | AsyncItemWriter writer = new AsyncItemWriter<>(); 95 | 96 | writer.setDelegate(writer(null)); 97 | 98 | return writer; 99 | } 100 | 101 | 102 | // Adds 5 minutes to the processing of the job where the file is 60000 records long 103 | @Bean 104 | public ItemProcessor processor() { 105 | return (transaction) -> { 106 | Thread.sleep(5); 107 | return transaction; 108 | }; 109 | } 110 | 111 | @Bean 112 | public Job asyncJob(JobRepository jobRepository) { 113 | return new JobBuilder("asyncJob", jobRepository) 114 | .start(step1async(null, null)) 115 | .build(); 116 | } 117 | 118 | // @Bean 119 | // public Job job1(JobRepository jobRepository) { 120 | // return new JobBuilder("job1", jobRepository) 121 | // .start(step1(null, null)) 122 | // .build(); 123 | // } 124 | 125 | @Bean 126 | public Step step1async(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 127 | return new StepBuilder("step1async", jobRepository) 128 | .>chunk(100, transactionManager) 129 | .reader(fileTransactionReader(null)) 130 | .processor(asyncItemProcessor()) 131 | .writer(asyncItemWriter()) 132 | .build(); 133 | } 134 | 135 | // @Bean 136 | // public Step step1(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 137 | // return new StepBuilder("step1", jobRepository) 138 | // .chunk(100, transactionManager) 139 | // .reader(fileTransactionReader(null)) 140 | // .processor(processor()) 141 | // .writer(writer(null)) 142 | // .build(); 143 | // } 144 | 145 | public static void main(String[] args) { 146 | String [] newArgs = new String[] {"inputFlatFile=/data/csv/bigtransactions.csv"}; 147 | 148 | SpringApplication.run(AsyncProcessorJobApplication.class, newArgs); 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /single-jvm-demos/src/main/java/io/spring/batch/scalingdemos/domain/JaxbDateSerializer.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.scalingdemos.domain; 17 | 18 | import java.text.SimpleDateFormat; 19 | import java.util.Date; 20 | import jakarta.xml.bind.annotation.adapters.XmlAdapter; 21 | 22 | /** 23 | * @author Michael Minella 24 | */ 25 | public class JaxbDateSerializer extends XmlAdapter { 26 | 27 | private SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 28 | 29 | @Override 30 | public String marshal(Date date) throws Exception { 31 | return dateFormat.format(date); 32 | } 33 | 34 | @Override 35 | public Date unmarshal(String date) throws Exception { 36 | return dateFormat.parse(date); 37 | } 38 | } -------------------------------------------------------------------------------- /single-jvm-demos/src/main/java/io/spring/batch/scalingdemos/domain/Transaction.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.scalingdemos.domain; 17 | 18 | import java.math.BigDecimal; 19 | import java.util.Date; 20 | import jakarta.xml.bind.annotation.XmlRootElement; 21 | import jakarta.xml.bind.annotation.adapters.XmlJavaTypeAdapter; 22 | 23 | /** 24 | * @author Michael Minella 25 | */ 26 | @XmlRootElement(name="transaction") 27 | public class Transaction { 28 | 29 | private String account; 30 | 31 | private Date timestamp; 32 | 33 | private BigDecimal amount; 34 | 35 | @XmlJavaTypeAdapter(JaxbDateSerializer.class) 36 | public Date getTimestamp() { 37 | return timestamp; 38 | } 39 | 40 | public void setTimestamp(Date timestamp) { 41 | this.timestamp = timestamp; 42 | } 43 | 44 | public BigDecimal getAmount() { 45 | return amount; 46 | } 47 | 48 | public void setAmount(BigDecimal amount) { 49 | this.amount = amount; 50 | } 51 | 52 | public String getAccount() { 53 | return account; 54 | } 55 | 56 | public void setAccount(String account) { 57 | this.account = account; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /single-jvm-demos/src/main/java/io/spring/batch/scalingdemos/multithreaded/MultithreadedJobApplication.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.scalingdemos.multithreaded; 17 | 18 | import javax.sql.DataSource; 19 | 20 | import io.spring.batch.scalingdemos.domain.Transaction; 21 | 22 | import org.springframework.batch.core.Job; 23 | import org.springframework.batch.core.Step; 24 | import org.springframework.batch.core.configuration.annotation.StepScope; 25 | import org.springframework.batch.core.job.builder.JobBuilder; 26 | import org.springframework.batch.core.repository.JobRepository; 27 | import org.springframework.batch.core.step.builder.StepBuilder; 28 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 29 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 30 | import org.springframework.batch.item.file.FlatFileItemReader; 31 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 32 | import org.springframework.beans.factory.annotation.Value; 33 | import org.springframework.boot.SpringApplication; 34 | import org.springframework.boot.autoconfigure.SpringBootApplication; 35 | import org.springframework.context.annotation.Bean; 36 | import org.springframework.core.io.Resource; 37 | import org.springframework.core.task.VirtualThreadTaskExecutor; 38 | import org.springframework.transaction.PlatformTransactionManager; 39 | 40 | /** 41 | * @author Michael Minella 42 | */ 43 | @SpringBootApplication 44 | public class MultithreadedJobApplication { 45 | 46 | @StepScope 47 | @Bean 48 | public FlatFileItemReader fileTransactionReader( 49 | @Value("#{jobParameters['inputFlatFile']}") Resource resource) { 50 | 51 | return new FlatFileItemReaderBuilder() 52 | .saveState(false) 53 | .resource(resource) 54 | .delimited() 55 | .names(new String[] {"account", "amount", "timestamp"}) 56 | .fieldSetMapper(fieldSet -> { 57 | Transaction transaction = new Transaction(); 58 | 59 | transaction.setAccount(fieldSet.readString("account")); 60 | transaction.setAmount(fieldSet.readBigDecimal("amount")); 61 | transaction.setTimestamp(fieldSet.readDate("timestamp", "yyyy-MM-dd HH:mm:ss")); 62 | 63 | return transaction; 64 | }) 65 | .build(); 66 | } 67 | 68 | @Bean 69 | @StepScope 70 | public JdbcBatchItemWriter writer(DataSource dataSource) { 71 | return new JdbcBatchItemWriterBuilder() 72 | .dataSource(dataSource) 73 | .beanMapped() 74 | .sql("INSERT INTO TRANSACTION (ACCOUNT, AMOUNT, TIMESTAMP) VALUES (:account, :amount, :timestamp)") 75 | .build(); 76 | } 77 | 78 | @Bean 79 | public Step step1(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 80 | VirtualThreadTaskExecutor taskExecutor = new VirtualThreadTaskExecutor(); 81 | 82 | return new StepBuilder("step1", jobRepository) 83 | .chunk(100, transactionManager) 84 | .reader(fileTransactionReader(null)) 85 | .writer(writer(null)) 86 | .taskExecutor(taskExecutor) 87 | .build(); 88 | } 89 | 90 | @Bean 91 | public Job multithreadedJob(JobRepository jobRepository) { 92 | return new JobBuilder("multithreadedJob", jobRepository) 93 | .start(step1(null, null)) 94 | .build(); 95 | } 96 | 97 | public static void main(String[] args) { 98 | String [] newArgs = new String[] {"inputFlatFile=/data/csv/bigtransactions.csv"}; 99 | 100 | SpringApplication.run(MultithreadedJobApplication.class, newArgs); 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /single-jvm-demos/src/main/java/io/spring/batch/scalingdemos/parallel/ParallelStepsJobApplication.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2018 the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package io.spring.batch.scalingdemos.parallel; 17 | 18 | import javax.sql.DataSource; 19 | 20 | import io.spring.batch.scalingdemos.domain.Transaction; 21 | 22 | import org.springframework.batch.core.Job; 23 | import org.springframework.batch.core.Step; 24 | import org.springframework.batch.core.job.flow.Flow; 25 | import org.springframework.batch.core.configuration.annotation.StepScope; 26 | import org.springframework.batch.core.job.builder.FlowBuilder; 27 | import org.springframework.batch.core.job.builder.JobBuilder; 28 | import org.springframework.batch.core.repository.JobRepository; 29 | import org.springframework.batch.core.step.builder.StepBuilder; 30 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 31 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 32 | import org.springframework.batch.item.file.FlatFileItemReader; 33 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 34 | import org.springframework.batch.item.xml.StaxEventItemReader; 35 | import org.springframework.batch.item.xml.builder.StaxEventItemReaderBuilder; 36 | import org.springframework.beans.factory.annotation.Value; 37 | import org.springframework.boot.SpringApplication; 38 | import org.springframework.boot.autoconfigure.SpringBootApplication; 39 | import org.springframework.context.annotation.Bean; 40 | import org.springframework.core.io.Resource; 41 | import org.springframework.core.task.VirtualThreadTaskExecutor; 42 | import org.springframework.oxm.jaxb.Jaxb2Marshaller; 43 | import org.springframework.transaction.PlatformTransactionManager; 44 | 45 | /** 46 | * @author Michael Minella 47 | */ 48 | @SpringBootApplication 49 | public class ParallelStepsJobApplication { 50 | 51 | @Bean 52 | public Job parallelStepsJob(JobRepository jobRepository) { 53 | Flow secondFlow = new FlowBuilder("secondFlow") 54 | .start(step2(null, null)) 55 | .build(); 56 | 57 | Flow parallelFlow = new FlowBuilder("parallelFlow") 58 | .start(step1(null, null)) 59 | .split(new VirtualThreadTaskExecutor()) 60 | .add(secondFlow) 61 | .build(); 62 | 63 | return new JobBuilder("parallelStepsJob", jobRepository) 64 | .start(parallelFlow) 65 | .end() 66 | .build(); 67 | } 68 | 69 | // @Bean 70 | // public Job sequentialStepsJob(JobRepository jobRepository) { 71 | // return new JobBuilder("sequentialStepsJob", jobRepository) 72 | // .start(step1(null, null)) 73 | // .next(step2(null, null)) 74 | // .build(); 75 | // } 76 | 77 | @Bean 78 | @StepScope 79 | public FlatFileItemReader fileTransactionReader( 80 | @Value("#{jobParameters['inputFlatFile']}") Resource resource) { 81 | 82 | return new FlatFileItemReaderBuilder() 83 | .name("flatFileTransactionReader") 84 | .resource(resource) 85 | .delimited() 86 | .names(new String[] {"account", "amount", "timestamp"}) 87 | .fieldSetMapper(fieldSet -> { 88 | Transaction transaction = new Transaction(); 89 | 90 | transaction.setAccount(fieldSet.readString("account")); 91 | transaction.setAmount(fieldSet.readBigDecimal("amount")); 92 | transaction.setTimestamp(fieldSet.readDate("timestamp", "yyyy-MM-dd HH:mm:ss")); 93 | 94 | return transaction; 95 | }) 96 | .build(); 97 | } 98 | 99 | @Bean 100 | @StepScope 101 | public StaxEventItemReader xmlTransactionReader( 102 | @Value("#{jobParameters['inputXmlFile']}") Resource resource) { 103 | Jaxb2Marshaller unmarshaller = new Jaxb2Marshaller(); 104 | unmarshaller.setClassesToBeBound(Transaction.class); 105 | 106 | return new StaxEventItemReaderBuilder() 107 | .name("xmlFileTransactionReader") 108 | .resource(resource) 109 | .addFragmentRootElements("transaction") 110 | .unmarshaller(unmarshaller) 111 | .build(); 112 | } 113 | 114 | @Bean 115 | @StepScope 116 | public JdbcBatchItemWriter writer(DataSource dataSource) { 117 | return new JdbcBatchItemWriterBuilder() 118 | .dataSource(dataSource) 119 | .beanMapped() 120 | .sql("INSERT INTO TRANSACTION (ACCOUNT, AMOUNT, TIMESTAMP) VALUES (:account, :amount, :timestamp)") 121 | .build(); 122 | } 123 | 124 | @Bean 125 | public Step step1(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 126 | return new StepBuilder("step1", jobRepository) 127 | .chunk(100, transactionManager) 128 | .reader(xmlTransactionReader(null)) 129 | .writer(writer(null)) 130 | .build(); 131 | } 132 | 133 | @Bean 134 | public Step step2(JobRepository jobRepository, PlatformTransactionManager transactionManager) { 135 | return new StepBuilder("step2", jobRepository) 136 | .chunk(100, transactionManager) 137 | .reader(fileTransactionReader(null)) 138 | .writer(writer(null)) 139 | .build(); 140 | } 141 | 142 | public static void main(String[] args) { 143 | String [] newArgs = new String[] {"inputFlatFile=/data/csv/bigtransactions.csv", 144 | "inputXmlFile=/data/xml/bigtransactions.xml"}; 145 | 146 | SpringApplication.run(ParallelStepsJobApplication.class, newArgs); 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /single-jvm-demos/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.sql.init.platform=mysql 2 | spring.datasource.driverClassName=com.mysql.cj.jdbc.Driver 3 | spring.datasource.url=jdbc:mysql://localhost:3306/scaling?cachePrepStmts=true&elideSetAutoCommits=true&alwaysSendSetIsolation=false&cacheResultSetMetadata=true&useSSL=false 4 | spring.datasource.username=root 5 | spring.datasource.password=p@ssw0rd 6 | #spring.datasource.schema=schema-mysql.sql 7 | spring.sql.init.mode=always 8 | spring.batch.jdbc.initialize-schema=always 9 | -------------------------------------------------------------------------------- /single-jvm-demos/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS TRANSACTION; 2 | 3 | CREATE TABLE TRANSACTION ( 4 | ACCOUNT VARCHAR(250) NOT NULL , 5 | TIMESTAMP DATETIME NOT NULL , 6 | AMOUNT DOUBLE NOT NULL 7 | ) ENGINE=InnoDB; 8 | 9 | -- CREATE TABLE BATCH_JOB_INSTANCE ( 10 | -- JOB_INSTANCE_ID BIGINT NOT NULL PRIMARY KEY , 11 | -- VERSION BIGINT , 12 | -- JOB_NAME VARCHAR(100) NOT NULL, 13 | -- JOB_KEY VARCHAR(32) NOT NULL, 14 | -- constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY) 15 | -- ) ENGINE=InnoDB; 16 | -- 17 | -- CREATE TABLE BATCH_JOB_EXECUTION ( 18 | -- JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY , 19 | -- VERSION BIGINT , 20 | -- JOB_INSTANCE_ID BIGINT NOT NULL, 21 | -- CREATE_TIME DATETIME NOT NULL, 22 | -- START_TIME DATETIME DEFAULT NULL , 23 | -- END_TIME DATETIME DEFAULT NULL , 24 | -- STATUS VARCHAR(10) , 25 | -- EXIT_CODE VARCHAR(2500) , 26 | -- EXIT_MESSAGE VARCHAR(2500) , 27 | -- LAST_UPDATED DATETIME, 28 | -- JOB_CONFIGURATION_LOCATION VARCHAR(2500) NULL, 29 | -- constraint JOB_INST_EXEC_FK foreign key (JOB_INSTANCE_ID) 30 | -- references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID) 31 | -- ) ENGINE=InnoDB; 32 | -- 33 | -- CREATE TABLE BATCH_JOB_EXECUTION_PARAMS ( 34 | -- JOB_EXECUTION_ID BIGINT NOT NULL , 35 | -- TYPE_CD VARCHAR(6) NOT NULL , 36 | -- KEY_NAME VARCHAR(100) NOT NULL , 37 | -- STRING_VAL VARCHAR(250) , 38 | -- DATE_VAL DATETIME DEFAULT NULL , 39 | -- LONG_VAL BIGINT , 40 | -- DOUBLE_VAL DOUBLE PRECISION , 41 | -- IDENTIFYING CHAR(1) NOT NULL , 42 | -- constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID) 43 | -- references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 44 | -- ) ENGINE=InnoDB; 45 | -- 46 | -- CREATE TABLE BATCH_STEP_EXECUTION ( 47 | -- STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY , 48 | -- VERSION BIGINT NOT NULL, 49 | -- STEP_NAME VARCHAR(100) NOT NULL, 50 | -- JOB_EXECUTION_ID BIGINT NOT NULL, 51 | -- START_TIME DATETIME NOT NULL , 52 | -- END_TIME DATETIME DEFAULT NULL , 53 | -- STATUS VARCHAR(10) , 54 | -- COMMIT_COUNT BIGINT , 55 | -- READ_COUNT BIGINT , 56 | -- FILTER_COUNT BIGINT , 57 | -- WRITE_COUNT BIGINT , 58 | -- READ_SKIP_COUNT BIGINT , 59 | -- WRITE_SKIP_COUNT BIGINT , 60 | -- PROCESS_SKIP_COUNT BIGINT , 61 | -- ROLLBACK_COUNT BIGINT , 62 | -- EXIT_CODE VARCHAR(2500) , 63 | -- EXIT_MESSAGE VARCHAR(2500) , 64 | -- LAST_UPDATED DATETIME, 65 | -- constraint JOB_EXEC_STEP_FK foreign key (JOB_EXECUTION_ID) 66 | -- references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 67 | -- ) ENGINE=InnoDB; 68 | -- 69 | -- CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT ( 70 | -- STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY, 71 | -- SHORT_CONTEXT VARCHAR(2500) NOT NULL, 72 | -- SERIALIZED_CONTEXT TEXT , 73 | -- constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID) 74 | -- references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID) 75 | -- ) ENGINE=InnoDB; 76 | -- 77 | -- CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT ( 78 | -- JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY, 79 | -- SHORT_CONTEXT VARCHAR(2500) NOT NULL, 80 | -- SERIALIZED_CONTEXT TEXT , 81 | -- constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID) 82 | -- references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 83 | -- ) ENGINE=InnoDB; 84 | -- 85 | -- CREATE TABLE BATCH_STEP_EXECUTION_SEQ ( 86 | -- ID BIGINT NOT NULL, 87 | -- UNIQUE_KEY CHAR(1) NOT NULL, 88 | -- constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 89 | -- ) ENGINE=InnoDB; 90 | -- 91 | -- INSERT INTO BATCH_STEP_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_STEP_EXECUTION_SEQ); 92 | -- 93 | -- CREATE TABLE BATCH_JOB_EXECUTION_SEQ ( 94 | -- ID BIGINT NOT NULL, 95 | -- UNIQUE_KEY CHAR(1) NOT NULL, 96 | -- constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 97 | -- ) ENGINE=InnoDB; 98 | -- 99 | -- INSERT INTO BATCH_JOB_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_EXECUTION_SEQ); 100 | -- 101 | -- CREATE TABLE BATCH_JOB_SEQ ( 102 | -- ID BIGINT NOT NULL, 103 | -- UNIQUE_KEY CHAR(1) NOT NULL, 104 | -- constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 105 | -- ) ENGINE=InnoDB; 106 | -- 107 | -- INSERT INTO BATCH_JOB_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_SEQ); 108 | --------------------------------------------------------------------------------