├── .github └── workflows │ └── maven.yml ├── .gitignore ├── .mvn └── wrapper │ ├── MavenWrapperDownloader.java │ ├── maven-wrapper.jar │ └── maven-wrapper.properties ├── .travis.yml ├── Dockerfile ├── README.md ├── Utilities.md ├── docker-compose.yml ├── mvnw ├── mvnw.cmd ├── pom.xml └── src ├── main ├── java │ └── org │ │ └── hung │ │ ├── SpringBatchMultiDatasrcApplication.java │ │ ├── config │ │ ├── BatchConfig.java │ │ ├── DatabaseConfig.java │ │ └── batch │ │ │ ├── JobConfig.java │ │ │ └── MyListener.java │ │ └── pojo │ │ └── AccountTxn.java └── resources │ ├── application.yml │ ├── data-mysql-account-db.sql │ ├── data-mysql-batch.sql │ ├── data-mysql-card-db.sql │ ├── schema-mysql-account-db.sql │ ├── schema-mysql-card-db.sql │ └── schema-mysql-spring-batch.sql └── test └── java └── org └── hung └── SpringBatchMultiDatasrcApplicationTests.java /.github/workflows/maven.yml: -------------------------------------------------------------------------------- 1 | name: Build project and docker image 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - name: Checkout repository 10 | uses: actions/checkout@v1 11 | 12 | - name: Set up JDK 1.8 13 | uses: actions/setup-java@v1 14 | with: 15 | java-version: 1.8 16 | 17 | - name : Enable maven cache 18 | uses: actions/cache@v1 19 | with: 20 | key: ${{ runner.os }}-maven-cache 21 | path: ~/.m2/repository 22 | 23 | - name: Build Spring Boot executable jar with Maven 24 | run: | 25 | chmod u+x ./mvnw 26 | ./mvnw -B clean package -DskipTests 27 | - name: Build Docker Image 28 | run: | 29 | ls -l target 30 | docker -v --debug 31 | docker build \ 32 | --cache-from library/openjdk \ 33 | --tag kwonghung/spring-batch-multi-datasource \ 34 | --build-arg BUILD_JAR_FILE=spring-batch-multi-datasrc-0.0.1-SNAPSHOT.jar . 35 | echo ${{ secrets.docker_hub_passwd }} | docker login --username kwonghung --password-stdin 36 | docker push kwonghung/spring-batch-multi-datasource 37 | 38 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | -------------------------------------------------------------------------------- /.mvn/wrapper/MavenWrapperDownloader.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2007-present the original author or authors. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * https://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | import java.net.*; 17 | import java.io.*; 18 | import java.nio.channels.*; 19 | import java.util.Properties; 20 | 21 | public class MavenWrapperDownloader { 22 | 23 | private static final String WRAPPER_VERSION = "0.5.6"; 24 | /** 25 | * Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided. 26 | */ 27 | private static final String DEFAULT_DOWNLOAD_URL = "https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/" 28 | + WRAPPER_VERSION + "/maven-wrapper-" + WRAPPER_VERSION + ".jar"; 29 | 30 | /** 31 | * Path to the maven-wrapper.properties file, which might contain a downloadUrl property to 32 | * use instead of the default one. 33 | */ 34 | private static final String MAVEN_WRAPPER_PROPERTIES_PATH = 35 | ".mvn/wrapper/maven-wrapper.properties"; 36 | 37 | /** 38 | * Path where the maven-wrapper.jar will be saved to. 39 | */ 40 | private static final String MAVEN_WRAPPER_JAR_PATH = 41 | ".mvn/wrapper/maven-wrapper.jar"; 42 | 43 | /** 44 | * Name of the property which should be used to override the default download url for the wrapper. 45 | */ 46 | private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl"; 47 | 48 | public static void main(String args[]) { 49 | System.out.println("- Downloader started"); 50 | File baseDirectory = new File(args[0]); 51 | System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath()); 52 | 53 | // If the maven-wrapper.properties exists, read it and check if it contains a custom 54 | // wrapperUrl parameter. 55 | File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH); 56 | String url = DEFAULT_DOWNLOAD_URL; 57 | if(mavenWrapperPropertyFile.exists()) { 58 | FileInputStream mavenWrapperPropertyFileInputStream = null; 59 | try { 60 | mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile); 61 | Properties mavenWrapperProperties = new Properties(); 62 | mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream); 63 | url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url); 64 | } catch (IOException e) { 65 | System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'"); 66 | } finally { 67 | try { 68 | if(mavenWrapperPropertyFileInputStream != null) { 69 | mavenWrapperPropertyFileInputStream.close(); 70 | } 71 | } catch (IOException e) { 72 | // Ignore ... 73 | } 74 | } 75 | } 76 | System.out.println("- Downloading from: " + url); 77 | 78 | File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH); 79 | if(!outputFile.getParentFile().exists()) { 80 | if(!outputFile.getParentFile().mkdirs()) { 81 | System.out.println( 82 | "- ERROR creating output directory '" + outputFile.getParentFile().getAbsolutePath() + "'"); 83 | } 84 | } 85 | System.out.println("- Downloading to: " + outputFile.getAbsolutePath()); 86 | try { 87 | downloadFileFromURL(url, outputFile); 88 | System.out.println("Done"); 89 | System.exit(0); 90 | } catch (Throwable e) { 91 | System.out.println("- Error downloading"); 92 | e.printStackTrace(); 93 | System.exit(1); 94 | } 95 | } 96 | 97 | private static void downloadFileFromURL(String urlString, File destination) throws Exception { 98 | if (System.getenv("MVNW_USERNAME") != null && System.getenv("MVNW_PASSWORD") != null) { 99 | String username = System.getenv("MVNW_USERNAME"); 100 | char[] password = System.getenv("MVNW_PASSWORD").toCharArray(); 101 | Authenticator.setDefault(new Authenticator() { 102 | @Override 103 | protected PasswordAuthentication getPasswordAuthentication() { 104 | return new PasswordAuthentication(username, password); 105 | } 106 | }); 107 | } 108 | URL website = new URL(urlString); 109 | ReadableByteChannel rbc; 110 | rbc = Channels.newChannel(website.openStream()); 111 | FileOutputStream fos = new FileOutputStream(destination); 112 | fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE); 113 | fos.close(); 114 | rbc.close(); 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/kwonghung-YIP/spring-batch-multi-datasource/77109513d06053b70310a7a56d9d6986cce775c3/.mvn/wrapper/maven-wrapper.jar -------------------------------------------------------------------------------- /.mvn/wrapper/maven-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.6.3/apache-maven-3.6.3-bin.zip 2 | wrapperUrl=https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | install: true 3 | 4 | services: 5 | - docker 6 | 7 | cache: 8 | directories: 9 | - $HOME/.m2 10 | 11 | #before_cache: 12 | #- rm -f $HOME/.gradle/caches/modules-2/modules-2.lock 13 | #- rm -fr $HOME/.gradle/caches/*/plugin-resolution/ 14 | 15 | before_install: 16 | - chmod +x mvnw 17 | 18 | jdk: 19 | - openjdk8 20 | 21 | script: 22 | - ./mvnw install -DskipTests=true -Dmaven.javadoc.skip=true -B -V 23 | - ./mvnw test -B 24 | - docker build -t kwonghung/spring-boot-k8s-secret --build-arg BUILD_JAR_FILE=spring-batch-multi-datasource-0.0.1-SNAPSHOT.jar . 25 | - echo "$DOCKER_HUB_PASSWD" | docker login --username "$DOCKER_HUB_LOGIN" --password-stdin 26 | - docker push kwonghung/spring-boot-k8s-secret 27 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM openjdk:8u242-slim 2 | LABEL maintainer="kwonghung.yip@gmail.com" 3 | 4 | ARG BUILD_JAR_FILE 5 | ENV BUILD_JAR_FILE=$BUILD_JAR_FILE 6 | ENV JAVA_OPTS="-Djava.security.egd=file:/dev/./urandom" 7 | 8 | WORKDIR /usr/local/springboot 9 | 10 | COPY target/$BUILD_JAR_FILE /usr/local/springboot 11 | 12 | #CMD ["./gradlew", "bootRun"] 13 | CMD ["sh","-c","sleep 20s && java $JAVA_OPTS -jar $BUILD_JAR_FILE"] 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Run this example with docker-compose 2 | 3 | ```bash 4 | wget --no-cache https://raw.githubusercontent.com/kwonghung-YIP/spring-batch-multi-datasource/master/docker-compose.yml 5 | docker-compose up 6 | ``` 7 | 8 | ## Deploy MySQL databases for local development 9 | 10 | ```bash 11 | docker run --name=mysql01 \ 12 | --detach \ 13 | --restart=always \ 14 | -p 23306:3306 \ 15 | -e MYSQL_DATABASE=spring-batch \ 16 | -e MYSQL_ROOT_PASSWORD=passw0rd \ 17 | -e MYSQL_USER=batch \ 18 | -e MYSQL_PASSWORD=testing \ 19 | mysql:8 20 | 21 | docker run --name=mysql02 \ 22 | --detach \ 23 | --restart=always \ 24 | -p 23307:3306 \ 25 | -e MYSQL_DATABASE=db1 \ 26 | -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ 27 | -e MYSQL_USER=user02 \ 28 | -e MYSQL_PASSWORD=testing \ 29 | mysql:8 30 | 31 | docker run --name=mysql03 \ 32 | --detach \ 33 | --restart=always \ 34 | -p 23308:3306 \ 35 | -e MYSQL_DATABASE=db2 \ 36 | -e MYSQL_RANDOM_ROOT_PASSWORD=yes \ 37 | -e MYSQL_USER=user03 \ 38 | -e MYSQL_PASSWORD=testing \ 39 | mysql:8 40 | ``` 41 | -------------------------------------------------------------------------------- /Utilities.md: -------------------------------------------------------------------------------- 1 | docker run \ 2 | -it --rm --link mysql01 -v `pwd`/schema-mysql-card-db.sql:/tmp/test.sql \ 3 | mysql:8 mysql --host=mysql01 --port=3306 -ubatch -ptesting < test.sql 4 | 5 | docker run \ 6 | -it --rm --link mysql02 \ 7 | mysql:8 mysql --host=mysql02 --port=3306 -uuser02 -ptesting 8 | 9 | docker run \ 10 | -it --rm \ 11 | mysql:8 mysql --help 12 | 13 | docker run \ 14 | -it --rm --network host \ 15 | mysql:8 mysql --host=localhost --port=23306 -ubatch -ptesting 16 | 17 | docker run \ 18 | -it --rm --network host \ 19 | mysql:8 mysql --host=192.168.79.129 --port=23306 -ubatch -ptesting 20 | 21 | INSERT INTO BATCH_STEP_EXECUTION_SEQ values(0,0); 22 | INSERT INTO BATCH_JOB_EXECUTION_SEQ values(0,0); 23 | INSERT INTO BATCH_JOB_SEQ values(0,0); 24 | 25 | https://github.com/spring-projects/spring-batch/blob/master/spring-batch-core/src/main/resources/org/springframework/batch/core/schema-mysql.sql 26 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | spring-batch-db: 5 | image: mysql:8 6 | environment: 7 | - MYSQL_RANDOM_ROOT_PASSWORD=yes 8 | - MYSQL_DATABASE=db 9 | - MYSQL_USER=batch 10 | - MYSQL_PASSWORD=testing 11 | 12 | card-db: 13 | image: mysql:8 14 | environment: 15 | - MYSQL_RANDOM_ROOT_PASSWORD=yes 16 | - MYSQL_DATABASE=db 17 | - MYSQL_USER=card 18 | - MYSQL_PASSWORD=testing 19 | 20 | account-db: 21 | image: mysql:8 22 | environment: 23 | - MYSQL_RANDOM_ROOT_PASSWORD=yes 24 | - MYSQL_DATABASE=db 25 | - MYSQL_USER=account 26 | - MYSQL_PASSWORD=testing 27 | 28 | spring-batch: 29 | image: kwonghung/spring-batch-multi-datasource:latest 30 | environment: 31 | - SPRING_PROFILES_ACTIVE=docker-compose 32 | depends_on: 33 | - spring-batch-db 34 | - card-db 35 | - account-db 36 | 37 | 38 | -------------------------------------------------------------------------------- /mvnw: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | # ---------------------------------------------------------------------------- 3 | # Licensed to the Apache Software Foundation (ASF) under one 4 | # or more contributor license agreements. See the NOTICE file 5 | # distributed with this work for additional information 6 | # regarding copyright ownership. The ASF licenses this file 7 | # to you under the Apache License, Version 2.0 (the 8 | # "License"); you may not use this file except in compliance 9 | # with the License. You may obtain a copy of the License at 10 | # 11 | # https://www.apache.org/licenses/LICENSE-2.0 12 | # 13 | # Unless required by applicable law or agreed to in writing, 14 | # software distributed under the License is distributed on an 15 | # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 16 | # KIND, either express or implied. See the License for the 17 | # specific language governing permissions and limitations 18 | # under the License. 19 | # ---------------------------------------------------------------------------- 20 | 21 | # ---------------------------------------------------------------------------- 22 | # Maven Start Up Batch script 23 | # 24 | # Required ENV vars: 25 | # ------------------ 26 | # JAVA_HOME - location of a JDK home dir 27 | # 28 | # Optional ENV vars 29 | # ----------------- 30 | # M2_HOME - location of maven2's installed home dir 31 | # MAVEN_OPTS - parameters passed to the Java VM when running Maven 32 | # e.g. to debug Maven itself, use 33 | # set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 34 | # MAVEN_SKIP_RC - flag to disable loading of mavenrc files 35 | # ---------------------------------------------------------------------------- 36 | 37 | if [ -z "$MAVEN_SKIP_RC" ] ; then 38 | 39 | if [ -f /etc/mavenrc ] ; then 40 | . /etc/mavenrc 41 | fi 42 | 43 | if [ -f "$HOME/.mavenrc" ] ; then 44 | . "$HOME/.mavenrc" 45 | fi 46 | 47 | fi 48 | 49 | # OS specific support. $var _must_ be set to either true or false. 50 | cygwin=false; 51 | darwin=false; 52 | mingw=false 53 | case "`uname`" in 54 | CYGWIN*) cygwin=true ;; 55 | MINGW*) mingw=true;; 56 | Darwin*) darwin=true 57 | # Use /usr/libexec/java_home if available, otherwise fall back to /Library/Java/Home 58 | # See https://developer.apple.com/library/mac/qa/qa1170/_index.html 59 | if [ -z "$JAVA_HOME" ]; then 60 | if [ -x "/usr/libexec/java_home" ]; then 61 | export JAVA_HOME="`/usr/libexec/java_home`" 62 | else 63 | export JAVA_HOME="/Library/Java/Home" 64 | fi 65 | fi 66 | ;; 67 | esac 68 | 69 | if [ -z "$JAVA_HOME" ] ; then 70 | if [ -r /etc/gentoo-release ] ; then 71 | JAVA_HOME=`java-config --jre-home` 72 | fi 73 | fi 74 | 75 | if [ -z "$M2_HOME" ] ; then 76 | ## resolve links - $0 may be a link to maven's home 77 | PRG="$0" 78 | 79 | # need this for relative symlinks 80 | while [ -h "$PRG" ] ; do 81 | ls=`ls -ld "$PRG"` 82 | link=`expr "$ls" : '.*-> \(.*\)$'` 83 | if expr "$link" : '/.*' > /dev/null; then 84 | PRG="$link" 85 | else 86 | PRG="`dirname "$PRG"`/$link" 87 | fi 88 | done 89 | 90 | saveddir=`pwd` 91 | 92 | M2_HOME=`dirname "$PRG"`/.. 93 | 94 | # make it fully qualified 95 | M2_HOME=`cd "$M2_HOME" && pwd` 96 | 97 | cd "$saveddir" 98 | # echo Using m2 at $M2_HOME 99 | fi 100 | 101 | # For Cygwin, ensure paths are in UNIX format before anything is touched 102 | if $cygwin ; then 103 | [ -n "$M2_HOME" ] && 104 | M2_HOME=`cygpath --unix "$M2_HOME"` 105 | [ -n "$JAVA_HOME" ] && 106 | JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 107 | [ -n "$CLASSPATH" ] && 108 | CLASSPATH=`cygpath --path --unix "$CLASSPATH"` 109 | fi 110 | 111 | # For Mingw, ensure paths are in UNIX format before anything is touched 112 | if $mingw ; then 113 | [ -n "$M2_HOME" ] && 114 | M2_HOME="`(cd "$M2_HOME"; pwd)`" 115 | [ -n "$JAVA_HOME" ] && 116 | JAVA_HOME="`(cd "$JAVA_HOME"; pwd)`" 117 | fi 118 | 119 | if [ -z "$JAVA_HOME" ]; then 120 | javaExecutable="`which javac`" 121 | if [ -n "$javaExecutable" ] && ! [ "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then 122 | # readlink(1) is not available as standard on Solaris 10. 123 | readLink=`which readlink` 124 | if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then 125 | if $darwin ; then 126 | javaHome="`dirname \"$javaExecutable\"`" 127 | javaExecutable="`cd \"$javaHome\" && pwd -P`/javac" 128 | else 129 | javaExecutable="`readlink -f \"$javaExecutable\"`" 130 | fi 131 | javaHome="`dirname \"$javaExecutable\"`" 132 | javaHome=`expr "$javaHome" : '\(.*\)/bin'` 133 | JAVA_HOME="$javaHome" 134 | export JAVA_HOME 135 | fi 136 | fi 137 | fi 138 | 139 | if [ -z "$JAVACMD" ] ; then 140 | if [ -n "$JAVA_HOME" ] ; then 141 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 142 | # IBM's JDK on AIX uses strange locations for the executables 143 | JAVACMD="$JAVA_HOME/jre/sh/java" 144 | else 145 | JAVACMD="$JAVA_HOME/bin/java" 146 | fi 147 | else 148 | JAVACMD="`which java`" 149 | fi 150 | fi 151 | 152 | if [ ! -x "$JAVACMD" ] ; then 153 | echo "Error: JAVA_HOME is not defined correctly." >&2 154 | echo " We cannot execute $JAVACMD" >&2 155 | exit 1 156 | fi 157 | 158 | if [ -z "$JAVA_HOME" ] ; then 159 | echo "Warning: JAVA_HOME environment variable is not set." 160 | fi 161 | 162 | CLASSWORLDS_LAUNCHER=org.codehaus.plexus.classworlds.launcher.Launcher 163 | 164 | # traverses directory structure from process work directory to filesystem root 165 | # first directory with .mvn subdirectory is considered project base directory 166 | find_maven_basedir() { 167 | 168 | if [ -z "$1" ] 169 | then 170 | echo "Path not specified to find_maven_basedir" 171 | return 1 172 | fi 173 | 174 | basedir="$1" 175 | wdir="$1" 176 | while [ "$wdir" != '/' ] ; do 177 | if [ -d "$wdir"/.mvn ] ; then 178 | basedir=$wdir 179 | break 180 | fi 181 | # workaround for JBEAP-8937 (on Solaris 10/Sparc) 182 | if [ -d "${wdir}" ]; then 183 | wdir=`cd "$wdir/.."; pwd` 184 | fi 185 | # end of workaround 186 | done 187 | echo "${basedir}" 188 | } 189 | 190 | # concatenates all lines of a file 191 | concat_lines() { 192 | if [ -f "$1" ]; then 193 | echo "$(tr -s '\n' ' ' < "$1")" 194 | fi 195 | } 196 | 197 | BASE_DIR=`find_maven_basedir "$(pwd)"` 198 | if [ -z "$BASE_DIR" ]; then 199 | exit 1; 200 | fi 201 | 202 | ########################################################################################## 203 | # Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 204 | # This allows using the maven wrapper in projects that prohibit checking in binary data. 205 | ########################################################################################## 206 | if [ -r "$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" ]; then 207 | if [ "$MVNW_VERBOSE" = true ]; then 208 | echo "Found .mvn/wrapper/maven-wrapper.jar" 209 | fi 210 | else 211 | if [ "$MVNW_VERBOSE" = true ]; then 212 | echo "Couldn't find .mvn/wrapper/maven-wrapper.jar, downloading it ..." 213 | fi 214 | if [ -n "$MVNW_REPOURL" ]; then 215 | jarUrl="$MVNW_REPOURL/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 216 | else 217 | jarUrl="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 218 | fi 219 | while IFS="=" read key value; do 220 | case "$key" in (wrapperUrl) jarUrl="$value"; break ;; 221 | esac 222 | done < "$BASE_DIR/.mvn/wrapper/maven-wrapper.properties" 223 | if [ "$MVNW_VERBOSE" = true ]; then 224 | echo "Downloading from: $jarUrl" 225 | fi 226 | wrapperJarPath="$BASE_DIR/.mvn/wrapper/maven-wrapper.jar" 227 | if $cygwin; then 228 | wrapperJarPath=`cygpath --path --windows "$wrapperJarPath"` 229 | fi 230 | 231 | if command -v wget > /dev/null; then 232 | if [ "$MVNW_VERBOSE" = true ]; then 233 | echo "Found wget ... using wget" 234 | fi 235 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 236 | wget "$jarUrl" -O "$wrapperJarPath" 237 | else 238 | wget --http-user=$MVNW_USERNAME --http-password=$MVNW_PASSWORD "$jarUrl" -O "$wrapperJarPath" 239 | fi 240 | elif command -v curl > /dev/null; then 241 | if [ "$MVNW_VERBOSE" = true ]; then 242 | echo "Found curl ... using curl" 243 | fi 244 | if [ -z "$MVNW_USERNAME" ] || [ -z "$MVNW_PASSWORD" ]; then 245 | curl -o "$wrapperJarPath" "$jarUrl" -f 246 | else 247 | curl --user $MVNW_USERNAME:$MVNW_PASSWORD -o "$wrapperJarPath" "$jarUrl" -f 248 | fi 249 | 250 | else 251 | if [ "$MVNW_VERBOSE" = true ]; then 252 | echo "Falling back to using Java to download" 253 | fi 254 | javaClass="$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.java" 255 | # For Cygwin, switch paths to Windows format before running javac 256 | if $cygwin; then 257 | javaClass=`cygpath --path --windows "$javaClass"` 258 | fi 259 | if [ -e "$javaClass" ]; then 260 | if [ ! -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 261 | if [ "$MVNW_VERBOSE" = true ]; then 262 | echo " - Compiling MavenWrapperDownloader.java ..." 263 | fi 264 | # Compiling the Java class 265 | ("$JAVA_HOME/bin/javac" "$javaClass") 266 | fi 267 | if [ -e "$BASE_DIR/.mvn/wrapper/MavenWrapperDownloader.class" ]; then 268 | # Running the downloader 269 | if [ "$MVNW_VERBOSE" = true ]; then 270 | echo " - Running MavenWrapperDownloader.java ..." 271 | fi 272 | ("$JAVA_HOME/bin/java" -cp .mvn/wrapper MavenWrapperDownloader "$MAVEN_PROJECTBASEDIR") 273 | fi 274 | fi 275 | fi 276 | fi 277 | ########################################################################################## 278 | # End of extension 279 | ########################################################################################## 280 | 281 | export MAVEN_PROJECTBASEDIR=${MAVEN_BASEDIR:-"$BASE_DIR"} 282 | if [ "$MVNW_VERBOSE" = true ]; then 283 | echo $MAVEN_PROJECTBASEDIR 284 | fi 285 | MAVEN_OPTS="$(concat_lines "$MAVEN_PROJECTBASEDIR/.mvn/jvm.config") $MAVEN_OPTS" 286 | 287 | # For Cygwin, switch paths to Windows format before running java 288 | if $cygwin; then 289 | [ -n "$M2_HOME" ] && 290 | M2_HOME=`cygpath --path --windows "$M2_HOME"` 291 | [ -n "$JAVA_HOME" ] && 292 | JAVA_HOME=`cygpath --path --windows "$JAVA_HOME"` 293 | [ -n "$CLASSPATH" ] && 294 | CLASSPATH=`cygpath --path --windows "$CLASSPATH"` 295 | [ -n "$MAVEN_PROJECTBASEDIR" ] && 296 | MAVEN_PROJECTBASEDIR=`cygpath --path --windows "$MAVEN_PROJECTBASEDIR"` 297 | fi 298 | 299 | # Provide a "standardized" way to retrieve the CLI args that will 300 | # work with both Windows and non-Windows executions. 301 | MAVEN_CMD_LINE_ARGS="$MAVEN_CONFIG $@" 302 | export MAVEN_CMD_LINE_ARGS 303 | 304 | WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 305 | 306 | exec "$JAVACMD" \ 307 | $MAVEN_OPTS \ 308 | -classpath "$MAVEN_PROJECTBASEDIR/.mvn/wrapper/maven-wrapper.jar" \ 309 | "-Dmaven.home=${M2_HOME}" "-Dmaven.multiModuleProjectDirectory=${MAVEN_PROJECTBASEDIR}" \ 310 | ${WRAPPER_LAUNCHER} $MAVEN_CONFIG "$@" 311 | -------------------------------------------------------------------------------- /mvnw.cmd: -------------------------------------------------------------------------------- 1 | @REM ---------------------------------------------------------------------------- 2 | @REM Licensed to the Apache Software Foundation (ASF) under one 3 | @REM or more contributor license agreements. See the NOTICE file 4 | @REM distributed with this work for additional information 5 | @REM regarding copyright ownership. The ASF licenses this file 6 | @REM to you under the Apache License, Version 2.0 (the 7 | @REM "License"); you may not use this file except in compliance 8 | @REM with the License. You may obtain a copy of the License at 9 | @REM 10 | @REM https://www.apache.org/licenses/LICENSE-2.0 11 | @REM 12 | @REM Unless required by applicable law or agreed to in writing, 13 | @REM software distributed under the License is distributed on an 14 | @REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY 15 | @REM KIND, either express or implied. See the License for the 16 | @REM specific language governing permissions and limitations 17 | @REM under the License. 18 | @REM ---------------------------------------------------------------------------- 19 | 20 | @REM ---------------------------------------------------------------------------- 21 | @REM Maven Start Up Batch script 22 | @REM 23 | @REM Required ENV vars: 24 | @REM JAVA_HOME - location of a JDK home dir 25 | @REM 26 | @REM Optional ENV vars 27 | @REM M2_HOME - location of maven2's installed home dir 28 | @REM MAVEN_BATCH_ECHO - set to 'on' to enable the echoing of the batch commands 29 | @REM MAVEN_BATCH_PAUSE - set to 'on' to wait for a keystroke before ending 30 | @REM MAVEN_OPTS - parameters passed to the Java VM when running Maven 31 | @REM e.g. to debug Maven itself, use 32 | @REM set MAVEN_OPTS=-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000 33 | @REM MAVEN_SKIP_RC - flag to disable loading of mavenrc files 34 | @REM ---------------------------------------------------------------------------- 35 | 36 | @REM Begin all REM lines with '@' in case MAVEN_BATCH_ECHO is 'on' 37 | @echo off 38 | @REM set title of command window 39 | title %0 40 | @REM enable echoing by setting MAVEN_BATCH_ECHO to 'on' 41 | @if "%MAVEN_BATCH_ECHO%" == "on" echo %MAVEN_BATCH_ECHO% 42 | 43 | @REM set %HOME% to equivalent of $HOME 44 | if "%HOME%" == "" (set "HOME=%HOMEDRIVE%%HOMEPATH%") 45 | 46 | @REM Execute a user defined script before this one 47 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPre 48 | @REM check for pre script, once with legacy .bat ending and once with .cmd ending 49 | if exist "%HOME%\mavenrc_pre.bat" call "%HOME%\mavenrc_pre.bat" 50 | if exist "%HOME%\mavenrc_pre.cmd" call "%HOME%\mavenrc_pre.cmd" 51 | :skipRcPre 52 | 53 | @setlocal 54 | 55 | set ERROR_CODE=0 56 | 57 | @REM To isolate internal variables from possible post scripts, we use another setlocal 58 | @setlocal 59 | 60 | @REM ==== START VALIDATION ==== 61 | if not "%JAVA_HOME%" == "" goto OkJHome 62 | 63 | echo. 64 | echo Error: JAVA_HOME not found in your environment. >&2 65 | echo Please set the JAVA_HOME variable in your environment to match the >&2 66 | echo location of your Java installation. >&2 67 | echo. 68 | goto error 69 | 70 | :OkJHome 71 | if exist "%JAVA_HOME%\bin\java.exe" goto init 72 | 73 | echo. 74 | echo Error: JAVA_HOME is set to an invalid directory. >&2 75 | echo JAVA_HOME = "%JAVA_HOME%" >&2 76 | echo Please set the JAVA_HOME variable in your environment to match the >&2 77 | echo location of your Java installation. >&2 78 | echo. 79 | goto error 80 | 81 | @REM ==== END VALIDATION ==== 82 | 83 | :init 84 | 85 | @REM Find the project base dir, i.e. the directory that contains the folder ".mvn". 86 | @REM Fallback to current working directory if not found. 87 | 88 | set MAVEN_PROJECTBASEDIR=%MAVEN_BASEDIR% 89 | IF NOT "%MAVEN_PROJECTBASEDIR%"=="" goto endDetectBaseDir 90 | 91 | set EXEC_DIR=%CD% 92 | set WDIR=%EXEC_DIR% 93 | :findBaseDir 94 | IF EXIST "%WDIR%"\.mvn goto baseDirFound 95 | cd .. 96 | IF "%WDIR%"=="%CD%" goto baseDirNotFound 97 | set WDIR=%CD% 98 | goto findBaseDir 99 | 100 | :baseDirFound 101 | set MAVEN_PROJECTBASEDIR=%WDIR% 102 | cd "%EXEC_DIR%" 103 | goto endDetectBaseDir 104 | 105 | :baseDirNotFound 106 | set MAVEN_PROJECTBASEDIR=%EXEC_DIR% 107 | cd "%EXEC_DIR%" 108 | 109 | :endDetectBaseDir 110 | 111 | IF NOT EXIST "%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config" goto endReadAdditionalConfig 112 | 113 | @setlocal EnableExtensions EnableDelayedExpansion 114 | for /F "usebackq delims=" %%a in ("%MAVEN_PROJECTBASEDIR%\.mvn\jvm.config") do set JVM_CONFIG_MAVEN_PROPS=!JVM_CONFIG_MAVEN_PROPS! %%a 115 | @endlocal & set JVM_CONFIG_MAVEN_PROPS=%JVM_CONFIG_MAVEN_PROPS% 116 | 117 | :endReadAdditionalConfig 118 | 119 | SET MAVEN_JAVA_EXE="%JAVA_HOME%\bin\java.exe" 120 | set WRAPPER_JAR="%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.jar" 121 | set WRAPPER_LAUNCHER=org.apache.maven.wrapper.MavenWrapperMain 122 | 123 | set DOWNLOAD_URL="https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 124 | 125 | FOR /F "tokens=1,2 delims==" %%A IN ("%MAVEN_PROJECTBASEDIR%\.mvn\wrapper\maven-wrapper.properties") DO ( 126 | IF "%%A"=="wrapperUrl" SET DOWNLOAD_URL=%%B 127 | ) 128 | 129 | @REM Extension to allow automatically downloading the maven-wrapper.jar from Maven-central 130 | @REM This allows using the maven wrapper in projects that prohibit checking in binary data. 131 | if exist %WRAPPER_JAR% ( 132 | if "%MVNW_VERBOSE%" == "true" ( 133 | echo Found %WRAPPER_JAR% 134 | ) 135 | ) else ( 136 | if not "%MVNW_REPOURL%" == "" ( 137 | SET DOWNLOAD_URL="%MVNW_REPOURL%/io/takari/maven-wrapper/0.5.6/maven-wrapper-0.5.6.jar" 138 | ) 139 | if "%MVNW_VERBOSE%" == "true" ( 140 | echo Couldn't find %WRAPPER_JAR%, downloading it ... 141 | echo Downloading from: %DOWNLOAD_URL% 142 | ) 143 | 144 | powershell -Command "&{"^ 145 | "$webclient = new-object System.Net.WebClient;"^ 146 | "if (-not ([string]::IsNullOrEmpty('%MVNW_USERNAME%') -and [string]::IsNullOrEmpty('%MVNW_PASSWORD%'))) {"^ 147 | "$webclient.Credentials = new-object System.Net.NetworkCredential('%MVNW_USERNAME%', '%MVNW_PASSWORD%');"^ 148 | "}"^ 149 | "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; $webclient.DownloadFile('%DOWNLOAD_URL%', '%WRAPPER_JAR%')"^ 150 | "}" 151 | if "%MVNW_VERBOSE%" == "true" ( 152 | echo Finished downloading %WRAPPER_JAR% 153 | ) 154 | ) 155 | @REM End of extension 156 | 157 | @REM Provide a "standardized" way to retrieve the CLI args that will 158 | @REM work with both Windows and non-Windows executions. 159 | set MAVEN_CMD_LINE_ARGS=%* 160 | 161 | %MAVEN_JAVA_EXE% %JVM_CONFIG_MAVEN_PROPS% %MAVEN_OPTS% %MAVEN_DEBUG_OPTS% -classpath %WRAPPER_JAR% "-Dmaven.multiModuleProjectDirectory=%MAVEN_PROJECTBASEDIR%" %WRAPPER_LAUNCHER% %MAVEN_CONFIG% %* 162 | if ERRORLEVEL 1 goto error 163 | goto end 164 | 165 | :error 166 | set ERROR_CODE=1 167 | 168 | :end 169 | @endlocal & set ERROR_CODE=%ERROR_CODE% 170 | 171 | if not "%MAVEN_SKIP_RC%" == "" goto skipRcPost 172 | @REM check for post script, once with legacy .bat ending and once with .cmd ending 173 | if exist "%HOME%\mavenrc_post.bat" call "%HOME%\mavenrc_post.bat" 174 | if exist "%HOME%\mavenrc_post.cmd" call "%HOME%\mavenrc_post.cmd" 175 | :skipRcPost 176 | 177 | @REM pause the script if MAVEN_BATCH_PAUSE is set to 'on' 178 | if "%MAVEN_BATCH_PAUSE%" == "on" pause 179 | 180 | if "%MAVEN_TERMINATE_CMD%" == "on" exit %ERROR_CODE% 181 | 182 | exit /B %ERROR_CODE% 183 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.2.4.RELEASE 9 | 10 | 11 | hung-poc 12 | spring-batch-multi-datasrc 13 | 0.0.1-SNAPSHOT 14 | spring-batch-multi-datasrc 15 | Spring Batch Multiple Data Source Example 16 | 17 | 18 | 1.8 19 | 20 | 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-batch 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-jdbc 29 | 30 | 36 | 37 | org.springframework.data 38 | spring-data-commons 39 | 40 | 41 | 42 | org.springframework.boot 43 | spring-boot-devtools 44 | runtime 45 | true 46 | 47 | 48 | mysql 49 | mysql-connector-java 50 | runtime 51 | 52 | 53 | org.projectlombok 54 | lombok 55 | true 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-starter-test 60 | test 61 | 62 | 63 | org.junit.vintage 64 | junit-vintage-engine 65 | 66 | 67 | 68 | 69 | org.springframework.batch 70 | spring-batch-test 71 | test 72 | 73 | 74 | 75 | 76 | 77 | 78 | org.springframework.boot 79 | spring-boot-maven-plugin 80 | 81 | 82 | 83 | 84 | 85 | -------------------------------------------------------------------------------- /src/main/java/org/hung/SpringBatchMultiDatasrcApplication.java: -------------------------------------------------------------------------------- 1 | package org.hung; 2 | 3 | import java.util.Date; 4 | 5 | import org.springframework.batch.core.Job; 6 | import org.springframework.batch.core.JobParameters; 7 | import org.springframework.batch.core.JobParametersBuilder; 8 | import org.springframework.batch.core.JobParametersInvalidException; 9 | import org.springframework.batch.core.launch.JobLauncher; 10 | import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; 11 | import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; 12 | import org.springframework.batch.core.repository.JobRestartException; 13 | import org.springframework.boot.SpringApplication; 14 | import org.springframework.boot.autoconfigure.SpringBootApplication; 15 | import org.springframework.context.ApplicationContext; 16 | 17 | @SpringBootApplication 18 | public class SpringBatchMultiDatasrcApplication { 19 | 20 | public static void main(String[] args) { 21 | ApplicationContext appCtx = SpringApplication.run(SpringBatchMultiDatasrcApplication.class, args); 22 | 23 | JobLauncher joblauncher = appCtx.getBean(JobLauncher.class); 24 | 25 | Job job = appCtx.getBean("runBalanceJob",Job.class); 26 | 27 | 28 | JobParameters jobParameters = new JobParametersBuilder() 29 | .addDate("runTime", new Date()) 30 | .toJobParameters(); 31 | 32 | try { 33 | joblauncher.run(job, jobParameters); 34 | } catch (JobExecutionAlreadyRunningException e) { 35 | // TODO Auto-generated catch block 36 | e.printStackTrace(); 37 | } catch (JobRestartException e) { 38 | // TODO Auto-generated catch block 39 | e.printStackTrace(); 40 | } catch (JobInstanceAlreadyCompleteException e) { 41 | // TODO Auto-generated catch block 42 | e.printStackTrace(); 43 | } catch (JobParametersInvalidException e) { 44 | // TODO Auto-generated catch block 45 | e.printStackTrace(); 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/org/hung/config/BatchConfig.java: -------------------------------------------------------------------------------- 1 | package org.hung.config; 2 | 3 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 4 | import org.springframework.context.annotation.Configuration; 5 | 6 | @Configuration 7 | @EnableBatchProcessing 8 | public class BatchConfig { 9 | 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/org/hung/config/DatabaseConfig.java: -------------------------------------------------------------------------------- 1 | package org.hung.config; 2 | 3 | import javax.sql.DataSource; 4 | 5 | import org.springframework.beans.factory.annotation.Value; 6 | import org.springframework.boot.context.properties.ConfigurationProperties; 7 | import org.springframework.boot.jdbc.DataSourceBuilder; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.context.annotation.Primary; 11 | import org.springframework.core.io.Resource; 12 | import org.springframework.data.transaction.ChainedTransactionManager; 13 | import org.springframework.jdbc.datasource.DataSourceTransactionManager; 14 | import org.springframework.jdbc.datasource.init.DataSourceInitializer; 15 | import org.springframework.jdbc.datasource.init.ResourceDatabasePopulator; 16 | import org.springframework.transaction.PlatformTransactionManager; 17 | 18 | import com.zaxxer.hikari.HikariDataSource; 19 | 20 | @Configuration 21 | public class DatabaseConfig { 22 | 23 | @Bean 24 | @Primary 25 | @ConfigurationProperties("spring-batch-db.datasource") 26 | DataSource springBatchDb() { 27 | DataSourceBuilder builder = DataSourceBuilder.create(); 28 | builder.type(HikariDataSource.class); 29 | return builder.build(); 30 | } 31 | 32 | /*@Bean 33 | DataSourceInitializer springBatchDbInit(@Value("${spring-batch-db.init-script}") Resource initScript) { 34 | DataSourceInitializer initializer = new DataSourceInitializer(); 35 | initializer.setDataSource(cardDb()); 36 | initializer.setEnabled(true); 37 | 38 | ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); 39 | populator.addScript(initScript); 40 | initializer.setDatabasePopulator(populator); 41 | 42 | ResourceDatabasePopulator cleaner = new ResourceDatabasePopulator(); 43 | initializer.setDatabaseCleaner(cleaner); 44 | 45 | return initializer; 46 | }*/ 47 | 48 | @Bean 49 | @ConfigurationProperties("card-db.datasource") 50 | DataSource cardDb() { 51 | DataSourceBuilder builder = DataSourceBuilder.create(); 52 | builder.type(HikariDataSource.class); 53 | return builder.build(); 54 | } 55 | 56 | @Bean 57 | PlatformTransactionManager cardDbTxManager() { 58 | return new DataSourceTransactionManager(cardDb()); 59 | } 60 | 61 | @Bean 62 | DataSourceInitializer cardDbInit(@Value("${card-db.init-script}") Resource initScript) { 63 | DataSourceInitializer initializer = new DataSourceInitializer(); 64 | initializer.setDataSource(cardDb()); 65 | initializer.setEnabled(true); 66 | 67 | ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); 68 | populator.addScript(initScript); 69 | populator.setSeparator("//"); 70 | initializer.setDatabasePopulator(populator); 71 | 72 | ResourceDatabasePopulator cleaner = new ResourceDatabasePopulator(); 73 | initializer.setDatabaseCleaner(cleaner); 74 | 75 | return initializer; 76 | } 77 | 78 | @Bean 79 | @ConfigurationProperties("account-db.datasource") 80 | DataSource accountDb() { 81 | DataSourceBuilder builder = DataSourceBuilder.create(); 82 | builder.type(HikariDataSource.class); 83 | return builder.build(); 84 | } 85 | 86 | @Bean 87 | DataSourceInitializer accountDbInit(@Value("${account-db.init-script}") Resource initScript) { 88 | DataSourceInitializer initializer = new DataSourceInitializer(); 89 | initializer.setDataSource(accountDb()); 90 | initializer.setEnabled(true); 91 | 92 | ResourceDatabasePopulator populator = new ResourceDatabasePopulator(); 93 | populator.addScript(initScript); 94 | initializer.setDatabasePopulator(populator); 95 | 96 | ResourceDatabasePopulator cleaner = new ResourceDatabasePopulator(); 97 | initializer.setDatabaseCleaner(cleaner); 98 | 99 | return initializer; 100 | } 101 | 102 | @Bean 103 | PlatformTransactionManager accountDbTxManager() { 104 | return new DataSourceTransactionManager(accountDb()); 105 | //return new JtaTransactionManager(); 106 | } 107 | 108 | @Bean 109 | PlatformTransactionManager chainTxManager() { 110 | ChainedTransactionManager txManager = new ChainedTransactionManager(cardDbTxManager(),accountDbTxManager()); 111 | return txManager; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/org/hung/config/batch/JobConfig.java: -------------------------------------------------------------------------------- 1 | package org.hung.config.batch; 2 | 3 | import java.sql.Timestamp; 4 | import java.time.LocalDateTime; 5 | 6 | import javax.sql.DataSource; 7 | 8 | import org.hung.pojo.AccountTxn; 9 | import org.springframework.batch.core.Job; 10 | import org.springframework.batch.core.Step; 11 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 12 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 13 | import org.springframework.batch.item.ItemProcessor; 14 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 15 | import org.springframework.batch.item.database.JdbcCursorItemReader; 16 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 17 | import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; 18 | import org.springframework.batch.item.support.CompositeItemWriter; 19 | import org.springframework.batch.item.support.builder.CompositeItemWriterBuilder; 20 | import org.springframework.beans.factory.annotation.Autowired; 21 | import org.springframework.beans.factory.annotation.Qualifier; 22 | import org.springframework.context.annotation.Bean; 23 | import org.springframework.context.annotation.Configuration; 24 | import org.springframework.jdbc.core.JdbcTemplate; 25 | import org.springframework.transaction.PlatformTransactionManager; 26 | import org.springframework.transaction.TransactionDefinition; 27 | import org.springframework.transaction.interceptor.DefaultTransactionAttribute; 28 | 29 | import lombok.extern.slf4j.Slf4j; 30 | 31 | @Slf4j 32 | @Configuration 33 | public class JobConfig { 34 | 35 | @Autowired 36 | private JobBuilderFactory jobFactory; 37 | 38 | @Autowired 39 | private StepBuilderFactory stepFactory; 40 | 41 | @Bean 42 | JdbcCursorItemReader txnReader(@Qualifier("cardDb") DataSource datasource) { 43 | return new JdbcCursorItemReaderBuilder() 44 | .name("account-txn-reader") 45 | .dataSource(datasource) 46 | .sql("select t.*, c.credit_limit from CARD_TXN t join CARD c on t.card_no = c.card_no where t.post = 'N' order by t.tx_datetime") 47 | .rowMapper((rs,rownum) -> { 48 | AccountTxn bean = new AccountTxn(); 49 | 50 | bean.setTxRef(rs.getLong("tx_ref")); 51 | Timestamp txTs = rs.getTimestamp("tx_datetime"); 52 | bean.setTxDatetime(txTs==null?null:txTs.toLocalDateTime()); 53 | bean.setCardNo(rs.getString("card_no")); 54 | bean.setCreditLimit(rs.getBigDecimal("credit_limit")); 55 | bean.setCrAmt(rs.getBigDecimal("cr_amount")); 56 | bean.setDrAmt(rs.getBigDecimal("dr_amount")); 57 | bean.setPosted("Y".equalsIgnoreCase(rs.getString("post"))); 58 | Timestamp postTs = rs.getTimestamp("post_datetime"); 59 | bean.setPostDatetime(postTs==null?null:postTs.toLocalDateTime()); 60 | 61 | return bean; 62 | }) 63 | .build(); 64 | } 65 | 66 | ItemProcessor txnProcessor() { 67 | return (item) -> { 68 | item.setPosted(true); 69 | item.setPostDatetime(LocalDateTime.now()); 70 | return item; 71 | }; 72 | } 73 | 74 | @Bean 75 | CompositeItemWriter txnWriter() { 76 | return new CompositeItemWriterBuilder() 77 | .delegates(balanceWriter(null),postTxnWriter(null)) 78 | .build(); 79 | } 80 | 81 | @Bean 82 | JdbcBatchItemWriter balanceWriter(@Qualifier("accountDb") DataSource datasource) { 83 | return new JdbcBatchItemWriterBuilder() 84 | .dataSource(datasource) 85 | .assertUpdates(true) 86 | .sql("insert into ACCOUNT (account_no, credit_limit, balance, last_tx_ref, last_tx_datetime) values (:cardNo, :creditLimit, :netAmt, :txRef, :txDatetime)" + 87 | " on duplicate key update BALANCE = BALANCE + :netAmt, LAST_TX_REF = :txRef, LAST_TX_DATETIME = :txDatetime") 88 | .beanMapped() 89 | .build(); 90 | } 91 | 92 | @Bean 93 | JdbcBatchItemWriter postTxnWriter(@Qualifier("cardDb") DataSource datasource) { 94 | return new JdbcBatchItemWriterBuilder() 95 | .dataSource(datasource) 96 | .assertUpdates(true) 97 | .sql("update CARD_TXN set POST = 'Y', POST_DATETIME = :postDatetime where TX_REF = :txRef") 98 | .beanMapped() 99 | .build(); 100 | } 101 | 102 | @Bean 103 | MyListener listener(@Qualifier("cardDb") DataSource datasource) { 104 | return new MyListener(new JdbcTemplate(datasource)); 105 | } 106 | 107 | @Bean 108 | Step runBalanceStep(@Qualifier("chainTxManager") PlatformTransactionManager transactionManager) { 109 | 110 | DefaultTransactionAttribute txAttribute = new DefaultTransactionAttribute(); 111 | txAttribute.setPropagationBehavior(TransactionDefinition.PROPAGATION_REQUIRES_NEW); 112 | txAttribute.setIsolationLevel(TransactionDefinition.ISOLATION_READ_COMMITTED); 113 | txAttribute.setTimeout(600); 114 | 115 | return stepFactory.get("run-balance-step") 116 | .transactionManager(transactionManager) 117 | .chunk(100) 118 | .faultTolerant() 119 | //.skipPolicy(new AlwaysSkipItemSkipPolicy()) 120 | .reader(txnReader(null)) 121 | .processor(txnProcessor()) 122 | .writer(txnWriter()) 123 | .transactionAttribute(txAttribute) 124 | .listener(listener(null)) 125 | .build(); 126 | } 127 | 128 | @Bean 129 | Job runBalanceJob() { 130 | return jobFactory.get("run-balance-job") 131 | .start(runBalanceStep(null)) 132 | .build(); 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /src/main/java/org/hung/config/batch/MyListener.java: -------------------------------------------------------------------------------- 1 | package org.hung.config.batch; 2 | 3 | import java.util.List; 4 | 5 | import org.springframework.batch.core.annotation.AfterChunk; 6 | import org.springframework.batch.core.annotation.OnWriteError; 7 | import org.springframework.batch.core.scope.context.ChunkContext; 8 | import org.springframework.jdbc.core.JdbcTemplate; 9 | 10 | import lombok.extern.slf4j.Slf4j; 11 | 12 | @Slf4j 13 | public class MyListener { 14 | 15 | private JdbcTemplate cardDb; 16 | 17 | public MyListener(JdbcTemplate jdbcTemplate) { 18 | cardDb = jdbcTemplate; 19 | } 20 | 21 | /*@AfterWrite 22 | void afterWrite(List items) { 23 | log.info("afterWrite"); 24 | }*/ 25 | 26 | @AfterChunk 27 | void afterChunk(ChunkContext context) { 28 | int count = cardDb.queryForObject("select count(1) from CARD_TXN where post = 'Y'", Integer.class); 29 | log.info("afterChunk {}",count); 30 | 31 | } 32 | 33 | @OnWriteError 34 | void writeError(Exception e, List items) { 35 | int count = cardDb.queryForObject("select count(1) from CARD_TXN where post = 'Y'", Integer.class); 36 | log.error("writeError {}",count); 37 | } 38 | 39 | /*@AfterChunkError 40 | void afterChunkError(ChunkContext context) { 41 | log.info("afterChunkError"); 42 | }*/ 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/org/hung/pojo/AccountTxn.java: -------------------------------------------------------------------------------- 1 | package org.hung.pojo; 2 | 3 | import java.math.BigDecimal; 4 | import java.time.LocalDateTime; 5 | 6 | import lombok.Data; 7 | 8 | @Data 9 | public class AccountTxn { 10 | 11 | private long txRef; 12 | private LocalDateTime txDatetime; 13 | private String cardNo; 14 | private BigDecimal creditLimit; 15 | private BigDecimal crAmt; 16 | private BigDecimal drAmt; 17 | private boolean posted; 18 | private LocalDateTime postDatetime; 19 | 20 | public BigDecimal getNetAmt() { 21 | BigDecimal netAmt = BigDecimal.ZERO; 22 | if (crAmt!=null) { 23 | netAmt = netAmt.add(crAmt); 24 | } 25 | if (drAmt!=null) { 26 | netAmt = netAmt.subtract(drAmt); 27 | } 28 | return netAmt; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/resources/application.yml: -------------------------------------------------------------------------------- 1 | logging: 2 | level: 3 | org.springframework.batch: info 4 | org.springframework.batch.repeat: info 5 | com.zaxxer.hikari: info 6 | org.springframework.jdbc: info 7 | org.springframework.transaction: info 8 | org.springframework.data.transaction: info 9 | #com.atomikos: debug 10 | 11 | --- 12 | spring: 13 | profiles: local 14 | batch: 15 | job: 16 | enabled: false 17 | initialize-schema: never 18 | 19 | spring-batch-db: 20 | datasource: 21 | driver-class-name: com.mysql.cj.jdbc.Driver 22 | jdbc-url: jdbc:mysql://192.168.79.129:23306/spring-batch?useSSL=false&allowPublicKeyRetrieval=true 23 | username: batch 24 | password: testing 25 | auto-commit: false 26 | 27 | card-db: 28 | datasource: 29 | driver-class-name: com.mysql.cj.jdbc.Driver 30 | jdbc-url: jdbc:mysql://192.168.79.129:23307/db1 31 | username: user02 32 | password: testing 33 | auto-commit: false 34 | init-script: classpath:data-mysql-card-db.sql 35 | 36 | account-db: 37 | datasource: 38 | driver-class-name: com.mysql.cj.jdbc.Driver 39 | jdbc-url: jdbc:mysql://192.168.79.129:23308/db2 40 | username: user03 41 | password: testing 42 | auto-commit: false 43 | init-script: classpath:data-mysql-account-db.sql 44 | 45 | --- 46 | spring: 47 | profiles: docker-compose 48 | batch: 49 | job: 50 | enabled: false 51 | initialize-schema: always 52 | 53 | spring-batch-db: 54 | datasource: 55 | driver-class-name: com.mysql.cj.jdbc.Driver 56 | jdbc-url: jdbc:mysql://spring-batch-db:3306/db?useSSL=false&allowPublicKeyRetrieval=true 57 | username: batch 58 | password: testing 59 | auto-commit: true 60 | #init-script: classpath:schema-mysql-spring-batch.sql 61 | 62 | card-db: 63 | datasource: 64 | driver-class-name: com.mysql.cj.jdbc.Driver 65 | jdbc-url: jdbc:mysql://card-db:3306/db 66 | username: card 67 | password: testing 68 | auto-commit: false 69 | init-script: classpath:schema-mysql-card-db.sql 70 | 71 | account-db: 72 | datasource: 73 | driver-class-name: com.mysql.cj.jdbc.Driver 74 | jdbc-url: jdbc:mysql://account-db:3306/db 75 | username: account 76 | password: testing 77 | auto-commit: false 78 | init-script: classpath:schema-mysql-account-db.sql 79 | -------------------------------------------------------------------------------- /src/main/resources/data-mysql-account-db.sql: -------------------------------------------------------------------------------- 1 | delete from ACCOUNT; 2 | 3 | commit; -------------------------------------------------------------------------------- /src/main/resources/data-mysql-batch.sql: -------------------------------------------------------------------------------- 1 | delete from BATCH_STEP_EXECUTION_CONTEXT; 2 | delete from BATCH_STEP_EXECUTION; 3 | delete from BATCH_JOB_EXECUTION_PARAMS; 4 | delete from BATCH_JOB_EXECUTION_CONTEXT; 5 | delete from BATCH_JOB_EXECUTION; 6 | delete from BATCH_JOB_INSTANCE; 7 | 8 | 9 | select 10 | STEP_EXECUTION_ID 11 | , VERSION 12 | , STEP_NAME 13 | , JOB_EXECUTION_ID 14 | , START_TIME 15 | , END_TIME 16 | , STATUS 17 | , COMMIT_COUNT 18 | , READ_COUNT 19 | , FILTER_COUNT 20 | , WRITE_COUNT 21 | , READ_SKIP_COUNT 22 | , WRITE_SKIP_COUNT 23 | , PROCESS_SKIP_COUNT 24 | , ROLLBACK_COUNT 25 | , EXIT_CODE 26 | from BATCH_STEP_EXECUTION; -------------------------------------------------------------------------------- /src/main/resources/data-mysql-card-db.sql: -------------------------------------------------------------------------------- 1 | update CARD_TXN set POST = 'N', POST_DATETIME = null; 2 | 3 | commit; -------------------------------------------------------------------------------- /src/main/resources/schema-mysql-account-db.sql: -------------------------------------------------------------------------------- 1 | drop table if exists ACCOUNT; 2 | 3 | create table ACCOUNT ( 4 | ACCOUNT_NO varchar(20) not null primary key, 5 | BALANCE decimal(10,2) not null default 0, 6 | CREDIT_LIMIT decimal(10,2) not null check (CREDIT_LIMIT > 0), 7 | LAST_TX_REF int, 8 | LAST_TX_DATETIME datetime(4), 9 | constraint CREDIT_LIMIT_CHK check (BALANCE * -1 <= CREDIT_LIMIT) 10 | ); -------------------------------------------------------------------------------- /src/main/resources/schema-mysql-card-db.sql: -------------------------------------------------------------------------------- 1 | drop procedure if exists GENERATE_CARD_TXN// 2 | 3 | drop table if exists CARD_TXN// 4 | 5 | drop table if exists CARD// 6 | 7 | create table CARD ( 8 | CARD_NO varchar(20) not null primary key, 9 | CREDIT_LIMIT decimal(10,2) not null 10 | )// 11 | 12 | create table CARD_TXN ( 13 | TX_REF int not null primary key, 14 | TX_DATETIME datetime(4) not null, 15 | CARD_NO varchar(20) not null, 16 | CR_AMOUNT decimal(10,2), 17 | DR_AMOUNT decimal(10,2), 18 | POST char(1) not null default 'N' check (POST in ('Y','N')), 19 | POST_DATETIME datetime(4), 20 | foreign key FK_CARD_NO(CARD_NO) references CARD(CARD_NO) 21 | on update cascade on delete restrict 22 | )// 23 | 24 | create procedure GENERATE_CARD_TXN( 25 | in no_of_card decimal(10), 26 | in no_of_txn decimal(10) 27 | ) 28 | begin 29 | declare n int; 30 | declare d1 date; 31 | declare d2 date; 32 | declare no_of_days int; 33 | declare v_card_no varchar(20); 34 | declare cr_amt decimal(10,2); 35 | declare dr_amt decimal(10,2); 36 | 37 | start transaction; 38 | 39 | set n = 1; 40 | while n <= no_of_card do 41 | insert into CARD ( 42 | card_no, credit_limit 43 | ) values ( 44 | concat(lpad(floor(rand()*9999),4,'0'),'-',lpad(floor(rand()*9999),4,'0'),'-',lpad(floor(rand()*9999),4,'0'),'-',lpad(floor(rand()*9999),4,'0')), 45 | round(rand()*40,0)*100+1000 46 | ); 47 | set n = n + 1; 48 | end while; 49 | 50 | set d1 = last_day(now()); 51 | set d2 = last_day(date_add(now(), interval -1 month)); 52 | set no_of_days=timestampdiff(day,d2,d1); 53 | set n = 1; 54 | while n <= no_of_txn do 55 | 56 | select card_no from CARD order by rand() limit 1 into v_card_no; 57 | 58 | if rand() > 0.8 then 59 | set cr_amt = round(rand()*30,2)*100; 60 | set dr_amt = null; 61 | else 62 | set cr_amt = null; 63 | set dr_amt = round(rand()*500,2); 64 | end if; 65 | 66 | insert into CARD_TXN ( 67 | tx_ref, tx_datetime, card_no, cr_amount, dr_amount 68 | ) values ( 69 | n, date_add(date_add(d2, interval round(rand()*no_of_days) day), interval round(rand()*24*60*60) second), v_card_no, cr_amt, dr_amt 70 | ); 71 | 72 | set n = n + 1; 73 | end while; 74 | 75 | commit; 76 | end // 77 | 78 | 79 | call GENERATE_CARD_TXN(500,10000)// 80 | 81 | 82 | -------------------------------------------------------------------------------- /src/main/resources/schema-mysql-spring-batch.sql: -------------------------------------------------------------------------------- 1 | -- Autogenerated: do not edit this file 2 | 3 | CREATE TABLE BATCH_JOB_INSTANCE ( 4 | JOB_INSTANCE_ID BIGINT NOT NULL PRIMARY KEY , 5 | VERSION BIGINT , 6 | JOB_NAME VARCHAR(100) NOT NULL, 7 | JOB_KEY VARCHAR(32) NOT NULL, 8 | constraint JOB_INST_UN unique (JOB_NAME, JOB_KEY) 9 | ) ENGINE=InnoDB; 10 | 11 | CREATE TABLE BATCH_JOB_EXECUTION ( 12 | JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY , 13 | VERSION BIGINT , 14 | JOB_INSTANCE_ID BIGINT NOT NULL, 15 | CREATE_TIME DATETIME NOT NULL, 16 | START_TIME DATETIME DEFAULT NULL , 17 | END_TIME DATETIME DEFAULT NULL , 18 | STATUS VARCHAR(10) , 19 | EXIT_CODE VARCHAR(2500) , 20 | EXIT_MESSAGE VARCHAR(2500) , 21 | LAST_UPDATED DATETIME, 22 | JOB_CONFIGURATION_LOCATION VARCHAR(2500) NULL, 23 | constraint JOB_INST_EXEC_FK foreign key (JOB_INSTANCE_ID) 24 | references BATCH_JOB_INSTANCE(JOB_INSTANCE_ID) 25 | ) ENGINE=InnoDB; 26 | 27 | CREATE TABLE BATCH_JOB_EXECUTION_PARAMS ( 28 | JOB_EXECUTION_ID BIGINT NOT NULL , 29 | TYPE_CD VARCHAR(6) NOT NULL , 30 | KEY_NAME VARCHAR(100) NOT NULL , 31 | STRING_VAL VARCHAR(250) , 32 | DATE_VAL DATETIME DEFAULT NULL , 33 | LONG_VAL BIGINT , 34 | DOUBLE_VAL DOUBLE PRECISION , 35 | IDENTIFYING CHAR(1) NOT NULL , 36 | constraint JOB_EXEC_PARAMS_FK foreign key (JOB_EXECUTION_ID) 37 | references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 38 | ) ENGINE=InnoDB; 39 | 40 | CREATE TABLE BATCH_STEP_EXECUTION ( 41 | STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY , 42 | VERSION BIGINT NOT NULL, 43 | STEP_NAME VARCHAR(100) NOT NULL, 44 | JOB_EXECUTION_ID BIGINT NOT NULL, 45 | START_TIME DATETIME NOT NULL , 46 | END_TIME DATETIME DEFAULT NULL , 47 | STATUS VARCHAR(10) , 48 | COMMIT_COUNT BIGINT , 49 | READ_COUNT BIGINT , 50 | FILTER_COUNT BIGINT , 51 | WRITE_COUNT BIGINT , 52 | READ_SKIP_COUNT BIGINT , 53 | WRITE_SKIP_COUNT BIGINT , 54 | PROCESS_SKIP_COUNT BIGINT , 55 | ROLLBACK_COUNT BIGINT , 56 | EXIT_CODE VARCHAR(2500) , 57 | EXIT_MESSAGE VARCHAR(2500) , 58 | LAST_UPDATED DATETIME, 59 | constraint JOB_EXEC_STEP_FK foreign key (JOB_EXECUTION_ID) 60 | references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 61 | ) ENGINE=InnoDB; 62 | 63 | CREATE TABLE BATCH_STEP_EXECUTION_CONTEXT ( 64 | STEP_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY, 65 | SHORT_CONTEXT VARCHAR(2500) NOT NULL, 66 | SERIALIZED_CONTEXT TEXT , 67 | constraint STEP_EXEC_CTX_FK foreign key (STEP_EXECUTION_ID) 68 | references BATCH_STEP_EXECUTION(STEP_EXECUTION_ID) 69 | ) ENGINE=InnoDB; 70 | 71 | CREATE TABLE BATCH_JOB_EXECUTION_CONTEXT ( 72 | JOB_EXECUTION_ID BIGINT NOT NULL PRIMARY KEY, 73 | SHORT_CONTEXT VARCHAR(2500) NOT NULL, 74 | SERIALIZED_CONTEXT TEXT , 75 | constraint JOB_EXEC_CTX_FK foreign key (JOB_EXECUTION_ID) 76 | references BATCH_JOB_EXECUTION(JOB_EXECUTION_ID) 77 | ) ENGINE=InnoDB; 78 | 79 | CREATE TABLE BATCH_STEP_EXECUTION_SEQ ( 80 | ID BIGINT NOT NULL, 81 | UNIQUE_KEY CHAR(1) NOT NULL, 82 | constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 83 | ) ENGINE=InnoDB; 84 | 85 | INSERT INTO BATCH_STEP_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_STEP_EXECUTION_SEQ); 86 | 87 | CREATE TABLE BATCH_JOB_EXECUTION_SEQ ( 88 | ID BIGINT NOT NULL, 89 | UNIQUE_KEY CHAR(1) NOT NULL, 90 | constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 91 | ) ENGINE=InnoDB; 92 | 93 | INSERT INTO BATCH_JOB_EXECUTION_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_EXECUTION_SEQ); 94 | 95 | CREATE TABLE BATCH_JOB_SEQ ( 96 | ID BIGINT NOT NULL, 97 | UNIQUE_KEY CHAR(1) NOT NULL, 98 | constraint UNIQUE_KEY_UN unique (UNIQUE_KEY) 99 | ) ENGINE=InnoDB; 100 | 101 | INSERT INTO BATCH_JOB_SEQ (ID, UNIQUE_KEY) select * from (select 0 as ID, '0' as UNIQUE_KEY) as tmp where not exists(select * from BATCH_JOB_SEQ); 102 | -------------------------------------------------------------------------------- /src/test/java/org/hung/SpringBatchMultiDatasrcApplicationTests.java: -------------------------------------------------------------------------------- 1 | package org.hung; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class SpringBatchMultiDatasrcApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | --------------------------------------------------------------------------------