├── .gitignore ├── data ├── empty_vgsales.csv └── vgsales.csv ├── docker-compose.yml ├── leader ├── build.gradle ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── native.sh ├── settings.gradle └── src │ └── main │ ├── java │ └── com │ │ └── example │ │ └── batch │ │ ├── BatchApplication.java │ │ ├── GameByYear.java │ │ ├── YearPlatformSales.java │ │ └── YearReport.java │ └── resources │ ├── application.properties │ └── schema.sql ├── readme.md └── worker ├── build.gradle ├── data ├── empty_vgsales.csv └── vgsales.csv ├── gradle └── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties ├── gradlew ├── gradlew.bat ├── native.sh ├── settings.gradle └── src ├── main ├── java │ └── com │ │ └── example │ │ └── batch │ │ └── BatchApplication.java └── resources │ └── application.properties └── test └── java └── com └── example └── batch └── BatchApplicationTests.java /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | .gradle 3 | build/ 4 | !gradle/wrapper/gradle-wrapper.jar 5 | !**/src/main/**/build/ 6 | !**/src/test/**/build/ 7 | 8 | ### STS ### 9 | .apt_generated 10 | .classpath 11 | .factorypath 12 | .project 13 | .settings 14 | .springBeans 15 | .sts4-cache 16 | bin/ 17 | !**/src/main/**/bin/ 18 | !**/src/test/**/bin/ 19 | 20 | ### IntelliJ IDEA ### 21 | .idea 22 | *.iws 23 | *.iml 24 | *.ipr 25 | out/ 26 | !**/src/main/**/out/ 27 | !**/src/test/**/out/ 28 | 29 | ### NetBeans ### 30 | /nbproject/private/ 31 | /nbbuild/ 32 | /dist/ 33 | /nbdist/ 34 | /.nb-gradle/ 35 | 36 | ### VS Code ### 37 | .vscode/ 38 | .DS_Store 39 | -------------------------------------------------------------------------------- /data/empty_vgsales.csv: -------------------------------------------------------------------------------- 1 | Rank,Name,Platform,Year,Genre,Publisher,NA_Sales,EU_Sales,JP_Sales,Other_Sales,Global_Sales -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | postgres: 5 | image: postgres:latest 6 | environment: 7 | - POSTGRES_USER=postgres 8 | - PGUSER=postgres 9 | - POSTGRES_NAME=postgres 10 | - POSTGRES_PASSWORD=postgres 11 | ports: 12 | - "5432:5432" 13 | rabbitmq: 14 | image: rabbitmq:3-management 15 | environment: 16 | - RABBITMQ_DEFAULT_USER=user 17 | - RABBITMQ_DEFAULT_PASS=password 18 | ports: 19 | - "5672:5672" 20 | - "15672:15672" -------------------------------------------------------------------------------- /leader/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'org.springframework.boot' version '3.0.2' 4 | id 'io.spring.dependency-management' version '1.1.0' 5 | id 'org.graalvm.buildtools.native' version '0.9.18' 6 | } 7 | 8 | group = 'com.example' 9 | version = '0.0.1-SNAPSHOT' 10 | sourceCompatibility = '17' 11 | 12 | repositories { 13 | mavenCentral() 14 | mavenLocal() 15 | } 16 | 17 | dependencies { 18 | implementation 'com.joshlong:batch-remotechunking-spring-boot-starter:1.2' 19 | implementation 'org.springframework.batch:spring-batch-integration' 20 | implementation 'org.springframework.integration:spring-integration-amqp' 21 | implementation 'org.springframework.boot:spring-boot-starter-integration' 22 | implementation 'org.springframework.boot:spring-boot-starter-batch' 23 | implementation 'org.springframework.boot:spring-boot-starter-amqp' 24 | implementation 'org.springframework.boot:spring-boot-starter-json' 25 | implementation 'org.springframework.boot:spring-boot-starter-jdbc' 26 | implementation 'org.postgresql:postgresql' 27 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 28 | testImplementation 'org.springframework.batch:spring-batch-test' 29 | } 30 | 31 | tasks.named('test') { 32 | useJUnitPlatform() 33 | } 34 | -------------------------------------------------------------------------------- /leader/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coffee-software-show/lets-code-spring-batch/987fb894669f3b074590234883c14b98703cf71e/leader/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /leader/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /leader/gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Stop when "xargs" is not available. 209 | if ! command -v xargs >/dev/null 2>&1 210 | then 211 | die "xargs is not available" 212 | fi 213 | 214 | # Use "xargs" to parse quoted args. 215 | # 216 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 217 | # 218 | # In Bash we could simply go: 219 | # 220 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 221 | # set -- "${ARGS[@]}" "$@" 222 | # 223 | # but POSIX shell has neither arrays nor command substitution, so instead we 224 | # post-process each arg (as a line of input to sed) to backslash-escape any 225 | # character that might be a shell metacharacter, then use eval to reverse 226 | # that process (while maintaining the separation between arguments), and wrap 227 | # the whole thing up as a single "set" statement. 228 | # 229 | # This will of course break if any of these variables contains a newline or 230 | # an unmatched quote. 231 | # 232 | 233 | eval "set -- $( 234 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 235 | xargs -n1 | 236 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 237 | tr '\n' ' ' 238 | )" '"$@"' 239 | 240 | exec "$JAVACMD" "$@" 241 | -------------------------------------------------------------------------------- /leader/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if %ERRORLEVEL% equ 0 goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if %ERRORLEVEL% equ 0 goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | set EXIT_CODE=%ERRORLEVEL% 84 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 85 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 86 | exit /b %EXIT_CODE% 87 | 88 | :mainEnd 89 | if "%OS%"=="Windows_NT" endlocal 90 | 91 | :omega 92 | -------------------------------------------------------------------------------- /leader/native.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ./gradlew clean 3 | ./gradlew nativeCompile 4 | ./build/native/nativeCompile/leader 5 | -------------------------------------------------------------------------------- /leader/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'leader' 2 | -------------------------------------------------------------------------------- /leader/src/main/java/com/example/batch/BatchApplication.java: -------------------------------------------------------------------------------- 1 | package com.example.batch; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.joshlong.batch.remotechunking.leader.LeaderChunkStep; 5 | import com.joshlong.batch.remotechunking.leader.LeaderInboundChunkChannel; 6 | import com.joshlong.batch.remotechunking.leader.LeaderItemWriter; 7 | import com.joshlong.batch.remotechunking.leader.LeaderOutboundChunkChannel; 8 | import org.springframework.amqp.core.Queue; 9 | import org.springframework.amqp.core.*; 10 | import org.springframework.amqp.rabbit.connection.ConnectionFactory; 11 | import org.springframework.aop.SpringProxy; 12 | import org.springframework.aop.framework.Advised; 13 | import org.springframework.aot.hint.MemberCategory; 14 | import org.springframework.aot.hint.RuntimeHints; 15 | import org.springframework.aot.hint.RuntimeHintsRegistrar; 16 | import org.springframework.batch.core.*; 17 | import org.springframework.batch.core.configuration.annotation.StepScope; 18 | import org.springframework.batch.core.job.builder.JobBuilder; 19 | import org.springframework.batch.core.launch.JobOperator; 20 | import org.springframework.batch.core.launch.support.RunIdIncrementer; 21 | import org.springframework.batch.core.repository.JobRepository; 22 | import org.springframework.batch.core.step.builder.StepBuilder; 23 | import org.springframework.batch.core.step.tasklet.TaskletStep; 24 | import org.springframework.batch.integration.chunk.ChunkMessageChannelItemWriter; 25 | import org.springframework.batch.item.ItemReader; 26 | import org.springframework.batch.item.ItemWriter; 27 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 28 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 29 | import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; 30 | import org.springframework.batch.item.file.FlatFileItemReader; 31 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 32 | import org.springframework.batch.repeat.RepeatStatus; 33 | import org.springframework.beans.factory.annotation.Value; 34 | import org.springframework.boot.SpringApplication; 35 | import org.springframework.boot.autoconfigure.SpringBootApplication; 36 | import org.springframework.boot.autoconfigure.batch.JobExecutionEvent; 37 | import org.springframework.context.annotation.Bean; 38 | import org.springframework.context.annotation.Configuration; 39 | import org.springframework.context.annotation.ImportRuntimeHints; 40 | import org.springframework.context.event.EventListener; 41 | import org.springframework.core.DecoratingProxy; 42 | import org.springframework.core.io.Resource; 43 | import org.springframework.integration.amqp.dsl.Amqp; 44 | import org.springframework.integration.dsl.IntegrationFlow; 45 | import org.springframework.jdbc.core.JdbcTemplate; 46 | import org.springframework.jdbc.core.RowMapper; 47 | import org.springframework.jdbc.core.namedparam.MapSqlParameterSource; 48 | import org.springframework.messaging.MessageChannel; 49 | import org.springframework.messaging.PollableChannel; 50 | import org.springframework.transaction.PlatformTransactionManager; 51 | import org.springframework.transaction.support.TransactionTemplate; 52 | 53 | import javax.sql.DataSource; 54 | import java.util.*; 55 | import java.util.concurrent.ConcurrentHashMap; 56 | 57 | @SpringBootApplication 58 | @ImportRuntimeHints(BatchApplication.Hints.class) 59 | public class BatchApplication { 60 | 61 | static class Hints implements RuntimeHintsRegistrar { 62 | 63 | @Override 64 | public void registerHints(RuntimeHints hints, ClassLoader classLoader) { 65 | // todo https://github.com/spring-projects/spring-batch/issues/4294 66 | hints.proxies().registerJdkProxy(JobOperator.class, SpringProxy.class, Advised.class, DecoratingProxy.class); 67 | Set.of(YearReport.class, GameByYear.class, YearPlatformSales.class) 68 | .forEach(clzz -> hints.reflection().registerType(clzz, MemberCategory.values())); 69 | 70 | } 71 | } 72 | 73 | public static void main(String[] args) { 74 | System.setProperty("csvFile", "file:///Users/jlong/Desktop/lets-code-spring-batch/data/vgsales.csv"); 75 | SpringApplication.run(BatchApplication.class, args); 76 | } 77 | 78 | public static final String EMPTY_CSV_STATUS = "EMPTY"; 79 | 80 | @Bean 81 | Job job( 82 | JobRepository jobRepository, 83 | ErrorStepConfiguration errorStepConfiguration, 84 | CsvToDbStepConfiguration csvToDbStepConfiguration, 85 | YearPlatformReportStepConfiguration yearPlatformReportStepConfiguration, 86 | YearReportStepConfiguration yearReportStepConfiguration, 87 | EndStepConfiguration endStepConfiguration) { 88 | var gameByYearStep = csvToDbStepConfiguration.gameByYearStep(); 89 | return new JobBuilder("job", jobRepository)// 90 | .incrementer(new RunIdIncrementer())// 91 | .start(gameByYearStep).on(EMPTY_CSV_STATUS).to(errorStepConfiguration.errorStep()) // 92 | .from(gameByYearStep).on("*").to(yearPlatformReportStepConfiguration.yearPlatformReportStep()) // 93 | .next(yearReportStepConfiguration.yearReportStep()) 94 | .next(endStepConfiguration.endStep()) // 95 | .build() // 96 | .build(); 97 | 98 | } 99 | } 100 | 101 | @Configuration 102 | class YearReportStepConfiguration { 103 | 104 | // todo: clear this out after the job is done. some sort of listener? 105 | private final Map reportMap = new ConcurrentHashMap<>(); 106 | private final DataSource dataSource; 107 | private final JobRepository repository; 108 | private final PlatformTransactionManager transactionManager; 109 | private final ObjectMapper objectMapper; 110 | private final ItemWriter itemWriter; 111 | 112 | YearReportStepConfiguration(JobRepository repository, DataSource dataSource, 113 | PlatformTransactionManager transactionManager, ObjectMapper objectMapper, 114 | @LeaderItemWriter ChunkMessageChannelItemWriter itemWriter) { 115 | this.dataSource = dataSource; 116 | this.repository = repository; 117 | this.transactionManager = transactionManager; 118 | this.objectMapper = objectMapper; 119 | this.itemWriter = itemWriter; 120 | } 121 | 122 | @EventListener 123 | void batchJobCompleted(JobExecutionEvent event) { 124 | var running = Map.of(// 125 | "running", event.getJobExecution().getStatus().isRunning(),// 126 | "finished", event.getJobExecution().getExitStatus().getExitCode() // 127 | );// 128 | System.out.println("jobExecutionEvent: [" + running + "]"); 129 | this.reportMap.clear(); 130 | } 131 | 132 | private final RowMapper rowMapper = (rs, rowNum) -> { 133 | var year = rs.getInt("year"); 134 | if (!this.reportMap.containsKey(year)) 135 | this.reportMap.put(year, new YearReport(year, new ArrayList<>())); 136 | var yr = this.reportMap.get(year); 137 | yr.breakout().add(new YearPlatformSales(rs.getInt("year"), rs.getString("platform"), rs.getFloat("sales"))); 138 | return yr; 139 | }; 140 | 141 | @Bean 142 | ItemReader yearPlatformSalesItemReader() { 143 | var sql = """ 144 | select year , 145 | ypr.platform, 146 | ypr.sales, 147 | (select count(yps.year) from year_platform_report yps where yps.year = ypr.year ) 148 | from year_platform_report ypr 149 | where ypr.year != 0 150 | order by year 151 | """; 152 | return new JdbcCursorItemReaderBuilder() 153 | .sql(sql) 154 | .name("yearPlatformSalesItemReader") 155 | .dataSource(this.dataSource) 156 | .rowMapper(this.rowMapper) 157 | .build(); 158 | 159 | } 160 | 161 | @Bean 162 | @LeaderChunkStep 163 | TaskletStep yearReportStep() { 164 | return new StepBuilder("yearReportStep", this.repository) 165 | .chunk(1000, this.transactionManager) 166 | .reader(yearPlatformSalesItemReader()) 167 | .processor(this.objectMapper::writeValueAsString) 168 | .writer(this.itemWriter) 169 | .build(); 170 | } 171 | 172 | @Bean 173 | IntegrationFlow replyFlow( 174 | @LeaderInboundChunkChannel PollableChannel replies, 175 | ConnectionFactory connectionFactory) { 176 | return IntegrationFlow 177 | .from(Amqp.inboundAdapter(connectionFactory, "replies")) 178 | .channel(replies) 179 | .get(); 180 | } 181 | 182 | @Bean 183 | IntegrationFlow outboundFlow(@LeaderOutboundChunkChannel MessageChannel requests, 184 | AmqpTemplate amqpTemplate) { 185 | return IntegrationFlow // 186 | .from(requests) 187 | .handle(Amqp.outboundAdapter(amqpTemplate).routingKey("requests")) 188 | .get(); 189 | } 190 | } 191 | 192 | /** 193 | * installs all the infrastructure for RabbitMQ 194 | */ 195 | @Configuration 196 | class RabbitConfiguration { 197 | 198 | @Bean 199 | org.springframework.amqp.core.Queue requestQueue() { 200 | return new org.springframework.amqp.core.Queue("requests", false); 201 | } 202 | 203 | @Bean 204 | org.springframework.amqp.core.Queue repliesQueue() { 205 | return new Queue("replies", false); 206 | } 207 | 208 | @Bean 209 | TopicExchange exchange() { 210 | return new TopicExchange("remote-chunking-exchange"); 211 | } 212 | 213 | @Bean 214 | Binding repliesBinding(TopicExchange exchange) { 215 | return BindingBuilder.bind(repliesQueue()).to(exchange).with("replies"); 216 | } 217 | 218 | @Bean 219 | Binding requestBinding(TopicExchange exchange) { 220 | return BindingBuilder.bind(requestQueue()).to(exchange).with("requests"); 221 | } 222 | 223 | 224 | } 225 | 226 | 227 | @Configuration 228 | class EndStepConfiguration { 229 | 230 | private final JobRepository repository; 231 | private final PlatformTransactionManager tx; 232 | 233 | EndStepConfiguration(JobRepository repository, PlatformTransactionManager tx) { 234 | this.repository = repository; 235 | this.tx = tx; 236 | } 237 | 238 | @Bean 239 | Step endStep() { 240 | return new StepBuilder("end", repository) 241 | .tasklet((contribution, chunkContext) -> { 242 | System.out.println("the job is finished"); 243 | return RepeatStatus.FINISHED; 244 | }, tx) 245 | .build(); 246 | } 247 | 248 | 249 | } 250 | 251 | @Configuration 252 | class ErrorStepConfiguration { 253 | 254 | private final JobRepository repository; 255 | private final PlatformTransactionManager tx; 256 | 257 | ErrorStepConfiguration(JobRepository repository, PlatformTransactionManager tx) { 258 | this.repository = repository; 259 | this.tx = tx; 260 | } 261 | 262 | @Bean 263 | Step errorStep() { 264 | return new StepBuilder("errorStep", repository) 265 | .tasklet((contribution, chunkContext) -> { 266 | System.out.println("oops!"); 267 | return RepeatStatus.FINISHED; 268 | }, tx) 269 | .build(); 270 | } 271 | } 272 | 273 | @Configuration 274 | class CsvToDbStepConfiguration { 275 | 276 | private final DataSource dataSource; 277 | private final Resource resource; 278 | private final JobRepository repository; 279 | private final PlatformTransactionManager tx; 280 | private final JdbcTemplate jdbc; 281 | 282 | CsvToDbStepConfiguration( 283 | @Value("${csvFile}") Resource resource, 284 | DataSource dataSource, JobRepository repository, 285 | PlatformTransactionManager txm, JdbcTemplate template) { 286 | this.dataSource = dataSource; 287 | this.repository = repository; 288 | this.resource = resource; 289 | this.tx = txm; 290 | this.jdbc = template; 291 | } 292 | 293 | @Bean 294 | @StepScope 295 | FlatFileItemReader gameByYearReader() { 296 | return new FlatFileItemReaderBuilder()// 297 | .resource(resource)// 298 | .name("gameByYearReader")// 299 | .delimited().delimiter(",")// 300 | .names("rank,name,platform,year,genre,publisher,na,eu,jp,other,global".split(",")) // 301 | .linesToSkip(1)// 302 | .fieldSetMapper(fieldSet -> new GameByYear(// 303 | fieldSet.readInt("rank"),// 304 | fieldSet.readString("name"),// 305 | fieldSet.readString("platform"), 306 | parseInt(fieldSet.readString("year")),// 307 | fieldSet.readString("genre"),// 308 | fieldSet.readString("publisher"),// 309 | fieldSet.readFloat("na"),// 310 | fieldSet.readFloat("eu"),// 311 | fieldSet.readFloat("jp"),// 312 | fieldSet.readFloat("other"),// 313 | fieldSet.readFloat("global") 314 | ))// 315 | .build(); 316 | 317 | } 318 | 319 | @Bean 320 | JdbcBatchItemWriter gameByYearWriter() { 321 | var sql = """ 322 | insert into video_game_sales( 323 | rank , 324 | name , 325 | platform , 326 | year , 327 | genre , 328 | publisher , 329 | na_sales , 330 | eu_sales , 331 | jp_sales , 332 | other_sales , 333 | global_sales 334 | ) 335 | values ( 336 | :rank, 337 | :name, 338 | :platform, 339 | :year, 340 | :genre, 341 | :publisher, 342 | :na_sales, 343 | :eu_sales, 344 | :jp_sales, 345 | :other_sales, 346 | :global_sales 347 | ) 348 | ON CONFLICT ON CONSTRAINT video_game_sales_name_platform_year_genre_key 349 | DO UPDATE 350 | SET 351 | rank=excluded.rank , 352 | na_sales=excluded.na_sales , 353 | eu_sales=excluded.eu_sales , 354 | jp_sales=excluded.jp_sales , 355 | other_sales=excluded.other_sales , 356 | global_sales=excluded.global_sales 357 | ; 358 | """; 359 | return new JdbcBatchItemWriterBuilder()// 360 | .sql(sql)// 361 | .dataSource(dataSource)// 362 | .itemSqlParameterSourceProvider(item -> { 363 | var map = new HashMap(); 364 | map.putAll(Map.of( 365 | "rank", item.rank(),// 366 | "name", item.name().trim(),// 367 | "platform", item.platform().trim(),// 368 | "year", item.year(),// 369 | "genre", item.genre().trim(),// 370 | "publisher", item.publisher().trim()// 371 | )); 372 | map.putAll(Map.of( 373 | "na_sales", item.na(),// 374 | "eu_sales", item.eu(),// 375 | "jp_sales", item.jp(),// 376 | "other_sales", item.other(),// 377 | "global_sales", item.global()// 378 | )); 379 | return new MapSqlParameterSource(map); 380 | }) // 381 | .build(); 382 | } 383 | 384 | @Bean 385 | Step gameByYearStep() { 386 | return new StepBuilder("csvToDb", repository)// 387 | .chunk(100, tx)// 388 | .reader(gameByYearReader())// 389 | .writer(gameByYearWriter())// 390 | .listener(new StepExecutionListener() { 391 | @Override 392 | public ExitStatus afterStep(StepExecution stepExecution) { 393 | var count = Objects.requireNonNull( 394 | jdbc.queryForObject("select coalesce(count(*) ,0) from video_game_sales", Integer.class)); 395 | var status = count == 0 ? new ExitStatus(BatchApplication.EMPTY_CSV_STATUS) : ExitStatus.COMPLETED; 396 | System.out.println("the status is " + status); 397 | return status; 398 | } 399 | }) 400 | .build(); 401 | } 402 | 403 | private static int parseInt(String text) { 404 | if (text != null && !text.contains("NA") && !text.contains("N/A")) return Integer.parseInt(text); 405 | return 0; 406 | } 407 | 408 | } 409 | 410 | @Configuration 411 | class YearPlatformReportStepConfiguration { 412 | 413 | private final JobRepository repository; 414 | private final JdbcTemplate jdbc; 415 | private final PlatformTransactionManager transactionManager; 416 | private final TransactionTemplate tx; 417 | 418 | YearPlatformReportStepConfiguration(JobRepository repository, JdbcTemplate jdbc, PlatformTransactionManager transactionManager, TransactionTemplate tx) { 419 | this.repository = repository; 420 | this.jdbc = jdbc; 421 | this.transactionManager = transactionManager; 422 | this.tx = tx; 423 | } 424 | 425 | @Bean 426 | Step yearPlatformReportStep() { 427 | return new StepBuilder("yearPlatformReportStep", repository)// 428 | .tasklet((contribution, chunkContext) ->// 429 | tx.execute(status -> { 430 | jdbc.execute( 431 | """ 432 | insert into year_platform_report (year, platform) 433 | select year, platform from video_game_sales 434 | on conflict on constraint year_platform_report_year_platform_key do nothing; 435 | """); 436 | jdbc.execute(""" 437 | insert into year_platform_report (year, platform, sales) 438 | select yp1.year, 439 | yp1.platform, ( 440 | select sum(vgs.global_sales) from video_game_sales vgs 441 | where vgs.platform = yp1.platform and vgs.year = yp1.year 442 | ) 443 | from year_platform_report as yp1 444 | on conflict on constraint year_platform_report_year_platform_key 445 | do update set 446 | year = excluded.year, 447 | platform = excluded.platform, 448 | sales = excluded.sales; 449 | """); 450 | return RepeatStatus.FINISHED; 451 | }), transactionManager)// 452 | .build(); 453 | } 454 | } -------------------------------------------------------------------------------- /leader/src/main/java/com/example/batch/GameByYear.java: -------------------------------------------------------------------------------- 1 | package com.example.batch; 2 | 3 | public record GameByYear(int rank, String name, String platform, int year, String genre, String publisher, float na, 4 | float eu, float jp, float other, float global) { 5 | } 6 | -------------------------------------------------------------------------------- /leader/src/main/java/com/example/batch/YearPlatformSales.java: -------------------------------------------------------------------------------- 1 | package com.example.batch; 2 | 3 | public record YearPlatformSales(int year, String platform, float sales) { 4 | } 5 | -------------------------------------------------------------------------------- /leader/src/main/java/com/example/batch/YearReport.java: -------------------------------------------------------------------------------- 1 | package com.example.batch; 2 | 3 | import java.util.Collection; 4 | 5 | public record YearReport(int year, Collection breakout) { 6 | } 7 | -------------------------------------------------------------------------------- /leader/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.password=postgres 2 | spring.datasource.username=postgres 3 | spring.datasource.url=jdbc:postgresql://localhost/postgres 4 | spring.batch.jdbc.initialize-schema=always 5 | spring.sql.init.mode=always 6 | spring.rabbitmq.host=localhost 7 | spring.rabbitmq.password=password 8 | spring.rabbitmq.username=user 9 | bootiful.batch.chunk.leader=true -------------------------------------------------------------------------------- /leader/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | drop table if exists video_game_sales; 2 | create table if not exists video_game_sales 3 | ( 4 | rank int, 5 | name text, 6 | platform text, 7 | year int, 8 | genre text, 9 | publisher text, 10 | na_sales numeric(4, 2), 11 | eu_sales numeric(4, 2), 12 | jp_sales numeric(4, 2), 13 | other_sales numeric(4, 2), 14 | global_sales numeric(4, 2), 15 | unique (name, platform, year, genre) 16 | 17 | ); 18 | 19 | drop table if exists year_platform_report; 20 | create table if not exists year_platform_report 21 | ( 22 | year int, 23 | platform text, 24 | sales numeric(8, 2), 25 | unique (year, platform) 26 | ); -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # Spring Batch, up and running 2 | 3 | 4 | ## Notes 5 | 6 | I took this data that we use in this application from [https://www.kaggle.com/datasets/gregorut/videogamesales](this dataset). I have no idea if it works or not. 7 | 8 | ## Outline 9 | 10 | ## part 1 11 | * motivating spring batch 12 | * a new project on the spring initializr 13 | * `docker-compose.yml` 14 | * setup the db connection properties 15 | * tell batch to initialize its own schema 16 | * introducing the spring batch metadata tables 17 | * building a `Job` with the `JobBuilder` and `StepBuilder` 18 | ** a "hello, world!" `Tasklet` 19 | * see the results in the `job_execution\*` tables in the SQL DB. 20 | * giving our batch job a unique run id 21 | * `JobParameters` 22 | 23 | 24 | ## part 2 25 | * `StepScope` and SpEL (`#{ jobParameters['date'] }`). 26 | * running the job yourself with the `JobLauncher` and disabling the property in `application.properties`. 27 | * introduce `.chunk(N)` 28 | * introduce `ItemReader` and `ItemProcessor` and `ItemWriter` 29 | * using the `ListItemReader` to read some hardcoded data ready manually from our `.csv` 30 | * writing my own `ItemWriter` to dump the data that was read in 31 | * creating a proper `FlatFileItemReader` to read the data in in a paging fashion. 32 | * create a `JdbcBatchItemWriterBuilder` to write the CSV data to the DB with an upsert. A quick digression to appreciate the beauty of ye `ole "upsert" 33 | 34 | ## part 3 35 | 36 | ...was just me faffing about trying to figure out how to analyze the data and get it into a separate table. it was a huge flop. so let's not do that. 37 | 38 | ## part 4 39 | 40 | * introduce step outcomes and conditional flows - do this _if_ that, etc. 41 | * if an error occurs, go to an error handling step, otherwise go to the year and platform report step. 42 | * the year platform step is a `Tasklet` that updates a new table, that serves as sort of a view, with the new data. this data should contain the amount of sales each platform did by year. 43 | * then set the stage for distribution and concurrence. i could do things on the same node. i should maybe show how to use an `Executor` or something? but what i really want to do is introduce remote chunking. 44 | * have a dicusssion around remote chunking vs partitioning 45 | * maybe to make this a little more realistic i could have another microservice that renders an `.svg` drawing a graph of of how much each platform made in a given year? 46 | * in order to understand Spring Batch's remote chunking we need a _very_ quick discussion of Spring Integration and channels. Show a quick AMQP example that sends records from one node to another. Or maybe it just uses Apache Kafka? 47 | * setup the final step, to read the data from the year platform report table with a reader, convert it to JSON with a processor, and then send it using a chunk itemwriter out over a `MessageChannel`. 48 | * setup `MessagingTemplate`, `ChunkMessageChannelItemWriter`, the two outbound and inbound `IntegrationFlow`s, etc. 49 | * setup a new module, worker node, that has the spring batch integration support and spring integration but does not necessarily have spring batch 50 | * setup the dummy item ItemProcessor and ItemWriters on the worker node. 51 | 52 | ## part 5-8 53 | these are all about hacking on the autoconfiguration and then integrating it into the demo. 54 | 55 | ## part 9 56 | * integrate the graalvm aot hints and profit -------------------------------------------------------------------------------- /worker/build.gradle: -------------------------------------------------------------------------------- 1 | plugins { 2 | id 'java' 3 | id 'org.springframework.boot' version '3.0.2' 4 | id 'io.spring.dependency-management' version '1.1.0' 5 | id 'org.graalvm.buildtools.native' version '0.9.18' 6 | } 7 | 8 | group = 'com.example' 9 | version = '0.0.1-SNAPSHOT' 10 | sourceCompatibility = '17' 11 | 12 | repositories { 13 | mavenCentral() 14 | mavenLocal() 15 | } 16 | 17 | dependencies { 18 | implementation 'com.joshlong:batch-remotechunking-spring-boot-starter:1.2' 19 | implementation 'org.springframework.batch:spring-batch-integration' 20 | implementation 'org.springframework.integration:spring-integration-amqp' 21 | implementation 'org.springframework.boot:spring-boot-starter-integration' 22 | implementation 'org.springframework.boot:spring-boot-starter-batch' 23 | implementation 'org.springframework.boot:spring-boot-starter-amqp' 24 | implementation 'org.springframework.boot:spring-boot-starter-json' 25 | implementation 'org.springframework.boot:spring-boot-starter-jdbc' 26 | implementation 'org.postgresql:postgresql' 27 | testImplementation 'org.springframework.boot:spring-boot-starter-test' 28 | testImplementation 'org.springframework.batch:spring-batch-test' 29 | } 30 | 31 | tasks.named('test') { 32 | useJUnitPlatform() 33 | } 34 | -------------------------------------------------------------------------------- /worker/data/empty_vgsales.csv: -------------------------------------------------------------------------------- 1 | Rank,Name,Platform,Year,Genre,Publisher,NA_Sales,EU_Sales,JP_Sales,Other_Sales,Global_Sales -------------------------------------------------------------------------------- /worker/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/coffee-software-show/lets-code-spring-batch/987fb894669f3b074590234883c14b98703cf71e/worker/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /worker/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | distributionBase=GRADLE_USER_HOME 2 | distributionPath=wrapper/dists 3 | distributionUrl=https\://services.gradle.org/distributions/gradle-7.6-bin.zip 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | -------------------------------------------------------------------------------- /worker/gradlew: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # 4 | # Copyright © 2015-2021 the original authors. 5 | # 6 | # Licensed under the Apache License, Version 2.0 (the "License"); 7 | # you may not use this file except in compliance with the License. 8 | # You may obtain a copy of the License at 9 | # 10 | # https://www.apache.org/licenses/LICENSE-2.0 11 | # 12 | # Unless required by applicable law or agreed to in writing, software 13 | # distributed under the License is distributed on an "AS IS" BASIS, 14 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | # See the License for the specific language governing permissions and 16 | # limitations under the License. 17 | # 18 | 19 | ############################################################################## 20 | # 21 | # Gradle start up script for POSIX generated by Gradle. 22 | # 23 | # Important for running: 24 | # 25 | # (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is 26 | # noncompliant, but you have some other compliant shell such as ksh or 27 | # bash, then to run this script, type that shell name before the whole 28 | # command line, like: 29 | # 30 | # ksh Gradle 31 | # 32 | # Busybox and similar reduced shells will NOT work, because this script 33 | # requires all of these POSIX shell features: 34 | # * functions; 35 | # * expansions «$var», «${var}», «${var:-default}», «${var+SET}», 36 | # «${var#prefix}», «${var%suffix}», and «$( cmd )»; 37 | # * compound commands having a testable exit status, especially «case»; 38 | # * various built-in commands including «command», «set», and «ulimit». 39 | # 40 | # Important for patching: 41 | # 42 | # (2) This script targets any POSIX shell, so it avoids extensions provided 43 | # by Bash, Ksh, etc; in particular arrays are avoided. 44 | # 45 | # The "traditional" practice of packing multiple parameters into a 46 | # space-separated string is a well documented source of bugs and security 47 | # problems, so this is (mostly) avoided, by progressively accumulating 48 | # options in "$@", and eventually passing that to Java. 49 | # 50 | # Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS, 51 | # and GRADLE_OPTS) rely on word-splitting, this is performed explicitly; 52 | # see the in-line comments for details. 53 | # 54 | # There are tweaks for specific operating systems such as AIX, CygWin, 55 | # Darwin, MinGW, and NonStop. 56 | # 57 | # (3) This script is generated from the Groovy template 58 | # https://github.com/gradle/gradle/blob/master/subprojects/plugins/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt 59 | # within the Gradle project. 60 | # 61 | # You can find Gradle at https://github.com/gradle/gradle/. 62 | # 63 | ############################################################################## 64 | 65 | # Attempt to set APP_HOME 66 | 67 | # Resolve links: $0 may be a link 68 | app_path=$0 69 | 70 | # Need this for daisy-chained symlinks. 71 | while 72 | APP_HOME=${app_path%"${app_path##*/}"} # leaves a trailing /; empty if no leading path 73 | [ -h "$app_path" ] 74 | do 75 | ls=$( ls -ld "$app_path" ) 76 | link=${ls#*' -> '} 77 | case $link in #( 78 | /*) app_path=$link ;; #( 79 | *) app_path=$APP_HOME$link ;; 80 | esac 81 | done 82 | 83 | APP_HOME=$( cd "${APP_HOME:-./}" && pwd -P ) || exit 84 | 85 | APP_NAME="Gradle" 86 | APP_BASE_NAME=${0##*/} 87 | 88 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 89 | DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' 90 | 91 | # Use the maximum available, or set MAX_FD != -1 to use that value. 92 | MAX_FD=maximum 93 | 94 | warn () { 95 | echo "$*" 96 | } >&2 97 | 98 | die () { 99 | echo 100 | echo "$*" 101 | echo 102 | exit 1 103 | } >&2 104 | 105 | # OS specific support (must be 'true' or 'false'). 106 | cygwin=false 107 | msys=false 108 | darwin=false 109 | nonstop=false 110 | case "$( uname )" in #( 111 | CYGWIN* ) cygwin=true ;; #( 112 | Darwin* ) darwin=true ;; #( 113 | MSYS* | MINGW* ) msys=true ;; #( 114 | NONSTOP* ) nonstop=true ;; 115 | esac 116 | 117 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 118 | 119 | 120 | # Determine the Java command to use to start the JVM. 121 | if [ -n "$JAVA_HOME" ] ; then 122 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 123 | # IBM's JDK on AIX uses strange locations for the executables 124 | JAVACMD=$JAVA_HOME/jre/sh/java 125 | else 126 | JAVACMD=$JAVA_HOME/bin/java 127 | fi 128 | if [ ! -x "$JAVACMD" ] ; then 129 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 130 | 131 | Please set the JAVA_HOME variable in your environment to match the 132 | location of your Java installation." 133 | fi 134 | else 135 | JAVACMD=java 136 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 137 | 138 | Please set the JAVA_HOME variable in your environment to match the 139 | location of your Java installation." 140 | fi 141 | 142 | # Increase the maximum file descriptors if we can. 143 | if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then 144 | case $MAX_FD in #( 145 | max*) 146 | MAX_FD=$( ulimit -H -n ) || 147 | warn "Could not query maximum file descriptor limit" 148 | esac 149 | case $MAX_FD in #( 150 | '' | soft) :;; #( 151 | *) 152 | ulimit -n "$MAX_FD" || 153 | warn "Could not set maximum file descriptor limit to $MAX_FD" 154 | esac 155 | fi 156 | 157 | # Collect all arguments for the java command, stacking in reverse order: 158 | # * args from the command line 159 | # * the main class name 160 | # * -classpath 161 | # * -D...appname settings 162 | # * --module-path (only if needed) 163 | # * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables. 164 | 165 | # For Cygwin or MSYS, switch paths to Windows format before running java 166 | if "$cygwin" || "$msys" ; then 167 | APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) 168 | CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) 169 | 170 | JAVACMD=$( cygpath --unix "$JAVACMD" ) 171 | 172 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 173 | for arg do 174 | if 175 | case $arg in #( 176 | -*) false ;; # don't mess with options #( 177 | /?*) t=${arg#/} t=/${t%%/*} # looks like a POSIX filepath 178 | [ -e "$t" ] ;; #( 179 | *) false ;; 180 | esac 181 | then 182 | arg=$( cygpath --path --ignore --mixed "$arg" ) 183 | fi 184 | # Roll the args list around exactly as many times as the number of 185 | # args, so each arg winds up back in the position where it started, but 186 | # possibly modified. 187 | # 188 | # NB: a `for` loop captures its iteration list before it begins, so 189 | # changing the positional parameters here affects neither the number of 190 | # iterations, nor the values presented in `arg`. 191 | shift # remove old arg 192 | set -- "$@" "$arg" # push replacement arg 193 | done 194 | fi 195 | 196 | # Collect all arguments for the java command; 197 | # * $DEFAULT_JVM_OPTS, $JAVA_OPTS, and $GRADLE_OPTS can contain fragments of 198 | # shell script including quotes and variable substitutions, so put them in 199 | # double quotes to make sure that they get re-expanded; and 200 | # * put everything else in single quotes, so that it's not re-expanded. 201 | 202 | set -- \ 203 | "-Dorg.gradle.appname=$APP_BASE_NAME" \ 204 | -classpath "$CLASSPATH" \ 205 | org.gradle.wrapper.GradleWrapperMain \ 206 | "$@" 207 | 208 | # Stop when "xargs" is not available. 209 | if ! command -v xargs >/dev/null 2>&1 210 | then 211 | die "xargs is not available" 212 | fi 213 | 214 | # Use "xargs" to parse quoted args. 215 | # 216 | # With -n1 it outputs one arg per line, with the quotes and backslashes removed. 217 | # 218 | # In Bash we could simply go: 219 | # 220 | # readarray ARGS < <( xargs -n1 <<<"$var" ) && 221 | # set -- "${ARGS[@]}" "$@" 222 | # 223 | # but POSIX shell has neither arrays nor command substitution, so instead we 224 | # post-process each arg (as a line of input to sed) to backslash-escape any 225 | # character that might be a shell metacharacter, then use eval to reverse 226 | # that process (while maintaining the separation between arguments), and wrap 227 | # the whole thing up as a single "set" statement. 228 | # 229 | # This will of course break if any of these variables contains a newline or 230 | # an unmatched quote. 231 | # 232 | 233 | eval "set -- $( 234 | printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" | 235 | xargs -n1 | 236 | sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' | 237 | tr '\n' ' ' 238 | )" '"$@"' 239 | 240 | exec "$JAVACMD" "$@" 241 | -------------------------------------------------------------------------------- /worker/gradlew.bat: -------------------------------------------------------------------------------- 1 | @rem 2 | @rem Copyright 2015 the original author or authors. 3 | @rem 4 | @rem Licensed under the Apache License, Version 2.0 (the "License"); 5 | @rem you may not use this file except in compliance with the License. 6 | @rem You may obtain a copy of the License at 7 | @rem 8 | @rem https://www.apache.org/licenses/LICENSE-2.0 9 | @rem 10 | @rem Unless required by applicable law or agreed to in writing, software 11 | @rem distributed under the License is distributed on an "AS IS" BASIS, 12 | @rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | @rem See the License for the specific language governing permissions and 14 | @rem limitations under the License. 15 | @rem 16 | 17 | @if "%DEBUG%"=="" @echo off 18 | @rem ########################################################################## 19 | @rem 20 | @rem Gradle startup script for Windows 21 | @rem 22 | @rem ########################################################################## 23 | 24 | @rem Set local scope for the variables with windows NT shell 25 | if "%OS%"=="Windows_NT" setlocal 26 | 27 | set DIRNAME=%~dp0 28 | if "%DIRNAME%"=="" set DIRNAME=. 29 | set APP_BASE_NAME=%~n0 30 | set APP_HOME=%DIRNAME% 31 | 32 | @rem Resolve any "." and ".." in APP_HOME to make it shorter. 33 | for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi 34 | 35 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 36 | set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m" 37 | 38 | @rem Find java.exe 39 | if defined JAVA_HOME goto findJavaFromJavaHome 40 | 41 | set JAVA_EXE=java.exe 42 | %JAVA_EXE% -version >NUL 2>&1 43 | if %ERRORLEVEL% equ 0 goto execute 44 | 45 | echo. 46 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 47 | echo. 48 | echo Please set the JAVA_HOME variable in your environment to match the 49 | echo location of your Java installation. 50 | 51 | goto fail 52 | 53 | :findJavaFromJavaHome 54 | set JAVA_HOME=%JAVA_HOME:"=% 55 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 56 | 57 | if exist "%JAVA_EXE%" goto execute 58 | 59 | echo. 60 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 61 | echo. 62 | echo Please set the JAVA_HOME variable in your environment to match the 63 | echo location of your Java installation. 64 | 65 | goto fail 66 | 67 | :execute 68 | @rem Setup the command line 69 | 70 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 71 | 72 | 73 | @rem Execute Gradle 74 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %* 75 | 76 | :end 77 | @rem End local scope for the variables with windows NT shell 78 | if %ERRORLEVEL% equ 0 goto mainEnd 79 | 80 | :fail 81 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 82 | rem the _cmd.exe /c_ return code! 83 | set EXIT_CODE=%ERRORLEVEL% 84 | if %EXIT_CODE% equ 0 set EXIT_CODE=1 85 | if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% 86 | exit /b %EXIT_CODE% 87 | 88 | :mainEnd 89 | if "%OS%"=="Windows_NT" endlocal 90 | 91 | :omega 92 | -------------------------------------------------------------------------------- /worker/native.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | ./gradlew clean 3 | ./gradlew nativeCompile 4 | ./build/native/nativeCompile/worker 5 | -------------------------------------------------------------------------------- /worker/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'worker' 2 | -------------------------------------------------------------------------------- /worker/src/main/java/com/example/batch/BatchApplication.java: -------------------------------------------------------------------------------- 1 | package com.example.batch; 2 | 3 | import com.fasterxml.jackson.core.JsonProcessingException; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import com.joshlong.batch.remotechunking.worker.WorkerInboundChunkChannel; 6 | import com.joshlong.batch.remotechunking.worker.WorkerItemProcessor; 7 | import com.joshlong.batch.remotechunking.worker.WorkerItemWriter; 8 | import com.joshlong.batch.remotechunking.worker.WorkerOutboundChunkChannel; 9 | import org.springframework.amqp.core.AmqpTemplate; 10 | import org.springframework.amqp.rabbit.connection.ConnectionFactory; 11 | import org.springframework.aop.SpringProxy; 12 | import org.springframework.aop.framework.Advised; 13 | import org.springframework.aot.hint.MemberCategory; 14 | import org.springframework.aot.hint.RuntimeHints; 15 | import org.springframework.aot.hint.RuntimeHintsRegistrar; 16 | import org.springframework.batch.core.launch.JobOperator; 17 | import org.springframework.batch.item.ItemProcessor; 18 | import org.springframework.batch.item.ItemWriter; 19 | import org.springframework.boot.SpringApplication; 20 | import org.springframework.boot.autoconfigure.SpringBootApplication; 21 | import org.springframework.context.annotation.Bean; 22 | import org.springframework.context.annotation.Configuration; 23 | import org.springframework.context.annotation.ImportRuntimeHints; 24 | import org.springframework.core.DecoratingProxy; 25 | import org.springframework.integration.amqp.dsl.Amqp; 26 | import org.springframework.integration.channel.DirectChannel; 27 | import org.springframework.integration.dsl.IntegrationFlow; 28 | import org.springframework.messaging.MessageChannel; 29 | 30 | import java.util.Collection; 31 | import java.util.Set; 32 | 33 | @SpringBootApplication 34 | @ImportRuntimeHints(BatchApplication.Hints.class) 35 | public class BatchApplication { 36 | 37 | 38 | static class Hints implements RuntimeHintsRegistrar { 39 | 40 | @Override 41 | public void registerHints(RuntimeHints hints, ClassLoader classLoader) { 42 | // todo https://github.com/spring-projects/spring-batch/issues/4294 43 | hints.proxies().registerJdkProxy(JobOperator.class, SpringProxy.class, Advised.class, DecoratingProxy.class); 44 | Set.of(YearReport.class, YearPlatformSales.class) 45 | .forEach(clzz -> hints.reflection().registerType(clzz, MemberCategory.values())); 46 | 47 | } 48 | } 49 | 50 | public static void main(String[] args) { 51 | SpringApplication.run(BatchApplication.class, args); 52 | } 53 | } 54 | 55 | 56 | @Configuration 57 | class WorkerConfiguration { 58 | 59 | private final ObjectMapper objectMapper; 60 | 61 | WorkerConfiguration(ObjectMapper objectMapper) { 62 | this.objectMapper = objectMapper; 63 | } 64 | 65 | @Bean 66 | IntegrationFlow inbound( 67 | @WorkerInboundChunkChannel DirectChannel requests, 68 | ConnectionFactory connectionFactory) { 69 | return IntegrationFlow 70 | .from(Amqp.inboundAdapter(connectionFactory, "requests")) 71 | .channel(requests) 72 | .get(); 73 | } 74 | 75 | @Bean 76 | IntegrationFlow outboundReplies(@WorkerOutboundChunkChannel MessageChannel replies, 77 | AmqpTemplate template) { 78 | return IntegrationFlow // 79 | .from(replies) 80 | .handle(Amqp.outboundAdapter(template).routingKey("replies")) 81 | .get(); 82 | } 83 | 84 | private YearReport deserializeYearReportJson(String json) { 85 | try { 86 | return objectMapper.readValue(json, YearReport.class); 87 | } catch (JsonProcessingException e) { 88 | throw new IllegalArgumentException("oops! couldn't parse the JSON!", e); 89 | } 90 | } 91 | 92 | private static void doSomethingTimeIntensive(YearReport yearReport) { 93 | System.out.println("===================="); 94 | System.out.println("got yearReport"); 95 | System.out.println(yearReport.toString()); 96 | } 97 | 98 | @Bean 99 | @WorkerItemProcessor 100 | ItemProcessor itemProcessor() { 101 | return yearReportJson -> { 102 | System.out.println(">> processing YearReport JSON: " + yearReportJson); 103 | Thread.sleep(5); 104 | return deserializeYearReportJson(yearReportJson); 105 | }; 106 | } 107 | 108 | @Bean 109 | @WorkerItemWriter 110 | ItemWriter writer() { 111 | return chunk -> chunk.getItems().forEach(WorkerConfiguration::doSomethingTimeIntensive); 112 | } 113 | } 114 | 115 | record YearPlatformSales(int year, String platform, float sales) { 116 | } 117 | 118 | record YearReport(int year, Collection breakout) { 119 | } 120 | -------------------------------------------------------------------------------- /worker/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.password=postgres 2 | spring.datasource.username=postgres 3 | spring.datasource.url=jdbc:postgresql://localhost/postgres 4 | spring.batch.jdbc.initialize-schema=always 5 | spring.sql.init.mode=always 6 | spring.rabbitmq.host=localhost 7 | spring.rabbitmq.password=password 8 | spring.rabbitmq.username=user 9 | bootiful.batch.chunk.worker=true -------------------------------------------------------------------------------- /worker/src/test/java/com/example/batch/BatchApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example.batch; 2 | 3 | import org.junit.jupiter.api.Test; 4 | import org.springframework.boot.test.context.SpringBootTest; 5 | 6 | @SpringBootTest 7 | class BatchApplicationTests { 8 | 9 | @Test 10 | void contextLoads() { 11 | } 12 | 13 | } 14 | --------------------------------------------------------------------------------