├── .gitignore ├── .gitmodules ├── .travis.yml ├── activiti-integration ├── manifest-leader.yml ├── manifest-worker.yml ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ ├── DemoApplication.java │ │ │ ├── LeaderChannels.java │ │ │ ├── LeaderConfiguration.java │ │ │ ├── ProcessRestController.java │ │ │ ├── Profiles.java │ │ │ ├── WorkerChannels.java │ │ │ └── WorkerConfiguration.java │ └── resources │ │ ├── application-default.properties │ │ ├── application-leader.properties │ │ ├── application-worker.properties │ │ ├── application.properties │ │ └── processes │ │ └── async.bpmn20.xml │ └── test │ └── java │ └── com │ └── example │ └── DemoApplicationTests.java ├── activiti ├── pom.xml └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── example │ │ │ ├── CheckForm.java │ │ │ ├── Customer.java │ │ │ ├── CustomerRepository.java │ │ │ ├── DemoApplication.java │ │ │ ├── SendConfirmationEmail.java │ │ │ ├── SignupRestController.java │ │ │ └── email │ │ │ ├── EmailValidationService.java │ │ │ ├── InvalidEmailException.java │ │ │ ├── MashapeEmailValidationService.java │ │ │ └── SimpleEmailValidationService.java │ └── resources │ │ ├── application.properties │ │ └── processes │ │ └── signup.bpmn20.xml │ └── test │ └── java │ └── com │ └── example │ ├── EmailValidationServiceTest.java │ └── SignupRestControllerTests.java ├── batch-and-integration ├── pom.xml └── src │ └── main │ ├── java │ ├── eda │ │ ├── IntegrationApplication.java │ │ └── IntegrationConfiguration.java │ └── edabatch │ │ ├── BatchChannels.java │ │ ├── BatchConfiguration.java │ │ ├── BatchIntegrationApplication.java │ │ ├── Contact.java │ │ ├── EtlFlowConfiguration.java │ │ ├── FinishedFileFlowConfiguration.java │ │ ├── InvalidFileFlowConfiguration.java │ │ ├── JobExecutionListener.java │ │ ├── Utils.java │ │ └── email │ │ ├── EmailValidationService.java │ │ ├── InvalidEmailException.java │ │ ├── MashapeEmailValidationService.java │ │ └── SimpleEmailValidationService.java │ └── resources │ ├── application.properties │ ├── data.csv │ └── schema.sql ├── batch ├── in.csv ├── pom.xml └── src │ └── main │ ├── java │ └── processing │ │ ├── BatchApplication.java │ │ ├── BatchConfiguration.java │ │ ├── Person.java │ │ ├── Step1Configuration.java │ │ ├── Step2Configuration.java │ │ ├── Step3Configuration.java │ │ └── email │ │ ├── EmailValidationService.java │ │ ├── InvalidEmailException.java │ │ ├── MashapeEmailValidationService.java │ │ └── SimpleEmailValidationService.java │ └── resources │ ├── application.properties │ └── schema.sql ├── dataflow ├── pom.xml ├── server-definitions │ ├── manifest.yml │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── demo │ │ │ └── DataFlowDefinitionsApplication.java │ │ └── resources │ │ └── static │ │ └── dataflow-example-apps.properties ├── server │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── dataflow │ │ │ │ ├── DataFlowInitializer.java │ │ │ │ └── DataFlowServer.java │ │ └── resources │ │ │ ├── application.properties │ │ │ └── static │ │ │ ├── app.properties │ │ │ └── maven-rabbitmq-1.0.1.properties │ │ └── test │ │ └── java │ │ └── com │ │ └── example │ │ └── DemoApplicationTests.java ├── shell │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── com │ │ │ └── example │ │ │ └── ShellApplication.java │ │ └── resources │ │ └── application.properties ├── stream-example │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── stream │ │ │ │ └── ProcessorStreamExample.java │ │ └── resources │ │ │ └── application.properties │ │ └── test │ │ └── java │ │ └── com │ │ └── example │ │ └── Stream101ApplicationTests.java └── task-example │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── task │ │ ├── BatchTaskExample.java │ │ └── BatchTaskProperties.java │ └── resources │ └── application.properties ├── integration-it ├── pom.xml └── src │ ├── integration-test │ ├── java │ │ └── integration │ │ │ ├── ActivitiIT.java │ │ │ ├── DataFlowIT.java │ │ │ ├── RemotePartitioningIT.java │ │ │ └── ShakyServiceIT.java │ └── resources │ │ └── application.properties │ ├── main │ └── java │ │ └── demo │ │ └── Main.java │ └── test │ └── java │ └── demo │ └── RemotePartitionIntegrationTest.java ├── pom.xml ├── remote-partitioning ├── ddl │ ├── data-init.sql │ └── schema-init.sql ├── manifest-leader.yml ├── manifest-worker.yml ├── pom.xml └── src │ └── main │ ├── java │ └── partition │ │ ├── IdRangePartitioner.java │ │ ├── JobConfiguration.java │ │ ├── LeaderChannels.java │ │ ├── LeaderStepConfiguration.java │ │ ├── PartitionApplication.java │ │ ├── PartitionRestController.java │ │ ├── Person.java │ │ ├── Profiles.java │ │ ├── WorkerChannels.java │ │ ├── WorkerConfiguration.java │ │ └── WorkerStepConfiguration.java │ └── resources │ ├── application-worker.properties │ ├── application.properties │ ├── data.sql │ └── schema.sql ├── shaky-service-to-service-calls ├── pom.xml ├── shaky-client │ ├── pom.xml │ └── src │ │ ├── main │ │ ├── java │ │ │ └── demo │ │ │ │ ├── CircuitBreakerGreetingClient.java │ │ │ │ ├── GreetingClient.java │ │ │ │ ├── GreetingClientApplication.java │ │ │ │ └── RetryableGreetingClient.java │ │ └── resources │ │ │ └── application.properties │ │ └── test │ │ ├── java │ │ └── demo │ │ │ ├── AbstractGreetingClientTest.java │ │ │ ├── CircuitBreakerGreetingClientTest.java │ │ │ └── RetryableGreetingClientTest.java │ │ └── resources │ │ └── application.properties └── shaky-service │ ├── pom.xml │ └── src │ ├── main │ └── java │ │ └── service │ │ ├── GreetingApplication.java │ │ └── GreetingServiceRestController.java │ └── test │ └── java │ └── service │ └── GreetingServiceRestControllerTest.java ├── stream ├── pom.xml ├── stream-consumer │ ├── pom.xml │ └── src │ │ └── main │ │ ├── java │ │ └── stream │ │ │ └── consumer │ │ │ ├── ConsumerChannels.java │ │ │ ├── integration │ │ │ └── StreamConsumer.java │ │ │ └── listeners │ │ │ └── StreamConsumer.java │ │ └── resources │ │ └── application.properties └── stream-producer │ ├── pom.xml │ └── src │ └── main │ ├── java │ └── stream │ │ └── producer │ │ ├── ProducerChannels.java │ │ ├── channels │ │ └── StreamProducer.java │ │ └── gateway │ │ └── StreamProducer.java │ └── resources │ └── application.properties └── task ├── pom.xml └── src └── main └── java └── task └── HelloTask.java /.gitignore: -------------------------------------------------------------------------------- 1 | # 2 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "build"] 2 | path = build 3 | url = git@github.com:cloud-native-java/build.git 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | 3 | #sudo: required 4 | 5 | jdk: 6 | - oraclejdk8 7 | 8 | cache: 9 | directories: 10 | - $HOME/.m2/repository/ 11 | 12 | script: 13 | - mvn -e -P integration-test verify deploy 14 | 15 | 16 | -------------------------------------------------------------------------------- /activiti-integration/manifest-leader.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: activiti-leader 4 | instances: 1 5 | buildpack: https://github.com/cloudfoundry/java-buildpack.git 6 | host: activiti-leader-${random-word} 7 | path: target/activiti-integration.jar 8 | services: 9 | - activiti-mysql 10 | - activiti-rabbitmq 11 | env: 12 | DEBUG: "true" 13 | SERVER_PORT: 80 14 | SPRING_PROFILES_ACTIVE: leader, cloud 15 | -------------------------------------------------------------------------------- /activiti-integration/manifest-worker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: activiti-worker 4 | instances: 2 5 | host: activiti-worker-${random-word} 6 | buildpack: https://github.com/cloudfoundry/java-buildpack.git 7 | path: target/activiti-integration.jar 8 | services: 9 | - activiti-mysql 10 | - activiti-rabbitmq 11 | env: 12 | DEBUG: "true" 13 | SPRING_PROFILES_ACTIVE: worker, cloud 14 | SERVER_PORT: 80 -------------------------------------------------------------------------------- /activiti-integration/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 5.19.0 8 | 9 | 10 | cnj 11 | integration 12 | 1.0.0-SNAPSHOT 13 | 14 | integration/activiti-integration 15 | activiti-integration 16 | 17 | 18 | org.codehaus.groovy 19 | groovy-all 20 | 21 | 25 | 26 | com.h2database 27 | h2 28 | 29 | 30 | 31 | org.activiti 32 | activiti-spring-boot-starter-rest-api 33 | ${activiti-spring-boot.version} 34 | 35 | 36 | 41 | 42 | org.springframework.boot 43 | spring-boot-starter-web 44 | 45 | 46 | org.springframework.boot 47 | spring-boot-starter-integration 48 | 49 | 50 | org.springframework.cloud 51 | spring-cloud-starter-stream-rabbit 52 | 53 | 54 | org.springframework.boot 55 | spring-boot-starter-test 56 | test 57 | 58 | 59 | 60 | 61 | 62 | 63 | org.springframework.boot 64 | spring-boot-maven-plugin 65 | 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/DemoApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.activiti.engine.IdentityService; 4 | import org.activiti.engine.identity.Group; 5 | import org.activiti.engine.identity.User; 6 | import org.springframework.beans.factory.InitializingBean; 7 | import org.springframework.boot.SpringApplication; 8 | import org.springframework.boot.autoconfigure.EnableAutoConfiguration; 9 | import org.springframework.boot.autoconfigure.SpringBootApplication; 10 | import org.springframework.cloud.stream.annotation.EnableBinding; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | import org.springframework.context.annotation.Profile; 14 | 15 | @SpringBootApplication 16 | @EnableAutoConfiguration 17 | public class DemoApplication { 18 | 19 | @Configuration 20 | @Profile(Profiles.LEADER) 21 | @EnableBinding(LeaderChannels.class) 22 | public static class Leader { 23 | 24 | @Bean 25 | InitializingBean init(IdentityService identityService) { 26 | return () -> { 27 | 28 | String usersGroup = "users"; 29 | if (0 == identityService.createGroupQuery().groupId(usersGroup).count()) { 30 | Group group = identityService.newGroup(usersGroup); 31 | group.setName(usersGroup); 32 | group.setType("security-role"); 33 | identityService.saveGroup(group); 34 | } 35 | 36 | String adminUser = "operator"; 37 | if (0 == identityService.createUserQuery().userId(adminUser).count()) { 38 | User admin = identityService.newUser(adminUser); 39 | admin.setPassword(adminUser); 40 | identityService.saveUser(admin); 41 | } 42 | }; 43 | } 44 | } 45 | 46 | @Configuration 47 | @Profile(Profiles.WORKER) 48 | @EnableBinding(WorkerChannels.class) 49 | public static class Worker { 50 | } 51 | 52 | 53 | public static void main(String[] args) { 54 | SpringApplication.run(DemoApplication.class, args); 55 | } 56 | } 57 | 58 | 59 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/LeaderChannels.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.cloud.stream.annotation.Input; 4 | import org.springframework.cloud.stream.annotation.Output; 5 | import org.springframework.messaging.MessageChannel; 6 | 7 | public interface LeaderChannels { 8 | 9 | @Output 10 | MessageChannel leaderRequests(); 11 | 12 | @Input 13 | MessageChannel leaderReplies(); 14 | } 15 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/LeaderConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.activiti.engine.ProcessEngine; 4 | import org.activiti.engine.impl.bpmn.behavior.ReceiveTaskActivityBehavior; 5 | import org.activiti.engine.impl.pvm.delegate.ActivityBehavior; 6 | import org.activiti.engine.impl.pvm.delegate.ActivityExecution; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.context.annotation.Profile; 10 | import org.springframework.integration.dsl.IntegrationFlow; 11 | import org.springframework.integration.dsl.IntegrationFlows; 12 | import org.springframework.integration.support.MessageBuilder; 13 | import org.springframework.messaging.Message; 14 | 15 | @Configuration 16 | @Profile(Profiles.LEADER) 17 | class LeaderConfiguration { 18 | 19 | // <1> 20 | @Bean 21 | ActivityBehavior gateway(LeaderChannels channels) { 22 | return new ReceiveTaskActivityBehavior() { 23 | 24 | @Override 25 | public void execute(ActivityExecution execution) throws Exception { 26 | 27 | Message executionMessage = MessageBuilder 28 | .withPayload(execution.getId()) 29 | .build(); 30 | 31 | channels.leaderRequests().send(executionMessage); 32 | } 33 | }; 34 | } 35 | 36 | // <2> 37 | @Bean 38 | IntegrationFlow repliesFlow(LeaderChannels channels, 39 | ProcessEngine engine) { 40 | return IntegrationFlows 41 | .from(channels.leaderReplies()) 42 | .handle(String.class, (executionId, map) -> { 43 | engine.getRuntimeService().signal(executionId); 44 | return null; 45 | }) 46 | .get(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/ProcessRestController.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.activiti.engine.ProcessEngine; 4 | import org.activiti.engine.runtime.ProcessInstance; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.context.annotation.Profile; 7 | import org.springframework.web.bind.annotation.GetMapping; 8 | import org.springframework.web.bind.annotation.RestController; 9 | 10 | import java.util.Collections; 11 | import java.util.Map; 12 | 13 | @Profile(Profiles.LEADER) 14 | @RestController 15 | class ProcessRestController { 16 | 17 | private final ProcessEngine processEngine; 18 | 19 | @Autowired 20 | ProcessRestController(ProcessEngine processEngine) { 21 | this.processEngine = processEngine; 22 | } 23 | 24 | @GetMapping("/start") 25 | Map launch() { 26 | ProcessInstance pi = this.processEngine.getRuntimeService() 27 | .startProcessInstanceByKey("asyncProcess"); 28 | return Collections.singletonMap("processInstanceId", pi .getProcessInstanceId()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/Profiles.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | 4 | public class Profiles { 5 | public static final String LEADER = "leader"; 6 | public static final String WORKER = "worker"; 7 | } 8 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/WorkerChannels.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | 4 | import org.springframework.cloud.stream.annotation.Input; 5 | import org.springframework.cloud.stream.annotation.Output; 6 | import org.springframework.messaging.MessageChannel; 7 | 8 | public interface WorkerChannels { 9 | 10 | @Input 11 | MessageChannel workerRequests(); 12 | 13 | @Output 14 | MessageChannel workerReplies(); 15 | } 16 | -------------------------------------------------------------------------------- /activiti-integration/src/main/java/com/example/WorkerConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.context.annotation.Profile; 8 | import org.springframework.integration.dsl.IntegrationFlow; 9 | import org.springframework.integration.dsl.IntegrationFlows; 10 | import org.springframework.integration.dsl.support.GenericHandler; 11 | 12 | 13 | @Configuration 14 | @Profile(Profiles.WORKER) 15 | class WorkerConfiguration { 16 | 17 | @Bean 18 | IntegrationFlow requestsFlow(WorkerChannels channels) { 19 | 20 | Log log = LogFactory.getLog(getClass()); 21 | 22 | // <1> 23 | return IntegrationFlows 24 | .from(channels.workerRequests()) 25 | .handle((GenericHandler) (executionId, headers) -> { 26 | // <2> 27 | headers 28 | .entrySet() 29 | .forEach(e -> log.info(e.getKey() + '=' + e.getValue())); 30 | log.info("sending executionId (" + executionId + ") to workerReplies."); 31 | return executionId; 32 | }) 33 | .channel(channels.workerReplies()) // <3> 34 | .get(); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /activiti-integration/src/main/resources/application-default.properties: -------------------------------------------------------------------------------- 1 | ## MySQL DB 2 | # 3 | #spring.datasource.url=jdbc:mysql://localhost:3306/batch?useSSL=false 4 | #spring.datasource.password=batch 5 | #spring.datasource.username=batch 6 | -------------------------------------------------------------------------------- /activiti-integration/src/main/resources/application-leader.properties: -------------------------------------------------------------------------------- 1 | server.port=8080 2 | 3 | # leader node channels 4 | spring.cloud.stream.bindings.leaderRequests.destination=${as}-requests 5 | spring.cloud.stream.bindings.leaderReplies.destination=${as}-replies 6 | spring.cloud.stream.bindings.leaderReplies.group=${as}-replies 7 | spring.cloud.stream.bindings.leaderReplies.durableSubscription=true 8 | -------------------------------------------------------------------------------- /activiti-integration/src/main/resources/application-worker.properties: -------------------------------------------------------------------------------- 1 | server.port=0 2 | 3 | # worker node channels 4 | spring.cloud.stream.bindings.workerReplies.destination=${as}-replies 5 | spring.cloud.stream.bindings.workerRequests.destination=${as}-requests 6 | spring.cloud.stream.bindings.workerRequests.group=${as}-requests 7 | spring.cloud.stream.bindings.workerRequests.durableSubscription=true 8 | -------------------------------------------------------------------------------- /activiti-integration/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | # channel definitions 3 | as=activiti-signup 4 | 5 | spring.jpa.hibernate.ddl-auto=update 6 | -------------------------------------------------------------------------------- /activiti-integration/src/main/resources/processes/async.bpmn20.xml: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /activiti-integration/src/test/java/com/example/DemoApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.SpringApplicationConfiguration; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | @RunWith(SpringJUnit4ClassRunner.class) 9 | @SpringApplicationConfiguration(classes = DemoApplication.class) 10 | public class DemoApplicationTests { 11 | 12 | @Test 13 | public void contextLoads() { 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /activiti/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | 5.19.0 8 | 9 | 10 | cnj 11 | integration 12 | 1.0.0-SNAPSHOT 13 | 14 | integration/activiti 15 | activiti 16 | 17 | 18 | org.codehaus.groovy 19 | groovy-all 20 | 21 | 22 | com.h2database 23 | h2 24 | 25 | 26 | org.activiti 27 | activiti-spring-boot-starter-basic 28 | ${activiti-spring-boot.version} 29 | 30 | 35 | 36 | 37 | org.springframework.retry 38 | spring-retry 39 | 40 | 41 | 42 | org.springframework.boot 43 | spring-boot-starter-web 44 | 45 | 46 | org.springframework.boot 47 | spring-boot-starter-data-jpa 48 | 49 | 50 | org.springframework.boot 51 | spring-boot-starter-test 52 | test 53 | 54 | 55 | 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-maven-plugin 60 | 61 | 62 | 63 | 64 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/CheckForm.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import com.example.email.EmailValidationService; 4 | import org.activiti.engine.RuntimeService; 5 | import org.activiti.engine.impl.pvm.delegate.ActivityExecution; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.stereotype.Service; 8 | 9 | import java.util.Collections; 10 | import java.util.Map; 11 | 12 | import static org.apache.commons.lang3.StringUtils.isEmpty; 13 | 14 | @Service 15 | class CheckForm { 16 | 17 | private final RuntimeService runtimeService; // <1> 18 | private final CustomerRepository customerRepository; 19 | private final EmailValidationService emailValidationService; 20 | 21 | @Autowired 22 | public CheckForm(EmailValidationService emailValidationService, 23 | RuntimeService runtimeService, CustomerRepository customerRepository) { 24 | this.runtimeService = runtimeService; 25 | this.customerRepository = customerRepository; 26 | this.emailValidationService = emailValidationService; 27 | } 28 | 29 | // <2> 30 | public void execute(ActivityExecution e) throws Exception { 31 | Long customerId = Long.parseLong(e.getVariable("customerId", 32 | String.class)); 33 | Map vars = Collections.singletonMap("formOK", 34 | validated(this.customerRepository.findOne(customerId))); 35 | this.runtimeService.setVariables(e.getId(), vars); // <3> 36 | } 37 | 38 | private boolean validated(Customer customer) { 39 | return !isEmpty(customer.getFirstName()) 40 | && !isEmpty(customer.getLastName()) 41 | && this.emailValidationService 42 | .isEmailValid(customer.getEmail()); 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/Customer.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import javax.persistence.Entity; 4 | import javax.persistence.GeneratedValue; 5 | import javax.persistence.Id; 6 | 7 | @Entity 8 | public class Customer { 9 | 10 | @Id 11 | @GeneratedValue 12 | private Long id; 13 | private String firstName, lastName; 14 | private String email; 15 | 16 | Customer() { 17 | } 18 | 19 | public void setFirstName(String firstName) { 20 | this.firstName = firstName; 21 | } 22 | 23 | public void setLastName(String lastName) { 24 | this.lastName = lastName; 25 | } 26 | 27 | public void setEmail(String email) { 28 | this.email = email; 29 | } 30 | 31 | public Long getId() { 32 | 33 | return id; 34 | } 35 | 36 | public String getFirstName() { 37 | return firstName; 38 | } 39 | 40 | public String getLastName() { 41 | return lastName; 42 | } 43 | 44 | public String getEmail() { 45 | return email; 46 | } 47 | 48 | public Customer(String firstName, String lastName, String email) { 49 | this.firstName = firstName; 50 | this.lastName = lastName; 51 | this.email = email; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/CustomerRepository.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.data.jpa.repository.JpaRepository; 4 | 5 | import java.util.Optional; 6 | 7 | public interface CustomerRepository extends JpaRepository { 8 | 9 | Optional findByEmail(String e); 10 | } 11 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/DemoApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.retry.annotation.EnableRetry; 7 | import org.springframework.web.client.RestTemplate; 8 | 9 | @SpringBootApplication 10 | public class DemoApplication { 11 | 12 | @Bean 13 | RestTemplate restTemplate() { 14 | return new RestTemplate(); 15 | } 16 | 17 | public static void main(String[] args) { 18 | SpringApplication.run(DemoApplication.class, args); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/SendConfirmationEmail.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.activiti.engine.impl.pvm.delegate.ActivityExecution; 4 | import org.apache.commons.logging.Log; 5 | import org.apache.commons.logging.LogFactory; 6 | import org.springframework.stereotype.Service; 7 | 8 | @Service 9 | class SendConfirmationEmail { 10 | 11 | private Log log = LogFactory.getLog(getClass()); 12 | 13 | public void execute(ActivityExecution execution) throws Exception { 14 | this.log.info("in " + getClass().getName() + ", customerId = " 15 | + execution.getVariable("customerId")); 16 | // exercise to reader: send an email, perhaps 17 | // using Sendgrid? 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/SignupRestController.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.activiti.engine.RuntimeService; 4 | import org.activiti.engine.TaskService; 5 | import org.activiti.engine.task.TaskInfo; 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.http.ResponseEntity; 10 | import org.springframework.util.Assert; 11 | import org.springframework.web.bind.annotation.*; 12 | 13 | import java.util.Collections; 14 | import java.util.List; 15 | import java.util.stream.Collectors; 16 | 17 | @RestController 18 | @RequestMapping("/customers") 19 | class SignupRestController { 20 | 21 | public static final String CUSTOMER_ID_PV_KEY = "customerId"; 22 | 23 | private final RuntimeService runtimeService; 24 | private final TaskService taskService; 25 | private final CustomerRepository customerRepository; 26 | private Log log = LogFactory.getLog(getClass()); 27 | 28 | // <1> 29 | @Autowired 30 | public SignupRestController(RuntimeService runtimeService, 31 | TaskService taskService, CustomerRepository repository) { 32 | this.runtimeService = runtimeService; 33 | this.taskService = taskService; 34 | this.customerRepository = repository; 35 | } 36 | 37 | // <2> 38 | @PostMapping 39 | public ResponseEntity startProcess(@RequestBody Customer customer) { 40 | Assert.notNull(customer); 41 | Customer save = this.customerRepository.save(new Customer(customer 42 | .getFirstName(), customer.getLastName(), customer.getEmail())); 43 | 44 | String processInstanceId = this.runtimeService 45 | .startProcessInstanceByKey( 46 | "signup", 47 | Collections.singletonMap(CUSTOMER_ID_PV_KEY, 48 | Long.toString(save.getId()))).getId(); 49 | this.log.info("started sign-up. the processInstance ID is " 50 | + processInstanceId); 51 | 52 | return ResponseEntity.ok(save.getId()); 53 | } 54 | 55 | // <3> 56 | @GetMapping("/{customerId}/signup/errors") 57 | public List readErrors(@PathVariable String customerId) { 58 | // @formatter:off 59 | return this.taskService 60 | .createTaskQuery() 61 | .active() 62 | .taskName("fix-errors") 63 | .includeProcessVariables() 64 | .processVariableValueEquals(CUSTOMER_ID_PV_KEY, customerId) 65 | .list() 66 | .stream() 67 | .map(TaskInfo::getId) 68 | .collect(Collectors.toList()); 69 | // @formatter:on 70 | } 71 | 72 | // <4> 73 | @PostMapping("/{customerId}/signup/errors/{taskId}") 74 | public void fixErrors(@PathVariable String customerId, 75 | @PathVariable String taskId, @RequestBody Customer fixedCustomer) { 76 | 77 | Customer customer = this.customerRepository.findOne(Long 78 | .parseLong(customerId)); 79 | customer.setEmail(fixedCustomer.getEmail()); 80 | customer.setFirstName(fixedCustomer.getFirstName()); 81 | customer.setLastName(fixedCustomer.getLastName()); 82 | this.customerRepository.save(customer); 83 | 84 | this.taskService 85 | .createTaskQuery() 86 | .active() 87 | .taskId(taskId) 88 | .includeProcessVariables() 89 | .processVariableValueEquals(CUSTOMER_ID_PV_KEY, customerId) 90 | .list() 91 | .forEach( 92 | t -> { 93 | log.info("fixing customer# " + customerId 94 | + " for taskId " + taskId); 95 | taskService.complete(t.getId(), 96 | Collections.singletonMap("formOK", true)); 97 | }); 98 | } 99 | 100 | // <5> 101 | @PostMapping("/{customerId}/signup/confirmation") 102 | public void confirm(@PathVariable String customerId) { 103 | this.taskService 104 | .createTaskQuery() 105 | .active() 106 | .taskName("confirm-email") 107 | .includeProcessVariables() 108 | .processVariableValueEquals(CUSTOMER_ID_PV_KEY, customerId) 109 | .list().forEach(t -> { 110 | log.info(t.toString()); 111 | taskService.complete(t.getId()); 112 | }); 113 | this.log.info("confirmed email receipt for " + customerId); 114 | } 115 | } -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/email/EmailValidationService.java: -------------------------------------------------------------------------------- 1 | package com.example.email; 2 | 3 | public interface EmailValidationService { 4 | boolean isEmailValid(String email); 5 | } 6 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/email/InvalidEmailException.java: -------------------------------------------------------------------------------- 1 | package com.example.email; 2 | 3 | public class InvalidEmailException extends Exception { 4 | public InvalidEmailException(String email) { 5 | super(String.format("the email %s isn't valid", email)); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/email/MashapeEmailValidationService.java: -------------------------------------------------------------------------------- 1 | package com.example.email; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.core.ParameterizedTypeReference; 7 | import org.springframework.http.RequestEntity; 8 | import org.springframework.http.ResponseEntity; 9 | import org.springframework.stereotype.Service; 10 | import org.springframework.web.client.RestTemplate; 11 | import org.springframework.web.util.UriComponents; 12 | import org.springframework.web.util.UriComponentsBuilder; 13 | 14 | import java.util.Map; 15 | 16 | @Service 17 | @Profile("production") 18 | class MashapeEmailValidationService implements EmailValidationService { 19 | 20 | private final String mashapeKey; 21 | private final RestTemplate restTemplate; 22 | private final String uri; 23 | 24 | @Autowired 25 | public MashapeEmailValidationService( 26 | @Value("${mashape.key}") String key, 27 | @Value("${email-validator.uri}") String uri, 28 | RestTemplate restTemplate) { 29 | this.mashapeKey = key; 30 | this.uri = uri; 31 | this.restTemplate = restTemplate; 32 | } 33 | 34 | public boolean isEmailValid(String email) { 35 | UriComponents emailValidatedUri = UriComponentsBuilder.fromHttpUrl(uri) 36 | .buildAndExpand(email); 37 | 38 | RequestEntity requestEntity = RequestEntity 39 | .get(emailValidatedUri.toUri()) 40 | .header("X-Mashape-Key", mashapeKey).build(); 41 | 42 | ParameterizedTypeReference> ptr = 43 | new ParameterizedTypeReference>() { }; 44 | 45 | ResponseEntity> responseEntity = restTemplate.exchange(requestEntity, ptr); 46 | 47 | return responseEntity.getBody().get("isValid"); 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /activiti/src/main/java/com/example/email/SimpleEmailValidationService.java: -------------------------------------------------------------------------------- 1 | package com.example.email; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.stereotype.Service; 7 | import org.springframework.util.StringUtils; 8 | 9 | @Service 10 | @Profile("!production") 11 | class SimpleEmailValidationService implements EmailValidationService { 12 | 13 | private Log log = LogFactory.getLog(getClass()); 14 | 15 | @Override 16 | public boolean isEmailValid(String email) { 17 | boolean emailIsValid = StringUtils.hasText(email) && 18 | email.length() > 1 && 19 | email.contains("@"); 20 | log.debug("emailIsValid: " + emailIsValid); 21 | return emailIsValid; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /activiti/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.jpa.hibernate.ddl-auto=update 2 | 3 | emailvalidator.uri=https://pozzad-email-validator.p.mashape.com/emailvalidator/validateEmail/{email} 4 | 5 | logging.level.org.springframework.cloud.netflix.feign.valid = DEBUG 6 | 7 | -------------------------------------------------------------------------------- /activiti/src/main/resources/processes/signup.bpmn20.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | ${formOK == true} 28 | 29 | 30 | 31 | ${formOK == false} 32 | 33 | 34 | 35 | 36 | 37 | 38 | customer 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | customer 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | -------------------------------------------------------------------------------- /activiti/src/test/java/com/example/EmailValidationServiceTest.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import com.example.email.EmailValidationService; 4 | import org.junit.Assert; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.test.context.SpringBootTest; 9 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 10 | 11 | @RunWith(SpringJUnit4ClassRunner.class) 12 | @SpringBootTest(classes = DemoApplication.class) 13 | public class EmailValidationServiceTest { 14 | 15 | @Autowired 16 | private EmailValidationService emailValidationService; 17 | 18 | @Test 19 | public void testEmailValidation() throws Exception { 20 | Assert.assertTrue(this.emailValidationService 21 | .isEmailValid("george@email.com")); 22 | Assert.assertFalse(this.emailValidationService.isEmailValid("george")); 23 | } 24 | } -------------------------------------------------------------------------------- /activiti/src/test/java/com/example/SignupRestControllerTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import com.example.email.EmailValidationService; 4 | import com.fasterxml.jackson.core.type.TypeReference; 5 | import com.fasterxml.jackson.databind.ObjectMapper; 6 | import org.aopalliance.intercept.MethodInterceptor; 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | import org.junit.Before; 10 | import org.junit.Test; 11 | import org.junit.runner.RunWith; 12 | import org.springframework.aop.framework.ProxyFactoryBean; 13 | import org.springframework.beans.BeansException; 14 | import org.springframework.beans.factory.annotation.Autowired; 15 | import org.springframework.beans.factory.config.BeanPostProcessor; 16 | import org.springframework.boot.test.autoconfigure.web.servlet.AutoConfigureMockMvc; 17 | import org.springframework.boot.test.context.SpringBootTest; 18 | import org.springframework.boot.test.mock.mockito.MockBean; 19 | import org.springframework.http.MediaType; 20 | import org.springframework.stereotype.Component; 21 | import org.springframework.test.context.junit4.SpringRunner; 22 | import org.springframework.test.web.servlet.MockMvc; 23 | 24 | import java.util.List; 25 | import java.util.concurrent.atomic.AtomicInteger; 26 | 27 | import static org.junit.Assert.*; 28 | import static org.mockito.BDDMockito.given; 29 | import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; 30 | import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post; 31 | import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; 32 | 33 | @RunWith(SpringRunner.class) 34 | @AutoConfigureMockMvc 35 | @SpringBootTest(classes = DemoApplication.class, 36 | webEnvironment = SpringBootTest.WebEnvironment.MOCK) 37 | public class SignupRestControllerTests { 38 | 39 | private static final AtomicInteger counter = new AtomicInteger(); 40 | 41 | private String validEmail = "dsyer@email.com"; 42 | private String invalidEmail = "pwebb"; 43 | 44 | @MockBean 45 | private EmailValidationService emailValidationService; 46 | 47 | @Autowired 48 | private MockMvc mockMvc; 49 | 50 | @Autowired 51 | private ObjectMapper objectMapper; 52 | 53 | @Before 54 | public void before() throws Exception { 55 | this.repository.deleteAll(); 56 | counter.set(0); 57 | 58 | given(this.emailValidationService.isEmailValid(invalidEmail)).willReturn(false); 59 | given(this.emailValidationService.isEmailValid(validEmail)).willReturn(true); 60 | } 61 | 62 | @Autowired 63 | private CustomerRepository repository; 64 | 65 | private static Log log = LogFactory.getLog(SignupRestController.class); 66 | 67 | protected void doTestSignup(Customer input) throws Exception { 68 | 69 | String email = input.getEmail(); 70 | 71 | String inputJson = jsonForCustomer(input); 72 | 73 | String rootUrl = "/customers"; 74 | 75 | // start signup 76 | this.mockMvc 77 | .perform( 78 | post(rootUrl).content(inputJson).contentType( 79 | MediaType.APPLICATION_JSON)) 80 | .andExpect(status().isOk()) 81 | .andExpect( 82 | mvcResult -> { 83 | String contentAsString = mvcResult.getResponse() 84 | .getContentAsString(); 85 | Long customerId = Long.parseLong(contentAsString); 86 | assertNotNull(customerId); 87 | assertTrue(customerId > 0); 88 | }); 89 | 90 | Customer customer = this.repository.findByEmail(email).orElseThrow( 91 | () -> new AssertionError( 92 | "no record stored in the database for email '" + email 93 | + "'")); 94 | 95 | String customerId = Long.toString(customer.getId()); 96 | 97 | // see if there are any errors to be corrected 98 | String contentAsString = this.mockMvc 99 | .perform(get(rootUrl + "/" + customerId + "/signup/errors")) 100 | .andExpect(status().isOk()).andReturn().getResponse() 101 | .getContentAsString(); 102 | ObjectMapper mapper = new ObjectMapper(); 103 | TypeReference> typeReference = new TypeReference>() { 104 | }; 105 | List errantSignupFixTaskIds = mapper.readerFor(typeReference) 106 | .readValue(contentAsString); 107 | log.info("errant signups: " + errantSignupFixTaskIds.toString()); 108 | 109 | // if necessary, fix them 110 | errantSignupFixTaskIds.forEach(taskId -> { 111 | try { 112 | customer.setEmail("valid@email.com"); 113 | this.mockMvc.perform( 114 | post(rootUrl + "/" + customerId + "/signup/errors/" + taskId) 115 | .content(jsonForCustomer(customer)) 116 | .contentType(MediaType.APPLICATION_JSON)) 117 | .andExpect(status().isOk()); 118 | 119 | } catch (Exception e) { 120 | throw new RuntimeException(e); 121 | } 122 | }); 123 | 124 | // confirm receipt of email 125 | this.mockMvc.perform( 126 | post(rootUrl + "/" + customerId + "/signup/confirmation")) 127 | .andExpect(status().isOk()); 128 | } 129 | 130 | @Test 131 | public void signupFlowHappyPath() throws Exception { 132 | 133 | this.doTestSignup(new Customer("Dave", "Syer", validEmail)); 134 | int i = counter.get(); 135 | assertEquals(i, 1); 136 | log.info("signupFlowHappyPath: " + i); 137 | } 138 | 139 | @Test 140 | public void signupFlowSadPath() throws Exception { 141 | 142 | this.doTestSignup(new Customer("Phil", "Webb", invalidEmail)); 143 | int i = counter.get(); 144 | assertEquals(i, 2); 145 | log.info("signupFlowSadPath: " + i); 146 | } 147 | 148 | @Component 149 | public static class CheckFormBPP implements BeanPostProcessor { 150 | 151 | @Override 152 | public Object postProcessBeforeInitialization(Object o, String s) 153 | throws BeansException { 154 | return o; 155 | } 156 | 157 | @Override 158 | public Object postProcessAfterInitialization(Object o, String s) 159 | throws BeansException { 160 | if (o.getClass().isAssignableFrom(CheckForm.class)) { 161 | return this.counting(CheckForm.class.cast(o)); 162 | } 163 | return o; 164 | } 165 | 166 | private Object counting(CheckForm cf) { 167 | log.info("creating a counting proxy for " + cf.getClass()); 168 | ProxyFactoryBean pfb = new ProxyFactoryBean(); 169 | pfb.addAdvice((MethodInterceptor) invocation -> { 170 | if (invocation.getMethod().getName().equals("execute")) { 171 | counter.incrementAndGet(); 172 | } 173 | return invocation.proceed(); 174 | }); 175 | pfb.setTarget(cf); 176 | return pfb.getObject(); 177 | } 178 | } 179 | 180 | private String jsonForCustomer(Customer customer) throws Exception { 181 | return this.objectMapper.writerFor(Customer.class).writeValueAsString( 182 | customer); 183 | } 184 | } -------------------------------------------------------------------------------- /batch-and-integration/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | integration/batch-and-integration 6 | batch-and-integration 7 | 8 | cnj 9 | integration 10 | 1.0.0-SNAPSHOT 11 | 12 | 13 | 14 | org.springframework.integration 15 | spring-integration-java-dsl 16 | 1.1.1.RELEASE 17 | 18 | 19 | org.springframework.integration 20 | spring-integration-file 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-jdbc 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-web 29 | 30 | 31 | com.h2database 32 | h2 33 | 34 | 35 | org.springframework.batch 36 | spring-batch-integration 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-starter-test 41 | test 42 | 43 | 44 | 45 | 46 | 47 | org.springframework.boot 48 | spring-boot-maven-plugin 49 | 50 | processing.ProcessingApplication 51 | 52 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/eda/IntegrationApplication.java: -------------------------------------------------------------------------------- 1 | package eda; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class IntegrationApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(IntegrationApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/eda/IntegrationConfiguration.java: -------------------------------------------------------------------------------- 1 | package eda; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.beans.factory.annotation.Value; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.integration.dsl.IntegrationFlow; 9 | import org.springframework.integration.dsl.IntegrationFlows; 10 | import org.springframework.integration.dsl.channel.MessageChannels; 11 | import org.springframework.integration.dsl.file.Files; 12 | import org.springframework.messaging.MessageChannel; 13 | 14 | import java.io.File; 15 | 16 | @Configuration 17 | public class IntegrationConfiguration { 18 | 19 | private final Log log = LogFactory.getLog(getClass()); 20 | 21 | @Bean 22 | IntegrationFlow etlFlow( 23 | @Value("${input-directory:${HOME}/Desktop/in}") File dir) { 24 | // @formatter:off 25 | return IntegrationFlows 26 | // <1> 27 | .from(Files 28 | .inboundAdapter(dir) 29 | .autoCreateDirectory(true), 30 | consumer -> consumer.poller( 31 | spec -> spec.fixedRate(1000))) 32 | // <2> 33 | .handle(File.class, (file, headers) -> { 34 | log.info("we noticed a new file, " + file); 35 | return file; 36 | }) 37 | // <3> 38 | .routeToRecipients( 39 | spec -> spec 40 | .recipient(csv(), 41 | msg -> hasExt(msg.getPayload(), ".csv")) 42 | .recipient(txt(), 43 | msg -> hasExt(msg.getPayload(), ".txt"))) 44 | .get(); 45 | // @formatter:on 46 | } 47 | 48 | private boolean hasExt(Object f, String ext) { 49 | File file = File.class.cast(f); 50 | return file.getName().toLowerCase().endsWith(ext.toLowerCase()); 51 | } 52 | 53 | // <4> 54 | @Bean 55 | MessageChannel txt() { 56 | return MessageChannels.direct().get(); 57 | } 58 | 59 | // <5> 60 | @Bean 61 | MessageChannel csv() { 62 | return MessageChannels.direct().get(); 63 | } 64 | 65 | // <6> 66 | @Bean 67 | IntegrationFlow txtFlow() { 68 | return IntegrationFlows.from(txt()).handle(File.class, (f, h) -> { 69 | log.info("file is .txt!"); 70 | return null; 71 | }).get(); 72 | } 73 | 74 | // <7> 75 | @Bean 76 | IntegrationFlow csvFlow() { 77 | return IntegrationFlows.from(csv()).handle(File.class, (f, h) -> { 78 | log.info("file is .csv!"); 79 | return null; 80 | }).get(); 81 | } 82 | } -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/BatchChannels.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.integration.dsl.channel.MessageChannels; 6 | import org.springframework.messaging.MessageChannel; 7 | 8 | @Configuration 9 | class BatchChannels { 10 | 11 | @Bean 12 | MessageChannel invalid() { 13 | return MessageChannels.direct().get(); 14 | } 15 | 16 | @Bean 17 | MessageChannel completed() { 18 | return MessageChannels.direct().get(); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/BatchConfiguration.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import edabatch.email.EmailValidationService; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.batch.core.Job; 6 | import org.springframework.batch.core.Step; 7 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 8 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 9 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 10 | import org.springframework.batch.core.configuration.annotation.StepScope; 11 | import org.springframework.batch.item.ItemProcessor; 12 | import org.springframework.batch.item.ItemReader; 13 | import org.springframework.batch.item.ItemWriter; 14 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 15 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 16 | import org.springframework.batch.item.file.FlatFileItemReader; 17 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 18 | import org.springframework.batch.repeat.RepeatStatus; 19 | import org.springframework.beans.factory.annotation.Value; 20 | import org.springframework.context.annotation.Bean; 21 | import org.springframework.context.annotation.Configuration; 22 | import org.springframework.core.io.Resource; 23 | import org.springframework.jdbc.core.JdbcTemplate; 24 | import org.springframework.web.client.HttpStatusCodeException; 25 | import org.springframework.web.client.RestTemplate; 26 | 27 | import javax.sql.DataSource; 28 | 29 | @Configuration 30 | @EnableBatchProcessing 31 | public class BatchConfiguration { 32 | 33 | @Bean 34 | RestTemplate restTemplate() { 35 | return new RestTemplate(); 36 | } 37 | 38 | @Bean 39 | JdbcTemplate jdbcTemplate(DataSource dataSource) { 40 | return new JdbcTemplate(dataSource); 41 | } 42 | 43 | @Bean 44 | Job job(JobBuilderFactory jobBuilderFactory, 45 | StepBuilderFactory stepBuilderFactory, JdbcTemplate template, 46 | ItemReader fileReader, 47 | ItemProcessor emailProcessor, 48 | ItemWriter jdbcWriter) { 49 | 50 | Step setup = stepBuilderFactory.get("clean-contact-table") 51 | .tasklet((contribution, chunkContext) -> { 52 | template.update("delete from CONTACT"); 53 | return RepeatStatus.FINISHED; 54 | }) 55 | .build(); 56 | 57 | Step fileToJdbc = stepBuilderFactory 58 | .get("file-to-jdbc-fileToJdbc") 59 | .chunk(5) 60 | // <1> 61 | .reader(fileReader).processor(emailProcessor) 62 | .writer(jdbcWriter) 63 | .faultTolerant() 64 | .skip(InvalidEmailException.class) 65 | // <2> 66 | .skipPolicy( 67 | (Throwable t, int skipCount) -> { 68 | LogFactory.getLog(getClass()).info("skipping "); 69 | return t.getClass().isAssignableFrom( 70 | InvalidEmailException.class); 71 | }).retry(HttpStatusCodeException.class) // <3> 72 | .retryLimit(2).build(); 73 | 74 | return jobBuilderFactory.get("etl") // <4> 75 | .start(setup).next(fileToJdbc).build(); 76 | } 77 | 78 | // <5> 79 | @Bean 80 | @StepScope 81 | FlatFileItemReader fileReader( 82 | @Value("file://#{jobParameters['file']}") Resource pathToFile) throws Exception { 83 | return new FlatFileItemReaderBuilder() 84 | .name("file-reader") 85 | .resource(pathToFile) 86 | .targetType(Contact.class) 87 | .delimited().names("fullName,email".split(",")) 88 | .build(); 89 | } 90 | 91 | public static class InvalidEmailException extends Exception { 92 | public InvalidEmailException(String email) { 93 | super(String.format("the email %s isn't valid", email)); 94 | } 95 | } 96 | 97 | // <6> 98 | @Bean 99 | ItemProcessor validatingProcessor( 100 | EmailValidationService emailValidationService) { 101 | return item -> { 102 | boolean valid = emailValidationService 103 | .isEmailValid(item.getEmail()); 104 | item.setValidEmail(valid); 105 | if (!valid) 106 | throw new InvalidEmailException(item.getEmail()); 107 | return item; 108 | }; 109 | } 110 | 111 | // <7> 112 | @Bean 113 | JdbcBatchItemWriter jdbcWriter(DataSource dataSource) { 114 | return new JdbcBatchItemWriterBuilder() 115 | .dataSource(dataSource) 116 | .beanMapped() 117 | .sql("insert into CONTACT( full_name, email, valid_email ) values ( :fullName, :email, :validEmail )") 118 | .build(); 119 | } 120 | } -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/BatchIntegrationApplication.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class BatchIntegrationApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(BatchIntegrationApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/Contact.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | public class Contact { 4 | 5 | public Contact(String full_name, String email, long id) { 6 | this.fullName = full_name; 7 | this.email = email; 8 | this.id = id; 9 | } 10 | 11 | public Contact(boolean validEmail, String full_name, String email, long id) { 12 | this.fullName = full_name; 13 | this.validEmail = validEmail; 14 | this.email = email; 15 | this.id = id; 16 | } 17 | 18 | public Contact() { 19 | } 20 | 21 | public String getFullName() { 22 | return fullName; 23 | } 24 | 25 | @Override 26 | public String toString() { 27 | return "Contact{" + "fullName='" + fullName + '\'' + ", email='" 28 | + email + '\'' + ", validEmail=" + validEmail + ", id=" + id 29 | + '}'; 30 | } 31 | 32 | public void setFullName(String fullName) { 33 | this.fullName = fullName; 34 | } 35 | 36 | public String getEmail() { 37 | return email; 38 | } 39 | 40 | public void setEmail(String email) { 41 | this.email = email; 42 | } 43 | 44 | public boolean isValidEmail() { 45 | return validEmail; 46 | } 47 | 48 | public boolean getValidEmail() { 49 | return validEmail; 50 | } 51 | 52 | public void setValidEmail(boolean validEmail) { 53 | this.validEmail = validEmail; 54 | } 55 | 56 | public long getId() { 57 | return id; 58 | } 59 | 60 | public void setId(long id) { 61 | this.id = id; 62 | } 63 | 64 | private String fullName, email; 65 | private boolean validEmail; 66 | private long id; 67 | } 68 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/EtlFlowConfiguration.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.springframework.batch.core.*; 4 | import org.springframework.batch.core.launch.JobLauncher; 5 | import org.springframework.batch.integration.launch.JobLaunchRequest; 6 | import org.springframework.batch.integration.launch.JobLaunchingGateway; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.integration.dsl.IntegrationFlow; 11 | import org.springframework.integration.dsl.IntegrationFlows; 12 | import org.springframework.integration.dsl.file.Files; 13 | import org.springframework.integration.support.MessageBuilder; 14 | import org.springframework.messaging.Message; 15 | 16 | import java.io.File; 17 | 18 | import static org.springframework.integration.file.FileHeaders.ORIGINAL_FILE; 19 | 20 | @Configuration 21 | class EtlFlowConfiguration { 22 | 23 | // <1> 24 | @Bean 25 | IntegrationFlow etlFlow( 26 | @Value("${input-directory:${HOME}/Desktop/in}") File directory, 27 | BatchChannels c, 28 | JobLauncher launcher, 29 | Job job) { 30 | 31 | // @formatter:off 32 | return 33 | IntegrationFlows 34 | .from( 35 | Files.inboundAdapter(directory).autoCreateDirectory(true), 36 | cs -> cs.poller(p -> p.fixedRate(1000))) 37 | .handle(File.class, (file, headers) -> { 38 | 39 | String absolutePath = file.getAbsolutePath(); 40 | 41 | JobParameters params = new JobParametersBuilder() 42 | .addString("file", absolutePath) 43 | .toJobParameters(); 44 | 45 | return MessageBuilder 46 | .withPayload(new JobLaunchRequest(job, params)) 47 | .setHeader(ORIGINAL_FILE, absolutePath) 48 | .copyHeadersIfAbsent(headers) 49 | .build(); 50 | }) 51 | .handle(new JobLaunchingGateway(launcher)) 52 | .routeToRecipients( 53 | spec -> spec 54 | .recipient(c.invalid(), this::notFinished) 55 | .recipient(c.completed(), this::finished)) 56 | .get(); 57 | 58 | // @formatter:on 59 | } 60 | 61 | private boolean finished(Message msg) { 62 | Object payload = msg.getPayload(); 63 | return JobExecution.class.cast(payload).getExitStatus() 64 | .equals(ExitStatus.COMPLETED); 65 | } 66 | 67 | private boolean notFinished(Message msg) { 68 | return !this.finished(msg); 69 | } 70 | 71 | } 72 | 73 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/FinishedFileFlowConfiguration.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.batch.core.JobExecution; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.context.annotation.Bean; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.integration.dsl.IntegrationFlow; 10 | import org.springframework.integration.dsl.IntegrationFlows; 11 | import org.springframework.integration.file.FileHeaders; 12 | import org.springframework.jdbc.core.JdbcTemplate; 13 | 14 | import java.io.File; 15 | import java.util.List; 16 | 17 | import static edabatch.Utils.mv; 18 | 19 | @Configuration 20 | class FinishedFileFlowConfiguration { 21 | 22 | private Log log = LogFactory.getLog(getClass()); 23 | 24 | @Bean 25 | IntegrationFlow finishedJobsFlow( 26 | BatchChannels channels, 27 | @Value("${completed-directory:${HOME}/Desktop/completed}") File finished, 28 | JdbcTemplate jdbcTemplate) { 29 | // @formatter:off 30 | return IntegrationFlows 31 | .from(channels.completed()) 32 | .handle(JobExecution.class, 33 | (je, headers) -> { 34 | String ogFileName = String.class.cast(headers 35 | .get(FileHeaders.ORIGINAL_FILE)); 36 | File file = new File(ogFileName); 37 | mv(file, finished); 38 | List contacts = jdbcTemplate.query( 39 | "select * from CONTACT", 40 | (rs, i) -> new Contact( 41 | rs.getBoolean("valid_email"), 42 | rs.getString("full_name"), 43 | rs.getString("email"), 44 | rs.getLong("id"))); 45 | contacts.forEach(log::info); 46 | return null; 47 | }).get(); 48 | // @formatter:on 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/InvalidFileFlowConfiguration.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.springframework.batch.core.JobExecution; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.context.annotation.Bean; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.integration.dsl.IntegrationFlow; 8 | import org.springframework.integration.dsl.IntegrationFlows; 9 | import org.springframework.integration.file.FileHeaders; 10 | 11 | import java.io.File; 12 | 13 | import static edabatch.Utils.mv; 14 | 15 | 16 | @Configuration 17 | class InvalidFileFlowConfiguration { 18 | 19 | @Bean 20 | IntegrationFlow invalidFileFlow( 21 | BatchChannels channels, 22 | @Value("${error-directory:${HOME}/Desktop/errors}") File errors) { 23 | // @formatter:off 24 | return IntegrationFlows 25 | .from(channels.invalid()) 26 | .handle(JobExecution.class, 27 | (je, headers) -> { 28 | String ogFileName = String.class.cast(headers 29 | .get(FileHeaders.ORIGINAL_FILE)); 30 | File file = new File(ogFileName); 31 | mv(file, errors); 32 | return null; 33 | }).get(); 34 | // @formatter:on 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/JobExecutionListener.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.autoconfigure.batch.JobExecutionEvent; 7 | import org.springframework.context.event.EventListener; 8 | import org.springframework.jdbc.core.JdbcTemplate; 9 | import org.springframework.jdbc.core.RowCallbackHandler; 10 | import org.springframework.stereotype.Component; 11 | 12 | @Component 13 | class JobExecutionListener { 14 | 15 | private final JdbcTemplate jdbcTemplate; 16 | 17 | private Log log = LogFactory.getLog(getClass()); 18 | 19 | @Autowired 20 | public JobExecutionListener(JdbcTemplate template) { 21 | this.jdbcTemplate = template; 22 | } 23 | 24 | @EventListener(JobExecutionEvent.class) 25 | public void job(JobExecutionEvent executionEvent) { 26 | log.info("jobExecutionEvent: " 27 | + executionEvent.getJobExecution().toString()); 28 | jdbcTemplate.query("select * from CONTACT", 29 | (RowCallbackHandler) rs -> log.info(String.format( 30 | "id=%s, full_name=%s, email=%s, valid_email=%s", 31 | rs.getLong("id"), rs.getString("full_name"), 32 | rs.getString("email"), rs.getBoolean("valid_email")))); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/Utils.java: -------------------------------------------------------------------------------- 1 | package edabatch; 2 | 3 | import org.springframework.util.Assert; 4 | 5 | import java.io.File; 6 | import java.io.IOException; 7 | 8 | import static java.nio.file.StandardCopyOption.REPLACE_EXISTING; 9 | 10 | /** 11 | * @author Josh Long 12 | */ 13 | abstract class Utils { 14 | public static void mv(File in, File out) { 15 | try { 16 | Assert.isTrue(out.exists() || out.mkdirs()); 17 | File target = new File(out, in.getName()); 18 | java.nio.file.Files.copy(in.toPath(), target.toPath(), REPLACE_EXISTING); 19 | } catch (IOException e) { 20 | throw new RuntimeException(e); 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/email/EmailValidationService.java: -------------------------------------------------------------------------------- 1 | package edabatch.email; 2 | 3 | public interface EmailValidationService { 4 | boolean isEmailValid(String email); 5 | } 6 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/email/InvalidEmailException.java: -------------------------------------------------------------------------------- 1 | package edabatch.email; 2 | 3 | public class InvalidEmailException extends Exception { 4 | public InvalidEmailException(String email) { 5 | super(String.format("the email %s isn't valid", email)); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/email/MashapeEmailValidationService.java: -------------------------------------------------------------------------------- 1 | package edabatch.email; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.core.ParameterizedTypeReference; 7 | import org.springframework.http.RequestEntity; 8 | import org.springframework.http.ResponseEntity; 9 | import org.springframework.stereotype.Service; 10 | import org.springframework.web.client.RestTemplate; 11 | import org.springframework.web.util.UriComponents; 12 | import org.springframework.web.util.UriComponentsBuilder; 13 | 14 | import java.util.Map; 15 | 16 | @Service 17 | @Profile("production") 18 | class MashapeEmailValidationService implements EmailValidationService { 19 | 20 | private final String mashapeKey; 21 | private final RestTemplate restTemplate; 22 | private final String uri; 23 | 24 | @Autowired 25 | public MashapeEmailValidationService( 26 | @Value("${mashape.key}") String key, 27 | @Value("${email-validator.uri}") String uri, 28 | RestTemplate restTemplate) { 29 | this.mashapeKey = key; 30 | this.uri = uri; 31 | this.restTemplate = restTemplate; 32 | } 33 | 34 | public boolean isEmailValid(String email) { 35 | UriComponents emailValidatedUri = UriComponentsBuilder.fromHttpUrl(uri) 36 | .buildAndExpand(email); 37 | 38 | RequestEntity requestEntity = RequestEntity 39 | .get(emailValidatedUri.toUri()) 40 | .header("X-Mashape-Key", mashapeKey).build(); 41 | 42 | ParameterizedTypeReference> ptr = 43 | new ParameterizedTypeReference>() { }; 44 | 45 | ResponseEntity> responseEntity = restTemplate.exchange(requestEntity, ptr); 46 | 47 | return responseEntity.getBody().get("isValid"); 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/java/edabatch/email/SimpleEmailValidationService.java: -------------------------------------------------------------------------------- 1 | package edabatch.email; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.stereotype.Service; 7 | import org.springframework.util.StringUtils; 8 | 9 | @Service 10 | @Profile("!production") 11 | class SimpleEmailValidationService implements EmailValidationService { 12 | 13 | private Log log = LogFactory.getLog(getClass()); 14 | 15 | @Override 16 | public boolean isEmailValid(String email) { 17 | boolean emailIsValid = StringUtils.hasText(email) && 18 | email.length() > 1 && 19 | email.contains("@"); 20 | log.debug("emailIsValid: " + emailIsValid); 21 | return emailIsValid; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /batch-and-integration/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | email-validator.uri=https://pozzad-email-validator.p.mashape.com/emailvalidator/validateEmail/{email} 2 | spring.main.web-environment=false 3 | 4 | spring.batch.job.enabled=false -------------------------------------------------------------------------------- /batch-and-integration/src/main/resources/data.csv: -------------------------------------------------------------------------------- 1 | Josh Long,jlong 2 | Kenny Bastani,kBastani@email.com 3 | Phil Webb,pwebb@email.com 4 | Dave Syer,dsyer@email.com 5 | Spencer Gibb,sgibb@email.com 6 | Juergen Hoeller,jhoeller@email.com -------------------------------------------------------------------------------- /batch-and-integration/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE contact IF EXISTS; 2 | 3 | CREATE TABLE contact( 4 | id IDENTITY primary key, 5 | email VARCHAR (20), 6 | full_name VARCHAR(20), 7 | valid_email BOOLEAN NULL 8 | ); -------------------------------------------------------------------------------- /batch/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | cnj 7 | integration 8 | 1.0.0-SNAPSHOT 9 | 10 | integration/batch 11 | batch 12 | 13 | 14 | org.springframework.boot 15 | spring-boot-starter-jdbc 16 | 17 | 18 | mysql 19 | mysql-connector-java 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter-batch 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-web 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-test 33 | test 34 | 35 | 36 | 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-maven-plugin 41 | 42 | 43 | 44 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/BatchApplication.java: -------------------------------------------------------------------------------- 1 | package processing; 2 | 3 | import org.springframework.batch.core.Job; 4 | import org.springframework.batch.core.JobParametersBuilder; 5 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 6 | import org.springframework.batch.core.launch.JobLauncher; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.boot.CommandLineRunner; 9 | import org.springframework.boot.SpringApplication; 10 | import org.springframework.boot.autoconfigure.SpringBootApplication; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.jdbc.core.JdbcTemplate; 13 | import org.springframework.web.client.RestTemplate; 14 | 15 | import javax.sql.DataSource; 16 | import java.io.File; 17 | 18 | @EnableBatchProcessing 19 | @SpringBootApplication 20 | public class BatchApplication { 21 | 22 | @Bean 23 | RestTemplate restTemplate() { 24 | return new RestTemplate(); 25 | } 26 | 27 | @Bean 28 | JdbcTemplate jdbcTemplate(DataSource dataSource) { 29 | return new JdbcTemplate(dataSource); 30 | } 31 | 32 | // <1> 33 | @Bean 34 | CommandLineRunner run( 35 | JobLauncher launcher, 36 | Job job, 37 | @Value("${user.home}") String home) { 38 | return args -> 39 | launcher.run(job, 40 | new JobParametersBuilder() 41 | .addString("input", path(home, "in.csv")) 42 | .addString("output", path(home, "out.csv")) 43 | .toJobParameters()); 44 | } 45 | 46 | public static void main(String[] args) { 47 | SpringApplication.run(BatchApplication.class, args); 48 | } 49 | 50 | private String path(String home, String fileName) { 51 | return new File(home, fileName).getAbsolutePath(); 52 | } 53 | } -------------------------------------------------------------------------------- /batch/src/main/java/processing/BatchConfiguration.java: -------------------------------------------------------------------------------- 1 | package processing; 2 | 3 | import org.springframework.batch.core.Job; 4 | import org.springframework.batch.core.Step; 5 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 6 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 7 | import org.springframework.batch.core.launch.support.RunIdIncrementer; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.web.client.HttpStatusCodeException; 11 | import processing.email.InvalidEmailException; 12 | 13 | import java.util.Map; 14 | 15 | @Configuration 16 | class BatchConfiguration { 17 | 18 | // <1> 19 | @Bean 20 | Job etl(JobBuilderFactory jbf, 21 | StepBuilderFactory sbf, 22 | Step1Configuration step1, // <2> 23 | Step2Configuration step2, 24 | Step3Configuration step3) throws Exception { 25 | 26 | Step setup = sbf.get("clean-contact-table") 27 | .tasklet(step1.tasklet(null)) // <3> 28 | .build(); 29 | 30 | Step s2 = sbf.get("file-db") 31 | .chunk(1000) // <4> 32 | .faultTolerant() // <5> 33 | .skip(InvalidEmailException.class) 34 | .retry(HttpStatusCodeException.class) 35 | .retryLimit(2) 36 | .reader(step2.fileReader(null)) // <6> 37 | .processor(step2.emailValidatingProcessor(null)) // <7> 38 | .writer(step2.jdbcWriter(null)) // <8> 39 | .build(); 40 | 41 | // <9> 42 | Step s3 = sbf.get("db-file") 43 | ., Map>chunk(100) 44 | .reader(step3.jdbcReader(null)) 45 | .writer(step3.fileWriter(null)) 46 | .build(); 47 | 48 | return jbf.get("etl") 49 | .incrementer(new RunIdIncrementer()) // <10> 50 | .start(setup) // <11> 51 | .next(s2) 52 | .next(s3) 53 | .build(); // <12> 54 | } 55 | } -------------------------------------------------------------------------------- /batch/src/main/java/processing/Person.java: -------------------------------------------------------------------------------- 1 | package processing; 2 | 3 | 4 | public class Person { 5 | private int age; 6 | private String firstName, email; 7 | 8 | public Person() { 9 | } 10 | 11 | public void setAge(int age) { 12 | this.age = age; 13 | } 14 | 15 | public void setFirstName(String firstName) { 16 | this.firstName = firstName; 17 | } 18 | 19 | public void setEmail(String email) { 20 | this.email = email; 21 | } 22 | 23 | public Person(int age, String firstName, String email) { 24 | this.age = age; 25 | this.firstName = firstName; 26 | this.email = email; 27 | } 28 | 29 | public int getAge() { 30 | return age; 31 | } 32 | 33 | public String getFirstName() { 34 | return firstName; 35 | } 36 | 37 | public String getEmail() { 38 | return email; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/Step1Configuration.java: -------------------------------------------------------------------------------- 1 | package processing; 2 | 3 | 4 | import org.apache.commons.logging.Log; 5 | import org.apache.commons.logging.LogFactory; 6 | import org.springframework.batch.core.step.tasklet.Tasklet; 7 | import org.springframework.batch.repeat.RepeatStatus; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.jdbc.core.JdbcTemplate; 11 | 12 | @Configuration 13 | class Step1Configuration { 14 | 15 | @Bean 16 | Tasklet tasklet(JdbcTemplate jdbcTemplate) { 17 | 18 | Log log = LogFactory.getLog(getClass()); 19 | 20 | return (contribution, chunkContext) -> { // <1> 21 | log.info("starting the ETL job."); 22 | jdbcTemplate.update("delete from PEOPLE"); 23 | return RepeatStatus.FINISHED; 24 | }; 25 | } 26 | } -------------------------------------------------------------------------------- /batch/src/main/java/processing/Step2Configuration.java: -------------------------------------------------------------------------------- 1 | package processing; 2 | 3 | import org.springframework.batch.core.configuration.annotation.StepScope; 4 | import org.springframework.batch.item.ItemProcessor; 5 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 6 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 7 | import org.springframework.batch.item.file.FlatFileItemReader; 8 | import org.springframework.batch.item.file.builder.FlatFileItemReaderBuilder; 9 | import org.springframework.beans.factory.annotation.Value; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.core.io.Resource; 13 | import processing.email.EmailValidationService; 14 | import processing.email.InvalidEmailException; 15 | 16 | import javax.sql.DataSource; 17 | 18 | @Configuration 19 | class Step2Configuration { 20 | 21 | @Bean 22 | @StepScope // <1> 23 | FlatFileItemReader fileReader( 24 | @Value("file://#{jobParameters['input']}") Resource in) // <2> 25 | throws Exception { 26 | 27 | // <3> 28 | return new FlatFileItemReaderBuilder() 29 | .name("file-reader") 30 | .resource(in) 31 | .targetType(Person.class) 32 | .delimited() 33 | .delimiter(",") 34 | .names(new String[]{"firstName", "age", "email"}) 35 | .build(); 36 | } 37 | 38 | @Bean 39 | ItemProcessor emailValidatingProcessor( 40 | EmailValidationService emailValidator) { // <4> 41 | return item -> { 42 | String email = item.getEmail(); 43 | if (!emailValidator.isEmailValid(email)) { 44 | throw new InvalidEmailException(email); 45 | } 46 | return item; 47 | }; 48 | } 49 | 50 | @Bean 51 | JdbcBatchItemWriter jdbcWriter(DataSource ds) { // <5> 52 | return new JdbcBatchItemWriterBuilder() 53 | .dataSource(ds) 54 | .sql("insert into PEOPLE( AGE, FIRST_NAME, EMAIL)" + 55 | " values (:age, :firstName, :email)") 56 | .beanMapped() 57 | .build(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/Step3Configuration.java: -------------------------------------------------------------------------------- 1 | package processing; 2 | 3 | import org.springframework.batch.core.configuration.annotation.StepScope; 4 | import org.springframework.batch.item.database.JdbcCursorItemReader; 5 | import org.springframework.batch.item.database.builder.JdbcCursorItemReaderBuilder; 6 | import org.springframework.batch.item.file.FlatFileItemWriter; 7 | import org.springframework.batch.item.file.builder.FlatFileItemWriterBuilder; 8 | import org.springframework.batch.item.file.transform.DelimitedLineAggregator; 9 | import org.springframework.beans.factory.annotation.Value; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.core.io.Resource; 13 | 14 | import javax.sql.DataSource; 15 | import java.util.Collections; 16 | import java.util.Map; 17 | 18 | @Configuration 19 | class Step3Configuration { 20 | 21 | // <1> 22 | @Bean 23 | JdbcCursorItemReader> jdbcReader(DataSource dataSource) { 24 | return new JdbcCursorItemReaderBuilder>() 25 | .dataSource(dataSource) 26 | .name("jdbc-reader") 27 | .sql("select COUNT(age) c, age a from PEOPLE group by age") 28 | .rowMapper((rs, i) -> Collections.singletonMap( 29 | rs.getInt("a"), rs.getInt("c"))) 30 | .build(); 31 | } 32 | 33 | //<2> 34 | @Bean 35 | @StepScope 36 | FlatFileItemWriter> fileWriter( 37 | @Value("file://#{jobParameters['output']}") Resource out) { 38 | 39 | DelimitedLineAggregator> aggregator = 40 | new DelimitedLineAggregator>() { 41 | { 42 | // <3> 43 | setDelimiter(","); 44 | setFieldExtractor(ageAndCount -> { 45 | Map.Entry next = ageAndCount.entrySet().iterator().next(); 46 | return new Object[]{next.getKey(), next.getValue()}; 47 | }); 48 | } 49 | }; 50 | 51 | return new FlatFileItemWriterBuilder>() 52 | .name("file-writer") 53 | .resource(out) 54 | .lineAggregator(aggregator) 55 | .build(); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/email/EmailValidationService.java: -------------------------------------------------------------------------------- 1 | package processing.email; 2 | 3 | public interface EmailValidationService { 4 | boolean isEmailValid(String email); 5 | } 6 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/email/InvalidEmailException.java: -------------------------------------------------------------------------------- 1 | package processing.email; 2 | 3 | public class InvalidEmailException extends Exception { 4 | public InvalidEmailException(String email) { 5 | super(String.format("the email %s isn't valid", email)); 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/email/MashapeEmailValidationService.java: -------------------------------------------------------------------------------- 1 | package processing.email; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.core.ParameterizedTypeReference; 7 | import org.springframework.http.RequestEntity; 8 | import org.springframework.http.ResponseEntity; 9 | import org.springframework.stereotype.Service; 10 | import org.springframework.web.client.RestTemplate; 11 | import org.springframework.web.util.UriComponents; 12 | import org.springframework.web.util.UriComponentsBuilder; 13 | 14 | import java.util.Map; 15 | 16 | @Service 17 | @Profile("production") 18 | class MashapeEmailValidationService implements EmailValidationService { 19 | 20 | private final String mashapeKey; 21 | private final RestTemplate restTemplate; 22 | private final String uri; 23 | 24 | @Autowired 25 | public MashapeEmailValidationService( 26 | @Value("${mashape.key}") String key, 27 | @Value("${email-validator.uri}") String uri, 28 | RestTemplate restTemplate) { 29 | this.mashapeKey = key; 30 | this.uri = uri; 31 | this.restTemplate = restTemplate; 32 | } 33 | 34 | public boolean isEmailValid(String email) { 35 | UriComponents emailValidatedUri = UriComponentsBuilder.fromHttpUrl(uri) 36 | .buildAndExpand(email); 37 | 38 | RequestEntity requestEntity = RequestEntity 39 | .get(emailValidatedUri.toUri()) 40 | .header("X-Mashape-Key", mashapeKey).build(); 41 | 42 | ParameterizedTypeReference> ptr = 43 | new ParameterizedTypeReference>() { }; 44 | 45 | ResponseEntity> responseEntity = restTemplate.exchange(requestEntity, ptr); 46 | 47 | return responseEntity.getBody().get("isValid"); 48 | } 49 | } 50 | 51 | -------------------------------------------------------------------------------- /batch/src/main/java/processing/email/SimpleEmailValidationService.java: -------------------------------------------------------------------------------- 1 | package processing.email; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.context.annotation.Profile; 6 | import org.springframework.stereotype.Service; 7 | import org.springframework.util.StringUtils; 8 | 9 | @Service 10 | @Profile("!production") 11 | class SimpleEmailValidationService implements EmailValidationService { 12 | 13 | private Log log = LogFactory.getLog(getClass()); 14 | 15 | @Override 16 | public boolean isEmailValid(String email) { 17 | boolean emailIsValid = StringUtils.hasText(email) && 18 | email.length() > 1 && 19 | email.contains("@"); 20 | log.debug("emailIsValid: " + emailIsValid); 21 | return emailIsValid; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /batch/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | email-validator.uri=https://pozzad-email-validator.p.mashape.com/emailvalidator/validateEmail/{email} 3 | 4 | spring.main.web-environment=false 5 | 6 | spring.datasource.url=jdbc:mysql://localhost:3306/batch?useSSL=false 7 | spring.datasource.password=batch 8 | spring.datasource.username=batch 9 | 10 | spring.batch.job.enabled=false 11 | 12 | logging.level.org.springframework.context.annotation=off 13 | 14 | -------------------------------------------------------------------------------- /batch/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | SET foreign_key_checks = 0; 2 | 3 | CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION ( 4 | ID INT 5 | ); 6 | CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION_CONTEXT ( 7 | ID INT 8 | ); 9 | CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION_PARAMS ( 10 | ID INT 11 | ); 12 | CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION_SEQ ( 13 | ID INT 14 | ); 15 | CREATE TABLE IF NOT EXISTS BATCH_JOB_INSTANCE ( 16 | ID INT 17 | ); 18 | CREATE TABLE IF NOT EXISTS BATCH_JOB_SEQ ( 19 | ID INT 20 | ); 21 | CREATE TABLE IF NOT EXISTS BATCH_STEP_EXECUTION ( 22 | ID INT 23 | ); 24 | CREATE TABLE IF NOT EXISTS BATCH_STEP_EXECUTION_CONTEXT ( 25 | ID INT 26 | ); 27 | CREATE TABLE IF NOT EXISTS BATCH_STEP_EXECUTION_SEQ ( 28 | ID INT 29 | ); 30 | TRUNCATE BATCH_JOB_EXECUTION; 31 | TRUNCATE BATCH_JOB_EXECUTION_CONTEXT; 32 | TRUNCATE BATCH_JOB_EXECUTION_PARAMS; 33 | TRUNCATE BATCH_JOB_EXECUTION_SEQ; 34 | TRUNCATE BATCH_JOB_INSTANCE; 35 | TRUNCATE BATCH_JOB_SEQ; 36 | TRUNCATE BATCH_STEP_EXECUTION; 37 | TRUNCATE BATCH_STEP_EXECUTION_CONTEXT; 38 | TRUNCATE BATCH_STEP_EXECUTION_SEQ; 39 | 40 | DROP TABLE BATCH_JOB_EXECUTION, BATCH_JOB_EXECUTION_CONTEXT, BATCH_JOB_EXECUTION_PARAMS, BATCH_JOB_EXECUTION_SEQ, BATCH_JOB_INSTANCE, BATCH_JOB_SEQ, BATCH_STEP_EXECUTION, BATCH_STEP_EXECUTION_CONTEXT, BATCH_STEP_EXECUTION_SEQ CASCADE; 41 | 42 | SET foreign_key_checks = 1; 43 | 44 | # customer data 45 | 46 | CREATE TABLE IF NOT EXISTS PEOPLE ( 47 | ID BIGINT(10) NOT NULL AUTO_INCREMENT PRIMARY KEY, 48 | EMAIL VARCHAR(255) NOT NULL, 49 | AGE INT(3) NOT NULL, 50 | FIRST_NAME VARCHAR(255) NOT NULL 51 | ); -------------------------------------------------------------------------------- /dataflow/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 7 | cnj 8 | integration 9 | 1.0.0-SNAPSHOT 10 | 11 | integration/dataflow 12 | dataflow 13 | 14 | pom 15 | 16 | server 17 | server-definitions 18 | shell 19 | stream-example 20 | task-example 21 | 22 | 23 | -------------------------------------------------------------------------------- /dataflow/server-definitions/manifest.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: server-definitions 4 | memory: 256MB 5 | instances: 1 6 | host: server-definitions-${random-word} 7 | path: target/server-definitions.jar 8 | env: 9 | SPRING_PROFILES_ACTIVE: cloud 10 | -------------------------------------------------------------------------------- /dataflow/server-definitions/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | dataflow 9 | 1.0.0-SNAPSHOT 10 | 11 | jar 12 | integration/dataflow/server-definitions 13 | server-definitions 14 | 15 | 16 | org.springframework.boot 17 | spring-boot-starter-web 18 | 19 | 20 | 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-maven-plugin 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /dataflow/server-definitions/src/main/java/demo/DataFlowDefinitionsApplication.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class DataFlowDefinitionsApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(DataFlowDefinitionsApplication.class, args); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /dataflow/server-definitions/src/main/resources/static/dataflow-example-apps.properties: -------------------------------------------------------------------------------- 1 | 2 | task.simple-task=maven://cnj:task:1.0.0-SNAPSHOT 3 | task.batch-task=maven://cnj:task-example:1.0.0-SNAPSHOT 4 | 5 | processor.brackets=maven://cnj:stream-example:1.0.0-SNAPSHOT 6 | -------------------------------------------------------------------------------- /dataflow/server/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | dataflow 9 | 1.0.0-SNAPSHOT 10 | 11 | integration/dataflow/server 12 | server 13 | 14 | true 15 | 16 | 17 | 18 | org.json 19 | json 20 | 21 | 22 | org.springframework.cloud 23 | spring-cloud-starter-dataflow-server-local 24 | 25 | 26 | org.springframework.boot 27 | spring-boot-starter-test 28 | test 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-starter-actuator 33 | 34 | 35 | org.springframework.cloud 36 | spring-cloud-dataflow-rest-client 37 | 38 | 39 | org.projectlombok 40 | lombok 41 | true 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | org.apache.maven.plugins 51 | maven-surefire-plugin 52 | 53 | ${skipTests} 54 | 55 | 56 | 57 | org.springframework.boot 58 | spring-boot-maven-plugin 59 | 60 | 61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /dataflow/server/src/main/java/dataflow/DataFlowInitializer.java: -------------------------------------------------------------------------------- 1 | package dataflow; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.beans.factory.annotation.Value; 5 | import org.springframework.boot.context.event.ApplicationReadyEvent; 6 | import org.springframework.cloud.dataflow.rest.client.DataFlowTemplate; 7 | import org.springframework.cloud.dataflow.rest.client.TaskOperations; 8 | import org.springframework.context.event.EventListener; 9 | import org.springframework.stereotype.Component; 10 | import org.springframework.web.client.RestTemplate; 11 | 12 | import java.net.URI; 13 | import java.net.URL; 14 | import java.util.*; 15 | import java.util.stream.Stream; 16 | 17 | @Component 18 | class DataFlowInitializer { 19 | 20 | private final URI baseUri; 21 | 22 | @Autowired 23 | public DataFlowInitializer(@Value("${server.port}") int port) { 24 | this.baseUri = URI.create("http://localhost:" + port); 25 | } 26 | 27 | @EventListener(ApplicationReadyEvent.class) 28 | public void deployOnStart(ApplicationReadyEvent e) throws Exception { 29 | 30 | // <1> 31 | DataFlowTemplate df = new DataFlowTemplate(baseUri, new RestTemplate()); 32 | 33 | // <2> 34 | Stream.of( 35 | "http://bit.ly/stream-applications-rabbit-maven", 36 | new URL(baseUri.toURL(), "app.properties").toString() 37 | ) 38 | .parallel() 39 | .forEach(u -> df.appRegistryOperations().importFromResource(u, true)); 40 | 41 | 42 | TaskOperations taskOperations = df.taskOperations(); 43 | Stream.of("batch-task", "simple-task") 44 | .forEach(tn -> { 45 | String name = "my-" + tn; 46 | taskOperations.create(name, tn); // <3> 47 | Map properties = Collections.singletonMap("simple-batch-task.input", System.getenv("HOME") + "Desktop/in.csv"); 48 | List arguments = Arrays.asList("input=in", "output=out"); 49 | taskOperations.launch(name, properties, arguments); // <4> 50 | }); 51 | 52 | 53 | // <3> 54 | Map streams = new HashMap<>(); 55 | streams.put("bracket-time", "time | brackets | log"); 56 | streams.entrySet().parallelStream() 57 | .forEach(stream -> df.streamOperations() 58 | .createStream(stream.getKey(), stream.getValue(), true)); 59 | } 60 | } -------------------------------------------------------------------------------- /dataflow/server/src/main/java/dataflow/DataFlowServer.java: -------------------------------------------------------------------------------- 1 | package dataflow; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.dataflow.server.EnableDataFlowServer; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.web.client.RestTemplate; 8 | 9 | // <1> 10 | @EnableDataFlowServer 11 | @SpringBootApplication 12 | public class DataFlowServer { 13 | 14 | public static void main(String[] args) { 15 | SpringApplication.run(DataFlowServer.class, args); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /dataflow/server/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | #spring.cloud.dataflow.registry.populator.locations = classpath:rabbit-apps.properties 2 | -------------------------------------------------------------------------------- /dataflow/server/src/main/resources/static/app.properties: -------------------------------------------------------------------------------- 1 | 2 | task.simple-task=maven://cnj:task:1.0.0-SNAPSHOT 3 | task.batch-task=maven://cnj:task-example:1.0.0-SNAPSHOT 4 | 5 | processor.brackets=maven://cnj:stream-example:1.0.0-SNAPSHOT 6 | -------------------------------------------------------------------------------- /dataflow/server/src/main/resources/static/maven-rabbitmq-1.0.1.properties: -------------------------------------------------------------------------------- 1 | # "http://bit.ly/stream-applications-rabbit-maven" 2 | source.file=maven://org.springframework.cloud.stream.app:file-source-rabbit:1.0.0.BUILD-SNAPSHOT 3 | source.ftp=maven://org.springframework.cloud.stream.app:ftp-source-rabbit:1.0.0.BUILD-SNAPSHOT 4 | source.gemfire=maven://org.springframework.cloud.stream.app:gemfire-source-rabbit:1.0.0.BUILD-SNAPSHOT 5 | source.gemfire-cq=maven://org.springframework.cloud.stream.app:gemfire-cq-source-rabbit:1.0.0.BUILD-SNAPSHOT 6 | source.jdbc=maven://org.springframework.cloud.stream.app:jdbc-source-rabbit:1.0.0.BUILD-SNAPSHOT 7 | source.jms=maven://org.springframework.cloud.stream.app:jms-source-rabbit:1.0.0.BUILD-SNAPSHOT 8 | source.http=maven://org.springframework.cloud.stream.app:http-source-rabbit:1.0.0.BUILD-SNAPSHOT 9 | source.load-generator=maven://org.springframework.cloud.stream.app:load-generator-source-rabbit:1.0.0.BUILD-SNAPSHOT 10 | source.mail=maven://org.springframework.cloud.stream.app:mail-source-rabbit:1.0.0.BUILD-SNAPSHOT 11 | source.mongodb=maven://org.springframework.cloud.stream.app:mongodb-source-rabbit:1.0.0.BUILD-SNAPSHOT 12 | source.rabbit=maven://org.springframework.cloud.stream.app:rabbit-source-rabbit:1.0.0.BUILD-SNAPSHOT 13 | source.s3=maven://org.springframework.cloud.stream.app:s3-source-rabbit:1.0.0.BUILD-SNAPSHOT 14 | source.sftp=maven://org.springframework.cloud.stream.app:sftp-source-rabbit:1.0.0.BUILD-SNAPSHOT 15 | source.syslog=maven://org.springframework.cloud.stream.app:syslog-source-rabbit:1.0.0.BUILD-SNAPSHOT 16 | source.tcp=maven://org.springframework.cloud.stream.app:tcp-source-rabbit:1.0.0.BUILD-SNAPSHOT 17 | source.tcp-client=maven://org.springframework.cloud.stream.app:tcp-client-source-rabbit:1.0.0.BUILD-SNAPSHOT 18 | source.time=maven://org.springframework.cloud.stream.app:time-source-rabbit:1.0.0.BUILD-SNAPSHOT 19 | source.trigger=maven://org.springframework.cloud.stream.app:trigger-source-rabbit:1.0.0.BUILD-SNAPSHOT 20 | source.triggertask=maven://org.springframework.cloud.stream.app:triggertask-source-rabbit:1.0.0.BUILD-SNAPSHOT 21 | source.twitterstream=maven://org.springframework.cloud.stream.app:twitterstream-source-rabbit:1.0.0.BUILD-SNAPSHOT 22 | processor.bridge=maven://org.springframework.cloud.stream.app:bridge-processor-rabbit:1.0.0.BUILD-SNAPSHOT 23 | processor.filter=maven://org.springframework.cloud.stream.app:filter-processor-rabbit:1.0.0.BUILD-SNAPSHOT 24 | processor.groovy-filter=maven://org.springframework.cloud.stream.app:groovy-filter-processor-rabbit:1.0.0.BUILD-SNAPSHOT 25 | processor.groovy-transform=maven://org.springframework.cloud.stream.app:groovy-transform-processor-rabbit:1.0.0.BUILD-SNAPSHOT 26 | processor.httpclient=maven://org.springframework.cloud.stream.app:httpclient-processor-rabbit:1.0.0.BUILD-SNAPSHOT 27 | processor.pmml=maven://org.springframework.cloud.stream.app:pmml-processor-rabbit:1.0.0.BUILD-SNAPSHOT 28 | processor.scriptable-transform=maven://org.springframework.cloud.stream.app:scriptable-transform-processor-rabbit:1.0.0.BUILD-SNAPSHOT 29 | processor.splitter=maven://org.springframework.cloud.stream.app:splitter-processor-rabbit:1.0.0.BUILD-SNAPSHOT 30 | processor.tcp-client=maven://org.springframework.cloud.stream.app:tcp-client-processor-rabbit:1.0.0.BUILD-SNAPSHOT 31 | processor.transform=maven://org.springframework.cloud.stream.app:transform-processor-rabbit:1.0.0.BUILD-SNAPSHOT 32 | sink.aggregate-counter=maven://org.springframework.cloud.stream.app:aggregate-counter-sink-rabbit:1.0.0.BUILD-SNAPSHOT 33 | sink.cassandra=maven://org.springframework.cloud.stream.app:cassandra-sink-rabbit:1.0.0.BUILD-SNAPSHOT 34 | sink.counter=maven://org.springframework.cloud.stream.app:counter-sink-rabbit:1.0.0.BUILD-SNAPSHOT 35 | sink.field-value-counter=maven://org.springframework.cloud.stream.app:field-value-counter-sink-rabbit:1.0.0.BUILD-SNAPSHOT 36 | sink.file=maven://org.springframework.cloud.stream.app:file-sink-rabbit:1.0.0.BUILD-SNAPSHOT 37 | sink.ftp=maven://org.springframework.cloud.stream.app:ftp-sink-rabbit:1.0.0.BUILD-SNAPSHOT 38 | sink.gemfire=maven://org.springframework.cloud.stream.app:gemfire-sink-rabbit:1.0.0.BUILD-SNAPSHOT 39 | sink.gpfdist=maven://org.springframework.cloud.stream.app:gpfdist-sink-rabbit:1.0.0.BUILD-SNAPSHOT 40 | sink.hdfs=maven://org.springframework.cloud.stream.app:hdfs-sink-rabbit:1.0.0.BUILD-SNAPSHOT 41 | sink.hdfs-dataset=maven://org.springframework.cloud.stream.app:hdfs-dataset-sink-rabbit:1.0.0.BUILD-SNAPSHOT 42 | sink.jdbc=maven://org.springframework.cloud.stream.app:jdbc-sink-rabbit:1.0.0.BUILD-SNAPSHOT 43 | sink.log=maven://org.springframework.cloud.stream.app:log-sink-rabbit:1.0.0.BUILD-SNAPSHOT 44 | sink.rabbit=maven://org.springframework.cloud.stream.app:rabbit-sink-rabbit:1.0.0.BUILD-SNAPSHOT 45 | sink.redis-pubsub=maven://org.springframework.cloud.stream.app:redis-sink-rabbit:1.0.0.BUILD-SNAPSHOT 46 | sink.router=maven://org.springframework.cloud.stream.app:router-sink-rabbit:1.0.0.BUILD-SNAPSHOT 47 | sink.s3=maven://org.springframework.cloud.stream.app:s3-sink-rabbit:1.0.0.BUILD-SNAPSHOT 48 | sink.tcp=maven://org.springframework.cloud.stream.app:tcp-sink-rabbit:1.0.0.BUILD-SNAPSHOT 49 | sink.throughput=maven://org.springframework.cloud.stream.app:throughput-sink-rabbit:1.0.0.BUILD-SNAPSHOT 50 | sink.websocket=maven://org.springframework.cloud.stream.app:websocket-sink-rabbit:1.0.0.BUILD-SNAPSHOT 51 | -------------------------------------------------------------------------------- /dataflow/server/src/test/java/com/example/DemoApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import dataflow.DataFlowServer; 4 | import org.junit.Test; 5 | import org.junit.runner.RunWith; 6 | import org.springframework.boot.test.SpringApplicationConfiguration; 7 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 8 | 9 | @RunWith(SpringJUnit4ClassRunner.class) 10 | @SpringApplicationConfiguration(classes = DataFlowServer.class) 11 | public class DemoApplicationTests { 12 | 13 | @Test 14 | public void contextLoads() { 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dataflow/shell/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | dataflow 9 | 1.0.0-SNAPSHOT 10 | 11 | integration/dataflow/shell 12 | shell 13 | 14 | 15 | org.springframework.cloud 16 | spring-cloud-dataflow-shell 17 | 18 | 19 | org.springframework.boot 20 | spring-boot-starter-test 21 | test 22 | 23 | 24 | 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-maven-plugin 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /dataflow/shell/src/main/java/com/example/ShellApplication.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.dataflow.shell.EnableDataFlowShell; 6 | 7 | @EnableDataFlowShell 8 | @SpringBootApplication 9 | public class ShellApplication { 10 | 11 | public static void main(String[] args) { 12 | SpringApplication.run(ShellApplication.class, args); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /dataflow/shell/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.main.banner-mode=off -------------------------------------------------------------------------------- /dataflow/stream-example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | dataflow 9 | 1.0.0-SNAPSHOT 10 | 11 | integration/dataflow/stream-example 12 | stream-example 13 | 14 | 15 | org.springframework.integration 16 | spring-integration-java-dsl 17 | 18 | 19 | org.springframework.cloud 20 | spring-cloud-starter-stream-rabbit 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-test 25 | test 26 | 27 | 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-maven-plugin 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /dataflow/stream-example/src/main/java/stream/ProcessorStreamExample.java: -------------------------------------------------------------------------------- 1 | package stream; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.stream.annotation.EnableBinding; 6 | import org.springframework.cloud.stream.messaging.Processor; 7 | import org.springframework.integration.annotation.MessageEndpoint; 8 | import org.springframework.integration.annotation.ServiceActivator; 9 | import org.springframework.integration.support.MessageBuilder; 10 | import org.springframework.messaging.Message; 11 | 12 | @MessageEndpoint // <1> 13 | @EnableBinding(Processor.class) // <2> 14 | @SpringBootApplication 15 | public class ProcessorStreamExample { 16 | 17 | @ServiceActivator(inputChannel = Processor.INPUT, outputChannel = Processor.OUTPUT) 18 | public Message process(Message in) { 19 | return MessageBuilder 20 | .withPayload("{" + in.getPayload() + "}") // <3> 21 | .copyHeadersIfAbsent(in.getHeaders()) 22 | .build(); 23 | } 24 | 25 | public static void main(String[] args) { 26 | SpringApplication.run(ProcessorStreamExample.class, args); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /dataflow/stream-example/src/main/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloud-native-java/integration/23c827f4cde1eb29a15a73d87f3632e6b4dd63af/dataflow/stream-example/src/main/resources/application.properties -------------------------------------------------------------------------------- /dataflow/stream-example/src/test/java/com/example/Stream101ApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.example; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringRunner; 7 | import stream.ProcessorStreamExample; 8 | 9 | @RunWith(SpringRunner.class) 10 | @SpringBootTest(classes = ProcessorStreamExample.class) 11 | public class Stream101ApplicationTests { 12 | 13 | @Test 14 | public void contextLoads() { 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /dataflow/task-example/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | dataflow 9 | 1.0.0-SNAPSHOT 10 | 11 | integration/dataflow/task-example 12 | task-example 13 | 14 | 15 | com.h2database 16 | h2 17 | 18 | 19 | org.springframework.cloud 20 | spring-cloud-task-core 21 | 22 | 23 | org.springframework.cloud 24 | spring-cloud-task-batch 25 | 26 | 27 | org.springframework.boot 28 | spring-boot-starter-batch 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-configuration-processor 33 | true 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-starter-test 38 | test 39 | 40 | 41 | 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-maven-plugin 46 | 47 | 48 | 49 | 50 | -------------------------------------------------------------------------------- /dataflow/task-example/src/main/java/task/BatchTaskExample.java: -------------------------------------------------------------------------------- 1 | package task; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.batch.core.Job; 6 | import org.springframework.batch.core.Step; 7 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 8 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 9 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 10 | import org.springframework.batch.repeat.RepeatStatus; 11 | import org.springframework.boot.SpringApplication; 12 | import org.springframework.boot.autoconfigure.SpringBootApplication; 13 | import org.springframework.cloud.task.configuration.EnableTask; 14 | import org.springframework.context.annotation.Bean; 15 | 16 | @EnableTask // <1> 17 | @EnableBatchProcessing 18 | @SpringBootApplication 19 | public class BatchTaskExample { 20 | 21 | private Log log = LogFactory.getLog(getClass()); 22 | 23 | @Bean 24 | Step tasklet(StepBuilderFactory sbf, BatchTaskProperties btp) { 25 | return sbf 26 | .get("tasklet") 27 | .tasklet((contribution, chunkContext) -> { 28 | log.info("input = " + btp.getInput()); 29 | return RepeatStatus.FINISHED; 30 | }) 31 | .build(); 32 | } 33 | 34 | @Bean 35 | Job hello(JobBuilderFactory jbf) { 36 | Step step = this.tasklet(null, null); 37 | return jbf 38 | .get("batch-task") 39 | .start(step) 40 | .build(); 41 | } 42 | 43 | public static void main(String[] args) { 44 | SpringApplication.run(BatchTaskExample.class, args); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /dataflow/task-example/src/main/java/task/BatchTaskProperties.java: -------------------------------------------------------------------------------- 1 | package task; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | import org.springframework.core.io.Resource; 5 | import org.springframework.stereotype.Component; 6 | 7 | @Component 8 | @ConfigurationProperties(prefix = "simple-batch-task") 9 | public class BatchTaskProperties { 10 | private Resource input, output; 11 | 12 | public Resource getInput() { 13 | return input; 14 | } 15 | 16 | public void setInput(Resource input) { 17 | this.input = input; 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /dataflow/task-example/src/main/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloud-native-java/integration/23c827f4cde1eb29a15a73d87f3632e6b4dd63af/dataflow/task-example/src/main/resources/application.properties -------------------------------------------------------------------------------- /integration-it/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | integration 9 | 1.0.0-SNAPSHOT 10 | 11 | integration-it 12 | 13 | 14 | 15 | cnj 16 | it-support 17 | ${project.version} 18 | 19 | 20 | org.springframework.cloud 21 | spring-cloud-dataflow-rest-client 22 | 23 | 24 | org.springframework.retry 25 | spring-retry 26 | 27 | 28 | org.springframework.boot 29 | spring-boot-starter-web 30 | test 31 | 32 | 33 | org.springframework.boot 34 | spring-boot-starter-test 35 | test 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /integration-it/src/integration-test/java/integration/ActivitiIT.java: -------------------------------------------------------------------------------- 1 | package integration; 2 | 3 | import cnj.CloudFoundryService; 4 | import org.apache.commons.logging.Log; 5 | import org.apache.commons.logging.LogFactory; 6 | import org.junit.After; 7 | import org.junit.Assert; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.springframework.beans.factory.annotation.Autowired; 12 | import org.springframework.boot.autoconfigure.SpringBootApplication; 13 | import org.springframework.boot.test.context.SpringBootTest; 14 | import org.springframework.boot.web.client.RestTemplateBuilder; 15 | import org.springframework.core.ParameterizedTypeReference; 16 | import org.springframework.http.HttpMethod; 17 | import org.springframework.http.HttpStatus; 18 | import org.springframework.http.ResponseEntity; 19 | import org.springframework.retry.RetryCallback; 20 | import org.springframework.retry.support.RetryTemplate; 21 | import org.springframework.test.context.junit4.SpringRunner; 22 | import org.springframework.web.client.RestTemplate; 23 | 24 | import java.io.File; 25 | import java.util.Arrays; 26 | import java.util.Map; 27 | import java.util.stream.Stream; 28 | 29 | @SpringBootTest(classes = ActivitiIT.Config.class) 30 | @RunWith(SpringRunner.class) 31 | public class ActivitiIT { 32 | 33 | private final RestTemplate restTemplate = new RestTemplateBuilder() 34 | .basicAuthorization("operator", "operator") 35 | .build(); 36 | 37 | @Autowired 38 | private RetryTemplate retryTemplate; 39 | 40 | @Autowired 41 | private CloudFoundryService cloudFoundryService; 42 | 43 | private File leaderManifest, workerManifest; 44 | 45 | private Log log = LogFactory.getLog(getClass()); 46 | 47 | @Before 48 | public void before() throws Throwable { 49 | 50 | // deploy the activiti application, twice, to CF as a leader and a worker node. 51 | String mysql = "activiti-mysql", rmq = "activiti-rabbitmq", 52 | leader = "activiti-leader", worker = "activiti-worker"; 53 | 54 | File projectFolder = new File(new File("."), "../activiti-integration"); 55 | this.leaderManifest = new File(projectFolder, "manifest-leader.yml"); 56 | this.workerManifest = new File(projectFolder, "manifest-worker.yml"); 57 | 58 | log.debug("activiti folder: " 59 | + projectFolder.getAbsolutePath()); 60 | 61 | // reset 62 | Runnable apps = () -> Stream.of(leader, worker) 63 | .parallel().forEach(app -> this.cloudFoundryService.destroyApplicationIfExists(app)); 64 | 65 | Runnable routes = () -> this.cloudFoundryService.destroyOrphanedRoutes(); 66 | 67 | Runnable services = () -> Stream.of(mysql, rmq) 68 | .parallel().forEach(svc -> this.cloudFoundryService.destroyServiceIfExists(svc)); 69 | 70 | // apps must be reset first! 71 | Stream.of(apps, routes, services).forEach(Runnable::run); 72 | 73 | // create services required 74 | Stream.of("p-mysql 100mb " + mysql, "cloudamqp lemur " + rmq) 75 | .map(x -> x.split(" ")) 76 | .parallel() 77 | .forEach(t -> this.cloudFoundryService.createService(t[0], t[1], t[2])); 78 | 79 | // deploy 80 | Arrays.asList(leaderManifest, workerManifest) 81 | .parallelStream() 82 | .forEach(mf -> this.cloudFoundryService.pushApplicationUsingManifest(mf)); 83 | } 84 | 85 | @After 86 | public void after() throws Throwable { 87 | Stream.of(this.leaderManifest, workerManifest) 88 | .forEach(this.cloudFoundryService::destroyApplicationUsingManifest); 89 | } 90 | 91 | @Test 92 | public void testDistributedWorkflows() throws Throwable { 93 | 94 | String url = this.cloudFoundryService 95 | .urlForApplication("activiti-leader"); 96 | 97 | ResponseEntity> entity = 98 | restTemplate.exchange(url + "/start", 99 | HttpMethod.GET, 100 | null, 101 | new ParameterizedTypeReference>() { 102 | }); 103 | Assert.assertEquals(entity.getStatusCode(), HttpStatus.OK); 104 | String pid = entity.getBody().get("processInstanceId"); 105 | log.info("process instance ID: " + pid); 106 | 107 | RetryCallback rt = retryContext -> { 108 | String pidUrl = url + "/history/historic-process-instances/" + pid; 109 | log.info("calling the " + url + " endpoint to confirm the process ran and completed successfully."); 110 | Map instanceInformation = restTemplate.exchange( 111 | pidUrl, HttpMethod.GET, null, 112 | new ParameterizedTypeReference>() { 113 | }) 114 | .getBody(); 115 | 116 | if (instanceInformation.get("endTime") != null) { 117 | log.info("endTime was not null.."); 118 | return true; 119 | } 120 | log.info("endTime was null.."); 121 | throw new RuntimeException("the endTime attribute was null"); 122 | }; 123 | Boolean endTimeNull = retryTemplate.execute(rt, retryContext -> false); 124 | Assert.assertTrue("the endTime attribute should eventually return null", 125 | endTimeNull); 126 | } 127 | 128 | @SpringBootApplication 129 | public static class Config { 130 | } 131 | } -------------------------------------------------------------------------------- /integration-it/src/integration-test/java/integration/DataFlowIT.java: -------------------------------------------------------------------------------- 1 | package integration; 2 | 3 | import cnj.CloudFoundryService; 4 | import org.apache.commons.logging.Log; 5 | import org.apache.commons.logging.LogFactory; 6 | import org.cloudfoundry.operations.CloudFoundryOperations; 7 | import org.cloudfoundry.operations.applications.PushApplicationRequest; 8 | import org.cloudfoundry.operations.applications.SetEnvironmentVariableApplicationRequest; 9 | import org.cloudfoundry.operations.applications.StartApplicationRequest; 10 | import org.cloudfoundry.operations.services.BindServiceInstanceRequest; 11 | import org.junit.Assert; 12 | import org.junit.Before; 13 | import org.junit.Test; 14 | import org.junit.runner.RunWith; 15 | import org.springframework.beans.factory.annotation.Autowired; 16 | import org.springframework.boot.autoconfigure.SpringBootApplication; 17 | import org.springframework.boot.test.context.SpringBootTest; 18 | import org.springframework.cloud.dataflow.rest.client.DataFlowTemplate; 19 | import org.springframework.cloud.dataflow.rest.client.TaskOperations; 20 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 21 | import org.springframework.web.client.RestTemplate; 22 | 23 | import java.io.File; 24 | import java.io.InputStream; 25 | import java.net.URI; 26 | import java.nio.file.Path; 27 | import java.nio.file.StandardCopyOption; 28 | import java.time.Duration; 29 | import java.util.*; 30 | import java.util.concurrent.ConcurrentHashMap; 31 | import java.util.stream.Stream; 32 | import java.util.stream.StreamSupport; 33 | 34 | /** 35 | * @author Josh Long 36 | */ 37 | @RunWith(SpringJUnit4ClassRunner.class) 38 | @SpringBootTest(classes = DataFlowIT.Config.class) 39 | public class DataFlowIT { 40 | 41 | private Log log = LogFactory.getLog(getClass()); 42 | @Autowired 43 | private CloudFoundryOperations cloudFoundryOperations; 44 | private File serviceDefinitionsManifest; 45 | @Autowired 46 | private CloudFoundryService cloudFoundryService; 47 | private String appName = "cfdf"; 48 | 49 | @Before 50 | public void deploy() throws Throwable { 51 | deployServiceDefinitions(); 52 | deployDataFlowServer(); 53 | } 54 | 55 | @Test 56 | public void deployTasksAndStreams() throws Exception { 57 | DataFlowTemplate df = this.dataFlowTemplate(this.appName); 58 | appDefinitions() 59 | .parallelStream() 60 | .forEach(u -> { 61 | log.info("importing " + u); 62 | df.appRegistryOperations().importFromResource(u, true); 63 | log.info("imported " + u); 64 | }); 65 | 66 | Arrays.asList(deployStreams(df), deployTasks(df)) 67 | .parallelStream() 68 | .map(r -> r) 69 | .forEach(Runnable::run); 70 | log.info("deployed tasks and streams."); 71 | } 72 | 73 | private void deployServiceDefinitions() { 74 | File projectFolder = new File(new File("."), "../dataflow/server-definitions"); 75 | serviceDefinitionsManifest = new File(projectFolder, "manifest.yml"); 76 | this.cloudFoundryService.pushApplicationUsingManifest(serviceDefinitionsManifest); 77 | } 78 | 79 | private void deployDataFlowServer() throws Throwable { 80 | 81 | // deploy the DF server 82 | String serverRedis = "cfdf-redis", serverMysql = "cfdf-mysql", serverRabbit = "cfdf-rabbit"; 83 | Stream.of("rediscloud 100mb " + serverRedis, 84 | "cloudamqp lemur " + serverRabbit, 85 | "p-mysql 100mb " + serverMysql) 86 | .parallel() 87 | .map(x -> x.split(" ")) 88 | .forEach(tpl -> this.cloudFoundryService.createServiceIfMissing(tpl[0], tpl[1], tpl[2])); 89 | 90 | String urlForServerJarDistribution = this.serverJarUrl(); 91 | File cfdfJar = new File(System.getProperty("user.home"), "cfdf.jar"); 92 | Assert.assertTrue(cfdfJar.getParentFile().exists() || cfdfJar.getParentFile().mkdirs()); 93 | 94 | Path targetFile = cfdfJar.toPath(); 95 | if (!cfdfJar.exists()) { 96 | URI uri = URI.create(urlForServerJarDistribution); 97 | try (InputStream inputStream = uri.toURL().openStream()) { 98 | java.nio.file.Files.copy(inputStream, targetFile, StandardCopyOption.REPLACE_EXISTING); 99 | } 100 | this.log.info("downloaded Data Flow server to " + targetFile.toFile().getAbsolutePath() + "."); 101 | } 102 | log.info("Data Flow Server jar lives at " + cfdfJar.getAbsolutePath()); 103 | 104 | int twoG = 1024 * 2; 105 | this.cloudFoundryOperations.applications() 106 | .push(PushApplicationRequest 107 | .builder() 108 | .application(targetFile) 109 | .buildpack("https://github.com/cloudfoundry/java-buildpack.git") 110 | .noStart(true) 111 | .name(appName) 112 | .host("cfdf-" + UUID.randomUUID().toString()) 113 | .memory(twoG) 114 | .diskQuota(twoG) 115 | .build()) 116 | .block(); 117 | log.info("pushed (but didn't start) the Data Flow server"); 118 | 119 | Map env = new ConcurrentHashMap<>(); 120 | 121 | // CF authentication 122 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_ORG", System.getenv("CF_ORG")); 123 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_SPACE", System.getenv("CF_SPACE")); 124 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_USERNAME", System.getenv("CF_USER")); 125 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_PASSWORD", System.getenv("CF_PASSWORD")); 126 | 127 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_SERVICES", serverRabbit); 128 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_TASK_SERVICES", serverMysql); 129 | 130 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_SKIP_SSL_VALIDATION", "false"); 131 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_URL", "https://api.run.pivotal.io"); 132 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_DOMAIN", "cfapps.io"); 133 | 134 | env.put("MAVEN_REMOTE_REPOSITORIES_LR_URL", "https://cloudnativejava.artifactoryonline.com/cloudnativejava/libs-release"); 135 | env.put("MAVEN_REMOTE_REPOSITORIES_LS_URL", "https://cloudnativejava.artifactoryonline.com/cloudnativejava/libs-snapshot"); 136 | env.put("MAVEN_REMOTE_REPOSITORIES_PR_URL", "https://cloudnativejava.artifactoryonline.com/cloudnativejava/plugins-release"); 137 | env.put("MAVEN_REMOTE_REPOSITORIES_PS_URL", "https://cloudnativejava.artifactoryonline.com/cloudnativejava/plugins-snapshot"); 138 | 139 | env.put("SPRING_CLOUD_DEPLOYER_CLOUDFOUNDRY_STREAM_INSTANCES", "1"); 140 | 141 | env.entrySet() 142 | .forEach(e -> { 143 | this.cloudFoundryOperations 144 | .applications() 145 | .setEnvironmentVariable( 146 | SetEnvironmentVariableApplicationRequest 147 | .builder() 148 | .name(appName) 149 | .variableName(e.getKey()) 150 | .variableValue(e.getValue()) 151 | .build()) 152 | .block(); 153 | 154 | log.info("set environment variable for " + appName + ": " + e.getKey() + '=' + e.getValue()); 155 | }); 156 | 157 | log.info("set all " + env.size() + " environment variables."); 158 | 159 | // bind the relevant services to DF 160 | Stream.of(serverMysql, serverRedis) 161 | .forEach(svc -> { 162 | 163 | this.cloudFoundryOperations.services() 164 | .bind(BindServiceInstanceRequest 165 | .builder() 166 | .applicationName(appName) 167 | .serviceInstanceName(svc) 168 | .build()) 169 | .block(); 170 | log.info("binding " + svc + " to " + appName); 171 | }); 172 | 173 | // start 174 | this.cloudFoundryOperations 175 | .applications() 176 | .start(StartApplicationRequest 177 | .builder() 178 | .stagingTimeout(Duration.ofMinutes(10)) 179 | .startupTimeout(Duration.ofMinutes(10)) 180 | .name(appName) 181 | .build()) 182 | .block(); 183 | 184 | log.info("started the Spring Cloud Data Flow Cloud Foundry server."); 185 | } 186 | 187 | private Runnable deployStreams(DataFlowTemplate df) { 188 | return () -> { 189 | Map streams = new HashMap<>(); 190 | streams.put("ttl", "time | brackets | log"); 191 | 192 | log.info("going to deploy " + streams.size() + " new stream(s)."); 193 | streams.entrySet() 194 | .parallelStream() 195 | .forEach(stream -> { 196 | String streamName = stream.getKey(); 197 | 198 | StreamSupport.stream(Spliterators.spliteratorUnknownSize(df.streamOperations().list().iterator(), 199 | Spliterator.ORDERED), false) 200 | .filter(sdr -> sdr.getName().equals(streamName)).forEach(tdr -> { 201 | log.info("deploying stream " + streamName); 202 | df.streamOperations().destroy(streamName); 203 | }); 204 | 205 | df.streamOperations() 206 | .createStream(streamName, stream.getValue(), true); 207 | }); 208 | }; 209 | } 210 | 211 | private Runnable deployTasks(DataFlowTemplate df) { 212 | return () -> { 213 | Map tasks = new HashMap<>(); 214 | tasks.put("my-simple-task", "simple-task"); 215 | 216 | log.info("going to deploy " + tasks.size() + " new task(s)."); 217 | tasks.entrySet() 218 | .parallelStream() 219 | .forEach(task -> { 220 | 221 | String taskName = task.getKey(); 222 | 223 | StreamSupport.stream(Spliterators.spliteratorUnknownSize(df.taskOperations().list().iterator(), 224 | Spliterator.ORDERED), false) 225 | .filter(tdr -> tdr.getName().equals(taskName)).forEach(tdr -> { 226 | log.info("destroying task " + taskName); 227 | df.taskOperations().destroy(taskName); 228 | }); 229 | 230 | log.info("deploying task " + taskName); 231 | TaskOperations to = df.taskOperations(); 232 | to.create(taskName, task.getValue()); 233 | to.launch(taskName, 234 | Collections.emptyMap(), 235 | Collections.singletonList(System.currentTimeMillis() + "")); 236 | }); 237 | }; 238 | } 239 | 240 | private DataFlowTemplate dataFlowTemplate(String cfDfServerName) throws Exception { 241 | String urlForApplication = this.cloudFoundryService.urlForApplication(cfDfServerName); 242 | log.info("attempting to create a DataFlowTemplate using the following API endpoint " + urlForApplication); 243 | return Optional.ofNullable(urlForApplication) 244 | .map(u -> new DataFlowTemplate(URI.create(u), new RestTemplate())) 245 | .orElseThrow(() -> new RuntimeException( 246 | "can't find a URI for the Spring Cloud Data Flow server!")); 247 | } 248 | 249 | private String serverJarUrl() { 250 | String serverJarVersion = "1.1.0.BUILD-SNAPSHOT"; 251 | String serverJarUrl = "http://repo.spring.io/${server_jar_url_prefix}/org/springframework/cloud/" + 252 | "spring-cloud-dataflow-server-cloudfoundry/${server_jar_version}" + 253 | "/spring-cloud-dataflow-server-cloudfoundry-${server_jar_version}.jar"; 254 | String prefix = serverJarVersion.toUpperCase().contains("RELEASE") ? "release" : "snapshot"; 255 | return serverJarUrl 256 | .replace("${server_jar_url_prefix}", prefix) 257 | .replace("${server_jar_version}", serverJarVersion); 258 | } 259 | 260 | private List appDefinitions() { 261 | List apps = new ArrayList<>(); 262 | apps.add("http://repo.spring.io/libs-release-local/org/springframework/cloud/task/app/spring-cloud-task-app-descriptor/Addison.RELEASE/spring-cloud-task-app-descriptor-Addison.RELEASE.task-apps-maven"); 263 | apps.add("http://repo.spring.io/libs-release/org/springframework/cloud/stream/app/spring-cloud-stream-app-descriptor/Avogadro.SR1/spring-cloud-stream-app-descriptor-Avogadro.SR1.stream-apps-rabbit-maven"); 264 | 265 | Optional.ofNullable(this.cloudFoundryService.urlForApplication("server-definitions")) 266 | .map(x -> x + "/dataflow-example-apps.properties") 267 | .ifPresent(apps::add); 268 | 269 | apps.forEach(x -> log.info("registering: " + x)); 270 | return apps; 271 | } 272 | 273 | @SpringBootApplication 274 | public static class Config { 275 | } 276 | } -------------------------------------------------------------------------------- /integration-it/src/integration-test/java/integration/RemotePartitioningIT.java: -------------------------------------------------------------------------------- 1 | package integration; 2 | 3 | import cnj.CloudFoundryService; 4 | import org.junit.After; 5 | import org.junit.Assert; 6 | import org.junit.Before; 7 | import org.junit.Test; 8 | import org.junit.runner.RunWith; 9 | import org.springframework.beans.factory.annotation.Autowired; 10 | import org.springframework.boot.autoconfigure.SpringBootApplication; 11 | import org.springframework.boot.test.context.SpringBootTest; 12 | import org.springframework.context.annotation.Bean; 13 | import org.springframework.core.ParameterizedTypeReference; 14 | import org.springframework.http.ResponseEntity; 15 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 16 | import org.springframework.web.client.RestTemplate; 17 | 18 | import java.io.File; 19 | import java.util.Map; 20 | import java.util.Objects; 21 | import java.util.stream.Stream; 22 | 23 | import static org.junit.Assert.assertTrue; 24 | import static org.springframework.http.HttpMethod.GET; 25 | 26 | /** 27 | * @author Josh Long 28 | */ 29 | @RunWith(SpringJUnit4ClassRunner.class) 30 | @SpringBootTest(classes = RemotePartitioningIT.Config.class) 31 | public class RemotePartitioningIT { 32 | 33 | @Autowired 34 | private RestTemplate restTemplate; 35 | 36 | @Autowired 37 | private CloudFoundryService cloudFoundryService; 38 | 39 | private String mysql = "batch-mysql", rmq = "batch-rmq"; 40 | 41 | private File leader, worker; 42 | 43 | @Before 44 | public void before() throws Throwable { 45 | Stream.of("p-mysql 100mb " + mysql, "cloudamqp lemur " + rmq) 46 | .map(x -> x.split(" ")) 47 | .forEach(t -> this.cloudFoundryService.createServiceIfMissing(t[0], t[1], t[2])); 48 | File projectFolder = new File(new File("."), "../remote-partitioning"); 49 | this.leader = new File(projectFolder, "manifest-leader.yml"); 50 | this.worker = new File(projectFolder, "manifest-worker.yml"); 51 | Assert.assertTrue("the manifest files " + this.leader.getAbsolutePath() + 52 | "and " + this.worker.getAbsolutePath() + " must exist", 53 | leader.exists() && worker.exists()); 54 | Stream.of(this.leader, this.worker).parallel() 55 | .forEach(f -> this.cloudFoundryService.pushApplicationUsingManifest(f)); 56 | } 57 | 58 | @After 59 | public void after() throws Throwable { 60 | Stream.of(this.leader, this.worker) 61 | .filter(Objects::nonNull) 62 | .forEach(this.cloudFoundryService::destroyApplicationUsingManifest); 63 | } 64 | 65 | @Test 66 | public void partitionedJob() { 67 | 68 | String leaderUrl = cloudFoundryService.urlForApplication("partition-leader"); 69 | 70 | String migrationJobUrl = leaderUrl + "/migrate"; 71 | ResponseEntity> jsonResponse = 72 | this.restTemplate.exchange( 73 | migrationJobUrl, GET, null, 74 | new ParameterizedTypeReference>() { 75 | }); 76 | assertTrue(jsonResponse.getBody().get("exitCode") 77 | .equalsIgnoreCase("completed")); 78 | assertTrue(jsonResponse.getBody().get("running") 79 | .equalsIgnoreCase("false")); 80 | assertTrue("the job should have completed successfully.", 81 | jsonResponse.getStatusCode().is2xxSuccessful()); 82 | Map status = 83 | this.restTemplate 84 | .exchange(leaderUrl + "/status", GET, null, 85 | new ParameterizedTypeReference>() { 86 | }) 87 | .getBody(); 88 | 89 | Assert.assertEquals("there should be an identical number" + 90 | " of records in the source and destination table", 91 | status.get("people.count"), status.get("new_people.count")); 92 | } 93 | 94 | @SpringBootApplication 95 | public static class Config { 96 | 97 | @Bean 98 | public RestTemplate restTemplate() { 99 | return new RestTemplate(); 100 | } 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /integration-it/src/integration-test/java/integration/ShakyServiceIT.java: -------------------------------------------------------------------------------- 1 | package integration; 2 | 3 | import org.junit.runner.RunWith; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.boot.test.context.SpringBootTest; 6 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 7 | 8 | /** 9 | * @author Josh Long 10 | */ 11 | @RunWith(SpringJUnit4ClassRunner.class) 12 | @SpringBootTest(classes = ShakyServiceIT.Config.class) 13 | public class ShakyServiceIT { 14 | 15 | @SpringBootApplication 16 | public static class Config {} 17 | 18 | 19 | 20 | } 21 | -------------------------------------------------------------------------------- /integration-it/src/integration-test/resources/application.properties: -------------------------------------------------------------------------------- 1 | logging.level.cnj=DEBUG 2 | logging.level.integration=DEBUG 3 | logging.level.org.cloudfoundry.operations=DEBUG 4 | spring.main.web-environment=false -------------------------------------------------------------------------------- /integration-it/src/main/java/demo/Main.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | public class Main { 4 | public static void main(String[] a) {} 5 | } 6 | -------------------------------------------------------------------------------- /integration-it/src/test/java/demo/RemotePartitionIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | /* 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.test.context.SpringBootTest; 7 | import org.springframework.core.ParameterizedTypeReference; 8 | import org.springframework.http.ResponseEntity; 9 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 10 | import org.springframework.web.client.RestTemplate; 11 | 12 | import java.util.Map; 13 | import java.util.Optional; 14 | 15 | import static org.junit.Assert.assertTrue; 16 | import static org.springframework.http.HttpMethod.GET; 17 | 18 | @RunWith(SpringJUnit4ClassRunner.class) 19 | @SpringBootTest(classes = ClientConfiguration.class) 20 | public class RemotePartitionIntegrationTest { 21 | 22 | @Autowired 23 | private CloudFoundryHelper helper; 24 | 25 | private final RestTemplate restTemplate = new RestTemplate(); 26 | 27 | @Test 28 | public void remotePartitionBatchJob() { 29 | Optional optional = this.helper.uriFor("partition-leader") 30 | .map(pm -> { 31 | String migrationJobUrl = pm + "/migrate"; 32 | ResponseEntity> jsonResponse = 33 | this.restTemplate.exchange( 34 | migrationJobUrl, GET, null, 35 | new ParameterizedTypeReference>() { 36 | }); 37 | assertTrue(jsonResponse.getBody().get("exitCode") 38 | .equalsIgnoreCase("completed")); 39 | assertTrue(jsonResponse.getBody().get("running") 40 | .equalsIgnoreCase("false")); 41 | assertTrue("the job should have completed successfully.", 42 | jsonResponse.getStatusCode().is2xxSuccessful()); 43 | Map status = 44 | this.restTemplate 45 | .exchange(pm + "/status", GET, null, 46 | new ParameterizedTypeReference>() { 47 | }) 48 | .getBody(); 49 | return status.get("people.count") 50 | .equals(status.get("new_people.count")); 51 | }); 52 | assertTrue(optional.orElse(false)); 53 | } 54 | } */ 55 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | integration 8 | pom 9 | 10 | cnj 11 | parent 12 | 1.0.0-SNAPSHOT 13 | 14 | 15 | 16 | 17 | activiti 18 | activiti-integration 19 | remote-partitioning 20 | shaky-service-to-service-calls 21 | dataflow 22 | stream 23 | batch-and-integration 24 | batch 25 | task 26 | integration-it 27 | 28 | 29 | 30 | 31 | 32 | org.jfrog.buildinfo 33 | artifactory-maven-plugin 34 | 2.4.0 35 | false 36 | 37 | 38 | build-info 39 | 40 | publish 41 | 42 | 43 | 44 | {{TRAVIS_COMMIT}} 45 | 46 | 47 | 48 | https://cloudnativejava.artifactoryonline.com/cloudnativejava 49 | 50 | ${env.ARTIFACTORY_USERNAME} 51 | ${env.ARTIFACTORY_PASSWORD} 52 | libs-release-local 53 | libs-snapshot-local 54 | 55 | 56 | Travis CI 57 | {{TRAVIS_BUILD_NUMBER}} 58 | 59 | http://travis-ci.org/{{TRAVIS_REPO_SLUG}}/builds/{{TRAVIS_BUILD_ID}} 60 | 61 | {{USER}} 62 | {{TRAVIS_COMMIT}} 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | false 75 | 76 | central 77 | libs-release 78 | 79 | https://cloudnativejava.artifactoryonline.com/cloudnativejava/libs-release 80 | 81 | 82 | 83 | 84 | snapshots 85 | libs-snapshot 86 | 87 | https://cloudnativejava.artifactoryonline.com/cloudnativejava/libs-snapshot 88 | 89 | 90 | 91 | 92 | 93 | 94 | false 95 | 96 | central 97 | plugins-release 98 | 99 | https://cloudnativejava.artifactoryonline.com/cloudnativejava/plugins-release 100 | 101 | 102 | 103 | 104 | snapshots 105 | plugins-snapshot 106 | 107 | https://cloudnativejava.artifactoryonline.com/cloudnativejava/plugins-snapshot 108 | 109 | 110 | 111 | 112 | 113 | 114 | -------------------------------------------------------------------------------- /remote-partitioning/ddl/schema-init.sql: -------------------------------------------------------------------------------- 1 | -- BATCH METADATA TABLES 2 | -- SET foreign_key_checks = 0; 3 | DROP TABLE IF EXISTS "BATCH_JOB_EXECUTION" ; 4 | DROP TABLE IF EXISTS "BATCH_JOB_EXECUTION_CONTEXT" ; 5 | DROP TABLE IF EXISTS "BATCH_JOB_EXECUTION_PARAMS" ; 6 | DROP TABLE IF EXISTS "BATCH_JOB_EXECUTION_SEQ" ; 7 | DROP TABLE IF EXISTS "BATCH_JOB_INSTANCE" ; 8 | DROP TABLE IF EXISTS "BATCH_JOB_SEQ" ; 9 | DROP TABLE IF EXISTS "BATCH_STEP_EXECUTION" ; 10 | DROP TABLE IF EXISTS "BATCH_STEP_EXECUTION_CONTEXT" ; 11 | DROP TABLE IF EXISTS "BATCH_STEP_EXECUTION_SEQ" ; 12 | -- SET foreign_key_checks = 1; 13 | 14 | -- CUSTOMERS 15 | 16 | DROP TABLE IF EXISTS "customer"; 17 | DROP TABLE IF EXISTS "new_customer"; 18 | 19 | CREATE TABLE "customer" ( 20 | id SERIAL PRIMARY KEY, 21 | firstName varchar(255) default NULL, 22 | lastName varchar(255) default NULL, 23 | birthdate varchar(255) 24 | ); 25 | 26 | CREATE TABLE "new_customer" ( 27 | id SERIAL PRIMARY KEY, 28 | firstName varchar(255) default NULL, 29 | lastName varchar(255) default NULL, 30 | birthdate varchar(255) 31 | ); 32 | -------------------------------------------------------------------------------- /remote-partitioning/manifest-leader.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: partition-leader 4 | instances: 1 5 | memory: 1GB 6 | host: partition-leader-${random-word} 7 | path: target/remote-partitioning.jar 8 | buildpack: https://github.com/cloudfoundry/java-buildpack.git 9 | services: 10 | - batch-mysql 11 | - batch-rmq 12 | env: 13 | DEBUG: "true" 14 | SERVER_PORT: 80 -------------------------------------------------------------------------------- /remote-partitioning/manifest-worker.yml: -------------------------------------------------------------------------------- 1 | --- 2 | applications: 3 | - name: partition-worker 4 | instances: 4 5 | memory: 1GB 6 | host: partition-worker-${random-word} 7 | path: target/remote-partitioning.jar 8 | buildpack: https://github.com/cloudfoundry/java-buildpack.git 9 | services: 10 | - batch-mysql 11 | - batch-rmq 12 | env: 13 | DEBUG: "true" 14 | SPRING_PROFILES_ACTIVE: worker, cloud 15 | SERVER_PORT: 80 -------------------------------------------------------------------------------- /remote-partitioning/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | integration 9 | 1.0.0-SNAPSHOT 10 | 11 | remote-partitioning 12 | integration/remote-partitioning 13 | 14 | 15 | org.projectlombok 16 | lombok 17 | true 18 | 19 | 20 | org.springframework.batch 21 | spring-batch-integration 22 | 23 | 24 | org.springframework.boot 25 | spring-boot-starter-jdbc 26 | 27 | 28 | mysql 29 | mysql-connector-java 30 | 31 | 32 | org.springframework.boot 33 | spring-boot-starter-web 34 | 35 | 36 | org.springframework.boot 37 | spring-boot-starter-actuator 38 | 39 | 40 | org.springframework.integration 41 | spring-integration-redis 42 | 43 | 44 | org.springframework.cloud 45 | spring-cloud-starter-stream-rabbit 46 | 47 | 48 | org.springframework.boot 49 | spring-boot-starter-batch 50 | 51 | 52 | org.springframework.boot 53 | spring-boot-starter-integration 54 | 55 | 56 | org.springframework.boot 57 | spring-boot-starter-test 58 | test 59 | 60 | 61 | 62 | 63 | 64 | org.springframework.boot 65 | spring-boot-maven-plugin 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/IdRangePartitioner.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.partition.support.Partitioner; 4 | import org.springframework.batch.item.ExecutionContext; 5 | import org.springframework.jdbc.core.JdbcOperations; 6 | 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | class IdRangePartitioner implements Partitioner { 11 | private final JdbcOperations jdbcTemplate; 12 | private final String column, table; 13 | 14 | IdRangePartitioner(JdbcOperations jdbcTemplate, String table, String column) { 15 | this.jdbcTemplate = jdbcTemplate; 16 | this.column = column; 17 | this.table = table; 18 | } 19 | 20 | @Override 21 | public Map partition(int gridSize) { 22 | Map result = new HashMap<>(); 23 | int min = jdbcTemplate.queryForObject("SELECT MIN(" + column 24 | + ") from " + table, Integer.class); 25 | int max = jdbcTemplate.queryForObject("SELECT MAX(" + column 26 | + ") from " + table, Integer.class); 27 | int targetSize = (max - min) / gridSize + 1; 28 | int number = 0; 29 | int start = min; 30 | int end = start + targetSize - 1; 31 | 32 | while (start <= max) { 33 | ExecutionContext value = new ExecutionContext(); 34 | result.put("partition" + number, value); 35 | if (end >= max) { 36 | end = max; 37 | } 38 | value.putInt("minValue", start); 39 | value.putInt("maxValue", end); 40 | start += targetSize; 41 | end += targetSize; 42 | number++; 43 | } 44 | 45 | return result; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/JobConfiguration.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.Job; 4 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 5 | import org.springframework.batch.core.launch.support.RunIdIncrementer; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.context.annotation.Profile; 9 | 10 | @Configuration 11 | @Profile(Profiles.LEADER_PROFILE) // <1> 12 | class JobConfiguration { 13 | 14 | @Bean 15 | Job job(JobBuilderFactory jbf, 16 | LeaderStepConfiguration lsc) { 17 | return jbf 18 | .get("job") 19 | .incrementer(new RunIdIncrementer()) 20 | .start(lsc.stagingStep(null, null)) // <2> 21 | .next(lsc.partitionStep(null, null, null, null)) // <3> 22 | .build(); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/LeaderChannels.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.cloud.stream.annotation.EnableBinding; 5 | import org.springframework.cloud.stream.annotation.Output; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.integration.channel.QueueChannel; 9 | import org.springframework.integration.dsl.channel.MessageChannels; 10 | import org.springframework.messaging.MessageChannel; 11 | 12 | @Configuration 13 | @EnableBinding(LeaderChannels.Leader.class) 14 | class LeaderChannels { 15 | 16 | private final Leader leader; 17 | 18 | @Autowired 19 | public LeaderChannels(Leader leader) { 20 | this.leader = leader; 21 | } 22 | 23 | @Bean 24 | public QueueChannel leaderRepliesAggregatedChannel() { 25 | return MessageChannels.queue().get(); 26 | } 27 | 28 | public MessageChannel leaderRequestsChannel() { 29 | return leader.leaderRequests(); 30 | } 31 | 32 | public interface Leader { 33 | 34 | @Output 35 | MessageChannel leaderRequests(); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/LeaderStepConfiguration.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.Step; 4 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 5 | import org.springframework.batch.core.explore.JobExplorer; 6 | import org.springframework.batch.core.partition.PartitionHandler; 7 | import org.springframework.batch.core.partition.support.Partitioner; 8 | import org.springframework.batch.integration.partition.MessageChannelPartitionHandler; 9 | import org.springframework.batch.repeat.RepeatStatus; 10 | import org.springframework.beans.factory.annotation.Value; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | import org.springframework.integration.core.MessagingTemplate; 14 | import org.springframework.jdbc.core.JdbcOperations; 15 | import org.springframework.jdbc.core.JdbcTemplate; 16 | 17 | @Configuration 18 | class LeaderStepConfiguration { 19 | 20 | // <1> 21 | @Bean 22 | Step stagingStep(StepBuilderFactory sbf, 23 | JdbcTemplate jdbc) { 24 | return sbf 25 | .get("staging") 26 | .tasklet((contribution, chunkContext) -> { 27 | jdbc.execute("truncate NEW_PEOPLE"); 28 | return RepeatStatus.FINISHED; 29 | }) 30 | .build(); 31 | } 32 | 33 | // <2> 34 | @Bean 35 | Step partitionStep(StepBuilderFactory sbf, 36 | Partitioner p, 37 | PartitionHandler ph, 38 | WorkerStepConfiguration wsc) { 39 | Step workerStep = wsc.workerStep(null); 40 | return sbf.get("partitionStep") 41 | .partitioner(workerStep.getName(), p) 42 | .partitionHandler(ph) 43 | .build(); 44 | } 45 | 46 | // <3> 47 | @Bean 48 | MessageChannelPartitionHandler partitionHandler( 49 | @Value("${partition.grid-size:4}") int gridSize, 50 | MessagingTemplate messagingTemplate, 51 | JobExplorer jobExplorer) { 52 | MessageChannelPartitionHandler partitionHandler = new MessageChannelPartitionHandler(); 53 | partitionHandler.setMessagingOperations(messagingTemplate); 54 | partitionHandler.setJobExplorer(jobExplorer); 55 | partitionHandler.setStepName("workerStep"); 56 | partitionHandler.setGridSize(gridSize); 57 | return partitionHandler; 58 | } 59 | 60 | // <4> 61 | @Bean 62 | MessagingTemplate messagingTemplate(LeaderChannels channels) { 63 | return new MessagingTemplate(channels.leaderRequestsChannel()); 64 | } 65 | 66 | // <5> 67 | @Bean 68 | Partitioner partitioner(JdbcOperations jdbcTemplate, 69 | @Value("${partition.table:PEOPLE}") String table, 70 | @Value("${partition.column:ID}") String column) { 71 | return new IdRangePartitioner(jdbcTemplate, table, column); 72 | } 73 | } -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/PartitionApplication.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.integration.annotation.IntegrationComponentScan; 8 | import org.springframework.integration.dsl.core.Pollers; 9 | import org.springframework.integration.scheduling.PollerMetadata; 10 | import org.springframework.jdbc.core.JdbcTemplate; 11 | 12 | import javax.sql.DataSource; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | @EnableBatchProcessing // <1> 16 | @IntegrationComponentScan 17 | @SpringBootApplication 18 | public class PartitionApplication { 19 | 20 | public static void main(String args[]) { 21 | SpringApplication.run(PartitionApplication.class, args); 22 | } 23 | 24 | // <2> 25 | @Bean(name = PollerMetadata.DEFAULT_POLLER) 26 | PollerMetadata defaultPoller() { 27 | return Pollers.fixedRate(10, TimeUnit.SECONDS).get(); 28 | } 29 | 30 | @Bean 31 | JdbcTemplate jdbcTemplate(DataSource dataSource) { 32 | return new JdbcTemplate(dataSource); 33 | } 34 | } -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/PartitionRestController.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.Job; 4 | import org.springframework.batch.core.JobExecution; 5 | import org.springframework.batch.core.JobExecutionException; 6 | import org.springframework.batch.core.JobParametersBuilder; 7 | import org.springframework.batch.core.launch.JobLauncher; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.context.annotation.Profile; 10 | import org.springframework.http.ResponseEntity; 11 | import org.springframework.jdbc.core.JdbcOperations; 12 | import org.springframework.web.bind.annotation.GetMapping; 13 | import org.springframework.web.bind.annotation.RequestMapping; 14 | import org.springframework.web.bind.annotation.RequestMethod; 15 | import org.springframework.web.bind.annotation.RestController; 16 | 17 | import java.util.Date; 18 | import java.util.HashMap; 19 | import java.util.Map; 20 | 21 | @RestController 22 | @Profile(Profiles.LEADER_PROFILE) 23 | class PartitionRestController { 24 | 25 | private final Job job; 26 | private final JobLauncher jobLauncher; 27 | private final JdbcOperations jdbcOperations; 28 | 29 | @Autowired 30 | public PartitionRestController( 31 | JdbcOperations jdbcOperations, 32 | JobConfiguration jobConfiguration, 33 | JobLauncher jobLauncher) throws Exception { 34 | this.jobLauncher = jobLauncher; 35 | this.jdbcOperations = jdbcOperations; 36 | this.job = jobConfiguration.job(null, null); 37 | } 38 | 39 | @RequestMapping(value = "/migrate", 40 | method = {RequestMethod.POST, RequestMethod.GET}) 41 | ResponseEntity start() throws JobExecutionException { 42 | JobExecution execution = this.jobLauncher.run(this.job, 43 | new JobParametersBuilder() 44 | .addDate("date", new Date()) 45 | .toJobParameters()); 46 | return ResponseEntity.ok(execution.getExitStatus()); 47 | } 48 | 49 | @GetMapping("/status") 50 | ResponseEntity status() { 51 | Map status = new HashMap<>(); 52 | status.put("people.count", this.jdbcOperations.queryForObject("select count(*) from PEOPLE", Number.class)); 53 | status.put("new_people.count", this.jdbcOperations.queryForObject("select count(*) from NEW_PEOPLE", Number.class)); 54 | return ResponseEntity.ok(status); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/Person.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | @Data 8 | @NoArgsConstructor 9 | @AllArgsConstructor 10 | public class Person { 11 | private int id; 12 | private int age; 13 | private String firstName, email; 14 | } 15 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/Profiles.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | public class Profiles { 4 | public static final String WORKER_PROFILE = "worker"; // <1> 5 | public static final String LEADER_PROFILE = "!" + WORKER_PROFILE; // <2> 6 | } 7 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/WorkerChannels.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.cloud.stream.annotation.EnableBinding; 5 | import org.springframework.cloud.stream.annotation.Input; 6 | import org.springframework.cloud.stream.annotation.Output; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.context.annotation.Profile; 9 | import org.springframework.messaging.MessageChannel; 10 | 11 | @Configuration 12 | @EnableBinding(WorkerChannels.Worker.class) 13 | @Profile(Profiles.WORKER_PROFILE) 14 | class WorkerChannels { 15 | 16 | private final Worker worker; 17 | 18 | @Autowired 19 | public WorkerChannels(Worker worker) { 20 | this.worker = worker; 21 | } 22 | 23 | MessageChannel workerRequestsChannels() { 24 | return this.worker.workerRequests(); 25 | } 26 | 27 | MessageChannel workerRepliesChannels() { 28 | return this.worker.workerReplies(); 29 | } 30 | 31 | public interface Worker { 32 | 33 | @Input 34 | MessageChannel workerRequests(); 35 | 36 | @Output 37 | MessageChannel workerReplies(); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/WorkerConfiguration.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.explore.JobExplorer; 4 | import org.springframework.batch.core.step.StepLocator; 5 | import org.springframework.batch.integration.partition.BeanFactoryStepLocator; 6 | import org.springframework.batch.integration.partition.StepExecutionRequest; 7 | import org.springframework.batch.integration.partition.StepExecutionRequestHandler; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.context.annotation.Profile; 11 | import org.springframework.integration.dsl.IntegrationFlow; 12 | import org.springframework.integration.dsl.IntegrationFlows; 13 | import org.springframework.integration.dsl.support.GenericHandler; 14 | import org.springframework.messaging.MessageChannel; 15 | 16 | @Configuration 17 | @Profile(Profiles.WORKER_PROFILE) // <1> 18 | class WorkerConfiguration { 19 | 20 | // <2> 21 | @Bean 22 | StepLocator stepLocator() { 23 | return new BeanFactoryStepLocator(); 24 | } 25 | 26 | // <3> 27 | @Bean 28 | StepExecutionRequestHandler stepExecutionRequestHandler( 29 | JobExplorer explorer, StepLocator stepLocator) { 30 | StepExecutionRequestHandler handler = new StepExecutionRequestHandler(); 31 | handler.setStepLocator(stepLocator); 32 | handler.setJobExplorer(explorer); 33 | return handler; 34 | } 35 | 36 | // <4> 37 | @Bean 38 | IntegrationFlow stepExecutionRequestHandlerFlow( 39 | WorkerChannels channels, 40 | StepExecutionRequestHandler handler) { 41 | 42 | MessageChannel channel = channels.workerRequestsChannels(); 43 | GenericHandler executionHandler = 44 | (payload, headers) -> handler.handle(payload); 45 | 46 | return IntegrationFlows.from(channel) 47 | .handle(StepExecutionRequest.class, executionHandler) 48 | .channel(channels.workerRepliesChannels()) 49 | .get(); 50 | } 51 | } -------------------------------------------------------------------------------- /remote-partitioning/src/main/java/partition/WorkerStepConfiguration.java: -------------------------------------------------------------------------------- 1 | package partition; 2 | 3 | import org.springframework.batch.core.Step; 4 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 5 | import org.springframework.batch.core.configuration.annotation.StepScope; 6 | import org.springframework.batch.item.database.JdbcBatchItemWriter; 7 | import org.springframework.batch.item.database.JdbcPagingItemReader; 8 | import org.springframework.batch.item.database.Order; 9 | import org.springframework.batch.item.database.builder.JdbcBatchItemWriterBuilder; 10 | import org.springframework.batch.item.database.support.MySqlPagingQueryProvider; 11 | import org.springframework.beans.factory.annotation.Value; 12 | import org.springframework.context.annotation.Bean; 13 | import org.springframework.context.annotation.Configuration; 14 | 15 | import javax.sql.DataSource; 16 | import java.util.Collections; 17 | 18 | @Configuration 19 | class WorkerStepConfiguration { 20 | 21 | // <1> 22 | @Value("${partition.chunk-size}") 23 | private int chunk; 24 | 25 | // <2> 26 | @Bean 27 | @StepScope 28 | JdbcPagingItemReader reader( 29 | DataSource dataSource, 30 | @Value("#{stepExecutionContext['minValue']}") Long min, 31 | @Value("#{stepExecutionContext['maxValue']}") Long max) { 32 | 33 | MySqlPagingQueryProvider queryProvider = new MySqlPagingQueryProvider(); 34 | queryProvider.setSelectClause("id as id, email as email, age as age, first_name as firstName"); 35 | queryProvider.setFromClause("from PEOPLE"); 36 | queryProvider.setWhereClause("where id >= " + min + " and id <= " + max); 37 | queryProvider.setSortKeys(Collections.singletonMap("id", Order.ASCENDING)); 38 | 39 | JdbcPagingItemReader reader = new JdbcPagingItemReader<>(); 40 | reader.setDataSource(dataSource); 41 | reader.setFetchSize(this.chunk); 42 | reader.setQueryProvider(queryProvider); 43 | reader.setRowMapper((rs, i) -> new Person( 44 | rs.getInt("id"), 45 | rs.getInt("age"), 46 | rs.getString("firstName"), 47 | rs.getString("email"))); 48 | return reader; 49 | } 50 | 51 | // <3> 52 | @Bean 53 | JdbcBatchItemWriter writer(DataSource ds) { 54 | return new JdbcBatchItemWriterBuilder() 55 | .beanMapped() 56 | .dataSource(ds) 57 | .sql("INSERT INTO NEW_PEOPLE(age,first_name,email) VALUES(:age, :firstName, :email )") 58 | .build(); 59 | } 60 | 61 | // <4> 62 | @Bean 63 | Step workerStep(StepBuilderFactory sbf) { 64 | return sbf 65 | .get("workerStep") 66 | .chunk(this.chunk) 67 | .reader(reader(null, null, null)) 68 | .writer(writer(null)) 69 | .build(); 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/resources/application-worker.properties: -------------------------------------------------------------------------------- 1 | 2 | server.port=0 3 | 4 | spring.datasource.initialize=false -------------------------------------------------------------------------------- /remote-partitioning/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.driverClassName=com.mysql.jdbc.Driver 2 | spring.datasource.username=batch 3 | spring.datasource.password=batch 4 | spring.datasource.url=jdbc:mysql://localhost:3306/batch?useSSL=false 5 | spring.datasource.platform=mysql 6 | spring.datasource.continueOnError=false 7 | partition.requests=partition-requests 8 | partition.replies=partition-replies 9 | partition.chunk-size=1000 10 | partition.grid-size=4 11 | partition.column=ID 12 | partition.table=PEOPLE 13 | spring.batch.job.enabled=false 14 | server.port=8080 15 | logging.level.org.springframework.batch.item.database.JdbcPagingItemReader=DEBUG 16 | 17 | spring.cloud.stream.bindings.workerRequests.destination=${partition.requests} 18 | spring.cloud.stream.bindings.workerRequests.group=${partition.requests} 19 | spring.cloud.stream.bindings.workerRequests.durableSubscription=true 20 | 21 | spring.cloud.stream.bindings.leaderRequests.destination=${partition.requests} 22 | 23 | spring.cloud.stream.bindings.workerReplies.destination=${partition.replies} 24 | 25 | -------------------------------------------------------------------------------- /remote-partitioning/src/main/resources/data.sql: -------------------------------------------------------------------------------- 1 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) VALUES ('DS@EMAIL.COM', 150, 'DOCTOR'); 2 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) VALUES ('JL@EMAIL.COM', 33, 'JOSH'); 3 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) VALUES ('KB@EMAIL.COM', 30, 'KENNY'); 4 | 5 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 6 | P.EMAIL, 7 | P.AGE, 8 | P.FIRST_NAME 9 | FROM PEOPLE P; 10 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 11 | P.EMAIL, 12 | P.AGE, 13 | P.FIRST_NAME 14 | FROM PEOPLE P; 15 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 16 | P.EMAIL, 17 | P.AGE, 18 | P.FIRST_NAME 19 | FROM PEOPLE P; 20 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 21 | P.EMAIL, 22 | P.AGE, 23 | P.FIRST_NAME 24 | FROM PEOPLE P; 25 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 26 | P.EMAIL, 27 | P.AGE, 28 | P.FIRST_NAME 29 | FROM PEOPLE P; 30 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 31 | P.EMAIL, 32 | P.AGE, 33 | P.FIRST_NAME 34 | FROM PEOPLE P; 35 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 36 | P.EMAIL, 37 | P.AGE, 38 | P.FIRST_NAME 39 | FROM PEOPLE P; 40 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 41 | P.EMAIL, 42 | P.AGE, 43 | P.FIRST_NAME 44 | FROM PEOPLE P; 45 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 46 | P.EMAIL, 47 | P.AGE, 48 | P.FIRST_NAME 49 | FROM PEOPLE P; 50 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 51 | P.EMAIL, 52 | P.AGE, 53 | P.FIRST_NAME 54 | FROM PEOPLE P; 55 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 56 | P.EMAIL, 57 | P.AGE, 58 | P.FIRST_NAME 59 | FROM PEOPLE P; 60 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 61 | P.EMAIL, 62 | P.AGE, 63 | P.FIRST_NAME 64 | FROM PEOPLE P; 65 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 66 | P.EMAIL, 67 | P.AGE, 68 | P.FIRST_NAME 69 | FROM PEOPLE P; 70 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 71 | P.EMAIL, 72 | P.AGE, 73 | P.FIRST_NAME 74 | FROM PEOPLE P; 75 | INSERT INTO PEOPLE (EMAIL, AGE, FIRST_NAME) SELECT 76 | P.EMAIL, 77 | P.AGE, 78 | P.FIRST_NAME 79 | FROM PEOPLE P; -------------------------------------------------------------------------------- /remote-partitioning/src/main/resources/schema.sql: -------------------------------------------------------------------------------- 1 | -- SET foreign_key_checks = 0; 2 | -- 3 | -- CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION ( 4 | -- ID INT 5 | -- ); 6 | -- CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION_CONTEXT ( 7 | -- ID INT 8 | -- ); 9 | -- CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION_PARAMS ( 10 | -- ID INT 11 | -- ); 12 | -- CREATE TABLE IF NOT EXISTS BATCH_JOB_EXECUTION_SEQ ( 13 | -- ID INT 14 | -- ); 15 | -- CREATE TABLE IF NOT EXISTS BATCH_JOB_INSTANCE ( 16 | -- ID INT 17 | -- ); 18 | -- CREATE TABLE IF NOT EXISTS BATCH_JOB_SEQ ( 19 | -- ID INT 20 | -- ); 21 | -- CREATE TABLE IF NOT EXISTS BATCH_STEP_EXECUTION ( 22 | -- ID INT 23 | -- ); 24 | -- CREATE TABLE IF NOT EXISTS BATCH_STEP_EXECUTION_CONTEXT ( 25 | -- ID INT 26 | -- ); 27 | -- CREATE TABLE IF NOT EXISTS BATCH_STEP_EXECUTION_SEQ ( 28 | -- ID INT 29 | -- ); 30 | -- TRUNCATE BATCH_JOB_EXECUTION; 31 | -- TRUNCATE BATCH_JOB_EXECUTION_CONTEXT; 32 | -- TRUNCATE BATCH_JOB_EXECUTION_PARAMS; 33 | -- TRUNCATE BATCH_JOB_EXECUTION_SEQ; 34 | -- TRUNCATE BATCH_JOB_INSTANCE; 35 | -- TRUNCATE BATCH_JOB_SEQ; 36 | -- TRUNCATE BATCH_STEP_EXECUTION; 37 | -- TRUNCATE BATCH_STEP_EXECUTION_CONTEXT; 38 | -- TRUNCATE BATCH_STEP_EXECUTION_SEQ; 39 | -- 40 | -- DROP TABLE BATCH_JOB_EXECUTION, BATCH_JOB_EXECUTION_CONTEXT, BATCH_JOB_EXECUTION_PARAMS, BATCH_JOB_EXECUTION_SEQ, BATCH_JOB_INSTANCE, BATCH_JOB_SEQ, BATCH_STEP_EXECUTION, BATCH_STEP_EXECUTION_CONTEXT, BATCH_STEP_EXECUTION_SEQ CASCADE; 41 | -- 42 | -- SET foreign_key_checks = 1; 43 | -- 44 | 45 | DROP TABLE IF EXISTS PEOPLE; 46 | 47 | CREATE TABLE PEOPLE ( 48 | ID BIGINT(10) NOT NULL AUTO_INCREMENT PRIMARY KEY, 49 | EMAIL VARCHAR(255) NOT NULL, 50 | AGE INT(3) NOT NULL, 51 | FIRST_NAME VARCHAR(255) NOT NULL 52 | ); 53 | 54 | 55 | DROP TABLE IF EXISTS NEW_PEOPLE; 56 | 57 | CREATE TABLE NEW_PEOPLE ( 58 | ID BIGINT(10) NOT NULL AUTO_INCREMENT PRIMARY KEY, 59 | EMAIL VARCHAR(255) NOT NULL, 60 | AGE INT(3) NOT NULL, 61 | FIRST_NAME VARCHAR(255) NOT NULL 62 | ); -------------------------------------------------------------------------------- /shaky-service-to-service-calls/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 5.19.0 7 | 8 | 9 | cnj 10 | integration 11 | 1.0.0-SNAPSHOT 12 | 13 | integration/shaky-service-to-service-calls 14 | shaky-service-to-service-calls 15 | pom 16 | 17 | shaky-service 18 | shaky-client 19 | 20 | 21 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 5.19.0 7 | 8 | 9 | cnj 10 | shaky-service-to-service-calls 11 | 1.0.0-SNAPSHOT 12 | 13 | integration/shaky-service-to-service-calls/shaky-client 14 | shaky-client 15 | 16 | 17 | 18 | com.h2database 19 | h2 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-starter-web 24 | 25 | 26 | cnj 27 | shaky-service 28 | ${project.version} 29 | 30 | 31 | 32 | org.springframework.retry 33 | spring-retry 34 | 35 | 36 | org.springframework.cloud 37 | spring-cloud-starter-hystrix 38 | 39 | 40 | org.springframework.cloud 41 | spring-cloud-starter-stream-rabbit 42 | 43 | 44 | org.springframework.boot 45 | spring-boot-starter-test 46 | test 47 | 48 | 49 | 50 | 51 | 52 | org.springframework.boot 53 | spring-boot-maven-plugin 54 | 55 | 56 | 57 | 58 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/main/java/demo/CircuitBreakerGreetingClient.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import com.netflix.hystrix.contrib.javanica.annotation.HystrixCommand; 4 | import org.apache.commons.logging.Log; 5 | import org.apache.commons.logging.LogFactory; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.beans.factory.annotation.Value; 8 | import org.springframework.core.ParameterizedTypeReference; 9 | import org.springframework.http.HttpMethod; 10 | import org.springframework.stereotype.Component; 11 | import org.springframework.web.client.RestTemplate; 12 | 13 | import java.util.Date; 14 | import java.util.Map; 15 | 16 | @Component 17 | public class CircuitBreakerGreetingClient implements GreetingClient { 18 | 19 | private Log log = LogFactory.getLog(getClass()); 20 | private final RestTemplate restTemplate; 21 | private final String serviceUri; 22 | 23 | @Autowired 24 | public CircuitBreakerGreetingClient(RestTemplate restTemplate, 25 | @Value("${greeting-service.domain:127.0.0.1}") String domain, 26 | @Value("${greeting-service.port:8080}") int port) { 27 | this.restTemplate = restTemplate; 28 | this.serviceUri = "http://" + domain + ":" + port + "/hi/{name}"; 29 | } 30 | 31 | @Override 32 | @HystrixCommand(fallbackMethod = "fallback") 33 | public String greet(String name) { 34 | long time = System.currentTimeMillis(); 35 | Date now = new Date(time); 36 | this.log.info("attempting to call " + "the greeting-service " + time 37 | + "/" + now.toString()); 38 | 39 | ParameterizedTypeReference> ptr = new ParameterizedTypeReference>() { 40 | }; 41 | 42 | return this.restTemplate 43 | .exchange(this.serviceUri, HttpMethod.GET, null, ptr, name) 44 | .getBody().get("greeting"); 45 | } 46 | 47 | public String fallback(String name) { 48 | return "OHAI"; 49 | } 50 | 51 | } 52 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/main/java/demo/GreetingClient.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | public interface GreetingClient { 4 | 5 | String greet(String name); 6 | } 7 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/main/java/demo/GreetingClientApplication.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.cloud.client.circuitbreaker.EnableCircuitBreaker; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.EnableAspectJAutoProxy; 8 | import org.springframework.retry.annotation.EnableRetry; 9 | import org.springframework.web.client.RestTemplate; 10 | 11 | @EnableRetry 12 | @EnableCircuitBreaker 13 | @EnableAspectJAutoProxy(proxyTargetClass = true) 14 | @SpringBootApplication 15 | public class GreetingClientApplication { 16 | 17 | @Bean 18 | RestTemplate restTemplate() { 19 | return new RestTemplate(); 20 | } 21 | 22 | public static void main(String args[]) { 23 | SpringApplication.run(GreetingClientApplication.class, args); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/main/java/demo/RetryableGreetingClient.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.beans.factory.annotation.Value; 7 | import org.springframework.core.ParameterizedTypeReference; 8 | import org.springframework.http.HttpMethod; 9 | import org.springframework.retry.annotation.Backoff; 10 | import org.springframework.retry.annotation.Recover; 11 | import org.springframework.retry.annotation.Retryable; 12 | import org.springframework.stereotype.Component; 13 | import org.springframework.web.client.RestTemplate; 14 | 15 | import java.util.Date; 16 | import java.util.Map; 17 | 18 | @Component 19 | public class RetryableGreetingClient implements GreetingClient { 20 | 21 | private Log log = LogFactory.getLog(getClass()); 22 | 23 | private final RestTemplate restTemplate; 24 | 25 | private final String serviceUri; 26 | 27 | @Autowired 28 | public RetryableGreetingClient(RestTemplate restTemplate, 29 | @Value("${greeting-service.domain:127.0.0.1}") String domain, 30 | @Value("${greeting-service.port:8080}") int port) { 31 | this.restTemplate = restTemplate; 32 | this.serviceUri = "http://" + domain + ":" + port + "/hi/{name}"; 33 | } 34 | 35 | // <1> 36 | @Retryable(include = Exception.class, 37 | maxAttempts = 4, 38 | backoff = @Backoff(multiplier = 5)) 39 | @Override 40 | public String greet(String name) { 41 | long time = System.currentTimeMillis(); 42 | 43 | Date now = new Date(time); 44 | 45 | this.log.info("attempting to call the greeting-service " 46 | + time + "/" + now.toString()); 47 | 48 | ParameterizedTypeReference> ptr = 49 | new ParameterizedTypeReference>() { }; 50 | 51 | return this.restTemplate 52 | .exchange(this.serviceUri, HttpMethod.GET, null, ptr, name) 53 | .getBody() 54 | .get("greeting"); 55 | } 56 | 57 | // <2> 58 | @Recover 59 | public String recoverForGreeting(Exception e) { 60 | return "OHAI"; 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/main/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloud-native-java/integration/23c827f4cde1eb29a15a73d87f3632e6b4dd63af/shaky-service-to-service-calls/shaky-client/src/main/resources/application.properties -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/test/java/demo/AbstractGreetingClientTest.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.junit.Test; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.context.embedded.EmbeddedServletContainerInitializedEvent; 8 | import org.springframework.context.ConfigurableApplicationContext; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.context.annotation.Import; 11 | import org.springframework.context.event.EventListener; 12 | 13 | import java.util.ArrayList; 14 | import java.util.Arrays; 15 | import java.util.List; 16 | import java.util.concurrent.atomic.AtomicInteger; 17 | import java.util.stream.Collectors; 18 | 19 | import static org.junit.Assert.assertEquals; 20 | 21 | public abstract class AbstractGreetingClientTest { 22 | 23 | private static AtomicInteger PORT; 24 | 25 | private final Log log = LogFactory.getLog(getClass()); 26 | 27 | protected ConfigurableApplicationContext serviceContext; 28 | protected ConfigurableApplicationContext clientContext; 29 | 30 | protected T client; 31 | 32 | @Test 33 | public void happy() throws Exception { 34 | this.run(true, "Hello, world!!"); 35 | } 36 | 37 | protected void run(boolean setup, String expectedResult) { 38 | PORT = new AtomicInteger(); 39 | this.init(setup); 40 | this.doTest(expectedResult); 41 | this.clientContext.stop(); 42 | if (null != this.serviceContext) { 43 | this.serviceContext.stop(); 44 | } 45 | this.serviceContext = null; 46 | this.clientContext = null; 47 | } 48 | 49 | @Test 50 | public void sad() throws Exception { 51 | this.run(false, "OHAI"); 52 | } 53 | 54 | protected void doTest(String expected) { 55 | String greet = this.client.greet("world!"); 56 | this.log.info("greeting = " + greet); 57 | assertEquals(greet, expected); 58 | } 59 | 60 | @Configuration 61 | @Import(service.GreetingApplication.class) 62 | public static class ApplicationListenerConfig { 63 | 64 | @EventListener(EmbeddedServletContainerInitializedEvent.class) 65 | public void ready(EmbeddedServletContainerInitializedEvent evt) { 66 | PORT.set(evt.getEmbeddedServletContainer().getPort()); 67 | } 68 | } 69 | 70 | protected void init(boolean working) { 71 | if (working) { 72 | this.serviceContext = SpringApplication.run( 73 | ApplicationListenerConfig.class, argsFor("server.port=0")); 74 | } 75 | int port = PORT.get(); 76 | log.info("application initialized on port " + port); 77 | this.clientContext = SpringApplication.run( 78 | GreetingClientApplication.class, 79 | argsFor("greeting-service.port=" + port, "server.port=0")); 80 | this.client = this.obtainClient(); 81 | 82 | } 83 | 84 | protected abstract T obtainClient(); 85 | 86 | protected String[] argsFor(String... args) { 87 | List argsList = Arrays.asList("spring.jmx.default-domain=" 88 | + Math.random(), "spring.jmx.enabled=false", 89 | "endpoints.jmx.enabled=false"); 90 | List result = new ArrayList<>(); 91 | result.addAll(argsList); 92 | result.addAll(Arrays.asList(args)); 93 | 94 | List strings = result.stream() 95 | .map(a -> a.startsWith("--") ? a : "--" + a) 96 | .collect(Collectors.toList()); 97 | return strings.toArray(new String[strings.size()]); 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/test/java/demo/CircuitBreakerGreetingClientTest.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | public class CircuitBreakerGreetingClientTest 4 | extends AbstractGreetingClientTest { 5 | 6 | @Override 7 | protected CircuitBreakerGreetingClient obtainClient() { 8 | return this.clientContext.getBean(CircuitBreakerGreetingClient.class); 9 | } 10 | } -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/test/java/demo/RetryableGreetingClientTest.java: -------------------------------------------------------------------------------- 1 | package demo; 2 | 3 | public class RetryableGreetingClientTest 4 | extends AbstractGreetingClientTest { 5 | 6 | @Override 7 | protected RetryableGreetingClient obtainClient() { 8 | return clientContext.getBean(RetryableGreetingClient.class); 9 | } 10 | } -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-client/src/test/resources/application.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/cloud-native-java/integration/23c827f4cde1eb29a15a73d87f3632e6b4dd63af/shaky-service-to-service-calls/shaky-client/src/test/resources/application.properties -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-service/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | 5.19.0 7 | 8 | 9 | cnj 10 | shaky-service-to-service-calls 11 | 1.0.0-SNAPSHOT 12 | 13 | integration/shaky-service-to-service-calls/shaky-service 14 | shaky-service 15 | 16 | 17 | com.h2database 18 | h2 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-starter-web 23 | 24 | 25 | org.springframework.cloud 26 | spring-cloud-starter-stream-rabbit 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-test 31 | test 32 | 33 | 34 | 44 | 45 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-service/src/main/java/service/GreetingApplication.java: -------------------------------------------------------------------------------- 1 | package service; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | 6 | @SpringBootApplication 7 | public class GreetingApplication { 8 | 9 | public static void main(String[] args) { 10 | SpringApplication.run(GreetingApplication.class); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-service/src/main/java/service/GreetingServiceRestController.java: -------------------------------------------------------------------------------- 1 | package service; 2 | 3 | import org.springframework.web.bind.annotation.PathVariable; 4 | import org.springframework.web.bind.annotation.RequestMapping; 5 | import org.springframework.web.bind.annotation.RequestMethod; 6 | import org.springframework.web.bind.annotation.RestController; 7 | 8 | import java.util.Collections; 9 | import java.util.Map; 10 | 11 | @RestController 12 | public class GreetingServiceRestController { 13 | 14 | @RequestMapping(method = RequestMethod.GET, value = "/hi/{name}") 15 | Map greetings(@PathVariable String name) { 16 | return Collections.singletonMap("greeting", "Hello, " + name + "!"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /shaky-service-to-service-calls/shaky-service/src/test/java/service/GreetingServiceRestControllerTest.java: -------------------------------------------------------------------------------- 1 | package service ; 2 | 3 | import com.fasterxml.jackson.core.type.TypeReference; 4 | import com.fasterxml.jackson.databind.ObjectMapper; 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.boot.test.context.SpringBootTest; 10 | import org.springframework.http.MediaType; 11 | import org.springframework.test.context.junit4.SpringRunner; 12 | import org.springframework.test.web.servlet.MockMvc; 13 | import org.springframework.test.web.servlet.setup.MockMvcBuilders; 14 | import org.springframework.web.context.WebApplicationContext; 15 | import service.GreetingApplication; 16 | 17 | import java.util.Map; 18 | 19 | import static org.junit.Assert.assertTrue; 20 | import static org.springframework.boot.test.context.SpringBootTest.WebEnvironment.MOCK; 21 | import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.get; 22 | import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status; 23 | 24 | @RunWith(SpringRunner.class) 25 | @SpringBootTest(classes = GreetingApplication.class, webEnvironment = MOCK) 26 | public class GreetingServiceRestControllerTest { 27 | 28 | private MockMvc mockMvc; 29 | 30 | @Autowired 31 | private WebApplicationContext webApplicationContext; 32 | 33 | @Autowired 34 | private ObjectMapper objectMapper; 35 | 36 | @Before 37 | public void before() throws Exception { 38 | this.mockMvc = MockMvcBuilders.webAppContextSetup(this.webApplicationContext).build(); 39 | } 40 | 41 | @Test 42 | public void greetings() throws Exception { 43 | String cnj = "CNJ"; 44 | this.mockMvc 45 | .perform(get("/hi/" + cnj).contentType(MediaType.APPLICATION_JSON)) 46 | .andExpect(status().isOk()) 47 | .andExpect( 48 | mvcResult -> { 49 | String json = mvcResult.getResponse().getContentAsString(); 50 | TypeReference> typeReference = 51 | new TypeReference>() { 52 | }; 53 | Map mapOfData = objectMapper.readerFor(typeReference) 54 | .readValue(json.getBytes()); 55 | assertTrue(mapOfData.get("greeting").contains(cnj)); 56 | }); 57 | } 58 | } -------------------------------------------------------------------------------- /stream/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 4.0.0 7 | stream 8 | pom 9 | integration/stream 10 | 11 | cnj 12 | integration 13 | 1.0.0-SNAPSHOT 14 | 15 | 16 | stream-producer 17 | stream-consumer 18 | 19 | 20 | -------------------------------------------------------------------------------- /stream/stream-consumer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | integration/stream-consumer 6 | stream-consumer 7 | 8 | cnj 9 | stream 10 | 1.0.0-SNAPSHOT 11 | 12 | 13 | 14 | org.springframework.cloud 15 | spring-cloud-starter-stream-rabbit 16 | 17 | 18 | org.springframework.boot 19 | spring-boot-starter-integration 20 | 21 | 22 | 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-maven-plugin 27 | 28 | stream.consumer.integration.StreamConsumer 29 | 30 | 31 | 32 | 33 | 34 | -------------------------------------------------------------------------------- /stream/stream-consumer/src/main/java/stream/consumer/ConsumerChannels.java: -------------------------------------------------------------------------------- 1 | package stream.consumer; 2 | 3 | import org.springframework.cloud.stream.annotation.Input; 4 | import org.springframework.messaging.SubscribableChannel; 5 | 6 | public interface ConsumerChannels { 7 | 8 | String DIRECTED = "directed"; 9 | String BROADCASTS = "broadcasts"; 10 | 11 | // <1> 12 | @Input(DIRECTED) 13 | SubscribableChannel directed(); 14 | 15 | @Input(BROADCASTS) 16 | SubscribableChannel broadcasts(); 17 | 18 | } 19 | -------------------------------------------------------------------------------- /stream/stream-consumer/src/main/java/stream/consumer/integration/StreamConsumer.java: -------------------------------------------------------------------------------- 1 | package stream.consumer.integration; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.cloud.stream.annotation.EnableBinding; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.integration.dsl.IntegrationFlow; 10 | import org.springframework.integration.dsl.IntegrationFlows; 11 | import org.springframework.messaging.SubscribableChannel; 12 | import stream.consumer.ConsumerChannels; 13 | 14 | @SpringBootApplication 15 | @EnableBinding(ConsumerChannels.class) // <1> 16 | public class StreamConsumer { 17 | 18 | public static void main(String args[]) { 19 | SpringApplication.run(StreamConsumer.class, args); 20 | } 21 | 22 | // <2> 23 | private IntegrationFlow incomingMessageFlow( 24 | SubscribableChannel incoming, 25 | String prefix) { 26 | 27 | Log log = LogFactory.getLog(getClass()); 28 | 29 | return IntegrationFlows 30 | .from(incoming) 31 | .transform(String.class, String::toUpperCase) 32 | .handle(String.class, 33 | (greeting, headers) -> { 34 | log.info("greeting received in IntegrationFlow (" 35 | + prefix + "): " + greeting); 36 | return null; 37 | }).get(); 38 | } 39 | 40 | @Bean 41 | IntegrationFlow direct(ConsumerChannels channels) { 42 | return incomingMessageFlow(channels.directed(), "directed"); 43 | } 44 | 45 | @Bean 46 | IntegrationFlow broadcast(ConsumerChannels channels) { 47 | return incomingMessageFlow(channels.broadcasts(), "broadcast"); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /stream/stream-consumer/src/main/java/stream/consumer/listeners/StreamConsumer.java: -------------------------------------------------------------------------------- 1 | package stream.consumer.listeners; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | import org.springframework.cloud.stream.annotation.EnableBinding; 8 | import org.springframework.cloud.stream.annotation.StreamListener; 9 | import org.springframework.stereotype.Component; 10 | import stream.consumer.ConsumerChannels; 11 | 12 | @SpringBootApplication 13 | @EnableBinding(ConsumerChannels.class) 14 | public class StreamConsumer { 15 | 16 | public static void main(String args[]) { 17 | SpringApplication.run(StreamConsumer.class, args); 18 | } 19 | } 20 | 21 | @Component 22 | class GreetingProcessor { 23 | 24 | private Log log = LogFactory.getLog(getClass()); 25 | 26 | @StreamListener(ConsumerChannels.DIRECTED) 27 | public void onNewDirectedGreetings(String greeting) { 28 | this.onNewGreeting(ConsumerChannels.DIRECTED, greeting); 29 | } 30 | 31 | @StreamListener(ConsumerChannels.BROADCASTS) 32 | public void onNewBroadcastGreeting(String greeting) { 33 | this.onNewGreeting(ConsumerChannels.BROADCASTS, greeting); 34 | } 35 | 36 | private void onNewGreeting(String prefix, String greeting) { 37 | log.info("greeting received in @StreamListener (" + prefix + "): " 38 | + greeting); 39 | } 40 | } -------------------------------------------------------------------------------- /stream/stream-consumer/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # <1> 2 | spring.cloud.stream.bindings.broadcasts.destination = greetings-pub-sub 3 | 4 | # <2> 5 | spring.cloud.stream.bindings.directed.destination = greetings-p2p 6 | spring.cloud.stream.bindings.directed.group = greetings-p2p-group 7 | spring.cloud.stream.bindings.directed.durableSubscription = true 8 | 9 | server.port=0 10 | 11 | spring.rabbitmq.addresses=localhost -------------------------------------------------------------------------------- /stream/stream-producer/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | integration/stream-producer 7 | stream-producer 8 | 9 | cnj 10 | stream 11 | 1.0.0-SNAPSHOT 12 | 13 | 14 | 15 | org.springframework.cloud 16 | spring-cloud-starter-stream-rabbit 17 | 18 | 19 | org.springframework.boot 20 | spring-boot-starter-web 21 | 22 | 23 | org.springframework.boot 24 | spring-boot-starter-integration 25 | 26 | 27 | 28 | 29 | 30 | org.springframework.boot 31 | spring-boot-maven-plugin 32 | 33 | stream.producer.gateway.StreamProducer 34 | 35 | 36 | 37 | 38 | 39 | -------------------------------------------------------------------------------- /stream/stream-producer/src/main/java/stream/producer/ProducerChannels.java: -------------------------------------------------------------------------------- 1 | package stream.producer; 2 | 3 | import org.springframework.cloud.stream.annotation.Output; 4 | import org.springframework.messaging.MessageChannel; 5 | 6 | public interface ProducerChannels { 7 | 8 | // <1> 9 | String DIRECT = "directGreetings"; 10 | String BROADCAST = "broadcastGreetings"; 11 | 12 | @Output(DIRECT) 13 | // <2> 14 | MessageChannel directGreetings(); 15 | 16 | @Output(BROADCAST) 17 | MessageChannel broadcastGreetings(); 18 | } 19 | -------------------------------------------------------------------------------- /stream/stream-producer/src/main/java/stream/producer/channels/StreamProducer.java: -------------------------------------------------------------------------------- 1 | package stream.producer.channels; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.stream.annotation.EnableBinding; 7 | import org.springframework.http.ResponseEntity; 8 | import org.springframework.messaging.MessageChannel; 9 | import org.springframework.messaging.support.MessageBuilder; 10 | import org.springframework.web.bind.annotation.PathVariable; 11 | import org.springframework.web.bind.annotation.RequestMapping; 12 | import org.springframework.web.bind.annotation.RestController; 13 | import stream.producer.ProducerChannels; 14 | 15 | @SpringBootApplication 16 | @EnableBinding(ProducerChannels.class) // <1> 17 | public class StreamProducer { 18 | 19 | public static void main(String args[]) { 20 | SpringApplication.run(StreamProducer.class, args); 21 | } 22 | } 23 | 24 | @RestController 25 | class GreetingProducer { 26 | 27 | private final MessageChannel broadcast, direct; 28 | 29 | // <2> 30 | @Autowired 31 | GreetingProducer(ProducerChannels channels) { 32 | this.broadcast = channels.broadcastGreetings(); 33 | this.direct = channels.directGreetings(); 34 | } 35 | 36 | @RequestMapping("/hi/{name}") 37 | ResponseEntity hi(@PathVariable String name) { 38 | String message = "Hello, " + name + "!"; 39 | 40 | // <3> 41 | this.direct.send(MessageBuilder.withPayload("Direct: " + message) 42 | .build()); 43 | 44 | this.broadcast.send(MessageBuilder.withPayload("Broadcast: " + message) 45 | .build()); 46 | return ResponseEntity.ok(message); 47 | } 48 | } -------------------------------------------------------------------------------- /stream/stream-producer/src/main/java/stream/producer/gateway/StreamProducer.java: -------------------------------------------------------------------------------- 1 | package stream.producer.gateway; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.cloud.stream.annotation.EnableBinding; 7 | import org.springframework.http.ResponseEntity; 8 | import org.springframework.integration.annotation.Gateway; 9 | import org.springframework.integration.annotation.IntegrationComponentScan; 10 | import org.springframework.integration.annotation.MessagingGateway; 11 | import org.springframework.web.bind.annotation.PathVariable; 12 | import org.springframework.web.bind.annotation.RequestMapping; 13 | import org.springframework.web.bind.annotation.RequestMethod; 14 | import org.springframework.web.bind.annotation.RestController; 15 | import stream.producer.ProducerChannels; 16 | 17 | @SpringBootApplication 18 | @EnableBinding(ProducerChannels.class) 19 | // <1> 20 | @IntegrationComponentScan 21 | // <2> 22 | public class StreamProducer { 23 | 24 | public static void main(String args[]) { 25 | SpringApplication.run(StreamProducer.class, args); 26 | } 27 | } 28 | 29 | // <3> 30 | @MessagingGateway 31 | interface GreetingGateway { 32 | 33 | @Gateway(requestChannel = ProducerChannels.BROADCAST) 34 | void broadcastGreet(String msg); 35 | 36 | @Gateway(requestChannel = ProducerChannels.DIRECT) 37 | void directGreet(String msg); 38 | } 39 | 40 | @RestController 41 | class GreetingProducer { 42 | 43 | private final GreetingGateway gateway; 44 | 45 | // <4> 46 | @Autowired 47 | GreetingProducer(GreetingGateway gateway) { 48 | this.gateway = gateway; 49 | } 50 | 51 | @RequestMapping(method = RequestMethod.GET, value = "/hi/{name}") 52 | ResponseEntity hi(@PathVariable String name) { 53 | String message = "Hello, " + name + "!"; 54 | this.gateway.directGreet("Direct: " + message); 55 | this.gateway.broadcastGreet("Broadcast: " + message); 56 | return ResponseEntity.ok(message); 57 | } 58 | } -------------------------------------------------------------------------------- /stream/stream-producer/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | 2 | # <1> 3 | spring.cloud.stream.bindings.broadcastGreetings.destination = greetings-pub-sub 4 | spring.cloud.stream.bindings.directGreetings.destination = greetings-p2p 5 | 6 | # <2> 7 | spring.rabbitmq.addresses=localhost -------------------------------------------------------------------------------- /task/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | cnj 8 | integration 9 | 1.0.0-SNAPSHOT 10 | 11 | integration/task 12 | task 13 | 14 | 15 | org.springframework.cloud 16 | spring-cloud-task-starter 17 | 18 | 19 | 20 | 21 | 22 | org.springframework.boot 23 | spring-boot-maven-plugin 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /task/src/main/java/task/HelloTask.java: -------------------------------------------------------------------------------- 1 | package task; 2 | 3 | import org.apache.commons.logging.Log; 4 | import org.apache.commons.logging.LogFactory; 5 | import org.springframework.boot.CommandLineRunner; 6 | import org.springframework.boot.SpringApplication; 7 | import org.springframework.boot.autoconfigure.SpringBootApplication; 8 | import org.springframework.cloud.task.configuration.EnableTask; 9 | import org.springframework.cloud.task.repository.TaskExplorer; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.data.domain.PageRequest; 12 | 13 | import java.util.stream.Stream; 14 | 15 | @EnableTask // <1> 16 | @SpringBootApplication 17 | public class HelloTask { 18 | 19 | private Log log = LogFactory.getLog(getClass()); 20 | 21 | @Bean 22 | CommandLineRunner runAndExplore(TaskExplorer taskExplorer) { 23 | return args -> { 24 | Stream.of(args).forEach(log::info); 25 | 26 | // <2> 27 | taskExplorer.findAll(new PageRequest(0, 1)) 28 | .forEach(taskExecution -> log.info(taskExecution.toString())); 29 | }; 30 | } 31 | 32 | public static void main(String args[]) { 33 | SpringApplication.run(HelloTask.class, args); 34 | } 35 | } 36 | --------------------------------------------------------------------------------