├── README.md ├── src ├── main │ ├── java │ │ └── com │ │ │ └── javatechie │ │ │ └── spring │ │ │ └── batch │ │ │ ├── repository │ │ │ └── CustomerRepository.java │ │ │ ├── config │ │ │ ├── CustomerProcessor.java │ │ │ ├── ExceptionSkipPolicy.java │ │ │ ├── CustomerItemWriter.java │ │ │ ├── JobCompletionNotificationListener.java │ │ │ └── SpringBatchConfig.java │ │ │ ├── SpringBatchExampleApplication.java │ │ │ ├── entity │ │ │ └── Customer.java │ │ │ ├── listener │ │ │ └── StepSkipListener.java │ │ │ └── controller │ │ │ └── BatchJobController.java │ └── resources │ │ ├── application.properties │ │ └── customers.csv └── test │ └── java │ └── com │ └── javatechie │ └── spring │ └── batch │ └── SpringBatchExampleApplicationTests.java └── pom.xml /README.md: -------------------------------------------------------------------------------- 1 | # spring-batch-file-upload -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/repository/CustomerRepository.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.repository; 2 | 3 | import com.javatechie.spring.batch.entity.Customer; 4 | import org.springframework.data.jpa.repository.JpaRepository; 5 | 6 | public interface CustomerRepository extends JpaRepository { 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/config/CustomerProcessor.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.config; 2 | 3 | import com.javatechie.spring.batch.entity.Customer; 4 | import org.springframework.batch.item.ItemProcessor; 5 | 6 | public class CustomerProcessor implements ItemProcessor { 7 | @Override 8 | public Customer process(Customer customer) { 9 | int age = Integer.parseInt(customer.getAge()); 10 | if (age >= 18) { 11 | return customer; 12 | } 13 | return null; 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver 2 | spring.datasource.url = jdbc:mysql://localhost:3306/javatechie 3 | spring.datasource.username = root 4 | spring.datasource.password = Password 5 | spring.jpa.hibernate.ddl-auto = update 6 | spring.jpa.properties.hibernate.dialect = org.hibernate.dialect.MySQL5Dialect 7 | spring.jpa.hibernate.naming.physical-strategy=org.hibernate.boot.model.naming.PhysicalNamingStrategyStandardImpl 8 | server.port=9292 9 | spring.batch.initialize-schema=ALWAYS 10 | spring.jpa.show-sql = true 11 | #disabled job run at startup 12 | spring.batch.job.enabled=false 13 | 14 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/SpringBatchExampleApplication.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch; 2 | 3 | import com.ulisesbocchio.jasyptspringboot.annotation.EnableEncryptableProperties; 4 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 5 | import org.springframework.boot.SpringApplication; 6 | import org.springframework.boot.autoconfigure.SpringBootApplication; 7 | 8 | @SpringBootApplication 9 | @EnableBatchProcessing 10 | public class SpringBatchExampleApplication { 11 | 12 | public static void main(String[] args) { 13 | SpringApplication.run(SpringBatchExampleApplication.class, args); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/config/ExceptionSkipPolicy.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.config; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.batch.core.step.skip.SkipLimitExceededException; 7 | import org.springframework.batch.core.step.skip.SkipPolicy; 8 | import org.springframework.batch.item.file.FlatFileParseException; 9 | 10 | import java.io.FileNotFoundException; 11 | //@Slf4j 12 | public class ExceptionSkipPolicy implements SkipPolicy { 13 | 14 | @Override 15 | public boolean shouldSkip(Throwable throwable, int i) throws SkipLimitExceededException { 16 | return throwable instanceof NumberFormatException ; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/config/CustomerItemWriter.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.config; 2 | 3 | import com.javatechie.spring.batch.entity.Customer; 4 | import com.javatechie.spring.batch.repository.CustomerRepository; 5 | import org.springframework.batch.core.configuration.annotation.StepScope; 6 | import org.springframework.batch.item.ItemWriter; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.stereotype.Component; 9 | 10 | import java.util.List; 11 | 12 | @Component 13 | public class CustomerItemWriter implements ItemWriter { 14 | 15 | @Autowired 16 | private CustomerRepository repository; 17 | 18 | @Override 19 | public void write(List list) throws Exception { 20 | System.out.println("Writer Thread "+Thread.currentThread().getName()); 21 | repository.saveAll(list); 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/config/JobCompletionNotificationListener.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.config; 2 | 3 | 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | import org.springframework.batch.core.BatchStatus; 7 | import org.springframework.batch.core.JobExecution; 8 | import org.springframework.batch.core.listener.JobExecutionListenerSupport; 9 | import org.springframework.stereotype.Component; 10 | @ Component 11 | public class JobCompletionNotificationListener extends JobExecutionListenerSupport { 12 | 13 | private static final Logger log = 14 | LoggerFactory.getLogger(JobCompletionNotificationListener.class); 15 | 16 | @Override 17 | public void afterJob(JobExecution jobExecution) { 18 | if (jobExecution.getStatus() == BatchStatus.COMPLETED) { 19 | log.info("!!! JOB FINISHED! Time to verify the results"); 20 | 21 | } 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/entity/Customer.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.entity; 2 | 3 | import lombok.AllArgsConstructor; 4 | import lombok.Data; 5 | import lombok.NoArgsConstructor; 6 | 7 | import javax.persistence.Column; 8 | import javax.persistence.Entity; 9 | import javax.persistence.Id; 10 | import javax.persistence.Table; 11 | 12 | @Entity 13 | @Table(name = "CUSTOMER_INFO") 14 | @Data 15 | @AllArgsConstructor 16 | @NoArgsConstructor 17 | public class Customer { 18 | @Id 19 | @Column(name = "CUSTOMER_ID") 20 | private int id; 21 | @Column(name = "FIRST_NAME") 22 | private String firstName; 23 | @Column(name = "LAST_NAME") 24 | private String lastName; 25 | @Column(name = "EMAIL") 26 | private String email; 27 | @Column(name = "GENDER") 28 | private String gender; 29 | @Column(name = "CONTACT") 30 | private String contactNo; 31 | @Column(name = "COUNTRY") 32 | private String country; 33 | @Column(name = "DOB") 34 | private String dob; 35 | @Column(name = "AGE") 36 | private String age; 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/com/javatechie/spring/batch/SpringBatchExampleApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch; 2 | 3 | import org.jasypt.encryption.pbe.PooledPBEStringEncryptor; 4 | import org.jasypt.encryption.pbe.config.SimpleStringPBEConfig; 5 | import org.junit.jupiter.api.Test; 6 | import org.springframework.boot.test.context.SpringBootTest; 7 | 8 | //@SpringBootTest 9 | class SpringBatchExampleApplicationTests { 10 | 11 | @Test 12 | public void testPasswordEncryption() { 13 | PooledPBEStringEncryptor encryptor = new PooledPBEStringEncryptor(); 14 | SimpleStringPBEConfig config = new SimpleStringPBEConfig(); 15 | config.setPassword("javatechie"); // encryptor's private key 16 | config.setAlgorithm("PBEWithMD5AndDES"); 17 | config.setKeyObtentionIterations("1000"); 18 | config.setPoolSize("1"); 19 | config.setProviderName("SunJCE"); 20 | config.setSaltGeneratorClassName("org.jasypt.salt.RandomSaltGenerator"); 21 | config.setStringOutputType("base64"); 22 | encryptor.setConfig(config); 23 | String plainText = "Password"; 24 | String encryptedPassword = encryptor.encrypt(plainText); 25 | System.out.println("encryptedPassword : " + encryptedPassword); 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/listener/StepSkipListener.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.listener; 2 | 3 | import com.fasterxml.jackson.databind.ObjectMapper; 4 | import com.javatechie.spring.batch.entity.Customer; 5 | import lombok.SneakyThrows; 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.batch.core.SkipListener; 9 | import org.springframework.batch.core.StepExecution; 10 | 11 | public class StepSkipListener implements SkipListener { 12 | 13 | 14 | Logger logger = LoggerFactory.getLogger(StepSkipListener.class); 15 | 16 | @Override // item reader 17 | public void onSkipInRead(Throwable throwable) { 18 | logger.info("A failure on read {} ", throwable.getMessage()); 19 | } 20 | 21 | @Override // item writter 22 | public void onSkipInWrite(Number item, Throwable throwable) { 23 | logger.info("A failure on write {} , {}", throwable.getMessage(), item); 24 | } 25 | 26 | @SneakyThrows 27 | @Override // item processor 28 | public void onSkipInProcess(Customer customer, Throwable throwable) { 29 | logger.info("Item {} was skipped due to the exception {}", new ObjectMapper().writeValueAsString(customer), 30 | throwable.getMessage()); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/resources/customers.csv: -------------------------------------------------------------------------------- 1 | id,firstName,lastName,email,gender,contactNo,country,dob,age 2 | 1,Truda,Cahan,tcahan0@xinhuanet.com,Female,506-108-7773,China,01/05/2001,60 3 | 2,Janey,Schoffel,jschoffel1@wisc.edu,Female,716-626-1084,Japan,3/31/2016,22 4 | 3,Quincy,Hansana,qhansana2@oracle.com,Male,853-785-2314,Portugal,9/28/2005,98 5 | 4,Margarette,Greenrod,mgreenrod3@chicagotribune.com,Female,911-424-3500,Peru,12/02/2019,27 6 | 5,Erminie,Fortnum,efortnum4@ustream.tv,Female,913-708-6039,Guyana,03/06/2002,29 7 | 6,Cyndie,Siaspinski,csiaspinski5@discuz.net,Female,279-842-5440,China,03/08/2012,36 8 | 7,Drusilla,Etheredge,detheredge6@nationalgeographic.com,Female,409-725-2578,Colombia,05/06/2014,70 9 | 8,Myrah,Davidofski,mdavidofski7@about.com,Female,811-739-2181,China,10/26/2006,77 10 | 9,Rosemarie,Iannelli,riannelli8@icq.com,Female,770-415-7996,Venezuela,3/29/1996,85 11 | 10,Griff,Bouldstridge,gbouldstridge9@eventbrite.com,Male,296-358-9767,Mauritania,05/05/2008,58 12 | 11,Peta,Friedlos,pfriedlosa@mail.ru,Female,892-349-0426,Portugal,12/20/1994,vnby4 13 | 12,Gerianna,Barbera,gbarberab@ning.com,Female,959-693-3764,Japan,9/18/1991,72 14 | 13,Selby,Palfreman,spalfremanc@youku.com,Male,662-375-8690,Philippines,07/09/2005,73 15 | 14,Alvinia,Pillifant,apillifantd@deviantart.com,Female,967-729-2595,Russia,3/24/2004,65 16 | 15,Kinna,Simione,ksimionee@ning.com,Female,305-203-0936,Venezuela,9/20/2012,85 17 | 16,Desmund,Caldecot,dcaldecotf@blogger.com,Male,883-122-1043,El Salvador,11/04/2017,89 18 | 17,Mic,Cloonan,mcloonang@wisc.edu,Male,849-587-4062,Yemen,1/20/1991,73 19 | 18,Lawton,Andersch,landerschh@opensource.org,Male,529-898-6409,Russia,10/19/2020, 20 | 19,Jobyna,Van,jvani@vimeo.com,Female,919-850-8135,China,12/23/1991,82 21 | 20,Filide,Swayland,fswaylandj@over-blog.com,Female,904-397-6549,China,06/10/2000,17 -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.6.7 9 | 10 | 11 | com.javatechie 12 | spring-batch-example 13 | 0.0.1-SNAPSHOT 14 | spring-batch-example 15 | Demo project for Spring Boot 16 | 17 | 1.8 18 | 19 | 20 | 21 | org.springframework.boot 22 | spring-boot-starter-data-jpa 23 | 24 | 25 | org.springframework.boot 26 | spring-boot-starter-web 27 | 28 | 29 | org.springframework.boot 30 | spring-boot-starter-batch 31 | 32 | 33 | mysql 34 | mysql-connector-java 35 | runtime 36 | 37 | 38 | org.projectlombok 39 | lombok 40 | true 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-starter-test 45 | test 46 | 47 | 48 | com.github.ulisesbocchio 49 | jasypt-spring-boot-starter 50 | 3.0.3 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | org.springframework.boot 59 | spring-boot-maven-plugin 60 | 61 | 62 | 63 | org.projectlombok 64 | lombok 65 | 66 | 67 | 68 | 69 | 70 | com.github.ulisesbocchio 71 | jasypt-maven-plugin 72 | 3.0.3 73 | 74 | 75 | 76 | 77 | 78 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/controller/BatchJobController.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.controller; 2 | 3 | import com.javatechie.spring.batch.entity.Customer; 4 | import com.javatechie.spring.batch.repository.CustomerRepository; 5 | import org.apache.tomcat.util.http.fileupload.IOUtils; 6 | import org.springframework.batch.core.*; 7 | import org.springframework.batch.core.launch.JobLauncher; 8 | import org.springframework.batch.core.repository.JobExecutionAlreadyRunningException; 9 | import org.springframework.batch.core.repository.JobInstanceAlreadyCompleteException; 10 | import org.springframework.batch.core.repository.JobRepository; 11 | import org.springframework.batch.core.repository.JobRestartException; 12 | import org.springframework.beans.factory.annotation.Autowired; 13 | import org.springframework.core.io.ClassPathResource; 14 | import org.springframework.web.bind.annotation.GetMapping; 15 | import org.springframework.web.bind.annotation.PostMapping; 16 | import org.springframework.web.bind.annotation.RequestParam; 17 | import org.springframework.web.bind.annotation.RestController; 18 | import org.springframework.web.multipart.MultipartFile; 19 | 20 | import java.io.File; 21 | import java.io.FileOutputStream; 22 | import java.io.IOException; 23 | import java.io.OutputStream; 24 | import java.nio.file.Files; 25 | import java.nio.file.Paths; 26 | import java.util.List; 27 | 28 | @RestController 29 | public class BatchJobController { 30 | 31 | public static final String TEMP_STORAGE_PATH = "/Users/javatechie/Desktop/temp/"; 32 | @Autowired 33 | private JobLauncher jobLauncher; 34 | 35 | @Autowired 36 | private Job job; 37 | 38 | @Autowired 39 | private CustomerRepository repository; 40 | @Autowired 41 | private JobRepository jobRepository; 42 | 43 | private final String TEMP_STORAGE = "/Users/javatechie/Desktop/batch-files/"; 44 | 45 | @PostMapping(path = "/importData") 46 | public void startBatch(@RequestParam("file") MultipartFile multipartFile) { 47 | 48 | 49 | // file -> path we don't know 50 | //copy the file to some storage in your VM : get the file path 51 | //copy the file to DB : get the file path 52 | 53 | try { 54 | String originalFileName = multipartFile.getOriginalFilename(); 55 | File fileToImport = new File(TEMP_STORAGE + originalFileName); 56 | multipartFile.transferTo(fileToImport); 57 | 58 | JobParameters jobParameters = new JobParametersBuilder() 59 | .addString("fullPathFileName", TEMP_STORAGE + originalFileName) 60 | .addLong("startAt", System.currentTimeMillis()).toJobParameters(); 61 | 62 | JobExecution execution = jobLauncher.run(job, jobParameters); 63 | 64 | // if(execution.getExitStatus().getExitCode().equals(ExitStatus.COMPLETED)){ 65 | // //delete the file from the TEMP_STORAGE 66 | // Files.deleteIfExists(Paths.get(TEMP_STORAGE + originalFileName)); 67 | // } 68 | 69 | } catch (JobExecutionAlreadyRunningException | JobRestartException | JobInstanceAlreadyCompleteException | JobParametersInvalidException | IOException e) { 70 | 71 | e.printStackTrace(); 72 | } 73 | } 74 | 75 | @GetMapping("/customers") 76 | public List getAll() { 77 | return repository.findAll(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/main/java/com/javatechie/spring/batch/config/SpringBatchConfig.java: -------------------------------------------------------------------------------- 1 | package com.javatechie.spring.batch.config; 2 | 3 | import com.javatechie.spring.batch.entity.Customer; 4 | import com.javatechie.spring.batch.listener.StepSkipListener; 5 | import com.javatechie.spring.batch.repository.CustomerRepository; 6 | import lombok.AllArgsConstructor; 7 | import org.springframework.batch.core.Job; 8 | import org.springframework.batch.core.JobParameters; 9 | import org.springframework.batch.core.SkipListener; 10 | import org.springframework.batch.core.Step; 11 | import org.springframework.batch.core.configuration.annotation.EnableBatchProcessing; 12 | import org.springframework.batch.core.configuration.annotation.JobBuilderFactory; 13 | import org.springframework.batch.core.configuration.annotation.StepBuilderFactory; 14 | import org.springframework.batch.core.configuration.annotation.StepScope; 15 | import org.springframework.batch.core.step.skip.SkipPolicy; 16 | import org.springframework.batch.item.data.RepositoryItemWriter; 17 | import org.springframework.batch.item.file.FlatFileItemReader; 18 | import org.springframework.batch.item.file.LineMapper; 19 | import org.springframework.batch.item.file.mapping.BeanWrapperFieldSetMapper; 20 | import org.springframework.batch.item.file.mapping.DefaultLineMapper; 21 | import org.springframework.batch.item.file.transform.DelimitedLineTokenizer; 22 | import org.springframework.beans.factory.annotation.Autowired; 23 | import org.springframework.beans.factory.annotation.Value; 24 | import org.springframework.context.annotation.Bean; 25 | import org.springframework.context.annotation.Configuration; 26 | import org.springframework.context.annotation.Scope; 27 | import org.springframework.context.annotation.ScopedProxyMode; 28 | import org.springframework.core.io.FileSystemResource; 29 | import org.springframework.core.task.SimpleAsyncTaskExecutor; 30 | import org.springframework.core.task.TaskExecutor; 31 | 32 | import java.io.File; 33 | 34 | @Configuration 35 | @EnableBatchProcessing 36 | //@AllArgsConstructor 37 | public class SpringBatchConfig { 38 | 39 | @Autowired 40 | private JobBuilderFactory jobBuilderFactory; 41 | @Autowired 42 | private StepBuilderFactory stepBuilderFactory; 43 | @Autowired 44 | private CustomerRepository customerRepository; 45 | @Autowired 46 | private CustomerItemWriter customerItemWriter; 47 | 48 | 49 | @Bean 50 | @StepScope 51 | public FlatFileItemReader itemReader(@Value("#{jobParameters[fullPathFileName]}") String pathToFIle) { 52 | FlatFileItemReader flatFileItemReader = new FlatFileItemReader<>(); 53 | flatFileItemReader.setResource(new FileSystemResource(new File(pathToFIle))); 54 | flatFileItemReader.setName("CSV-Reader"); 55 | flatFileItemReader.setLinesToSkip(1); 56 | flatFileItemReader.setLineMapper(lineMapper()); 57 | return flatFileItemReader; 58 | } 59 | 60 | private LineMapper lineMapper() { 61 | DefaultLineMapper lineMapper = new DefaultLineMapper<>(); 62 | 63 | DelimitedLineTokenizer lineTokenizer = new DelimitedLineTokenizer(); 64 | lineTokenizer.setDelimiter(","); 65 | lineTokenizer.setStrict(false); 66 | lineTokenizer.setNames("id", "firstName", "lastName", "email", "gender", "contactNo", "country", "dob", "age"); 67 | //lineTokenizer.setNames("id", "firstName", "lastName", "email", "gender", "contactNo", "country", "dob"); 68 | 69 | BeanWrapperFieldSetMapper fieldSetMapper = new BeanWrapperFieldSetMapper<>(); 70 | fieldSetMapper.setTargetType(Customer.class); 71 | 72 | lineMapper.setLineTokenizer(lineTokenizer); 73 | lineMapper.setFieldSetMapper(fieldSetMapper); 74 | 75 | return lineMapper; 76 | } 77 | 78 | @Bean 79 | public CustomerProcessor processor() { 80 | return new CustomerProcessor(); 81 | } 82 | 83 | @Bean 84 | public RepositoryItemWriter writer() { 85 | RepositoryItemWriter writer = new RepositoryItemWriter<>(); 86 | writer.setRepository(customerRepository); 87 | writer.setMethodName("save"); 88 | return writer; 89 | } 90 | 91 | 92 | @Bean 93 | public Step step1(FlatFileItemReader itemReader) { 94 | return stepBuilderFactory.get("slaveStep").chunk(10) 95 | .reader(itemReader) 96 | .processor(processor()) 97 | .writer(customerItemWriter) 98 | .faultTolerant() 99 | .listener(skipListener()) 100 | .skipPolicy(skipPolicy()) 101 | .taskExecutor(taskExecutor()) 102 | .build(); 103 | } 104 | 105 | 106 | @Bean 107 | public Job runJob(FlatFileItemReader itemReader) { 108 | return jobBuilderFactory.get("importCustomer").flow(step1(itemReader)).end().build(); 109 | } 110 | 111 | 112 | @Bean 113 | public SkipPolicy skipPolicy() { 114 | return new ExceptionSkipPolicy(); 115 | } 116 | 117 | @Bean 118 | public SkipListener skipListener() { 119 | return new StepSkipListener(); 120 | } 121 | 122 | @Bean 123 | public TaskExecutor taskExecutor() { 124 | SimpleAsyncTaskExecutor taskExecutor = new SimpleAsyncTaskExecutor(); 125 | taskExecutor.setConcurrencyLimit(10); 126 | return taskExecutor; 127 | } 128 | 129 | 130 | } 131 | --------------------------------------------------------------------------------