├── img.png ├── img ├── img.png ├── img_1.png ├── img_2.png └── img_3.png ├── src ├── main │ ├── java │ │ └── com │ │ │ └── wyj │ │ │ ├── task │ │ │ ├── module │ │ │ │ ├── enums │ │ │ │ │ ├── TaskTypeEnum.java │ │ │ │ │ ├── TaskExecResult.java │ │ │ │ │ ├── TaskStatusEnum.java │ │ │ │ │ └── TaskSplitStatusEnum.java │ │ │ │ ├── Task.java │ │ │ │ └── TaskSplit.java │ │ │ ├── JobShardingStrategy.java │ │ │ ├── core │ │ │ │ ├── TaskService.java │ │ │ │ ├── TaskScheduler.java │ │ │ │ ├── TaskStrategyContext.java │ │ │ │ ├── CoreTaskService.java │ │ │ │ └── TaskMQConsumer.java │ │ │ ├── repository │ │ │ │ ├── entity │ │ │ │ │ ├── TaskPO.java │ │ │ │ │ └── TaskSplitPO.java │ │ │ │ ├── mapper │ │ │ │ │ └── TaskMapper.java │ │ │ │ ├── TaskRepository.java │ │ │ │ ├── transfer │ │ │ │ │ └── TaskTransfer.java │ │ │ │ └── impl │ │ │ │ │ └── TaskRepositoryImpl.java │ │ │ ├── TaskApi.java │ │ │ ├── TaskStrategy.java │ │ │ ├── TaskHandler.java │ │ │ └── util │ │ │ │ └── JsonUtil.java │ │ │ └── test │ │ │ ├── impl │ │ │ ├── MyTaskTypeEnum.java │ │ │ ├── SimpleCache.java │ │ │ ├── MultiTaskStrategy.java │ │ │ └── SimpleTaskStrategy.java │ │ │ └── BatchTaskApplication.java │ └── resources │ │ ├── application.properties │ │ ├── logback-spring.xml │ │ └── mapper │ │ └── task-mapper.xml └── test │ └── java │ └── com │ └── wyj │ └── task │ └── BatchTaskApplicationTests.java ├── uml ├── 1.1创建任务.puml ├── 2.1任务分发.puml ├── 3.1任务完结.puml └── 2.2任务执行.puml ├── .gitignore ├── cmd └── rocketmq ├── README.MD └── pom.xml /img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wengyingjian/scheduler_batch/HEAD/img.png -------------------------------------------------------------------------------- /img/img.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wengyingjian/scheduler_batch/HEAD/img/img.png -------------------------------------------------------------------------------- /img/img_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wengyingjian/scheduler_batch/HEAD/img/img_1.png -------------------------------------------------------------------------------- /img/img_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wengyingjian/scheduler_batch/HEAD/img/img_2.png -------------------------------------------------------------------------------- /img/img_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wengyingjian/scheduler_batch/HEAD/img/img_3.png -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/module/enums/TaskTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.module.enums; 2 | 3 | public interface TaskTypeEnum { 4 | Integer getType(); 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/JobShardingStrategy.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task; 2 | 3 | import com.wyj.task.module.Task; 4 | import com.wyj.task.module.TaskSplit; 5 | 6 | import java.util.List; 7 | 8 | public interface JobShardingStrategy { 9 | List sharding(Task task); 10 | } 11 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/core/TaskService.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.core; 2 | 3 | import org.apache.rocketmq.client.apis.ClientException; 4 | 5 | public interface TaskService { 6 | void dispatch(TaskScheduler.StopChecker checker) throws ClientException; 7 | 8 | void scan(TaskScheduler.StopChecker checker); 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/module/enums/TaskExecResult.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.module.enums; 2 | 3 | public enum TaskExecResult { 4 | /** 5 | * 分片任务全部完成 6 | */ 7 | SUCCESS, 8 | /** 9 | * 分片任务出错，需要终止 10 | */ 11 | STOP, 12 | /** 13 | * 分片任务未完成，需要再次调度 14 | */ 15 | RETRY; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/repository/entity/TaskPO.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.repository.entity; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Date; 6 | 7 | @Data 8 | public class TaskPO { 9 | private Long id; 10 | private Integer status; 11 | private Integer taskType; 12 | private Date createTime; 13 | private Date updateTime; 14 | private Date taskTime; 15 | private String bizData; 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/test/impl/MyTaskTypeEnum.java: -------------------------------------------------------------------------------- 1 | package com.wyj.test.impl; 2 | 3 | import com.wyj.task.module.enums.TaskTypeEnum; 4 | import lombok.AllArgsConstructor; 5 | 6 | @AllArgsConstructor 7 | public enum MyTaskTypeEnum implements TaskTypeEnum { 8 | SIMPLE_FLUSH_CACHE(0), MULTI_TASK_TEST(1); 9 | 10 | private final int type; 11 | 12 | @Override 13 | public Integer getType() { 14 | return type; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /uml/1.1创建任务.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title 创建任务 3 | 4 | participant batch 5 | participant biz 6 | database batch_db 7 | 8 | biz --> batch: 创建任务 9 | 10 | activate batch 11 | batch -> batch: 1.初始化任务 12 | group 2.初始化任务分片 13 | batch -> batch: 获取任务处理器 14 | batch -> biz 15 | activate biz 16 | biz -> biz: 任务分片 17 | deactivate biz 18 | end group 19 | 20 | batch -> batch_db: 3.保存任务及分片 21 | activate batch_db 22 | deactivate batch 23 | 24 | @enduml -------------------------------------------------------------------------------- /uml/2.1任务分发.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title 任务分发 3 | 4 | participant schedule 5 | database mq 6 | participant batch 7 | database batch_db 8 | 9 | schedule --> batch: trigger 10 | 11 | activate batch 12 | batch -> batch_db: 1.扫描待处理任务 13 | activate batch_db 14 | deactivate batch_db 15 | 16 | loop 17 | batch -> mq: 2.发送分片消息 18 | activate mq 19 | deactivate mq 20 | 21 | batch -> batch_db: 3.更新任务状态 22 | activate batch_db 23 | deactivate batch_db 24 | end loop 25 | @enduml -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/TaskApi.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task; 2 | 3 | import com.wyj.task.module.enums.TaskTypeEnum; 4 | 5 | import java.util.Date; 6 | 7 | /** 8 | * 给业务系统暴露的接口 9 | */ 10 | public interface TaskApi { 11 | /** 12 | * 创建任务 13 | * 若使用周期任务，任务系统可借助schedule通过cron，手动创建任务 14 | * 15 | * @param taskType 任务类型 16 | * @param taskTime 任务时间 17 | * @param bizData 业务字段 18 | */ 19 | void submit(TaskTypeEnum taskType, Date taskTime, String bizData); 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/repository/entity/TaskSplitPO.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.repository.entity; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.Date; 6 | 7 | @Data 8 | public class TaskSplitPO { 9 | private Long id; 10 | private Long taskId; 11 | private Integer status; 12 | private Integer taskType; 13 | private Date createTime; 14 | private Date updateTime; 15 | private Date taskTime; 16 | private String bizData; 17 | private Long execCount; 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/test/impl/SimpleCache.java: -------------------------------------------------------------------------------- 1 | package com.wyj.test.impl; 2 | 3 | import lombok.Data; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | @Data 9 | public class SimpleCache { 10 | private String dataId; 11 | private List storeIds = new ArrayList<>(); 12 | 13 | public static SimpleCache init(String strategyId) { 14 | SimpleCache strategy = new SimpleCache(); 15 | strategy.setDataId(strategyId); 16 | return strategy; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /uml/3.1任务完结.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title 任务完结 3 | 4 | participant schedule 5 | participant batch 6 | participant biz 7 | database batch_db 8 | 9 | schedule --> batch: trigger 10 | 11 | activate batch 12 | batch -> batch_db: 1.扫描未完结主任务 13 | activate batch_db 14 | deactivate batch_db 15 | 16 | batch -> batch_db: 2.找到分片已完成任务 17 | activate batch_db 18 | deactivate batch_db 19 | 20 | batch -> biz: 3.调用reduce方法 21 | activate biz 22 | deactivate biz 23 | 24 | batch -> batch_db: 4.更新任务状态 25 | activate batch_db 26 | deactivate batch_db 27 | @enduml -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/TaskStrategy.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task; 2 | 3 | import com.wyj.task.module.enums.TaskTypeEnum; 4 | import com.wyj.task.module.Task; 5 | 6 | /** 7 | * 任务策略 8 | * 业务系统实现 9 | * 由框架调用 10 | */ 11 | public interface TaskStrategy { 12 | 13 | /** 14 | * 获取任务类型 15 | */ 16 | TaskTypeEnum getTaskType(); 17 | 18 | /** 19 | * 任务分片策略 20 | */ 21 | JobShardingStrategy shardingStrategy(); 22 | 23 | /** 24 | * 任务处理逻辑，需要做好幂等处理 25 | */ 26 | TaskHandler taskHandler(); 27 | 28 | } 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/**/target/ 5 | !**/src/test/**/target/ 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | !**/src/main/**/build/ 30 | !**/src/test/**/build/ 31 | 32 | ### VS Code ### 33 | .vscode/ 34 | -------------------------------------------------------------------------------- /src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # port 2 | server.port=9999 3 | 4 | ## db 5 | spring.datasource.driver-class-name=com.mysql.cj.jdbc.Driver 6 | spring.datasource.url=jdbc:mysql://localhost:3306/test?serverTimezone=UTC&useUnicode=true&characterEncoding=utf-8 7 | spring.datasource.username=root 8 | spring.datasource.password=123456aa 9 | 10 | ## mybatis 11 | mybatis.mapper-locations: classpath:mapper/* 12 | mybatis.type-aliases-package: com.wyj.task.repository.entity 13 | mybatis.configuration.map-underscore-to-camel-case: true 14 | 15 | ## mq 16 | rocketmq.name-server=127.0.0.1:9876 17 | rocketmq.producer.group=task_producer_group -------------------------------------------------------------------------------- /src/main/java/com/wyj/test/BatchTaskApplication.java: -------------------------------------------------------------------------------- 1 | package com.wyj.test; 2 | 3 | import org.mybatis.spring.annotation.MapperScan; 4 | import org.springframework.boot.SpringApplication; 5 | import org.springframework.boot.autoconfigure.SpringBootApplication; 6 | import org.springframework.context.annotation.ComponentScan; 7 | 8 | @SpringBootApplication 9 | @ComponentScan(basePackages = {"com.wyj"}) 10 | @MapperScan("com.wyj.task.repository.mapper") 11 | public class BatchTaskApplication { 12 | 13 | public static void main(String[] args) { 14 | SpringApplication.run(BatchTaskApplication.class, args); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/module/Task.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.module; 2 | 3 | import com.wyj.task.repository.transfer.TaskTransfer; 4 | import lombok.Data; 5 | 6 | import java.util.Date; 7 | 8 | @Data 9 | public class Task { 10 | private Long id; 11 | private Integer status; 12 | private Integer taskType; 13 | private Date createTime; 14 | private Date updateTime; 15 | private Date taskTime; 16 | /** 17 | * 不进行持久化 18 | */ 19 | private String bizData; 20 | 21 | public static Task init(Integer taskType, Date taskTime, String bizData) { 22 | return TaskTransfer.init(taskType, taskTime, bizData); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/module/enums/TaskStatusEnum.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.module.enums; 2 | 3 | public enum TaskStatusEnum { 4 | /** 5 | * 任务初始化 6 | */ 7 | INIT(0), 8 | /** 9 | * 任务处理中：已分发->未完成 10 | */ 11 | EXECUTING(1), 12 | /** 13 | * 任务完结：全部成功 14 | */ 15 | SUCCESS(2), 16 | /** 17 | * 任务完结：部分失败 18 | */ 19 | FINISH(3); 20 | 21 | int status; 22 | 23 | 24 | public int getStatus() { 25 | return status; 26 | } 27 | 28 | public void setStatus(int status) { 29 | this.status = status; 30 | } 31 | 32 | TaskStatusEnum(int status) { 33 | this.status = status; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/module/TaskSplit.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.module; 2 | 3 | import com.wyj.task.repository.transfer.TaskTransfer; 4 | import lombok.Data; 5 | 6 | import java.io.Serializable; 7 | import java.util.Date; 8 | 9 | @Data 10 | public class TaskSplit implements Serializable { 11 | private Long id; 12 | private Long taskId; 13 | private Integer status; 14 | private Integer taskType; 15 | private Date createTime; 16 | private Date updateTime; 17 | private Date taskTime; 18 | private Long exec_count; 19 | private String bizData; 20 | 21 | public static TaskSplit init(String bizData, Task task) { 22 | return TaskTransfer.initSplit(bizData, task); 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/TaskHandler.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task; 2 | 3 | import com.wyj.task.module.Task; 4 | import com.wyj.task.module.enums.TaskExecResult; 5 | import com.wyj.task.module.TaskSplit; 6 | 7 | public interface TaskHandler { 8 | 9 | /** 10 | * 可以一次性执行完所有任务，返回成功 11 | * 或者一次处理部分任务，返回继续（但是需要内部记录状态） 12 | *

13 | * 业务系统需要做逻辑的幂等处理，同一个任务，因为网络问题可能会重复执行 14 | * 框架会尽量保证任务不丢，但无法保证任务不重复 15 | *

16 | * mq默认超时时间为15min，若单次任务执行超过15min，则判断为任务失败 17 | * 18 | * @param split 19 | * @return 20 | */ 21 | TaskExecResult execute(TaskSplit split); 22 | 23 | /** 24 | * 所有分片任务完结后的回调 25 | * 业务系统可选择实现 26 | * 需要做好幂等处理 27 | */ 28 | void reduce(Task task); 29 | } 30 | -------------------------------------------------------------------------------- /uml/2.2任务执行.puml: -------------------------------------------------------------------------------- 1 | @startuml 2 | title 任务执行 3 | 4 | database mq 5 | participant batch 6 | participant biz 7 | database batch_db 8 | database biz_db 9 | 10 | mq --> batch: trigger（分片） 11 | 12 | activate batch 13 | batch -> batch: 1.获取任务处理器 14 | 15 | group 2.处理任务 16 | alt 任务单次处理 17 | batch -> biz: 处理任务（分片） 18 | activate biz 19 | biz -> biz: 业务逻辑 20 | batch <- biz: 处理完成（成功/失败） 21 | deactivate biz 22 | else if 任务分多次处理 23 | batch -> biz: 处理任务（分片） 24 | activate biz 25 | biz -> biz_db: 获取待处理任务 26 | activate biz_db 27 | deactivate 28 | biz -> biz: 业务逻辑 29 | biz -> biz_db: 更新任务处理进度 30 | activate biz_db 31 | deactivate 32 | batch <- biz: 处理完成（需要重试） 33 | deactivate biz 34 | end 35 | end group 36 | 37 | batch -> batch_db: 3.更新任务状态（成功/失败/重试） 38 | alt 任务失败/重试 39 | batch -> mq: 4.重新发送分片消息 40 | activate mq 41 | deactivate mq 42 | end 43 | activate batch_db 44 | deactivate batch_db 45 | @enduml -------------------------------------------------------------------------------- /src/main/resources/logback-spring.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | ${logPattern} 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /cmd/rocketmq: -------------------------------------------------------------------------------- 1 | 2 | # aliyun centos安装java8 3 | yum install java-1.8.0-openjdk 4 | cd /usr/lib/javm/java 5 | vi /etc/profile 6 | JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.352.b08-2.el7_9.x86_64/jre 7 | PATH=$JAVA_HOME/bin:$PATH 8 | export JAVA_HOME PATH 9 | source /etc/profile 10 | 11 | # 安装rocketmq（前提安装java $JAVA_HOME) 12 | wget https://archive.apache.org/dist/rocketmq/4.9.4/rocketmq-all-4.9.4-bin-release.zip 13 | unzip rocketmq-ala-4.9.4-bin-release.zip 14 | 15 | # 启动rocketmq 16 | cd /root/rocketmq-all-4.9.4-bin-release/bin 17 | nohup sh ./mqnamesrv & 18 | nohup sh ./mqbroker -n 39.101.66.97:9876 -c ../conf/broker.conf & 19 | tail -f nohup.out 20 | 21 | ## 启动失败 22 | vi runserver.sh 23 | JAVA_OPT="${JAVA_OPT} -server -Xms256m -Xmx256m -Xmn128m -XX:MetaspaceSize=128m -XX:MaxMetaspaceSize=320m" 24 | vi runbroker.sh 25 | JAVA_OPT="${JAVA_OPT} -server -Xms1g -Xmx1g" 26 | 27 | # 安装控制台 28 | cd ~ 29 | yum install git && git clone https://github.com/apache/rocketmq-dashboard.git 30 | cd rocketmq-dashboard/ 31 | mvn spring-boot:run 32 | http://127.0.0.1:8080/#/ 33 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/module/enums/TaskSplitStatusEnum.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.module.enums; 2 | 3 | public enum TaskSplitStatusEnum { 4 | /** 5 | * 任务初始化 6 | */ 7 | INIT(0), 8 | /** 9 | * 任务处理中：已分发->未完成 10 | */ 11 | EXECUTING(1), 12 | /** 13 | * 任务完结：全部成功 14 | */ 15 | SUCCESS(2), 16 | /** 17 | * 任务完结：部分失败 18 | */ 19 | STOP(3); 20 | int status; 21 | 22 | public static TaskSplitStatusEnum valueOf(Integer status) { 23 | if (status == null) { 24 | throw new RuntimeException(); 25 | } 26 | for (TaskSplitStatusEnum statusEnum : values()) { 27 | if (statusEnum.getStatus() == status) { 28 | return statusEnum; 29 | } 30 | } 31 | throw new RuntimeException("status not supported"); 32 | } 33 | 34 | public int getStatus() { 35 | return status; 36 | } 37 | 38 | public void setStatus(int status) { 39 | this.status = status; 40 | } 41 | 42 | TaskSplitStatusEnum(int status) { 43 | this.status = status; 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/core/TaskScheduler.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.core; 2 | 3 | import org.apache.rocketmq.client.apis.ClientException; 4 | 5 | public class TaskScheduler { 6 | 7 | private TaskService taskService; 8 | 9 | //cron 1分钟 10 | public void schedule() throws ClientException { 11 | long start = System.currentTimeMillis(); 12 | long gap = (60 - 5) * 1000L;//任务1分钟调度一次，每次任务执行55秒， 13 | StopChecker checker = new StopChecker(start + gap); 14 | 15 | //分发任务 16 | taskService.dispatch(checker); 17 | //扫描任务是否完成 18 | taskService.scan(checker); 19 | } 20 | 21 | 22 | public static class StopChecker { 23 | long deadLine; 24 | 25 | public StopChecker(long deadLine) { 26 | this.deadLine = deadLine; 27 | } 28 | 29 | /*** 30 | * 检查是否要中止任务 31 | * 返回true代表需要中止 32 | */ 33 | boolean check() { 34 | if (deadLine == -1) { 35 | return false; 36 | } 37 | return System.currentTimeMillis() >= deadLine; 38 | } 39 | 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/repository/mapper/TaskMapper.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.repository.mapper; 2 | 3 | import com.wyj.task.repository.entity.TaskPO; 4 | import com.wyj.task.repository.entity.TaskSplitPO; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | public interface TaskMapper { 10 | int insert(TaskPO taskPO); 11 | 12 | int batchInsertSplit(List splitPO); 13 | 14 | /** 15 | * 查询待分发的任务 16 | */ 17 | List queryTaskSplitToDispatch(long idStart, int limit); 18 | 19 | /** 20 | * 更新任务分片状态：初始化->执行中 21 | */ 22 | int updateTaskSplitStatus(long splitId, int preStatus, int targetStatus); 23 | 24 | /** 25 | * 更新任务执行状态，自增执行次数 26 | * 27 | * @param targetStatus 若为null，代表不修改状态 28 | */ 29 | void updateTaskSplitExecStatus(Long splitId, Integer targetStatus); 30 | 31 | /** 32 | * 查询未终态的任务 33 | */ 34 | List queryNotFinishTaskIds(long idStart, int limit); 35 | 36 | /** 37 | * 根据任务id统计分片的状态 38 | */ 39 | List querySplitStatusMapByTaskIds(List taskIds); 40 | 41 | /** 42 | * 根据id查询任务 43 | */ 44 | TaskPO queryTask(Long taskId); 45 | 46 | /** 47 | * 更新任务终态 48 | */ 49 | void updateTaskStatus(Long taskId, int taskStatus); 50 | } 51 | -------------------------------------------------------------------------------- /src/test/java/com/wyj/task/BatchTaskApplicationTests.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task; 2 | 3 | import com.wyj.task.core.TaskScheduler; 4 | import com.wyj.test.BatchTaskApplication; 5 | import com.wyj.task.core.TaskService; 6 | import com.wyj.test.impl.MyTaskTypeEnum; 7 | import org.apache.rocketmq.client.apis.ClientException; 8 | import org.junit.jupiter.api.Test; 9 | import org.springframework.boot.test.context.SpringBootTest; 10 | 11 | import javax.annotation.Resource; 12 | import java.util.Date; 13 | import java.util.Random; 14 | 15 | @SpringBootTest(classes = BatchTaskApplication.class) 16 | class BatchTaskApplicationTests { 17 | 18 | @Resource 19 | private TaskService taskService; 20 | @Resource 21 | private TaskApi taskClient; 22 | 23 | @Test 24 | public void testCreateSimpleTask() { 25 | taskClient.submit(MyTaskTypeEnum.SIMPLE_FLUSH_CACHE, new Date(), String.valueOf(new Random().nextInt(100000))); 26 | } 27 | 28 | @Test 29 | public void testCreateMultiTask() { 30 | taskClient.submit(MyTaskTypeEnum.MULTI_TASK_TEST, new Date(), null); 31 | } 32 | 33 | @Test 34 | public void testDispatch() throws ClientException { 35 | taskService.dispatch(new TaskScheduler.StopChecker(-1)); 36 | } 37 | 38 | @Test 39 | public void testConsume() { 40 | } 41 | 42 | @Test 43 | public void testScan() { 44 | taskService.scan(new TaskScheduler.StopChecker(-1)); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/repository/TaskRepository.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.repository; 2 | 3 | import com.wyj.task.module.enums.TaskSplitStatusEnum; 4 | import com.wyj.task.module.enums.TaskStatusEnum; 5 | import com.wyj.task.module.Task; 6 | import com.wyj.task.module.TaskSplit; 7 | 8 | import java.util.List; 9 | import java.util.Map; 10 | 11 | public interface TaskRepository { 12 | 13 | /** 14 | * 初始化任务+任务分片 15 | */ 16 | Task init(Task task, List taskSplit); 17 | 18 | /** 19 | * 获取需要分发的任务分片 20 | */ 21 | List queryTaskSplitToDispatch(long idStart, int limit); 22 | 23 | /** 24 | * 同步任务分片状态 25 | */ 26 | boolean statusSplit2Executing(long splitId, TaskSplitStatusEnum preStatus, TaskSplitStatusEnum targetStatus); 27 | 28 | /** 29 | * 更新任务执行状态，自增执行次数 30 | * 31 | * @param targetStatus 若为null，代表不修改状态 32 | */ 33 | void updateSplitExecStatus(Long splitId, TaskSplitStatusEnum targetStatus); 34 | 35 | /** 36 | * 同步任务分片状态 37 | */ 38 | void synTaskStatus(long id, TaskSplitStatusEnum preStatus, TaskSplitStatusEnum targetStatus); 39 | 40 | /** 41 | * 查询已开始、未完结的任务 42 | */ 43 | List loadNoFinishTasksIds(long idStart, int limit); 44 | 45 | /** 46 | * 根据任务id统计分片的状态 47 | * 48 | * @return key：taskId，value#key：taskSplitStatus，value#value：count 49 | */ 50 | Map> querySplitStatusMapByTaskIds(List taskIds); 51 | 52 | Task queryTask(Long taskId); 53 | 54 | /** 55 | * 更新任务终态 56 | */ 57 | void updateTaskFinalizeStatus(Long taskId, TaskStatusEnum taskStatus); 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/core/TaskStrategyContext.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.core; 2 | 3 | import com.wyj.task.TaskStrategy; 4 | import com.wyj.task.module.enums.TaskTypeEnum; 5 | import org.springframework.beans.factory.InitializingBean; 6 | import org.springframework.context.annotation.Bean; 7 | import org.springframework.context.annotation.DependsOn; 8 | import org.springframework.stereotype.Component; 9 | import org.springframework.stereotype.Service; 10 | 11 | import javax.annotation.PostConstruct; 12 | import javax.annotation.Resource; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.Map; 16 | import java.util.function.Function; 17 | import java.util.stream.Collectors; 18 | 19 | @Component("taskStrategyContext") 20 | public class TaskStrategyContext { 21 | @Resource 22 | private List taskStrategyList; 23 | 24 | private static Map taskStrategyMap; 25 | 26 | @PostConstruct 27 | public void init() { 28 | taskStrategyMap = taskStrategyList.stream().collect(Collectors.toMap( 29 | taskServiceTemplate -> taskServiceTemplate.getTaskType().getType(), 30 | Function.identity() 31 | )); 32 | } 33 | 34 | public static TaskStrategy getTask(Integer taskType) { 35 | return taskStrategyMap.get(taskType); 36 | } 37 | 38 | public static List getTasks() { 39 | return new ArrayList<>(taskStrategyMap.values()); 40 | } 41 | 42 | public static String getTaskTopic(Integer taskType) { 43 | String suffix = getTask(taskType).getTaskType().toString(); 44 | return "task_topic_" + suffix; 45 | } 46 | 47 | public static String getTaskConsumerGroup(Integer taskType) { 48 | String suffix = getTask(taskType).getTaskType().toString(); 49 | return "task_consumer_group_" + suffix; 50 | } 51 | 52 | 53 | public static TaskStrategy getTask(TaskTypeEnum taskType) { 54 | return taskStrategyMap.get(taskType.getType()); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/test/impl/MultiTaskStrategy.java: -------------------------------------------------------------------------------- 1 | 2 | package com.wyj.test.impl; 3 | 4 | import com.wyj.task.JobShardingStrategy; 5 | import com.wyj.task.TaskHandler; 6 | import com.wyj.task.TaskStrategy; 7 | import com.wyj.task.module.Task; 8 | import com.wyj.task.module.TaskSplit; 9 | import com.wyj.task.module.enums.TaskExecResult; 10 | import com.wyj.task.util.JsonUtil; 11 | import org.springframework.stereotype.Service; 12 | 13 | import java.util.*; 14 | import java.util.concurrent.ConcurrentHashMap; 15 | 16 | @Service 17 | public class MultiTaskStrategy implements TaskStrategy { 18 | 19 | @Override 20 | public MyTaskTypeEnum getTaskType() { 21 | return MyTaskTypeEnum.MULTI_TASK_TEST; 22 | } 23 | 24 | @Override 25 | public JobShardingStrategy shardingStrategy() { 26 | return task -> { 27 | List splits = new ArrayList<>(); 28 | for (int i = 0; i < 100; i++) { 29 | TaskSplit taskSplit = TaskSplit.init(i + "", task); 30 | splits.add(taskSplit); 31 | } 32 | return splits; 33 | }; 34 | } 35 | 36 | private static final Map splitRetryRecord = new ConcurrentHashMap<>(); 37 | 38 | @Override 39 | public TaskHandler taskHandler() { 40 | return new TaskHandler() { 41 | @Override 42 | public TaskExecResult execute(TaskSplit split) { 43 | //mock 44 | splitRetryRecord.putIfAbsent(split.getBizData(), 1); 45 | //进行2000次重试后再返回成功 46 | int count = splitRetryRecord.get(split.getBizData()); 47 | if (count <= 100) { 48 | splitRetryRecord.put(split.getBizData(), count + 1); 49 | try { 50 | Thread.sleep(100); 51 | } catch (InterruptedException e) { 52 | throw new RuntimeException(e); 53 | } 54 | return TaskExecResult.RETRY; 55 | } 56 | splitRetryRecord.remove(split.getBizData()); 57 | return TaskExecResult.SUCCESS; 58 | 59 | } 60 | 61 | @Override 62 | public void reduce(Task task) { 63 | System.out.println("finalize:" + JsonUtil.obj2String(task)); 64 | } 65 | }; 66 | } 67 | 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/test/impl/SimpleTaskStrategy.java: -------------------------------------------------------------------------------- 1 | package com.wyj.test.impl; 2 | 3 | import com.wyj.task.JobShardingStrategy; 4 | import com.wyj.task.TaskHandler; 5 | import com.wyj.task.module.enums.TaskExecResult; 6 | import com.wyj.task.module.Task; 7 | import com.wyj.task.module.TaskSplit; 8 | import com.wyj.task.TaskStrategy; 9 | import com.wyj.task.util.JsonUtil; 10 | import org.springframework.stereotype.Service; 11 | 12 | import java.util.*; 13 | 14 | @Service 15 | public class SimpleTaskStrategy implements TaskStrategy { 16 | 17 | @Override 18 | public com.wyj.task.module.enums.TaskTypeEnum getTaskType() { 19 | return MyTaskTypeEnum.SIMPLE_FLUSH_CACHE; 20 | } 21 | 22 | @Override 23 | public JobShardingStrategy shardingStrategy() { 24 | return task -> { 25 | //提取task中的业务字段 26 | String strategyId = task.getBizData(); 27 | List storeIdList = queryStoreIdByStrategyIds(strategyId); 28 | 29 | //对task结合业务属性进行分片 30 | Map splitMap = new HashMap<>(); 31 | for (String storeId : storeIdList) { 32 | Long key = Long.parseLong(storeId) % 1000; 33 | 34 | TaskSplit taskSplit = splitMap.get(key); 35 | if (taskSplit == null) { 36 | taskSplit = TaskSplit.init(JsonUtil.obj2String(SimpleCache.init(strategyId)), task); 37 | splitMap.put(key, taskSplit); 38 | } 39 | SimpleCache strategy = JsonUtil.string2Obj(taskSplit.getBizData(), SimpleCache.class); 40 | taskSplit.setBizData(JsonUtil.obj2String(strategy)); 41 | } 42 | 43 | return new ArrayList<>(splitMap.values()); 44 | }; 45 | } 46 | 47 | 48 | //mock，此处模拟数据库查询 49 | //为了减少没用的分片(按照门店id分片），此处进行数据库查询，根据关联的门店id，仅分发有任务的分片 50 | private List queryStoreIdByStrategyIds(String strategyId) { 51 | List storeIds = new ArrayList<>(); 52 | for (int i = 0; i < 20; i++) { 53 | storeIds.add(strategyId + new Random().nextInt(10000)); 54 | } 55 | return storeIds; 56 | } 57 | 58 | @Override 59 | public TaskHandler taskHandler() { 60 | return new TaskHandler() { 61 | @Override 62 | public TaskExecResult execute(TaskSplit split) { 63 | //1.获取当前任务所有门店 64 | //2.处理门店数据 65 | //3.返回结果 66 | //mock 67 | int random = new Random().nextInt(4); 68 | if (random == 0) { 69 | return TaskExecResult.SUCCESS; 70 | } 71 | if (random == 1) { 72 | return TaskExecResult.STOP; 73 | } 74 | if (random == 2) { 75 | return TaskExecResult.RETRY; 76 | } 77 | //timeout 78 | try { 79 | Thread.sleep(3000 * 60); 80 | return TaskExecResult.SUCCESS; 81 | } catch (InterruptedException e) { 82 | throw new RuntimeException(e); 83 | } 84 | } 85 | 86 | @Override 87 | public void reduce(Task task) { 88 | System.out.println("finalize:" + JsonUtil.obj2String(task)); 89 | } 90 | }; 91 | } 92 | 93 | 94 | } 95 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | ## 一、概述 2 | 该框架用于分布式环境下批量任务分发处理 3 | 4 | InfoQ文章：[分布式任务批处理技术选型与实践](https://xie.infoq.cn/article/7ecb2f26d6c0b6f000af6994f) 5 | 6 | ### 1.主要功能： 7 | ``` 8 | 1.业务系统提供分片规则，创建任务 9 | 2.任务分发，由框架完成 10 | 3.执行任务，回调业务任务处理器 11 | 4.任务完结，回调业务完结处理器 12 | ``` 13 | 14 | ### 2.特性功能： 15 | ``` 16 | 任务分发，可用于集群下多机器处理任务分片 17 | 任务可视，基于Mysql数据库的任务管理 18 | ``` 19 | 20 | ### 3.依赖的技术有： 21 | ``` 22 | Mysql 23 | RocketMQ 24 | Spring（当前未从项目拆离） 25 | Schedule（当前还未集成，需要业务系统手动调度） 26 | ``` 27 | ## 二、架构 28 | ### 1.部署架构 29 | ![img.png](img/img.png) 30 | ### 2.系统架构 31 | ![img_2.png](img/img_2.png) 32 | ### 3.技术架构 33 | ![img_1.png](img/img_1.png) 34 | ### 4.状态机 35 | ![img_3.png](img/img_3.png) 36 | 37 | ## 三、快速开始 38 | ### 0.准备数据库 39 | ``` 40 | CREATE TABLE `task` ( 41 | `id` bigint unsigned NOT NULL AUTO_INCREMENT, 42 | `status` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, 43 | `create_time` datetime NOT NULL, 44 | `update_time` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP, 45 | `task_type` int NOT NULL, 46 | `task_time` datetime NOT NULL, 47 | `biz_data` varchar(1000) DEFAULT NULL, 48 | PRIMARY KEY (`id`) 49 | ) ENGINE=InnoDB AUTO_INCREMENT=13 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 50 | 51 | CREATE TABLE `task_split` ( 52 | `id` bigint unsigned NOT NULL AUTO_INCREMENT, 53 | `task_id` bigint NOT NULL, 54 | `status` varchar(10) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci NOT NULL, 55 | `create_time` datetime NOT NULL, 56 | `update_time` datetime NOT NULL ON UPDATE CURRENT_TIMESTAMP, 57 | `task_type` int NOT NULL, 58 | `task_time` datetime NOT NULL, 59 | `exec_count` bigint NOT NULL DEFAULT '0', 60 | `biz_data` varchar(1000) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci DEFAULT NULL, 61 | PRIMARY KEY (`id`) 62 | ) ENGINE=InnoDB AUTO_INCREMENT=618 DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci; 63 | ``` 64 | ### 1.注册任务 65 | **a.确定任务类型** 66 | 67 | 实现`TaskTypeEnum`接口，通过枚举类维护不同的任务类型 68 | 69 | ``` 70 | public enum MyTaskTypeEnum implements TaskTypeEnum { 71 | SIMPLE_FLUSH_CACHE(0), MULTI_TASK_TEST(1); 72 | } 73 | ``` 74 | **b.实现任务策略** 75 | 实现`TaskStrategy`接口，实现以下功能： 76 | - 返回任务类型 77 | - 提供任务分片规则 78 | - 提供业务处理器（任务处理逻辑+完结回调） 79 | 80 | ``` 81 | public class MyTaskStrategy implements TaskStrategy { 82 | TaskTypeEnum getTaskType(){} 83 | JobShardingStrategy shardingStrategy(){} 84 | TaskHandler handler(){} 85 | } 86 | 87 | public interface JobShardingStrategy { 88 | List sharding(Task task); 89 | } 90 | 91 | public interface TaskHandler { 92 | TaskExecResult execute(TaskSplit split); 93 | void reduce(Task task); 94 | } 95 | ``` 96 | 97 | ### 2.初始化消息队列 98 | 为了保证不同任务消费者线程之间互不影响，不同的任务通过不同的 topic 隔离。 99 | - topic 为`task_topic_ + ${task_name} ` 100 | - consumer.group 为`task_consumer_group_ + ${task_name}`。 101 | 102 | 如SIMPLE_FLUSH_CACHE这个任务： 103 | - topic `为task_topic_SIMPLE_FLUSH_CACHE`， 104 | - consumer.group `为task_consumer_group_SIMPLE_FLUSH_CACHE`。 105 | 106 | ``` 107 | rocketmq.name-server=39.101.66.97:9876 108 | ``` 109 | 110 | ### 3.创建任务 111 | ``` 112 | @Resource 113 | private TaskApi taskClient; 114 | 115 | @Test 116 | public void testCreateSimpleTask() { 117 | taskClient.submit(MyTaskTypeEnum.SIMPLE_FLUSH_CACHE, new Date(), String.valueOf(new Random().nextInt(100000))); 118 | } 119 | ``` 120 | 121 | 122 | ### 4.配置调度系统 123 | 当前没有集成调度工具，可通过公司技术栈的分布式调度框架完成。 124 | 当前手动调度示范： 125 | ``` 126 | @Resource 127 | private TaskService taskService; 128 | 129 | @Test 130 | public void testDispatch() { 131 | taskService.dispatch(new TaskScheduler.StopChecker(-1)); 132 | } 133 | ``` -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/repository/transfer/TaskTransfer.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.repository.transfer; 2 | 3 | import com.wyj.task.module.enums.TaskStatusEnum; 4 | import com.wyj.task.module.Task; 5 | import com.wyj.task.module.TaskSplit; 6 | import com.wyj.task.repository.entity.TaskPO; 7 | import com.wyj.task.repository.entity.TaskSplitPO; 8 | 9 | import java.util.Date; 10 | import java.util.List; 11 | import java.util.stream.Collectors; 12 | 13 | public class TaskTransfer { 14 | 15 | public static Task tranfer(TaskPO taskPO) { 16 | Task task = new Task(); 17 | task.setId(taskPO.getId()); 18 | task.setStatus(taskPO.getStatus()); 19 | task.setTaskType(taskPO.getTaskType()); 20 | task.setCreateTime(taskPO.getCreateTime()); 21 | task.setUpdateTime(taskPO.getUpdateTime()); 22 | task.setTaskTime(taskPO.getTaskTime()); 23 | task.setBizData(taskPO.getBizData()); 24 | return task; 25 | } 26 | 27 | public static TaskPO tranfer(Task task) { 28 | TaskPO taskPO = new TaskPO(); 29 | taskPO.setStatus(task.getStatus()); 30 | taskPO.setTaskType(task.getTaskType()); 31 | taskPO.setCreateTime(task.getCreateTime()); 32 | taskPO.setUpdateTime(task.getUpdateTime()); 33 | taskPO.setTaskTime(task.getTaskTime()); 34 | taskPO.setBizData(task.getBizData()); 35 | return taskPO; 36 | } 37 | 38 | public static Task init(Integer type, Date taskTime, String bizData) { 39 | Task task = new Task(); 40 | task.setStatus(TaskStatusEnum.INIT.getStatus()); 41 | task.setTaskType(type); 42 | task.setTaskTime(taskTime); 43 | task.setBizData(bizData); 44 | return task; 45 | } 46 | 47 | public static TaskSplit initSplit(String bizData, Task task) { 48 | TaskSplit taskSplit = new TaskSplit(); 49 | taskSplit.setStatus(TaskStatusEnum.INIT.getStatus()); 50 | taskSplit.setTaskType(task.getTaskType()); 51 | taskSplit.setTaskTime(task.getTaskTime()); 52 | taskSplit.setBizData(bizData); 53 | return taskSplit; 54 | } 55 | 56 | public static List tranfer(TaskPO task, List taskSplit) { 57 | return taskSplit.stream().map(fTaskSplit -> tranfer(task, fTaskSplit)).collect(Collectors.toList()); 58 | } 59 | 60 | public static TaskSplitPO tranfer(TaskPO task, TaskSplit taskSplit) { 61 | TaskSplitPO splitPO = new TaskSplitPO(); 62 | splitPO.setTaskId(task.getId()); 63 | splitPO.setCreateTime(task.getCreateTime()); 64 | splitPO.setUpdateTime(task.getUpdateTime()); 65 | splitPO.setTaskTime(task.getTaskTime()); 66 | splitPO.setTaskType(task.getTaskType()); 67 | 68 | splitPO.setStatus(taskSplit.getStatus()); 69 | splitPO.setBizData(taskSplit.getBizData()); 70 | return splitPO; 71 | } 72 | 73 | public static List tranferSplitList(List splitPO) { 74 | return splitPO.stream().map(TaskTransfer::transferSplit).collect(Collectors.toList()); 75 | } 76 | 77 | public static TaskSplit transferSplit(TaskSplitPO splitPO) { 78 | TaskSplit taskSplit = new TaskSplit(); 79 | taskSplit.setId(splitPO.getId()); 80 | taskSplit.setTaskId(splitPO.getTaskId()); 81 | taskSplit.setStatus(splitPO.getStatus()); 82 | taskSplit.setTaskType(splitPO.getTaskType()); 83 | taskSplit.setCreateTime(splitPO.getCreateTime()); 84 | taskSplit.setUpdateTime(splitPO.getUpdateTime()); 85 | taskSplit.setTaskTime(splitPO.getTaskTime()); 86 | taskSplit.setExec_count(splitPO.getExecCount()); 87 | taskSplit.setBizData(splitPO.getBizData()); 88 | return taskSplit; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 4.0.0 5 | 6 | org.springframework.boot 7 | spring-boot-starter-parent 8 | 2.7.5 9 | 10 | 11 | 12 | com.wyj 13 | task 14 | 0.0.1-SNAPSHOT 15 | batch-task 16 | batch-task 17 | 18 | 1.8 19 | 20 | 21 | 22 | 23 | 24 | org.codehaus.jackson 25 | jackson-mapper-asl 26 | 1.9.13 27 | 28 | 29 | 30 | 31 | org.apache.rocketmq 32 | rocketmq-spring-boot-starter 33 | 2.2.2 34 | 35 | 36 | org.apache.rocketmq 37 | rocketmq-client-java 38 | 5.0.2 39 | 40 | 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-starter-web 45 | 46 | 47 | 48 | mysql 49 | mysql-connector-java 50 | runtime 51 | 52 | 53 | 54 | org.mybatis.spring.boot 55 | mybatis-spring-boot-starter 56 | 2.2.0 57 | 58 | 59 | 60 | org.springframework.boot 61 | spring-boot-starter-jdbc 62 | 63 | 64 | 65 | org.springframework.boot 66 | spring-boot-devtools 67 | runtime 68 | true 69 | 70 | 71 | 72 | org.projectlombok 73 | lombok 74 | true 75 | 76 | 77 | 78 | org.springframework.boot 79 | spring-boot-starter-test 80 | test 81 | 82 | 83 | 84 | 85 | 86 | 87 | org.springframework.boot 88 | spring-boot-maven-plugin 89 | 90 | 91 | 92 | org.projectlombok 93 | lombok 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/repository/impl/TaskRepositoryImpl.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.repository.impl; 2 | 3 | import com.wyj.task.module.enums.TaskSplitStatusEnum; 4 | import com.wyj.task.module.enums.TaskStatusEnum; 5 | import com.wyj.task.module.Task; 6 | import com.wyj.task.module.TaskSplit; 7 | import com.wyj.task.repository.TaskRepository; 8 | import com.wyj.task.repository.entity.TaskPO; 9 | import com.wyj.task.repository.entity.TaskSplitPO; 10 | import com.wyj.task.repository.mapper.TaskMapper; 11 | import com.wyj.task.repository.transfer.TaskTransfer; 12 | import org.springframework.stereotype.Service; 13 | import org.springframework.util.CollectionUtils; 14 | 15 | import javax.annotation.Resource; 16 | import java.util.*; 17 | 18 | @Service 19 | public class TaskRepositoryImpl implements TaskRepository { 20 | 21 | @Resource 22 | private TaskMapper taskMapper; 23 | 24 | private String mqTopicName; 25 | 26 | @Override 27 | public Task init(Task task, List taskSplit) { 28 | TaskPO taskPO = TaskTransfer.tranfer(task); 29 | taskPO.setStatus(TaskStatusEnum.INIT.getStatus()); 30 | taskPO.setCreateTime(new Date()); 31 | taskPO.setUpdateTime(new Date()); 32 | taskMapper.insert(taskPO); 33 | 34 | List splitPO = TaskTransfer.tranfer(taskPO, taskSplit); 35 | taskMapper.batchInsertSplit(splitPO); 36 | 37 | return TaskTransfer.tranfer(taskPO); 38 | } 39 | 40 | @Override 41 | public List queryTaskSplitToDispatch(long idStart, int limit) { 42 | List splitPO = taskMapper.queryTaskSplitToDispatch(idStart, limit); 43 | return TaskTransfer.tranferSplitList(splitPO); 44 | } 45 | 46 | @Override 47 | public boolean statusSplit2Executing(long splitId, TaskSplitStatusEnum preStatus, TaskSplitStatusEnum targetStatus) { 48 | return taskMapper.updateTaskSplitStatus(splitId, preStatus.getStatus(), targetStatus.getStatus()) != 0; 49 | } 50 | 51 | @Override 52 | public void updateSplitExecStatus(Long splitId, TaskSplitStatusEnum targetStatus) { 53 | taskMapper.updateTaskSplitExecStatus(splitId, targetStatus == null ? null : targetStatus.getStatus()); 54 | } 55 | 56 | @Override 57 | public void synTaskStatus(long id, TaskSplitStatusEnum preStatus, TaskSplitStatusEnum targetStatus) { 58 | 59 | } 60 | 61 | @Override 62 | public List loadNoFinishTasksIds(long idStart, int limit) { 63 | return taskMapper.queryNotFinishTaskIds(idStart, limit); 64 | } 65 | 66 | @Override 67 | public Map> querySplitStatusMapByTaskIds(List taskIds) { 68 | if (CollectionUtils.isEmpty(taskIds)) { 69 | return Collections.emptyMap(); 70 | } 71 | 72 | List groupList = taskMapper.querySplitStatusMapByTaskIds(taskIds); 73 | 74 | Map> map = new HashMap<>(); 75 | for (Map result : groupList) { 76 | long count = (long) (result.get("value")); 77 | String[] keys = ((String) result.get("key")).split("_"); 78 | long taskId = Long.parseLong(keys[0]); 79 | Integer status = Integer.parseInt(keys[1]); 80 | 81 | Map taskIdMap = map.computeIfAbsent(taskId, k -> new HashMap<>()); 82 | taskIdMap.put(TaskSplitStatusEnum.valueOf(status), (int) count); 83 | } 84 | return map; 85 | } 86 | 87 | @Override 88 | public Task queryTask(Long taskId) { 89 | TaskPO taskPO = taskMapper.queryTask(taskId); 90 | return TaskTransfer.tranfer(taskPO); 91 | } 92 | 93 | @Override 94 | public void updateTaskFinalizeStatus(Long taskId, TaskStatusEnum taskStatus) { 95 | taskMapper.updateTaskStatus(taskId,taskStatus.getStatus()); 96 | } 97 | 98 | 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/util/JsonUtil.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.util; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import org.codehaus.jackson.map.DeserializationConfig; 5 | import org.codehaus.jackson.map.ObjectMapper; 6 | import org.codehaus.jackson.map.SerializationConfig; 7 | import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion; 8 | import org.codehaus.jackson.type.JavaType; 9 | import org.codehaus.jackson.type.TypeReference; 10 | import org.springframework.util.StringUtils; 11 | 12 | import java.text.SimpleDateFormat; 13 | 14 | 15 | @Slf4j 16 | public class JsonUtil { 17 | 18 | private static ObjectMapper objectMapper = new ObjectMapper(); 19 | 20 | static { 21 | //对象的所有字段全部列入 22 | objectMapper.setSerializationInclusion(Inclusion.ALWAYS); 23 | 24 | //取消默认转换timestamps形式 25 | objectMapper.configure(SerializationConfig.Feature.WRITE_DATES_AS_TIMESTAMPS, false); 26 | 27 | //忽略空Bean转json的错误 28 | objectMapper.configure(SerializationConfig.Feature.FAIL_ON_EMPTY_BEANS, false); 29 | 30 | //所有的日期格式都统一为以下的样式，即yyyy-MM-dd HH:mm:ss 31 | objectMapper.setDateFormat(new SimpleDateFormat("yyyy-MM-dd HH:mm:ss")); 32 | 33 | //忽略在json字符串中存在，但是在java对象中不存在对应属性的情况。防止错误 34 | objectMapper.configure(DeserializationConfig.Feature.FAIL_ON_UNKNOWN_PROPERTIES, false); 35 | } 36 | 37 | /** 38 | * @Description: 对象转字符串 39 | * @Auther: GALAace 40 | * @Date: 2019/6/21 23:15 41 | */ 42 | public static String obj2String(T obj) { 43 | if (obj == null) { 44 | return null; 45 | } 46 | try { 47 | return obj instanceof String ? (String) obj : objectMapper.writeValueAsString(obj); 48 | } catch (Exception e) { 49 | log.warn("Parse Object to String error", e); 50 | return null; 51 | } 52 | } 53 | 54 | /** 55 | * @Description: 对象转的字符串（格式化后） 56 | * @Auther: GALAace 57 | * @Date: 2019/6/21 23:15 58 | */ 59 | public static String obj2StringPretty(T obj) { 60 | if (obj == null) { 61 | return null; 62 | } 63 | try { 64 | return obj instanceof String ? (String) obj : objectMapper.writerWithDefaultPrettyPrinter().writeValueAsString(obj); 65 | } catch (Exception e) { 66 | log.warn("Parse Object to String error", e); 67 | return null; 68 | } 69 | } 70 | 71 | /** 72 | * @Description: 字符串转对象 73 | * @Auther: GALAace 74 | * @Date: 2019/6/21 23:17 75 | */ 76 | public static T string2Obj(String str, Class clazz) { 77 | if (StringUtils.isEmpty(str) || clazz == null) { 78 | return null; 79 | } 80 | 81 | try { 82 | return clazz.equals(String.class) ? (T) str : objectMapper.readValue(str, clazz); 83 | } catch (Exception e) { 84 | log.warn("Parse String to Object error", e); 85 | return null; 86 | } 87 | } 88 | 89 | /** 90 | * @Description: 字符串转复杂对象（List，Map，Set等） 91 | * @Auther: GALAace 92 | * @Date: 2019/6/22 0:20 93 | */ 94 | public static T string2Obj(String str, TypeReference typeReference) { 95 | if (StringUtils.isEmpty(str) || typeReference == null) { 96 | return null; 97 | } 98 | try { 99 | return (T) (typeReference.getType().equals(String.class) ? str : objectMapper.readValue(str, typeReference)); 100 | } catch (Exception e) { 101 | log.warn("Parse String to Object error", e); 102 | return null; 103 | } 104 | } 105 | 106 | /** 107 | * @Description: 字符串转复杂对象（可变长） 108 | * @Auther: GALAace 109 | * @Date: 2019/6/22 0:21 110 | */ 111 | public static T string2Obj(String str, Class collectionClass, Class... elementClasses) { 112 | JavaType javaType = objectMapper.getTypeFactory().constructParametricType(collectionClass, elementClasses); 113 | try { 114 | return objectMapper.readValue(str, javaType); 115 | } catch (Exception e) { 116 | log.warn("Parse String to Object error", e); 117 | return null; 118 | } 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/main/resources/mapper/task-mapper.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | id 10 | ,status, task_type, create_time, update_time, task_time,biz_data 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | id 24 | , task_id, status, task_type, create_time, update_time, task_time,biz_data,exec_count 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | insert into task 41 | (status, task_type, create_time, update_time, task_time, biz_data) 42 | values (#{status}, #{taskType}, #{createTime}, #{updateTime}, #{taskTime}, #{bizData}) 43 | 44 | 45 | 46 | INSERT INTO task_split 47 | (task_id, status, task_type, create_time, update_time, task_time,biz_data,exec_count) 48 | VALUES 49 | 50 | (#{item.taskId},#{item.status}, #{item.taskType}, #{item.createTime}, #{item.updateTime}, 51 | #{item.taskTime},#{item.bizData},0) 52 | 53 | 54 | 55 | 63 | 64 | update task_split 65 | set status= #{targetStatus}, 66 | update_time=sysdate() 67 | where id = #{splitId} 68 | and status = #{preStatus} 69 | 70 | 71 | update task_split 72 | set exec_count=exec_count + 1, 73 | update_time=sysdate() 74 | 75 | ,status= #{targetStatus} 76 | 77 | where id = #{splitId} 78 | 79 | 88 | 89 | 100 | 101 | 107 | 108 | 109 | update task 110 | set status=#{taskStatus}, 111 | update_time=sysdate() 112 | where id = #{taskId} 113 | 114 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/core/CoreTaskService.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.core; 2 | 3 | import com.wyj.task.TaskApi; 4 | import com.wyj.task.module.enums.TaskSplitStatusEnum; 5 | import com.wyj.task.module.enums.TaskStatusEnum; 6 | import com.wyj.task.module.enums.TaskTypeEnum; 7 | import com.wyj.task.module.Task; 8 | import com.wyj.task.module.TaskSplit; 9 | import com.wyj.task.repository.TaskRepository; 10 | import com.wyj.task.util.JsonUtil; 11 | import org.apache.rocketmq.client.apis.ClientException; 12 | import org.apache.rocketmq.client.apis.ClientServiceProvider; 13 | import org.apache.rocketmq.client.apis.message.Message; 14 | import org.apache.rocketmq.client.apis.producer.Producer; 15 | import org.apache.rocketmq.spring.core.RocketMQTemplate; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | import org.springframework.messaging.support.MessageBuilder; 19 | import org.springframework.stereotype.Service; 20 | import org.springframework.util.CollectionUtils; 21 | 22 | import javax.annotation.Resource; 23 | import java.nio.charset.StandardCharsets; 24 | import java.util.Date; 25 | import java.util.List; 26 | import java.util.Map; 27 | 28 | @Service 29 | public class CoreTaskService implements TaskService, TaskApi { 30 | 31 | private final Logger logger = LoggerFactory.getLogger(this.getClass()); 32 | @Resource 33 | private TaskRepository taskRepository; 34 | @Resource 35 | private RocketMQTemplate rocketMQTemplate; 36 | 37 | @Override 38 | public void submit(TaskTypeEnum taskType, Date taskTime, String bizData) { 39 | //1.分片 40 | Task task = Task.init(taskType.getType(), taskTime, bizData); 41 | List taskSplit = TaskStrategyContext.getTask(taskType).shardingStrategy().sharding(task); 42 | 43 | //2.数据库task、taskSplit 44 | taskRepository.init(task, taskSplit); 45 | } 46 | 47 | @Override 48 | public void dispatch(TaskScheduler.StopChecker checker) throws ClientException { 49 | List taskSplitList; 50 | long idStart = 0; 51 | int limit = 100; 52 | while (!checker.check()) { 53 | //查询待处理任务：任务状态，任务处理时间 54 | taskSplitList = taskRepository.queryTaskSplitToDispatch(idStart, limit); 55 | if (CollectionUtils.isEmpty(taskSplitList)) { 56 | break; 57 | } 58 | idStart = taskSplitList.get(taskSplitList.size() - 1).getId(); 59 | 60 | //分发+修改状态 61 | for (TaskSplit split : taskSplitList) { 62 | doDispatch(split); 63 | } 64 | } 65 | } 66 | 67 | /** 68 | * 因为是任务，可以重试，所以不需要事务消息 69 | * 先发消息，再更新数据库；如果数据库操作失败，重试即可 70 | * 71 | * @param split 72 | */ 73 | private void doDispatch(TaskSplit split) throws ClientException { 74 | //分发任务 75 | String mqTopic = TaskStrategyContext.getTaskTopic(split.getTaskType()); 76 | rocketMQTemplate.convertAndSend(mqTopic, JsonUtil.obj2String(split)); 77 | 78 | //修改任务状态 79 | taskRepository.statusSplit2Executing(split.getId(), TaskSplitStatusEnum.INIT, TaskSplitStatusEnum.EXECUTING); 80 | 81 | logger.info("task dispatch succ,splitId={},split={}", split.getId(), JsonUtil.obj2String(split)); 82 | } 83 | 84 | @Override 85 | public void scan(TaskScheduler.StopChecker checker) { 86 | long idStart = 0L; 87 | int size = 100; 88 | //查询所有未完结任务 89 | List taskIds = taskRepository.loadNoFinishTasksIds(idStart, size); 90 | //统计未完结任务的分片状态 91 | Map> taskSplitStatusMap = 92 | taskRepository.querySplitStatusMapByTaskIds(taskIds); 93 | 94 | for (Long taskId : taskIds) { 95 | Map taskSplitStatus = taskSplitStatusMap.get(taskId); 96 | //该未完结任务没有分片数据 97 | if (taskSplitStatus == null) { 98 | continue; 99 | } 100 | //该未完结任务仍存在未完结的分片：不处理 101 | if (taskSplitStatus.get(TaskSplitStatusEnum.INIT) != null 102 | || taskSplitStatus.get(TaskSplitStatusEnum.EXECUTING) != null) { 103 | continue; 104 | } 105 | //剩下的情况就是：所有子任务已完结 106 | Task task = taskRepository.queryTask(taskId); 107 | 108 | //1.需要调用finalize，同样，业务系统需要做好幂等处理 109 | TaskStrategyContext.getTask(task.getTaskType()).taskHandler().reduce(task); 110 | 111 | //2.需要更新任务状态：若全部成功，更新为SUCCESS；若部分停止，更新为FINISH 112 | TaskStatusEnum taskStatus = 113 | (taskSplitStatus.get(TaskSplitStatusEnum.STOP) == null || taskSplitStatus.get(TaskSplitStatusEnum.STOP) == 0) 114 | ? TaskStatusEnum.SUCCESS : TaskStatusEnum.FINISH; 115 | taskRepository.updateTaskFinalizeStatus(task.getId(), taskStatus); 116 | } 117 | } 118 | 119 | } 120 | -------------------------------------------------------------------------------- /src/main/java/com/wyj/task/core/TaskMQConsumer.java: -------------------------------------------------------------------------------- 1 | package com.wyj.task.core; 2 | 3 | import com.wyj.task.TaskHandler; 4 | import com.wyj.task.TaskStrategy; 5 | import com.wyj.task.module.TaskSplit; 6 | import com.wyj.task.module.enums.TaskExecResult; 7 | import com.wyj.task.module.enums.TaskSplitStatusEnum; 8 | import com.wyj.task.repository.TaskRepository; 9 | import com.wyj.task.util.JsonUtil; 10 | import org.apache.rocketmq.client.apis.ClientConfiguration; 11 | import org.apache.rocketmq.client.apis.ClientException; 12 | import org.apache.rocketmq.client.apis.ClientServiceProvider; 13 | import org.apache.rocketmq.client.apis.consumer.ConsumeResult; 14 | import org.apache.rocketmq.client.apis.consumer.FilterExpression; 15 | import org.apache.rocketmq.client.apis.consumer.FilterExpressionType; 16 | import org.apache.rocketmq.client.apis.consumer.PushConsumer; 17 | import org.apache.rocketmq.client.consumer.DefaultMQPushConsumer; 18 | import org.apache.rocketmq.client.consumer.listener.ConsumeConcurrentlyStatus; 19 | import org.apache.rocketmq.client.consumer.listener.MessageListenerConcurrently; 20 | import org.apache.rocketmq.client.exception.MQClientException; 21 | import org.apache.rocketmq.common.message.MessageExt; 22 | import org.apache.rocketmq.spring.core.RocketMQTemplate; 23 | import org.slf4j.Logger; 24 | import org.slf4j.LoggerFactory; 25 | import org.springframework.beans.factory.annotation.Value; 26 | import org.springframework.context.annotation.DependsOn; 27 | import org.springframework.stereotype.Component; 28 | 29 | import javax.annotation.PostConstruct; 30 | import javax.annotation.Resource; 31 | import java.util.Collections; 32 | import java.util.List; 33 | import java.util.function.Consumer; 34 | 35 | @DependsOn(value = {"taskStrategyContext"}) 36 | @Component 37 | public class TaskMQConsumer { 38 | 39 | private final Logger logger = LoggerFactory.getLogger(this.getClass()); 40 | @Resource 41 | private TaskRepository taskRepository; 42 | // @Resource 43 | // private RocketMQTemplate rocketMQTemplate; 44 | @Value("${rocketmq.name-server}") 45 | private String nameservAddr; 46 | 47 | @Resource 48 | private TaskStrategyContext taskStrategyContext; 49 | 50 | /** 51 | * 任务层的处理 52 | */ 53 | public TaskExecResult handleWithTaskWrapper(TaskSplit split) { 54 | logger.info("task mq consumer:taskId={},splitId={}", split.getTaskId(), split.getId()); 55 | 56 | //get handler 57 | TaskHandler handler = TaskStrategyContext.getTask(split.getTaskType()).taskHandler(); 58 | 59 | //handle：不进行资源抢占，直接执行，业务系统需要保证幂等 60 | TaskExecResult result = TaskExecResult.RETRY; 61 | boolean error = false; 62 | try { 63 | result = handler.execute(split); 64 | } catch (Exception e) { 65 | logger.error("TaskHandler exec exception,splitId={}", split.getId(), e); 66 | error = true; 67 | } finally { 68 | logger.info("task mq result:taskId={},splitId={},err={},result={}", 69 | split.getTaskId(), split.getId(), error, error ? null : result); 70 | } 71 | 72 | //process result 73 | //情况1：返回：执行成功，（任何状态）->（成功状态） 74 | //情况2：返回：执行终止，（任何状态）->（终止状态） 75 | //情况3：返回：重试，需要保证无限重试次数，这里需要调用客户端重发消息 76 | //情况4：返回：空，可能是客户端忘记实现，或者异常，处理方式同异常 77 | //情况5：返回：异常，需要保证有限重试次数。这里跟随rocketmq即可，通过中间件自动重试 78 | 79 | //空、异常，数据库更新执行次数，消息自动重试 80 | if (error || result == null) { 81 | taskRepository.updateSplitExecStatus(split.getId(), null); 82 | throw new RuntimeException("task exec failed,throw to retry mq"); 83 | } 84 | 85 | if (result == TaskExecResult.SUCCESS) { 86 | //执行成功，数据库：（任何状态）->（成功状态），消息：处理成功 87 | taskRepository.updateSplitExecStatus(split.getId(), TaskSplitStatusEnum.SUCCESS); 88 | 89 | } else if (result == TaskExecResult.STOP) { 90 | //执行终止，数据库（任何状态）->（终止状态），消息：处理成功 91 | taskRepository.updateSplitExecStatus(split.getId(), TaskSplitStatusEnum.STOP); 92 | 93 | } else if (result == TaskExecResult.RETRY) { 94 | //执行终止，数据库更新执行次数，消息：返回成功，重新发送 95 | taskRepository.updateSplitExecStatus(split.getId(), null); 96 | } 97 | return result; 98 | } 99 | 100 | /** 101 | * 消息层的处理 102 | *

103 | * 性能优化：若期望处理时间较短，重试的消息不会放回mq，而是在实例内部进行重试，最多重试1000次 104 | * 对于重试的任务，期望时间可能超过mq超时时间（或重试超过1000次），将放弃优化，放回到mq 105 | */ 106 | public void handleWithMessageWrapper(TaskSplit split) { 107 | //mq默认超时时间，单次执行需要控制在该时间内 108 | long deadLine = System.currentTimeMillis() + (15 - 1) * 60 * 1000L; 109 | int retry = 0; 110 | long maxCost = 1; 111 | 112 | TaskExecResult result; 113 | long predict; 114 | do { 115 | retry++; 116 | //before-统计信息 117 | long start = System.currentTimeMillis(); 118 | 119 | //处理消息 120 | result = handleWithTaskWrapper(split); 121 | 122 | //after-统计信息 123 | maxCost = Math.max(maxCost, System.currentTimeMillis() - start); 124 | predict = System.currentTimeMillis() + maxCost; 125 | } while (result == TaskExecResult.RETRY && predict < deadLine && retry < 1000); 126 | 127 | //对于重试的任务，期望时间可能超过mq超时时间（或重试超过1000次），将放弃优化，放回到mq 128 | if (result == TaskExecResult.RETRY) { 129 | String mqTopic = TaskStrategyContext.getTaskTopic(split.getTaskType()); 130 | // rocketMQTemplate.convertAndSend(mqTopic, JsonUtil.obj2String(split)); 131 | } 132 | } 133 | 134 | /** 135 | * 因为不同的任务使用不同的topic、consumer group，所以这里根据task注册不同的consumer listener 136 | * topic = task_topic_ + ${task_name} 137 | * consumerGroup = task_consumer_group_ + ${task_name} 138 | */ 139 | @PostConstruct 140 | public void init() throws Exception { 141 | //1.获取所有的任务 142 | List tasks = TaskStrategyContext.getTasks(); 143 | 144 | //2.注册各个任务的consumer 145 | for (TaskStrategy task : tasks) { 146 | String mqTopic = TaskStrategyContext.getTaskTopic(task.getTaskType().getType()); 147 | String consumerGroup = TaskStrategyContext.getTaskConsumerGroup(task.getTaskType().getType()); 148 | initConsumer(mqTopic, consumerGroup, this::handleWithMessageWrapper); 149 | } 150 | } 151 | 152 | private void initConsumer(String mqTopic, String consumerGroup, Consumer c) throws MQClientException { 153 | DefaultMQPushConsumer consumer = new DefaultMQPushConsumer(); 154 | consumer.setNamesrvAddr(nameservAddr); 155 | consumer.setConsumerGroup(consumerGroup); 156 | consumer.subscribe(mqTopic, "*"); 157 | //timeout 158 | // consumer.setConsumeTimeout(15); 159 | // consumer.setConsumeThreadMax(20); 160 | 161 | consumer.registerMessageListener((MessageListenerConcurrently) (msgs, context) -> { 162 | //获取消息 163 | for (MessageExt ext : msgs) { 164 | TaskSplit split = JsonUtil.string2Obj(new String(ext.getBody()), TaskSplit.class); 165 | c.accept(split); 166 | } 167 | //处理结果 168 | return ConsumeConcurrentlyStatus.CONSUME_SUCCESS; 169 | }); 170 | //启动消费者 171 | consumer.start(); 172 | } 173 | 174 | 175 | } 176 | --------------------------------------------------------------------------------