├── .gitignore ├── README.MD ├── pom.xml └── src ├── main ├── java │ └── me │ │ └── j360 │ │ └── rad │ │ ├── action │ │ ├── ActionCenter.java │ │ ├── ActionCenterDelegate.java │ │ ├── ActionCenterFactory.java │ │ └── ActionConfig.java │ │ ├── core │ │ └── ActionTemplate.java │ │ ├── exception │ │ └── ActionException.java │ │ ├── model │ │ ├── Action.java │ │ └── ActionTarget.java │ │ ├── redis │ │ └── RedisActionCenter.java │ │ ├── spring │ │ └── ActionCenterFactoryBean.java │ │ └── util │ │ └── NamedThreadFactory.java └── resources │ └── applicationContext.xml └── test ├── java └── me │ └── j360 │ └── rad │ └── test │ └── ActionTest.java └── logback-test.xml /.gitignore: -------------------------------------------------------------------------------- 1 | *.jar 2 | *.war 3 | *~ 4 | *.class 5 | *.lock 6 | *.DS_Store 7 | *.swp 8 | *.out 9 | target/ 10 | *.iml 11 | *.ipr 12 | *.iws 13 | .settings/ 14 | .classpath 15 | .project 16 | .metadata/ 17 | .idea/ 18 | logs/ 19 | 20 | dependency-reduced-pom.xml 21 | aixforce-search/data 22 | *.rdb 23 | db/ 24 | *.versionsBackup 25 | .flattened-pom.xml 26 | 27 | node_modules/ 28 | sass/ 29 | package.json 30 | Gruntfile.js 31 | config.js 32 | .sass-cache/ 33 | 34 | 35 | -------------------------------------------------------------------------------- /README.MD: -------------------------------------------------------------------------------- 1 | j360-redis-async-db 2 | ============== 3 | 4 | j360系列 - 缓存异步写数据库的框架 5 | 6 | 7 | ===v1.0功能实现=== 8 | - 针对统计类的接口缓存（pv、uv等）信息后，需要异步将数据同步到持久层，先想几天再码代码 9 | 10 | 11 | ##相关前置说明## 12 | - 依赖Spring配置 13 | - 依赖redis读写接口 14 | - 需要另外存储redis对象(key、value) 15 | - 写数据库默认mysql(未来支持mongodb、es等) 16 | 17 | 18 | ##异步写策略## 19 | - 1.达到方法调用次数增量（可配置，默认100），触发同步数据库事件 20 | - 2.超过方法调用的时间增量（可配置，默认30s），触发同步数据库事件 21 | - 3.方法调用后会补一条异步触发消息（可配置，默认30s）,满足条件（1.2未执行）触发同步数据库事件 22 | 23 | ##同步数据库## 24 | - 1.单条数据同步:立即执行 25 | - 2.批量执行同步:按照以下策略 26 | 27 | ##批量同步数据库策略## 28 | - 1.时间增量（可配置，默认3s） 29 | - 2.内容增量（可配置，默认1000条，请根据数据库性能配置） 30 | 31 | ##分布式支持## 32 | - 1.批量更新：单个jvm使用原子类锁进行队列执行 33 | - 2.分布式环境默认使用单条数据同步，批量同步需要依赖分布式队列框架（activemq、rabbitmq、kafka等）进行id分片策略同步(v2.0支持) 34 | - 3.id分片下的无状态集群需要抢分布式锁执行同步功能（redis、zk） 35 | 36 | 37 | ##简单版本定时策略## 38 | - 使用异步写3，通过队列定时30S后消费，消费时判断上一次同步时间是否超出30S，否则消费丢弃 -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | me.j360.rad 8 | j360-redis-async-db 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 1.14.8 13 | 18.0 14 | 1.4.2.RELEASE 15 | 4.1.6.RELEASE 16 | 1.1.2 17 | 18 | 19 | 20 | org.projectlombok 21 | lombok 22 | ${lombok.version} 23 | 24 | 25 | com.google.guava 26 | guava 27 | ${guava.version} 28 | 29 | 30 | 31 | ch.qos.logback 32 | logback-classic 33 | ${logback.version} 34 | 35 | 36 | ch.qos.logback 37 | logback-core 38 | ${logback.version} 39 | 40 | 41 | ch.qos.logback 42 | logback-access 43 | ${logback.version} 44 | 45 | 46 | 47 | 48 | junit 49 | junit 50 | 4.11 51 | test 52 | 53 | 54 | org.springframework 55 | spring-test 56 | 4.1.6.RELEASE 57 | test 58 | 59 | 60 | 61 | org.springframework 62 | spring-tx 63 | ${spring.version} 64 | 65 | 66 | org.springframework 67 | spring-web 68 | ${spring.version} 69 | 70 | 71 | org.springframework 72 | spring-webmvc 73 | ${spring.version} 74 | 75 | 76 | org.springframework 77 | spring-core 78 | ${spring.version} 79 | 80 | 81 | org.springframework 82 | spring-expression 83 | ${spring.version} 84 | 85 | 86 | 87 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/action/ActionCenter.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.action; 2 | 3 | import me.j360.rad.model.Action; 4 | import me.j360.rad.model.ActionTarget; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Package: me.j360.rad.action 10 | * User: min_xu 11 | * Date: 16/4/13 下午4:10 12 | * 说明： 13 | */ 14 | public interface ActionCenter { 15 | 16 | public void writeToCache(ActionTarget actionTarget); 17 | public void writeToDatabase(Action action); 18 | 19 | public void writeToDatabase(List actions); 20 | 21 | public ActionTarget getActionTarget(long targetId); 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/action/ActionCenterDelegate.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.action; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import me.j360.rad.model.Action; 6 | import me.j360.rad.model.ActionTarget; 7 | import me.j360.rad.util.NamedThreadFactory; 8 | 9 | import java.util.Date; 10 | import java.util.List; 11 | import java.util.concurrent.Executors; 12 | import java.util.concurrent.ScheduledExecutorService; 13 | import java.util.concurrent.ScheduledFuture; 14 | import java.util.concurrent.TimeUnit; 15 | import java.util.concurrent.atomic.AtomicBoolean; 16 | 17 | /** 18 | * Package: me.j360.rad.action 19 | * User: min_xu 20 | * Date: 16/4/13 下午4:12 21 | * 说明： 22 | */ 23 | public class ActionCenterDelegate implements ActionCenter{ 24 | 25 | @Setter 26 | @Getter 27 | private ActionCenter actionCenter; 28 | 29 | private ScheduledExecutorService executor; 30 | 31 | private ScheduledFuture scheduledFuture; 32 | 33 | private AtomicBoolean flushing = new AtomicBoolean(false); 34 | 35 | public ActionCenterDelegate(){ 36 | 37 | } 38 | 39 | //入口 40 | public void call(Action action) { 41 | long targetId = 0; 42 | ActionTarget actionTarget = getActionTarget(targetId); 43 | //写入缓存新的数据 44 | writeToCache(actionTarget); 45 | 46 | boolean isWrite = false; 47 | 48 | if(actionTarget.getCount() == actionTarget.getNextCount()){ 49 | //同步到数据库 50 | isWrite = true; 51 | writeToDatabase(action); 52 | } 53 | if(isWrite == false){ 54 | if(new Date(actionTarget.getTimestamp()).after(new Date())){ 55 | //同步到数据库 56 | isWrite = true; 57 | writeToDatabase(action); 58 | } 59 | } 60 | 61 | //补一个定时队列去判断是否有必要在下一个call之前检查 62 | schedulerCheck(action); 63 | } 64 | 65 | public void writeToCache(ActionTarget actionTarget) { 66 | actionCenter.writeToCache(actionTarget); 67 | } 68 | 69 | public void writeToDatabase(Action action) { 70 | actionCenter.writeToDatabase(action); 71 | } 72 | 73 | @Override 74 | public void writeToDatabase(List actions) { 75 | 76 | } 77 | 78 | @Override 79 | public ActionTarget getActionTarget(long targetId) { 80 | return actionCenter.getActionTarget(targetId); 81 | } 82 | 83 | private void schedulerCheck(final Action action){ 84 | // 85 | executor = Executors.newSingleThreadScheduledExecutor(new NamedThreadFactory("LazyAction")); 86 | scheduledFuture = executor.scheduleWithFixedDelay(new Runnable() { 87 | @Override 88 | public void run() { 89 | try { 90 | if (flushing.compareAndSet(false, true)) { 91 | // 92 | writeToDatabase(action); 93 | } 94 | } catch (Throwable t) { 95 | 96 | } 97 | } 98 | }, 30, 30, TimeUnit.SECONDS); 99 | 100 | 101 | } 102 | 103 | } 104 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/action/ActionCenterFactory.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.action; 2 | 3 | /** 4 | * Package: me.j360.rad.action 5 | * User: min_xu 6 | * Date: 16/4/13 下午4:09 7 | * 说明： 8 | */ 9 | public class ActionCenterFactory { 10 | 11 | private ActionCenter actionCenter; 12 | 13 | public ActionCenter getActionCenter() { 14 | return actionCenter; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/action/ActionConfig.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.action; 2 | 3 | /** 4 | * Package: me.j360.rad.action 5 | * User: min_xu 6 | * Date: 16/4/13 下午4:11 7 | * 说明：异步写数据库的配置 8 | */ 9 | public class ActionConfig { 10 | 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/core/ActionTemplate.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.core; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import me.j360.rad.action.ActionCenter; 6 | import me.j360.rad.action.ActionCenterDelegate; 7 | import me.j360.rad.model.Action; 8 | 9 | /** 10 | * Package: me.j360.rad.core 11 | * User: min_xu 12 | * Date: 16/4/13 下午4:08 13 | * 说明： 14 | */ 15 | public class ActionTemplate{ 16 | 17 | @Getter 18 | @Setter 19 | private ActionCenterDelegate actionDelegate; 20 | 21 | public void call(Action action){ 22 | //调用delegate 23 | actionDelegate.call(action); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/exception/ActionException.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.exception; 2 | 3 | /** 4 | * Package: me.j360.rad.exception 5 | * User: min_xu 6 | * Date: 16/4/13 下午4:12 7 | * 说明： 8 | */ 9 | public class ActionException extends RuntimeException{ 10 | 11 | public ActionException(){ 12 | super(); 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/model/Action.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.model; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | /** 7 | * Package: me.j360.rad.model 8 | * User: min_xu 9 | * Date: 16/4/13 下午4:03 10 | * 说明： 11 | */ 12 | public class Action { 13 | @Getter 14 | @Setter 15 | private long actionId; 16 | @Getter 17 | @Setter 18 | private String table; 19 | @Getter 20 | @Setter 21 | private String field; 22 | @Getter 23 | @Setter 24 | private String targetId; 25 | @Getter 26 | @Setter 27 | private long number; 28 | @Getter 29 | @Setter 30 | private String timestamp; 31 | @Getter 32 | @Setter 33 | private long delay; 34 | @Getter 35 | @Setter 36 | private String destimate; 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/model/ActionTarget.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.model; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | 6 | /** 7 | * Package: me.j360.rad.model 8 | * User: min_xu 9 | * Date: 16/4/13 下午4:53 10 | * 说明： 11 | */ 12 | public class ActionTarget { 13 | 14 | @Getter 15 | @Setter 16 | private long targetId; 17 | @Getter 18 | @Setter 19 | private String timestamp; 20 | @Getter 21 | @Setter 22 | private long count; 23 | @Getter 24 | @Setter 25 | private long nextCount; 26 | @Getter 27 | @Setter 28 | private long data; 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/redis/RedisActionCenter.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.redis; 2 | 3 | import me.j360.rad.action.ActionCenter; 4 | import me.j360.rad.model.Action; 5 | import me.j360.rad.model.ActionTarget; 6 | 7 | import java.util.List; 8 | 9 | /** 10 | * Package: me.j360.rad.redis 11 | * User: min_xu 12 | * Date: 16/4/13 下午4:25 13 | * 说明： 14 | */ 15 | public class RedisActionCenter implements ActionCenter { 16 | 17 | 18 | public void writeToCache(ActionTarget actionTarget) { 19 | 20 | } 21 | 22 | @Override 23 | public void writeToDatabase(Action action) { 24 | //将单条数据写到数据库 25 | 26 | } 27 | 28 | @Override 29 | public void writeToDatabase(List actions) { 30 | //批量将数据写到数据库 31 | 32 | } 33 | 34 | @Override 35 | public ActionTarget getActionTarget(long targetId) { 36 | 37 | return new ActionTarget(); 38 | } 39 | 40 | 41 | private boolean checkDelay(){ 42 | return true; 43 | } 44 | 45 | private void asyncCall(){ 46 | 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/spring/ActionCenterFactoryBean.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.spring; 2 | 3 | import lombok.Getter; 4 | import lombok.Setter; 5 | import me.j360.rad.action.ActionCenter; 6 | import me.j360.rad.action.ActionCenterFactory; 7 | import org.springframework.beans.factory.DisposableBean; 8 | import org.springframework.beans.factory.FactoryBean; 9 | import org.springframework.beans.factory.InitializingBean; 10 | 11 | /** 12 | * Package: me.j360.rad.spring 13 | * User: min_xu 14 | * Date: 16/4/13 下午7:18 15 | * 说明： 16 | */ 17 | public class ActionCenterFactoryBean implements FactoryBean, 18 | InitializingBean, DisposableBean { 19 | 20 | @Getter 21 | @Setter 22 | private ActionCenter actionCenter; 23 | 24 | public void destroy() throws Exception { 25 | 26 | } 27 | 28 | public ActionCenter getObject() throws Exception { 29 | return actionCenter; 30 | } 31 | 32 | public Class getObjectType() { 33 | return actionCenter.getClass(); 34 | } 35 | 36 | public boolean isSingleton() { 37 | return false; 38 | } 39 | 40 | public void afterPropertiesSet() throws Exception { 41 | 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/me/j360/rad/util/NamedThreadFactory.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.util; 2 | 3 | import java.util.concurrent.ThreadFactory; 4 | import java.util.concurrent.atomic.AtomicInteger; 5 | 6 | /** 7 | * Package: me.j360.rad.util 8 | * User: min_xu 9 | * Date: 16/4/13 下午5:22 10 | * 说明： 11 | */ 12 | public class NamedThreadFactory implements ThreadFactory { 13 | private static final AtomicInteger POOL_SEQ = new AtomicInteger(1); 14 | 15 | private final AtomicInteger mThreadNum = new AtomicInteger(1); 16 | 17 | private final String mPrefix; 18 | 19 | private final boolean mDaemo; 20 | 21 | private final ThreadGroup mGroup; 22 | 23 | public NamedThreadFactory() { 24 | this("pool-" + POOL_SEQ.getAndIncrement(), false); 25 | } 26 | 27 | public NamedThreadFactory(String prefix) { 28 | this(prefix, false); 29 | } 30 | 31 | public NamedThreadFactory(String prefix, boolean daemo) { 32 | mPrefix = prefix + "-thread-"; 33 | mDaemo = daemo; 34 | SecurityManager s = System.getSecurityManager(); 35 | mGroup = (s == null) ? Thread.currentThread().getThreadGroup() : s.getThreadGroup(); 36 | } 37 | 38 | public Thread newThread(Runnable runnable) { 39 | String name = mPrefix + mThreadNum.getAndIncrement(); 40 | Thread ret = new Thread(mGroup, runnable, name, 0); 41 | ret.setDaemon(mDaemo); 42 | return ret; 43 | } 44 | 45 | public ThreadGroup getThreadGroup() { 46 | return mGroup; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/main/resources/applicationContext.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/test/java/me/j360/rad/test/ActionTest.java: -------------------------------------------------------------------------------- 1 | package me.j360.rad.test; 2 | 3 | import lombok.extern.slf4j.Slf4j; 4 | import me.j360.rad.core.ActionTemplate; 5 | import me.j360.rad.model.Action; 6 | import org.junit.Test; 7 | import org.junit.runner.RunWith; 8 | import org.springframework.beans.factory.annotation.Autowired; 9 | import org.springframework.test.context.ActiveProfiles; 10 | import org.springframework.test.context.ContextConfiguration; 11 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 12 | import org.springframework.test.context.transaction.TransactionConfiguration; 13 | 14 | /** 15 | * Package: me.j360.rad.test 16 | * User: min_xu 17 | * Date: 16/4/13 下午7:29 18 | * 说明： 19 | */ 20 | @Slf4j 21 | @ActiveProfiles("dev") 22 | @RunWith(SpringJUnit4ClassRunner.class) 23 | @TransactionConfiguration(defaultRollback=false) 24 | @ContextConfiguration(locations = {"classpath:/applicationContext.xml"}) 25 | public class ActionTest { 26 | 27 | @Autowired 28 | private ActionTemplate actionTemplate; 29 | 30 | @Test 31 | public void call(){ 32 | Action action = new Action(); 33 | actionTemplate.call(action); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/test/logback-test.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | %date{HH:mm:ss.SSS} [%thread] %-5level %logger{36} - %msg%n 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | --------------------------------------------------------------------------------