├── src ├── main │ ├── resources │ │ ├── META-INF │ │ │ ├── services │ │ │ │ └── io.sdmq.queue.core.ConsumeQueueProvider │ │ │ ├── spring.factories │ │ │ └── spring-configuration-metadata.json │ │ └── config │ │ │ ├── application.yml │ │ │ └── logback.xml │ ├── java │ │ └── io │ │ │ └── sdmq │ │ │ ├── queue │ │ │ ├── rabbitmq │ │ │ │ └── package-info.java │ │ │ ├── redis │ │ │ │ ├── package-info.java │ │ │ │ ├── event │ │ │ │ │ ├── JobEvent.java │ │ │ │ │ ├── JobEventListener.java │ │ │ │ │ ├── RedisJobTraceEvent.java │ │ │ │ │ ├── RedisJobEventListener.java │ │ │ │ │ └── JobEventBus.java │ │ │ │ ├── support │ │ │ │ │ ├── Lifecycle.java │ │ │ │ │ ├── DistributedLock.java │ │ │ │ │ ├── RedisQueueProperties.java │ │ │ │ │ ├── RedisDistributedLock.java │ │ │ │ │ └── RedisSupport.java │ │ │ │ ├── JobWrapp.java │ │ │ │ ├── JobOperationService.java │ │ │ │ ├── ready │ │ │ │ │ ├── ReadyQueueManager.java │ │ │ │ │ └── RealTimeTask.java │ │ │ │ ├── bucket │ │ │ │ │ ├── BucketQueueManager.java │ │ │ │ │ └── BucketTask.java │ │ │ │ ├── JobOperationServiceImpl.java │ │ │ │ ├── RedisQueueImpl.java │ │ │ │ └── RdbStore.java │ │ │ ├── core │ │ │ │ ├── ConsumeQueueProvider.java │ │ │ │ ├── Job.java │ │ │ │ └── Queue.java │ │ │ ├── cqp │ │ │ │ ├── JmsConsumeQueue.java │ │ │ │ └── ConsoleConsumeQueue.java │ │ │ └── JobMsg.java │ │ │ ├── util │ │ │ ├── RdbOperation.java │ │ │ ├── Status.java │ │ │ ├── Constants.java │ │ │ ├── BlockUtils.java │ │ │ ├── StartGetReady.java │ │ │ ├── NamedUtil.java │ │ │ ├── JobIdGenerator.java │ │ │ ├── SnowFlake.java │ │ │ ├── IpUtils.java │ │ │ ├── FastJsonConvert.java │ │ │ ├── DateUtils.java │ │ │ └── ResponseMessage.java │ │ │ ├── leader │ │ │ ├── LeaderManager.java │ │ │ ├── ServerNode.java │ │ │ ├── LeaderWorkListener.java │ │ │ └── SimpleLeaderManager.java │ │ │ ├── exception │ │ │ ├── JobNotFoundException.java │ │ │ ├── ConsumeQueueException.java │ │ │ └── DelayQueueException.java │ │ │ ├── restful │ │ │ ├── AdminController.java │ │ │ └── JobController.java │ │ │ ├── common │ │ │ ├── extension │ │ │ │ ├── ExtNamed.java │ │ │ │ ├── SPI.java │ │ │ │ └── ExtensionLoader.java │ │ │ ├── autoconfigure │ │ │ │ ├── RocketMQProperties.java │ │ │ │ ├── RocketmqAutoConfiguration.java │ │ │ │ ├── QueueHealthIndicator.java │ │ │ │ ├── RegistryProperties.java │ │ │ │ ├── HealthAutoConfiguration.java │ │ │ │ ├── LeaderAutoConfiguration.java │ │ │ │ ├── RedisQueueAutoConfiguration.java │ │ │ │ ├── MessageProducer.java │ │ │ │ └── DruidConfig.java │ │ │ └── conf │ │ │ │ ├── FailedEventListener.java │ │ │ │ ├── StartEventListener.java │ │ │ │ ├── RedisConfig.java │ │ │ │ └── AppEnvContext.java │ │ │ └── Bootstarp.java │ ├── docker │ │ ├── Dockerfile │ │ └── k8s-sdmq.yaml │ ├── bin │ │ ├── env.sh │ │ ├── dockerenv.sh │ │ └── sdmq │ ├── assembly │ │ └── assembly.xml │ └── sql │ │ └── sdmq.sql └── test │ ├── java │ └── io │ │ └── sdmq │ │ ├── TestDelayQueue.java │ │ ├── AppTest.java │ │ ├── RoundRobinTest.java │ │ ├── FixTest.java │ │ ├── DistributedLockTest.java │ │ └── transactional │ │ └── RedisTemplateTransactional.java │ └── resources │ └── config │ ├── application.yml │ └── logback.xml ├── .gitignore ├── .travis.yml ├── README.md └── pom.xml /src/main/resources/META-INF/services/io.sdmq.queue.core.ConsumeQueueProvider: -------------------------------------------------------------------------------- 1 | io.sdmq.queue.cqp.JmsConsumeQueue 2 | io.sdmq.queue.cqp.ConsoleConsumeQueue 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Package Files # 4 | *.jar 5 | *.war 6 | *.ear 7 | *.iml 8 | target 9 | classes 10 | *.idea 11 | *.soft.logs 12 | soft.logs 13 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/rabbitmq/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * using rabbitmq support 3 | * Created by Xs.Tao on 2017/7/18. 4 | */ 5 | package io.sdmq.queue.rabbitmq; -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * default useing redis 3 | * 4 | * Created by Xs.Tao on 2017/7/18. 5 | */ 6 | package io.sdmq.queue.redis; -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/RdbOperation.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/19. 5 | */ 6 | public enum RdbOperation { 7 | INSERT, UPDATE, DELETE 8 | } 9 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring.factories: -------------------------------------------------------------------------------- 1 | org.springframework.boot.autoconfigure.EnableAutoConfiguration=io.sdmq.common.autoconfigure.RedisQueueAutoConfiguration,io.sdmq.common.autoconfigure.LeaderAutoConfiguration -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/event/JobEvent.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.event; 2 | 3 | import io.sdmq.queue.JobMsg; 4 | 5 | /** 6 | * Created by Xs.Tao on 2017/7/19. 7 | */ 8 | public interface JobEvent { 9 | 10 | JobMsg getJob(); 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/support/Lifecycle.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.support; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/19. 5 | */ 6 | public interface Lifecycle { 7 | 8 | void start(); 9 | 10 | void stop(); 11 | 12 | boolean isRunning(); 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/leader/LeaderManager.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.leader; 2 | 3 | import io.sdmq.queue.redis.support.Lifecycle; 4 | 5 | /** 6 | * Created by Xs.Tao on 2017/7/28. 7 | */ 8 | public interface LeaderManager extends Lifecycle { 9 | 10 | boolean isLeader(); 11 | 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/event/JobEventListener.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.event; 2 | 3 | import com.google.common.eventbus.Subscribe; 4 | 5 | /** 6 | * Created by Xs.Tao on 2017/7/19. 7 | */ 8 | public interface JobEventListener { 9 | 10 | @Subscribe 11 | void listen(JobEvent event); 12 | 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/exception/JobNotFoundException.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.exception; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/19. 5 | */ 6 | public class JobNotFoundException extends DelayQueueException { 7 | 8 | public JobNotFoundException(String errorMessage, Object... args) { 9 | super(errorMessage, args); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/exception/ConsumeQueueException.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.exception; 2 | 3 | /** 4 | * Created by Xs.Tao on 2018/3/17. 5 | */ 6 | public class ConsumeQueueException extends DelayQueueException { 7 | 8 | public ConsumeQueueException(String errorMessage, Object... args) { 9 | super(errorMessage, args); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/restful/AdminController.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.restful; 2 | 3 | import org.springframework.stereotype.Controller; 4 | import org.springframework.web.bind.annotation.RequestMapping; 5 | 6 | /** 7 | * Created by Xs.Tao on 2017/7/25. 8 | */ 9 | @RequestMapping("/admin") 10 | @Controller 11 | public class AdminController { 12 | 13 | } 14 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | notifications: 2 | email: 3 | recipients: 4 | - hixstao@gmail.com 5 | language: java 6 | 7 | jdk: 8 | - oraclejdk8 9 | 10 | cache: 11 | directories: 12 | - $HOME/.m2 13 | 14 | before_install: 15 | - echo "MAVEN_OPTS='-Dlicense.skip=true -Dmaven.test.skip=true'" > ~/.mavenrc 16 | 17 | script: 18 | - mvn -B clean install 19 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/support/DistributedLock.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.support; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/20. 5 | */ 6 | public interface DistributedLock { 7 | 8 | boolean tryLock(String key); 9 | 10 | boolean tryLock(String key, long timeout); 11 | 12 | boolean lock(String key); 13 | 14 | void unlock(String key); 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/Status.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/18. 5 | */ 6 | public enum Status { 7 | WaitPut,//待加入 8 | Delay,//已经进入延时队列 9 | Ready,//已经出了延时队列 客户端可以方法此数据 10 | Finish,//客户端已经处理完数据了 11 | Delete,//客户端已经把数据删除了 12 | Restore,//手动恢复重发状态/或者是在实时队列中验证时间出现异常 再次放入buck中 13 | ConsumerFailRestore//消费失败 14 | 15 | } 16 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/leader/ServerNode.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.leader; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/28. 5 | */ 6 | public class ServerNode { 7 | 8 | public static final String DIR_SPLITE = "/"; 9 | public static final String ROOT = "dq"; 10 | public static final String LEADERLATCH = DIR_SPLITE + ROOT + DIR_SPLITE + "latch"; 11 | public static String NAMESPACE = "io-sdmq"; 12 | } 13 | -------------------------------------------------------------------------------- /src/main/docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM registry.cn-hangzhou.aliyuncs.com/peachyy/java8base 2 | LABEL maintainer="peachyy" 3 | ENV TZ=Asia/Shanghai 4 | RUN ln -snf /usr/share/zoneinfo/${TZ} /etc/localtime && \ 5 | echo ${TZ} > /etc/timezone 6 | RUN mkdir -p /opt 7 | ADD target/peachyy-sdmq.tar peachyy-sdmq.tar 8 | RUN tar -zxf peachyy-sdmq.tar && cd peachyy-sdmq 9 | WORKDIR /opt/peachyy-sdmq 10 | 11 | ENTRYPOINT ["bin/sdmq","start"] 12 | EXPOSE 6355 -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/core/ConsumeQueueProvider.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.core; 2 | 3 | import io.sdmq.common.extension.SPI; 4 | import io.sdmq.exception.ConsumeQueueException; 5 | 6 | /** 7 | * Created by Xs.Tao on 2018/3/17. 8 | */ 9 | @SPI("consoleCQ") 10 | public interface ConsumeQueueProvider { 11 | 12 | void init(); 13 | 14 | void consumer(Job job) throws ConsumeQueueException; 15 | 16 | void destory(); 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/JobWrapp.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis; 2 | 3 | import io.sdmq.queue.JobMsg; 4 | 5 | /** 6 | * Created by Xs.Tao on 2017/7/24. 7 | */ 8 | public class JobWrapp extends JobMsg { 9 | 10 | private String buckedName; 11 | 12 | public String getBuckedName() { 13 | return buckedName; 14 | } 15 | 16 | public void setBuckedName(String buckedName) { 17 | this.buckedName = buckedName; 18 | } 19 | 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/extension/ExtNamed.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.extension; 2 | 3 | import java.lang.annotation.Documented; 4 | import java.lang.annotation.ElementType; 5 | import java.lang.annotation.Retention; 6 | import java.lang.annotation.RetentionPolicy; 7 | import java.lang.annotation.Target; 8 | 9 | /** 10 | * Created by Xs.Tao on 2018/3/17. 11 | */ 12 | @Documented 13 | @Retention(RetentionPolicy.RUNTIME) 14 | @Target({ElementType.TYPE}) 15 | public @interface ExtNamed { 16 | 17 | String value(); 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/core/Job.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.core; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/8/7. 5 | */ 6 | public interface Job { 7 | 8 | 9 | String getBizKey(); 10 | 11 | 12 | String getTopic(); 13 | 14 | 15 | String getId(); 16 | 17 | 18 | long getDelay(); 19 | 20 | 21 | long getTtl(); 22 | 23 | 24 | String getBody(); 25 | 26 | 27 | long getCreateTime(); 28 | 29 | 30 | int getStatus(); 31 | 32 | 33 | String getSubtopic(); 34 | String getExtendData(); 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/exception/DelayQueueException.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.exception; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/18. 5 | */ 6 | public class DelayQueueException extends RuntimeException { 7 | 8 | private static final long serialVersionUID = 5018901344199973515L; 9 | 10 | public DelayQueueException(final String errorMessage, final Object... args) { 11 | super(String.format(errorMessage, args)); 12 | } 13 | 14 | public DelayQueueException(final Throwable cause) { 15 | super(cause); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/Constants.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | /** 4 | * Created by Xs.Tao on 2018/1/12. 5 | */ 6 | public class Constants { 7 | 8 | public static final String USER_DIR = "user.dir"; 9 | public static String SOFT_HOME_KEY = "soft.home"; 10 | public static String SOFT_LOG_HOME_KEY = "soft.logs"; 11 | public static String SOFT_HOME = System.getProperty(SOFT_HOME_KEY); 12 | public static String SOFT_LOG_HOME = System.getProperty(SOFT_LOG_HOME_KEY); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/extension/SPI.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.extension; 2 | 3 | import java.lang.annotation.Documented; 4 | import java.lang.annotation.ElementType; 5 | import java.lang.annotation.Retention; 6 | import java.lang.annotation.RetentionPolicy; 7 | import java.lang.annotation.Target; 8 | 9 | /** 10 | * Created by Xs.Tao on 2018/3/17. 11 | */ 12 | @Documented 13 | @Retention(RetentionPolicy.RUNTIME) 14 | @Target({ElementType.TYPE}) 15 | public @interface SPI { 16 | 17 | /** 18 | * 默认的实现方式 19 | */ 20 | String value() default ""; 21 | } 22 | -------------------------------------------------------------------------------- /src/test/java/io/sdmq/TestDelayQueue.java: -------------------------------------------------------------------------------- 1 | package io.sdmq; 2 | 3 | import org.springframework.boot.SpringApplication; 4 | import org.springframework.boot.autoconfigure.SpringBootApplication; 5 | import org.springframework.context.ApplicationContext; 6 | 7 | import io.sdmq.util.StartGetReady; 8 | 9 | /** 10 | * Created by Xs.Tao on 2017/7/20. 11 | */ 12 | @SpringBootApplication 13 | public class TestDelayQueue { 14 | 15 | public static void main(String[] args) { 16 | StartGetReady.ready(); 17 | ApplicationContext ctx = SpringApplication.run(TestDelayQueue.class, args); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/core/Queue.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.core; 2 | 3 | import io.sdmq.exception.DelayQueueException; 4 | import io.sdmq.queue.JobMsg; 5 | import io.sdmq.queue.redis.support.Lifecycle; 6 | 7 | /** 8 | * Created by Xs.Tao on 2017/7/18. 9 | */ 10 | public interface Queue extends Lifecycle { 11 | 12 | void push(JobMsg job) throws DelayQueueException; 13 | 14 | 15 | boolean ack(String jobMsgId) throws DelayQueueException; 16 | 17 | long getSize(); 18 | 19 | void clear(); 20 | 21 | boolean delete(String jobMsgId); 22 | 23 | JobMsg getJob(String jobId); 24 | 25 | String getImplementType(); 26 | 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/Bootstarp.java: -------------------------------------------------------------------------------- 1 | package io.sdmq; 2 | 3 | import org.springframework.boot.autoconfigure.SpringBootApplication; 4 | import org.springframework.boot.builder.SpringApplicationBuilder; 5 | import org.springframework.context.ConfigurableApplicationContext; 6 | 7 | import io.sdmq.util.StartGetReady; 8 | 9 | /** 10 | * Created by Xs.Tao on 2017/7/18. 11 | */ 12 | @SpringBootApplication 13 | public class Bootstarp { 14 | 15 | public static void main(String[] args) { 16 | StartGetReady.ready(); 17 | ConfigurableApplicationContext context = 18 | new SpringApplicationBuilder(Bootstarp.class) 19 | .run(args); 20 | 21 | 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/bin/env.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BUILD_ID=5555555555buildid 4 | LOG_HOME=/var/logs/sdmq/ 5 | LOG_PATH=$LOG_HOME 6 | export JAVA_HOME=/usr/local/jdk 7 | JAVA_OPTS=' -server -Xms256m -Xmx256m -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./heap -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dcom.sun.management.jmxremote -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.port=6598 -Dcom.sun.management.jmxremote.authenticate=false 8 | -Dcom.sun.management.jmxremote.ssl=false' 9 | #JAVA_OPTS=' -server -Dcom.sun.management.jmxremote -Djava.rmi.server.hostname=219.239.88.245 -Dcom.sun.management.jmxremote.port=6598 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false' -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/cqp/JmsConsumeQueue.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.cqp; 2 | 3 | import io.sdmq.common.autoconfigure.MessageProducer; 4 | import io.sdmq.common.extension.ExtNamed; 5 | import io.sdmq.exception.ConsumeQueueException; 6 | import io.sdmq.queue.core.ConsumeQueueProvider; 7 | import io.sdmq.queue.core.Job; 8 | 9 | /** 10 | * Created by Xs.Tao on 2018/3/17. 11 | */ 12 | @ExtNamed("jmsCQ") 13 | public class JmsConsumeQueue implements ConsumeQueueProvider { 14 | 15 | @Override 16 | public void init() { 17 | 18 | } 19 | 20 | @Override 21 | public void consumer(Job job) throws ConsumeQueueException { 22 | MessageProducer.send(job); 23 | } 24 | 25 | @Override 26 | public void destory() { 27 | 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/BlockUtils.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | /** 4 | * Created by Xs.Tao on 2017/7/21. 5 | */ 6 | public final class BlockUtils { 7 | 8 | public static final long DEF_SLEEP_TIMES = 100L; 9 | 10 | public static void waitingShortTime() { 11 | sleep(DEF_SLEEP_TIMES); 12 | } 13 | 14 | public static void sleep(final long millis, final boolean isInterrupt) { 15 | try { 16 | Thread.sleep(millis); 17 | } catch (final InterruptedException ex) { 18 | if (isInterrupt) { 19 | Thread.currentThread().interrupt(); 20 | } 21 | 22 | } 23 | } 24 | 25 | public static void sleep(final long millis) { 26 | sleep(millis, false); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/io/sdmq/AppTest.java: -------------------------------------------------------------------------------- 1 | package io.sdmq; 2 | 3 | import junit.framework.Test; 4 | import junit.framework.TestCase; 5 | import junit.framework.TestSuite; 6 | 7 | /** 8 | * Unit test for simple App. 9 | */ 10 | public class AppTest 11 | extends TestCase { 12 | 13 | /** 14 | * Create the test case 15 | * 16 | * @param testName name of the test case 17 | */ 18 | public AppTest(String testName) { 19 | super(testName); 20 | } 21 | 22 | /** 23 | * @return the suite of tests being tested 24 | */ 25 | public static Test suite() { 26 | return new TestSuite(AppTest.class); 27 | } 28 | 29 | /** 30 | * Rigourous Test :-) 31 | */ 32 | public void testApp() { 33 | assertTrue(true); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/main/bin/dockerenv.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | BUILD_ID=5555555555buildid 4 | LOG_HOME=/var/logs/sdmq/ 5 | LOG_PATH=$LOG_HOME 6 | #export JAVA_HOME=/usr/local/jdk 7 | JAVA_OPTS=' -server -Djava.security.egd=file:/dev/./urandom -XX:+UnlockExperimentalVMOptions -XX:+UseCGroupMemoryLimitForHeap -XX:MaxRAMPercentage=80.0 -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=./heap -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dcom.sun.management.jmxremote -Djava.rmi.server.hostname=127.0.0.1 -Dcom.sun.management.jmxremote.port=6598 -Dcom.sun.management.jmxremote.authenticate=false 8 | -Dcom.sun.management.jmxremote.ssl=false' 9 | #JAVA_OPTS=' -server -Dcom.sun.management.jmxremote -Djava.rmi.server.hostname=219.239.88.245 -Dcom.sun.management.jmxremote.port=6598 -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false' -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/event/RedisJobTraceEvent.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.event; 2 | 3 | import io.sdmq.queue.JobMsg; 4 | import io.sdmq.util.RdbOperation; 5 | 6 | /** 7 | * Created by Xs.Tao on 2017/7/19. 8 | */ 9 | public class RedisJobTraceEvent implements JobEvent { 10 | 11 | private JobMsg jobMsg = null; 12 | private RdbOperation operation = RdbOperation.UPDATE; 13 | 14 | public RedisJobTraceEvent(JobMsg jobMsg) { 15 | this.jobMsg = jobMsg; 16 | } 17 | 18 | public RedisJobTraceEvent(JobMsg jobMsg, RdbOperation operation) { 19 | this.jobMsg = jobMsg; 20 | this.operation = operation; 21 | } 22 | 23 | @Override 24 | public JobMsg getJob() { 25 | return jobMsg; 26 | } 27 | 28 | public RdbOperation getOperation() { 29 | return operation; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/cqp/ConsoleConsumeQueue.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.cqp; 2 | 3 | import io.sdmq.common.extension.ExtNamed; 4 | import io.sdmq.exception.ConsumeQueueException; 5 | import io.sdmq.queue.core.ConsumeQueueProvider; 6 | import io.sdmq.queue.core.Job; 7 | import io.sdmq.util.FastJsonConvert; 8 | 9 | /** 10 | * Created by Xs.Tao on 2018/3/17. 11 | */ 12 | @ExtNamed("consoleCQ") 13 | public class ConsoleConsumeQueue implements ConsumeQueueProvider { 14 | 15 | @Override 16 | public void init() { 17 | 18 | } 19 | 20 | @Override 21 | public void consumer(Job job) throws ConsumeQueueException { 22 | System.out.println(String.format("invoke topic %s json:%s", job.getTopic(), 23 | FastJsonConvert.convertObjectToJSON(job))); 24 | } 25 | 26 | @Override 27 | public void destory() { 28 | 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/leader/LeaderWorkListener.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.leader; 2 | 3 | import org.apache.curator.framework.recipes.leader.LeaderLatchListener; 4 | import org.slf4j.Logger; 5 | import org.slf4j.LoggerFactory; 6 | 7 | import io.sdmq.queue.core.Queue; 8 | 9 | /** 10 | * Created by Xs.Tao on 2017/7/28. 11 | */ 12 | public class LeaderWorkListener implements LeaderLatchListener { 13 | 14 | public static final Logger LOGGER = LoggerFactory.getLogger(LeaderWorkListener.class); 15 | private Queue queue; 16 | 17 | @Override 18 | public void isLeader() { 19 | LOGGER.warn("is starting work!"); 20 | queue.start(); 21 | } 22 | 23 | @Override 24 | public void notLeader() { 25 | LOGGER.warn("is stoping work!"); 26 | queue.stop(); 27 | } 28 | 29 | public void setQueue(Queue queue) { 30 | this.queue = queue; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/RocketMQProperties.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | /** 6 | * Created by Xs.Tao on 2017/7/20. 7 | */ 8 | @ConfigurationProperties(prefix = RocketMQProperties.SDMQ_MQ_PREFIX) 9 | public class RocketMQProperties { 10 | 11 | public static final String SDMQ_MQ_PREFIX = "sdmq.reocketmq"; 12 | private String namesrvAddr; 13 | private String filterSourceRoot = "/home/"; 14 | 15 | public String getNamesrvAddr() { 16 | return namesrvAddr; 17 | } 18 | 19 | public void setNamesrvAddr(String namesrvAddr) { 20 | this.namesrvAddr = namesrvAddr; 21 | } 22 | 23 | public String getFilterSourceRoot() { 24 | return filterSourceRoot; 25 | } 26 | 27 | public void setFilterSourceRoot(String filterSourceRoot) { 28 | this.filterSourceRoot = filterSourceRoot; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/event/RedisJobEventListener.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.event; 2 | 3 | import io.sdmq.queue.JobMsg; 4 | import io.sdmq.queue.redis.RdbStore; 5 | import io.sdmq.util.RdbOperation; 6 | import io.sdmq.util.Status; 7 | 8 | /** 9 | * Created by Xs.Tao on 2017/7/19. 10 | */ 11 | public class RedisJobEventListener implements JobEventListener { 12 | 13 | private RdbStore store; 14 | 15 | @Override 16 | public void listen(JobEvent event) { 17 | if (event instanceof RedisJobTraceEvent) { 18 | RedisJobTraceEvent e = (RedisJobTraceEvent) event; 19 | JobMsg job = e.getJob(); 20 | if (e.getOperation() == RdbOperation.INSERT && job.getStatus() != Status.Restore.ordinal()) { 21 | store.insertJob(job); 22 | } else { 23 | store.updateJobsStatus(job); 24 | } 25 | } 26 | } 27 | 28 | public void setStore(RdbStore store) { 29 | this.store = store; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/StartGetReady.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import org.springframework.util.ResourceUtils; 4 | 5 | import java.io.File; 6 | import java.io.FileNotFoundException; 7 | 8 | /** 9 | * Created by Xs.Tao on 2018/1/12. 10 | */ 11 | public class StartGetReady { 12 | 13 | public static void ready() { 14 | if (System.getProperty(Constants.SOFT_HOME_KEY) == null) { 15 | System.setProperty(Constants.SOFT_HOME_KEY, getClazzPathUrl()); 16 | } 17 | if (System.getProperty(Constants.SOFT_LOG_HOME_KEY) == null) { 18 | System.setProperty(Constants.SOFT_LOG_HOME_KEY, "".concat("/logs")); 19 | } 20 | } 21 | 22 | private static String getClazzPathUrl() { 23 | File path = null; 24 | try { 25 | path = new File(ResourceUtils.getURL("classpath:").getPath()); 26 | } catch (FileNotFoundException e) { 27 | e.printStackTrace(); 28 | } 29 | if (!path.exists()) 30 | path = new File(""); 31 | return path.getAbsolutePath(); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/conf/FailedEventListener.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.conf; 2 | 3 | import org.springframework.boot.context.event.ApplicationFailedEvent; 4 | import org.springframework.context.ApplicationContext; 5 | import org.springframework.context.ApplicationListener; 6 | import org.springframework.context.annotation.Configuration; 7 | 8 | import io.sdmq.queue.redis.RedisQueueImpl; 9 | 10 | /** 11 | * Created by Xs.Tao on 2017/7/28. 12 | */ 13 | @Configuration 14 | public class FailedEventListener implements ApplicationListener { 15 | 16 | @Override 17 | public void onApplicationEvent(ApplicationFailedEvent event) { 18 | Throwable throwable = event.getException(); 19 | handler(throwable, event); 20 | } 21 | 22 | private void handler(Throwable throwable, ApplicationFailedEvent event) { 23 | ApplicationContext ctx = event.getApplicationContext(); 24 | if (ctx != null) { 25 | RedisQueueImpl redisQueue = ctx.getBean(RedisQueueImpl.class); 26 | if (redisQueue != null && redisQueue.isRunning()) { 27 | redisQueue.stop(); 28 | } 29 | } 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/RocketmqAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import com.alibaba.rocketmq.client.producer.DefaultMQProducer; 4 | 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 7 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.util.Assert; 11 | 12 | /** 13 | * Created by Xs.Tao on 2017/7/20. 14 | */ 15 | @Configuration 16 | @EnableConfigurationProperties(RocketMQProperties.class) 17 | @ConditionalOnClass(value = {DefaultMQProducer.class}) 18 | public class RocketmqAutoConfiguration { 19 | 20 | @Autowired 21 | private RocketMQProperties properties; 22 | 23 | @Bean(initMethod = "init", destroyMethod = "close") 24 | public MessageProducer newMessageProducer() { 25 | Assert.notNull(properties.getNamesrvAddr(), "请正确配置NamesrvAddr"); 26 | MessageProducer producer = new MessageProducer(); 27 | producer.setNamesrvAddr(properties.getNamesrvAddr()); 28 | return producer; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/NamedUtil.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import com.google.common.base.Joiner; 4 | import com.google.common.collect.Lists; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * Created by Xs.Tao on 2017/7/18. 10 | */ 11 | public class NamedUtil { 12 | 13 | public static final String SPLITE_CHAR = ":"; 14 | public static final String LOCK_CHAR = "LOCK"; 15 | 16 | public static String buildBucketName(String prefix, String name, int index) { 17 | List lst = Lists.newArrayList(); 18 | lst.add(prefix); 19 | lst.add(name); 20 | lst.add(index); 21 | return Joiner.on(SPLITE_CHAR).join(lst); 22 | } 23 | 24 | public static String buildPoolName(String prefix, String name, String pool) { 25 | return Joiner.on(SPLITE_CHAR).join(Lists.newArrayList(prefix, name, pool)); 26 | } 27 | 28 | public static String buildRealTimeName(String prefix, String name, String readTimeName) { 29 | return Joiner.on(SPLITE_CHAR).join(Lists.newArrayList(prefix, name, readTimeName)); 30 | } 31 | 32 | public static String buildLockName(String prefix) { 33 | return Joiner.on(SPLITE_CHAR).join(Lists.newArrayList(prefix.concat(":" + LOCK_CHAR))); 34 | } 35 | 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/main/assembly/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | assembly 3 | 4 | tar.gz 5 | 6 | true 7 | 8 | 9 | 10 | src/main/bin 11 | bin 12 | 0755 13 | 14 | sdmq 15 | 16 | 17 | 18 | 19 | ${project.build.directory}/classes/config 20 | config 21 | 0644 22 | 23 | 24 | 25 | 26 | src/main/bin/env.sh 27 | bin 28 | env.sh 29 | 0755 30 | 31 | 32 | 33 | 34 | lib 35 | 36 | ${project.groupId}:${project.artifactId} 37 | 38 | 39 | 40 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/conf/StartEventListener.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.conf; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.context.ApplicationContext; 6 | import org.springframework.context.ApplicationListener; 7 | import org.springframework.context.annotation.Configuration; 8 | import org.springframework.context.event.ContextRefreshedEvent; 9 | 10 | import io.sdmq.queue.redis.RedisQueueImpl; 11 | 12 | /** 13 | * Created by Xs.Tao on 2017/7/28. 14 | */ 15 | @Configuration 16 | public class StartEventListener implements ApplicationListener { 17 | 18 | public static final Logger LOGGER = LoggerFactory.getLogger(StartEventListener.class); 19 | 20 | @Override 21 | public void onApplicationEvent(ContextRefreshedEvent event) { 22 | ApplicationContext ctx = event.getApplicationContext(); 23 | if (ctx != null) { 24 | RedisQueueImpl redisQueue = ctx.getBean(RedisQueueImpl.class); 25 | String regEnable = AppEnvContext.getProperty("jikexiu.registry.enable", "false"); 26 | if (!redisQueue.isRunning() && Boolean.parseBoolean(regEnable) == false) { 27 | LOGGER.info("starting Queue StandAlone Model ..."); 28 | redisQueue.start(); 29 | } 30 | } 31 | } 32 | 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/event/JobEventBus.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.event; 2 | 3 | import com.google.common.eventbus.AsyncEventBus; 4 | import com.google.common.eventbus.EventBus; 5 | import com.google.common.util.concurrent.MoreExecutors; 6 | 7 | import java.util.concurrent.atomic.AtomicBoolean; 8 | 9 | /** 10 | * Created by Xs.Tao on 2017/7/19. 11 | */ 12 | public class JobEventBus { 13 | 14 | private EventBus bus = null; 15 | private AtomicBoolean register = new AtomicBoolean(false); 16 | 17 | private JobEventBus() { 18 | bus = new AsyncEventBus(MoreExecutors.newDirectExecutorService()); 19 | } 20 | 21 | public static JobEventBus getInstance() { 22 | return LazyHolder.JEB; 23 | } 24 | 25 | public void register(JobEventListener listener) { 26 | if (register.compareAndSet(false, true)) { 27 | bus.register(listener); 28 | } 29 | 30 | } 31 | 32 | public void unregister(JobEventListener listener) { 33 | if (register.get() == true) { 34 | bus.unregister(listener); 35 | } 36 | } 37 | 38 | public void post(JobEvent event) { 39 | if (register.get() == true) { 40 | bus.post(event); 41 | } 42 | 43 | } 44 | 45 | private static class LazyHolder { 46 | 47 | private static final JobEventBus JEB = new JobEventBus(); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/docker/k8s-sdmq.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: apps/v1 2 | kind: Deployment 3 | metadata: 4 | labels: 5 | appname: sdmq 6 | name: sdmq 7 | namespace: default 8 | spec: 9 | replicas: 1 10 | selector: 11 | matchLabels: 12 | appname: sdmq 13 | template: 14 | metadata: 15 | labels: 16 | appname: sdmq 17 | spec: 18 | # imagePullSecrets: 19 | # - name: vps-docker-registry 20 | containers: 21 | - image: sdmq:latest 22 | env: 23 | - name: aliyun_logs_api 24 | value: /var/logs/sdmq/*.log 25 | - name: aliyun_logs_sdmq_logstore 26 | value: sdmq-log 27 | - name: aliyun_logs_tag_tags 28 | value: tag=sdmq 29 | - name: aliyun_logs_sdmq_ttl 30 | value: "365" 31 | volumeMounts: 32 | - name: log-volumn 33 | mountPath: /var/logs/sdmq 34 | name: sdmq 35 | ports: 36 | - containerPort: 6355 37 | resources: 38 | limits: 39 | cpu: "1" 40 | memory: 2048Mi 41 | requests: 42 | cpu: "1" 43 | memory: 2048Mi 44 | 45 | volumes: 46 | - name: log-volumn 47 | emptyDir: {} 48 | 49 | --- 50 | 51 | apiVersion: v1 52 | kind: Service 53 | metadata: 54 | name: sdmq-service 55 | namespace: default 56 | spec: 57 | selector: 58 | appname: sdmq 59 | type: NodePort 60 | ports: 61 | - protocol: TCP 62 | port: 7653 63 | targetPort: 7653 -------------------------------------------------------------------------------- /src/main/sql/sdmq.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat MySQL Data Transfer 3 | 4 | Source Server : 127.0.0.1 5 | Source Server Version : 50615 6 | Source Host : 127.0.0.1:3306 7 | Source Database : sdmq 8 | 9 | Target Server Type : MYSQL 10 | Target Server Version : 50615 11 | File Encoding : 65001 12 | 13 | Date: 2018-06-08 11:19:56 14 | */ 15 | 16 | SET FOREIGN_KEY_CHECKS=0; 17 | 18 | -- ---------------------------- 19 | -- Table structure for t_delay_queue_job 20 | -- ---------------------------- 21 | DROP TABLE IF EXISTS `t_delay_queue_job`; 22 | CREATE TABLE `t_delay_queue_job` ( 23 | `id` varchar(128) NOT NULL, 24 | `bizkey` varchar(128) DEFAULT NULL, 25 | `topic` varchar(128) DEFAULT NULL, 26 | `subtopic` varchar(250) DEFAULT NULL, 27 | `delay` bigint(20) DEFAULT NULL, 28 | `create_time` bigint(20) DEFAULT NULL, 29 | `body` text, 30 | `status` int(11) DEFAULT NULL, 31 | `ttl` int(11) DEFAULT NULL, 32 | `update_time` datetime(3) DEFAULT NULL, 33 | `extend_data` varchar(2000) DEFAULT NULL, 34 | PRIMARY KEY (`id`), 35 | KEY `T_DELAY_QUEUE_JOB_ID_STATUS` (`id`,`status`), 36 | KEY `T_DELAY_QUEUE_JOB_STATUS` (`status`) 37 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 38 | 39 | -- ---------------------------- 40 | -- Table structure for t_delay_queue_job_log 41 | -- ---------------------------- 42 | DROP TABLE IF EXISTS `t_delay_queue_job_log`; 43 | CREATE TABLE `t_delay_queue_job_log` ( 44 | `id` varchar(128) NOT NULL, 45 | `status` int(11) DEFAULT NULL, 46 | `thread` varchar(60) DEFAULT NULL, 47 | `update_time` datetime(3) DEFAULT NULL, 48 | `host` varchar(128) DEFAULT NULL, 49 | KEY `T_DELAY_QUEUE_JOB_LOG_ID_STATUS` (`id`,`status`) 50 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 51 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/JobOperationService.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis; 2 | 3 | import java.util.List; 4 | 5 | import io.sdmq.queue.JobMsg; 6 | import io.sdmq.util.Status; 7 | 8 | /** 9 | * Created by Xs.Tao on 2017/7/19. 10 | */ 11 | public interface JobOperationService { 12 | 13 | /** 14 | * 获取Job元数据 15 | */ 16 | JobMsg getJob(String jobId); 17 | 18 | /** 19 | * 添加Job到元数据池 20 | */ 21 | void addJobToPool(JobMsg jobMsg); 22 | 23 | /** 24 | * 删除元数据此任务 25 | */ 26 | void removeJobToPool(String jobId); 27 | 28 | /** 29 | * 更新元任务池任务的状态 30 | */ 31 | void updateJobStatus(String jobId, Status status); 32 | 33 | /** 34 | * 根据JobId删除元数据 35 | */ 36 | void deleteJobToPool(String jobId); 37 | 38 | /** 39 | * 加一个Job到指定Bucket 40 | */ 41 | void addBucketJob(String bucketName, String JobId, double score); 42 | 43 | /** 44 | * 从指定Bucket删除一个Job 45 | */ 46 | void removeBucketKey(String bucketName, String jobId); 47 | 48 | /** 49 | * 添加一个Job到 可执行队列 50 | */ 51 | void addReadyTime(String readyName, String jobId); 52 | 53 | 54 | /** 55 | * 获取一个实时队列中的第一个数据 56 | */ 57 | String getReadyJob(); 58 | 59 | /** 60 | * 获取指定个数实时队列中的数据 不是用的POP方式 需要手動刪除 61 | */ 62 | List getReadyJob(int size); 63 | 64 | /** 65 | * 刪除实时队列中的一个数据 66 | */ 67 | boolean removeReadyJob(String jobId); 68 | 69 | /** 70 | * 获取bucket中最顶端的一个Job 71 | */ 72 | String getBucketTop1Job(String bucketName); 73 | 74 | /** 75 | * 批量获取顶端数据 只获取满足条件的数据 最多size行 76 | */ 77 | List getBucketTopJobs(String bucketName, int size); 78 | 79 | void clearAll(); 80 | } 81 | -------------------------------------------------------------------------------- /src/test/resources/config/application.yml: -------------------------------------------------------------------------------- 1 | 2 | sdmq: 3 | logpath: ../logs 4 | registry: 5 | enable: false 6 | namespace: io-sdmq 7 | serverList: 127.0.0.1:2181 8 | reocketmq: 9 | namesrvAddr: 127.0.0.1:9876 10 | filterSourceRoot: /home/ 11 | rqueue: 12 | #是否已集群模式运行 会涉及到分布式锁竞争的情况 默认为false 13 | cluster: false 14 | name: dq 15 | bucketSize: 1 16 | 17 | spring: 18 | jmx: 19 | enabled: true 20 | application: 21 | name: 'sdmq' 22 | version: '1.0' 23 | datasource: 24 | url: jdbc:mysql://localhost:3306/sdmq?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true 25 | driverClassName: com.mysql.jdbc.Driver 26 | username: sdmq 27 | password: sdmq 28 | type: com.alibaba.druid.pool.DruidDataSource 29 | filters: stat 30 | maxActive: 30 31 | initialSize: 3 32 | maxWait: 60000 33 | maxIdle: 30 34 | minIdle: 3 35 | timeBetweenEvictionRunsMillis: 60000 36 | minEvictableIdleTimeMillis: 300000 37 | validationQuery: select 'x' 38 | testWhileIdle: true 39 | testOnBorrow: false 40 | testOnReturn: false 41 | poolPreparedStatements: true 42 | maxOpenPreparedStatements: 20 43 | redis: 44 | host: 127.0.0.1 45 | password: aJjbuDRYw4 46 | port: 6380 47 | timeout: 60000 48 | database: 11 49 | # cluster: 50 | # nodes: 51 | # - 127.0.0.1:30001 52 | # - 127.0.0.1:30002 53 | # - 127.0.0.1:30003 54 | # - 127.0.0.1:30004 55 | # - 127.0.0.1:30005 56 | # - 127.0.0.1:30006 57 | pool: 58 | max-idle: 300 59 | min-idle: 20 60 | max-active: 200 61 | max-wait: -1 62 | 63 | server: 64 | port: 6357 65 | 66 | logging: 67 | config: classpath:config/logback.xml -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/QueueHealthIndicator.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import org.springframework.boot.actuate.health.Health; 4 | import org.springframework.boot.actuate.health.HealthIndicator; 5 | 6 | import io.sdmq.leader.LeaderManager; 7 | import io.sdmq.leader.ServerNode; 8 | import io.sdmq.queue.core.Queue; 9 | import io.sdmq.queue.redis.RedisQueueImpl; 10 | import io.sdmq.queue.redis.support.RedisQueueProperties; 11 | 12 | public class QueueHealthIndicator implements HealthIndicator { 13 | 14 | 15 | private Queue queue; 16 | private LeaderManager leaderManager; 17 | private RedisQueueProperties redisQueueProperties; 18 | 19 | public QueueHealthIndicator(RedisQueueImpl queue, LeaderManager leaderManager, RedisQueueProperties 20 | redisQueueProperties) { 21 | this.queue = queue; 22 | this.leaderManager = leaderManager; 23 | this.redisQueueProperties = redisQueueProperties; 24 | } 25 | 26 | @Override 27 | public Health health() { 28 | try { 29 | Health.Builder builder = Health.up(); 30 | if (leaderManager == null) { 31 | builder.withDetail("run", queue.isRunning()); 32 | } else { 33 | builder.withDetail("run", queue.isRunning()).withDetail("isMaster", leaderManager.isLeader()); 34 | } 35 | return builder 36 | .withDetail("isCluster", redisQueueProperties.isCluster()) 37 | .withDetail("bucketSize", redisQueueProperties.getBucketSize()) 38 | .withDetail("prefix", redisQueueProperties.getPrefix()) 39 | .withDetail("namespace", ServerNode.NAMESPACE) 40 | .build(); 41 | } catch (Exception e) { 42 | return Health.down(e).build(); 43 | } 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/resources/META-INF/spring-configuration-metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "groups": [ 3 | { 4 | "sourceType": "io.sdmq.common.autoconfigure.RegistryProperties", 5 | "name": "sdmq.registry", 6 | "type": "io.sdmq.common.autoconfigure.RegistryProperties", 7 | "description": "sdmq.registry 配置项" 8 | }, 9 | { 10 | "sourceType": "io.sdmq.queue.redis.support.RedisQueueProperties", 11 | "name": "sdmq.rqueue", 12 | "type": "io.sdmq.queue.redis.support.RedisQueueProperties", 13 | "description": "sdmq.rqueue 配置项" 14 | } 15 | ], 16 | "properties": [ 17 | { 18 | "sourceType": "io.sdmq.common.autoconfigure.RegistryProperties", 19 | "name": "sdmq.registry.enable", 20 | "type": "java.lang.Boolean", 21 | "description": "是否启用注册中心模式" 22 | }, 23 | { 24 | "sourceType": "io.sdmq.common.autoconfigure.RegistryProperties", 25 | "name": "sdmq.registry.namespace", 26 | "type": "java.lang.String", 27 | "description": "namespace" 28 | }, 29 | { 30 | "sourceType": "io.sdmq.common.autoconfigure.RegistryProperties", 31 | "name": "sdmq.registry.serverList", 32 | "type": "java.lang.String", 33 | "description": "zk serever list" 34 | }, 35 | { 36 | "sourceType": "io.sdmq.queue.redis.support.RedisQueueProperties", 37 | "name": "sdmq.rqueue.cluster", 38 | "type": "java.lang.Boolean", 39 | "description": "是否启用集群模式" 40 | }, 41 | { 42 | "sourceType": "io.sdmq.queue.redis.support.RedisQueueProperties", 43 | "name": "sdmq.rqueue.name", 44 | "type": "java.lang.String", 45 | "description": "队列名称" 46 | }, 47 | { 48 | "sourceType": "io.sdmq.queue.redis.support.RedisQueueProperties", 49 | "name": "sdmq.rqueue.bucketSize", 50 | "type": "java.lang.Integer", 51 | "description": "分区bucket大小" 52 | } 53 | ] 54 | } -------------------------------------------------------------------------------- /src/test/java/io/sdmq/RoundRobinTest.java: -------------------------------------------------------------------------------- 1 | package io.sdmq; 2 | 3 | import io.sdmq.queue.redis.RedisQueueImpl; 4 | import io.sdmq.queue.redis.support.RedisQueueProperties; 5 | 6 | /** 7 | * Created by Xs.Tao on 2017/7/19. 8 | */ 9 | public class RoundRobinTest { 10 | 11 | public static void main(String[] args) { 12 | RedisQueueProperties properties = new RedisQueueProperties(); 13 | properties.setBucketSize(1); 14 | properties.setPrefix("com.tmk"); 15 | properties.setName("b"); 16 | final RedisQueueImpl redisQueue = new RedisQueueImpl(); 17 | redisQueue.setProperties(properties); 18 | new Thread(new Runnable() { 19 | @Override 20 | public void run() { 21 | for (int i = 0; i < 5; i++) { 22 | System.out.println(redisQueue.buildQueueName()); 23 | System.out.println(redisQueue.buildQueueName()); 24 | System.out.println(redisQueue.buildQueueName()); 25 | System.out.println(redisQueue.buildQueueName()); 26 | } 27 | } 28 | }).start(); 29 | new Thread(new Runnable() { 30 | @Override 31 | public void run() { 32 | for (int i = 0; i < 5; i++) { 33 | System.out.println(redisQueue.buildQueueName()); 34 | System.out.println(redisQueue.buildQueueName()); 35 | System.out.println(redisQueue.buildQueueName()); 36 | System.out.println(redisQueue.buildQueueName()); 37 | System.out.println(redisQueue.buildQueueName()); 38 | System.out.println(redisQueue.buildQueueName()); 39 | System.out.println(redisQueue.buildQueueName()); 40 | System.out.println(redisQueue.buildQueueName()); 41 | } 42 | } 43 | }).start(); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/main/resources/config/application.yml: -------------------------------------------------------------------------------- 1 | 2 | sdmq: 3 | logpath: ../logs 4 | registry: 5 | enable: false 6 | namespace: io-sdmq 7 | serverList: 127.0.0.1:2181 8 | reocketmq: 9 | namesrvAddr: 127.0.0.1:9876 10 | filterSourceRoot: /home/ 11 | rqueue: 12 | #是否已集群模式运行 会涉及到分布式锁竞争的情况 默认为false 13 | cluster: false 14 | name: dq 15 | bucketSize: 4 16 | 17 | spring: 18 | jmx: 19 | enabled: true 20 | application: 21 | name: 'sdmq' 22 | version: '1.0' 23 | datasource: 24 | url: jdbc:mysql://127.0.0.1:3306/sdmq?useUnicode=true&characterEncoding=UTF-8&allowMultiQueries=true 25 | driverClassName: com.mysql.jdbc.Driver 26 | username: root 27 | password: root 28 | type: com.alibaba.druid.pool.DruidDataSource 29 | filters: stat 30 | maxActive: 30 31 | initialSize: 3 32 | maxWait: 60000 33 | maxIdle: 30 34 | minIdle: 3 35 | timeBetweenEvictionRunsMillis: 60000 36 | minEvictableIdleTimeMillis: 300000 37 | validationQuery: select 'x' 38 | testWhileIdle: true 39 | testOnBorrow: false 40 | testOnReturn: false 41 | poolPreparedStatements: true 42 | maxOpenPreparedStatements: 20 43 | redis: 44 | host: localhost 45 | password: aJjbuDRYw4 46 | port: 6380 47 | timeout: 60000 48 | database: 11 49 | # cluster: 50 | # nodes: 51 | # - 127.0.0.1:30001 52 | # - 127.0.0.1:30002 53 | # - 127.0.0.1:30003 54 | # - 127.0.0.1:30004 55 | # - 127.0.0.1:30005 56 | # - 127.0.0.1:30006 57 | pool: 58 | max-idle: 300 59 | min-idle: 10 60 | max-active: 200 61 | max-wait: 10000 62 | 63 | server: 64 | port: 6355 65 | 66 | logging: 67 | config: file:${soft.home}/config/logback.xml 68 | -------------------------------------------------------------------------------- /src/test/java/io/sdmq/FixTest.java: -------------------------------------------------------------------------------- 1 | package io.sdmq; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.test.SpringApplicationConfiguration; 7 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 8 | import org.springframework.test.context.web.WebAppConfiguration; 9 | 10 | import java.util.Calendar; 11 | import java.util.Date; 12 | import java.util.Random; 13 | 14 | import io.sdmq.queue.JobMsg; 15 | import io.sdmq.queue.redis.JobWrapp; 16 | import io.sdmq.queue.redis.RedisQueueImpl; 17 | import io.sdmq.util.JobIdGenerator; 18 | 19 | /** 20 | * Created by Xs.Tao on 2017/7/19. 21 | */ 22 | @RunWith(SpringJUnit4ClassRunner.class) 23 | @SpringApplicationConfiguration(classes = TestDelayQueue.class) 24 | @WebAppConfiguration 25 | public class FixTest { 26 | 27 | @Autowired 28 | private RedisQueueImpl redisQueue; 29 | 30 | @Test 31 | public void pushTest() { 32 | for (int i = 0; i < 500; i++) { 33 | long time = 1000 * (60 * new Random().nextInt(100) + 1); 34 | Calendar calendar = Calendar.getInstance(); 35 | calendar.setTimeInMillis(System.currentTimeMillis() + time); 36 | int hour = calendar.get(Calendar.HOUR_OF_DAY); 37 | int min = calendar.get(Calendar.MINUTE); 38 | int src = calendar.get(Calendar.SECOND); 39 | JobMsg job = new JobWrapp(); 40 | job.setBody(String.format("{你应该在 %s 运行}", hour + ":" + min + ":" + src)); 41 | job.setTopic("test1".concat(new Date().getSeconds() + "")); 42 | job.setDelay(time); 43 | job.setId(JobIdGenerator.getStringId()); 44 | redisQueue.push(job); 45 | 46 | // try { 47 | // Thread.sleep(1000L); 48 | // } catch (InterruptedException e) { 49 | // e.printStackTrace(); 50 | // } 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/conf/RedisConfig.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.conf; 2 | 3 | import org.springframework.context.annotation.Bean; 4 | import org.springframework.context.annotation.Configuration; 5 | import org.springframework.data.redis.connection.RedisConnectionFactory; 6 | import org.springframework.data.redis.core.RedisTemplate; 7 | import org.springframework.data.redis.serializer.RedisSerializer; 8 | import org.springframework.data.redis.serializer.SerializationException; 9 | import org.springframework.data.redis.serializer.StringRedisSerializer; 10 | import org.springframework.util.SerializationUtils; 11 | 12 | @Configuration 13 | public class RedisConfig { 14 | 15 | @Bean 16 | public RedisTemplate redisTemplate(RedisConnectionFactory factory) { 17 | final RedisTemplate template = new RedisTemplate(); 18 | template.setConnectionFactory(factory); 19 | template.setKeySerializer(new StringRedisSerializer()); 20 | template.setHashValueSerializer(new RedisSerializer() { 21 | @Override 22 | public byte[] serialize(Object o) throws SerializationException { 23 | return SerializationUtils.serialize(o); 24 | } 25 | 26 | @Override 27 | public Object deserialize(byte[] bytes) throws SerializationException { 28 | return SerializationUtils.deserialize(bytes); 29 | } 30 | }); 31 | //template.setHashValueSerializer( new GenericToStringSerializer< Object >( Object.class ) ); 32 | template.setValueSerializer(new RedisSerializer() { 33 | @Override 34 | public byte[] serialize(Object o) throws SerializationException { 35 | return SerializationUtils.serialize(o); 36 | } 37 | 38 | @Override 39 | public Object deserialize(byte[] bytes) throws SerializationException { 40 | return SerializationUtils.deserialize(bytes); 41 | } 42 | }); 43 | return template; 44 | } 45 | } -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/RegistryProperties.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | import java.util.Objects; 6 | 7 | /** 8 | * Created by Xs.Tao on 2017/7/28. 9 | */ 10 | @ConfigurationProperties(prefix = RegistryProperties.SDMQ_REGISTRY_PREFIX) 11 | public class RegistryProperties { 12 | 13 | public static final String SDMQ_REGISTRY_PREFIX = "sdmq.registry"; 14 | 15 | 16 | private String enable = Objects.toString(Boolean.FALSE); 17 | private String serverList; 18 | 19 | private int maxRetries = 100; 20 | 21 | private int maxSleepTimeMilliseconds; 22 | 23 | private int baseSleepTimeMilliseconds; 24 | 25 | private String namespace = "io-sdmq"; 26 | 27 | public String getServerList() { 28 | return serverList; 29 | } 30 | 31 | public void setServerList(String serverList) { 32 | this.serverList = serverList; 33 | } 34 | 35 | public int getMaxRetries() { 36 | return maxRetries; 37 | } 38 | 39 | public void setMaxRetries(int maxRetries) { 40 | this.maxRetries = maxRetries; 41 | } 42 | 43 | public int getMaxSleepTimeMilliseconds() { 44 | return maxSleepTimeMilliseconds; 45 | } 46 | 47 | public void setMaxSleepTimeMilliseconds(int maxSleepTimeMilliseconds) { 48 | this.maxSleepTimeMilliseconds = maxSleepTimeMilliseconds; 49 | } 50 | 51 | public int getBaseSleepTimeMilliseconds() { 52 | return baseSleepTimeMilliseconds; 53 | } 54 | 55 | public void setBaseSleepTimeMilliseconds(int baseSleepTimeMilliseconds) { 56 | this.baseSleepTimeMilliseconds = baseSleepTimeMilliseconds; 57 | } 58 | 59 | public String getEnable() { 60 | return enable; 61 | } 62 | 63 | public void setEnable(String enable) { 64 | this.enable = enable; 65 | } 66 | 67 | public String getNamespace() { 68 | return namespace; 69 | } 70 | 71 | public void setNamespace(String namespace) { 72 | this.namespace = namespace; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/HealthAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import org.springframework.beans.factory.annotation.Autowired; 4 | import org.springframework.boot.actuate.health.CompositeHealthIndicator; 5 | import org.springframework.boot.actuate.health.HealthAggregator; 6 | import org.springframework.boot.actuate.health.HealthIndicator; 7 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; 8 | import org.springframework.context.annotation.Bean; 9 | import org.springframework.context.annotation.Configuration; 10 | import org.springframework.core.Ordered; 11 | import org.springframework.core.annotation.Order; 12 | 13 | import java.util.Map; 14 | 15 | import io.sdmq.common.conf.AppEnvContext; 16 | import io.sdmq.leader.LeaderManager; 17 | import io.sdmq.leader.SimpleLeaderManager; 18 | import io.sdmq.queue.redis.RedisQueueImpl; 19 | import io.sdmq.queue.redis.support.RedisQueueProperties; 20 | 21 | /** 22 | * Created by Xs.Tao on 2017/7/28. 23 | */ 24 | @Configuration 25 | @Order(Ordered.LOWEST_PRECEDENCE + 1000) 26 | public class HealthAutoConfiguration { 27 | 28 | 29 | @Autowired 30 | private HealthAggregator healthAggregator; 31 | 32 | 33 | @Bean 34 | @Autowired(required = false) 35 | @ConditionalOnMissingBean 36 | public HealthIndicator jikexiuHealthIndicator(RedisQueueImpl redisQueue, 37 | RedisQueueProperties properties) { 38 | CompositeHealthIndicator compositeHealthIndicator = new 39 | CompositeHealthIndicator(healthAggregator); 40 | Map leaderManagerMap = AppEnvContext.getCtx().getBeansOfType(LeaderManager.class); 41 | LeaderManager manager = null; 42 | if (leaderManagerMap != null && !leaderManagerMap.isEmpty()) { 43 | manager = AppEnvContext.getCtx().getBean(SimpleLeaderManager.class); 44 | } 45 | 46 | compositeHealthIndicator.addHealthIndicator("dq", new QueueHealthIndicator( 47 | redisQueue, manager, properties)); 48 | return compositeHealthIndicator; 49 | } 50 | 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/conf/AppEnvContext.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.conf; 2 | 3 | import org.springframework.beans.BeansException; 4 | import org.springframework.boot.bind.RelaxedPropertyResolver; 5 | import org.springframework.context.ApplicationContext; 6 | import org.springframework.context.ApplicationContextAware; 7 | import org.springframework.context.EnvironmentAware; 8 | import org.springframework.context.annotation.Configuration; 9 | import org.springframework.core.Ordered; 10 | import org.springframework.core.annotation.Order; 11 | import org.springframework.core.env.Environment; 12 | import org.springframework.util.StringUtils; 13 | 14 | @Configuration 15 | @Order(Ordered.HIGHEST_PRECEDENCE - 50) 16 | public class AppEnvContext implements EnvironmentAware, ApplicationContextAware { 17 | 18 | private static Environment env; 19 | private static ApplicationContext ctx; 20 | 21 | public static String getProperty(String key) { 22 | 23 | return env.getProperty(key); 24 | } 25 | 26 | public static String getProperty(String key, String defaultValue) { 27 | 28 | String v = getProperty(key); 29 | return StringUtils.isEmpty(v) ? defaultValue : v; 30 | } 31 | 32 | /** 33 | * 34 | * getRelaxedPropertyResolver( "spring.datasource.").getProperty("url"); 35 | *

36 | * 37 | */ 38 | public static RelaxedPropertyResolver getRelaxedPropertyResolver(String prefix) { 39 | 40 | RelaxedPropertyResolver propertyResolver = new RelaxedPropertyResolver(env, prefix); 41 | return propertyResolver; 42 | } 43 | 44 | public static Environment getEnv() { 45 | return env; 46 | } 47 | 48 | public static ApplicationContext getCtx() { 49 | return ctx; 50 | } 51 | 52 | protected void finalize() throws Throwable { 53 | super.finalize(); 54 | env = null; 55 | } 56 | 57 | @Override 58 | public void setEnvironment(Environment environment) { 59 | env = environment; 60 | } 61 | 62 | @Override 63 | public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { 64 | ctx = applicationContext; 65 | } 66 | } -------------------------------------------------------------------------------- /src/test/java/io/sdmq/DistributedLockTest.java: -------------------------------------------------------------------------------- 1 | package io.sdmq; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.test.SpringApplicationConfiguration; 9 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 10 | import org.springframework.test.context.web.WebAppConfiguration; 11 | 12 | import java.util.concurrent.ExecutorService; 13 | import java.util.concurrent.Executors; 14 | 15 | import io.sdmq.queue.redis.support.DistributedLock; 16 | import io.sdmq.queue.redis.support.RedisDistributedLock; 17 | import io.sdmq.queue.redis.support.RedisSupport; 18 | 19 | /** 20 | * Created by Xs.Tao on 2017/7/20. 21 | */ 22 | @RunWith(SpringJUnit4ClassRunner.class) 23 | @SpringApplicationConfiguration(classes = TestDelayQueue.class) 24 | @WebAppConfiguration 25 | public class DistributedLockTest { 26 | 27 | public static final Logger LOGGER = LoggerFactory.getLogger(DistributedLockTest.class); 28 | @Autowired 29 | private RedisSupport redisSupport; 30 | 31 | @Test 32 | public void test1() { 33 | final DistributedLock lock = new RedisDistributedLock(redisSupport); 34 | ExecutorService executorService = Executors.newFixedThreadPool(5); 35 | for (int i = 0; i < 20; i++) { 36 | final int index = i; 37 | executorService.execute(new Runnable() { 38 | @Override 39 | public void run() { 40 | try { 41 | if (index >= 10) { 42 | lock.lock("test002"); 43 | } else { 44 | lock.lock("test001"); 45 | } 46 | 47 | LOGGER.info("我得到锁了 {} ", index); 48 | Thread.sleep(500); 49 | } catch (InterruptedException e) { 50 | e.printStackTrace(); 51 | } finally { 52 | if (index >= 10) { 53 | lock.unlock("test002"); 54 | } else { 55 | lock.unlock("test001"); 56 | } 57 | } 58 | } 59 | }); 60 | 61 | } 62 | try { 63 | Thread.sleep(1000000L); 64 | } catch (InterruptedException e) { 65 | e.printStackTrace(); 66 | } 67 | } 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/resources/config/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 7 | 9 | 10 | 11 | 12 | 15 | %clr(%d{HH:mm:ss.SSS}){faint} %clr([%thread]){faint} %clr(%-5level){magenta}%clr(%-40.40logger{40}){cyan} %L %clr(:){faint}- %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | ${logpath}/sdmq-%d{yyyy-MM-dd}.log 23 | 30 24 | 25 | 26 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{40} - %msg%n 27 | 28 | 29 | 30 | 31 | ${logpath}/sdmq-track-%d{yyyy-MM-dd}.log 32 | 30 33 | 34 | 35 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{40} - %msg%n 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/test/resources/config/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 7 | 9 | 10 | 11 | 12 | 15 | %clr(%d{HH:mm:ss.SSS}){faint} %clr([%thread]){faint} %clr(%-5level){magenta}%clr(%-40.40logger{40}){cyan} %L %clr(:){faint}- %msg%n 16 | 17 | 18 | 19 | 20 | 21 | 22 | ${logpath}/sdmq-%d{yyyy-MM-dd}.log 23 | 30 24 | 25 | 26 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{40} - %msg%n 27 | 28 | 29 | 30 | 31 | ${logpath}/sdmq-track-%d{yyyy-MM-dd}.log 32 | 30 33 | 34 | 35 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{40} - %msg%n 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/LeaderAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import org.apache.curator.framework.CuratorFrameworkFactory; 4 | import org.apache.curator.framework.recipes.leader.LeaderLatchListener; 5 | import org.apache.zookeeper.server.ZooKeeperServer; 6 | import org.springframework.beans.factory.annotation.Autowired; 7 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 8 | import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; 9 | import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; 10 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 11 | import org.springframework.context.annotation.Bean; 12 | import org.springframework.context.annotation.Configuration; 13 | import org.springframework.core.Ordered; 14 | import org.springframework.core.annotation.Order; 15 | 16 | import io.sdmq.common.conf.AppEnvContext; 17 | import io.sdmq.leader.LeaderManager; 18 | import io.sdmq.leader.LeaderWorkListener; 19 | import io.sdmq.leader.ServerNode; 20 | import io.sdmq.leader.SimpleLeaderManager; 21 | import io.sdmq.queue.redis.RedisQueueImpl; 22 | import io.sdmq.util.IpUtils; 23 | 24 | /** 25 | * Created by Xs.Tao on 2017/7/28. 26 | */ 27 | @Configuration 28 | @EnableConfigurationProperties(RegistryProperties.class) 29 | @ConditionalOnProperty(prefix = RegistryProperties.SDMQ_REGISTRY_PREFIX, value = "enable", havingValue = "true") 30 | @ConditionalOnClass(value = {ZooKeeperServer.class, CuratorFrameworkFactory.class}) 31 | @Order(Ordered.LOWEST_PRECEDENCE + 50) 32 | public class LeaderAutoConfiguration { 33 | 34 | @Autowired 35 | private RegistryProperties registryProperties; 36 | 37 | 38 | @Bean 39 | @Autowired 40 | @ConditionalOnMissingBean 41 | public LeaderLatchListener leaderLatchListenerImpl(RedisQueueImpl redisQueueImpl) { 42 | LeaderWorkListener listener = new LeaderWorkListener(); 43 | listener.setQueue(redisQueueImpl); 44 | return listener; 45 | } 46 | 47 | @Bean(name = "simpleLeaderManager", initMethod = "init", destroyMethod = "stop") 48 | @Autowired 49 | @ConditionalOnMissingBean 50 | public LeaderManager leaderManager(LeaderLatchListener leaderLatchListener) { 51 | SimpleLeaderManager slm = new SimpleLeaderManager(); 52 | slm.setProperties(registryProperties); 53 | slm.addListener(leaderLatchListener); 54 | ServerNode.NAMESPACE = registryProperties.getNamespace(); 55 | slm.setServerName(IpUtils.getIp() + ":" + AppEnvContext.getProperty("server.port")); 56 | return slm; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/JobMsg.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue; 2 | 3 | import io.sdmq.queue.core.Job; 4 | import io.sdmq.util.Status; 5 | 6 | /** 7 | * Created by Xs.Tao on 2017/7/18. 8 | */ 9 | public class JobMsg implements java.io.Serializable, Job { 10 | 11 | private String topic; 12 | private String subtopic; 13 | /** 14 | * 预留字段 15 | **/ 16 | private String id; 17 | private String bizKey; 18 | private long delay; 19 | private long ttl; 20 | /***预留字段**/ 21 | private String body; 22 | /** 23 | * 扩展字段 24 | */ 25 | private String extendData; 26 | private long createTime = System.currentTimeMillis(); 27 | private int status = Status.WaitPut.ordinal(); 28 | 29 | @Override 30 | public String getExtendData() { 31 | return extendData; 32 | } 33 | 34 | public void setExtendData(String extendData) { 35 | this.extendData = extendData; 36 | } 37 | @Override 38 | public String getBizKey() { 39 | return bizKey; 40 | } 41 | 42 | public void setBizKey(String bizKey) { 43 | this.bizKey = bizKey; 44 | } 45 | @Override 46 | public String getTopic() { 47 | 48 | return topic; 49 | } 50 | 51 | public void setTopic(String topic) { 52 | this.topic = topic; 53 | } 54 | @Override 55 | public String getId() { 56 | return id; 57 | } 58 | 59 | public void setId(String id) { 60 | this.id = id; 61 | } 62 | @Override 63 | public long getDelay() { 64 | return delay; 65 | } 66 | 67 | public void setDelay(long delay) { 68 | this.delay = delay; 69 | } 70 | @Override 71 | public long getTtl() { 72 | return ttl; 73 | } 74 | 75 | public void setTtl(long ttl) { 76 | this.ttl = ttl; 77 | } 78 | @Override 79 | public String getBody() { 80 | return body; 81 | } 82 | 83 | public void setBody(String body) { 84 | this.body = body; 85 | } 86 | @Override 87 | public long getCreateTime() { 88 | return createTime; 89 | } 90 | 91 | public void setCreateTime(long createTime) { 92 | this.createTime = createTime; 93 | } 94 | @Override 95 | public int getStatus() { 96 | return status; 97 | } 98 | 99 | public void setStatus(int status) { 100 | this.status = status; 101 | } 102 | @Override 103 | public String getSubtopic() { 104 | return subtopic; 105 | } 106 | 107 | public void setSubtopic(String subtopic) { 108 | this.subtopic = subtopic; 109 | } 110 | } 111 | 112 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/support/RedisQueueProperties.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.support; 2 | 3 | import org.springframework.boot.context.properties.ConfigurationProperties; 4 | 5 | /** 6 | * Created by Xs.Tao on 2017/7/18. 7 | */ 8 | @ConfigurationProperties(prefix = RedisQueueProperties.REDIS_QUEUE_PREFIX) 9 | public class RedisQueueProperties { 10 | 11 | public static final String REDIS_QUEUE_PREFIX = "sdmq.rqueue"; 12 | private String name; 13 | private String prefix = "io.sdmq"; 14 | private String originPool = "pools"; 15 | private String readyName = "ready"; 16 | private int bucketSize = 3; 17 | 18 | /** 19 | * buck轮询时间 20 | **/ 21 | private long buckRoundRobinTime = 300; 22 | /** 23 | * ready轮询时间 24 | **/ 25 | private long readyRoundRobinTime = 200; 26 | private boolean cluster = false; 27 | 28 | public String getName() { 29 | return name; 30 | } 31 | 32 | public void setName(String name) { 33 | this.name = name; 34 | } 35 | 36 | public String getPrefix() { 37 | return prefix; 38 | } 39 | 40 | public void setPrefix(String prefix) { 41 | this.prefix = prefix; 42 | } 43 | 44 | public String getOriginPool() { 45 | return originPool; 46 | } 47 | 48 | public void setOriginPool(String originPool) { 49 | this.originPool = originPool; 50 | } 51 | 52 | public String getReadyName() { 53 | return readyName; 54 | } 55 | 56 | public void setReadyName(String readyName) { 57 | this.readyName = readyName; 58 | } 59 | 60 | public int getBucketSize() { 61 | return bucketSize; 62 | } 63 | 64 | public void setBucketSize(int bucketSize) { 65 | this.bucketSize = bucketSize; 66 | } 67 | 68 | public boolean isCluster() { 69 | return cluster; 70 | } 71 | 72 | public void setCluster(boolean cluster) { 73 | this.cluster = cluster; 74 | } 75 | 76 | public long getBuckRoundRobinTime() { 77 | if (buckRoundRobinTime <= 0) { 78 | buckRoundRobinTime = 500; 79 | } 80 | return buckRoundRobinTime; 81 | } 82 | 83 | public void setBuckRoundRobinTime(long buckRoundRobinTime) { 84 | this.buckRoundRobinTime = buckRoundRobinTime; 85 | } 86 | 87 | public long getReadyRoundRobinTime() { 88 | if (readyRoundRobinTime <= 0) { 89 | readyRoundRobinTime = 500; 90 | } 91 | return readyRoundRobinTime; 92 | } 93 | 94 | public void setReadyRoundRobinTime(long readyRoundRobinTime) { 95 | this.readyRoundRobinTime = readyRoundRobinTime; 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/JobIdGenerator.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.util.StringUtils; 6 | 7 | /** 8 | * 

 9 |  *     -DmachineId=num 机器标识
10 |  *
11 | * Created by Xs.Tao on 2017/9/18. 12 | */ 13 | public class JobIdGenerator { 14 | 15 | public static final Logger LOGGER = LoggerFactory.getLogger(JobIdGenerator.class); 16 | public static final int DATACENTER = 2; 17 | public static final int DEFAULT_MACHINED = 1; 18 | private static SnowFlake snowFlake = null; 19 | 20 | static { 21 | int m = machinedId(); 22 | LOGGER.info(" machined {}", m); 23 | snowFlake = new SnowFlake(DATACENTER, m); 24 | } 25 | 26 | private static int machinedId() { 27 | //通过获取IP地址最后一位来获取 28 | String MACHINED = System.getProperty("machineId"); 29 | if (!StringUtils.isEmpty(MACHINED)) { 30 | try { 31 | return Integer.parseInt(MACHINED); 32 | } catch (Exception e) { 33 | return DEFAULT_MACHINED; 34 | } 35 | } 36 | return DEFAULT_MACHINED; 37 | } 38 | 39 | public static long getLongId() { 40 | return snowFlake.nextId(); 41 | } 42 | 43 | public static String getStringId() { 44 | return String.valueOf(snowFlake.nextId()); 45 | } 46 | 47 | // public static void main(String[] args) { 48 | // final Map c= Maps.newHashMap(); 49 | // final Map c1=Maps.newHashMap(); 50 | // final AtomicLong atomicLong=new AtomicLong(0); 51 | // for(int i=0;i<20;i++){ 52 | // new Thread(new Runnable() { 53 | // @Override 54 | // public void run() { 55 | // for(int i=0;i<5000;i++){ 56 | // String k=getStringId(); 57 | // atomicLong.incrementAndGet(); 58 | // System.out.println(k+"=="+c.containsKey(k)+"--"+atomicLong.get()); 59 | // if(!c.containsKey(k)){ 60 | // c.put(k,i); 61 | // }else{ 62 | // throw new RuntimeException(String.format("id %s重复了",k)); 63 | // } 64 | // 65 | // 66 | // } 67 | // c1.put(Thread.currentThread().getName(),""); 68 | // } 69 | // }).start(); 70 | // } 71 | // while(c1.size()>=20){ 72 | // System.out.println(c.size()); 73 | // } 74 | // try { 75 | // Thread.sleep(5000000L); 76 | // } catch (InterruptedException e) { 77 | // e.printStackTrace(); 78 | // } 79 | // } 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/support/RedisDistributedLock.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.support; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import io.sdmq.util.BlockUtils; 7 | 8 | /** 9 | * Created by Xs.Tao on 2017/7/20. 10 | */ 11 | public class RedisDistributedLock implements DistributedLock { 12 | 13 | public static final Logger LOGGER = LoggerFactory.getLogger(RedisDistributedLock.class); 14 | private static final long DEFAULT_LOCK_TIMEOUT = 1000 * 60 * 3; 15 | private RedisSupport redisSupport; 16 | 17 | public RedisDistributedLock() { 18 | 19 | } 20 | 21 | public RedisDistributedLock(RedisSupport redisSupport) { 22 | this.redisSupport = redisSupport; 23 | } 24 | 25 | public static boolean isEmpty(Object str) { 26 | return (str == null || "".equals(str)); 27 | } 28 | 29 | @Override 30 | public boolean tryLock(String key) { 31 | throw new RuntimeException("待实现"); 32 | } 33 | 34 | private boolean interLock(String key) { 35 | //得到锁后设置的过期时间，未得到锁返回0 36 | while (true) { 37 | long expireTime = getCuurentMillis() + DEFAULT_LOCK_TIMEOUT + 1; 38 | if (redisSupport.setNx(key, String.valueOf(expireTime))) { 39 | redisSupport.pExpire(key, DEFAULT_LOCK_TIMEOUT); 40 | return true; 41 | } else { 42 | String curLockTimeStr = redisSupport.get(key); 43 | if (!isEmpty(curLockTimeStr) && 44 | getCuurentMillis() > Long.valueOf(curLockTimeStr)) { 45 | String setAftercurLockTimeStr = redisSupport.getSet(key, String.valueOf(expireTime)); 46 | //仍然过期,则得到锁 47 | if (setAftercurLockTimeStr != null && setAftercurLockTimeStr.equals(curLockTimeStr)) { 48 | redisSupport.pExpire(key, DEFAULT_LOCK_TIMEOUT); 49 | return true; 50 | } 51 | } 52 | } 53 | BlockUtils.sleep(10); 54 | if (LOGGER.isDebugEnabled()) { 55 | LOGGER.debug("没有获取锁 {} 正在等待...", key); 56 | } 57 | } 58 | } 59 | 60 | private long getCuurentMillis() { 61 | return System.currentTimeMillis(); 62 | } 63 | 64 | @Override 65 | public boolean tryLock(String key, long timeout) { 66 | throw new RuntimeException("待实现"); 67 | } 68 | 69 | @Override 70 | public boolean lock(String key) { 71 | return interLock(key); 72 | } 73 | 74 | @Override 75 | public void unlock(String key) { 76 | if (!isEmpty(key)) { 77 | redisSupport.deleteKey(key); 78 | } 79 | } 80 | 81 | public void setRedisSupport(RedisSupport redisSupport) { 82 | this.redisSupport = redisSupport; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sdmq [![Build Status](https://api.travis-ci.org/peachyy/sdmq.svg?branch=master)](https://travis-ci.org/peachyy/sdmq) [![Coverage Status](https://coveralls.io/repos/github/peachyy/sdmq/badge.svg?branch=master)](https://coveralls.io/github/peachyy/sdmq?branch=master) [![Gitter](https://badges.gitter.im/peachyy/sdmq.svg)](https://gitter.im/peachyy/sdmq?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge) 2 | is a simple delay message queue， based on redis and kotlin 3 | 4 | 设计 https://www.cnblogs.com/peachyy/p/7398430.html 5 | 6 | `一个简单、稳定、可扩展的延迟消息队列` 7 | 8 | # 运行模式 9 | 10 | * 支持 master,slave （HA）需要配置`sdmq.registry.serverList` zk集群地址列表 11 | * 支持 cluster 会涉及到分布式锁竞争 效果不是很明显 分布式锁采用`redis`的 `setNx`实现 12 | * StandAlone 13 | 14 | 推荐使用master slave的模式 15 | 16 | ##### Usage 17 | 18 | #### 消息体 19 | 20 | 以JSON数据格式参数 目前只提供了`http`协议 21 | 22 | 23 | * body 业务消息体 24 | * delay 延时毫秒 距`createTime`的间隔毫秒数 25 | * id 任务ID 系统自动生成 任务创建成功返回 26 | * status 状态 默认不填写 27 | * topic 标题 28 | * subtopic 保留字段 29 | * ttl 保留字段 30 | * createTime 创建任务时间 非必填 系统默认 31 | 32 | #### 添加任务 33 | 34 | 35 | ```` 36 | /push 37 | POST application/json 38 | 39 | {"body":"{ffff}","delay":56600,"id":"20","status":0,"topic":"ces","subtopic":"",ttl":12} 40 | ```` 41 | 42 | #### 删除任务 43 | 44 | 删除任务 需要记录一个JobId 45 | 46 | ```` 47 | /delete?jobId=xxx 48 | GET 49 | ```` 50 | #### 恢复单个任务 51 | 52 | 用于任务错乱 脑裂情况 根据日志恢复任务 53 | 54 | ```` 55 | /reStoreJob?JobId=xxx 56 | GET 57 | ```` 58 | #### 恢复所有未完成的任务 59 | 60 | 根据日志恢复任务 61 | 62 | ```` 63 | /reStore?expire=true 64 | GET 65 | ```` 66 | 67 | 参数`expire` 表示是否需要恢复已过期还未执行的数据 68 | 69 | #### 清空队列数据 70 | 71 | 根据日志中未完成的数据清空队列中全部数据 72 | 73 | `清空之后 会删除缓存中的所有任务` 74 | ```` 75 | /clearAll 76 | GET 77 | ```` 78 | 79 | 80 | 81 | #### 客户端获取队列方式 82 | 83 | 目前默认实现了`rocketmq`的推送方式。暂时就不用自己去实现推拉数据了。直接强依赖MQ。 84 | 85 | ##### 消息体中消息与`rocketmq`消息字段对应关系 86 | 87 | sdmq | rocketMQ | 备注| 88 | --- | --- |--- 89 | topic | topic | | 90 | subtopic | tag | | 91 | body | 消息内容 | 消息内容 | 92 | 93 | 94 | 95 | ## 后期优化 96 | 97 | * 分区(buck)支持动态设置 98 | * redis与数据库数据一致性的问题 （`重要`） 99 | * 实现自己的推拉机制 100 | * 支持可切换实现方式 当前强依赖redis 只有这么1个实现 101 | * 支持Web控制台管理队列 102 | * 实现消息消费`TTL`机制 103 | 104 | 定位是后期会改为基于`kotlin` `java`太多麻烦事了 105 | ## 测试 106 | 需要配置好数据库地址和redis的地址 如果不是单机模式 也需要配置好zookeep 107 | 108 | 运行测试类`io.sdmq.FixTest`添加任务到队列中 109 | 110 | 启动`Bootstarp`消费前面添加数据 为了方便查询效果 默认的消费方式是`consoleCQ` 控制台输出 111 | 112 | ## 更新日志 113 | 114 | * 2017年11月21日14:34:39 支持`restful`清空队列数据 115 | * 2018年03月19日14:26:56 支持配置消费方式 默认为`jmsCQ` 在不修改代码的情况下覆盖方式 `-DClassName=xxxx` 116 | 117 | 118 | 119 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/SnowFlake.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | public class SnowFlake { 4 | 5 | /** 6 | * 起始的时间戳 7 | */ 8 | private final static long START_STMP = 1505699432705L; 9 | 10 | /** 11 | * 每一部分占用的位数 12 | */ 13 | private final static long SEQUENCE_BIT = 12; //序列号占用的位数 14 | private final static long MACHINE_BIT = 5; //机器标识占用的位数 15 | private final static long DATACENTER_BIT = 5;//数据中心占用的位数 16 | 17 | /** 18 | * 每一部分的最大值 19 | */ 20 | private final static long MAX_DATACENTER_NUM = -1L ^ (-1L << DATACENTER_BIT); 21 | private final static long MAX_MACHINE_NUM = -1L ^ (-1L << MACHINE_BIT); 22 | private final static long MAX_SEQUENCE = -1L ^ (-1L << SEQUENCE_BIT); 23 | 24 | /** 25 | * 每一部分向左的位移 26 | */ 27 | private final static long MACHINE_LEFT = SEQUENCE_BIT; 28 | private final static long DATACENTER_LEFT = SEQUENCE_BIT + MACHINE_BIT; 29 | private final static long TIMESTMP_LEFT = DATACENTER_LEFT + DATACENTER_BIT; 30 | 31 | private long datacenterId; //数据中心 32 | private long machineId; //机器标识 33 | private long sequence = 0L; //序列号 34 | private long lastStmp = -1L;//上一次时间戳 35 | 36 | public SnowFlake(long datacenterId, long machineId) { 37 | if (datacenterId > MAX_DATACENTER_NUM || datacenterId < 0) { 38 | throw new IllegalArgumentException("datacenterId can't be greater than MAX_DATACENTER_NUM or less than 0"); 39 | } 40 | if (machineId > MAX_MACHINE_NUM || machineId < 0) { 41 | throw new IllegalArgumentException("machineId can't be greater than MAX_MACHINE_NUM or less than 0"); 42 | } 43 | this.datacenterId = datacenterId; 44 | this.machineId = machineId; 45 | } 46 | 47 | /** 48 | * 产生下一个ID 49 | */ 50 | public synchronized long nextId() { 51 | long currStmp = getNewstmp(); 52 | if (currStmp < lastStmp) { 53 | throw new RuntimeException("Clock moved backwards. Refusing to generate id"); 54 | } 55 | 56 | if (currStmp == lastStmp) { 57 | //相同毫秒内，序列号自增 58 | sequence = (sequence + 1) & MAX_SEQUENCE; 59 | //同一毫秒的序列数已经达到最大 60 | if (sequence == 0L) { 61 | currStmp = getNextMill(); 62 | } 63 | } else { 64 | //不同毫秒内，序列号置为0 65 | sequence = 0L; 66 | } 67 | 68 | lastStmp = currStmp; 69 | 70 | return (currStmp - START_STMP) << TIMESTMP_LEFT //时间戳部分 71 | | datacenterId << DATACENTER_LEFT //数据中心部分 72 | | machineId << MACHINE_LEFT //机器标识部分 73 | | sequence; //序列号部分 74 | } 75 | 76 | private long getNextMill() { 77 | long mill = getNewstmp(); 78 | while (mill <= lastStmp) { 79 | mill = getNewstmp(); 80 | } 81 | return mill; 82 | } 83 | 84 | private long getNewstmp() { 85 | return System.currentTimeMillis(); 86 | } 87 | 88 | 89 | } -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/ready/ReadyQueueManager.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.ready; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | import java.util.Timer; 7 | import java.util.concurrent.atomic.AtomicBoolean; 8 | 9 | import io.sdmq.common.extension.ExtensionLoader; 10 | import io.sdmq.queue.core.ConsumeQueueProvider; 11 | import io.sdmq.queue.core.Queue; 12 | import io.sdmq.queue.redis.JobOperationService; 13 | import io.sdmq.queue.redis.support.DistributedLock; 14 | import io.sdmq.queue.redis.support.Lifecycle; 15 | import io.sdmq.queue.redis.support.RedisQueueProperties; 16 | 17 | /** 18 | * Created by Xs.Tao on 2017/7/19. 19 | */ 20 | public class ReadyQueueManager implements Lifecycle { 21 | 22 | public static final Logger LOGGER = LoggerFactory.getLogger(ReadyQueueManager.class); 23 | public static final String THREAD_NAME = "sdmq-ready-queue-%s"; 24 | public boolean daemon = true; 25 | private volatile AtomicBoolean isRuning = new AtomicBoolean(false); 26 | private RedisQueueProperties properties; 27 | private Timer timer; 28 | private JobOperationService jobOperationService; 29 | private Queue delayQueue; 30 | private String threadName; 31 | private DistributedLock lock = null; 32 | 33 | 34 | @Override 35 | public void start() { 36 | if (isRuning.compareAndSet(false, true)) { 37 | threadName = String.format(THREAD_NAME, 1); 38 | timer = new Timer(threadName, daemon); 39 | RealTimeTask task = new RealTimeTask(); 40 | task.setProperties(properties); 41 | task.setJobOperationService(jobOperationService); 42 | task.setDelayQueue(delayQueue); 43 | task.setLock(lock); 44 | task.setConsumeQueueProvider(ExtensionLoader.getExtension(ConsumeQueueProvider.class)); 45 | timer.schedule(task, 500, properties.getReadyRoundRobinTime()); 46 | LOGGER.info(String.format("Starting Ready Thead %s ....", threadName)); 47 | } 48 | } 49 | 50 | @Override 51 | public void stop() { 52 | if (isRuning.compareAndSet(true, false)) { 53 | if (timer != null) { 54 | timer.cancel(); 55 | LOGGER.info(String.format("stoping timer %s .....", threadName)); 56 | } 57 | } 58 | } 59 | 60 | @Override 61 | public boolean isRunning() { 62 | return isRuning.get(); 63 | } 64 | 65 | public void setProperties(RedisQueueProperties properties) { 66 | this.properties = properties; 67 | } 68 | 69 | public void setDaemon(boolean daemon) { 70 | this.daemon = daemon; 71 | } 72 | 73 | public void setDelayQueue(Queue delayQueue) { 74 | this.delayQueue = delayQueue; 75 | } 76 | 77 | public void setJobOperationService(JobOperationService jobOperationService) { 78 | this.jobOperationService = jobOperationService; 79 | } 80 | 81 | public void setLock(DistributedLock lock) { 82 | this.lock = lock; 83 | } 84 | } 85 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/IpUtils.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import java.net.InetAddress; 4 | import java.net.NetworkInterface; 5 | import java.net.SocketException; 6 | import java.net.UnknownHostException; 7 | import java.util.Enumeration; 8 | 9 | public final class IpUtils { 10 | 11 | /** 12 | * IP地址的正则表达式. 13 | */ 14 | public static final String IP_REGEX = "((\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])(\\." + 15 | "(\\d|[1-9]\\d|1\\d{2}|2[0-4]\\d|25[0-5])){3})"; 16 | 17 | private static volatile String cachedIpAddress; 18 | 19 | /** 20 | * 获取本机IP地址. 21 | * 22 | *

23 | * 有限获取外网IP地址. 24 | * 也有可能是链接着路由器的最终IP地址. 25 | *

26 | * 27 | * @return 本机IP地址 28 | */ 29 | public static String getIp() { 30 | if (null != cachedIpAddress) { 31 | return cachedIpAddress; 32 | } 33 | Enumeration netInterfaces; 34 | try { 35 | netInterfaces = NetworkInterface.getNetworkInterfaces(); 36 | } catch (final SocketException ex) { 37 | throw new RuntimeException(ex); 38 | } 39 | String localIpAddress = null; 40 | OUTW: 41 | while (netInterfaces.hasMoreElements()) { 42 | NetworkInterface netInterface = netInterfaces.nextElement(); 43 | Enumeration ipAddresses = netInterface.getInetAddresses(); 44 | while (ipAddresses.hasMoreElements()) { 45 | InetAddress ipAddress = ipAddresses.nextElement(); 46 | if (isPublicIpAddress(ipAddress)) { 47 | String publicIpAddress = ipAddress.getHostAddress(); 48 | cachedIpAddress = publicIpAddress; 49 | return publicIpAddress; 50 | } 51 | if (isLocalIpAddress(ipAddress)) { 52 | localIpAddress = ipAddress.getHostAddress(); 53 | break OUTW; 54 | } 55 | } 56 | } 57 | cachedIpAddress = localIpAddress; 58 | return localIpAddress; 59 | } 60 | 61 | private static boolean isPublicIpAddress(final InetAddress ipAddress) { 62 | return !ipAddress.isSiteLocalAddress() && !ipAddress.isLoopbackAddress() && !isV6IpAddress(ipAddress); 63 | } 64 | 65 | private static boolean isLocalIpAddress(final InetAddress ipAddress) { 66 | return ipAddress.isSiteLocalAddress() && !ipAddress.isLoopbackAddress() && !isV6IpAddress(ipAddress); 67 | } 68 | 69 | private static boolean isV6IpAddress(final InetAddress ipAddress) { 70 | return ipAddress.getHostAddress().contains(":"); 71 | } 72 | 73 | /** 74 | * 获取本机Host名称. 75 | * 76 | * @return 本机Host名称 77 | */ 78 | public static String getHostName() { 79 | try { 80 | return InetAddress.getLocalHost().getHostName(); 81 | } catch (final UnknownHostException ex) { 82 | throw new RuntimeException(ex); 83 | } 84 | } 85 | 86 | public static String getHostAndIp() { 87 | String name = ""; 88 | try { 89 | name = getIp(); 90 | } catch (Exception e) { 91 | // 92 | } 93 | try { 94 | name = name.concat("/").concat(getHostName()); 95 | } catch (Exception e) { 96 | // 97 | } 98 | return name; 99 | } 100 | } -------------------------------------------------------------------------------- /src/main/java/io/sdmq/leader/SimpleLeaderManager.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.leader; 2 | 3 | 4 | import org.apache.curator.framework.CuratorFramework; 5 | import org.apache.curator.framework.CuratorFrameworkFactory; 6 | import org.apache.curator.framework.recipes.leader.LeaderLatch; 7 | import org.apache.curator.framework.recipes.leader.LeaderLatchListener; 8 | import org.apache.curator.retry.ExponentialBackoffRetry; 9 | import org.apache.curator.utils.CloseableUtils; 10 | import org.assertj.core.util.Lists; 11 | import org.slf4j.Logger; 12 | import org.slf4j.LoggerFactory; 13 | 14 | import java.util.List; 15 | import java.util.concurrent.atomic.AtomicBoolean; 16 | 17 | import io.sdmq.common.autoconfigure.RegistryProperties; 18 | import io.sdmq.util.BlockUtils; 19 | 20 | /** 21 | * Created by Xs.Tao on 2017/7/28. 22 | */ 23 | public class SimpleLeaderManager implements LeaderManager { 24 | 25 | public static final Logger LOGGER = LoggerFactory.getLogger(SimpleLeaderManager.class); 26 | 27 | private LeaderLatch leaderLatch; 28 | 29 | private CuratorFramework framework; 30 | 31 | private String serverName = ""; 32 | 33 | private volatile AtomicBoolean isLatch = new AtomicBoolean(false); 34 | private List listeners = Lists.newArrayList(); 35 | private RegistryProperties properties; 36 | 37 | public void init() { 38 | CuratorFrameworkFactory.Builder builder = CuratorFrameworkFactory.builder() 39 | .connectString(properties.getServerList()) 40 | .retryPolicy(new ExponentialBackoffRetry(properties.getBaseSleepTimeMilliseconds(), 41 | properties.getMaxRetries(), 42 | properties.getMaxSleepTimeMilliseconds())) 43 | .namespace(ServerNode.NAMESPACE); 44 | framework = builder.build(); 45 | framework.start(); 46 | leaderLatch = new LeaderLatch(framework, ServerNode.LEADERLATCH, serverName, LeaderLatch.CloseMode.NOTIFY_LEADER); 47 | for (LeaderLatchListener listener : listeners) { 48 | leaderLatch.addListener(listener); 49 | } 50 | LOGGER.info("starting Queue Master Slave Model ..."); 51 | start(); 52 | } 53 | 54 | 55 | @Override 56 | public void start() { 57 | if (isLatch.compareAndSet(false, true)) { 58 | try { 59 | LOGGER.info("starting latch...."); 60 | leaderLatch.start(); 61 | } catch (Exception e) { 62 | LOGGER.error(e.getMessage(), e); 63 | throw new RuntimeException(e); 64 | } 65 | } 66 | 67 | } 68 | 69 | @Override 70 | public void stop() { 71 | if (isLatch.compareAndSet(true, false)) { 72 | try { 73 | BlockUtils.sleep(500); 74 | LOGGER.info("stop latch...."); 75 | CloseableUtils.closeQuietly(leaderLatch); 76 | //CloseableUtils.closeQuietly(framework); 77 | } catch (Exception e) { 78 | LOGGER.error(e.getMessage(), e); 79 | throw new RuntimeException(e); 80 | } 81 | } 82 | } 83 | 84 | @Override 85 | public boolean isRunning() { 86 | return isLatch.get(); 87 | } 88 | 89 | @Override 90 | public boolean isLeader() { 91 | return leaderLatch.hasLeadership(); 92 | } 93 | 94 | public void setProperties(RegistryProperties properties) { 95 | this.properties = properties; 96 | } 97 | 98 | public void addListener(LeaderLatchListener leaderLatchListener) { 99 | if (!listeners.contains(leaderLatchListener)) { 100 | listeners.add(leaderLatchListener); 101 | } 102 | 103 | } 104 | 105 | public void setServerName(String serverName) { 106 | this.serverName = serverName; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/bin/sdmq: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ################################################### 3 | #Usage: ./sdmq {start|stop|restart|status} 4 | ################################################### 5 | #set -e 6 | PROG=delay-queue 7 | CMD_PRG="$0" 8 | PRGDIR=`dirname "$CMD_PRG"` 9 | PRO_HOME=`cd "$PRGDIR/.." >/dev/null; pwd` 10 | LOG_HOME=$PRO_HOME/logs 11 | echo "[Set 1] install dir : $PRO_HOME " 12 | if [ -x "$PRO_HOME"/"bin/env.sh" ];then 13 | echo "[Set 1] setting source $PRO_HOME/bin/env.sh" 14 | source $PRO_HOME/bin/env.sh ||exit 15 | fi 16 | 17 | if [ "$LOG_PATH" != "" ]; then 18 | LOG_HOME=$LOG_PATH 19 | fi 20 | echo "[Set 1.1] log path : $LOG_HOME " 21 | 22 | if [ "$JAVA_HOME" != "" ]; then 23 | JAVA_HOME=$JAVA_HOME 24 | fi 25 | 26 | if [ "$JAVA_HOME" = "" ]; then 27 | echo "WARN: JAVA_HOME is not set." 28 | # exit 1 29 | fi 30 | 31 | 32 | PIDFILE=$PRO_HOME/pid 33 | 34 | 35 | status() { 36 | if [ -f $PIDFILE ]; then 37 | PID=$(cat $PIDFILE) 38 | # ps_PIDLIST=`ps -ef|grep $CPS_PID|grep -v grep|awk -F" " '{print $2}'` 39 | if [ ! -x /proc/${PID} ]; then 40 | return 1 41 | else 42 | return 0 43 | fi 44 | else 45 | return 1 46 | fi 47 | } 48 | case "$1" in 49 | start) 50 | status 51 | RETVAL=$? 52 | if [ $RETVAL -eq 0 ]; then 53 | 54 | echo "$PIDFILE exists, process is already running($(cat $PIDFILE))" 55 | exit 1 56 | fi 57 | 58 | FILES=`ls $PRO_HOME/lib` 59 | echo $FILES 60 | 61 | for jarname in $FILES 62 | do 63 | JARS=$PRO_HOME/lib/$jarname 64 | done 65 | execmd=java 66 | if [ "$JAVA_HOME" != "" ]; then 67 | execmd=$JAVA_HOME/bin/java 68 | fi 69 | #echo "Starting $execmd $JAVA_OPTS $JAVA_MEM_OPTS $JAVA_DEBUG_OPTS $JAVA_JMX_OPTS -Dsoft.home=$PRO_HOME/ -Dsoft.logs=$LOG_HOME -jar $JARS --spring.config.location=file:$PRO_HOME/conf/ > /dev/null 2>&1 &" 70 | nohup $execmd $JAVA_OPTS $JAVA_MEM_OPTS $JAVA_DEBUG_OPTS $JAVA_JMX_OPTS -Dsoft.home=$PRO_HOME/ -Dsoft.logs=$LOG_HOME -jar $JARS --spring.config.location=file:$PRO_HOME/conf/ > /dev/null 2>&1 & 71 | #exec $execmd -jar $JARS $JAVA_OPTS $JAVA_MEM_OPTS $JAVA_DEBUG_OPTS $JAVA_JMX_OPTS --spring.config.location=file:$PRO_HOME/conf & 72 | RETVAL=$? 73 | if [ $RETVAL -eq 0 ]; then 74 | _pid=$! 75 | echo "$PROG is started($_pid)" 76 | echo $_pid > $PIDFILE 77 | exit 0 78 | else 79 | echo "Stopping $PROG" 80 | rm -f $PIDFILE 81 | exit 1 82 | fi 83 | ;; 84 | stop) 85 | status 86 | RETVAL=$? 87 | if [ $RETVAL -eq 0 ]; then 88 | echo "Shutting down $PROG `cat $PIDFILE`" 89 | kill `cat $PIDFILE` 90 | RETVAL=$? 91 | if [ $RETVAL -eq 0 ]; then 92 | rm -f $PIDFILE 93 | else 94 | echo "Failed to stopping $PROG" 95 | fi 96 | else 97 | echo 'not running' 98 | fi 99 | ;; 100 | status) 101 | status 102 | RETVAL=$? 103 | if [ $RETVAL -eq 0 ]; then 104 | PID=$(cat $PIDFILE) 105 | echo "$PROG is running ($PID) home:$PRO_HOME/ logs:$LOG_HOME" 106 | else 107 | echo "$PROG is not running" 108 | fi 109 | ;; 110 | watch) 111 | status 112 | RETVAL=$? 113 | if [ $RETVAL -eq 0 ]; then 114 | tail -f $2 | 115 | 116 | while IFS= read line 117 | do 118 | echo "$msgBuffer" "$line" 119 | 120 | if [[ "$line" == *"Started "* ]]; then 121 | echo Application Started... exiting buffer! 122 | pkill tail 123 | fi 124 | done 125 | else 126 | echo "$PROG is not running" 127 | fi 128 | ;; 129 | restart) 130 | $0 stop 131 | $0 start 132 | ;; 133 | *) 134 | echo "Usage: $0 {start|stop|restart|status}" 135 | ;; 136 | esac 137 | 138 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/bucket/BucketQueueManager.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.bucket; 2 | 3 | import com.google.common.collect.Maps; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.Map; 9 | import java.util.Timer; 10 | import java.util.concurrent.atomic.AtomicBoolean; 11 | 12 | import io.sdmq.queue.redis.JobOperationService; 13 | import io.sdmq.queue.redis.support.DistributedLock; 14 | import io.sdmq.queue.redis.support.Lifecycle; 15 | import io.sdmq.queue.redis.support.RedisQueueProperties; 16 | import io.sdmq.util.NamedUtil; 17 | 18 | /** 19 | * Created by Xs.Tao on 2017/7/18. 20 | */ 21 | public class BucketQueueManager implements Lifecycle { 22 | 23 | public static final Logger LOGGER = LoggerFactory.getLogger(BucketQueueManager.class); 24 | public static final String THREAD_NAME = "sdmq-delay-queue-%s"; 25 | public boolean daemon = true; 26 | private volatile AtomicBoolean isRuning = new AtomicBoolean(false); 27 | private RedisQueueProperties properties; 28 | private Map HOLD_TIMES = Maps.newConcurrentMap(); 29 | private JobOperationService jobOperationService; 30 | private DistributedLock lock = null; 31 | 32 | private int checkBucketNum(int bucketSize) { 33 | if (bucketSize <= 0) { 34 | bucketSize = 1; 35 | } 36 | return bucketSize; 37 | } 38 | 39 | // public static void main(String[] args) { 40 | // BucketQueueManager manager=new BucketQueueManager(); 41 | // RedisQueueProperties redisQueueProperties=new RedisQueueProperties(); 42 | // manager.setProperties(redisQueueProperties); 43 | // manager.start(); 44 | // try { 45 | // Thread.sleep(10000L); 46 | // } catch (InterruptedException e) { 47 | // e.printStackTrace(); 48 | // } 49 | // System.out.println(manager.isRunning()); 50 | // manager.stop(); 51 | // System.out.println(manager.isRunning()); 52 | // } 53 | 54 | 55 | @Override 56 | public void start() { 57 | int bucketSize = checkBucketNum(properties.getBucketSize()); 58 | if (isRuning.compareAndSet(false, true)) { 59 | for (int i = 1; i <= bucketSize; i++) { 60 | String bName = NamedUtil.buildBucketName(properties.getPrefix(), properties.getName(), i); 61 | BucketTask task = new BucketTask(bName); 62 | task.setJobOperationService(jobOperationService); 63 | task.setPoolName(NamedUtil.buildPoolName(properties.getPrefix(), properties.getName(), properties 64 | .getOriginPool())); 65 | task.setReadyName(NamedUtil.buildPoolName(properties.getPrefix(), properties.getName(), properties 66 | .getReadyName())); 67 | task.setProperties(properties); 68 | task.setLock(lock); 69 | String threadName = String.format(THREAD_NAME, i); 70 | Timer timer = new Timer(threadName, daemon); 71 | timer.schedule(task, 500, properties.getBuckRoundRobinTime()); 72 | HOLD_TIMES.put(threadName, timer); 73 | LOGGER.info(String.format("Starting Bucket Thead %s ....", threadName)); 74 | } 75 | 76 | } 77 | } 78 | 79 | @Override 80 | public void stop() { 81 | if (isRuning.compareAndSet(true, false)) { 82 | if (HOLD_TIMES != null && HOLD_TIMES.size() > 0) { 83 | for (Map.Entry entry : HOLD_TIMES.entrySet()) { 84 | String n = entry.getKey(); 85 | Timer timer = entry.getValue(); 86 | timer.cancel(); 87 | LOGGER.info(String.format("stoping timer %s .....", n)); 88 | } 89 | } 90 | } 91 | } 92 | 93 | @Override 94 | public boolean isRunning() { 95 | return isRuning.get(); 96 | } 97 | 98 | public void setProperties(RedisQueueProperties properties) { 99 | this.properties = properties; 100 | } 101 | 102 | public void setDaemon(boolean daemon) { 103 | this.daemon = daemon; 104 | } 105 | 106 | public void setJobOperationService(JobOperationService jobOperationService) { 107 | this.jobOperationService = jobOperationService; 108 | } 109 | 110 | public void setLock(DistributedLock lock) { 111 | this.lock = lock; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/extension/ExtensionLoader.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.extension; 2 | 3 | import org.springframework.util.StringUtils; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | import java.util.Map; 8 | import java.util.ServiceLoader; 9 | import java.util.concurrent.ConcurrentHashMap; 10 | 11 | import io.sdmq.queue.core.ConsumeQueueProvider; 12 | 13 | /** 14 | * Created by Xs.Tao on 2017/12/20. 15 | */ 16 | public final class ExtensionLoader { 17 | 18 | private static volatile Map, Object> extensionMap = new ConcurrentHashMap<>(); 19 | 20 | private static volatile Map, List> extensionListMap = new ConcurrentHashMap<>(); 21 | 22 | private ExtensionLoader() { 23 | } 24 | 25 | public static T getExtension(Class clazz) { 26 | T extension = (T) extensionMap.get(clazz); 27 | if (extension == null) { 28 | extension = newExtension(clazz); 29 | if (extension != null) { 30 | extensionMap.put(clazz, extension); 31 | } 32 | } 33 | return extension; 34 | } 35 | 36 | public static List getExtensionList(Class clazz) { 37 | List extensions = (List) extensionListMap.get(clazz); 38 | if (extensions == null) { 39 | extensions = newExtensionList(clazz); 40 | if (!extensions.isEmpty()) { 41 | extensionListMap.put(clazz, extensions); 42 | } 43 | } 44 | return extensions; 45 | } 46 | 47 | public static T newExtension(Class clazz) { 48 | String defaultImp = getDefaultSPI(clazz); 49 | if (StringUtils.isEmpty(defaultImp)) { 50 | throw new RuntimeException(String.format("请配置 %s SPI默认实现", clazz.getName())); 51 | } 52 | ServiceLoader serviceLoader = ServiceLoader.load(clazz); 53 | for (T service : serviceLoader) { 54 | if (service.getClass().isAnnotationPresent(ExtNamed.class) 55 | && defaultImp.equalsIgnoreCase(getExNamed(service.getClass()))) { 56 | return service; 57 | } 58 | } 59 | return null; 60 | } 61 | 62 | private static String getDefaultSPI(Class clazz) { 63 | String spi = System.getProperty(clazz.getName()); 64 | if (!StringUtils.isEmpty(spi)) { 65 | return spi; 66 | } 67 | if (clazz.isAnnotationPresent(SPI.class)) { 68 | SPI annotation = clazz.getAnnotation(SPI.class); 69 | return annotation.value(); 70 | } 71 | return null; 72 | } 73 | 74 | private static String getExNamed(Class clazz) { 75 | if (clazz.isAnnotationPresent(ExtNamed.class)) { 76 | ExtNamed annotation = clazz.getAnnotation(ExtNamed.class); 77 | return annotation.value(); 78 | } 79 | return null; 80 | } 81 | 82 | public static void main(String[] args) { 83 | System.out.println(ExtensionLoader.getExtension(ConsumeQueueProvider.class)); 84 | System.out.println(ExtensionLoader.getExtension(ConsumeQueueProvider.class)); 85 | System.out.println(ExtensionLoader.getExtension(ConsumeQueueProvider.class)); 86 | } 87 | 88 | public static List newExtensionList(Class clazz) { 89 | ServiceLoader serviceLoader = ServiceLoader.load(clazz); 90 | List extensions = new ArrayList<>(); 91 | for (T service : serviceLoader) { 92 | extensions.add(service); 93 | } 94 | return extensions; 95 | } 96 | 97 | // /** 98 | // * @param namespaceFun 命名空间函数 99 | // * @param namespaceValue 指定命名空间的值 100 | // */ 101 | // public static T getExtension(Class clazz, String namespaceFun, String namespaceValue) { 102 | // List lst = ExtensionLoader.getExtensionList(clazz); 103 | // if (lst == null || lst.size() == 0) { 104 | // throw new RuntimeException(String.format("请配置 %s 厂商实现", clazz.getName())); 105 | // } 106 | // if (lst != null && lst.size() > 0) { 107 | // for (T ext : lst) { 108 | // Method method = ReflectionUtils.findMethod(clazz, namespaceFun); 109 | // Assert.notNull(method, String.format("厂商接口 %s 没有找到命名空间函数 %s", clazz.getName(), namespaceFun)); 110 | // String namespaceValue_ = Objects.toString(ReflectionUtils.invokeMethod(method, ext)); 111 | // if (namespaceValue_.equalsIgnoreCase(namespaceValue)) { 112 | // 113 | // return ext; 114 | // } 115 | // } 116 | // } 117 | // return null; 118 | // } 119 | } -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/RedisQueueAutoConfiguration.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import com.alibaba.druid.pool.DruidDataSource; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | import org.springframework.beans.factory.annotation.Autowired; 8 | import org.springframework.boot.autoconfigure.condition.ConditionalOnClass; 9 | import org.springframework.boot.context.properties.EnableConfigurationProperties; 10 | import org.springframework.context.annotation.Bean; 11 | import org.springframework.context.annotation.Configuration; 12 | import org.springframework.data.redis.core.StringRedisTemplate; 13 | 14 | import io.sdmq.queue.redis.JobOperationService; 15 | import io.sdmq.queue.redis.JobOperationServiceImpl; 16 | import io.sdmq.queue.redis.RdbStore; 17 | import io.sdmq.queue.redis.RedisQueueImpl; 18 | import io.sdmq.queue.redis.bucket.BucketQueueManager; 19 | import io.sdmq.queue.redis.event.JobEventBus; 20 | import io.sdmq.queue.redis.event.JobEventListener; 21 | import io.sdmq.queue.redis.event.RedisJobEventListener; 22 | import io.sdmq.queue.redis.ready.ReadyQueueManager; 23 | import io.sdmq.queue.redis.support.RedisDistributedLock; 24 | import io.sdmq.queue.redis.support.RedisQueueProperties; 25 | import io.sdmq.queue.redis.support.RedisSupport; 26 | import redis.clients.jedis.Jedis; 27 | 28 | /** 29 | * Created by Xs.Tao on 2017/7/19. 30 | */ 31 | @Configuration 32 | @EnableConfigurationProperties(RedisQueueProperties.class) 33 | @ConditionalOnClass(value = {Jedis.class, RedisQueueImpl.class}) 34 | public class RedisQueueAutoConfiguration { 35 | 36 | public static final Logger LOGGER = LoggerFactory.getLogger(RedisQueueAutoConfiguration.class); 37 | @Autowired 38 | private DruidConfig druidConfig; 39 | @Autowired 40 | private RedisQueueProperties properties; 41 | @Autowired 42 | private StringRedisTemplate template; 43 | 44 | private JobOperationService jobOperationService; 45 | 46 | 47 | @Bean 48 | public RedisSupport redisSupport() { 49 | RedisSupport support = new RedisSupport(); 50 | support.setTemplate(template); 51 | return support; 52 | } 53 | 54 | /** 55 | * 分布式锁 56 | */ 57 | @Bean 58 | @Autowired 59 | public RedisDistributedLock redisDistributedLock(RedisSupport redisSupport) { 60 | return new RedisDistributedLock(redisSupport); 61 | } 62 | 63 | @Bean 64 | @Autowired 65 | public JobOperationService JobOperationService(RedisSupport redisSupport) { 66 | JobOperationServiceImpl jobOperationService = new JobOperationServiceImpl(); 67 | jobOperationService.setRedisSupport(redisSupport); 68 | jobOperationService.setProperties(properties); 69 | return jobOperationService; 70 | } 71 | 72 | @Bean 73 | @Autowired 74 | public BucketQueueManager BucketQueueManager(JobOperationService jobOperationService, RedisDistributedLock lock) { 75 | BucketQueueManager manager = new BucketQueueManager(); 76 | manager.setProperties(properties); 77 | manager.setJobOperationService(jobOperationService); 78 | manager.setLock(lock); 79 | return manager; 80 | } 81 | 82 | @Bean 83 | public RdbStore rdbStore() { 84 | try { 85 | DruidDataSource ds = (DruidDataSource) druidConfig.newInstanceDruidDataSource(); 86 | return new RdbStore(ds); 87 | } catch (Exception e) { 88 | LOGGER.error(e.getMessage()); 89 | throw new RuntimeException(e); 90 | } 91 | } 92 | 93 | @Bean 94 | @Autowired 95 | public JobEventListener jobEventListener(RdbStore store) { 96 | RedisJobEventListener eventListener = new RedisJobEventListener(); 97 | eventListener.setStore(store); 98 | JobEventBus.getInstance().register(eventListener); 99 | return eventListener; 100 | } 101 | 102 | @Bean 103 | @Autowired 104 | public ReadyQueueManager readyQueueManager(JobOperationService jobOperationService, RedisDistributedLock lock) { 105 | ReadyQueueManager manager = new ReadyQueueManager(); 106 | manager.setProperties(properties); 107 | manager.setJobOperationService(jobOperationService); 108 | manager.setLock(lock); 109 | return manager; 110 | } 111 | 112 | @Bean 113 | @Autowired 114 | public RedisQueueImpl redisQueueImpl(JobOperationService jobOperationService, 115 | BucketQueueManager bucketQueueManager, ReadyQueueManager readyQueueManager) { 116 | RedisQueueImpl redisQueue = new RedisQueueImpl(); 117 | redisQueue.setProperties(properties); 118 | redisQueue.setJobOperationService(jobOperationService); 119 | redisQueue.setBucketQueueManager(bucketQueueManager); 120 | redisQueue.setReadyQueueManager(readyQueueManager); 121 | readyQueueManager.setDelayQueue(redisQueue); 122 | return redisQueue; 123 | } 124 | 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/MessageProducer.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import com.google.common.util.concurrent.FutureCallback; 4 | import com.google.common.util.concurrent.Futures; 5 | import com.google.common.util.concurrent.ListenableFuture; 6 | import com.google.common.util.concurrent.ListeningExecutorService; 7 | import com.google.common.util.concurrent.MoreExecutors; 8 | 9 | import com.alibaba.rocketmq.client.exception.MQBrokerException; 10 | import com.alibaba.rocketmq.client.exception.MQClientException; 11 | import com.alibaba.rocketmq.client.producer.DefaultMQProducer; 12 | import com.alibaba.rocketmq.client.producer.SendResult; 13 | import com.alibaba.rocketmq.common.message.Message; 14 | import com.alibaba.rocketmq.remoting.exception.RemotingException; 15 | 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | import org.springframework.util.Assert; 19 | import org.springframework.util.CollectionUtils; 20 | import org.springframework.util.StringUtils; 21 | 22 | import java.io.Closeable; 23 | import java.io.IOException; 24 | import java.io.Serializable; 25 | import java.nio.charset.Charset; 26 | import java.util.Map; 27 | import java.util.Objects; 28 | import java.util.concurrent.Callable; 29 | import java.util.concurrent.ExecutorService; 30 | import java.util.concurrent.Executors; 31 | 32 | import io.sdmq.queue.JobMsg; 33 | import io.sdmq.queue.core.Job; 34 | import io.sdmq.queue.redis.RedisQueueImpl; 35 | import io.sdmq.util.FastJsonConvert; 36 | 37 | 38 | public class MessageProducer implements Closeable { 39 | 40 | public static final Logger LOGGER = LoggerFactory.getLogger(RedisQueueImpl.class); 41 | public static final ExecutorService EXECUTORS = Executors.newFixedThreadPool(2); 42 | private static DefaultMQProducer PRODUCER; 43 | private String namesrvAddr; 44 | private String groupName; 45 | 46 | /** 47 | * @return 48 | */ 49 | public static boolean send(Job msg) { 50 | Assert.notNull(msg, "参数错误"); 51 | Message message = new Message(); 52 | message.setTopic(msg.getTopic()); 53 | if (!StringUtils.isEmpty(msg.getSubtopic())) { 54 | message.setTags(msg.getSubtopic()); 55 | } 56 | message.setKeys(msg.getBizKey()); 57 | Serializable data = msg.getBody(); 58 | if (data != null) { 59 | message.setBody(((String) data).getBytes(Charset.forName("UTF-8"))); 60 | } else { 61 | message.setBody("".getBytes(Charset.forName("UTF-8"))); 62 | } 63 | if(!StringUtils.isEmpty(msg.getExtendData())){ 64 | Map map = FastJsonConvert.convertJSONMap(msg.getExtendData()); 65 | if(!CollectionUtils.isEmpty(map)){ 66 | for(Map.Entry entry:map.entrySet()){ 67 | message.putUserProperty(entry.getKey(),Objects.toString(entry.getValue())); 68 | } 69 | } 70 | } 71 | 72 | try { 73 | SendResult send = PRODUCER.send(message); 74 | } catch (MQClientException | MQBrokerException | RemotingException | InterruptedException e) { 75 | LOGGER.error(String.format("消息发送失败[%s]", message.toString()), e); 76 | return false; 77 | } 78 | return true; 79 | } 80 | 81 | //guava异步发送mq 82 | public static void sendAsyncMessage(final JobMsg job) { 83 | 84 | ListeningExecutorService guavaExecutor = MoreExecutors.listeningDecorator(EXECUTORS); 85 | 86 | final ListenableFuture listenableFuture = guavaExecutor.submit(new Callable() { 87 | 88 | @Override 89 | public Boolean call() throws Exception { 90 | return MessageProducer.send(job); 91 | } 92 | }); 93 | Futures.addCallback(listenableFuture, new FutureCallback() { 94 | @Override 95 | public void onSuccess(Boolean mqMessageStatus) { 96 | } 97 | 98 | @Override 99 | public void onFailure(Throwable throwable) { 100 | LOGGER.error(throwable.getMessage()); 101 | } 102 | },guavaExecutor); 103 | } 104 | 105 | public String getNamesrvAddr() { 106 | return namesrvAddr; 107 | } 108 | 109 | public void setNamesrvAddr(String namesrvAddr) { 110 | this.namesrvAddr = namesrvAddr; 111 | } 112 | 113 | public void setGroupName(String groupName) { 114 | this.groupName = groupName; 115 | } 116 | 117 | protected void init() { 118 | if (PRODUCER == null) { 119 | PRODUCER = new DefaultMQProducer(Objects.toString(groupName,"Producer")); 120 | PRODUCER.setNamesrvAddr(namesrvAddr); 121 | try { 122 | PRODUCER.start(); 123 | } catch (MQClientException e) { 124 | LOGGER.error("消息发送端初始化失败", e); 125 | throw new RuntimeException(e); 126 | } 127 | } 128 | } 129 | 130 | /** 131 | * 关闭消息发送端，同时释放资源，由容器自动处理，程序中不能调用此方法 132 | */ 133 | @Override 134 | public void close() throws IOException { 135 | if (PRODUCER != null) { 136 | LOGGER.info("shutdowing mq..."); 137 | PRODUCER.shutdown(); 138 | } 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/ready/RealTimeTask.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.ready; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.util.StringUtils; 6 | 7 | import java.util.Date; 8 | import java.util.List; 9 | import java.util.TimerTask; 10 | 11 | import io.sdmq.queue.JobMsg; 12 | import io.sdmq.queue.core.ConsumeQueueProvider; 13 | import io.sdmq.queue.core.Queue; 14 | import io.sdmq.queue.redis.JobOperationService; 15 | import io.sdmq.queue.redis.event.JobEventBus; 16 | import io.sdmq.queue.redis.event.RedisJobTraceEvent; 17 | import io.sdmq.queue.redis.support.DistributedLock; 18 | import io.sdmq.queue.redis.support.RedisQueueProperties; 19 | import io.sdmq.util.DateUtils; 20 | import io.sdmq.util.NamedUtil; 21 | import io.sdmq.util.Status; 22 | 23 | /** 24 | * 
 25 |  *     此客户端默认存在 并默认启动
 26 |  *
 27 |  *     发现数据 立即发MQ  后期后改善为push、pull 或者长连接之类的实现
 28 |  *
29 | * Created by Xs.Tao on 2017/7/19. 30 | */ 31 | public class RealTimeTask extends TimerTask { 32 | 33 | public static final Logger LOGGER = LoggerFactory.getLogger(RealTimeTask.class); 34 | private RedisQueueProperties properties; 35 | private JobOperationService jobOperationService; 36 | private Queue delayQueue; 37 | private DistributedLock lock = null; 38 | private ConsumeQueueProvider consumeQueueProvider = null; 39 | 40 | @Override 41 | public void run() { 42 | runTemplate(); 43 | } 44 | 45 | private void runTemplate() { 46 | if (properties.isCluster()) { 47 | String lockName = NamedUtil.buildLockName(NamedUtil.buildRealTimeName(properties.getPrefix(), properties 48 | .getName(), properties.getReadyName())); 49 | try { 50 | lock.lock(lockName); 51 | runInstance(); 52 | } finally { 53 | lock.unlock(lockName); 54 | } 55 | } else { 56 | runInstance(); 57 | } 58 | } 59 | 60 | private void runInstance() { 61 | try { 62 | if (LOGGER.isTraceEnabled()) { 63 | LOGGER.trace("开始轮询实时队列...%s"); 64 | } 65 | //获取ready队列中的一个数据 66 | List jobIds = jobOperationService.getReadyJob(10); 67 | if (jobIds != null && jobIds.size() > 0) { 68 | for (String jobId : jobIds) { 69 | if (!StringUtils.isEmpty(jobId)) { 70 | JobMsg j = jobOperationService.getJob(jobId); 71 | if (j == null) { 72 | this.jobOperationService.removeReadyJob(jobId); 73 | LOGGER.warn("任务ID {} 元数据池没有数据", jobId); 74 | continue; 75 | } 76 | if (j.getStatus() == Status.Delete.ordinal()) { 77 | this.jobOperationService.removeJobToPool(jobId); 78 | continue; 79 | } 80 | if (j.getStatus() != Status.Delete.ordinal()) { 81 | if (!check(j)) {//没有达到执行时间 从新发送延时Buck中 82 | j.setStatus(Status.Restore.ordinal()); 83 | delayQueue.push(j); 84 | continue; 85 | } 86 | if (LOGGER.isInfoEnabled()) { 87 | long runLong = j.getDelay() + j.getCreateTime(); 88 | String runDateString = DateUtils.format(new Date(runLong), DateUtils 89 | .FORMAT_YYYY_MM_DD_HH_MM_SS_SSS); 90 | LOGGER.info(String.format("invokeTask %s target time : %s", jobId, runDateString)); 91 | } 92 | consumeQueueProvider.consumer(j); 93 | j.setStatus(Status.Finish.ordinal()); 94 | this.jobOperationService.updateJobStatus(jobId, Status.Finish); 95 | this.jobOperationService.removeReadyJob(jobId); 96 | this.jobOperationService.removeJobToPool(jobId); 97 | JobEventBus.getInstance().post(new RedisJobTraceEvent(j)); 98 | } 99 | } 100 | } 101 | } 102 | } catch (Exception e) { 103 | LOGGER.error("处理实时队列发生错误", e); 104 | } 105 | } 106 | 107 | private boolean check(JobMsg job) { 108 | long runTime = job.getCreateTime() + job.getDelay(), 109 | currentTime = System.currentTimeMillis(); 110 | return runTime <= currentTime; 111 | } 112 | 113 | 114 | public void setProperties(RedisQueueProperties properties) { 115 | this.properties = properties; 116 | } 117 | 118 | public void setJobOperationService(JobOperationService jobOperationService) { 119 | this.jobOperationService = jobOperationService; 120 | } 121 | 122 | /** 123 | * 注入队列操作对象 便于时间没有到触发条件下 执行重新发送 124 | */ 125 | public void setDelayQueue(Queue delayQueue) { 126 | this.delayQueue = delayQueue; 127 | } 128 | 129 | public void setLock(DistributedLock lock) { 130 | this.lock = lock; 131 | } 132 | 133 | public void setConsumeQueueProvider(ConsumeQueueProvider consumeQueueProvider) { 134 | this.consumeQueueProvider = consumeQueueProvider; 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/FastJsonConvert.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | import com.alibaba.fastjson.JSONObject; 5 | import com.alibaba.fastjson.TypeReference; 6 | import com.alibaba.fastjson.serializer.SerializerFeature; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | /** 16 | * Created by Xs.Tao on 2017/7/18. 17 | */ 18 | public class FastJsonConvert { 19 | private final static Logger LOGGER = LoggerFactory 20 | .getLogger(FastJsonConvert.class); 21 | 22 | private static final SerializerFeature[] featuresWithNullValue = {SerializerFeature.WriteMapNullValue, 23 | SerializerFeature.WriteNullBooleanAsFalse, 24 | SerializerFeature.WriteNullListAsEmpty, SerializerFeature.WriteNullNumberAsZero, SerializerFeature 25 | .WriteNullStringAsEmpty}; 26 | 27 | /** 28 | * 方法名称：将JSON字符串转换为实体对象
29 | * 概要说明：将JSON字符串转换为实体对象
30 | * 31 | * @param data JSON字符串 32 | * @param clzss 转换对象 33 | * @return T 34 | */ 35 | public static T convertJSONToObject(String data, Class clzss) { 36 | try { 37 | T t = JSON.parseObject(data, clzss); 38 | return t; 39 | } catch (Exception e) { 40 | e.printStackTrace(); 41 | return null; 42 | } 43 | } 44 | 45 | /** 46 | * 方法名称：将JSONObject对象转换为实体对象
47 | * 概要说明：将JSONObject对象转换为实体对象
48 | * 49 | * @param data JSONObject对象 50 | * @param clzss 转换对象 51 | * @return T 52 | */ 53 | public static T convertJSONToObject(JSONObject data, Class clzss) { 54 | try { 55 | T t = JSONObject.toJavaObject(data, clzss); 56 | return t; 57 | } catch (Exception e) { 58 | e.printStackTrace(); 59 | return null; 60 | } 61 | } 62 | 63 | /** 64 | * 方法名称：将JSON字符串数组转为List集合对象
65 | * 概要说明：将JSON字符串数组转为List集合对象
66 | * 67 | * @param data JSON字符串数组 68 | * @param clzss 转换对象 69 | * @return List集合对象 70 | */ 71 | public static List convertJSONToArray(String data, Class clzss) { 72 | try { 73 | List t = JSON.parseArray(data, clzss); 74 | return t; 75 | } catch (Exception e) { 76 | e.printStackTrace(); 77 | return null; 78 | } 79 | } 80 | 81 | /** 82 | * 方法名称：将List转为List集合对象
83 | * 概要说明：将List转为List集合对象
84 | * 85 | * @param data List 86 | * @param clzss 转换对象 87 | * @return List集合对象 88 | */ 89 | public static List convertJSONToArray(List data, Class clzss) { 90 | try { 91 | List t = new ArrayList(); 92 | for (JSONObject jsonObject : data) { 93 | t.add(convertJSONToObject(jsonObject, clzss)); 94 | } 95 | return t; 96 | } catch (Exception e) { 97 | e.printStackTrace(); 98 | return null; 99 | } 100 | } 101 | 102 | /** 103 | * 方法名称：将对象转为JSON字符串
104 | * 概要说明：将对象转为JSON字符串
105 | * 106 | * @param obj 任意对象 107 | * @return JSON字符串 108 | */ 109 | public static String convertObjectToJSON(Object obj) { 110 | try { 111 | String text = JSON.toJSONString(obj); 112 | return text; 113 | } catch (Exception e) { 114 | e.printStackTrace(); 115 | return null; 116 | } 117 | } 118 | 119 | /** 120 | * 方法名称：将对象转为(JSON字符串)
121 | * 概要说明：将对象转为(JSON字符串)
122 | * 123 | * @param obj 任意对象 124 | * @return JSON字符串 125 | */ 126 | public static String convertObjectToJSONBracket(Object obj) { 127 | try { 128 | String text = JSON.toJSONString(obj); 129 | return "(" + text + ")"; 130 | } catch (Exception e) { 131 | e.printStackTrace(); 132 | return null; 133 | } 134 | } 135 | 136 | /** 137 | * 方法名称：将对象转为JSONObject对象
138 | * 概要说明：将对象转为JSONObject对象
139 | * 140 | * @param obj 任意对象 141 | * @return JSONObject对象 142 | */ 143 | public static JSONObject convertObjectToJSONObject(Object obj) { 144 | try { 145 | JSONObject jsonObject = (JSONObject) JSONObject.toJSON(obj); 146 | return jsonObject; 147 | } catch (Exception e) { 148 | e.printStackTrace(); 149 | return null; 150 | } 151 | } 152 | 153 | 154 | /** 155 | * 方法名称：
156 | * 概要说明：
157 | */ 158 | public static String convertObjectToJSONWithNullValue(Object obj) { 159 | try { 160 | String text = JSON.toJSONString(obj, featuresWithNullValue); 161 | return text; 162 | } catch (Exception e) { 163 | e.printStackTrace(); 164 | return null; 165 | } 166 | } 167 | public static Map convertJSONMap(String data) { 168 | return convertJSONTypeReference(data, 169 | new TypeReference>() { 170 | }); 171 | } 172 | public static T convertJSONTypeReference(String data, 173 | TypeReference typeReference) { 174 | try { 175 | T listMap = JSON.parseObject(data, typeReference); 176 | return listMap; 177 | } catch (Exception e) { 178 | LOGGER.error("转换JSON失败 异常消息{} \n原文为:{}", e.getMessage(), data); 179 | return null; 180 | } 181 | } 182 | } 183 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/bucket/BucketTask.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.bucket; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.util.StringUtils; 6 | 7 | import java.util.List; 8 | import java.util.TimerTask; 9 | 10 | import io.sdmq.queue.JobMsg; 11 | import io.sdmq.queue.redis.JobOperationService; 12 | import io.sdmq.queue.redis.event.JobEventBus; 13 | import io.sdmq.queue.redis.event.RedisJobTraceEvent; 14 | import io.sdmq.queue.redis.support.DistributedLock; 15 | import io.sdmq.queue.redis.support.RedisQueueProperties; 16 | import io.sdmq.util.NamedUtil; 17 | import io.sdmq.util.Status; 18 | 19 | /** 20 | * Created by Xs.Tao on 2017/7/18. 21 | */ 22 | public class BucketTask extends TimerTask { 23 | 24 | public static final Logger LOGGER = LoggerFactory.getLogger(BucketTask.class); 25 | public static final long TIME_OUT = 1000 * 30; 26 | private String bucketName; 27 | private String poolName; 28 | private String readyName; 29 | private JobOperationService jobOperationService; 30 | private DistributedLock lock = null; 31 | private RedisQueueProperties properties; 32 | 33 | 34 | protected BucketTask(String bucketName) { 35 | this.bucketName = bucketName; 36 | } 37 | 38 | @Override 39 | public void run() { 40 | runTemplate(); 41 | } 42 | 43 | private void runTemplate() { 44 | if (properties.isCluster()) { 45 | String lockName = NamedUtil.buildLockName(bucketName); 46 | try { 47 | lock.lock(lockName); 48 | runInstance(); 49 | } finally { 50 | lock.unlock(lockName); 51 | } 52 | } else { 53 | runInstance(); 54 | } 55 | } 56 | 57 | private void runInstance() { 58 | try { 59 | if (LOGGER.isTraceEnabled()) { 60 | LOGGER.trace(String.format("开始轮询...%s", bucketName)); 61 | } 62 | List jobs = peeks(); 63 | if (jobs != null && jobs.size() > 0) { 64 | for (String jobId : jobs) { 65 | if (!StringUtils.isEmpty(jobId)) { 66 | JobMsg job = jobOperationService.getJob(jobId); 67 | if (null == job) {//如果数据为null 68 | jobOperationService.removeBucketKey(bucketName, jobId); 69 | continue; 70 | } 71 | if (job.getStatus() == Status.Delete.ordinal()) { 72 | this.jobOperationService.removeJobToPool(jobId); 73 | this.jobOperationService.removeBucketKey(bucketName, jobId); 74 | continue; 75 | } 76 | //不是延迟状态的数据 不处理 以防数据被消费了 状态还没有更新的问题 77 | if (job.getStatus() != Status.Delay.ordinal()) { 78 | continue; 79 | } 80 | long delay = job.getCreateTime() + job.getDelay() - System.currentTimeMillis(); 81 | if (delay <= 0) {//到了当前时间了 把数据从buck移除 并添加到实时队列 82 | job.setStatus(Status.Ready.ordinal()); 83 | //首先执行修改元数据状态 避免timer竞争到数据了 而状态不一致而出现问题。 84 | JobEventBus.getInstance().post(new RedisJobTraceEvent(job)); 85 | this.jobOperationService.updateJobStatus(job.getId(), Status.Ready); 86 | jobOperationService.addReadyTime(readyName, jobId); 87 | jobOperationService.removeBucketKey(bucketName, jobId);//删除 88 | if (LOGGER.isDebugEnabled()) { 89 | LOGGER.debug(String.format("jobId %s 达到了运行时间了", jobId)); 90 | } 91 | } else {//还没有到当前时间 92 | //SKIP 93 | } 94 | } 95 | } 96 | } 97 | } catch (Exception e) { 98 | LOGGER.error(String.format("处理延时队 %s 列发生错误", bucketName), e); 99 | } 100 | } 101 | 102 | private String peek() { 103 | try { 104 | String jobMsgId = jobOperationService.getBucketTop1Job(bucketName); 105 | return jobMsgId; 106 | } catch (Exception e) { 107 | throw e; 108 | } 109 | } 110 | 111 | /** 112 | * 批量偷取 提高实时性 113 | */ 114 | private List peeks() { 115 | try { 116 | List lst = jobOperationService.getBucketTopJobs(bucketName, 10); 117 | return lst; 118 | } catch (Exception e) { 119 | throw e; 120 | } 121 | } 122 | 123 | public String getBucketName() { 124 | return bucketName; 125 | } 126 | 127 | public void setBucketName(String bucketName) { 128 | this.bucketName = bucketName; 129 | } 130 | 131 | public String getPoolName() { 132 | return poolName; 133 | } 134 | 135 | public void setPoolName(String poolName) { 136 | this.poolName = poolName; 137 | } 138 | 139 | public String getReadyName() { 140 | return readyName; 141 | } 142 | 143 | public void setReadyName(String readyName) { 144 | this.readyName = readyName; 145 | } 146 | 147 | public void setJobOperationService(JobOperationService jobOperationService) { 148 | this.jobOperationService = jobOperationService; 149 | } 150 | 151 | public void setLock(DistributedLock lock) { 152 | this.lock = lock; 153 | } 154 | 155 | public void setProperties(RedisQueueProperties properties) { 156 | this.properties = properties; 157 | } 158 | } 159 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/JobOperationServiceImpl.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis; 2 | 3 | import com.google.common.collect.Lists; 4 | 5 | import org.springframework.data.redis.core.ZSetOperations; 6 | import org.springframework.util.Assert; 7 | 8 | import java.util.Iterator; 9 | import java.util.List; 10 | import java.util.Objects; 11 | import java.util.Set; 12 | 13 | import io.sdmq.queue.JobMsg; 14 | import io.sdmq.queue.redis.bucket.BucketTask; 15 | import io.sdmq.queue.redis.support.RedisQueueProperties; 16 | import io.sdmq.queue.redis.support.RedisSupport; 17 | import io.sdmq.util.FastJsonConvert; 18 | import io.sdmq.util.NamedUtil; 19 | import io.sdmq.util.Status; 20 | 21 | /** 22 | * Created by Xs.Tao on 2017/7/19. 23 | */ 24 | public class JobOperationServiceImpl implements JobOperationService { 25 | 26 | private RedisSupport redisSupport; 27 | private RedisQueueProperties properties; 28 | 29 | public void setRedisSupport(RedisSupport redisSupport) { 30 | this.redisSupport = redisSupport; 31 | } 32 | 33 | private String getPoolName() { 34 | String name = NamedUtil.buildRealTimeName(properties.getPrefix(), properties.getName(), properties.getOriginPool()); 35 | return name; 36 | } 37 | 38 | private String geReadyName() { 39 | String name = NamedUtil.buildRealTimeName(properties.getPrefix(), properties.getName(), properties.getReadyName()); 40 | return name; 41 | } 42 | 43 | public JobMsg getJob(String jobId) { 44 | Assert.notNull(jobId, "非法参数"); 45 | String json = redisSupport.getHashKey(getPoolName(), jobId); 46 | JobMsg job = FastJsonConvert.convertJSONToObject(json, JobWrapp.class); 47 | return job; 48 | } 49 | 50 | @Override 51 | public void addJobToPool(JobMsg jobMsg) { 52 | redisSupport.hashPut(getPoolName(), jobMsg.getId(), FastJsonConvert.convertObjectToJSON(jobMsg)); 53 | } 54 | 55 | @Override 56 | public void removeJobToPool(String jobId) { 57 | redisSupport.deleteHashKeys(getPoolName(), jobId); 58 | } 59 | 60 | @Override 61 | public void updateJobStatus(String jobId, Status status) { 62 | JobMsg msg = getJob(jobId); 63 | Assert.notNull(msg, String.format("JobId %s 数据已不存在", jobId)); 64 | msg.setStatus(status.ordinal()); 65 | addJobToPool(msg); 66 | } 67 | 68 | @Override 69 | public void deleteJobToPool(String jobId) { 70 | redisSupport.deleteHashKeys(getPoolName(), jobId); 71 | } 72 | 73 | @Override 74 | public void addBucketJob(String bucketName, String JobId, double score) { 75 | redisSupport.zadd(bucketName, JobId, score); 76 | } 77 | 78 | 79 | @Override 80 | public void removeBucketKey(String bucketName, String jobId) { 81 | redisSupport.zrem(bucketName, jobId); 82 | } 83 | 84 | @Override 85 | public void addReadyTime(String readyName, String jobId) { 86 | 87 | redisSupport.rightPush(readyName, jobId); 88 | } 89 | 90 | 91 | @Override 92 | public String getReadyJob() { 93 | return redisSupport.leftPop(geReadyName()); 94 | } 95 | 96 | @Override 97 | public List getReadyJob(int size) { 98 | List root = redisSupport.lrange(geReadyName(), 0, size); 99 | if (null != root && root.size() > 0) { 100 | root = Lists.reverse(root); 101 | } 102 | return root; 103 | /* 104 | for(int i=0;i sets = redisSupport.zrangeByScore(bucketName, 0, to, 0, 1); 125 | if (sets != null && sets.size() > 0) { 126 | String jobMsgId = Objects.toString(sets.toArray()[0]); 127 | return jobMsgId; 128 | } 129 | return null; 130 | } 131 | 132 | @Override 133 | public List getBucketTopJobs(String bucketName, int size) { 134 | double to = Long.valueOf(System.currentTimeMillis() + BucketTask.TIME_OUT).doubleValue(); 135 | Set> sets = redisSupport.zrangeByScoreWithScores(bucketName, 0, to, 0, size); 136 | List lsts = Lists.newArrayList(); 137 | if (sets != null && sets.size() > 0) { 138 | Iterator> it = sets.iterator(); 139 | while (it.hasNext()) { 140 | ZSetOperations.TypedTuple curr = it.next(); 141 | if (curr.getScore() <= System.currentTimeMillis()) { 142 | lsts.add(curr.getValue()); 143 | } else { 144 | break; 145 | } 146 | } 147 | // String jobMsgId = Objects.toString(sets.toArray()[0]); 148 | return lsts; 149 | } 150 | return null; 151 | } 152 | 153 | @Override 154 | public void clearAll() { 155 | int buckSize = properties.getBucketSize(); 156 | if (buckSize <= 0) { 157 | buckSize = 1; 158 | } 159 | List lst = Lists.newArrayList(); 160 | for (int i = 1; i <= buckSize; i++) { 161 | lst.add(NamedUtil.buildBucketName(properties.getPrefix(), properties.getName(), i)); 162 | } 163 | lst.add(getPoolName()); 164 | lst.add(geReadyName()); 165 | redisSupport.deleteKey(lst.toArray(new String[]{})); 166 | } 167 | 168 | public void setProperties(RedisQueueProperties properties) { 169 | this.properties = properties; 170 | } 171 | } 172 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/RedisQueueImpl.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | import org.springframework.util.Assert; 6 | import org.springframework.util.StringUtils; 7 | 8 | import java.io.IOException; 9 | import java.util.concurrent.atomic.AtomicBoolean; 10 | import java.util.concurrent.atomic.AtomicInteger; 11 | 12 | import io.sdmq.exception.DelayQueueException; 13 | import io.sdmq.exception.JobNotFoundException; 14 | import io.sdmq.queue.JobMsg; 15 | import io.sdmq.queue.core.Queue; 16 | import io.sdmq.queue.redis.bucket.BucketQueueManager; 17 | import io.sdmq.queue.redis.event.JobEventBus; 18 | import io.sdmq.queue.redis.event.RedisJobTraceEvent; 19 | import io.sdmq.queue.redis.ready.ReadyQueueManager; 20 | import io.sdmq.queue.redis.support.RedisQueueProperties; 21 | import io.sdmq.util.NamedUtil; 22 | import io.sdmq.util.RdbOperation; 23 | import io.sdmq.util.Status; 24 | 25 | /** 26 | * Created by Xs.Tao on 2017/7/18. 27 | */ 28 | public class RedisQueueImpl implements Queue, java.io.Closeable { 29 | 30 | public static final Logger LOGGER = LoggerFactory.getLogger(RedisQueueImpl.class); 31 | private volatile AtomicBoolean isRuning = new AtomicBoolean(false); 32 | private volatile AtomicInteger pos = new AtomicInteger(0); 33 | 34 | private JobOperationService jobOperationService; 35 | 36 | private RedisQueueProperties properties; 37 | 38 | private BucketQueueManager bucketQueueManager; 39 | 40 | private ReadyQueueManager readyQueueManager; 41 | 42 | 43 | public void push(JobMsg job) throws DelayQueueException { 44 | try { 45 | Assert.notNull(job, "Job不能为空"); 46 | Assert.notNull(job.getId(), "JobId 不能为空"); 47 | Assert.notNull(job.getDelay(), "Job Delay不能为空"); 48 | if (job.getStatus() != Status.WaitPut.ordinal() && job.getStatus() != Status.Restore.ordinal()) { 49 | throw new IllegalArgumentException(String.format("任务%s状态异常", job.getId())); 50 | } 51 | String queueName = buildQueueName(); 52 | if (job instanceof JobWrapp) { 53 | ((JobWrapp) job).setBuckedName(queueName); 54 | } 55 | this.jobOperationService.addJobToPool(job); 56 | JobEventBus.getInstance().post(new RedisJobTraceEvent(job, RdbOperation.INSERT)); 57 | double score = Long.valueOf(job.getCreateTime() + job.getDelay()); 58 | 59 | this.jobOperationService.addBucketJob(queueName, job.getId(), score); 60 | job.setStatus(Status.Delay.ordinal()); 61 | this.jobOperationService.updateJobStatus(job.getId(), Status.Delay); 62 | JobEventBus.getInstance().post(new RedisJobTraceEvent(job)); 63 | if (LOGGER.isDebugEnabled()) { 64 | LOGGER.debug("task {} append success bucket to {} !", job.getId(), queueName); 65 | } 66 | } catch (Exception e) { 67 | LOGGER.error("添加任务失败", e); 68 | throw new DelayQueueException(e); 69 | } 70 | } 71 | 72 | 73 | @Override 74 | public boolean ack(String jobMsgId) throws DelayQueueException { 75 | throw new DelayQueueException("待实现"); 76 | } 77 | 78 | @Override 79 | public long getSize() { 80 | throw new DelayQueueException("待实现"); 81 | } 82 | 83 | @Override 84 | public void clear() { 85 | LOGGER.warn("正在执行清空队列操作 请注意"); 86 | this.jobOperationService.clearAll(); 87 | } 88 | 89 | @Override 90 | public boolean delete(String jobMsgId) { 91 | JobWrapp job = (JobWrapp) this.jobOperationService.getJob(jobMsgId); 92 | 93 | if (null == job) { 94 | return false; 95 | } 96 | if (job.getStatus() == Status.Finish.ordinal()) { 97 | throw new JobNotFoundException(String.format("任务 %s 已经完成", jobMsgId)); 98 | } 99 | job.setStatus(Status.Delete.ordinal()); 100 | this.jobOperationService.addJobToPool(job);//更新这个数据到池 101 | JobEventBus.getInstance().post(new RedisJobTraceEvent(job)); 102 | //是否需要删除buck？ 103 | if (!StringUtils.isEmpty(job.getBuckedName())) { 104 | this.jobOperationService.removeBucketKey(job.getBuckedName(), jobMsgId); 105 | this.jobOperationService.removeJobToPool(jobMsgId);//fix 删除源数据 这种放在源数据池中毫无意义 106 | } 107 | return true; 108 | } 109 | 110 | @Override 111 | public JobMsg getJob(String jobId) { 112 | Assert.notNull(jobId); 113 | JobMsg jobMsg = this.jobOperationService.getJob(jobId); 114 | return jobMsg; 115 | } 116 | 117 | @Override 118 | public String getImplementType() { 119 | return this.getClass().getSimpleName(); 120 | } 121 | 122 | @Override 123 | public void close() throws IOException { 124 | stop(); 125 | } 126 | 127 | /////////////////////////help //////////// 128 | 129 | /** 130 | * 根据BuckSize轮询获取 131 | */ 132 | public String buildQueueName() { 133 | return NamedUtil.buildBucketName(properties.getPrefix(), properties.getName(), getNextRoundRobin()); 134 | } 135 | 136 | /** 137 | * 轮询算法 目前只适用于单机 138 | */ 139 | private int getNextRoundRobin() { 140 | synchronized (this) { 141 | if (pos.get() >= properties.getBucketSize() || pos.get() < 0) { 142 | pos.set(1); 143 | } else { 144 | pos.getAndIncrement(); 145 | } 146 | } 147 | return pos.get(); 148 | } 149 | 150 | public void setJobOperationService(JobOperationService jobOperationService) { 151 | this.jobOperationService = jobOperationService; 152 | } 153 | 154 | public void setProperties(RedisQueueProperties properties) { 155 | this.properties = properties; 156 | } 157 | 158 | public void setBucketQueueManager(BucketQueueManager bucketQueueManager) { 159 | this.bucketQueueManager = bucketQueueManager; 160 | } 161 | 162 | public void setReadyQueueManager(ReadyQueueManager readyQueueManager) { 163 | this.readyQueueManager = readyQueueManager; 164 | } 165 | 166 | @Override 167 | public void start() { 168 | if (isRuning.compareAndSet(false, true)) { 169 | if (LOGGER.isInfoEnabled() && properties.isCluster()) { 170 | LOGGER.info("Cluster Model Starting..."); 171 | } 172 | bucketQueueManager.start(); 173 | readyQueueManager.start(); 174 | } 175 | } 176 | 177 | @Override 178 | public void stop() { 179 | if (isRuning.compareAndSet(true, false)) { 180 | bucketQueueManager.stop(); 181 | readyQueueManager.stop(); 182 | } 183 | 184 | } 185 | 186 | @Override 187 | public boolean isRunning() { 188 | return isRuning.get(); 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/DateUtils.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import java.text.ParseException; 4 | import java.text.SimpleDateFormat; 5 | import java.util.Date; 6 | import java.util.HashMap; 7 | import java.util.Locale; 8 | import java.util.Map; 9 | 10 | 11 | public class DateUtils { 12 | 13 | public static final String FORMAT_DEFAULT = "yyyy-MM-dd HH:mm:ss"; 14 | public static final String FORMAT_YYYY_MM_DD_HH_MM_SS_SSS = "yyyy-MM-dd HH:mm:ss SSS"; 15 | public static final String FORMAT_TIMESTAMP = "yyyy-MM-dd_HH_mm_ss_SSS"; 16 | public static final String FORMAT_YYYYMMDD = "yyyyMMdd"; 17 | 18 | public static final String FORMAT_DATE = "yyyy-MM-dd"; 19 | 20 | public static final String FORMAT_MONTH = "yyyy-MM"; 21 | 22 | public static final String FORMAT_TIME = "HH:mm:ss"; 23 | 24 | public static final String FORMAT_SHORT_DATE_TIME = "MM-dd HH:mm"; 25 | 26 | public static final String FORMAT_DATE_TIME = FORMAT_DEFAULT; 27 | 28 | public static final String FORMAT_NO_SECOND = "yyyy-MM-dd HH:mm"; 29 | 30 | public static final String FORMAT_JAPAN = "MM.dd(EEE) HH"; 31 | 32 | public static final String FORMAT_CHINESE_NO_SECOND = "yyyy年MM月dd日 HH:mm"; 33 | 34 | public static final String FORMAT_CHINESE_NO_SECOND_1 = "yyyy年MM月dd日HH:mm"; 35 | 36 | public static final String FORMAT_CHINESE = "yyyy年MM月dd日 HH点mm分"; 37 | 38 | public static final String FROMAT_CHAINESE_WEEK_SECOND = "yyyy-MM-dd(E) HH:mm"; 39 | 40 | public static final int TYPE_HTML_SPACE = 2; 41 | 42 | public static final int TYPE_DECREASE_SYMBOL = 3; 43 | 44 | public static final int TYPE_SPACE = 4; 45 | 46 | public static final int TYPE_NULL = 5; 47 | private static Map formaters = new HashMap(); 48 | 49 | static { 50 | SimpleDateFormat defaultFormater = new SimpleDateFormat(FORMAT_DEFAULT, Locale.CHINA); 51 | formaters.put(FORMAT_DEFAULT, defaultFormater); 52 | formaters.put(FORMAT_DATE, new SimpleDateFormat(FORMAT_DATE, Locale.CHINA)); 53 | formaters.put(FORMAT_MONTH, new SimpleDateFormat(FORMAT_MONTH, Locale.CHINA)); 54 | formaters.put(FORMAT_TIME, new SimpleDateFormat(FORMAT_TIME, Locale.CHINA)); 55 | formaters.put(FORMAT_SHORT_DATE_TIME, new SimpleDateFormat(FORMAT_SHORT_DATE_TIME, Locale.CHINA)); 56 | formaters.put(FORMAT_CHINESE_NO_SECOND, new SimpleDateFormat(FORMAT_CHINESE_NO_SECOND, Locale.CHINA)); 57 | formaters.put(FORMAT_CHINESE, new SimpleDateFormat(FORMAT_CHINESE, Locale.CHINA)); 58 | formaters.put(FORMAT_DATE_TIME, defaultFormater); 59 | formaters.put(FORMAT_NO_SECOND, new SimpleDateFormat(FORMAT_NO_SECOND, Locale.CHINA)); 60 | formaters.put(FORMAT_JAPAN, new SimpleDateFormat(FORMAT_JAPAN, Locale.JAPAN)); 61 | formaters.put(FORMAT_CHINESE_NO_SECOND_1, new SimpleDateFormat(FORMAT_CHINESE_NO_SECOND_1, Locale.CHINA)); 62 | formaters.put(FROMAT_CHAINESE_WEEK_SECOND, new SimpleDateFormat(FROMAT_CHAINESE_WEEK_SECOND, Locale.CHINA)); 63 | formaters.put(FORMAT_YYYY_MM_DD_HH_MM_SS_SSS, new SimpleDateFormat(FORMAT_YYYY_MM_DD_HH_MM_SS_SSS, Locale.CHINA)); 64 | formaters.put(FORMAT_TIMESTAMP, new SimpleDateFormat(FORMAT_TIMESTAMP, Locale.CHINA)); 65 | } 66 | 67 | public static Map getFormaters() { 68 | return formaters; 69 | } 70 | 71 | /** 72 | * 使用给定的 pattern 对日期进格式化为字符串 73 | * 74 | * @param date 待格式化的日期 75 | * @param pattern 格式字符串 76 | * @return 格式化后的日期字符串 77 | */ 78 | public static String format(Date date, String pattern) { 79 | SimpleDateFormat dateFormat; 80 | if (formaters.containsKey(pattern)) { 81 | dateFormat = formaters.get(pattern); 82 | } else { 83 | dateFormat = new SimpleDateFormat(pattern, Locale.CHINA); 84 | } 85 | return dateFormat.format(date); 86 | } 87 | 88 | /** 89 | * 以默认日期格式(yyyy-MM-dd HH:mm:ss)对日期进行格式化 90 | * 91 | * @param date 待格式化的日期 92 | * @return 格式化后的日期字符串 93 | */ 94 | public static String format(Date date) { 95 | return formaters.get(FORMAT_DEFAULT).format(date); 96 | } 97 | 98 | 99 | public static String format(Date date, 100 | String format, 101 | int type) { 102 | if (date != null) { 103 | //--------------------------------- 104 | // 日期不为空时才格式 105 | //--------------------------------- 106 | try { 107 | //--------------------------------- 108 | // 调用SimpleDateFormat来格式化 109 | //--------------------------------- 110 | return new SimpleDateFormat(format).format(date); 111 | } catch (Exception e) { 112 | //--------------------------------- 113 | // 格式化失败后，返回一个空串 114 | //--------------------------------- 115 | return ""; 116 | } 117 | } else { 118 | //--------------------------------- 119 | // 如果传入日期为空，则根据类型返回结果 120 | //--------------------------------- 121 | switch (type) { 122 | case TYPE_HTML_SPACE: // '\002' 123 | return " "; 124 | 125 | case TYPE_DECREASE_SYMBOL: // '\003' 126 | return "-"; 127 | 128 | case TYPE_SPACE: // '\004' 129 | return " "; 130 | 131 | case TYPE_NULL: 132 | return null; 133 | 134 | default: 135 | //--------------------------------- 136 | // 默认为空串 137 | //--------------------------------- 138 | return ""; 139 | } 140 | } 141 | } 142 | 143 | /** 144 | * 将给定字符串解析为对应格式的日期,循环尝试使用预定义的日期格式进行解析 145 | * 146 | * @param str 待解析的日期字符串 147 | * @return 解析成功的日期，解析失败返回null 148 | */ 149 | public static Date parse(String str) { 150 | Date date = null; 151 | for (String _pattern : formaters.keySet()) { 152 | if (_pattern.getBytes().length == str.getBytes().length) { 153 | try { 154 | date = formaters.get(_pattern).parse(str); 155 | //格式化成功则退出 156 | break; 157 | } catch (ParseException e) { 158 | //格式化失败，继续尝试下一个 159 | //e.printStackTrace(); 160 | } 161 | } else if (_pattern.equals(FORMAT_JAPAN)) { 162 | try { 163 | date = formaters.get(_pattern).parse(str); 164 | //格式化成功则退出 165 | break; 166 | } catch (ParseException e) { 167 | } 168 | } 169 | } 170 | return date; 171 | } 172 | } 173 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/restful/JobController.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.restful; 2 | 3 | import com.google.common.base.Joiner; 4 | import com.google.common.collect.Lists; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | import org.springframework.http.MediaType; 9 | import org.springframework.util.Assert; 10 | import org.springframework.util.StringUtils; 11 | import org.springframework.web.bind.annotation.RequestBody; 12 | import org.springframework.web.bind.annotation.RequestMapping; 13 | import org.springframework.web.bind.annotation.RequestMethod; 14 | import org.springframework.web.bind.annotation.RequestParam; 15 | import org.springframework.web.bind.annotation.RestController; 16 | 17 | import java.util.List; 18 | 19 | import javax.annotation.Resource; 20 | 21 | import io.sdmq.queue.JobMsg; 22 | import io.sdmq.queue.core.Queue; 23 | import io.sdmq.queue.redis.JobWrapp; 24 | import io.sdmq.queue.redis.RdbStore; 25 | import io.sdmq.util.IpUtils; 26 | import io.sdmq.util.JobIdGenerator; 27 | import io.sdmq.util.ResponseMessage; 28 | import io.sdmq.util.Status; 29 | 30 | 31 | /** 32 | * 
 33 |  *     提供HTTP方式操作任务
 34 |  *     1、/push 添加任务
 35 |  *     2、/delete 删除任务
 36 |  *     3、/finish 完成任务  暂未实现
 37 |  *
38 | * Created by Xs.Tao on 2017/7/19. 39 | */ 40 | @RestController 41 | @RequestMapping(value = "/") 42 | public class JobController { 43 | 44 | public static final Logger LOGGER = LoggerFactory.getLogger(JobController.class); 45 | private static final int PAGE_SIZE = 500; 46 | @Resource(name = "redisQueueImpl") 47 | private Queue reidsQueue; 48 | @Resource(name = "rdbStore") 49 | private RdbStore store; 50 | 51 | @RequestMapping(value = "/push", method = RequestMethod.POST, headers = "content-type=" + MediaType 52 | .APPLICATION_JSON_VALUE) 53 | public ResponseMessage push(@RequestBody JobWrapp jobMsg) { 54 | try { 55 | Assert.notNull(jobMsg.getTopic(), "参数topic错误"); 56 | if (StringUtils.isEmpty(jobMsg.getId())) { 57 | jobMsg.setId(createId(jobMsg)); 58 | } 59 | reidsQueue.push(jobMsg); 60 | return ResponseMessage.ok(jobMsg.getId()); 61 | } catch (Exception e) { 62 | return ResponseMessage.error(e.getMessage()); 63 | } 64 | } 65 | 66 | private String createId(JobMsg msg) { 67 | List r = Lists.newArrayList(msg.getTopic()); 68 | if (!StringUtils.isEmpty(msg.getBizKey())) { 69 | r.add(msg.getBizKey()); 70 | } 71 | r.add(IpUtils.getIp()); 72 | r.add(JobIdGenerator.getStringId()); 73 | return Joiner.on(":").join(r); 74 | } 75 | 76 | @RequestMapping(value = "/delete", method = RequestMethod.GET) 77 | public ResponseMessage delete(String jobId) { 78 | try { 79 | reidsQueue.delete(jobId); 80 | return ResponseMessage.ok(); 81 | } catch (Exception e) { 82 | return ResponseMessage.error(e.getMessage()); 83 | } 84 | } 85 | 86 | @RequestMapping(value = "/finish", method = RequestMethod.GET) 87 | public ResponseMessage finish(String jobId) { 88 | try { 89 | return ResponseMessage.error("功能暂未开放"); 90 | } catch (Exception e) { 91 | return ResponseMessage.error(e.getMessage()); 92 | } 93 | } 94 | 95 | /** 96 | * 恢复单个job 97 | */ 98 | @RequestMapping(value = "/reStoreJob", method = RequestMethod.GET) 99 | public ResponseMessage reStoreJob(@RequestParam("jobId") String jobId) { 100 | try { 101 | Assert.notNull(jobId, "JobId 不能为空"); 102 | JobMsg job = reidsQueue.getJob(jobId); 103 | if (job == null) { 104 | LOGGER.warn("Job在元数据池中不存在 {} ", jobId); 105 | } 106 | reidsQueue.delete(jobId); 107 | JobMsg msg = store.getJobMyId(jobId); 108 | Assert.notNull(msg, "Job不存在!"); 109 | msg.setStatus(Status.Restore.ordinal()); 110 | LOGGER.info("正在恢复任务{} ", msg.getId()); 111 | reidsQueue.push(msg); 112 | return ResponseMessage.ok(); 113 | } catch (Exception e) { 114 | return ResponseMessage.error(e.getMessage()); 115 | } 116 | } 117 | 118 | /** 119 | * 提供一个方法 假设缓存中间件出现异常 以及数据错乱的情况 提供恢复功能 120 | * 121 | * @param expire 过期的数据是否需要重发 true需要, false不需要 默认为true 122 | */ 123 | @RequestMapping(value = "/reStore", method = RequestMethod.GET) 124 | public ResponseMessage reStore(Boolean expire) { 125 | long startTime = System.currentTimeMillis(); 126 | int count = 0; 127 | try { 128 | if (expire == null) { 129 | expire = true; 130 | } 131 | count = store.getNotFinshDataCount(); 132 | LOGGER.info("正在恢复数据{}", count); 133 | int pageCount = (double) count / PAGE_SIZE == 0 ? count / PAGE_SIZE : count / PAGE_SIZE + 1; 134 | List msgs = Lists.newArrayList(); 135 | for (int i = 1; i <= pageCount; i++) { 136 | msgs.addAll(store.getNotFinshDataList((i - 1) * PAGE_SIZE, PAGE_SIZE)); 137 | } 138 | //1、首先情况数据 139 | this.reidsQueue.clear(); 140 | long time = System.currentTimeMillis(); 141 | int index = 1; 142 | for (JobMsg msg : msgs) { 143 | if (!expire && msg.getCreateTime() + msg.getDelay() < time) { 144 | continue; 145 | } 146 | msg.setStatus(Status.Restore.ordinal()); 147 | LOGGER.info("正在恢复任务{} ({}/{}) ", msg.getId(), index, count); 148 | this.reidsQueue.push(msg); 149 | index++; 150 | } 151 | } catch (Exception e) { 152 | LOGGER.error(e.getMessage(), e); 153 | return ResponseMessage.error(e.getMessage()); 154 | } 155 | long entTime = System.currentTimeMillis(); 156 | 157 | return ResponseMessage.ok(String.format("花费了%s ms，恢复 %s 行数据", entTime - startTime, count)); 158 | } 159 | 160 | 161 | @RequestMapping(value = "/clearAll", method = RequestMethod.GET) 162 | public ResponseMessage clearAll() { 163 | long startTime = System.currentTimeMillis(); 164 | int count = 0; 165 | try { 166 | count = store.getNotFinshDataCount(); 167 | LOGGER.info("正在清空队列数据 行数 {}", count); 168 | int pageCount = (double) count / PAGE_SIZE == 0 ? count / PAGE_SIZE : count / PAGE_SIZE + 1; 169 | List msgs = Lists.newArrayList(); 170 | for (int i = 1; i <= pageCount; i++) { 171 | msgs.addAll(store.getNotFinshDataList((i - 1) * PAGE_SIZE, PAGE_SIZE)); 172 | } 173 | int index = 1; 174 | for (JobMsg msg : msgs) { 175 | LOGGER.info("正在删除任务{} ({}/{}) ", msg.getId(), index, count); 176 | reidsQueue.delete(msg.getId()); 177 | index++; 178 | } 179 | //this.reidsQueue.clear(); 180 | } catch (Exception e) { 181 | LOGGER.error(e.getMessage(), e); 182 | return ResponseMessage.error(e.getMessage()); 183 | } 184 | long entTime = System.currentTimeMillis(); 185 | 186 | return ResponseMessage.ok(String.format("花费了%s ms，删除 %s 行数据", entTime - startTime, count)); 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /src/test/java/io/sdmq/transactional/RedisTemplateTransactional.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.transactional; 2 | 3 | import org.junit.Test; 4 | import org.junit.runner.RunWith; 5 | import org.springframework.beans.factory.annotation.Autowired; 6 | import org.springframework.boot.test.SpringApplicationConfiguration; 7 | import org.springframework.dao.DataAccessException; 8 | import org.springframework.data.redis.core.RedisOperations; 9 | import org.springframework.data.redis.core.RedisTemplate; 10 | import org.springframework.data.redis.core.SessionCallback; 11 | import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; 12 | import org.springframework.test.context.web.WebAppConfiguration; 13 | 14 | import java.util.List; 15 | import java.util.Objects; 16 | import java.util.concurrent.ExecutorService; 17 | import java.util.concurrent.Executors; 18 | 19 | import io.sdmq.TestDelayQueue; 20 | import io.sdmq.queue.redis.support.RedisSupport; 21 | import redis.clients.jedis.Jedis; 22 | import redis.clients.jedis.JedisPool; 23 | import redis.clients.jedis.JedisPoolConfig; 24 | import redis.clients.jedis.Transaction; 25 | 26 | /** 27 | * Created by Xs.Tao on 2017/8/1. 28 | */ 29 | @RunWith(SpringJUnit4ClassRunner.class) 30 | @SpringApplicationConfiguration(classes = TestDelayQueue.class) 31 | @WebAppConfiguration 32 | public class RedisTemplateTransactional { 33 | 34 | @Autowired 35 | private RedisTemplate redisTemplate; 36 | 37 | @Autowired 38 | private RedisSupport support; 39 | 40 | @Test 41 | public void redisSupportTest() throws InterruptedException { 42 | ExecutorService executorService = Executors.newFixedThreadPool(10); 43 | for (int i = 0; i < 500; i++) { 44 | final int tmp = i; 45 | executorService.execute(new Runnable() { 46 | @Override 47 | public void run() { 48 | final Jedis jedis = support.getJedis(); 49 | new callback() { 50 | @Override 51 | public void exe(Jedis jedis) { 52 | try { 53 | Transaction transaction = jedis.multi(); 54 | transaction.set("abc" + tmp, Objects.toString(tmp)); 55 | // jedis.set("avd"+tmp,Objects.toString(tmp)); 56 | // // transaction.set("abc"+tmp, Objects.toString(tmp)); 57 | // // transaction.set("avd"+tmp,Objects.toString(tmp)); 58 | saveDb(tmp); 59 | jedis.set("abc" + tmp, Objects.toString(tmp + "_2")); 60 | // transaction.set("abc"+tmp,Objects.toString(tmp)); 61 | List lst = transaction.exec(); 62 | System.out.println(tmp); 63 | 64 | 65 | } catch (Exception e) { 66 | e.printStackTrace(); 67 | } finally { 68 | jedis.close(); 69 | } 70 | } 71 | }.exe(jedis); 72 | 73 | } 74 | }); 75 | } 76 | Thread.sleep(Integer.MAX_VALUE); 77 | } 78 | 79 | @Test 80 | //@Transactional 81 | public void test() throws Exception { 82 | System.out.println(redisTemplate); 83 | redisTemplate.watch("test"); 84 | redisTemplate.multi(); 85 | 86 | redisTemplate.opsForValue().set("test", "123"); 87 | 88 | redisTemplate.opsForValue().set("test1", "456"); 89 | 90 | redisTemplate.exec(); 91 | 92 | } 93 | 94 | @Test 95 | public void test3() throws InterruptedException { 96 | JedisPoolConfig config = new JedisPoolConfig(); 97 | config.setMaxIdle(300); 98 | config.setMaxTotal(600); 99 | config.setMaxWaitMillis(1000 * 3); 100 | config.setMinIdle(200); 101 | config.setTestOnBorrow(true); 102 | final JedisPool pool = new JedisPool(config, "219.239.88.69", 6379, 300, "123", 7); 103 | 104 | // for(int i=0;i<200;i++){ 105 | // Jedis jedis=pool.getResource(); 106 | // jedis.set("aa",Objects.toString(i)); 107 | // System.out.println(jedis); 108 | // } 109 | ExecutorService executorService = Executors.newFixedThreadPool(10); 110 | for (int i = 0; i < 500; i++) { 111 | final int tmp = i; 112 | executorService.execute(new Runnable() { 113 | @Override 114 | public void run() { 115 | Jedis jedis = pool.getResource(); 116 | new callback() { 117 | @Override 118 | public void exe(Jedis jedis) { 119 | try { 120 | Transaction transaction = jedis.multi(); 121 | transaction.set("abc" + tmp, Objects.toString(tmp)); 122 | // jedis.set("avd"+tmp,Objects.toString(tmp)); 123 | // // transaction.set("abc"+tmp, Objects.toString(tmp)); 124 | // // transaction.set("avd"+tmp,Objects.toString(tmp)); 125 | saveDb(tmp); 126 | // jedis.set("abc"+tmp,Objects.toString(tmp)); 127 | // transaction.set("abc"+tmp,Objects.toString(tmp)); 128 | List lst = transaction.exec(); 129 | System.out.println(tmp); 130 | 131 | } catch (Exception e) { 132 | System.out.println(e.getMessage()); 133 | } finally { 134 | jedis.close(); 135 | } 136 | } 137 | }.exe(jedis); 138 | 139 | } 140 | }); 141 | 142 | } 143 | ///pool.close(); 144 | Thread.sleep(Integer.MAX_VALUE); 145 | } 146 | 147 | private void saveDb(int tmp) { 148 | if (tmp % 3 == 0) { 149 | throw new RuntimeException("失败父超时" + tmp); 150 | } 151 | 152 | } 153 | 154 | @Test 155 | public void test2() { 156 | System.out.println(11); 157 | List results = (List) redisTemplate.execute(new SessionCallback>() { 158 | @SuppressWarnings({"rawtypes", "unchecked"}) 159 | public List execute(RedisOperations operations) { 160 | operations.multi(); 161 | //new Jedis().multi(); 162 | operations.opsForValue().set("abb", "233332"); 163 | if (true) { 164 | throw new eeee("xx"); 165 | } 166 | operations.opsForValue().set("abb", "2"); 167 | // operations.opsForValue().set("11", "22"); 168 | // operations.opsForValue().get("11"); 169 | // operations.opsForList().leftPush("aaa", 1); 170 | // operations.opsForList().range("aaa", 0l, 1l); 171 | // operations.opsForSet().add("bbbb", 12); 172 | // operations.opsForSet().members("bbbb"); 173 | return operations.exec(); 174 | } 175 | }); 176 | for (Object o : results) { 177 | System.out.println(o); 178 | } 179 | } 180 | 181 | interface callback { 182 | 183 | void exe(Jedis jedis); 184 | } 185 | 186 | class eeee extends DataAccessException { 187 | 188 | public eeee(String msg) { 189 | super(msg); 190 | } 191 | } 192 | 193 | } 194 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/util/ResponseMessage.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.util; 2 | 3 | import com.alibaba.fastjson.JSON; 4 | 5 | import java.io.Serializable; 6 | import java.lang.reflect.Field; 7 | import java.util.Arrays; 8 | import java.util.Collection; 9 | import java.util.HashMap; 10 | import java.util.HashSet; 11 | import java.util.Map; 12 | import java.util.Set; 13 | 14 | /** 15 | * response message transform 16 | * Created by Xs.Tao on 2017/6/6. 17 | */ 18 | public class ResponseMessage implements Serializable { 19 | 20 | private static final long serialVersionUID = 262611675172299409L; 21 | /** 22 | * 是否成功 23 | */ 24 | private boolean success; 25 | 26 | /** 27 | * 反馈数据 28 | */ 29 | private Object data; 30 | 31 | /** 32 | * 反馈信息 33 | */ 34 | private String msg; 35 | 36 | /** 37 | * 响应码 38 | */ 39 | private int code; 40 | 41 | 42 | /** 43 | * 过滤字段：指定需要序列化的字段 44 | */ 45 | private transient Map, Set> includes; 46 | 47 | /** 48 | * 过滤字段：指定不需要序列化的字段 49 | */ 50 | private transient Map, Set> excludes; 51 | 52 | private transient boolean onlyData; 53 | 54 | private transient String callback; 55 | 56 | protected ResponseMessage(String message) { 57 | this.code = 500; 58 | this.msg = message; 59 | this.success = false; 60 | } 61 | 62 | protected ResponseMessage(boolean success, Object data) { 63 | this.code = success ? 200 : 500; 64 | this.data = data; 65 | this.success = success; 66 | } 67 | 68 | protected ResponseMessage(boolean success, Object data, int code) { 69 | this(success, data); 70 | this.code = code; 71 | } 72 | 73 | public static ResponseMessage fromJson(String json) { 74 | return JSON.parseObject(json, ResponseMessage.class); 75 | } 76 | 77 | public static ResponseMessage ok() { 78 | return ok(null); 79 | } 80 | 81 | public static ResponseMessage empty() { 82 | return new ResponseMessage(""); 83 | } 84 | 85 | public static ResponseMessage ok(Object data) { 86 | return new ResponseMessage(true, data); 87 | } 88 | 89 | public static ResponseMessage error(String message) { 90 | return new ResponseMessage(message); 91 | } 92 | 93 | public static ResponseMessage error(String message, int code) { 94 | return new ResponseMessage(message).setCode(code); 95 | } 96 | 97 | public Map toMap() { 98 | Map map = new HashMap<>(); 99 | map.put("success", this.success); 100 | if (data != null) 101 | map.put("data", this.getData()); 102 | if (msg != null) 103 | map.put("message", this.getMessage()); 104 | map.put("code", this.getCode()); 105 | return map; 106 | } 107 | 108 | public ResponseMessage include(Class type, String... fields) { 109 | return include(type, Arrays.asList(fields)); 110 | } 111 | 112 | public ResponseMessage include(Class type, Collection fields) { 113 | if (includes == null) 114 | includes = new HashMap<>(); 115 | if (fields == null || fields.isEmpty()) 116 | return this; 117 | for (String field : fields) { 118 | if (field.contains(".")) { 119 | String tmp[] = field.split("[.]", 2); 120 | try { 121 | Field field1 = type.getDeclaredField(tmp[0]); 122 | if (field1 != null) { 123 | include(field1.getType(), tmp[1]); 124 | } 125 | } catch (Throwable e) { 126 | } 127 | } else { 128 | getStringListFormMap(includes, type).add(field); 129 | } 130 | } 131 | 132 | 133 | return this; 134 | } 135 | 136 | public ResponseMessage exclude(Class type, Collection fields) { 137 | if (excludes == null) 138 | excludes = new HashMap<>(); 139 | if (fields == null || fields.isEmpty()) 140 | return this; 141 | for (String field : fields) { 142 | if (field.contains(".")) { 143 | String tmp[] = field.split("[.]", 2); 144 | try { 145 | Field field1 = type.getDeclaredField(tmp[0]); 146 | if (field1 != null) { 147 | exclude(field1.getType(), tmp[1]); 148 | } 149 | } catch (Throwable e) { 150 | } 151 | } else { 152 | getStringListFormMap(excludes, type).add(field); 153 | } 154 | } 155 | return this; 156 | } 157 | 158 | public ResponseMessage exclude(Collection fields) { 159 | if (excludes == null) 160 | excludes = new HashMap<>(); 161 | if (fields == null || fields.isEmpty()) 162 | return this; 163 | Class type; 164 | if (data != null) 165 | type = data.getClass(); 166 | else 167 | return this; 168 | exclude(type, fields); 169 | return this; 170 | } 171 | 172 | public ResponseMessage include(Collection fields) { 173 | if (includes == null) 174 | includes = new HashMap<>(); 175 | if (fields == null || fields.isEmpty()) 176 | return this; 177 | Class type; 178 | if (data != null) 179 | type = data.getClass(); 180 | else 181 | return this; 182 | include(type, fields); 183 | return this; 184 | } 185 | 186 | public ResponseMessage exclude(Class type, String... fields) { 187 | return exclude(type, Arrays.asList(fields)); 188 | } 189 | 190 | public ResponseMessage exclude(String... fields) { 191 | return exclude(Arrays.asList(fields)); 192 | } 193 | 194 | public ResponseMessage include(String... fields) { 195 | return include(Arrays.asList(fields)); 196 | } 197 | 198 | protected Set getStringListFormMap(Map, Set> map, Class type) { 199 | Set list = map.get(type); 200 | if (list == null) { 201 | list = new HashSet<>(); 202 | map.put(type, list); 203 | } 204 | return list; 205 | } 206 | 207 | public boolean isSuccess() { 208 | return success; 209 | } 210 | 211 | public void setSuccess(boolean success) { 212 | this.success = success; 213 | } 214 | 215 | public Object getData() { 216 | return data; 217 | } 218 | 219 | public ResponseMessage setData(Object data) { 220 | this.data = data; 221 | return this; 222 | } 223 | 224 | @Override 225 | public String toString() { 226 | return JSON.toJSONString(this); 227 | } 228 | 229 | public int getCode() { 230 | return code; 231 | } 232 | 233 | public ResponseMessage setCode(int code) { 234 | this.code = code; 235 | return this; 236 | } 237 | 238 | public Map, Set> getExcludes() { 239 | return excludes; 240 | } 241 | 242 | public Map, Set> getIncludes() { 243 | return includes; 244 | } 245 | 246 | public ResponseMessage onlyData() { 247 | setOnlyData(true); 248 | return this; 249 | } 250 | 251 | public boolean isOnlyData() { 252 | return onlyData; 253 | } 254 | 255 | public void setOnlyData(boolean onlyData) { 256 | this.onlyData = onlyData; 257 | } 258 | 259 | public ResponseMessage callback(String callback) { 260 | this.callback = callback; 261 | return this; 262 | } 263 | 264 | public String getCallback() { 265 | return callback; 266 | } 267 | 268 | public String getMessage() { 269 | return msg; 270 | } 271 | 272 | public void setMessage(String message) { 273 | this.msg = message; 274 | } 275 | 276 | } -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/support/RedisSupport.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis.support; 2 | 3 | import com.google.common.collect.Lists; 4 | 5 | import org.springframework.dao.DataAccessException; 6 | import org.springframework.data.redis.connection.RedisConnection; 7 | import org.springframework.data.redis.core.HashOperations; 8 | import org.springframework.data.redis.core.ListOperations; 9 | import org.springframework.data.redis.core.RedisCallback; 10 | import org.springframework.data.redis.core.StringRedisTemplate; 11 | import org.springframework.data.redis.core.ValueOperations; 12 | import org.springframework.data.redis.core.ZSetOperations; 13 | 14 | import java.util.Collection; 15 | import java.util.List; 16 | import java.util.Map; 17 | import java.util.Set; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import redis.clients.jedis.Jedis; 21 | 22 | 23 | public class RedisSupport { 24 | 25 | private StringRedisTemplate template; 26 | 27 | public StringRedisTemplate getTemplate() { 28 | return template; 29 | } 30 | 31 | public void setTemplate(StringRedisTemplate template) { 32 | this.template = template; 33 | } 34 | 35 | public void deleteKey(String... key) { 36 | this.template.delete(Lists.newArrayList(key)); 37 | } 38 | 39 | public void set(String k, String v) { 40 | ValueOperations ops = this.template.opsForValue(); 41 | ops.set(k, v); 42 | } 43 | 44 | public void set(String k, String v, long var3, TimeUnit var5) { 45 | ValueOperations ops = this.template.opsForValue(); 46 | ops.set(k, v, var3, var5); 47 | } 48 | 49 | public String get(String key) { 50 | ValueOperations ops = this.template.opsForValue(); 51 | return ops.get(key); 52 | } 53 | 54 | public void leftPush(String key, String item) { 55 | ListOperations listOperations = template.opsForList(); 56 | listOperations.leftPush(key, item); 57 | } 58 | 59 | public List lrange(String key, int start, int size) { 60 | ListOperations listOperations = template.opsForList(); 61 | return listOperations.range(key, start, size); 62 | } 63 | 64 | public boolean lrem(String key, String value) { 65 | try { 66 | ListOperations listOperations = template.opsForList(); 67 | listOperations.remove(key, 1, value); 68 | } catch (Exception e) { 69 | return false; 70 | } 71 | return true; 72 | } 73 | 74 | public String leftPop(String key) { 75 | ListOperations listOperations = template.opsForList(); 76 | return listOperations.leftPop(key); 77 | } 78 | 79 | public String rightPop(String key) { 80 | ListOperations listOperations = template.opsForList(); 81 | return listOperations.rightPop(key); 82 | } 83 | 84 | public void rightPush(String key, String item) { 85 | ListOperations listOperations = template.opsForList(); 86 | listOperations.rightPush(key, item); 87 | } 88 | 89 | public void hashPutAll(String key, Map map) { 90 | HashOperations hashOperations = template.opsForHash(); 91 | hashOperations.putAll(key, map); 92 | } 93 | 94 | public void hashPut(String key, String hashKey, String hashValue) { 95 | HashOperations hashOperations = template.opsForHash(); 96 | hashOperations.put(key, hashKey, hashValue); 97 | } 98 | 99 | public String getHashKey(String key, String mapKey) { 100 | HashOperations hashOperations = template.opsForHash(); 101 | return hashOperations.get(key, mapKey); 102 | } 103 | 104 | public Set getHashKeys(String key) { 105 | HashOperations hashOperations = template.opsForHash(); 106 | return hashOperations.keys(key); 107 | } 108 | 109 | /** 110 | * 删除hash中的指定key 111 | */ 112 | public void deleteHashKeys(String key, Object... keys) { 113 | HashOperations hashOperations = template.opsForHash(); 114 | hashOperations.delete(key, keys); 115 | } 116 | 117 | public List getHashKeys(String key, Collection keys) { 118 | HashOperations hashOperations = template.opsForHash(); 119 | return hashOperations.multiGet(key, keys); 120 | } 121 | 122 | public List hashgetAll(String key) { 123 | HashOperations hashOperations = template.opsForHash(); 124 | List lst = hashOperations.values(key); 125 | return lst; 126 | } //zset. 127 | 128 | public Set zrangeByScore(String key, double min, double max, int offset, int count) { 129 | ZSetOperations zset = template.opsForZSet(); 130 | 131 | Set datas = zset.rangeByScore(key, min, max, offset, count); 132 | return datas; 133 | } 134 | 135 | public Set> zrangeByScoreWithScores(String key, double min, double max, int offset, 136 | int count) { 137 | ZSetOperations zset = template.opsForZSet(); 138 | Set> set = zset.rangeByScoreWithScores(key, min, max, offset, count); 139 | return set; 140 | } 141 | 142 | public boolean zadd(String key, String itemKey, double score) { 143 | ZSetOperations zset = template.opsForZSet(); 144 | return zset.add(key, itemKey, score); 145 | } 146 | 147 | public Long zrem(String key, String itemKey) { 148 | ZSetOperations zset = template.opsForZSet(); 149 | return zset.remove(key, itemKey); 150 | } 151 | 152 | public Boolean setNx(final String key, final String value) { 153 | final org.springframework.data.redis.serializer.RedisSerializer redisSerializer = template.getKeySerializer(); 154 | final org.springframework.data.redis.serializer.RedisSerializer redisValueSerializer = template.getValueSerializer(); 155 | return template.execute(new RedisCallback() { 156 | @Override 157 | public Boolean doInRedis(RedisConnection connection) throws DataAccessException { 158 | return connection.setNX(redisSerializer.serialize(key), redisValueSerializer.serialize(value)); 159 | } 160 | }); 161 | } 162 | 163 | public Boolean pExpire(final String key, final long timeout) { 164 | final org.springframework.data.redis.serializer.RedisSerializer redisSerializer = template.getKeySerializer(); 165 | return template.execute(new RedisCallback() { 166 | @Override 167 | public Boolean doInRedis(RedisConnection connection) throws DataAccessException { 168 | return connection.pExpire(redisSerializer.serialize(key), timeout); 169 | } 170 | }); 171 | } 172 | 173 | /** 174 | * command GETSET key value 175 | */ 176 | public String getSet(final String key, final String value) { 177 | final org.springframework.data.redis.serializer.RedisSerializer redisSerializer = template.getKeySerializer(); 178 | final org.springframework.data.redis.serializer.RedisSerializer redisValueSerializer = template.getValueSerializer(); 179 | return template.execute(new RedisCallback() { 180 | @Override 181 | public String doInRedis(RedisConnection connection) throws DataAccessException { 182 | byte[] b = connection.getSet(redisSerializer.serialize(key), redisValueSerializer.serialize(value)); 183 | return template.getStringSerializer().deserialize(b); 184 | } 185 | }); 186 | } 187 | 188 | public Jedis getJedis() { 189 | return (Jedis) template.getConnectionFactory().getConnection().getNativeConnection(); 190 | } 191 | } -------------------------------------------------------------------------------- /src/main/java/io/sdmq/common/autoconfigure/DruidConfig.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.common.autoconfigure; 2 | 3 | import com.alibaba.druid.pool.DruidDataSource; 4 | 5 | import org.springframework.boot.context.properties.ConfigurationProperties; 6 | import org.springframework.context.annotation.Configuration; 7 | import org.springframework.core.Ordered; 8 | import org.springframework.core.annotation.Order; 9 | 10 | import javax.sql.DataSource; 11 | 12 | /** 13 | * Created by Xs.Tao on 2017/7/19. 14 | */ 15 | @SuppressWarnings("Duplicates") 16 | @Configuration 17 | @ConfigurationProperties(prefix = "spring.datasource") 18 | @Order(Ordered.HIGHEST_PRECEDENCE) 19 | public class DruidConfig { 20 | 21 | private String url; 22 | 23 | private String username; 24 | 25 | private String password; 26 | 27 | private String driverClassName; 28 | 29 | private int initialSize; 30 | 31 | private int maxActive; 32 | 33 | private int minIdle; 34 | 35 | private int maxWait; 36 | 37 | private long timeBetweenEvictionRunsMillis; 38 | 39 | private long minEvictableIdleTimeMillis; 40 | 41 | private String validationQuery; 42 | 43 | private boolean testWhileIdle; 44 | 45 | private boolean testOnBorrow; 46 | 47 | private boolean testOnReturn; 48 | 49 | private boolean poolPreparedStatements; 50 | 51 | private int maxPoolPreparedStatementPerConnectionSize; 52 | 53 | private String filters; 54 | 55 | private DruidDataSource mydataSource; 56 | 57 | public DruidConfig() { 58 | } 59 | 60 | public DruidConfig(String url, String username, String password, 61 | String driverClassName, int initialSize, 62 | int maxActive, int minIdle, int maxWait, 63 | long timeBetweenEvictionRunsMillis, 64 | long minEvictableIdleTimeMillis, String validationQuery, 65 | boolean testWhileIdle, boolean testOnBorrow, boolean testOnReturn, 66 | boolean poolPreparedStatements, int maxPoolPreparedStatementPerConnectionSize, 67 | String filters) { 68 | this.url = url; 69 | this.username = username; 70 | this.password = password; 71 | this.driverClassName = driverClassName; 72 | this.initialSize = initialSize; 73 | this.maxActive = maxActive; 74 | this.minIdle = minIdle; 75 | this.maxWait = maxWait; 76 | this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; 77 | this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; 78 | this.validationQuery = validationQuery; 79 | this.testWhileIdle = testWhileIdle; 80 | this.testOnBorrow = testOnBorrow; 81 | this.testOnReturn = testOnReturn; 82 | this.poolPreparedStatements = poolPreparedStatements; 83 | this.maxPoolPreparedStatementPerConnectionSize = maxPoolPreparedStatementPerConnectionSize; 84 | this.filters = filters; 85 | } 86 | 87 | public DataSource newInstanceDruidDataSource() throws Exception { 88 | if (mydataSource != null) { 89 | return mydataSource; 90 | } 91 | DruidDataSource druidDataSource = new DruidDataSource(); 92 | druidDataSource.setUrl(this.url); 93 | druidDataSource.setUsername(this.username); 94 | druidDataSource.setPassword(this.password); 95 | druidDataSource.setDriverClassName(this.driverClassName); 96 | druidDataSource.setInitialSize(this.initialSize); 97 | druidDataSource.setMaxActive(this.maxActive); 98 | druidDataSource.setMinIdle(this.minIdle); 99 | druidDataSource.setMaxWait(this.maxWait); 100 | druidDataSource.setTimeBetweenEvictionRunsMillis(this.timeBetweenEvictionRunsMillis); 101 | druidDataSource.setMinEvictableIdleTimeMillis(this.minEvictableIdleTimeMillis); 102 | druidDataSource.setValidationQuery(this.validationQuery); 103 | druidDataSource.setTestWhileIdle(this.testWhileIdle); 104 | druidDataSource.setTestOnBorrow(this.testOnBorrow); 105 | druidDataSource.setTestOnReturn(this.testOnReturn); 106 | druidDataSource.setPoolPreparedStatements(this.poolPreparedStatements); 107 | druidDataSource.setMaxPoolPreparedStatementPerConnectionSize(this.maxPoolPreparedStatementPerConnectionSize); 108 | druidDataSource.setFilters(this.filters); 109 | 110 | try { 111 | if (null != druidDataSource) { 112 | druidDataSource.setFilters("wall,stat"); 113 | druidDataSource.setUseGlobalDataSourceStat(true); 114 | druidDataSource.init(); 115 | } 116 | } catch (Exception e) { 117 | throw new RuntimeException("load datasource error, dbProperties is :", e); 118 | } 119 | synchronized (this) { 120 | mydataSource = druidDataSource; 121 | } 122 | druidDataSource.init(); 123 | return druidDataSource; 124 | } 125 | 126 | public String getUrl() { 127 | return url; 128 | } 129 | 130 | public void setUrl(String url) { 131 | this.url = url; 132 | } 133 | 134 | public String getUsername() { 135 | return username; 136 | } 137 | 138 | public void setUsername(String username) { 139 | this.username = username; 140 | } 141 | 142 | public String getPassword() { 143 | return password; 144 | } 145 | 146 | public void setPassword(String password) { 147 | this.password = password; 148 | } 149 | 150 | public String getDriverClassName() { 151 | return driverClassName; 152 | } 153 | 154 | public void setDriverClassName(String driverClassName) { 155 | this.driverClassName = driverClassName; 156 | } 157 | 158 | public int getInitialSize() { 159 | return initialSize; 160 | } 161 | 162 | public void setInitialSize(int initialSize) { 163 | this.initialSize = initialSize; 164 | } 165 | 166 | public int getMaxActive() { 167 | return maxActive; 168 | } 169 | 170 | public void setMaxActive(int maxActive) { 171 | this.maxActive = maxActive; 172 | } 173 | 174 | public int getMinIdle() { 175 | return minIdle; 176 | } 177 | 178 | public void setMinIdle(int minIdle) { 179 | this.minIdle = minIdle; 180 | } 181 | 182 | public int getMaxWait() { 183 | return maxWait; 184 | } 185 | 186 | public void setMaxWait(int maxWait) { 187 | this.maxWait = maxWait; 188 | } 189 | 190 | public long getTimeBetweenEvictionRunsMillis() { 191 | return timeBetweenEvictionRunsMillis; 192 | } 193 | 194 | public void setTimeBetweenEvictionRunsMillis(long timeBetweenEvictionRunsMillis) { 195 | this.timeBetweenEvictionRunsMillis = timeBetweenEvictionRunsMillis; 196 | } 197 | 198 | public long getMinEvictableIdleTimeMillis() { 199 | return minEvictableIdleTimeMillis; 200 | } 201 | 202 | public void setMinEvictableIdleTimeMillis(long minEvictableIdleTimeMillis) { 203 | this.minEvictableIdleTimeMillis = minEvictableIdleTimeMillis; 204 | } 205 | 206 | public String getValidationQuery() { 207 | return validationQuery; 208 | } 209 | 210 | public void setValidationQuery(String validationQuery) { 211 | this.validationQuery = validationQuery; 212 | } 213 | 214 | public boolean isTestWhileIdle() { 215 | return testWhileIdle; 216 | } 217 | 218 | public void setTestWhileIdle(boolean testWhileIdle) { 219 | this.testWhileIdle = testWhileIdle; 220 | } 221 | 222 | public boolean isTestOnBorrow() { 223 | return testOnBorrow; 224 | } 225 | 226 | public void setTestOnBorrow(boolean testOnBorrow) { 227 | this.testOnBorrow = testOnBorrow; 228 | } 229 | 230 | public boolean isTestOnReturn() { 231 | return testOnReturn; 232 | } 233 | 234 | public void setTestOnReturn(boolean testOnReturn) { 235 | this.testOnReturn = testOnReturn; 236 | } 237 | 238 | public boolean isPoolPreparedStatements() { 239 | return poolPreparedStatements; 240 | } 241 | 242 | public void setPoolPreparedStatements(boolean poolPreparedStatements) { 243 | this.poolPreparedStatements = poolPreparedStatements; 244 | } 245 | 246 | public int getMaxPoolPreparedStatementPerConnectionSize() { 247 | return maxPoolPreparedStatementPerConnectionSize; 248 | } 249 | 250 | public void setMaxPoolPreparedStatementPerConnectionSize(int maxPoolPreparedStatementPerConnectionSize) { 251 | this.maxPoolPreparedStatementPerConnectionSize = maxPoolPreparedStatementPerConnectionSize; 252 | } 253 | 254 | public String getFilters() { 255 | return filters; 256 | } 257 | 258 | public void setFilters(String filters) { 259 | this.filters = filters; 260 | } 261 | } 262 | -------------------------------------------------------------------------------- /src/main/java/io/sdmq/queue/redis/RdbStore.java: -------------------------------------------------------------------------------- 1 | package io.sdmq.queue.redis; 2 | 3 | import com.google.common.collect.Lists; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.sql.Connection; 9 | import java.sql.PreparedStatement; 10 | import java.sql.ResultSet; 11 | import java.sql.SQLException; 12 | import java.util.Date; 13 | import java.util.List; 14 | 15 | import javax.sql.DataSource; 16 | 17 | import io.sdmq.queue.JobMsg; 18 | import io.sdmq.util.IpUtils; 19 | 20 | /** 21 | * Created by Xs.Tao on 2017/7/19. 22 | */ 23 | public class RdbStore { 24 | 25 | public static final Logger LOGGER = LoggerFactory.getLogger(RdbStore.class); 26 | private static final String TABLE_NAME = "t_delay_queue_job"; 27 | private static final String LOG_TABLE_NAME = "t_delay_queue_job_log"; 28 | 29 | private DataSource dataSource; 30 | 31 | public RdbStore(final DataSource dataSource) { 32 | this.dataSource = dataSource; 33 | } 34 | 35 | public boolean insertJob(JobMsg jobMsg) { 36 | boolean result = false; 37 | String sql = "INSERT INTO `" + TABLE_NAME + "` (`id`, `topic`, `subtopic`, `delay`, `create_time`, `body`, " + 38 | "`status`, `ttl`, `update_time`,`bizKey`,`extend_data`) " 39 | + "VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?,?,?);"; 40 | try ( 41 | Connection conn = dataSource.getConnection(); 42 | PreparedStatement preparedStatement = conn.prepareStatement(sql)) { 43 | preparedStatement.setString(1, jobMsg.getId()); 44 | preparedStatement.setString(2, jobMsg.getTopic()); 45 | preparedStatement.setString(3, jobMsg.getSubtopic()); 46 | preparedStatement.setLong(4, jobMsg.getDelay()); 47 | preparedStatement.setLong(5, jobMsg.getCreateTime()); 48 | preparedStatement.setString(6, jobMsg.getBody()); 49 | preparedStatement.setInt(7, jobMsg.getStatus()); 50 | preparedStatement.setLong(8, jobMsg.getTtl()); 51 | preparedStatement.setObject(9, new Date()); 52 | preparedStatement.setString(10, jobMsg.getBizKey()); 53 | preparedStatement.setString(11, jobMsg.getExtendData()); 54 | preparedStatement.execute(); 55 | insertLogJob(jobMsg, conn); 56 | result = true; 57 | } catch (final SQLException ex) { 58 | LOGGER.error("INSERT数据发生错误", ex); 59 | } 60 | return result; 61 | } 62 | 63 | public boolean updateJobsStatus(JobMsg jobMsg) { 64 | boolean result = false; 65 | String sql = "UPDATE `" + TABLE_NAME + "` SET `status` =? , `update_time` = ? WHERE `id`=? "; 66 | try ( 67 | Connection conn = dataSource.getConnection(); 68 | PreparedStatement preparedStatement = conn.prepareStatement(sql)) { 69 | preparedStatement.setInt(1, jobMsg.getStatus()); 70 | preparedStatement.setObject(2, new Date()); 71 | preparedStatement.setString(3, jobMsg.getId()); 72 | preparedStatement.execute(); 73 | insertLogJob(jobMsg, conn); 74 | result = true; 75 | } catch (final SQLException ex) { 76 | LOGGER.error("UPDATE数据发生错误", ex); 77 | } finally { 78 | 79 | } 80 | return result; 81 | } 82 | 83 | /** 84 | * 添加运行规矩日志 注意 这个方法不会自动关闭数据库链接 85 | */ 86 | public boolean insertLogJob(JobMsg jobMsg, Connection conn) { 87 | boolean result = false; 88 | String sql = "INSERT INTO `" + LOG_TABLE_NAME + "` (`id`, `status`,`thread`,`update_time`,`host`) " 89 | + "VALUES (?, ?, ?, ?,?);"; 90 | try { 91 | PreparedStatement preparedStatement = conn.prepareStatement(sql); 92 | preparedStatement.setString(1, jobMsg.getId()); 93 | preparedStatement.setInt(2, jobMsg.getStatus()); 94 | preparedStatement.setString(3, Thread.currentThread().getName()); 95 | preparedStatement.setObject(4, new Date()); 96 | preparedStatement.setString(5, IpUtils.getHostAndIp()); 97 | preparedStatement.execute(); 98 | result = true; 99 | } catch (final SQLException ex) { 100 | LOGGER.error("INSERT日志数据发生错误", ex); 101 | } 102 | return result; 103 | } 104 | 105 | /** 106 | * 获取待恢复的数据行数 这里是获取 不是已完成 并且不是已删除的数据 107 | */ 108 | public int getNotFinshDataCount() { 109 | String countSql = "SELECT count(1) FROM `" + TABLE_NAME + "` where `status` <> 3 and `status` <> 4 "; 110 | int count = 0; 111 | try ( 112 | Connection conn = dataSource.getConnection(); 113 | PreparedStatement preparedStatement = conn.prepareStatement(countSql)) { 114 | ResultSet rs = preparedStatement.executeQuery(); 115 | if (rs != null && rs.next()) { 116 | count = rs.getInt(1); 117 | } 118 | } catch (final SQLException ex) { 119 | LOGGER.error("COUNT数据发生错误", ex); 120 | } 121 | return count; 122 | } 123 | 124 | public JobMsg getJobMyId(String jobId) { 125 | String sql = "SELECT `id`,`topic`,`subtopic`,`delay`,`create_time`,`body`,`status`,`ttl`,`bizkey`,`extend_data` FROM `" + 126 | TABLE_NAME + "` " + 127 | "where `id` = ?"; 128 | List msgList = Lists.newArrayList(); 129 | try ( 130 | Connection conn = dataSource.getConnection(); 131 | PreparedStatement preparedStatement = conn.prepareStatement(sql)) { 132 | preparedStatement.setString(1, jobId); 133 | ResultSet rs = preparedStatement.executeQuery(); 134 | while (rs != null && rs.next()) { 135 | String id = rs.getString("id"); 136 | String bizKey = rs.getString("bizkey"); 137 | String topic = rs.getString("topic"); 138 | String subtopic = rs.getString("subtopic"); 139 | long delay = rs.getLong("delay"); 140 | long createTime = rs.getLong("create_time"); 141 | String body = rs.getString("body"); 142 | int status = rs.getInt("status"); 143 | long ttl = rs.getLong("ttl"); 144 | String extend_data = rs.getString("extend_data"); 145 | JobMsg msg = new JobMsg(); 146 | msg.setStatus(status); 147 | msg.setTtl(ttl); 148 | msg.setBizKey(bizKey); 149 | msg.setBody(body); 150 | msg.setTopic(topic); 151 | msg.setCreateTime(createTime); 152 | msg.setSubtopic(subtopic); 153 | msg.setDelay(delay); 154 | msg.setId(id); 155 | msg.setExtendData(extend_data); 156 | return msg; 157 | } 158 | } catch (final SQLException ex) { 159 | LOGGER.error("getJobMyId数据发生错误", ex); 160 | } 161 | return null; 162 | } 163 | 164 | public List getNotFinshDataList(int start, int size) { 165 | String sql = "SELECT `id`,`topic`,`subtopic`,`delay`,`create_time`,`body`,`status`,`ttl`,`bizkey`,`extend_data` FROM `" + 166 | TABLE_NAME + "` " + 167 | "where `status` <> 3 and `status` <> 4 order by create_time ,delay limit ?, ?"; 168 | List msgList = Lists.newArrayList(); 169 | try ( 170 | Connection conn = dataSource.getConnection(); 171 | PreparedStatement preparedStatement = conn.prepareStatement(sql)) { 172 | preparedStatement.setInt(1, start); 173 | preparedStatement.setInt(2, size); 174 | ResultSet rs = preparedStatement.executeQuery(); 175 | while (rs != null && rs.next()) { 176 | String id = rs.getString("id"); 177 | String bizKey = rs.getString("bizkey"); 178 | String topic = rs.getString("topic"); 179 | String subtopic = rs.getString("subtopic"); 180 | long delay = rs.getLong("delay"); 181 | long createTime = rs.getLong("create_time"); 182 | String body = rs.getString("body"); 183 | int status = rs.getInt("status"); 184 | long ttl = rs.getLong("ttl"); 185 | String extend_data = rs.getString("extend_data"); 186 | JobMsg msg = new JobMsg(); 187 | msg.setStatus(status); 188 | msg.setTtl(ttl); 189 | msg.setBizKey(bizKey); 190 | msg.setBody(body); 191 | msg.setTopic(topic); 192 | msg.setCreateTime(createTime); 193 | msg.setSubtopic(subtopic); 194 | msg.setDelay(delay); 195 | msg.setId(id); 196 | msg.setExtendData(extend_data); 197 | msgList.add(msg); 198 | } 199 | } catch (final SQLException ex) { 200 | LOGGER.error("getNotFinshDataList数据发生错误", ex); 201 | } finally { 202 | 203 | } 204 | return msgList; 205 | } 206 | } 207 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 4 | 4.0.0 5 | 6 | io.sdmq 7 | sdmq-core 8 | 0.0.1 9 | jar 10 | 11 | sdmq 12 | https://github.com/peachyy/sdmq.git 13 | 14 | 15 | UTF-8 16 | UTF-8 17 | 4.2.3.RELEASE 18 | 1.4.3.RELEASE 19 | 1.7 20 | ${java.version} 21 | ${java.version} 22 | 32.0.0-jre 23 | 1.3.1 24 | 1.2.83 25 | 2.10.0 26 | false 27 | 28 | 29 | 30 | 31 | org.springframework.boot 32 | spring-boot-dependencies 33 | ${boot.version} 34 | pom 35 | import 36 | 37 | 38 | com.google.guava 39 | guava 40 | ${guava.version} 41 | 42 | 43 | org.apache.commons 44 | commons-lang3 45 | 3.1 46 | 47 | 48 | commons-codec 49 | commons-codec 50 | 1.9 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | io.springfox 60 | springfox-swagger2 61 | 2.4.0 62 | 63 | 64 | io.springfox 65 | springfox-swagger-ui 66 | 2.4.0 67 | 68 | 69 | 70 | com.alibaba 71 | fastjson 72 | ${fastjson.version} 73 | 74 | 75 | com.alibaba 76 | druid 77 | 1.0.14 78 | 79 | 80 | mysql 81 | mysql-connector-java 82 | 8.0.28 83 | 84 | 85 | org.apache.curator 86 | curator-recipes 87 | ${curator.version} 88 | 89 | 90 | org.apache.curator 91 | curator-client 92 | ${curator.version} 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | org.springframework.boot 102 | spring-boot-starter-thymeleaf 103 | true 104 | 105 | 106 | org.springframework.boot 107 | spring-boot-starter-web 108 | 109 | 110 | org.springframework.boot 111 | spring-boot-starter-tomcat 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | org.springframework.boot 122 | spring-boot-starter-undertow 123 | true 124 | 125 | 126 | org.springframework.boot 127 | spring-boot-starter-actuator 128 | true 129 | 130 | 131 | org.springframework.boot 132 | spring-boot-starter-data-redis 133 | true 134 | 135 | 136 | org.springframework.boot 137 | spring-boot-starter-test 138 | 139 | 140 | org.springframework 141 | spring-context-support 142 | 143 | 144 | mysql 145 | mysql-connector-java 146 | 147 | 148 | com.alibaba 149 | fastjson 150 | 151 | 152 | 153 | com.alibaba 154 | druid 155 | 156 | 157 | com.google.guava 158 | guava 159 | 160 | 161 | com.alibaba.rocketmq 162 | rocketmq-client 163 | 3.2.6 164 | 165 | 166 | org.apache.curator 167 | curator-recipes 168 | 169 | 170 | org.apache.curator 171 | curator-client 172 | 173 | 174 | 175 | 176 | 177 | src/main/java 178 | 179 | 180 | org.springframework.boot 181 | spring-boot-maven-plugin 182 | ${boot.version} 183 | 184 | true 185 | io.sdmq.Bootstarp 186 | 187 | 188 | 189 | 190 | repackage 191 | 192 | 193 | 194 | 195 | 196 | org.apache.maven.plugins 197 | maven-assembly-plugin 198 | 2.5.2 199 | 200 | false 201 | src/main/assembly/assembly.xml 202 | peachyy-${project.name} 203 | 204 | 205 | 206 | make-assembly 207 | install 208 | 209 | single 210 | 211 | 212 | 213 | 214 | 215 | org.apache.maven.plugins 216 | maven-surefire-plugin 217 | 2.18.1 218 | 219 | true 220 | 221 | **/*Test.java 222 | 223 | 224 | 225 | 226 | org.apache.maven.surefire 227 | surefire-junit47 228 | 2.18.1 229 | 230 | 231 | 232 | 233 | org.apache.maven.plugins 234 | maven-jar-plugin 235 | 2.6 236 | 237 | 238 | 239 | true 240 | lib/ 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | --------------------------------------------------------------------------------