├── .gitignore ├── README.md ├── pom.xml └── src ├── main ├── java │ └── me │ │ └── zhenchuan │ │ └── rmqc │ │ ├── AsyncRMQClient.java │ │ ├── ClientConfig.java │ │ ├── IRMQClient.java │ │ ├── RMQClient.java │ │ ├── SyncRMQClient.java │ │ ├── async │ │ ├── AsyncRMQSender.java │ │ └── RateLimiter.java │ │ ├── message │ │ ├── ByteArrayDataOutputStream.java │ │ ├── Compression.java │ │ ├── Message.java │ │ ├── MessageListener.java │ │ ├── MessageSerDe.java │ │ ├── MessageSetBuilder.java │ │ ├── MessageSetReader.java │ │ ├── SerDe.java │ │ └── TMessageSet.java │ │ └── queue │ │ ├── FileBlockingQueue.java │ │ ├── FileQueue4Sink.java │ │ ├── MemoryQueue4Sink.java │ │ ├── MessageQueue4Sink.java │ │ └── Queue4Client.java └── resources │ └── producer.properties └── test └── java └── me └── zhenchuan └── rmqc └── ClientTest.java /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | .classpath 3 | .project 4 | .settings -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | RocketMQClient 2 | ============== 3 | 4 | RocketMQ的异步发送的方式是用异步网络I/O的方式来完成的 5 | 6 | kafka的实现方式为本地内存维护一个队列,异步的过程就是把数据放入队列的过程. 7 | 8 | 如果由于网络的不稳定性,Kafka的方式会造成数据在内存堆积.同时,如果服务端重启等操作也会导致数据丢失. 9 | 10 | 而RocketMQ的同步会对主线程进行阻塞.异步的话又依赖于broker的处理能力.还是有可能阻塞主线程服务. 11 | 12 | 13 | 对RocketMQ的调用进行封装,加了一层Queue来尽可能减小对主服务的影响. 14 | 15 | 同时为了解决内存Queue的弊端,这里采用MappedFileQueue来保证写入速度的情况下来保证数据的安全性. 16 | 17 | 同时支持一次从Queue中取出N个message,进行打包压缩来减少网络消耗.(如果是采用这种方式,则需要消费端来知晓其设置来做对应的调整) 18 | 19 | 还支持发送Producer时的限流,防止对主服务的负载产生影响. 20 | 21 | #Note 22 | 本client的目的是即使broker挂掉,依然可以hold住大量的数据.对主服务不产生(或较小)影响 23 | 24 | 注意升级你的broker的处理能力!!! 25 | 26 | #配置 27 | 28 | public interface ClientConfig { 29 | 30 | @Config("client.type") 31 | @Default("sync") //支持sync和async 32 | public String clientType(); 33 | 34 | @Config("async.queue.type") 35 | @Default("memory")//支持file和memory 36 | public String asyncQueueType() ; 37 | 38 | @Config("async.memory.queue.capacity") 39 | @Default("100000")//内存队列ArrayBlockingQueue的容量 40 | public int asyncMemoryQueueCapacity() ; 41 | 42 | @Config("async.filequeue.path") 43 | @Default("/tmp/mqlocalfilequeue")//mapedfilequeue的目录 44 | public String asyncFileQueuePath() ; 45 | 46 | @Config("async.filequeue.name") 47 | @Default("localqueue")//mapedfilequeue的名称 48 | public String asyncFileQueueName() ; 49 | 50 | @Config("async.filequeue.gcperid") 51 | @Default("PT1h")//删除已经处理的数据,周期,默认为1小时.格式为joda 52 | public String asyncFileQueueGCPeriod() ; 53 | 54 | @Config("async.jobqueue.capacity") 55 | @Default("10")//缓冲的task(每次发送被封装为一个task)的个数. 56 | public int asyncJobQueueCapacity(); 57 | 58 | @Config("async.sender.threads") 59 | @Default("4")//异步发送到broker使用的线程数.线程池. 60 | public int asyncSenderThreads() ; 61 | 62 | @Config("retry.count") 63 | @Default("5")//发送broker时的重试次数. 64 | public int retryCount() ; 65 | 66 | @Config("app") 67 | @Default("") 68 | public String app() ; 69 | 70 | @Config("compression") 71 | @Default("0")//0表示不压缩,1表示使用lzf压缩 72 | public int getCompression() ; 73 | 74 | @Config("async.timeout") 75 | @Default("60000")//从queue中获得message的超时时间 76 | public long getAsyncTimeout(); 77 | 78 | @Config("async.batchsize") 79 | @Default("1")//批量发送的message的个数... 80 | public int asyncBatchSize() ; 81 | 82 | @Config("topic") 83 | @Default("") 84 | public String topic(); 85 | 86 | @Config("message.per.second") 87 | @Default("100000000")//限制向broker发送的messge的qps 88 | public int msgPerSec(); 89 | 90 | } 91 | 92 | #用法 93 | 94 | Properties properties = new Properties(); 95 | //properties.put("client.type", "async"); 96 | //properties.put("async.queue.type", "file"); 97 | 98 | RMQClient client = new RMQClient(properties); 99 | Message message = new Message("routekey","hello world".getBytes()); 100 | client.send(message); 101 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | me.zhenchuan 6 | RocketMQClient 7 | 0.0.1-SNAPSHOT 8 | jar 9 | 10 | RocketMQClient 11 | http://maven.apache.org 12 | 13 | 14 | UTF-8 15 | 16 | 17 | 18 | 19 | junit 20 | junit 21 | 4.10 22 | test 23 | 24 | 25 | 26 | org.slf4j 27 | slf4j-log4j12 28 | 1.5.6 29 | test 30 | 31 | 32 | 33 | commons-lang 34 | commons-lang 35 | 2.6 36 | test 37 | 38 | 39 | 40 | 41 | 42 | com.alibaba.rocketmq 43 | rocketmq-client 44 | 3.0.9 45 | 46 | 47 | 48 | com.google.guava 49 | guava 50 | 11.0.2 51 | 52 | 53 | 54 | joda-time 55 | joda-time 56 | 2.3 57 | 58 | 59 | 60 | com.ning 61 | compress-lzf 62 | 1.0.1 63 | 64 | 65 | 66 | com.leansoft 67 | bigqueue 68 | 0.7.0 69 | 70 | 71 | 72 | org.skife.config 73 | config-magic 74 | 0.9 75 | 76 | 77 | 78 | com.codahale.metrics 79 | metrics-core 80 | 3.0.1 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | org.apache.maven.plugins 89 | maven-compiler-plugin 90 | 91 | 1.7 92 | 1.7 93 | 94 | 95 | 96 | 97 | org.apache.maven.plugins 98 | maven-assembly-plugin 99 | 100 | false 101 | 102 | jar-with-dependencies 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | github.release.repo 113 | https://raw.github.com/bulldog2011/bulldog-repo/master/repo/releases/ 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/AsyncRMQClient.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc; 2 | 3 | import java.util.concurrent.*; 4 | import java.util.concurrent.atomic.AtomicLong; 5 | 6 | import javax.annotation.PreDestroy; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.google.common.util.concurrent.ThreadFactoryBuilder; 12 | 13 | import me.zhenchuan.rmqc.async.AsyncRMQSender; 14 | import me.zhenchuan.rmqc.async.RateLimiter; 15 | import me.zhenchuan.rmqc.message.Message; 16 | import me.zhenchuan.rmqc.message.Compression; 17 | import me.zhenchuan.rmqc.message.MessageSetBuilder; 18 | import me.zhenchuan.rmqc.message.MessageSetReader; 19 | import me.zhenchuan.rmqc.message.TMessageSet; 20 | import me.zhenchuan.rmqc.queue.Queue4Client; 21 | 22 | public class AsyncRMQClient implements IRMQClient { 23 | 24 | private static final Logger log = LoggerFactory 25 | .getLogger(AsyncRMQClient.class); 26 | 27 | private final ClientConfig config; 28 | private final Queue4Client messageQueue; 29 | 30 | private final BlockingQueue jobQueue; 31 | private final ThreadPoolExecutor senders; 32 | private final MessageSetBuilder builder; 33 | 34 | private AtomicLong lostMessages = new AtomicLong(0); 35 | private AtomicLong sentMessages = new AtomicLong(0); 36 | private AtomicLong restoredMessages = new AtomicLong(0); 37 | private AtomicLong retriedCount = new AtomicLong(0); 38 | 39 | private final RateLimiter rateLimiter; 40 | 41 | private ExecutorService poller = Executors 42 | .newSingleThreadExecutor(new ThreadFactoryBuilder().setNameFormat( 43 | "AsyncRMQClientPoller-%d").build()); 44 | 45 | public AsyncRMQClient(ClientConfig config){ 46 | this(config,new Queue4Client(config),config.msgPerSec()); 47 | } 48 | 49 | public AsyncRMQClient(ClientConfig config, Queue4Client messageQueue, 50 | int mesPerSec) { 51 | this.config = config; 52 | this.messageQueue = messageQueue; 53 | this.builder = new MessageSetBuilder(config) 54 | .withCompression(Compression.create(config.getCompression())); 55 | //使用builder从messageQueue中批量取出数据,交给senders处理. 56 | poller.execute(createPoller()); 57 | 58 | this.jobQueue = new ArrayBlockingQueue( 59 | config.asyncJobQueueCapacity()); 60 | 61 | this.rateLimiter = new RateLimiter(mesPerSec); 62 | 63 | this.senders = new ThreadPoolExecutor(config.asyncSenderThreads(), 64 | config.asyncSenderThreads(), 10, TimeUnit.SECONDS, jobQueue, 65 | new RejectedExecutionHandler() { 66 | @Override 67 | public void rejectedExecution(Runnable r, 68 | ThreadPoolExecutor executor) { 69 | TMessageSet messageSet = ((AsyncRMQSender) r) 70 | .getMessageSet(); 71 | for (Message m : new MessageSetReader(messageSet)) { 72 | restore(m); 73 | } 74 | } 75 | }); 76 | } 77 | 78 | private boolean running; 79 | 80 | private long lastBatch; 81 | 82 | private Runnable createPoller() { 83 | running = true; 84 | final AsyncRMQClient client = this; 85 | 86 | return new Runnable() { 87 | @Override 88 | public void run() { 89 | while (running || !messageQueue.isEmpty()) { 90 | try { 91 | Message msg = messageQueue.poll( 92 | Math.max(0, 93 | lastBatch + config.getAsyncTimeout() 94 | - System.currentTimeMillis()), 95 | TimeUnit.MILLISECONDS); 96 | 97 | boolean expired = (msg == null); 98 | if (!expired) { 99 | builder.withMessage(msg.getRoutingKey(), 100 | msg.getPayload()); 101 | builder.drainFrom(messageQueue, 102 | config.asyncBatchSize() - builder.size()); 103 | } 104 | 105 | boolean full = (builder.size() >= config 106 | .asyncBatchSize()); 107 | if ((expired || full) && builder.size() > 0) { 108 | lastBatch = System.currentTimeMillis(); 109 | rateLimiter.pause(builder.size()); 110 | senders.execute(new AsyncRMQSender(builder.build(), 111 | client, config)); 112 | } else if (builder.size() == 0) { 113 | Thread.sleep(config.getAsyncTimeout()); 114 | } 115 | } catch (Exception e) { 116 | log.error( 117 | "MessageConsumer poller exception: " 118 | + e.getMessage(), e); 119 | } 120 | } 121 | 122 | builder.drainFrom(messageQueue, (int) messageQueue.size()); 123 | if (builder.size() > 0) { 124 | try { 125 | senders.execute(new AsyncRMQSender(builder.build(), client, 126 | config)); 127 | } catch (Exception e) { 128 | log.error( 129 | "MessageConsumer poller exception: " 130 | + e.getMessage(), e); 131 | } 132 | } 133 | } 134 | }; 135 | } 136 | 137 | @PreDestroy 138 | public void shutdown() { 139 | running = false; 140 | poller.shutdown(); 141 | try { 142 | poller.awaitTermination(5000 + config.getAsyncTimeout(), 143 | TimeUnit.MILLISECONDS); 144 | senders.shutdown(); 145 | senders.awaitTermination(5000 + config.getAsyncTimeout(), 146 | TimeUnit.MILLISECONDS); 147 | if (!senders.isTerminated()) { 148 | log.error("AsyncSuroClient didn't terminate gracefully within 5 seconds"); 149 | senders.shutdownNow(); 150 | } 151 | } catch (InterruptedException e) { 152 | // ignore exceptions while shutting down 153 | } 154 | } 155 | 156 | @Override 157 | public void send(Message message) { 158 | if (!messageQueue.offer(message)) { 159 | lostMessages.incrementAndGet(); 160 | } 161 | } 162 | 163 | @Override 164 | public long getSentMessageCount() { 165 | return sentMessages.get(); 166 | } 167 | 168 | @Override 169 | public long getLostMessageCount() { 170 | return lostMessages.get(); 171 | } 172 | 173 | public void restore(Message message) { 174 | restoredMessages.incrementAndGet(); 175 | send(message); 176 | } 177 | 178 | private AtomicLong senderExceptionCount = new AtomicLong(0); 179 | 180 | public void updateSenderException() { 181 | senderExceptionCount.incrementAndGet(); 182 | } 183 | 184 | private long sendTime; 185 | public void updateSendTime(long sendTime) { 186 | this.sendTime = sendTime; 187 | } 188 | 189 | public void updateSentDataStats(TMessageSet messageSet, boolean retried) { 190 | if (retried) { 191 | retriedCount.incrementAndGet(); 192 | } 193 | } 194 | 195 | } 196 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/ClientConfig.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc; 2 | 3 | import org.skife.config.Config; 4 | import org.skife.config.Default; 5 | 6 | public interface ClientConfig { 7 | 8 | @Config("client.type") 9 | @Default("sync") //支持sync和async 10 | public String clientType(); 11 | 12 | @Config("async.queue.type") 13 | @Default("memory")//支持file和memory 14 | public String asyncQueueType() ; 15 | 16 | @Config("async.memory.queue.capacity") 17 | @Default("100000")//内存队列ArrayBlockingQueue的容量 18 | public int asyncMemoryQueueCapacity() ; 19 | 20 | @Config("async.filequeue.path") 21 | @Default("/tmp/mqlocalfilequeue")//mapedfilequeue的目录 22 | public String asyncFileQueuePath() ; 23 | 24 | @Config("async.filequeue.name") 25 | @Default("localqueue")//mapedfilequeue的名称 26 | public String asyncFileQueueName() ; 27 | 28 | @Config("async.filequeue.gcperid") 29 | @Default("PT1h")//删除已经处理的数据,周期,默认为1小时.格式为joda 30 | public String asyncFileQueueGCPeriod() ; 31 | 32 | @Config("async.jobqueue.capacity") 33 | @Default("10")//缓冲的task(每次发送被封装为一个task)的个数. 34 | public int asyncJobQueueCapacity(); 35 | 36 | @Config("async.sender.threads") 37 | @Default("4")//异步发送到broker使用的线程数.线程池. 38 | public int asyncSenderThreads() ; 39 | 40 | @Config("retry.count") 41 | @Default("5")//发送broker时的重试次数. 42 | public int retryCount() ; 43 | 44 | @Config("app") 45 | @Default("") 46 | public String app() ; 47 | 48 | @Config("compression") 49 | @Default("0")//0表示不压缩,1表示使用lzf压缩 50 | public int getCompression() ; 51 | 52 | @Config("async.timeout") 53 | @Default("60000")//从queue中获得message的超时时间 54 | public long getAsyncTimeout(); 55 | 56 | @Config("async.batchsize") 57 | @Default("1")//批量发送的message的个数... 58 | public int asyncBatchSize() ; 59 | 60 | @Config("topic") 61 | @Default("") 62 | public String topic(); 63 | 64 | @Config("message.per.second") 65 | @Default("100000000")//限制向broker发送的messge的qps 66 | public int msgPerSec(); 67 | 68 | 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/IRMQClient.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc; 2 | 3 | import me.zhenchuan.rmqc.message.Message; 4 | 5 | 6 | public interface IRMQClient { 7 | 8 | void send(Message message); 9 | 10 | long getSentMessageCount(); 11 | 12 | long getLostMessageCount(); 13 | 14 | } 15 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/RMQClient.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc; 2 | 3 | import java.util.Properties; 4 | 5 | import org.skife.config.ConfigurationObjectFactory; 6 | 7 | import com.alibaba.rocketmq.client.exception.MQClientException; 8 | 9 | import me.zhenchuan.rmqc.message.Message; 10 | 11 | 12 | public class RMQClient implements IRMQClient{ 13 | 14 | private final IRMQClient client; 15 | private final ClientConfig config; 16 | 17 | public RMQClient(Properties properties) throws MQClientException{ 18 | ConfigurationObjectFactory factory = new ConfigurationObjectFactory(properties); 19 | this.config = factory.build(ClientConfig.class); 20 | if("async".equalsIgnoreCase(config.clientType())){ 21 | client = new AsyncRMQClient(config); 22 | }else{ 23 | client = new SyncRMQClient(config); 24 | } 25 | } 26 | 27 | @Override 28 | public void send(Message message) { 29 | client.send(message); 30 | } 31 | 32 | @Override 33 | public long getSentMessageCount() { 34 | return client.getSentMessageCount(); 35 | } 36 | 37 | @Override 38 | public long getLostMessageCount() { 39 | return client.getLostMessageCount(); 40 | } 41 | 42 | public ClientConfig getConfig(){ 43 | return config; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/SyncRMQClient.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc; 2 | 3 | import java.util.concurrent.atomic.AtomicLong; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import com.alibaba.rocketmq.client.exception.MQClientException; 9 | import com.alibaba.rocketmq.client.producer.DefaultMQProducer; 10 | import com.alibaba.rocketmq.client.producer.SendResult; 11 | import com.alibaba.rocketmq.client.producer.SendStatus; 12 | 13 | import me.zhenchuan.rmqc.message.Compression; 14 | import me.zhenchuan.rmqc.message.Message; 15 | 16 | public class SyncRMQClient implements IRMQClient{ 17 | 18 | private static final Logger log = LoggerFactory.getLogger(SyncRMQClient.class); 19 | 20 | private final ClientConfig config; 21 | @Deprecated 22 | private final Compression compression; //因为RocksMQ默认可以设置在消息大于N时启动压缩.这里有点多余. 23 | private final DefaultMQProducer producer; 24 | 25 | private AtomicLong sentMessageCount = new AtomicLong(0); 26 | private AtomicLong lostMessageCount = new AtomicLong(0); 27 | private AtomicLong retriedCount = new AtomicLong(0); 28 | private AtomicLong senderExceptionCount = new AtomicLong(0); 29 | 30 | 31 | 32 | public SyncRMQClient(ClientConfig config) throws MQClientException{ 33 | this.config = config; 34 | this.compression = Compression.create(config.getCompression()); 35 | this.producer = new DefaultMQProducer(config.app()); 36 | this.producer.start(); 37 | } 38 | 39 | @Override 40 | public void send(Message message) { 41 | send((com.alibaba.rocketmq.common.message.Message)null); 42 | } 43 | 44 | public boolean send(com.alibaba.rocketmq.common.message.Message message) { 45 | if (message == null) { 46 | return false; 47 | } 48 | boolean sent = false; 49 | boolean retried = false; 50 | 51 | for (int i = 0; i < config.retryCount(); ++i) { 52 | try { 53 | SendResult sendResult = producer.send(message); 54 | if(sendResult.getSendStatus() == SendStatus.SEND_OK){ 55 | sent = true; 56 | retried = i > 0; 57 | break; 58 | } 59 | } catch (Exception e) { 60 | log.error("Exception in send: " + e.getMessage(), e); 61 | senderExceptionCount.incrementAndGet(); 62 | } 63 | } 64 | 65 | if (sent) { 66 | sentMessageCount.incrementAndGet(); 67 | if (retried) { 68 | retriedCount.incrementAndGet(); 69 | } 70 | 71 | } else { 72 | lostMessageCount.incrementAndGet(); 73 | } 74 | 75 | return sent; 76 | } 77 | 78 | @Override 79 | public long getSentMessageCount() { 80 | return sentMessageCount.get(); 81 | } 82 | 83 | @Override 84 | public long getLostMessageCount() { 85 | return lostMessageCount.get(); 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/async/AsyncRMQSender.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.async; 2 | 3 | import me.zhenchuan.rmqc.AsyncRMQClient; 4 | import me.zhenchuan.rmqc.ClientConfig; 5 | import me.zhenchuan.rmqc.message.MessageSetReader; 6 | import me.zhenchuan.rmqc.message.TMessageSet; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import com.alibaba.rocketmq.client.exception.MQClientException; 12 | import com.alibaba.rocketmq.client.producer.DefaultMQProducer; 13 | import com.alibaba.rocketmq.client.producer.SendResult; 14 | import com.alibaba.rocketmq.client.producer.SendStatus; 15 | import com.alibaba.rocketmq.common.message.Message; 16 | /**** 17 | * 发送MessageSet 18 | * @author crnsnl 19 | * 20 | */ 21 | public class AsyncRMQSender implements Runnable { 22 | private static final Logger log = LoggerFactory.getLogger(AsyncRMQSender.class); 23 | 24 | private final AsyncRMQClient client; 25 | private final ClientConfig config; 26 | private final TMessageSet messageSet; 27 | private final DefaultMQProducer producer; 28 | 29 | public AsyncRMQSender( 30 | TMessageSet tMessageSet, 31 | AsyncRMQClient client,ClientConfig config) throws MQClientException { 32 | this.client = client; 33 | this.config = config; 34 | this.messageSet = tMessageSet; 35 | this.producer = new DefaultMQProducer(config.app()); 36 | this.producer.start(); 37 | } 38 | 39 | public void run() { 40 | boolean sent = false; 41 | boolean retried = false; 42 | long startTS = System.currentTimeMillis(); 43 | Message message = messageSet.toRocketMQMessage(config.topic()); 44 | for (int i = 0; i < config.retryCount(); ++i) { 45 | try { 46 | SendResult sendResult = producer.send(message); 47 | if(sendResult.getSendStatus() == SendStatus.SEND_OK){ 48 | sent = true; 49 | retried = i > 0; 50 | break; 51 | } 52 | } catch (Exception e) { 53 | log.error("Exception in send: " + e.getMessage(), e); 54 | client.updateSenderException(); 55 | } 56 | } 57 | 58 | if (sent){ 59 | client.updateSendTime(System.currentTimeMillis() - startTS); 60 | client.updateSentDataStats(messageSet, retried); 61 | } else { 62 | for (me.zhenchuan.rmqc.message.Message m : new MessageSetReader(messageSet)) { 63 | client.restore(m); 64 | } 65 | } 66 | } 67 | 68 | public TMessageSet getMessageSet() { 69 | return messageSet; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/async/RateLimiter.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.async; 2 | 3 | /** 4 | * A simple rate limiter based on Lucene's RateLimiter.SimpleRateLimiter. 5 | */ 6 | public class RateLimiter { 7 | private volatile int msgPerSec; 8 | private volatile double nsPerMsg; 9 | private volatile long lastNS; 10 | 11 | public RateLimiter(int msgPerSec) { 12 | setMsgPerSec(msgPerSec); 13 | } 14 | 15 | public void setMsgPerSec(int msgPerSec) { 16 | this.msgPerSec = msgPerSec; 17 | nsPerMsg = 1000000000. / msgPerSec; 18 | 19 | } 20 | 21 | public int getMsgPerSec() { 22 | return this.msgPerSec; 23 | } 24 | 25 | /** Pauses, if necessary, to keep the instantaneous IO 26 | * rate at or below the target. NOTE: multiple threads 27 | * may safely use this, however the implementation is 28 | * not perfectly thread safe but likely in practice this 29 | * is harmless (just means in some rare cases the rate 30 | * might exceed the target). It's best to call this 31 | * with a biggish count, not one byte at a time. 32 | * @return the pause time in nano seconds 33 | * */ 34 | public long pause(int msgs) throws InterruptedException { 35 | if (msgs == 1) { 36 | return 0; 37 | } 38 | 39 | // TODO: this is purely instantaneous rate; maybe we 40 | // should also offer decayed recent history one? 41 | final long targetNS = lastNS = lastNS + ((long) (msgs * nsPerMsg)); 42 | final long startNS; 43 | long curNS = startNS = System.nanoTime(); 44 | if (lastNS < curNS) { 45 | lastNS = curNS; 46 | } 47 | 48 | // While loop because Thread.sleep doesn't always sleep 49 | // enough: 50 | while(true) { 51 | final long pauseNS = targetNS - curNS; 52 | if (pauseNS > 0) { 53 | Thread.sleep((int) (pauseNS/1000000), (int) (pauseNS % 1000000)); 54 | curNS = System.nanoTime(); 55 | continue; 56 | } 57 | break; 58 | } 59 | return curNS - startNS; 60 | } 61 | } -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/ByteArrayDataOutputStream.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import com.google.common.io.ByteArrayDataOutput; 4 | 5 | import java.io.ByteArrayOutputStream; 6 | import java.io.DataOutput; 7 | import java.io.DataOutputStream; 8 | import java.io.IOException; 9 | 10 | /** 11 | * Exposing Guava's ByteStreams.ByteArrayDataOutput, which is private static. 12 | */ 13 | public class ByteArrayDataOutputStream implements ByteArrayDataOutput { 14 | private final DataOutput output; 15 | private final ByteArrayOutputStream byteArrayOutputSteam; 16 | 17 | public ByteArrayDataOutputStream(ByteArrayOutputStream byteArrayOutputSteam) { 18 | this.byteArrayOutputSteam = byteArrayOutputSteam; 19 | output = new DataOutputStream(byteArrayOutputSteam); 20 | } 21 | 22 | @Override 23 | public void write(int b) { 24 | try { 25 | output.write(b); 26 | } catch (IOException impossible) { 27 | throw new AssertionError(impossible); 28 | } 29 | } 30 | 31 | @Override 32 | public void write(byte[] b) { 33 | try { 34 | output.write(b); 35 | } catch (IOException impossible) { 36 | throw new AssertionError(impossible); 37 | } 38 | } 39 | 40 | @Override 41 | public void write(byte[] b, int off, int len) { 42 | try { 43 | output.write(b, off, len); 44 | } catch (IOException impossible) { 45 | throw new AssertionError(impossible); 46 | } 47 | } 48 | 49 | @Override 50 | public void writeBoolean(boolean v) { 51 | try { 52 | output.writeBoolean(v); 53 | } catch (IOException impossible) { 54 | throw new AssertionError(impossible); 55 | } 56 | } 57 | 58 | @Override 59 | public void writeByte(int v) { 60 | try { 61 | output.writeByte(v); 62 | } catch (IOException impossible) { 63 | throw new AssertionError(impossible); 64 | } 65 | } 66 | 67 | @Override 68 | public void writeBytes(String s) { 69 | try { 70 | output.writeBytes(s); 71 | } catch (IOException impossible) { 72 | throw new AssertionError(impossible); 73 | } 74 | } 75 | 76 | @Override 77 | public void writeChar(int v) { 78 | try { 79 | output.writeChar(v); 80 | } catch (IOException impossible) { 81 | throw new AssertionError(impossible); 82 | } 83 | } 84 | 85 | @Override 86 | public void writeChars(String s) { 87 | try { 88 | output.writeChars(s); 89 | } catch (IOException impossible) { 90 | throw new AssertionError(impossible); 91 | } 92 | } 93 | 94 | @Override 95 | public void writeDouble(double v) { 96 | try { 97 | output.writeDouble(v); 98 | } catch (IOException impossible) { 99 | throw new AssertionError(impossible); 100 | } 101 | } 102 | 103 | @Override 104 | public void writeFloat(float v) { 105 | try { 106 | output.writeFloat(v); 107 | } catch (IOException impossible) { 108 | throw new AssertionError(impossible); 109 | } 110 | } 111 | 112 | @Override 113 | public void writeInt(int v) { 114 | try { 115 | output.writeInt(v); 116 | } catch (IOException impossible) { 117 | throw new AssertionError(impossible); 118 | } 119 | } 120 | 121 | @Override 122 | public void writeLong(long v) { 123 | try { 124 | output.writeLong(v); 125 | } catch (IOException impossible) { 126 | throw new AssertionError(impossible); 127 | } 128 | } 129 | 130 | @Override 131 | public void writeShort(int v) { 132 | try { 133 | output.writeShort(v); 134 | } catch (IOException impossible) { 135 | throw new AssertionError(impossible); 136 | } 137 | } 138 | 139 | @Override 140 | public void writeUTF(String s) { 141 | try { 142 | output.writeUTF(s); 143 | } catch (IOException impossible) { 144 | throw new AssertionError(impossible); 145 | } 146 | } 147 | 148 | @Override 149 | public byte[] toByteArray() { 150 | return byteArrayOutputSteam.toByteArray(); 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/Compression.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import com.ning.compress.lzf.LZFDecoder; 4 | import com.ning.compress.lzf.LZFEncoder; 5 | 6 | import java.io.IOException; 7 | 8 | /** 9 | * Suro message payload compression 10 | * 11 | * The method {@link #compress(byte[])} receives byte[] and returns compressed byte[] 12 | * The method {@link #decompress(byte[])} receives compressed byte[] and returns uncompressed one 13 | * 14 | * 0, NO no compression 15 | * 1, LZF LZF compression 16 | * 17 | * @author jbae 18 | */ 19 | public enum Compression { 20 | NO(0) { 21 | byte[] compress(byte[] buffer) { 22 | return buffer; 23 | } 24 | byte[] decompress(byte[] buffer) { 25 | return buffer; 26 | } 27 | }, 28 | LZF(1) { 29 | byte[] compress(byte[] buffer) { 30 | try { 31 | return LZFEncoder.encode(buffer); 32 | } catch (Exception e) { 33 | throw new RuntimeException(e); 34 | } 35 | } 36 | byte[] decompress(byte[] buffer) { 37 | try { 38 | return LZFDecoder.decode(buffer); 39 | } catch (IOException e) { 40 | throw new RuntimeException(e); 41 | } 42 | } 43 | }; 44 | private final int id; 45 | Compression(int id) { this.id = id; } 46 | 47 | public int getId() { return id; } 48 | 49 | public static Compression create(int id) { 50 | for (Compression compression : values()) { 51 | if (id == compression.getId()) { 52 | return compression; 53 | } 54 | } 55 | 56 | throw new IllegalArgumentException("invalid compression id: " + id); 57 | } 58 | 59 | abstract byte[] compress(byte[] buffer); 60 | abstract byte[] decompress(byte[] buffer); 61 | } -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/Message.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import com.google.common.collect.BiMap; 4 | import com.google.common.collect.HashBiMap; 5 | 6 | import java.io.DataInput; 7 | import java.io.DataOutput; 8 | import java.io.IOException; 9 | import java.util.Arrays; 10 | 11 | /** 12 | * Suro message payload contains routing key as String and payload as byte[]. 13 | * 14 | * @author jbae 15 | */ 16 | public class Message { 17 | public static final BiMap> classMap = HashBiMap.create(); 18 | static { 19 | classMap.put((byte) 0, Message.class); 20 | } 21 | 22 | private String routingKey; 23 | private byte[] payload; 24 | 25 | public Message() {} 26 | public Message(String routingKey, byte[] payload) { 27 | this.routingKey = routingKey; 28 | this.payload = payload; 29 | } 30 | 31 | public String getRoutingKey() { 32 | return routingKey; 33 | } 34 | 35 | public byte[] getPayload() { 36 | return payload; 37 | } 38 | 39 | @Override 40 | public String toString() { 41 | return String.format("routingKey: %s, payload byte size: %d", 42 | routingKey, 43 | payload.length); 44 | } 45 | 46 | @Override 47 | public boolean equals(Object o) { 48 | if (this == o) return true; 49 | if (o == null || getClass() != o.getClass()) return false; 50 | 51 | Message message = (Message) o; 52 | 53 | if (!Arrays.equals(payload, message.payload)) return false; 54 | return !(routingKey != null ? !routingKey.equals(message.routingKey) : message.routingKey != null); 55 | 56 | } 57 | 58 | @Override 59 | public int hashCode() { 60 | int result = routingKey != null ? routingKey.hashCode() : 0; 61 | result = 31 * result + (payload != null ? Arrays.hashCode(payload) : 0); 62 | return result; 63 | } 64 | 65 | 66 | public void write(DataOutput dataOutput) throws IOException { 67 | dataOutput.writeUTF(routingKey); 68 | dataOutput.writeInt(payload.length); 69 | dataOutput.write(payload); 70 | } 71 | 72 | 73 | public void readFields(DataInput dataInput) throws IOException { 74 | routingKey = dataInput.readUTF(); 75 | payload = new byte[dataInput.readInt()]; 76 | dataInput.readFully(payload); 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/MessageListener.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | public interface MessageListener { 4 | 5 | public void onLostMessage(Message message); 6 | 7 | } 8 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/MessageSerDe.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import com.google.common.io.ByteArrayDataOutput; 4 | import com.google.common.io.ByteStreams; 5 | 6 | import org.slf4j.Logger; 7 | import org.slf4j.LoggerFactory; 8 | 9 | import java.io.ByteArrayOutputStream; 10 | import java.io.DataInput; 11 | import java.io.IOException; 12 | 13 | /** 14 | * Message itself should be (de)serialized. This serde is mainly used to persist 15 | * messages to somewhere including file based queue 16 | * 17 | * @author jbae 18 | */ 19 | public class MessageSerDe implements SerDe { 20 | private final static Logger log = LoggerFactory.getLogger(MessageSerDe.class); 21 | 22 | private ThreadLocal outputStream = 23 | new ThreadLocal() { 24 | @Override 25 | protected ByteArrayOutputStream initialValue() { 26 | return new ByteArrayOutputStream(); 27 | } 28 | 29 | @Override 30 | public ByteArrayOutputStream get() { 31 | ByteArrayOutputStream b = super.get(); 32 | b.reset(); 33 | return b; 34 | } 35 | }; 36 | 37 | @Override 38 | public Message deserialize(byte[] payload) { 39 | try { 40 | DataInput dataInput = ByteStreams.newDataInput(payload); 41 | Class clazz = Message.classMap.get(dataInput.readByte()); 42 | Message msg = clazz.newInstance(); 43 | msg.readFields(dataInput); 44 | return msg; 45 | } catch (Exception e) { 46 | log.error("Exception on deserialize: " + e.getMessage(), e); 47 | return new Message(); 48 | } 49 | } 50 | 51 | @Override 52 | public byte[] serialize(Message payload) { 53 | try { 54 | ByteArrayDataOutput out = new ByteArrayDataOutputStream(outputStream.get()); 55 | out.writeByte(Message.classMap.inverse().get(payload.getClass())); 56 | payload.write(out); 57 | return out.toByteArray(); 58 | } catch (IOException e) { 59 | log.error("Exception on serialize: " + e.getMessage(), e); 60 | return new byte[]{}; 61 | } 62 | } 63 | 64 | @Override 65 | public String toString(byte[] payload) { 66 | return deserialize(payload).toString(); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/MessageSetBuilder.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import com.google.common.io.ByteArrayDataOutput; 4 | 5 | import me.zhenchuan.rmqc.ClientConfig; 6 | import me.zhenchuan.rmqc.queue.Queue4Client; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.ByteArrayOutputStream; 12 | import java.io.IOException; 13 | import java.nio.ByteBuffer; 14 | import java.util.ArrayList; 15 | import java.util.List; 16 | import java.util.zip.CRC32; 17 | 18 | /** 19 | * The payload for Suro's Thrift communication is {@link TMessageSet}, a thrift presentation of 20 | * a set of messages. This class can be helpful to easily create {@link TMessageSet} instances. 21 | * 22 | * @author jbae 23 | */ 24 | public class MessageSetBuilder { 25 | private static final Logger log = LoggerFactory.getLogger(MessageSetBuilder.class); 26 | 27 | private final ClientConfig config; 28 | private List messageList; 29 | private Compression compression = Compression.LZF; 30 | 31 | /** 32 | * @param config contains information including application name, etc 33 | */ 34 | public MessageSetBuilder(ClientConfig config) { 35 | this.config = config; 36 | messageList = new ArrayList(); 37 | } 38 | 39 | public MessageSetBuilder withMessage(String routingKey, byte[] payload) { 40 | this.messageList.add(new Message(routingKey, payload)); 41 | return this; 42 | } 43 | 44 | public MessageSetBuilder withCompression(Compression compresson) { 45 | this.compression = compresson; 46 | return this; 47 | } 48 | 49 | public TMessageSet build() { 50 | try { 51 | byte[] buffer = createPayload(messageList, compression); 52 | long crc = getCRC(buffer); 53 | 54 | 55 | return new TMessageSet( 56 | config.app(), 57 | messageList.size(), 58 | (byte) compression.getId(), 59 | crc, 60 | ByteBuffer.wrap(buffer)); 61 | } catch (IOException e) { 62 | log.error("Exception on building TMessageSet: " + e.getMessage(), e); 63 | return null; 64 | } finally { 65 | messageList.clear(); 66 | } 67 | } 68 | 69 | /** 70 | * @return number of messages in MessageSet 71 | */ 72 | public int size() { 73 | return messageList.size(); 74 | } 75 | 76 | /** 77 | * Create compressed byte[] from the list of messages. Each message contains 78 | * byte[] as its message body, so, this method is simply flattening byte[] 79 | * for all messages in the messageList 80 | * 81 | * @param messageList a list of messages for payload 82 | * @param compression Compression method to be applied to the payload 83 | * @return A byte array that encodes the build payload 84 | * @throws IOException 85 | */ 86 | public static byte[] createPayload(List messageList, Compression compression) throws IOException { 87 | ByteArrayDataOutput out = new ByteArrayDataOutputStream(outputStream.get()); 88 | for (Message message : messageList) { 89 | message.write(out); 90 | } 91 | 92 | return compression.compress(out.toByteArray()); 93 | } 94 | 95 | private static ThreadLocal outputStream = 96 | new ThreadLocal() { 97 | @Override 98 | protected ByteArrayOutputStream initialValue() { 99 | return new ByteArrayOutputStream(); 100 | } 101 | 102 | @Override 103 | public ByteArrayOutputStream get() { 104 | ByteArrayOutputStream b = super.get(); 105 | b.reset(); 106 | return b; 107 | } 108 | }; 109 | 110 | /** 111 | * Compute CRC32 value for byte[] 112 | * 113 | * @param buffer all the bytes in the buffer will be used for CRC32 calculation 114 | * 115 | * @return a CRC32 value for the given byte array 116 | */ 117 | public static long getCRC(byte[] buffer) { 118 | CRC32 crc = new CRC32(); 119 | crc.update(buffer); 120 | return crc.getValue(); 121 | } 122 | 123 | /** 124 | * Drains the given number of messages from the givne queue. Instead of calling {@link #withMessage(String, byte[])}, 125 | * we can call this method. 126 | * This method is reverse one of JDK BlockingQueue.drainTo. 127 | * 128 | * @param queue the queue to drain messages from 129 | * @param size the number of messages to drain from the given queue 130 | */ 131 | public void drainFrom(Queue4Client queue, int size) { 132 | queue.drain(size, messageList); 133 | } 134 | } 135 | 136 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/MessageSetReader.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import com.google.common.io.ByteArrayDataInput; 4 | import com.google.common.io.ByteStreams; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.io.IOException; 9 | import java.util.Iterator; 10 | 11 | /** 12 | * This class implements an {@link Iterable} reader for {@link TMessageSet} so we can iterate 13 | * over each message in a {@link TMessageSet}. 14 | * 15 | * @author jbae 16 | */ 17 | public class MessageSetReader implements Iterable { 18 | private static final Logger log = LoggerFactory.getLogger(MessageSetReader.class); 19 | 20 | private final TMessageSet messageSet; 21 | 22 | public MessageSetReader(TMessageSet messageSet) { 23 | this.messageSet = messageSet; 24 | } 25 | 26 | public boolean checkCRC() { 27 | long crcReceived = messageSet.getCrc(); 28 | long crc = MessageSetBuilder.getCRC(messageSet.getMessages()); 29 | 30 | return crcReceived == crc; 31 | } 32 | 33 | @Override 34 | public Iterator iterator() { 35 | try { 36 | final ByteArrayDataInput input = ByteStreams.newDataInput( 37 | Compression.create(messageSet.getCompression()).decompress(messageSet.getMessages())); 38 | 39 | return new Iterator() { 40 | private int messageCount = messageSet.getNumMessages(); 41 | 42 | @Override 43 | public boolean hasNext() { 44 | return messageCount > 0; 45 | } 46 | 47 | @Override 48 | public Message next() { 49 | Message m = new Message(); 50 | try { 51 | m.readFields(input); 52 | --messageCount; 53 | return m; 54 | } catch (IOException e) { 55 | log.error("Exception while iterating MessageSet:" + e.getMessage(), e); 56 | return null; 57 | } 58 | } 59 | 60 | @Override 61 | public void remove() { 62 | throw new UnsupportedOperationException("remove is not supported"); 63 | } 64 | }; 65 | } catch (Exception e) { 66 | log.error("Exception while reading: " + e.getMessage(), e); 67 | return new Iterator() { 68 | @Override 69 | public boolean hasNext() { 70 | return false; 71 | } 72 | @Override 73 | public Message next() { 74 | return null; 75 | } 76 | 77 | @Override 78 | public void remove() { 79 | throw new UnsupportedOperationException("remove is not supported"); 80 | } 81 | }; 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/SerDe.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | public interface SerDe { 4 | T deserialize(byte[] payload); 5 | 6 | byte[] serialize(T payload); 7 | 8 | String toString(byte[] payload); 9 | } 10 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/message/TMessageSet.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.message; 2 | 3 | import java.nio.ByteBuffer; 4 | 5 | public class TMessageSet { 6 | 7 | public String app; // required 8 | public int numMessages; // required 9 | public byte compression; // required 10 | public long crc; // required 11 | public ByteBuffer messages; // required 12 | 13 | public TMessageSet() { 14 | } 15 | 16 | public TMessageSet(String app, int numMessages, byte compression, long crc, 17 | ByteBuffer messages) { 18 | this.app = app; 19 | this.numMessages = numMessages; 20 | this.compression = compression; 21 | this.crc = crc; 22 | this.messages = messages; 23 | } 24 | 25 | public String getApp() { 26 | return app; 27 | } 28 | 29 | public void setApp(String app) { 30 | this.app = app; 31 | } 32 | 33 | public int getNumMessages() { 34 | return numMessages; 35 | } 36 | 37 | public void setNumMessages(int numMessages) { 38 | this.numMessages = numMessages; 39 | } 40 | 41 | public byte getCompression() { 42 | return compression; 43 | } 44 | 45 | public TMessageSet setCompression(byte compression) { 46 | this.compression = compression; 47 | return this; 48 | } 49 | 50 | public long getCrc() { 51 | return crc; 52 | } 53 | 54 | public void setCrc(long crc) { 55 | this.crc = crc; 56 | } 57 | 58 | public byte[] getMessages() { 59 | return messages == null ? null : messages.array(); 60 | } 61 | 62 | public TMessageSet setMessages(ByteBuffer messages) { 63 | this.messages = messages; 64 | return this; 65 | } 66 | 67 | public TMessageSet setMessages(byte[] messages) { 68 | setMessages(messages == null ? (ByteBuffer)null : ByteBuffer.wrap(messages)); 69 | return this; 70 | } 71 | 72 | public com.alibaba.rocketmq.common.message.Message toRocketMQMessage(String topic){ 73 | return new com.alibaba.rocketmq.common.message.Message(topic,getMessages()); 74 | } 75 | 76 | } 77 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/queue/FileBlockingQueue.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.queue; 2 | 3 | import com.google.common.base.Preconditions; 4 | import com.leansoft.bigqueue.BigArrayImpl; 5 | import com.leansoft.bigqueue.IBigArray; 6 | import com.leansoft.bigqueue.page.IMappedPage; 7 | import com.leansoft.bigqueue.page.IMappedPageFactory; 8 | import com.leansoft.bigqueue.page.MappedPageFactoryImpl; 9 | 10 | import me.zhenchuan.rmqc.message.SerDe; 11 | 12 | import org.slf4j.Logger; 13 | import org.slf4j.LoggerFactory; 14 | 15 | import java.io.IOException; 16 | import java.nio.ByteBuffer; 17 | import java.util.AbstractQueue; 18 | import java.util.Collection; 19 | import java.util.Iterator; 20 | import java.util.concurrent.BlockingQueue; 21 | import java.util.concurrent.Executors; 22 | import java.util.concurrent.TimeUnit; 23 | import java.util.concurrent.atomic.AtomicLong; 24 | import java.util.concurrent.locks.Condition; 25 | import java.util.concurrent.locks.Lock; 26 | import java.util.concurrent.locks.ReentrantLock; 27 | 28 | /** 29 | * File based blocking queue with BigQueue 30 | * @param Type name should be given and its SerDe should be implemented. 31 | * 32 | * With the argument path and name, files will be created under the directory 33 | * [path]/[name]. BigQueue needs to do garge collection, which is deleting 34 | * unnecessary page file. Garbage collection is done in the background every 35 | * gcPeriodInSec seconds. 36 | * 37 | * When the messages are retrieved from the queue, 38 | * we can control the behavior whether to remove messages immediately or wait 39 | * until we commit. autoCommit true means removing messages immediately. 40 | * 41 | * @author jbae 42 | */ 43 | public class FileBlockingQueue extends AbstractQueue implements BlockingQueue { 44 | private static final Logger log = LoggerFactory.getLogger(FileBlockingQueue.class); 45 | 46 | private final Lock lock = new ReentrantLock(); 47 | private final Condition notEmpty = lock.newCondition(); 48 | 49 | final IBigArray innerArray; 50 | // 2 ^ 3 = 8 51 | final static int QUEUE_FRONT_INDEX_ITEM_LENGTH_BITS = 3; 52 | // size in bytes of queue front index page 53 | final static int QUEUE_FRONT_INDEX_PAGE_SIZE = 1 << QUEUE_FRONT_INDEX_ITEM_LENGTH_BITS; 54 | // only use the first page 55 | static final long QUEUE_FRONT_PAGE_INDEX = 0; 56 | 57 | // folder name for queue front index page 58 | final static String QUEUE_FRONT_INDEX_PAGE_FOLDER = "front_index"; 59 | 60 | // front index of the big queue, 61 | final AtomicLong queueFrontIndex = new AtomicLong(); 62 | 63 | // factory for queue front index page management(acquire, release, cache) 64 | IMappedPageFactory queueFrontIndexPageFactory; 65 | 66 | private final SerDe serDe; 67 | private long consumedIndex; 68 | private final boolean autoCommit; 69 | 70 | public FileBlockingQueue( 71 | String path, 72 | String name, 73 | int gcPeriodInSec, 74 | SerDe serDe, 75 | boolean autoCommit) throws IOException { 76 | innerArray = new BigArrayImpl(path, name); 77 | // the ttl does not matter here since queue front index page is always cached 78 | this.queueFrontIndexPageFactory = new MappedPageFactoryImpl(QUEUE_FRONT_INDEX_PAGE_SIZE, 79 | ((BigArrayImpl)innerArray).getArrayDirectory() + QUEUE_FRONT_INDEX_PAGE_FOLDER, 80 | 10 * 1000/*does not matter*/); 81 | IMappedPage queueFrontIndexPage = this.queueFrontIndexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 82 | 83 | ByteBuffer queueFrontIndexBuffer = queueFrontIndexPage.getLocal(0); 84 | long front = queueFrontIndexBuffer.getLong(); 85 | queueFrontIndex.set(front); 86 | 87 | consumedIndex = front; 88 | 89 | Executors.newScheduledThreadPool(1).scheduleAtFixedRate(new Runnable() { 90 | @Override 91 | public void run() { 92 | try { 93 | gc(); 94 | } catch (IOException e) { 95 | log.error("IOException while gc: " + e.getMessage(), e); 96 | } 97 | } 98 | }, gcPeriodInSec, gcPeriodInSec, TimeUnit.SECONDS); 99 | 100 | this.serDe = serDe; 101 | this.autoCommit = autoCommit; 102 | } 103 | 104 | private void gc() throws IOException { 105 | long beforeIndex = this.queueFrontIndex.get(); 106 | if (beforeIndex > 0L) { // wrap 107 | beforeIndex--; 108 | try { 109 | this.innerArray.removeBeforeIndex(beforeIndex); 110 | } catch (IndexOutOfBoundsException e) { 111 | log.error("Exception on gc: " + e.getMessage(), e); 112 | } 113 | } 114 | } 115 | 116 | public void commit() { 117 | try { 118 | lock.lock(); 119 | commitInternal(true); 120 | } catch (IOException e) { 121 | log.error("IOException on commit: " + e.getMessage(), e); 122 | } finally { 123 | lock.unlock(); 124 | } 125 | } 126 | 127 | private void commitInternal(boolean doCommit) throws IOException { 128 | if (!doCommit) return; 129 | 130 | this.queueFrontIndex.set(consumedIndex); 131 | // persist the queue front 132 | IMappedPage queueFrontIndexPage = null; 133 | queueFrontIndexPage = this.queueFrontIndexPageFactory.acquirePage(QUEUE_FRONT_PAGE_INDEX); 134 | ByteBuffer queueFrontIndexBuffer = queueFrontIndexPage.getLocal(0); 135 | queueFrontIndexBuffer.putLong(consumedIndex); 136 | queueFrontIndexPage.setDirty(true); 137 | } 138 | 139 | public void close() { 140 | try { 141 | gc(); 142 | if (this.queueFrontIndexPageFactory != null) { 143 | this.queueFrontIndexPageFactory.releaseCachedPages(); 144 | } 145 | 146 | this.innerArray.close(); 147 | } catch(IOException e) { 148 | log.error("IOException while closing: " + e.getMessage(), e); 149 | } 150 | } 151 | 152 | @Override 153 | public E poll() { 154 | if (isEmpty()) { 155 | return null; 156 | } 157 | E x = null; 158 | lock.lock(); 159 | try { 160 | if (!isEmpty()) { 161 | x = consumeElement(); 162 | if (!isEmpty()) { 163 | notEmpty.signal(); 164 | } 165 | } 166 | } catch (IOException e) { 167 | log.error("IOException while poll: " + e.getMessage(), e); 168 | return null; 169 | } finally { 170 | lock.unlock(); 171 | } 172 | 173 | return x; 174 | } 175 | 176 | @Override 177 | public E peek() { 178 | if (isEmpty()) { 179 | return null; 180 | } 181 | lock.lock(); 182 | try { 183 | return consumeElement(); 184 | } catch (IOException e) { 185 | log.error("IOException while peek: " + e.getMessage(), e); 186 | return null; 187 | } finally { 188 | lock.unlock(); 189 | } 190 | } 191 | 192 | @Override 193 | public boolean offer(E e) { 194 | Preconditions.checkNotNull(e); 195 | try { 196 | innerArray.append(serDe.serialize(e)); 197 | signalNotEmpty(); 198 | return true; 199 | } catch (IOException ex) { 200 | return false; 201 | } 202 | } 203 | 204 | @Override 205 | public void put(E e) throws InterruptedException { 206 | offer(e); 207 | } 208 | 209 | @Override 210 | public boolean offer(E e, long timeout, TimeUnit unit) throws InterruptedException { 211 | return offer(e); 212 | } 213 | 214 | @Override 215 | public E take() throws InterruptedException { 216 | E x; 217 | lock.lockInterruptibly(); 218 | try { 219 | while (isEmpty()) { 220 | notEmpty.await(); 221 | } 222 | x = consumeElement(); 223 | } catch (IOException e) { 224 | log.error("IOException on take: " + e.getMessage(), e); 225 | return null; 226 | } finally { 227 | lock.unlock(); 228 | } 229 | 230 | return x; 231 | } 232 | 233 | private E consumeElement() throws IOException { 234 | // restore consumedIndex if not committed 235 | consumedIndex = this.queueFrontIndex.get(); 236 | E x = serDe.deserialize(innerArray.get(consumedIndex)); 237 | ++consumedIndex; 238 | commitInternal(autoCommit); 239 | return x; 240 | } 241 | 242 | @Override 243 | public E poll(long timeout, TimeUnit unit) throws InterruptedException { 244 | E x = null; 245 | long nanos = unit.toNanos(timeout); 246 | lock.lockInterruptibly(); 247 | try { 248 | while (isEmpty()) { 249 | if (nanos <= 0) 250 | return null; 251 | nanos = notEmpty.awaitNanos(nanos); 252 | } 253 | x = consumeElement(); 254 | } catch (IOException e) { 255 | log.error("IOException on poll: " + e.getMessage(), e); 256 | return null; 257 | } finally { 258 | lock.unlock(); 259 | } 260 | 261 | return x; 262 | } 263 | 264 | @Override 265 | public int remainingCapacity() { 266 | return Integer.MAX_VALUE; 267 | } 268 | 269 | @Override 270 | public int drainTo(Collection c) { 271 | return drainTo(c, Integer.MAX_VALUE); 272 | } 273 | 274 | @Override 275 | public int drainTo(Collection c, int maxElements) { 276 | if (c == null) 277 | throw new NullPointerException(); 278 | if (c == this) 279 | throw new IllegalArgumentException(); 280 | 281 | lock.lock(); 282 | // restore consumedIndex if not committed 283 | consumedIndex = this.queueFrontIndex.get(); 284 | try { 285 | int n = Math.min(maxElements, size()); 286 | // count.get provides visibility to first n Nodes 287 | int i = 0; 288 | while (i < n && consumedIndex < innerArray.getHeadIndex()) { 289 | c.add(serDe.deserialize(innerArray.get(consumedIndex))); 290 | ++consumedIndex; 291 | ++i; 292 | } 293 | commitInternal(autoCommit); 294 | return n; 295 | } catch (Exception e) { 296 | throw new RuntimeException(e); 297 | } finally { 298 | lock.unlock(); 299 | } 300 | } 301 | 302 | @Override 303 | public Iterator iterator() { 304 | return new Iterator() { 305 | 306 | @Override 307 | public boolean hasNext() { 308 | return !isEmpty(); 309 | } 310 | 311 | @Override 312 | public E next() { 313 | try { 314 | E x = consumeElement(); 315 | return x; 316 | } catch (IOException e) { 317 | throw new RuntimeException(e); 318 | } 319 | } 320 | 321 | @Override 322 | public void remove() { 323 | throw new UnsupportedOperationException("remove is not supported, use dequeue()"); 324 | } 325 | }; 326 | } 327 | 328 | @Override 329 | public int size() { 330 | return (int)(innerArray.getHeadIndex() - queueFrontIndex.get()); 331 | } 332 | 333 | private void signalNotEmpty() { 334 | lock.lock(); 335 | try { 336 | notEmpty.signal(); 337 | } finally { 338 | lock.unlock(); 339 | } 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/queue/FileQueue4Sink.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.queue; 2 | 3 | 4 | import me.zhenchuan.rmqc.message.Message; 5 | import me.zhenchuan.rmqc.message.MessageSerDe; 6 | 7 | import org.joda.time.Period; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.io.IOException; 12 | import java.util.List; 13 | import java.util.concurrent.TimeUnit; 14 | 15 | /** 16 | * File based {@link MessageQueue4Sink}, delegating actual logic to {@link FileBlockingQueue} 17 | * NOTE !!! NotThreadSafe 18 | * @author jbae 19 | */ 20 | public class FileQueue4Sink implements MessageQueue4Sink { 21 | private static final Logger log = LoggerFactory.getLogger(FileQueue4Sink.class); 22 | 23 | public static final String TYPE = "file"; 24 | 25 | private final FileBlockingQueue queue; 26 | 27 | public FileQueue4Sink( 28 | String path, 29 | String name, 30 | String gcPeriod) throws IOException { 31 | queue = new FileBlockingQueue( 32 | path, 33 | name, 34 | new Period(gcPeriod == null ? "PT1h" : gcPeriod).toStandardSeconds().getSeconds(), 35 | new MessageSerDe(), 36 | true); // auto-commit needed because of message copy 37 | } 38 | 39 | @Override 40 | public boolean offer(Message msg) { 41 | try { 42 | return queue.offer(msg); 43 | } catch (Exception e) { 44 | log.error("Exception on offer: " + e.getMessage(), e); 45 | return false; 46 | } 47 | } 48 | 49 | @Override 50 | public int drain(int batchSize, List msgList) { 51 | return queue.drainTo(msgList, batchSize); 52 | } 53 | 54 | @Override 55 | public Message poll(long timeout, TimeUnit unit) throws InterruptedException { 56 | return queue.poll(timeout, unit); 57 | } 58 | 59 | @Override 60 | public void close() { 61 | queue.close(); 62 | } 63 | 64 | @Override 65 | public boolean isEmpty() { 66 | return queue.isEmpty(); 67 | } 68 | 69 | @Override 70 | public long size() { 71 | return queue.size(); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/queue/MemoryQueue4Sink.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.queue; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.ArrayBlockingQueue; 5 | import java.util.concurrent.BlockingQueue; 6 | import java.util.concurrent.TimeUnit; 7 | 8 | import me.zhenchuan.rmqc.message.Message; 9 | 10 | 11 | /** 12 | * Memory based {@link MessageQueue4Sink}, delegating actual queueing work to {@link ArrayBlockingQueue} 13 | * NOTE !!! NotThreadSafe 14 | * @author jbae 15 | */ 16 | public class MemoryQueue4Sink implements MessageQueue4Sink { 17 | public static final String TYPE = "memory"; 18 | 19 | private final BlockingQueue queue; 20 | 21 | public MemoryQueue4Sink(int capacity) { 22 | this.queue = new ArrayBlockingQueue(capacity); 23 | } 24 | 25 | @Override 26 | public boolean offer(Message msg) { 27 | return queue.offer(msg); 28 | } 29 | 30 | @Override 31 | public Message poll(long timeout, TimeUnit timeUnit) throws InterruptedException { 32 | return queue.poll(timeout, timeUnit); 33 | } 34 | 35 | @Override 36 | public int drain(int batchSize, List msgList) { 37 | return queue.drainTo(msgList, batchSize); 38 | } 39 | 40 | @Override 41 | public void close() { 42 | // do nothing 43 | } 44 | 45 | @Override 46 | public boolean isEmpty() { 47 | return queue.isEmpty(); 48 | } 49 | 50 | @Override 51 | public long size() { 52 | return queue.size(); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/queue/MessageQueue4Sink.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.queue; 2 | 3 | import java.util.List; 4 | import java.util.concurrent.TimeUnit; 5 | 6 | import me.zhenchuan.rmqc.message.Message; 7 | 8 | 9 | public interface MessageQueue4Sink { 10 | 11 | boolean offer(Message msg); 12 | Message poll(long timeout, TimeUnit timeUnit) throws InterruptedException; 13 | /**** 14 | * 从queue中获取最多batchSize个message到msgList中. 15 | * @param batchSize 16 | * @param msgList 17 | * @return 18 | */ 19 | int drain(int batchSize, List msgList); 20 | void close(); 21 | boolean isEmpty(); 22 | long size(); 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/me/zhenchuan/rmqc/queue/Queue4Client.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc.queue; 2 | 3 | 4 | import me.zhenchuan.rmqc.ClientConfig; 5 | import me.zhenchuan.rmqc.message.Message; 6 | 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.IOException; 11 | import java.util.List; 12 | import java.util.concurrent.TimeUnit; 13 | 14 | /** 15 | * 根据配置来决定来wrapper哪一种Queue 16 | */ 17 | public class Queue4Client { 18 | private static final Logger logger = LoggerFactory.getLogger(Queue4Client.class); 19 | 20 | private MessageQueue4Sink queue; 21 | 22 | 23 | public Queue4Client(ClientConfig config) { 24 | if (config.asyncQueueType().equals("memory")) { 25 | queue = new MemoryQueue4Sink(config.asyncMemoryQueueCapacity()); 26 | } else { 27 | try { 28 | queue = new FileQueue4Sink( 29 | config.asyncFileQueuePath(), 30 | config.asyncFileQueueName(), 31 | config.asyncFileQueueGCPeriod()); 32 | } catch (IOException e) { 33 | logger.error("Exception on initializing Queue4Client: " + e.getMessage(), e); 34 | } 35 | } 36 | } 37 | 38 | public boolean offer(Message msg) { 39 | return queue.offer(msg); 40 | } 41 | 42 | public Message poll(long timeout, TimeUnit timeUnit) throws InterruptedException { 43 | return queue.poll(timeout, timeUnit); 44 | } 45 | 46 | public int drain(int batchSize, List msgList) { 47 | return queue.drain(batchSize, msgList); 48 | } 49 | 50 | public void close() { 51 | queue.close(); 52 | } 53 | 54 | public boolean isEmpty() { 55 | return queue.isEmpty(); 56 | } 57 | 58 | public long size() { 59 | return queue.size(); 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /src/main/resources/producer.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/zhenchuan/RocketMQClient/f0e9eace160f119177bb2f2bc10b2d3d505e093c/src/main/resources/producer.properties -------------------------------------------------------------------------------- /src/test/java/me/zhenchuan/rmqc/ClientTest.java: -------------------------------------------------------------------------------- 1 | package me.zhenchuan.rmqc; 2 | 3 | import java.util.Properties; 4 | 5 | import me.zhenchuan.rmqc.message.Message; 6 | 7 | import org.apache.commons.lang.RandomStringUtils; 8 | import org.junit.Test; 9 | 10 | 11 | public class ClientTest { 12 | 13 | private static final int bench_num = 100000; 14 | 15 | private static final String b1024 = RandomStringUtils.randomAscii(1024); 16 | private static final String b512 = RandomStringUtils.randomAscii(512); 17 | private static final String b256 = RandomStringUtils.randomAscii(256); 18 | 19 | @Test 20 | public void testSendWithAsyncFileBrokerOffline() throws Exception{ 21 | Properties properties = new Properties(); 22 | properties.put("client.type", "async"); 23 | properties.put("async.queue.type", "file"); 24 | 25 | RMQClient client = new RMQClient(properties); 26 | Message message = new Message("routekey",b1024.getBytes()); 27 | long s = System.currentTimeMillis(); 28 | for(int i = 0 ; i