filterData : multiFilters.getRight()) {
94 | String extVal = extJsonHeader.get(filterData.getKey());
95 | match = EXACT_MATCH_FILTER.apply(filterData.getRight(), extVal);
96 | if (match && relation == NSQConfig.MultiFiltersRelation.any) {
97 | return true;
98 | }
99 | if (!match && relation == NSQConfig.MultiFiltersRelation.all) {
100 | return false;
101 | }
102 | }
103 | return match;
104 | }
105 |
106 | @Override
107 | public int getType() {
108 | return 3;
109 | }
110 | };
111 | }
112 |
113 |
--------------------------------------------------------------------------------
/src/test/java/com/youzan/nsq/client/network/frame/TestMessageFrame.java:
--------------------------------------------------------------------------------
1 | package com.youzan.nsq.client.network.frame;
2 |
3 | import org.testng.Assert;
4 | import org.testng.annotations.Test;
5 |
6 | import java.nio.ByteBuffer;
7 | import java.util.Random;
8 |
9 | /**
10 | * http://nsq.io/clients/tcp_protocol_spec.html#data-format
11 | *
12 | * @author zhaoxi (linzuxiong)
13 | */
14 | public class TestMessageFrame {
15 | private Random _r = new Random();
16 |
17 | @Test
18 | public void newInstance() {
19 | MessageFrame frame = new MessageFrame();
20 | }
21 |
22 | @Test
23 | public void setData() {
24 | MessageFrame frame = new MessageFrame();
25 | int size = 100;
26 | int dataSize = size - 4;
27 | // size + type + data
28 | // timestamp + attempts + ID + messageBody
29 |
30 | byte[] timestamp = new byte[8];
31 | byte[] attempts = new byte[2];
32 | byte[] messageID = new byte[16];
33 | byte[] messageBody = new byte[dataSize - 8 - 2 - 16];
34 | _r.nextBytes(timestamp);
35 | _r.nextBytes(attempts);
36 | _r.nextBytes(messageID);
37 | _r.nextBytes(messageBody);
38 |
39 | ByteBuffer bb = java.nio.ByteBuffer.allocate(dataSize);
40 | bb.put(timestamp);
41 | bb.put(attempts);
42 | bb.put(messageID);
43 | bb.put(messageBody);
44 | byte[] data = bb.array();
45 |
46 | frame.setSize(dataSize);
47 | frame.setData(data);
48 |
49 | Assert.assertEquals(frame.getTimestamp(), timestamp);
50 | Assert.assertEquals(frame.getAttempts(), attempts);
51 | Assert.assertEquals(frame.getMessageID(), messageID);
52 | Assert.assertEquals(frame.getMessageBody(), messageBody);
53 | }
54 |
55 | @Test(expectedExceptions = Exception.class)
56 | public void dataException() {
57 | MessageFrame frame = new MessageFrame();
58 | int size = 29;
59 | int dataSize = size - 4;
60 | // size + type + data
61 | // timestamp + attempts + ID + messageBody
62 |
63 | byte[] timestamp = new byte[8];
64 | byte[] attempts = new byte[2];
65 | byte[] messageID = new byte[16];
66 | byte[] messageBody = new byte[dataSize - 8 - 2 - 16];
67 | _r.nextBytes(timestamp);
68 | _r.nextBytes(attempts);
69 | _r.nextBytes(messageID);
70 | _r.nextBytes(messageBody);
71 |
72 | ByteBuffer bb = java.nio.ByteBuffer.allocate(dataSize);
73 | bb.put(timestamp);
74 | bb.put(attempts);
75 | bb.put(messageID);
76 | bb.put(messageBody);
77 | byte[] data = bb.array();
78 |
79 | frame.setSize(dataSize);
80 | frame.setData(data);
81 |
82 | }
83 |
84 | @Test
85 | public void dataBoundary() {
86 | MessageFrame frame = new MessageFrame();
87 | int size = 30;
88 | int dataSize = size - 4;
89 | // size + type + data
90 | // timestamp + attempts + ID + messageBody
91 |
92 | byte[] timestamp = new byte[8];
93 | byte[] attempts = new byte[2];
94 | byte[] messageID = new byte[16];
95 | byte[] messageBody = new byte[dataSize - 8 - 2 - 16];
96 | _r.nextBytes(timestamp);
97 | _r.nextBytes(attempts);
98 | _r.nextBytes(messageID);
99 | _r.nextBytes(messageBody);
100 |
101 | ByteBuffer bb = java.nio.ByteBuffer.allocate(dataSize);
102 | bb.put(timestamp);
103 | bb.put(attempts);
104 | bb.put(messageID);
105 | bb.put(messageBody);
106 | byte[] data = bb.array();
107 |
108 | frame.setSize(dataSize);
109 | frame.setData(data);
110 | }
111 |
112 | }
113 |
--------------------------------------------------------------------------------
/src/test/java/com/youzan/nsq/client/utils/Lz4Util.java:
--------------------------------------------------------------------------------
1 | package com.youzan.nsq.client.utils; /**
2 | * Created by lin on 17/6/30.
3 | */
4 |
5 | import net.jpountz.lz4.LZ4Compressor;
6 | import net.jpountz.lz4.LZ4Factory;
7 | import net.jpountz.lz4.LZ4FastDecompressor;
8 | import net.jpountz.lz4.LZ4SafeDecompressor;
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 |
12 | /**
13 | * Created by suguoqing on 16/6/8.
14 | */
15 | public class Lz4Util {
16 | private static final Logger LOGGER = LoggerFactory.getLogger(Lz4Util.class);
17 | private static LZ4Factory factory = LZ4Factory.fastestInstance();
18 | private static LZ4SafeDecompressor decompressor = factory.safeDecompressor();
19 | private static LZ4FastDecompressor fastDecompressor = factory.fastDecompressor();
20 | private static LZ4Compressor compressor = factory.fastCompressor();
21 |
22 | /**
23 | * 压缩一个 data 数组
24 | * @param data 原始数据
25 | * @return 压缩后的数据
26 | */
27 | public static byte[] compress(byte[] data){
28 | final int decompressedLength = data.length;
29 |
30 | // compress data
31 | int maxCompressedLength = compressor.maxCompressedLength(decompressedLength);
32 | byte[] compressed = new byte[maxCompressedLength];
33 | int compressedLength = compressor.compress(data, 0, decompressedLength, compressed, 0, maxCompressedLength);
34 |
35 | byte[] result = new byte[compressedLength];
36 | System.arraycopy(compressed, 0, result, 0, compressedLength);
37 | return result;
38 | }
39 |
40 | /**
41 | * 快速解压一个数组
42 | * @param compressedArray 压缩后的数据
43 | * @param orinArrayLength
44 | * @param extInfo
45 | * @return
46 | */
47 | public static byte[] fastDecopress(final byte[] compressedArray, int orinArrayLength, String... extInfo) {
48 | try {
49 | return fastDecompressor.decompress(compressedArray, orinArrayLength);
50 | } catch (Exception e) {
51 | LOGGER.error("fast decompress Error:extInfo ={} ", extInfo, e);
52 | return null;
53 | }
54 | }
55 |
56 | /**
57 | * 解压一个数组
58 | * @param finalCompressedArray 原始数组
59 | * @param decompressLength 数组压缩前的长度
60 | * @param extInfo 异常信息
61 | * @return
62 | */
63 | public static byte[] decompress( byte[] finalCompressedArray, Integer decompressLength, String ... extInfo) {
64 | int ratio = 3;
65 | if (null != decompressLength) {
66 | // 知道长度用fastCompress
67 | return fastDecopress(finalCompressedArray, decompressLength, extInfo);
68 | } else {
69 | decompressLength = finalCompressedArray.length * ratio;
70 | }
71 |
72 | int i = 5;
73 | while (i > 0) {
74 | try {
75 | return decompress(finalCompressedArray, decompressLength);
76 | } catch (Exception e) {
77 | ratio = ratio * 2;
78 | i--;
79 | if (LOGGER.isInfoEnabled()) {
80 | LOGGER.info("decompress Error:ratio ={} extInfo ={} ", ratio, extInfo, e);
81 | }
82 |
83 | }
84 |
85 | }
86 |
87 | throw new RuntimeException("decompress error");
88 | }
89 |
90 | /**
91 | * 解压一个数组
92 | *
93 | * @param finalCompressedArray 压缩后的数据
94 | * @param length 原始数据长度, 长度不能小于压缩前的长度。
95 | * @return
96 | */
97 | private static byte[] decompress(byte[] finalCompressedArray, int length) {
98 |
99 | byte[] desc = new byte[length];
100 | int decompressLen = decompressor.decompress(finalCompressedArray, desc);
101 |
102 | byte[] result = new byte[decompressLen];
103 | System.arraycopy(desc, 0, result, 0, decompressLen);
104 | return result;
105 | }
106 |
107 | }
--------------------------------------------------------------------------------
/src/main/java/com/youzan/nsq/client/network/frame/OrderedMessageFrame.java:
--------------------------------------------------------------------------------
1 | package com.youzan.nsq.client.network.frame;
2 |
3 | import java.nio.ByteBuffer;
4 |
5 | /**
6 | * Created by lin on 16/9/26.
7 | */
8 | public class OrderedMessageFrame extends MessageFrame{
9 | /**
10 | * 8-byte : disk queue offset
11 | */
12 | final private byte[] diskQueueOffset = new byte[8];
13 | /**
14 | * 4-byte : disk queue data size
15 | */
16 | final private byte[] diskQueueDataSize = new byte[4];
17 |
18 | @Override
19 | public void setData(byte[] bytes, boolean shouldExt) {
20 | System.arraycopy(bytes, 0, timestamp, 0, 8);
21 | System.arraycopy(bytes, 8, attempts, 0, 2);
22 |
23 | //Sub Ordered incoming extra info, disk queue offset & disk queue data size
24 | System.arraycopy(bytes, 10, messageID, 0, 16);
25 | System.arraycopy(bytes, 10, internalID, 0, 8);
26 | System.arraycopy(bytes, 18, traceID, 0, 8);
27 |
28 | int messageBodyStart;
29 | int messageBodySize;
30 | if(!shouldExt) {
31 | messageBodyStart = 26;//8 + 2 + 16;
32 | } else {
33 | //read ext content & length here
34 | //version
35 | System.arraycopy(bytes, 26, extVerBytes, 0, 1);
36 | int extVer = (int)extVerBytes[0];
37 | switch (extVer) {
38 | case 0:
39 | messageBodyStart = 27;//8+2+16+1
40 | break;
41 | default:
42 | byte[] extBytesLenBytes = new byte[2];
43 | //ext content length
44 | System.arraycopy(bytes, 27, extBytesLenBytes, 0, 2);
45 | int extBytesLen = ByteBuffer.wrap(extBytesLenBytes).getShort();
46 | //allocate
47 | extBytes = new byte[extBytesLen];
48 | System.arraycopy(bytes, 29, extBytes, 0, extBytesLen);
49 | messageBodyStart = 29 + extBytesLen;//8 + 2 + 16 + 1 + 2 + extBytesLen;
50 | }
51 | }
52 |
53 | System.arraycopy(bytes, messageBodyStart, diskQueueOffset, 0, 8);
54 | System.arraycopy(bytes, messageBodyStart + 8, diskQueueDataSize, 0, 4);
55 |
56 | messageBodySize = bytes.length - (messageBodyStart + 8 + 4);
57 | messageBody = new byte[messageBodySize];
58 |
59 | System.arraycopy(bytes, messageBodyStart + 8 + 4, messageBody, 0, messageBodySize);
60 | }
61 |
62 | @Override
63 | public void setData(byte[] bytes) {
64 | //capability of message array bytes.length - (8 + 2 + 8 + 4 + 16)
65 | final int messageBodySize = bytes.length - (38);
66 | messageBody = new byte[messageBodySize];
67 | System.arraycopy(bytes, 0, timestamp, 0, 8);
68 | System.arraycopy(bytes, 8, attempts, 0, 2);
69 | //Sub Ordered incoming extra info, disk queue offset & disk queue data size
70 | System.arraycopy(bytes, 10, messageID, 0, 16);
71 | System.arraycopy(bytes, 10, internalID, 0, 8);
72 | System.arraycopy(bytes, 18, traceID, 0, 8);
73 |
74 | System.arraycopy(bytes, 26, diskQueueOffset, 0, 8);
75 | System.arraycopy(bytes, 34, diskQueueDataSize, 0, 4);
76 |
77 |
78 | System.arraycopy(bytes, 38, messageBody, 0, messageBodySize);
79 | }
80 |
81 | /**
82 | * function to get diskQueueOffset of current msg, diskqueue has meaning only when message contains advanced info of
83 | * SUB
84 | * @return diskQueueOffSet (int 64) in byte[]
85 | */
86 | public byte[] getDiskQueueOffset(){
87 | return this.diskQueueOffset;
88 | }
89 |
90 | /**
91 | * function to get diskQueueDataSize of current msg, disk queue dat size has meaning only when message contains
92 | * advanced info of SUB
93 | * @return diskQueueDataSize (int 64) in byte[]
94 | */
95 | public byte[] getDiskQueueDataSize(){
96 | return this.diskQueueDataSize;
97 | }
98 | }
99 |
--------------------------------------------------------------------------------
/src/main/java/com/youzan/util/Lists.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | * Copy from https://github.com/google/guava/blob/master/guava/src/com/google/common/collect/Lists.java
4 | * It's Apache License.
5 | *
6 | */
7 | package com.youzan.util;
8 |
9 | import org.slf4j.Logger;
10 | import org.slf4j.LoggerFactory;
11 |
12 | import java.math.RoundingMode;
13 | import java.util.AbstractList;
14 | import java.util.List;
15 | import java.util.RandomAccess;
16 |
17 | /**
18 | * @author zhaoxi (linzuxiong)
19 | *
20 | *
21 | */
22 | public class Lists {
23 | private static final Logger logger = LoggerFactory.getLogger(Lists.class);
24 |
25 | public static List> partition(List list, int size) {
26 | if (list == null || size <= 0) {
27 | throw new IllegalArgumentException("Your input is blank!");
28 | }
29 | return (list instanceof RandomAccess) ? new RandomAccessPartition(list, size) : new Partition(list, size);
30 | }
31 |
32 | private static class Partition extends AbstractList> {
33 | final List list;
34 | final int size;
35 |
36 | Partition(List list, int size) {
37 | this.list = list;
38 | this.size = size;
39 | }
40 |
41 | @Override
42 | public List get(int index) {
43 | if (index < 0 || index >= size()) {
44 | throw new IndexOutOfBoundsException(String.format("Size: %s, Index: %s", size(), index));
45 | }
46 | int start = index * size;
47 | int end = Math.min(start + size, list.size());
48 | return list.subList(start, end);
49 | }
50 |
51 | @Override
52 | public int size() {
53 | return divideInt(list.size(), size, RoundingMode.CEILING);
54 | }
55 |
56 | @Override
57 | public boolean isEmpty() {
58 | return list.isEmpty();
59 | }
60 | }
61 |
62 | private static class RandomAccessPartition extends Partition implements RandomAccess {
63 | RandomAccessPartition(List list, int size) {
64 | super(list, size);
65 | }
66 | }
67 |
68 | /**
69 | * Returns the base-2 logarithm of {@code x}, rounded according to the
70 | * specified rounding mode.
71 | *
72 | * @param p
73 | * the size
74 | * @param q
75 | * the small batch size
76 | * @param mode
77 | * RoundingMode
78 | *
79 | * @return batches
80 | * @throws IllegalArgumentException
81 | * if {@code x <= 0}
82 | * @throws ArithmeticException
83 | * if {@code mode} is {@link RoundingMode#UNNECESSARY} and
84 | * {@code x} is not a power of two
85 | */
86 | public static int divideInt(int p, int q, RoundingMode mode) {
87 | if (q == 0) {
88 | throw new ArithmeticException("/ by zero"); // for GWT
89 | }
90 | int div = p / q;
91 | int rem = p - q * div; // equal to p % q
92 |
93 | if (rem == 0) {
94 | return div;
95 | }
96 | int signum = 1 | ((p ^ q) >> (Integer.SIZE - 1));
97 | boolean increment;
98 | switch (mode) {
99 | case DOWN:
100 | increment = false;
101 | break;
102 | case UP:
103 | increment = true;
104 | break;
105 | case CEILING:
106 | increment = signum > 0;
107 | break;
108 | case FLOOR:
109 | increment = signum < 0;
110 | break;
111 | case HALF_EVEN:
112 | case HALF_DOWN:
113 | default:
114 | throw new AssertionError();
115 | }
116 | return increment ? div + signum : div;
117 | }
118 | }
119 |
--------------------------------------------------------------------------------
/src/test/java/com/youzan/nsq/client/utils/ConnectionUtil.java:
--------------------------------------------------------------------------------
1 | package com.youzan.nsq.client.utils;
2 |
3 | import com.youzan.nsq.client.MockedNSQConnectionImpl;
4 | import com.youzan.nsq.client.MockedNSQSimpleClient;
5 | import com.youzan.nsq.client.core.Client;
6 | import com.youzan.nsq.client.core.NSQConnection;
7 | import com.youzan.nsq.client.core.NSQSimpleClient;
8 | import com.youzan.nsq.client.core.command.Identify;
9 | import com.youzan.nsq.client.core.command.Magic;
10 | import com.youzan.nsq.client.core.command.Rdy;
11 | import com.youzan.nsq.client.core.command.Sub;
12 | import com.youzan.nsq.client.entity.Address;
13 | import com.youzan.nsq.client.entity.NSQConfig;
14 | import com.youzan.nsq.client.entity.Role;
15 | import com.youzan.nsq.client.entity.Topic;
16 | import com.youzan.nsq.client.exception.NSQNoConnectionException;
17 | import com.youzan.nsq.client.network.frame.NSQFrame;
18 | import com.youzan.nsq.client.network.netty.NSQClientInitializer;
19 | import io.netty.bootstrap.Bootstrap;
20 | import io.netty.channel.*;
21 | import io.netty.channel.nio.NioEventLoopGroup;
22 | import io.netty.channel.socket.nio.NioSocketChannel;
23 |
24 | import java.util.concurrent.CountDownLatch;
25 | import java.util.concurrent.ExecutionException;
26 | import java.util.concurrent.TimeUnit;
27 | import java.util.concurrent.TimeoutException;
28 |
29 | /**
30 | * Created by lin on 17/8/11.
31 | */
32 | public class ConnectionUtil {
33 |
34 | private static Bootstrap bootstrap;
35 | private static EventLoopGroup eventLoopGroup;
36 |
37 | static{
38 | //netty setup
39 | bootstrap = new Bootstrap();
40 | eventLoopGroup = new NioEventLoopGroup(Runtime.getRuntime().availableProcessors() * 2);
41 | bootstrap.option(ChannelOption.SO_KEEPALIVE, true);
42 | bootstrap.option(ChannelOption.TCP_NODELAY, true);
43 | bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, 500);
44 | bootstrap.group(eventLoopGroup);
45 | bootstrap.channel(NioSocketChannel.class);
46 | bootstrap.handler(new NSQClientInitializer());
47 | }
48 |
49 | public static NSQConnection connect(Address addr, String channel, NSQConfig config) throws InterruptedException, TimeoutException, NSQNoConnectionException, ExecutionException {
50 | ChannelFuture chFuture = bootstrap.connect(addr.getHost(), addr.getPort());
51 | final CountDownLatch connLatch = new CountDownLatch(1);
52 | chFuture.addListener(new ChannelFutureListener() {
53 | @Override
54 | public void operationComplete(ChannelFuture channelFuture) throws Exception {
55 | if (channelFuture.isSuccess())
56 | connLatch.countDown();
57 | }
58 | });
59 | connLatch.await(500, TimeUnit.MILLISECONDS);
60 | Channel ch = chFuture.channel();
61 | MockedNSQConnectionImpl con1 = new MockedNSQConnectionImpl(0, addr, ch, config);
62 | con1.setTopic(new Topic(addr.getTopic(), addr.getPartition()));
63 | NSQSimpleClient simpleClient = new MockedNSQSimpleClient(Role.Consumer, false);
64 | ch.attr(Client.STATE).set(simpleClient);
65 | ch.attr(NSQConnection.STATE).set(con1);
66 | con1.command(Magic.getInstance());
67 | NSQFrame resp = con1.commandAndGetResponse(null, new Identify(config, addr.isTopicExtend()));
68 | if (null == resp) {
69 | throw new IllegalStateException("Bad Identify Response!");
70 | }
71 | Thread.sleep(100);
72 | resp = con1.commandAndGetResponse(null, new Sub(new Topic(addr.getTopic(), addr.getPartition()), channel));
73 | if (null == resp) {
74 | throw new IllegalStateException("Bad Identify Response!");
75 | }
76 | if (resp.getType() == NSQFrame.FrameType.ERROR_FRAME) {
77 | throw new IllegalStateException(resp.getMessage());
78 | }
79 | con1.subSent();
80 | Thread.sleep(100);
81 | con1.command(new Rdy(1));
82 | Thread.sleep(100);
83 |
84 | return con1;
85 | }
86 | }
87 |
--------------------------------------------------------------------------------
/src/main/java/com/youzan/nsq/client/core/command/PubExt.java:
--------------------------------------------------------------------------------
1 | package com.youzan.nsq.client.core.command;
2 |
3 | import com.fasterxml.jackson.core.JsonProcessingException;
4 | import com.fasterxml.jackson.databind.JsonNode;
5 | import com.fasterxml.jackson.databind.node.ObjectNode;
6 | import com.youzan.nsq.client.entity.Message;
7 | import com.youzan.util.SystemUtil;
8 |
9 | import java.io.IOException;
10 | import java.nio.ByteBuffer;
11 | import java.util.IllegalFormatException;
12 |
13 | /**
14 | * PubExt command
15 | * Created by lin on 17/7/29.
16 | */
17 | public class PubExt extends Pub {
18 | private byte[] jsonHeaderBytes;
19 | public static final String CLIENT_TAG_KEY = "##client_dispatch_tag";
20 | public static final String TRACE_ID_KEY = "##trace_id";
21 | public static final String FILTER_EXT_KEY = "filter_ext_key";
22 | public static final String FILTER_DATA = "filter_data";
23 |
24 |
25 | /**
26 | * @param msg message object
27 | */
28 | public PubExt(final Message msg, boolean trace) throws IllegalFormatException {
29 | super(msg);
30 | String clientTag = msg.getDesiredTag();
31 | boolean jsonHeaderNeeded = (null != clientTag && !clientTag.isEmpty()) || (trace);
32 |
33 | Object jsonObj = msg.getJsonHeaderExt();
34 | ObjectNode jsonHeaderExt = null;
35 | //create one json
36 | if (null == jsonObj && jsonHeaderNeeded) {
37 | jsonHeaderExt = SystemUtil.getObjectMapper().createObjectNode();
38 | if(null != clientTag)
39 | jsonHeaderExt.put(CLIENT_TAG_KEY, clientTag);
40 | if(trace)
41 | jsonHeaderExt.put(TRACE_ID_KEY, msg.getTraceIDStr());
42 | } else if (null != jsonObj) {
43 | //parse message json header ext
44 | try {
45 | String jsonStr = SystemUtil.getObjectMapper().writeValueAsString(jsonObj);
46 | JsonNode json = SystemUtil.getObjectMapper().readTree(jsonStr);
47 | //override client tag
48 | if(json.isObject()) {
49 | jsonHeaderExt = (ObjectNode)json;
50 | if(null != clientTag)
51 | jsonHeaderExt.put(CLIENT_TAG_KEY, clientTag);
52 | if(trace)
53 | jsonHeaderExt.put(TRACE_ID_KEY, msg.getTraceIDStr());
54 | } else {
55 | //throw error
56 | throw new IllegalStateException("Invalid json header format, pass in json root is not object.");
57 | }
58 | } catch (IOException e) {
59 | throw new IllegalStateException("Could not parse json header.");
60 | }
61 | } else {
62 | //throw error
63 | throw new IllegalStateException("Invalid json header format. Json header not specified.");
64 | }
65 | try {
66 | this.jsonHeaderBytes = SystemUtil.getObjectMapper().writeValueAsBytes(jsonHeaderExt);
67 | } catch (JsonProcessingException e) {
68 | throw new IllegalStateException("Fail to convert object node to string.");
69 | }
70 | }
71 |
72 | @Override
73 | public byte[] getBytes() {
74 | if(null == bytes){
75 | byte[] header = this.getHeader().getBytes(NSQCommand.DEFAULT_CHARSET);
76 | byte[] jsonHeaderBytes = this.jsonHeaderBytes;
77 | byte[] body = this.getBody().get(0);
78 | ByteBuffer buf = ByteBuffer.allocate(header.length + 4/*total length*/ + 2/*json header length*/ + jsonHeaderBytes.length + body.length);
79 |
80 | buf.put(header)
81 | .putInt(2 + jsonHeaderBytes.length + body.length)
82 | .putShort((short) jsonHeaderBytes.length)
83 | .put(jsonHeaderBytes)
84 | .put(body);
85 | bytes = buf.array();
86 | }
87 | return bytes;
88 | }
89 |
90 | @Override
91 | public String getHeader() {
92 | return String.format("PUB_EXT %s%s\n", topic.getTopicText(), this.getPartitionStr());
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/java/com/youzan/util/ConcurrentSortedSet.java:
--------------------------------------------------------------------------------
1 | /**
2 | *
3 | */
4 | package com.youzan.util;
5 |
6 | import org.slf4j.Logger;
7 | import org.slf4j.LoggerFactory;
8 |
9 | import java.util.Collection;
10 | import java.util.SortedSet;
11 | import java.util.TreeSet;
12 | import java.util.concurrent.locks.ReentrantReadWriteLock;
13 | import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
14 | import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
15 |
16 | /**
17 | * Blocking when try-lock. It is for the small size collection. It consists of
18 | * one {@link SortedSet}
19 | *
20 | * @author zhaoxi (linzuxiong)
21 | *
22 | *
23 | */
24 | @ThreadSafe
25 | public class ConcurrentSortedSet implements java.io.Serializable {
26 | private static final long serialVersionUID = -4747846630389873940L;
27 | private static final Logger logger = LoggerFactory.getLogger(ConcurrentSortedSet.class);
28 |
29 | private SortedSet set = null;
30 | private final ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
31 | private final ReadLock r = lock.readLock();
32 | private final WriteLock w = lock.writeLock();
33 |
34 | public ConcurrentSortedSet() {
35 | set = new TreeSet<>();
36 | }
37 |
38 | public void clear() {
39 | w.lock();
40 | try {
41 | set.clear();
42 | } finally {
43 | w.unlock();
44 | }
45 | }
46 |
47 | public int size() {
48 | r.lock();
49 | try {
50 | return set.size();
51 | } finally {
52 | r.unlock();
53 | }
54 | }
55 |
56 | public boolean addAll(Collection c) {
57 | if (c == null || c.isEmpty()) {
58 | return true;
59 | }
60 | w.lock();
61 | try {
62 | return set.addAll(c);
63 | } finally {
64 | w.unlock();
65 | }
66 | }
67 |
68 | /**
69 | * Replace inner-data into the specified target.
70 | *
71 | * @param target
72 | * the new data
73 | */
74 | public void swap(SortedSet target) {
75 | if (target == null) {
76 | throw new IllegalArgumentException("Your input is null pointer!");
77 | }
78 | w.lock();
79 | final SortedSet tmp = set;
80 | try {
81 | set = target;
82 | } finally {
83 | w.unlock();
84 | }
85 | tmp.clear();
86 | }
87 |
88 | public void add(T e) {
89 | if (e == null) {
90 | return;
91 | }
92 | w.lock();
93 | try {
94 | set.add(e);
95 | } finally {
96 | w.unlock();
97 | }
98 | }
99 |
100 | /**
101 | * @param e
102 | * the element that need to be removed
103 | */
104 | public void remove(T e) {
105 | w.lock();
106 | try {
107 | set.remove(e);
108 | } finally {
109 | w.unlock();
110 | }
111 | }
112 |
113 | public boolean isEmpty() {
114 | r.lock();
115 | try {
116 | return set.isEmpty();
117 | } finally {
118 | r.unlock();
119 | }
120 | }
121 |
122 | public T[] newArray(T[] a) {
123 | r.lock();
124 | try {
125 | return set.toArray(a);
126 | } finally {
127 | r.unlock();
128 | }
129 | }
130 |
131 | /**
132 | * Never return null.
133 | *
134 | * @return the new {@link SortedSet}
135 | */
136 | public SortedSet newSortedSet() {
137 | r.lock();
138 | try {
139 | return new TreeSet<>(set);
140 | } finally {
141 | r.unlock();
142 | }
143 | }
144 |
145 | @Override
146 | public String toString() {
147 | r.lock();
148 | try {
149 | return set.toString();
150 | } finally {
151 | r.unlock();
152 | }
153 | }
154 | }
155 |
--------------------------------------------------------------------------------
/src/main/java/com/youzan/nsq/client/core/pool/consumer/FixedPool.java:
--------------------------------------------------------------------------------
1 | package com.youzan.nsq.client.core.pool.consumer;
2 |
3 | import com.youzan.nsq.client.core.Client;
4 | import com.youzan.nsq.client.core.NSQConnection;
5 | import com.youzan.nsq.client.core.NSQConnectionImpl;
6 | import com.youzan.nsq.client.entity.Address;
7 | import com.youzan.nsq.client.entity.NSQConfig;
8 | import com.youzan.nsq.client.exception.NSQNoConnectionException;
9 | import com.youzan.nsq.client.network.netty.NSQClientInitializer;
10 | import io.netty.bootstrap.Bootstrap;
11 | import io.netty.channel.Channel;
12 | import io.netty.channel.ChannelFuture;
13 | import io.netty.channel.ChannelOption;
14 | import io.netty.channel.EventLoopGroup;
15 | import io.netty.channel.nio.NioEventLoopGroup;
16 | import io.netty.channel.socket.nio.NioSocketChannel;
17 | import org.slf4j.Logger;
18 | import org.slf4j.LoggerFactory;
19 |
20 | import java.util.ArrayList;
21 | import java.util.List;
22 | import java.util.concurrent.TimeUnit;
23 | import java.util.concurrent.atomic.AtomicInteger;
24 |
25 | /**
26 | * @deprecated Deprecated as one {@link Address} does not more one topic anymore, after partition is introduced.
27 | * @author zhaoxi (linzuxiong)
28 | */
29 | public class FixedPool {
30 | private static final Logger logger = LoggerFactory.getLogger(FixedPool.class);
31 |
32 | private final AtomicInteger connectionIDGenerator = new AtomicInteger(0);
33 | private final Bootstrap bootstrap;
34 | private final EventLoopGroup eventLoopGroup;
35 | private final int size;
36 |
37 |
38 | private final NSQConfig config;
39 | private final Address address;
40 | private final Client client;
41 | private final List connections;
42 |
43 | public FixedPool(Address address, int size, Client client, NSQConfig config) {
44 | this.address = address;
45 | this.size = size;
46 | this.client = client;
47 | this.config = config;
48 | this.connections = new ArrayList<>(size);
49 | this.bootstrap = new Bootstrap();
50 | this.eventLoopGroup = new NioEventLoopGroup(config.getThreadPoolSize4IO());
51 | }
52 |
53 |
54 | public void prepare(boolean isOrdered) throws NSQNoConnectionException {
55 | bootstrap.option(ChannelOption.SO_KEEPALIVE, true);
56 | bootstrap.option(ChannelOption.TCP_NODELAY, true);
57 | bootstrap.option(ChannelOption.CONNECT_TIMEOUT_MILLIS, config.getConnectTimeoutInMillisecond());
58 | bootstrap.group(eventLoopGroup);
59 | bootstrap.channel(NioSocketChannel.class);
60 | bootstrap.handler(new NSQClientInitializer());
61 | for (int i = 0; i < size; i++) {
62 | final ChannelFuture future = bootstrap.connect(address.getHost(), address.getPort());
63 | // Wait until the connection attempt succeeds or fails.
64 | if (!future.awaitUninterruptibly(config.getConnectTimeoutInMillisecond(), TimeUnit.MILLISECONDS)) {
65 | throw new NSQNoConnectionException(future.cause());
66 | }
67 | final Channel channel = future.channel();
68 | if (!future.isSuccess()) {
69 | if (channel != null) {
70 | channel.close();
71 | }
72 | throw new NSQNoConnectionException("Connect " + address + " is wrong.", future.cause());
73 | }
74 |
75 | final NSQConnection conn = new NSQConnectionImpl(connectionIDGenerator.incrementAndGet(), address, channel,
76 | config);
77 | // Netty async+sync programming
78 | channel.attr(NSQConnection.STATE).set(conn);
79 | channel.attr(Client.STATE).set(client);
80 | channel.attr(Client.ORDERED).set(isOrdered);
81 | connections.add(conn);
82 | }
83 | logger.debug("Having created {} connections for {}", size, address);
84 | }
85 |
86 | public List getConnections() {
87 | return connections;
88 | }
89 |
90 | public void close() {
91 | connections.clear();
92 | if (!eventLoopGroup.isShuttingDown()) {
93 | eventLoopGroup.shutdownGracefully(1, 2, TimeUnit.SECONDS);
94 | }
95 | }
96 | }
97 |
--------------------------------------------------------------------------------