├── .gitignore ├── rocketmq-flume-sink ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── handu │ └── flume │ └── sink │ └── rocketmq │ ├── RocketMQSinkUtil.java │ └── RocketMQSink.java ├── rocketmq-flume-source ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── handu │ └── flume │ └── source │ └── rocketmq │ ├── RocketMQSourceUtil.java │ └── RocketMQSource.java ├── pom.xml └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | .* 2 | *.class 3 | *.jar 4 | *.war 5 | *.iml 6 | .settings 7 | target 8 | !.gitignore -------------------------------------------------------------------------------- /rocketmq-flume-sink/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | rocketmq-flume-sink 7 | 8 | 9 | rocketmq-flume 10 | com.handu 11 | 1.0.0 12 | 13 | 14 | -------------------------------------------------------------------------------- /rocketmq-flume-source/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | rocketmq-flume-source 7 | 8 | 9 | rocketmq-flume 10 | com.handu 11 | 1.0.0 12 | 13 | 14 | -------------------------------------------------------------------------------- /rocketmq-flume-sink/src/main/java/com/handu/flume/sink/rocketmq/RocketMQSinkUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2006-2014 handu.com. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.handu.flume.sink.rocketmq; 17 | 18 | import com.alibaba.rocketmq.client.producer.DefaultMQProducer; 19 | import com.alibaba.rocketmq.client.producer.MQProducer; 20 | import com.google.common.base.Preconditions; 21 | import org.apache.flume.Context; 22 | 23 | /** 24 | * Sink工具类 25 | * 26 | * @author Jinkai.Ma 27 | * @since 2015-01-16 28 | */ 29 | public class RocketMQSinkUtil { 30 | 31 | /** 32 | * Topic配置项，如：a1.sinks.s1.topic=TestTopic 33 | */ 34 | public static final String TOPIC_CONFIG = "topic"; 35 | public static final String TOPIC_DEFAULT = "FLUME_ROCKETMQ"; 36 | /** 37 | * Tags配置项，如：a1.sinks.s1.tags=Tag1,Tag2 38 | */ 39 | public static final String TAG_CONFIG = "tag"; 40 | public static final String TAG_DEFAULT = ""; 41 | /** 42 | * Producer分组配置项，如：a1.sinks.s1.producerGroup=please_rename_unique_group_name 43 | */ 44 | public static final String PRODUCER_GROUP_CONFIG = "producerGroup"; 45 | public static final String PRODUCER_GROUP_DEFAULT = "DEFAULT_PRODUCER"; 46 | /** 47 | * Namesrv地址配置项，如：a1.sinks.s1.namesrvAddr=localhost:9876 48 | */ 49 | public static final String NAMESRV_ADDR_CONFIG = "namesrvAddr"; 50 | 51 | public static MQProducer getProducer(Context context) { 52 | final String producerGroup = context.getString(PRODUCER_GROUP_CONFIG, PRODUCER_GROUP_DEFAULT); 53 | final String namesrvAddr = Preconditions.checkNotNull(context.getString(NAMESRV_ADDR_CONFIG), "RocketMQ namesrvAddr must be specified. For example: a1.sinks.s1.namesrvAddr=127.0.0.1:9876"); 54 | 55 | DefaultMQProducer producer = new DefaultMQProducer(producerGroup); 56 | producer.setNamesrvAddr(namesrvAddr); 57 | 58 | return producer; 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | com.handu 7 | rocketmq-flume 8 | pom 9 | 1.0.0 10 | http://www.handu.com 11 | 2014 12 | 13 | 14 | 韩都衣舍 15 | http://www.handu.com 16 | 17 | 18 | 19 | 20 | The Hando Apollo Team 21 | dev@handu.com 22 | http://apollo.handu.com/ 23 | 韩都衣舍 24 | http://www.handu.com/ 25 | 26 | 27 | 28 | 29 | scm:git:https://github.com/rocketmq/rocketmq-flume.git 30 | scm:git:https://github.com/rocketmq/rocketmq-flume.git 31 | 32 | 33 | 34 | UTF-8 35 | 36 | 18.0 37 | 1.5.2 38 | 3.2.6 39 | 4.11 40 | 1.9.0 41 | 42 | 43 | 44 | rocketmq-flume-sink 45 | rocketmq-flume-source 46 | 47 | 48 | 49 | 50 | com.google.guava 51 | guava 52 | ${guava.version} 53 | 54 | 55 | org.apache.flume 56 | flume-ng-core 57 | ${flume.version} 58 | 59 | 60 | com.alibaba.rocketmq 61 | rocketmq-client 62 | ${rocketmq.version} 63 | 64 | 65 | junit 66 | junit 67 | ${junit.version} 68 | test 69 | 70 | 71 | org.mockito 72 | mockito-all 73 | ${mockito.version} 74 | test 75 | 76 | 77 | 78 | 79 | clean install dependency:copy-dependencies 80 | 81 | 82 | org.apache.maven.plugins 83 | maven-compiler-plugin 84 | 3.3 85 | 86 | 1.6 87 | 1.6 88 | 89 | 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /rocketmq-flume-source/src/main/java/com/handu/flume/source/rocketmq/RocketMQSourceUtil.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2006-2014 handu.com. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.handu.flume.source.rocketmq; 17 | 18 | import com.alibaba.rocketmq.client.consumer.DefaultMQPullConsumer; 19 | import com.alibaba.rocketmq.client.consumer.MQPullConsumer; 20 | import com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel; 21 | import com.google.common.base.Preconditions; 22 | import org.apache.flume.Context; 23 | 24 | /** 25 | * Source工具类 26 | * 27 | * @author Jinkai.Ma 28 | * @since 2015-01-16 29 | */ 30 | public class RocketMQSourceUtil { 31 | 32 | /** 33 | * Topic配置项，如：a1.sources.r1.topic=TestTopic 34 | */ 35 | public static final String TOPIC_CONFIG = "topic"; 36 | /** 37 | * Tags配置项，如：a1.sources.r1.tags=Tag1,Tag2 38 | */ 39 | public static final String TAGS_CONFIG = "tags"; 40 | public static final String TAGS_DEFAULT = "*"; 41 | /** 42 | * Tags header name configuration, eg: a1.sources.r1.tagsHeaderName=name 43 | */ 44 | public static final String TAGS_HEADER_NAME_CONFIG = "tagsHeaderName"; 45 | public static final String TAGS_HEADER_NAME_DEFAULT = "tags"; 46 | /** 47 | * Topic header name configuration, eg: a1.sources.r1.topicHeaderName=name 48 | */ 49 | public static final String TOPIC_HEADER_NAME_CONFIG = "topicHeaderName"; 50 | public static final String TOPIC_HEADER_NAME_DEFAULT = "topic"; 51 | /** 52 | * 一次最多拉取条数配置项，如：a1.sources.r1.maxNums=150 53 | */ 54 | public static final String MAXNUMS_CONFIG = "maxNums"; 55 | public static final int MAXNUMS_DEFAULT = 32; 56 | /** 57 | * Consumer分组配置项，如：a1.sources.r1.consumerGroup=please_rename_unique_group_name 58 | */ 59 | public static final String CONSUMER_GROUP_CONFIG = "consumerGroup"; 60 | public static final String CONSUMER_GROUP_DEFAULT = "DEFAULT_CONSUMER"; 61 | /** 62 | * Namesrv地址配置项，如：a1.sinks.s1.namesrvAddr=localhost:9876 63 | */ 64 | public static final String NAMESRV_ADDR_CONFIG = "namesrvAddr"; 65 | /** 66 | * 订阅方式配置项，如：a1.sources.r1.messageModel=BROADCASTING 67 | */ 68 | public static final String MESSAGEMODEL_CONFIG = "messageModel"; 69 | public static final String MESSAGEMODEL_DEFAULT = "BROADCASTING"; 70 | 71 | public static MQPullConsumer getConsumer(Context context) { 72 | final String consumerGroup = context.getString(CONSUMER_GROUP_CONFIG, CONSUMER_GROUP_DEFAULT); 73 | final String namesrvAddr = Preconditions.checkNotNull(context.getString(NAMESRV_ADDR_CONFIG), "RocketMQ namesrvAddr must be specified. For example: a1.sources.r1.namesrvAddr=127.0.0.1:9876"); 74 | 75 | DefaultMQPullConsumer consumer = new DefaultMQPullConsumer(consumerGroup); 76 | consumer.setNamesrvAddr(namesrvAddr); 77 | consumer.setMessageModel(MessageModel.valueOf(context.getString(MESSAGEMODEL_CONFIG, MESSAGEMODEL_DEFAULT))); 78 | 79 | return consumer; 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /rocketmq-flume-sink/src/main/java/com/handu/flume/sink/rocketmq/RocketMQSink.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2006-2014 handu.com. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.handu.flume.sink.rocketmq; 17 | 18 | import com.alibaba.rocketmq.client.exception.MQClientException; 19 | import com.alibaba.rocketmq.client.producer.MQProducer; 20 | import com.alibaba.rocketmq.client.producer.SendResult; 21 | import com.alibaba.rocketmq.common.message.Message; 22 | import com.google.common.base.Preconditions; 23 | import com.google.common.base.Throwables; 24 | import org.apache.flume.*; 25 | import org.apache.flume.conf.Configurable; 26 | import org.apache.flume.sink.AbstractSink; 27 | import org.slf4j.Logger; 28 | import org.slf4j.LoggerFactory; 29 | 30 | /** 31 | * 发送消息 32 | * 33 | * @author Jinkai.Ma 34 | * @since 2015-01-16 35 | */ 36 | public class RocketMQSink extends AbstractSink implements Configurable { 37 | 38 | private static final Logger LOG = LoggerFactory.getLogger(RocketMQSink.class); 39 | 40 | private String topic; 41 | private String tag; 42 | private MQProducer producer; 43 | 44 | @Override 45 | public void configure(Context context) { 46 | // 获取配置项 47 | topic = context.getString(RocketMQSinkUtil.TOPIC_CONFIG, RocketMQSinkUtil.TOPIC_DEFAULT); 48 | tag = context.getString(RocketMQSinkUtil.TAG_CONFIG, RocketMQSinkUtil.TAG_DEFAULT); 49 | // 初始化Producer 50 | producer = Preconditions.checkNotNull(RocketMQSinkUtil.getProducer(context)); 51 | } 52 | 53 | @Override 54 | public Status process() throws EventDeliveryException { 55 | Channel channel = getChannel(); 56 | Transaction tx = channel.getTransaction(); 57 | try { 58 | tx.begin(); 59 | Event event = channel.take(); 60 | if (event == null || event.getBody() == null || event.getBody().length == 0) { 61 | tx.commit(); 62 | return Status.READY; 63 | } 64 | // 发送消息 65 | SendResult sendResult = producer.send(new Message(topic, tag, event.getBody())); 66 | if (LOG.isDebugEnabled()) { 67 | LOG.debug("SendResult={}, Message={}", sendResult, event.getBody()); 68 | } 69 | tx.commit(); 70 | return Status.READY; 71 | } catch (Exception e) { 72 | LOG.error("RocketMQSink send message exception", e); 73 | try { 74 | tx.rollback(); 75 | return Status.BACKOFF; 76 | } catch (Exception e2) { 77 | LOG.error("Rollback exception", e2); 78 | } 79 | return Status.BACKOFF; 80 | } finally { 81 | tx.close(); 82 | } 83 | } 84 | 85 | @Override 86 | public synchronized void start() { 87 | try { 88 | // 启动Producer 89 | producer.start(); 90 | } catch (MQClientException e) { 91 | LOG.error("RocketMQSink start producer failed", e); 92 | Throwables.propagate(e); 93 | } 94 | super.start(); 95 | } 96 | 97 | @Override 98 | public synchronized void stop() { 99 | // 停止Producer 100 | producer.shutdown(); 101 | super.stop(); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | rocketmq-flume Source&Sink 2 | ========================== 3 | 4 | 该项目用于[RocketMQ](https://github.com/alibaba/RocketMQ)与[Flume-ng](https://github.com/apache/flume)之间的消息接收和投递。 5 | 6 | 1. 首先请确定您已经对[RocketMQ](https://github.com/alibaba/RocketMQ)和[Flume-ng](https://github.com/apache/flume)有了基本的了解 7 | 2. 确保本地maven库中已经存在[RocketMQ相关的包](https://github.com/alibaba/RocketMQ/releases/download/v3.2.2/alibaba-rocketmq-client-java-3.2.2.tar.gz)，或者下载RocketMQ源码自行编译 8 | 3. 在rocketmq-flume项目根目录执行`mvn clean install dependency:copy-dependencies` 9 | 4. 将rocketmq-flume相关依赖jar包拷贝到`$FLUME_HOME/lib`目录中(具体包会在后面描述) 10 | 11 | ## Sink 12 | 13 | ### Sink配置说明 14 | 15 | | 配置项 | 必填 | 默认值 | 说明 | 16 | |---------------|-----|------------------|------| 17 | | namesrvAddr | 必填 | null | Name Server地址，遵循RocketMQ配置方式 | 18 | | producerGroup | 可选 | DEFAULT_PRODUCER | Producer分组 | 19 | | topic | 必填 | null | Topic名称 | 20 | | tags | 可选 | 空字符串 | Tag名称，遵循RocketMQ配置方式 | 21 | 22 | ### Sink综合示例 23 | 24 | - 编写Flume的配置文件 25 | 26 | ``` 27 | agent1.sources=source1 28 | agent1.channels=channel1 29 | agent1.sinks=sink1 30 | 31 | agent1.sources.source1.type=avro 32 | agent1.sources.source1.bind=0.0.0.0 33 | agent1.sources.source1.port=15151 34 | agent1.sources.source1.channels=channel1 35 | 36 | agent1.sinks.sink1.type=com.handu.flume.sink.rocketmq.RocketMQSink 37 | agent1.sinks.sink1.namesrvAddr=rocketmq_namesrv:9876 38 | agent1.sinks.sink1.producerGroup=MyProducerGroup_1 39 | agent1.sinks.sink1.topic=FromFlume 40 | agent1.sinks.sink1.tag=Tag1 41 | agent1.sinks.sink1.channel=channel1 42 | 43 | agent1.channels.channel1.type=memory 44 | agent1.channels.channel1.capacity=100 45 | agent1.channels.channel1.transactionCapacity=100 46 | agent1.channels.channel1.keep-alive=3 47 | ``` 48 | 49 | - 将下面jar包拷贝到`$FLUME_HOME/lib`目录中 50 | 51 | ``` 52 | rocketmq-flume-sink-1.0.0.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target) 53 | fastjson-1.1.41.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 54 | netty-all-4.0.23.Final.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 55 | rocketmq-client-3.2.2.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 56 | rocketmq-common-3.2.2.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 57 | rocketmq-remoting-3.2.2.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 58 | ``` 59 | 60 | - 执行测试命令查看 61 | 62 | ``` 63 | shell1> $FLUME_HOME/bin/flume-ng agent -c conf -f conf/flume.conf -n agent1 -Dflume.root.logger=INFO,console 64 | shell2> $FLUME_HOME/bin/flume-ng avro-client -H localhost -p 15151 -F $FLUME_HOME/README 65 | ``` 66 | 67 | - 查看shell1控制台输出的信息 68 | 69 | ## Source 70 | 71 | ### Source配置说明 72 | 73 | | 配置项 | 必填 | 默认值 | 说明 | 74 | |---------------|-----|------------------|------| 75 | | namesrvAddr | 必填 | null | Name Server地址，遵循RocketMQ配置方式 | 76 | | consumerGroup | 可选 | DEFAULT_CONSUMER | Consumer分组 | 77 | | topic | 必填 | null | Topic名称 | 78 | | tags | 可选 | * | Tag名称，遵循RocketMQ配置方式 | 79 | | messageModel | 可选 | BROADCASTING | BROADCASTING或CLUSTERING | 80 | | maxNums | 可选 | 32 | 一次读取消息数量 | 81 | 82 | ### Source综合示例 83 | 84 | ``` 85 | agent1.sources=source1 86 | agent1.channels=channel1 87 | agent1.sinks=sink1 88 | 89 | agent1.sources.source1.type=com.handu.flume.source.rocketmq.RocketMQSource 90 | agent1.sources.source1.namesrvAddr=rocketmq_namesrv:9876 91 | agent1.sources.source1.consumerGroup=MyConsumerGroup_1 92 | agent1.sources.source1.topic=TopicTest 93 | agent1.sources.source1.tags=* 94 | agent1.sources.source1.messageModel=BROADCASTING 95 | agent1.sources.source1.maxNums=32 96 | agent1.sources.source1.channels=channel1 97 | 98 | agent1.sinks.sink1.type=logger 99 | agent1.sinks.sink1.channel=channel1 100 | 101 | agent1.channels.channel1.type=memory 102 | agent1.channels.channel1.capacity=100 103 | agent1.channels.channel1.transactionCapacity=100 104 | agent1.channels.channel1.keep-alive=3 105 | ``` 106 | 107 | - 将下面jar包拷贝到`$FLUME_HOME/lib`目录中 108 | 109 | ``` 110 | rocketmq-flume-source-1.0.0.jar (文件位置: $PROJECT_HOME/rocketmq-flume-source/target) 111 | fastjson-1.1.41.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 112 | netty-all-4.0.23.Final.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 113 | rocketmq-client-3.2.2.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 114 | rocketmq-common-3.2.2.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 115 | rocketmq-remoting-3.2.2.jar (文件位置: $PROJECT_HOME/rocketmq-flume-sink/target/dependency) 116 | ``` 117 | 118 | - 向RocketMQ中投递一些测试消息 119 | 120 | - 执行测试命令查看控制台输出 121 | 122 | ``` 123 | $FLUME_HOME/bin/flume-ng agent -c conf -f conf/flume.conf -n agent1 -Dflume.root.logger=INFO,console 124 | ``` 125 | -------------------------------------------------------------------------------- /rocketmq-flume-source/src/main/java/com/handu/flume/source/rocketmq/RocketMQSource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2006-2014 handu.com. 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | package com.handu.flume.source.rocketmq; 17 | 18 | import com.alibaba.rocketmq.client.consumer.MQPullConsumer; 19 | import com.alibaba.rocketmq.client.consumer.PullResult; 20 | import com.alibaba.rocketmq.client.consumer.PullStatus; 21 | import com.alibaba.rocketmq.client.exception.MQClientException; 22 | import com.alibaba.rocketmq.common.message.MessageExt; 23 | import com.alibaba.rocketmq.common.message.MessageQueue; 24 | import com.google.common.base.Preconditions; 25 | import com.google.common.base.Throwables; 26 | import com.google.common.collect.Lists; 27 | import com.google.common.collect.Maps; 28 | import org.apache.flume.Context; 29 | import org.apache.flume.Event; 30 | import org.apache.flume.EventDeliveryException; 31 | import org.apache.flume.PollableSource; 32 | import org.apache.flume.conf.Configurable; 33 | import org.apache.flume.event.SimpleEvent; 34 | import org.apache.flume.source.AbstractSource; 35 | import org.slf4j.Logger; 36 | import org.slf4j.LoggerFactory; 37 | 38 | import java.util.HashMap; 39 | import java.util.List; 40 | import java.util.Map; 41 | import java.util.Set; 42 | 43 | /** 44 | * 接收消息 45 | * 46 | * @author Jinkai.Ma 47 | * @since 2015-01-16 48 | */ 49 | public class RocketMQSource extends AbstractSource implements Configurable, PollableSource { 50 | 51 | private static final Logger LOG = LoggerFactory.getLogger(RocketMQSource.class); 52 | 53 | private String topic; 54 | private String tags; 55 | private String topicHeaderName; 56 | private String tagsHeaderName; 57 | private int maxNums; 58 | private MQPullConsumer consumer; 59 | 60 | @Override 61 | public void configure(Context context) { 62 | // 初始化配置项 63 | topic = Preconditions.checkNotNull(context.getString(RocketMQSourceUtil.TOPIC_CONFIG), "RocketMQ topic must be specified. For example: a1.sources.r1.topic=TestTopic"); 64 | tags = context.getString(RocketMQSourceUtil.TAGS_CONFIG, RocketMQSourceUtil.TAGS_DEFAULT); 65 | topicHeaderName = context.getString(RocketMQSourceUtil.TOPIC_HEADER_NAME_CONFIG, RocketMQSourceUtil.TOPIC_HEADER_NAME_DEFAULT); 66 | tagsHeaderName = context.getString(RocketMQSourceUtil.TAGS_HEADER_NAME_CONFIG, RocketMQSourceUtil.TAGS_HEADER_NAME_DEFAULT); 67 | maxNums = context.getInteger(RocketMQSourceUtil.MAXNUMS_CONFIG, RocketMQSourceUtil.MAXNUMS_DEFAULT); 68 | 69 | // 初始化Consumer 70 | consumer = Preconditions.checkNotNull(RocketMQSourceUtil.getConsumer(context)); 71 | consumer.registerMessageQueueListener(topic, null); 72 | } 73 | 74 | @Override 75 | public Status process() throws EventDeliveryException { 76 | List eventList = Lists.newArrayList(); 77 | Map offsetMap = Maps.newHashMap(); 78 | Event event; 79 | Map headers; 80 | 81 | try { 82 | Set mqs = Preconditions.checkNotNull(consumer.fetchSubscribeMessageQueues(topic)); 83 | for (MessageQueue mq : mqs) { 84 | // 获取offset 85 | long offset = this.getMessageQueueOffset(mq); 86 | PullResult pullResult = consumer.pull(mq, tags, offset, maxNums); 87 | // 发现新消息写入Event 88 | if (pullResult.getPullStatus() == PullStatus.FOUND) { 89 | for (MessageExt messageExt : pullResult.getMsgFoundList()) { 90 | event = new SimpleEvent(); 91 | headers = new HashMap(); 92 | headers.put(topicHeaderName, messageExt.getTopic()); 93 | headers.put(tagsHeaderName, messageExt.getTags()); 94 | if (LOG.isDebugEnabled()) { 95 | LOG.debug("MessageQueue={}, Topic={}, Tags={}, Message: {}",new Object[] { 96 | mq, messageExt.getTopic(), messageExt.getTags(), messageExt.getBody()}); 97 | } 98 | event.setBody(messageExt.getBody()); 99 | event.setHeaders(headers); 100 | eventList.add(event); 101 | } 102 | offsetMap.put(mq, pullResult.getNextBeginOffset()); 103 | } 104 | } 105 | // 批量处理事件 106 | getChannelProcessor().processEventBatch(eventList); 107 | for (Map.Entry entry : offsetMap.entrySet()) { 108 | // 更新offset 109 | this.putMessageQueueOffset(entry.getKey(), entry.getValue()); 110 | } 111 | } catch (Exception e) { 112 | LOG.error("RocketMQSource consume message exception", e); 113 | return Status.BACKOFF; 114 | } 115 | return Status.READY; 116 | } 117 | 118 | @Override 119 | public synchronized void start() { 120 | try { 121 | // 启动Consumer 122 | consumer.start(); 123 | } catch (MQClientException e) { 124 | LOG.error("RocketMQSource start consumer failed", e); 125 | Throwables.propagate(e); 126 | } 127 | super.start(); 128 | } 129 | 130 | @Override 131 | public synchronized void stop() { 132 | // 停止Consumer 133 | consumer.shutdown(); 134 | super.stop(); 135 | } 136 | 137 | private void putMessageQueueOffset(MessageQueue mq, long offset) throws MQClientException { 138 | // 存储Offset，客户端每隔5s会定时刷新到Broker或者写入本地缓存文件 139 | consumer.updateConsumeOffset(mq, offset); 140 | } 141 | 142 | private long getMessageQueueOffset(MessageQueue mq) throws MQClientException { 143 | // 从Broker获取Offset 144 | long offset = consumer.fetchConsumeOffset(mq, false); 145 | 146 | if (offset < 0L) { 147 | offset = 0L; 148 | } 149 | return offset; 150 | } 151 | 152 | } 153 | --------------------------------------------------------------------------------