├── .gitignore ├── LICENSE ├── MQMessage.py ├── MQMessageListener.py ├── MQProducer.py ├── MQPullConsumer.py ├── MQPushConsumer.py ├── README.md ├── __init__.py ├── jpypetest.py ├── settings_MQ.py └── test ├── Producer.py ├── PullConsumer.py └── PushConsumer.py /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | downloads/ 16 | eggs/ 17 | .eggs/ 18 | lib/ 19 | lib64/ 20 | parts/ 21 | sdist/ 22 | var/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *,cover 46 | .hypothesis/ 47 | 48 | # Translations 49 | *.mo 50 | *.pot 51 | 52 | # Django stuff: 53 | *.log 54 | 55 | # Sphinx documentation 56 | docs/_build/ 57 | 58 | # PyBuilder 59 | target/ 60 | 61 | #Ipython Notebook 62 | .ipynb_checkpoints 63 | 64 | #idea 65 | .idea/ 66 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 GangLuICT 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MQMessage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | from jpype import * 5 | import logging 6 | import settings_MQ as settings 7 | 8 | logger = logging.getLogger("MQMessage") 9 | 10 | __all__ = ["MQMessage", "SendStatus", "PullStatus", "ConsumeFromWhere", "ConsumeConcurrentlyStatus", "ConsumeOrderlyStatus", 11 | "MessageModel"] 12 | 13 | Message = JPackage('com.alibaba.rocketmq.common.message').Message 14 | # enum classes: 15 | SENDSTATUS = JPackage('com.alibaba.rocketmq.client.producer').SendStatus 16 | PULLSTATUS = JPackage('com.alibaba.rocketmq.client.consumer').PullStatus 17 | CONSUMEFROMWHERE = JPackage('com.alibaba.rocketmq.common.consumer').ConsumeFromWhere 18 | CONSUMECONCURRENTLYSTATUS = JPackage('com.alibaba.rocketmq.client.consumer.listener').ConsumeConcurrentlyStatus 19 | CONSUMEORDERLYSTATUS = JPackage('com.alibaba.rocketmq.client.consumer.listener').ConsumeOrderlyStatus 20 | MESSAGEMODEL = JPackage('com.alibaba.rocketmq.common.protocol.heartbeat').MessageModel 21 | 22 | 23 | class MQMessage(object): 24 | def __init__(self, topic, tags, keys, body): 25 | self.topic = topic 26 | self.tags = tags 27 | self.keys = keys 28 | self.body = body 29 | self.msg = Message(JString(self.topic), JString(self.tags), JString(self.keys), self.body.encode(encoding = settings.MsgBodyEncoding)) 30 | # JAVA prototype of Message 31 | # public Message() 32 | # public Message(String topic, byte[] body) 33 | # public Message(String topic, String tags, String keys, byte[] body) 34 | # public Message(String topic, String tags, String keys, int flag, byte[] body, boolean waitStoreMsgOK) 35 | 36 | def tostr(self): 37 | """ 38 | Translate the object into a string 39 | """ 40 | return self.topic + "::" + self.tags + "::" + self.keys + "::" + self.body 41 | 42 | 43 | # PullResult的返回结果 44 | PullStatus = { 45 | #'FOUND': 0, # Founded 46 | 'FOUND': PULLSTATUS.FOUND, 47 | #'NO_NEW_MSG': 1, # No new message can be pull 48 | 'NO_NEW_MSG': PULLSTATUS.NO_NEW_MSG, 49 | #'NO_MATCHED_MSG': 2, # Filtering results can not match 50 | 'NO_MATCHED_MSG': PULLSTATUS.NO_MATCHED_MSG, 51 | #'OFFSET_ILLEGAL': 3 # Illegal offset，may be too big or too small 52 | 'OFFSET_ILLEGAL': PULLSTATUS.OFFSET_ILLEGAL 53 | } 54 | 55 | # SendResult的返回结果 56 | SendStatus = { 57 | #'SEND_OK': 0, 58 | 'SEND_OK': SENDSTATUS.SEND_OK, 59 | #'FLUSH_DISK_TIMEOUT': 1, 60 | 'FLUSH_DISK_TIMEOUT': SENDSTATUS.FLUSH_DISK_TIMEOUT, 61 | #'FLUSH_SLAVE_TIMEOUT': 2, 62 | 'FLUSH_SLAVE_TIMEOUT': SENDSTATUS.FLUSH_SLAVE_TIMEOUT, 63 | #'SLAVE_NOT_AVAILABLE': 3 64 | 'SLAVE_NOT_AVAILABLE': SENDSTATUS.SLAVE_NOT_AVAILABLE 65 | } 66 | 67 | # PushConsumer消费时选择第一次订阅时的消费位置 68 | ConsumeFromWhere = { 69 | # 一个新的订阅组第一次启动从队列的最后位置开始消费 70 | # 后续再启动接着上次消费的进度开始消费 71 | #'CONSUME_FROM_LAST_OFFSET': 0, 72 | 'CONSUME_FROM_LAST_OFFSET': CONSUMEFROMWHERE.CONSUME_FROM_LAST_OFFSET, 73 | #@Deprecated 74 | #'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST': 1, 75 | 'CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST': CONSUMEFROMWHERE.CONSUME_FROM_LAST_OFFSET_AND_FROM_MIN_WHEN_BOOT_FIRST, 76 | #@Deprecated 77 | #'CONSUME_FROM_MIN_OFFSET': 2, 78 | 'CONSUME_FROM_MIN_OFFSET': CONSUMEFROMWHERE.CONSUME_FROM_MIN_OFFSET, 79 | #@Deprecated 80 | #'CONSUME_FROM_MAX_OFFSET': 3, 81 | 'CONSUME_FROM_MAX_OFFSET': CONSUMEFROMWHERE.CONSUME_FROM_MAX_OFFSET, 82 | # 一个新的订阅组第一次启动从队列的最前位置开始消费
83 | # 后续再启动接着上次消费的进度开始消费 84 | #'CONSUME_FROM_FIRST_OFFSET': 4, 85 | 'CONSUME_FROM_FIRST_OFFSET': CONSUMEFROMWHERE.CONSUME_FROM_FIRST_OFFSET, 86 | # 一个新的订阅组第一次启动从指定时间点开始消费,时间点设置参见DefaultMQPushConsumer.consumeTimestamp参数 87 | # 后续再启动接着上次消费的进度开始消费 88 | #'CONSUME_FROM_TIMESTAMP': 5, 89 | 'CONSUME_FROM_TIMESTAMP': CONSUMEFROMWHERE.CONSUME_FROM_TIMESTAMP, 90 | } 91 | 92 | # PushConsumer消费后的返回值(并发消费时) 93 | ConsumeConcurrentlyStatus = { 94 | #'CONSUME_SUCCESS': 0, # Success consumption 95 | 'CONSUME_SUCCESS': CONSUMECONCURRENTLYSTATUS.CONSUME_SUCCESS, 96 | #'RECONSUME_LATER': 1, # Failure consumption,later try to consume 97 | 'RECONSUME_LATER': CONSUMECONCURRENTLYSTATUS.RECONSUME_LATER, 98 | } 99 | 100 | # PushConsumer消费后的返回值(顺序消费时) 101 | ConsumeOrderlyStatus ={ 102 | #'SUCCESS': 0, # Success consumption 103 | 'SUCCESS': CONSUMEORDERLYSTATUS.SUCCESS, 104 | #'ROLLBACK': 1, # Rollback consumption(only for binlog consumption) 105 | 'ROLLBACK': CONSUMEORDERLYSTATUS.ROLLBACK, 106 | #'COMMIT': 2, # Commit offset(only for binlog consumption) 107 | 'COMMIT': CONSUMEORDERLYSTATUS.COMMIT, 108 | #'SUSPEND_CURRENT_QUEUE_A_MOMENT': 3 # Suspend current queue a moment 109 | 'SUSPEND_CURRENT_QUEUE_A_MOMENT': CONSUMEORDERLYSTATUS.SUSPEND_CURRENT_QUEUE_A_MOMENT 110 | } 111 | 112 | # PushConsumer的消息model 113 | MessageModel = { 114 | #'BROADCASTING': 0, # broadcast 115 | 'BROADCASTING': MESSAGEMODEL.BROADCASTING, 116 | #'CLUSTERING': 1 # clustering 117 | 'CLUSTERING': MESSAGEMODEL.CLUSTERING 118 | } 119 | -------------------------------------------------------------------------------- /MQMessageListener.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | from jpype import * 5 | import logging 6 | import settings_MQ as settings 7 | 8 | logger = logging.getLogger("MQMessageListener") 9 | 10 | __all__ = ["msgListenerConcurrentlyProxy", "msgListenerOrderlyProxy"] 11 | 12 | from MQMessage import ConsumeConcurrentlyStatus, ConsumeOrderlyStatus 13 | 14 | class MessageListenerConcurrently: 15 | '''接口类MessageListenerConcurrently的实现 16 | public interface MessageListenerConcurrently extends MessageListener { 17 | ''' 18 | def consumeMessage(self, msgs, context): 19 | ''' 20 | # JAVA prototype 21 | # ConsumeConcurrentlyStatus consumeMessage(final List msgs, final ConsumeConcurrentlyContext context); 22 | ''' 23 | logger.debug("Into consumerMessage of MessageListenerConcurrently") 24 | #msg = msgs.get(JInt(0)) 25 | for msg in msgs: 26 | topic = msg.getTopic() 27 | tags = msg.getTags() 28 | body = str(msg.getBody()).decode(settings.MsgBodyEncoding) 29 | 30 | logger.debug(msg.toString()) 31 | # In Python 2.x, bytes is just an alias for str. 所以bytes解码时要注意了, msg.body.decode会出错(bytes没有decode方法)！ 32 | #logger.debug("Message body: " + str(msg.getBody())) 33 | #logger.debug("Message body: " + str(msg.getBody()).decode(settings.MsgBodyEncoding)) 34 | logger.debug("Message body: " + body) 35 | 36 | if topic == "RMQTopicTest": 37 | # 执行TopicTest的消费逻辑 38 | if tags == "TagA": 39 | # 执行TagA的消费 40 | logger.debug("Got message with topic " + topic + " and tags " + tags) 41 | elif tags == "TagB": 42 | # 执行TagB的消费 43 | logger.debug("Got message with topic " + topic + " and tags " + tags) 44 | elif tags == "TagC": 45 | # 执行TagC的消费 46 | logger.debug("Got message with topic " + topic + " and tags " + tags) 47 | else: 48 | # 错误的Tag 49 | logger.error("Got message with topic " + topic + " and UNKNOWN tags " + tags) 50 | elif topic == "TopicTest1": 51 | # 执行TopicTest1的消费逻辑 52 | logger.debug("Got message with topic " + topic + " and tags " + tags) 53 | else: 54 | logger.debug("Got message with UNKNOWN topic " + topic ) 55 | 56 | return ConsumeConcurrentlyStatus['CONSUME_SUCCESS'] 57 | 58 | class MessageListenerOrderly: 59 | '''接口类MessageListenerOrderly的实现 60 | public interface MessageListenerOrderly extends MessageListener { 61 | ''' 62 | def __init__(self): 63 | #JAVA原子类 64 | self.consumeTimes = java.util.concurrent.atomic.AtomicLong(0) 65 | 66 | def consumeMessage(self, msgs, context): 67 | ''' 68 | # JAVA prototype 69 | # ConsumeOrderlyStatus consumeMessage(final List msgs, final ConsumeOrderlyContext context); 70 | ''' 71 | context.setAutoCommit(False) 72 | logger.debug(java.lang.Thread.currentThread().getName() + " Receive New Messages: " + msgs.toString()) 73 | #TODO: msgs.toString()可能需要改成for msg in msgs: msg.toString() 74 | 75 | self.consumeTimes.incrementAndGet() 76 | consumeTimes = self.consumeTimes.get() 77 | #print consumeTimes 78 | #print type(consumeTimes) 79 | 80 | if (consumeTimes % 2) == 0: 81 | logger.debug("consumeTimes % 2 = 0, return SUCCESS") 82 | return ConsumeOrderlyStatus['SUCCESS'] 83 | elif (consumeTimes % 3) == 0: 84 | logger.debug("consumeTimes % 3 = 0, return ROLLBACK") 85 | return ConsumeOrderlyStatus['ROLLBACK'] 86 | elif (consumeTimes % 4) == 0: 87 | logger.debug("consumeTimes % 4 = 0, return COMMIT") 88 | return ConsumeOrderlyStatus['COMMIT'] 89 | elif (consumeTimes % 5) == 0: 90 | logger.debug("consumeTimes % 5 = 0, return SUSPEND_CURRENT_QUEUE_A_MOMENT") 91 | context.setSuspendCurrentQueueTimeMillis(3000) 92 | return ConsumeOrderlyStatus['SUSPEND_CURRENT_QUEUE_A_MOMENT'] 93 | else: 94 | logger.debug("consumeTimes is not times of 2, 3, 4, 5, return SUCCESS") 95 | return ConsumeOrderlyStatus['SUCCESS'] 96 | 97 | 98 | #实现 与 类代理 99 | msgListenerConcurrently = MessageListenerConcurrently() 100 | #JProxy("MessageListenerConcurrently", inst = msgListenerConcurrently) 101 | msgListenerConcurrentlyProxy = JProxy("com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently", inst = msgListenerConcurrently) 102 | 103 | 104 | #实现 与 类代理 105 | msgListenerOrderly = MessageListenerOrderly() 106 | #JProxy("MessageListenerOrderly", inst = msgListenerOrderly) 107 | msgListenerOrderlyProxy = JProxy("com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly", inst = msgListenerOrderly) 108 | 109 | -------------------------------------------------------------------------------- /MQProducer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | from jpype import * 5 | import logging 6 | import time 7 | 8 | logger = logging.getLogger("MQProducer") 9 | 10 | __all__ = ["MQProducer"] 11 | 12 | DefaultMQProducer = JPackage('com.alibaba.rocketmq.client.producer').DefaultMQProducer 13 | MQClientException = JPackage('com.alibaba.rocketmq.client.exception').MQClientException 14 | SendResult = JPackage('com.alibaba.rocketmq.client.producer').SendResult 15 | 16 | class MQProducer(object): 17 | def __init__(self, groupName, namesrvAddr): 18 | """ 19 | :param groupName: 20 | :param namesrvAddr: 21 | :return: 22 | # JAVA prototype(实现的是第2个,见init函数 实例化) 23 | # public DefaultMQProducer() { 24 | # public DefaultMQProducer(final String producerGroup) { 25 | # public DefaultMQProducer(RPCHook rpcHook) { 26 | # public DefaultMQProducer(final String producerGroup, RPCHook rpcHook) { 27 | """ 28 | self.producer = None #初始化放在了init函数中 29 | self.groupName = groupName 30 | self.namesrvAddr = namesrvAddr 31 | self.instanceName = str(int(time.time()*1000)) #毫秒值作为instance name 32 | self.compressMsgBodyOverHowmuch = 4096 #消息压缩阈值 33 | 34 | def init(self): 35 | """批量设置一些基本项(为了尽可能少实现这些API接口,如以后有需要,可以逐个移出init)""" 36 | logger.info('Initializing producer ' + self.instanceName + ' ...') 37 | self.producer = DefaultMQProducer(JString(self.groupName)) #创建实例 38 | self.producer.setNamesrvAddr(JString(self.namesrvAddr)) 39 | self.producer.setInstanceName(JString(self.instanceName)) 40 | self.producer.setCompressMsgBodyOverHowmuch(JInt(self.compressMsgBodyOverHowmuch)) 41 | 42 | def start(self): 43 | """ 44 | # JAVA prototype 45 | # public void start() throws MQClientException { 46 | """ 47 | logger.info('Starting producer ' + self.instanceName + ' ...') 48 | self.producer.start() 49 | 50 | def shutdown(self): 51 | """ 52 | # JAVA prototype 53 | # public void shutdown() { 54 | """ 55 | logger.info('Shutting down producer ' + self.instanceName + ' ...') 56 | self.producer.shutdown() 57 | 58 | def send(self, MQMsg): 59 | """ 60 | # JAVA prototype 61 | # public SendResult send(Message msg, long timeout) throws MQClientException, RemotingException, 62 | # MQBrokerException, InterruptedException { 63 | # public void send(Message msg, SendCallback sendCallback) throws MQClientException, RemotingException, 64 | # InterruptedException { 65 | # public void send(Message msg, SendCallback sendCallback, long timeout) throws MQClientException, 66 | # RemotingException, InterruptedException { 67 | # public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { 68 | # public SendResult send(Message msg, MessageQueue mq) throws MQClientException, RemotingException, 69 | # MQBrokerException, InterruptedException { 70 | # public SendResult send(Message msg, MessageQueue mq, long timeout) throws MQClientException, 71 | # RemotingException, MQBrokerException, InterruptedException { 72 | # public void send(Message msg, MessageQueue mq, SendCallback sendCallback) throws MQClientException, 73 | # RemotingException, InterruptedException { 74 | # public void send(Message msg, MessageQueue mq, SendCallback sendCallback, long timeout) 75 | # throws MQClientException, RemotingException, InterruptedException { 76 | # public SendResult send(Message msg, MessageQueueSelector selector, Object arg) throws MQClientException, 77 | # RemotingException, MQBrokerException, InterruptedException { 78 | # public SendResult send(Message msg, MessageQueueSelector selector, Object arg, long timeout) 79 | # throws MQClientException, RemotingException, MQBrokerException, InterruptedException { 80 | # public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback) 81 | # throws MQClientException, RemotingException, InterruptedException { 82 | # public void send(Message msg, MessageQueueSelector selector, Object arg, SendCallback sendCallback, 83 | # long timeout) throws MQClientException, RemotingException, InterruptedException { 84 | """ 85 | logger.debug('Producer ' + self.instanceName + ' sending message: ' + MQMsg.tostr()) 86 | return self.producer.send(MQMsg.msg) 87 | 88 | def sendOneway(self, MQMsg): 89 | """ 90 | # JAVA prototype 91 | # public void sendOneway(Message msg) throws MQClientException, RemotingException, InterruptedException { 92 | # public void sendOneway(Message msg, MessageQueue mq) throws MQClientException, RemotingException, 93 | # InterruptedException { 94 | # public void sendOneway(Message msg, MessageQueueSelector selector, Object arg) throws MQClientException, 95 | # RemotingException, InterruptedException { 96 | """ 97 | logger.debug('Producer ' + self.instanceName + ' sending one-way message: ' + MQMsg.tostr()) 98 | return self.producer.sendOneway(MQMsg.msg) 99 | -------------------------------------------------------------------------------- /MQPullConsumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | from jpype import * 5 | import logging 6 | import time 7 | 8 | logger = logging.getLogger("MQPullConsumer") 9 | 10 | __all__ = ["MQPullConsumer"] 11 | 12 | DefaultMQPullConsumer= JPackage('com.alibaba.rocketmq.client.consumer').DefaultMQPullConsumer 13 | MQClientException = JPackage('com.alibaba.rocketmq.client.exception').MQClientException 14 | PullResult = JPackage('com.alibaba.rocketmq.client.consumer').PullResult 15 | MessageQueue = JPackage('com.alibaba.rocketmq.common.message').MessageQueue 16 | 17 | 18 | class MQPullConsumer(object): 19 | 20 | def __init__(self, groupName, namesrvAddr): 21 | """ 22 | :param groupName: 23 | :param namesrvAddr: 24 | :return: 25 | """ 26 | self.consumer = None #初始化放在了init函数中 27 | self.groupName = groupName 28 | self.namesrvAddr = namesrvAddr 29 | self.instanceName = str(int(time.time()*1000)) #毫秒值作为instance name 30 | 31 | self.mqs = None 32 | self.offseTable = {} # map of message queue id to queue offset 33 | 34 | def init(self): 35 | """批量设置一些基本项(为了尽可能少实现这些API接口,如以后有需要,可以逐个移出init)""" 36 | logger.info('Initializing consumer ' + self.instanceName + ' ...') 37 | self.consumer = DefaultMQPullConsumer(JString(self.groupName)) #创建实例 38 | self.consumer.setNamesrvAddr(JString(self.namesrvAddr)) 39 | self.consumer.setInstanceName(JString(self.instanceName)) 40 | 41 | def start(self): 42 | """ 43 | # JAVA prototype 44 | # public void start() throws MQClientException { 45 | """ 46 | logger.info('Starting consumer ' + self.instanceName + ' ...') 47 | self.consumer.start() 48 | 49 | def shutdown(self): 50 | """ 51 | # JAVA prototype 52 | # public void shutdown() { 53 | """ 54 | logger.info('Shutting down consumer ' + self.instanceName + ' ...') 55 | self.consumer.shutdown() 56 | 57 | def pullBlockIfNotFound(self, mq, subExpression, offset, maxNums): 58 | """ 59 | # JAVA prototype 60 | # public PullResult pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums) 61 | # throws MQClientException, RemotingException, MQBrokerException, InterruptedException { 62 | # public void pullBlockIfNotFound(MessageQueue mq, String subExpression, long offset, int maxNums, 63 | # PullCallback pullCallback) throws MQClientException, RemotingException, InterruptedException { 64 | """ 65 | pullResult = self.consumer.pullBlockIfNotFound(mq, subExpression, self.getMessageQueueOffset(mq), maxNums) 66 | return pullResult 67 | 68 | def fetchSubscribeMessageQueues(self, topic): 69 | """ 70 | # JAVA prototype 71 | # public Set fetchSubscribeMessageQueues(String topic) throws MQClientException { 72 | """ 73 | self.mqs = self.consumer.fetchSubscribeMessageQueues(JString(topic)) 74 | 75 | def getMessageQueueOffset(self, mq): 76 | """ 77 | 获取某个MQ中的当前消息的offset 78 | :param mq: 79 | :return: 80 | """ 81 | haskey = self.offseTable.has_key(mq.queueId) 82 | if haskey: 83 | return self.offseTable[mq.queueId] 84 | else: 85 | return 0 86 | 87 | def putMessageQueueOffset(self, mq, offset): 88 | """ 89 | 设置某个MQ中的当前消息的offset(更新后的值) 90 | :param mq: 91 | :param offset: 92 | :return: 93 | """ 94 | self.offseTable[mq.queueId] = offset 95 | -------------------------------------------------------------------------------- /MQPushConsumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | from jpype import * 5 | import logging 6 | import time 7 | import settings_MQ as settings 8 | 9 | logger = logging.getLogger("MQPullConsumer") 10 | 11 | __all__ = ["MQPushConsumer"] 12 | 13 | DefaultMQPushConsumer= JPackage('com.alibaba.rocketmq.client.consumer').DefaultMQPushConsumer 14 | MQClientException = JPackage('com.alibaba.rocketmq.client.exception').MQClientException 15 | #MessageExt = JPackage('com.alibaba.rocketmq.common.message').MessageExt 16 | #ConsumeConsurrentlyContext = JPackage('com.alibaba.rocketmq.client.consumer.listener').ConsumeConsurrentlyContext 17 | #ConsumeConsurrentlyStatus = JPackage('com.alibaba.rocketmq.client.consumer.listener').ConsumeConsurrentlyStatus 18 | #MessageListenerConcurrently = JPackage('com.alibaba.rocketmq.client.consumer.listener').MessageListenerConcurrently 19 | 20 | #from MQMessage import ConsumeConcurrentlyStatus, ConsumeOrderlyStatus 21 | 22 | class MQPushConsumer(object): 23 | '''实现类 24 | public class DefaultMQPushConsumer extends ClientConfig implements MQPushConsumer { 25 | ''' 26 | 27 | def __init__(self, groupName, namesrvAddr): 28 | """ 29 | :param groupName: 30 | :param namesrvAddr: 31 | :return: 32 | """ 33 | self.consumer = None #初始化放在了init函数中 34 | self.groupName = groupName 35 | self.namesrvAddr = namesrvAddr 36 | self.instanceName = str(int(time.time()*1000)) #毫秒值作为instance name 37 | 38 | def init(self): 39 | """批量设置一些基本项(为了尽可能少实现这些API接口,如以后有需要,可以逐个移出init)""" 40 | logger.info('Initializing consumer ' + self.instanceName + ' ...') 41 | self.consumer = DefaultMQPushConsumer(JString(self.groupName)) #创建实例 42 | self.consumer.setNamesrvAddr(JString(self.namesrvAddr)) 43 | self.consumer.setInstanceName(JString(self.instanceName)) 44 | 45 | def start(self): 46 | """ 47 | # JAVA prototype 48 | # public void start() throws MQClientException { 49 | """ 50 | logger.info('Starting consumer ' + self.instanceName + ' ...') 51 | self.consumer.start() 52 | 53 | def shutdown(self): 54 | """ 55 | # JAVA prototype 56 | # public void shutdown() { 57 | """ 58 | logger.info('Shutting down consumer ' + self.instanceName + ' ...') 59 | self.consumer.shutdown() 60 | 61 | def setMessageModel(self, messageModel): 62 | """ 63 | # JAVA prototype 64 | # public void setMessageModel(MessageModel messageModel) 65 | """ 66 | logger.info('Setting message model of instance ' + self.instanceName + ' to ' + str(messageModel)) 67 | #self.consumer.setMessageModel(JObject(messageModel, "com.alibaba.rocketmq.common.protocol.heartbeat.MessageModel")) 68 | self.consumer.setMessageModel(messageModel) 69 | 70 | def subscribe(self, topic, subExpression): 71 | # JAVA prototype 72 | # public void subscribe(String topic, String subExpression) throws MQClientException { 73 | # public void subscribe(String topic, String fullClassName, String filterClassSource) throws MQClientException { 74 | self.consumer.subscribe(JString(topic), JString(subExpression)) 75 | 76 | def unsubscribe(self, topic): 77 | # JAVA prototype 78 | # public void unsubscribe(String topic) { 79 | self.consumer.unsubscribe(JString(topic)) 80 | 81 | def setConsumeFromWhere(self, fromwhere): 82 | # JAVA prototype 83 | # public void setConsumeFromWhere(ConsumeFromWhere consumeFromWhere) { 84 | #self.consumer.setConsumeFromWhere(JObject(fromwhere, "com.alibaba.rocketmq.common.consumer.ConsumeFromWhere")) 85 | self.consumer.setConsumeFromWhere(fromwhere) 86 | 87 | def registerMessageListener(self, listener): 88 | # JAVA prototype 89 | # public void registerMessageListener(MessageListenerConcurrently messageListener) { 90 | # public void registerMessageListener(MessageListenerOrderly messageListener) { 91 | self.consumer.registerMessageListener(listener) 92 | 93 | # 下面内容移除到MQMesasgeListener.py中 94 | # 95 | # class MessageListenerConcurrently: 96 | # '''接口类MessageListenerConcurrently的实现 97 | # public interface MessageListenerConcurrently extends MessageListener { 98 | # ''' 99 | # def consumeMessage(self, msgs, context): 100 | # ''' 101 | # # JAVA prototype 102 | # # ConsumeConcurrentlyStatus consumeMessage(final List msgs, final ConsumeConcurrentlyContext context); 103 | # ''' 104 | # logger.debug("Into consumerMessage of MessageListenerConcurrently") 105 | # #msg = msgs.get(JInt(0)) 106 | # for msg in msgs: 107 | # topic = msg.getTopic() 108 | # tags = msg.getTags() 109 | # body = str(msg.getBody()).decode(settings.MsgBodyEncoding) 110 | # 111 | # logger.debug(msg.toString()) 112 | # # In Python 2.x, bytes is just an alias for str. 所以bytes解码时要注意了, msg.body.decode会出错(bytes没有decode方法)！ 113 | # #logger.debug("Message body: " + str(msg.getBody())) 114 | # #logger.debug("Message body: " + str(msg.getBody()).decode(settings.MsgBodyEncoding)) 115 | # logger.debug("Message body: " + body) 116 | # 117 | # if topic == "RMQTopicTest": 118 | # # 执行TopicTest的消费逻辑 119 | # if tags == "TagA": 120 | # # 执行TagA的消费 121 | # logger.debug("Got message with topic " + topic + " and tags " + tags) 122 | # elif tags == "TagB": 123 | # # 执行TagB的消费 124 | # logger.debug("Got message with topic " + topic + " and tags " + tags) 125 | # elif tags == "TagC": 126 | # # 执行TagC的消费 127 | # logger.debug("Got message with topic " + topic + " and tags " + tags) 128 | # else: 129 | # # 错误的Tag 130 | # logger.error("Got message with topic " + topic + " and UNKNOWN tags " + tags) 131 | # elif topic == "TopicTest1": 132 | # # 执行TopicTest1的消费逻辑 133 | # logger.debug("Got message with topic " + topic + " and tags " + tags) 134 | # else: 135 | # logger.debug("Got message with UNKNOWN topic " + topic ) 136 | # 137 | # return ConsumeConcurrentlyStatus['CONSUME_SUCCESS'] 138 | # 139 | # #实现 140 | # msgListenerConcurrently = MessageListenerConcurrently() 141 | # #JProxy("MessageListenerConcurrently", inst = msgListenerConcurrently) 142 | # msgListenerConcurrentlyProxy = JProxy("com.alibaba.rocketmq.client.consumer.listener.MessageListenerConcurrently", inst = msgListenerConcurrently) 143 | # 144 | # class MessageListenerOrderly: 145 | # '''接口类MessageListenerOrderly的实现 146 | # public interface MessageListenerOrderly extends MessageListener { 147 | # ''' 148 | # def __init__(self): 149 | # #JAVA原子类 150 | # self.consumeTimes = java.util.concurrent.atomic.AtomicLong(0) 151 | # 152 | # def consumeMessage(self, msgs, context): 153 | # # JAVA prototype 154 | # # ConsumeOrderlyStatus consumeMessage(final List msgs, final ConsumeOrderlyContext context); 155 | # context.setAutoCommit(False) 156 | # logger.debug(java.lang.Thread.currentThread().getName() + " Receive New Messages: " + msgs.toString()) 157 | # #TODO: msgs.toString()可能需要改成for msg in msgs: msg.toString() 158 | # 159 | # self.consumeTimes.incrementAndGet() 160 | # consumeTimes = self.consumeTimes.get() 161 | # # print consumeTimes 162 | # # print type(consumeTimes) 163 | # 164 | # if (consumeTimes % 2) == 0: 165 | # logger.debug("consumeTimes % 2 = 0, return SUCCESS") 166 | # return ConsumeOrderlyStatus['SUCCESS'] 167 | # elif (consumeTimes % 3) == 0: 168 | # logger.debug("consumeTimes % 3 = 0, return ROLLBACK") 169 | # return ConsumeOrderlyStatus['ROLLBACK'] 170 | # elif (consumeTimes % 4) == 0: 171 | # logger.debug("consumeTimes % 4 = 0, return COMMIT") 172 | # return ConsumeOrderlyStatus['COMMIT'] 173 | # elif (consumeTimes % 5) == 0: 174 | # logger.debug("consumeTimes % 5 = 0, return SUSPEND_CURRENT_QUEUE_A_MOMENT") 175 | # context.setSuspendCurrentQueueTimeMillis(3000) 176 | # return ConsumeOrderlyStatus['SUSPEND_CURRENT_QUEUE_A_MOMENT'] 177 | # else: 178 | # logger.debug("consumeTimes is not times of 2, 3, 4, 5, return SUCCESS") 179 | # return ConsumeOrderlyStatus['SUCCESS'] 180 | # 181 | # #实现 182 | # msgListenerOrderly = MessageListenerOrderly() 183 | # #JProxy("MessageListenerOrderly", inst = msgListenerOrderly) 184 | # msgListenerOrderlyProxy = JProxy("com.alibaba.rocketmq.client.consumer.listener.MessageListenerOrderly", inst = msgListenerOrderly) 185 | # 186 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RMQ-Client4Python 2 | Python client module for RocketMQ 3 | 4 | 5 | ## Dependencies 6 | JAVA libraries: 7 | RocketMQ libs 8 | Python libraries: 9 | jpype 10 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GangLuICT/RMQ-Client4Python/5a7cfe2efc1e6c009090b891d0a4fb0dc03ce3e4/__init__.py -------------------------------------------------------------------------------- /jpypetest.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | import os 5 | import logging 6 | import jpype 7 | 8 | # 为了支持中文输入，要显式设置编码 9 | import sys 10 | if sys.getdefaultencoding() != 'utf-8': 11 | reload(sys) 12 | sys.setdefaultencoding('utf-8'); 13 | 14 | if __name__ == "__main__": 15 | jvmPath = jpype.getDefaultJVMPath() 16 | print jvmPath 17 | jpype.startJVM(jvmPath) 18 | #jpype.startJVM(jvmPath, "-Xms32m", "-Xmx256m", "-mx256m", "-Djava.class.path=/Users/tan9le/temp/some-lib.jar:") 19 | jpype.java.lang.System.out.println( " hello world! " ) 20 | map = jpype.JClass("java.util.HashMap")() 21 | map.put("123","123124测试中文") 22 | print map.get("123") 23 | jpype.shutdownJVM() 24 | 25 | -------------------------------------------------------------------------------- /settings_MQ.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | import logging 5 | logging.basicConfig(level=logging.DEBUG, 6 | format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s', 7 | datefmt='%a, %d %b %Y %H:%M:%S', 8 | filename='rmq.log', 9 | filemode='a') 10 | 11 | RMQClientJAR = '/home/deploy/rocketmq/alibaba-rocketmq/lib/' 12 | JAVA_EXT_DIRS = RMQClientJAR 13 | 14 | #startJVM中的options参数不能包含空格！只能一项一项填写 15 | #JVM_OPTIONS = '-Xms32m -Xmx256m -mx256m' 16 | JVM_OPTIONS = '-mx256m' 17 | 18 | pullMaxNums = 32 19 | MsgBodyEncoding = 'utf-8' 20 | -------------------------------------------------------------------------------- /test/Producer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | import logging 5 | logger = logging.getLogger("Producer") 6 | 7 | #导入上级目录模块 8 | import sys 9 | sys.path.append("..") 10 | import settings_MQ as settings 11 | 12 | #启动JVM 13 | from jpype import * 14 | jvmPath = getDefaultJVMPath() 15 | startJVM(jvmPath, settings.JVM_OPTIONS, "-Djava.ext.dirs="+settings.JAVA_EXT_DIRS) 16 | #startJVM(jvmPath, "-Djava.class.path=" + settings.RMQClientJAR + ":") 17 | logger.info(java.lang.System.getProperty("java.class.path")) 18 | logger.info(java.lang.System.getProperty("java.ext.dirs")) 19 | 20 | #启动JVM之后才能调用JPackage,否则找不到相关的jar包 21 | from MQProducer import * 22 | from MQMessage import MQMessage 23 | 24 | # 为了支持文本中文输入，要显式设置编码；该编码不影响Message的Body的编码 25 | import sys 26 | if sys.getdefaultencoding() != 'utf-8': 27 | reload(sys) 28 | sys.setdefaultencoding('utf-8'); 29 | 30 | ############################################################################### 31 | if __name__ == '__main__': 32 | #jpype.startJVM(jvmPath, "-Xms32m", "-Xmx256m", "-mx256m", "-Djava.class.path=/Users/tan9le/temp/some-lib.jar:") 33 | java.lang.System.out.println("Java runs correct!") 34 | 35 | producer = MQProducer('MQClient4Python-Producer', 'jfxr-7:9876;jfxr-6:9876') 36 | producer.init() 37 | producer.start() 38 | MQMsg = MQMessage('RMQTopicTest', # topic 39 | 'TagB', # tag 40 | 'OrderID001', # key 41 | 'Hello, RocketMQ!') # body 42 | sendResult = producer.send(MQMsg) 43 | MQMsg = MQMessage('RMQTopicTest', # topic 44 | 'TagC', # tag 45 | 'OrderID001', # key 46 | 'Hello, RocketMQ! I am 陆钢') # body 47 | sendResult = producer.send(MQMsg) 48 | 49 | producer.shutdown() 50 | 51 | shutdownJVM() 52 | -------------------------------------------------------------------------------- /test/PullConsumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | import logging 5 | logger = logging.getLogger("PullConsumer") 6 | 7 | #导入上级目录模块 8 | import sys 9 | sys.path.append("..") 10 | import settings_MQ as settings 11 | 12 | #启动JVM 13 | from jpype import * 14 | jvmPath = getDefaultJVMPath() 15 | startJVM(jvmPath, "-ea", "-Djava.ext.dirs=" + settings.JAVA_EXT_DIRS) 16 | #startJVM(jvmPath, "-Djava.class.path=" + settings.RMQClientJAR + ":") 17 | print settings.JAVA_EXT_DIRS 18 | logger.info(java.lang.System.getProperty("java.class.path")) 19 | logger.info(java.lang.System.getProperty("java.ext.dirs")) 20 | 21 | #启动JVM之后才能调用JPackage,否则找不到相关的jar包 22 | from MQPullConsumer import * 23 | from MQMessage import MQMessage, PullStatus 24 | 25 | # 为了支持文本中文输入，要显式设置编码；该编码不影响Message的Body的编码 26 | import sys 27 | if sys.getdefaultencoding() != 'utf-8': 28 | reload(sys) 29 | sys.setdefaultencoding('utf-8'); 30 | 31 | if __name__ == '__main__': 32 | consumer = MQPullConsumer('MQClient4Python-Consumer', 'jfxr-7:9876;jfxr-6:9876') 33 | consumer.init() 34 | consumer.start() 35 | 36 | consumer.fetchSubscribeMessageQueues("RMQTopicTest") #获取所有消息队列,返回值存储到consumer.mqs 37 | #TODO: 38 | # 1. fetchSubscribeMessageQueues可能返回异常：Can not find Message Queue for this topic 39 | # 如果不存在Topic，则创建topic：createTopic(String key, String newTopic, int queueNum) 40 | # 2. MessageQueue的数目，如何确定的！ 41 | # 3. PullConsumer没有ConsumeFromWhere这项设置 42 | 43 | while True: 44 | for mq in consumer.mqs: 45 | logger.debug("Pulling from message queue: " + str(mq.queueId)) 46 | while True: 47 | try: 48 | pullResult = consumer.pullBlockIfNotFound(mq, '', consumer.getMessageQueueOffset(mq), settings.pullMaxNums) # brokerSuspendMaxTimeMillis默认值是20s 49 | consumer.putMessageQueueOffset(mq, pullResult.getNextBeginOffset()) 50 | pullStatus = PullStatus[str(pullResult.getPullStatus())] # JAVA中的enum对应到Python中没有转换为Int，enum对象转换为string的时候是其枚举值的名字，而不是enum的值（0,1...）！ 51 | #print pullStatus 52 | #print type(pullStatus) 53 | #print type(PullStatus['NO_MATCHED_MSG']) 54 | #print pullStatus == PullStatus['NO_MATCHED_MSG'] # 比较对象时先将其转换成了string 55 | if pullStatus == PullStatus['FOUND']: 56 | logger.debug('Found') 57 | logger.debug(pullResult.toString()) 58 | msgList = pullResult.getMsgFoundList() 59 | for msg in msgList: 60 | logger.debug(msg.toString()) 61 | # In Python 2.x, bytes is just an alias for str. 所以bytes解码时要注意了, msg.body.decode会出错！ 62 | #logger.debug("Message body: " + str(msg.body)) 63 | logger.debug("Message body: " + str(msg.body).decode(settings.MsgBodyEncoding)) 64 | #TODO: 进一步分析pull下来的result 65 | elif pullStatus == PullStatus['NO_NEW_MSG']: 66 | logger.debug('NO_NEW_MSG') 67 | break 68 | elif pullStatus == PullStatus['NO_MATCHED_MSG']: 69 | logger.debug('NO_MATCHED_MSG') 70 | elif pullStatus == PullStatus['OFFSET_ILLEGAL']: 71 | logger.debug('OFFSET_ILLEGAL') 72 | else: 73 | logger.error('Wrong pull status: ' + str(pullStatus)) 74 | except JavaException, ex: 75 | logger.error(str(ex.javaClass()) + str(ex.message())) 76 | logger.error(str(ex.stacktrace())) 77 | 78 | consumer.shutdown() 79 | 80 | shutdownJVM() 81 | -------------------------------------------------------------------------------- /test/PushConsumer.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # -*- coding:utf-8 -*- 3 | 4 | import logging 5 | logger = logging.getLogger("PushConsumer") 6 | 7 | #导入上级目录模块 8 | import sys 9 | sys.path.append("..") 10 | import settings_MQ as settings 11 | 12 | #启动JVM 13 | from jpype import * 14 | jvmPath = getDefaultJVMPath() 15 | startJVM(jvmPath, settings.JVM_OPTIONS, "-Djava.ext.dirs="+settings.JAVA_EXT_DIRS) 16 | #startJVM(jvmPath, "-Djava.class.path=" + settings.RMQClientJAR + ":") 17 | logger.info(java.lang.System.getProperty("java.class.path")) 18 | logger.info(java.lang.System.getProperty("java.ext.dirs")) 19 | 20 | #启动JVM之后才能调用JPackage,否则找不到相关的jar包 21 | from MQPushConsumer import MQPushConsumer 22 | from MQMessageListener import msgListenerConcurrentlyProxy, msgListenerOrderlyProxy 23 | from MQMessage import ConsumeFromWhere, MessageModel 24 | 25 | # 为了支持文本中文输入，要显式设置编码；该编码不影响Message的Body的编码 26 | import sys 27 | if sys.getdefaultencoding() != 'utf-8': 28 | reload(sys) 29 | sys.setdefaultencoding('utf-8'); 30 | 31 | import time 32 | 33 | if __name__ == '__main__': 34 | consumer = MQPushConsumer('MQClient4Python-Consumer', 'jfxr-7:9876;jfxr-6:9876') 35 | consumer.init() 36 | 37 | consumer.setMessageModel(MessageModel['CLUSTERING']) # 默认是CLUSTERING 38 | #consumer.setMessageModel(MessageModel.CLUSTERING) # 默认是CLUSTERING 39 | 40 | consumer.subscribe("RMQTopicTest", "TagB") 41 | 42 | consumer.setConsumeFromWhere(ConsumeFromWhere['CONSUME_FROM_LAST_OFFSET']) 43 | #consumer.setConsumeFromWhere(ConsumeFromWhere.CONSUME_FROM_LAST_OFFSET) 44 | 45 | #consumer.registerMessageListener(msgListenerConcurrentlyProxy) 46 | consumer.registerMessageListener(msgListenerOrderlyProxy) 47 | 48 | consumer.start() 49 | 50 | while True: 51 | time.sleep(1) 52 | 53 | #监听状态时不需要shutdown,除非真实想退出! 54 | #consumer.shutdown() 55 | #监听状态时JVM也不能退出,除非真实想退出! 56 | #shutdownJVM() 57 | --------------------------------------------------------------------------------