├── .gitignore ├── LICENSE ├── README.md ├── README.zh-CN.md ├── composer.json ├── example.php ├── phpunit.sh ├── src ├── Consumer.php └── ZkUtils.php └── tests ├── README.md ├── ZkUtilsTest.php ├── consumerTest.sh ├── phpunit.xml └── runExample.php /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | vendor/ 3 | composer.lock 4 | *.swp 5 | *.swo 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # php-consumergroup 2 | 3 | php-consumergroup is a kafka consumer library with group and rebalance supports. 4 | 5 | [Chinese Doc](./README.zh-CN.md) 6 | 7 | ## Requirements 8 | 9 | * Apache Kafka 0.7.x, 0.8.x, 0.9.x, 0.10.x 10 | 11 | ## Dependencies 12 | 13 | * [php-zookeeper](https://github.com/php-zookeeper/php-zookeeper) 14 | * [php_rdkafka](https://github.com/arnaud-lb/php-rdkafka/releases/tag/1.0.0) (1.0.0 is recommended) 15 | * [librdkafka](https://github.com/edenhill/librdkafka/releases/tag/0.9.1) (0.9.1 is recommended) 16 | 17 | ## Performance 18 | 19 | `78,000+` messages/s for single process 20 | 21 | more detail [benchmark](#benchmark) 22 | 23 | ## Example 24 | 25 | * installing this library via composer 26 | 27 | ``` 28 | payload\n"; 33 | } 34 | 35 | function handle_error_call_back($msg) { 36 | echo $msg->errstr(); 37 | } 38 | 39 | $consumer = New Consumer("localhost:2181"); 40 | $consumer->setGroupId("group-test"); 41 | $consumer->setTopic("topic-test"); 42 | $consumer->setOffsetAutoReset(Consumer::SMALLEST); 43 | $consumer->setErrHandler("handle_error_call_back"); 44 | 45 | try { 46 | $consumer->start("echo_message"); 47 | } 48 | catch(Exception $e) { 49 | printf("error: %s\n", $e->getMessage()); 50 | } 51 | ``` 52 | 53 | see [example.php](./example.php) 54 | 55 | ## Consumer Options 56 | 57 | ##### Consumer::setMaxMessage() 58 | 59 | Number, defaults to `32` 60 | 61 | If partitions > 1, it forces consumers to switch to other partitons when max message is reached, or other partitons will be starved 62 | 63 | ##### Consumer::setCommitInterval() 64 | 65 | Millisecond, defaults to `500ms` 66 | 67 | Offset auto commit interval. 68 | 69 | ##### Consumer::setWatchInterval() 70 | 71 | Millisecond, defaults to `10,000 ms` 72 | 73 | Time interval to check rebalance. Rebalance is triggered when the number of partition or consumer changes. 74 | 75 | ##### Consumer::setConsumeTimeout() 76 | 77 | Millisecond, default is `1,000 ms` 78 | 79 | Kafka request timeout. 80 | 81 | ##### Consumer::setClientId() 82 | 83 | String, defaults to `"default"` 84 | 85 | Client id is used to identify consumers. 86 | 87 | ##### Consumer::setOffsetAutoReset() 88 | 89 | `smallest|largest`, defaults to `smallest` 90 | 91 | Consumer can choose whether to fetch the oldest or the lastest message when offset isn't present in zookeeper or is out of range. 92 | 93 | ##### Consumer::setConf() 94 | Attribute and value are passed in 95 | 96 | We can use this function to modify the librdkafka configuration。 97 | 98 | more detail about [librdkafka configuration](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md). 99 | 100 | ## Exception 101 | 102 | * Recoverable exceptption (e.g. request timeout), error handler will be called, and you can log error messages. 103 | * Unrecoverable exception (e.g. kafka/zookeeper is broken), exceptions will be thrown, and you should log message and stop the consumer. 104 | 105 | ## Benchmark 106 | 107 | |Type|Parmeter| 108 | |---|---| 109 | |CPU|Intel(R) Xeon(R) CPU E5-2420 0 @ 1.90GHz| 110 | |CPU Core|24| 111 | |Memory|16G| 112 | |Disk|SSD| 113 | |Network|1000Mbit/s| 114 | |Os|CentOS release 6.7| 115 | 116 | Benchmark is measured by produring `20,000,000` messages at single partition, and calculate the time it takes to consume those messages. 117 | 118 | QPS is `78,000` messages/s when process cpu utility is `100%`. 119 | -------------------------------------------------------------------------------- /README.zh-CN.md: -------------------------------------------------------------------------------- 1 | ## php-consumergroup 2 | ### 简介 3 | 主要是对php_rdkafka的consumer的api进行了一层封装，增加了原程序中所没有的与zookeeper交互的功能。在此基础上实现了rebalance功能以及group功能。 4 | producer及其他相关的内容可以参考[php_rdkafka](https://github.com/arnaud-lb/php-rdkafka) 中的相关文档。 5 | 经过简单的压力测试，单个进程的消费能力能达到每秒钟7.8W条，压测详细内容见[压力测试](#压力测试)。 6 | 7 | ### 依赖 8 | [php_zookeeper](https://github.com/php-zookeeper/php-zookeeper) 9 | 10 | [php_rdkafka](https://github.com/arnaud-lb/php-rdkafka/releases/tag/1.0.0) （建议使用1.0.0版本） 11 | 12 | [librdkafka](https://github.com/edenhill/librdkafka/releases/tag/0.9.1)（建议使用0.9.1版本） 13 | 14 | ### 使用 15 | 16 | * 通过composer引入该类库 17 | 18 | ``` 19 | payload\n"; 24 | } 25 | 26 | function handle_error_call_back($msg) { 27 | echo $msg->errstr(); 28 | } 29 | 30 | $consumer = New Consumer("localhost:2181"); 31 | $consumer->setGroupId("group-test"); 32 | $consumer->setTopic("topic-test"); 33 | $consumer->setOffsetAutoReset(Consumer::smallest); 34 | $consumer->setErrHandler("handle_error_call_back"); 35 | 36 | try { 37 | $consumer->start("echo_message"); 38 | } 39 | catch(Exception $e) { 40 | printf("error: %s\n", $e->getMessage()); 41 | } 42 | 43 | ``` 44 | 45 | 更详细的例子见[example](./example.php)。 46 | 47 | ### 关于异常处理 48 | * 对于一些可恢复的异常，比如获取消息超时之类的，提供回调函数，建议的做法是把这些异常都记录到日志中，方便到时排查问题。回调函数执行完毕后会继续进行消费。 49 | * 对于一些不可恢复的异常，诸如kafka挂掉了，zookeeper不可用之类的，抛出到最外层，由使用者决定该如何操作。建议的做法是记录日志之后退出消费，等到异常问题排查之后，再重新开始消费。 50 | 51 | ### 配置参数 52 | 目前支持的配置项较少，以后会根据需求再进行增加。 53 | #### Consumer::setMaxMessage() 54 | 一个整数，表示当一个consumer轮询消费多个partition的时候，每个partition最多能消费多少条信息。用来避免只消费一个partition，而忽略了其他的partition。默认值是1000。 55 | 56 | #### Consumer::setCommitInterval() 57 | 一个整数，超过这个时间，consumer就自动提交一次每个partition的offset到zookeeper。减少因consumer进程突然死亡而造成的重复消费的消息的条数。默认值是500ms。 58 | 59 | #### Consumer::setWatchInterval() 60 | 一个整数，超过这个时间，consumer就会开始检查是否有consumer或者partition的变动。如果有变化的话就会自行rebalance。默认值是10000ms。 61 | 62 | #### Consumer::setConsumeTimeout() 63 | 一个整数，向server发送请求获取的超时时间。默认是1000ms。 64 | 65 | #### Consumer::setClientId() 66 | 一个字符串，加在consumerId的最后，用来表示是当前consumer是属于哪个应用的。 67 | 68 | #### Consumer::setOffsetAutoReset() 69 | 可选值为smallest和largest。如果zookeeper中没有offset或者offset超出了kafka中消息的范围，将会自动选择从最小的offset开始消费还是从最新的offset开始消费。如果选择smallest，将会从kafka中保存的消息中最旧的一条开始消费。如果选择largest，将会从最新的一条消息开始消费，即consumer启动后写入到kafka的第一条开始。 70 | 71 | #### Consumer::setConf() 72 | 通过这个函数可以配置librdkafk的配置项，传入两个参数，分别是attribute及value。详细的配置项列表可以见[librdkafka的官方文档](https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md)。 73 | 74 | ### 注意事项 75 | 1) 虽然consumer在zookeeper上注册都是临时节点，超过一定时间没有心跳包发送将会自动掉线，也能触发rebalance。但是为了能使在主动关闭的情况下做到, 其他consumer更快察觉到有consumer关闭，更早进行rebalance 76 | 所以建议使用时，安装信号处理器，在收到进程退出的信号时调用Consumer::stop()，使consumer能正常退出, 关闭前提交offset，尽量避免重复消费。。详细例子见[example](./example.php)。 77 | 78 | 2) consumer在运行中如果连续报严重错误一定次数(当前写死为256次)之后会抛出一个"kafka server is not available."的Exception。严重错误包括kafka server挂掉、server端所在机器上的系统损坏等。如果正常消费会将计数清零。因为consumer内有缓存队列，每个partition 1M的大小，所以必须等这部分消耗完之后再报错连续一定次数才会抛出这个Exception。 79 | 80 | ### 压力测试 81 | 当前进行了简单的压力测试，创建了一个单partition的topic，开启一个客户端进程对客户端性能进行压测。配置项全部使用了默认配置。瓶颈点出现在客户端的cpu占用率上，达到了100%。读取2000W条数据，最后计算出来，每秒平均消费大概7.8W条数据。 82 | 83 | #### 测试时客户端的机器配置 84 | |类型|参数| 85 | |---|---| 86 | |CPU|Intel(R) Xeon(R) CPU E5-2420 0 @ 1.90GHz| 87 | |核数|24| 88 | |内存|16G| 89 | |硬盘|SSD| 90 | |网卡|千兆网卡| 91 | |操作系统|CentOS release 6.7| 92 | 93 | 94 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "meitu/php-consumergroup", 3 | "description": "php kafka consumer", 4 | "license": "Apache-2.0", 5 | "authors": [ 6 | { 7 | "name": "lzf", 8 | "email": "lzf@meitu.com" 9 | } 10 | ], 11 | "require-dev" : { 12 | "phpunit/phpunit": "5.6.*" 13 | }, 14 | "autoload" : { 15 | "psr-4" : {"MTKafka\\" : "src/"} 16 | } 17 | } -------------------------------------------------------------------------------- /example.php: -------------------------------------------------------------------------------- 1 | topic_name, $msg->partition, $msg->errstr()); 30 | } 31 | 32 | //ticks 33 | declare(ticks=1); 34 | 35 | //set config 36 | $zkAddress = "localhost:2181"; 37 | $topic = "php-test"; 38 | $groupId = "group-test-1"; 39 | $maxMessage = 10; 40 | 41 | $consumer = New Consumer($zkAddress); 42 | $consumer->setGroupId($groupId); 43 | $consumer->setTopic($topic); 44 | $consumer->setMaxMessage($maxMessage); 45 | $consumer->setClientId("test"); 46 | $consumer->setOffsetAutoReset(Consumer::SMALLEST); 47 | $consumer->setErrHandler("handleError"); 48 | 49 | //install signal processor 50 | pcntl_signal(SIGHUP, "sig_handler"); 51 | pcntl_signal(SIGINT, "sig_handler"); 52 | pcntl_signal(SIGQUIT, "sig_handler"); 53 | pcntl_signal(SIGTERM, "sig_handler"); 54 | 55 | //start to consume message 56 | try { 57 | $consumer->start("echo_message"); 58 | } 59 | catch(Exception $e) { 60 | printf("error: %s\n", $e->getMessage()); 61 | } 62 | -------------------------------------------------------------------------------- /phpunit.sh: -------------------------------------------------------------------------------- 1 | vendor/bin/phpunit --configuration tests/phpunit.xml --colors tests -------------------------------------------------------------------------------- /src/Consumer.php: -------------------------------------------------------------------------------- 1 | zkUtils = new ZkUtils($zkAddress, $sessionTimeout, $chroot); 44 | $this->topic = null; 45 | $this->maxMessage = 32; 46 | $this->consumerIdPrefix = gethostbyname(gethostname()) . "-" . getmypid() . "-" . microtime(true); 47 | $this->consumerId = $this->consumerIdPrefix . "default"; 48 | $this->commitInterval = 500; 49 | $this->watchInterval = 10000; 50 | $this->consumeTimeout = 1000; 51 | $this->errHandler = function ($msg) { 52 | printf("partition: %d , err: %s \n", $msg->partition, $msg->errstr()); 53 | }; 54 | 55 | $this->offsetAutoReset = self::SMALLEST; 56 | 57 | $this->conf = new \Rdkafka\Conf(); 58 | $this->conf->set('broker.version.fallback', '0.8.2'); 59 | $this->conf->set('queued.max.messages.kbytes', 1024); 60 | $this->conf->set('topic.metadata.refresh.interval.ms', 60000); 61 | $this->conf->set('fetch.message.max.bytes', 1048576); 62 | $this->conf->setErrorCb(function ($kafka, $err, $reason) { 63 | printf("Kafka error: %s (reason: %s)\n", rd_kafka_err2str($err), $reason); 64 | global $__serverDownTimes; 65 | $__serverDownTimes += 1; 66 | if ($__serverDownTimes > 256) { 67 | throw new \Exception ("broker is not available"); 68 | } 69 | }); 70 | } 71 | 72 | /** 73 | * Modify the librdkafka configuration 74 | * 75 | * @var String $attribute config attribute 76 | * @var $value config value 77 | */ 78 | public function setConf($attribute, $value) { 79 | $this->conf->set($attribute, $value); 80 | } 81 | 82 | /** 83 | * Set consumer group Id, this value must be set 84 | * 85 | * @var String $groupId consumer group Id 86 | */ 87 | public function setGroupId($groupId) { 88 | $this->groupId = $groupId; 89 | } 90 | 91 | /** 92 | * Set topic, this value must be set 93 | * 94 | * @var String $topic topic 95 | */ 96 | public function setTopic($topic) { 97 | $this->topic = $topic; 98 | } 99 | 100 | /** 101 | * If partitions > 1, it forces consumers to switch to other 102 | * partitons when max message is reached, or other partitons 103 | * will be starved. 104 | * 105 | * @var Int $maxMessage max message number, defaults to 32 106 | */ 107 | public function setMaxMessage($maxMessage) { 108 | $this->maxMessage = $maxMessage; 109 | } 110 | 111 | /** 112 | * Set offset auto commit interval. 113 | * 114 | * @var Ind $commitInterval the unit is milliseconds, defaults to 500 115 | */ 116 | public function setCommitInterval($commitInterval) { 117 | $this->commitInterval = $commitInterval; 118 | } 119 | 120 | /** 121 | * Set time interval to check rebalance. Rebalance is triggered 122 | * when the number of partition or consumer changes. 123 | * 124 | * @var Int $watchInterval the unit is milliseconds, defaults to 10000 125 | */ 126 | public function setWatchInterval($watchInterval) { 127 | $this->watchInterval = $watchInterval; 128 | } 129 | 130 | /** 131 | * Set kafka request timeout. 132 | * 133 | * @var Int $consumeTimeout the unit is milliseconds, defaults to 1000 134 | */ 135 | public function setConsumeTimeout($consumeTimeout) { 136 | $this->consumeTimeout = $consumeTimeout; 137 | } 138 | 139 | /** 140 | * Set client id is used to identify consumers 141 | * 142 | * @var String $clientId client id, defaults to "default" 143 | */ 144 | public function setClientId($clientId) { 145 | $this->consumerId = $this->consumerIdPrefix . '-' . $clientId; 146 | } 147 | 148 | /** 149 | * Set a callback function is used to handle error 150 | * 151 | * @var Function $errorHandler error handle functioin 152 | */ 153 | public function setErrHandler($errHandler) { 154 | $this->errHandler = $errHandler; 155 | } 156 | 157 | /** 158 | * Set a callback function is used to handle when consumer offset 159 | * reach the end of partition 160 | * 161 | * @var Function $eofHandler eof handle function 162 | */ 163 | public function setEofHandler($eofHandler) { 164 | $this->eofHandler = $eofHandler; 165 | } 166 | 167 | /** 168 | * Set offset auto reset rule. Consumer can choose whether to fetch 169 | * the oldest or the lastest message when offset isn't present in 170 | * zookeeper or is out of range. 171 | * 172 | * @var Int $autoReset smallest or largest, defaults to samllest 173 | */ 174 | public function setOffsetAutoReset($autoReset) { 175 | if ($autoReset === self::SMALLEST || $autoReset === self::LARGEST) { 176 | $this->offsetAutoReset = $autoReset; 177 | } else { 178 | throw new \Exception ("offsetAutoReset must be smallest or largest"); 179 | } 180 | } 181 | 182 | //get current time 183 | private function getTime() { 184 | return microtime(true) * 1000; 185 | } 186 | 187 | private function rebalance() { 188 | $cnt = count($this->consumers); 189 | $this->consumedPartitions = array(); 190 | for ($i=0; $ipartitions); $i++) { 191 | if ($this->consumers[$i % $cnt] === $this->consumerId) 192 | array_push($this->consumedPartitions, $this->partitions[$i]); 193 | } 194 | } 195 | 196 | private function checkOwner() { 197 | $curr = $this->currentPartitions; 198 | $csm = $this->consumedPartitions; 199 | if ($curr === $csm) return; 200 | 201 | //release unnecessary partition after rebalance 202 | $diff = array_values(array_diff($curr, $csm)); 203 | foreach($diff as $partition) { 204 | if (!$this->zkUtils->releasePartitionOwnership($this->topic, $this->groupId, 205 | $partition)) { 206 | throw new \Exception ("failed to release partition ownership"); 207 | } else { 208 | array_splice($curr, array_search($partition, $curr), 1); 209 | $this->zkUtils->commitOffset($this->topic, $this->groupId, 210 | $partition, $this->offsets[$partition]); 211 | $this->rkTopic->consumeStop($partition); 212 | } 213 | } 214 | 215 | //register owner for new partitions after rebalance 216 | $diff = array_values(array_diff($csm, $curr)); 217 | foreach($diff as $partition) { 218 | if ($this->zkUtils->registerOwner($this->topic, $this->groupId, 219 | $partition, $this->consumerId)) { 220 | array_push($curr, $partition); 221 | $offset = $this->zkUtils->getOffset($this->topic, $this->groupId, $partition); 222 | if ($offset < 0) { 223 | $this->rkTopic->consumeStart($partition, $this->offsetAutoReset === self::SMALLEST ? -2 : -1 ); 224 | } else { 225 | $this->rkTopic->consumeStart($partition, $offset); 226 | } 227 | $this->offsets[$partition] = $offset; 228 | $this->prevOffsets[$partition] = $offset; 229 | } 230 | } 231 | sort($curr); 232 | $this->currentPartitions = $curr; 233 | } 234 | 235 | private function needRebalance() { 236 | if (empty($this->currentPartitions)) { 237 | usleep(100000); 238 | } else if ($this->getTime() - $this->lastWatchTime < $this->watchInterval) { 239 | return false; 240 | } 241 | $needRebalance = false; 242 | $this->lastWatchTime = $this->getTime(); 243 | 244 | //trigger rebalance when partition or consumer number has changed 245 | $partitions = $this->zkUtils->getPartitions($this->topic); 246 | if (empty($partitions)) { 247 | return false; 248 | } 249 | sort($partitions); 250 | if ($partitions !== $this->partitions) { 251 | $this->partitions = $partitions; 252 | $needRebalance= true; 253 | } 254 | 255 | $consumers = $this->zkUtils->getConsumers($this->groupId, $this->topic); 256 | sort($consumers); 257 | if ($consumers !== $this->consumers) { 258 | $this->consumers = $consumers; 259 | $needRebalance = true; 260 | } 261 | 262 | return $needRebalance; 263 | } 264 | 265 | private function commitOffset($partitions) { 266 | foreach($partitions as $partition) { 267 | if ($this->offsets[$partition] != $this->prevOffsets[$partition]) { 268 | $this->zkUtils->commitOffset($this->topic, $this->groupId, 269 | $partition, $this->offsets[$partition]); 270 | $this->prevOffsets[$partition] = $this->offsets[$partition]; 271 | } 272 | } 273 | } 274 | 275 | /** 276 | * Group name can't be used by two or more topics in different instances, 277 | * while it may case partitions unconsumed. for example: 278 | * 279 | * Instance A register C1 group with T1 topic(2 partitions) 280 | * Instance B register C1 group with T2 topic(2 partitions) 281 | * 282 | * T1 partition 0 would be assigned to A, partition 1 would be assigned to B 283 | * but B would never consume partition while it didn't clamin the partition, 284 | * the same with T2. 285 | */ 286 | private function validateGroup($groupId, $topic) { 287 | $path = $this->consumer_dir."/$groupId"; 288 | $config = $this->zkUtils->get($path); 289 | if (empty($config)) { 290 | if(!$this->zkUtils->set($path, json_encode(array('topic' => $topic)))) { 291 | throw new \Exception("failed to set topic config to consumer group"); 292 | } 293 | return; 294 | } 295 | $config = json_decode($config, true); 296 | if (is_array($config) && count($config) > 0 && !isset($config["topic"])) { 297 | throw new \Exception("consumer group was written by someone"); 298 | } 299 | if (is_array($config) && isset($config["topic"]) && $config["topic"] != $topic) { 300 | throw new \Exception("consumer group [".$groupId."] was used by topic [".$config["topic"]."]"); 301 | } 302 | } 303 | 304 | /** 305 | * start to process messages by this callback function 306 | * 307 | * @var function $callback_func callback function to process message 308 | */ 309 | public function start($callback_func) { 310 | if (empty($this->groupId) || empty($this->topic)) { 311 | throw new \Exception("groupId and topic shouldn't be empty"); 312 | } 313 | $this->rk = new \Rdkafka\Consumer($this->conf); 314 | $brokerList = $this->zkUtils->getBrokerList(); 315 | if ($brokerList == "") { 316 | throw new \Exception ("broker list shouldn't be empty!"); 317 | } 318 | $this->rk->addBrokers($brokerList); 319 | $this->validateGroup($this->groupId, $this->topic); 320 | 321 | $topicConf = new \Rdkafka\TopicConf(); 322 | $topicConf->set('auto.offset.reset', $this->offsetAutoReset); 323 | $this->rkTopic = $this->rk->newTopic($this->topic, $topicConf); 324 | 325 | $this->lastCommitTime = $this->getTime(); 326 | if (!$this->zkUtils->registerConsumer($this->topic, $this->groupId, $this->consumerId)) { 327 | throw new \Exception("failed to register consumer"); 328 | } 329 | 330 | $this->lastWatchTime = 0; 331 | while (self::$running) { 332 | $this->consume($callback_func); 333 | } 334 | $this->shutdown(); 335 | } 336 | 337 | private function shutdown() { 338 | $this->commitOffset($this->currentPartitions); 339 | //release partitions, and commit offsets; 340 | foreach($this->currentPartitions as $partition) { 341 | if (!$this->zkUtils->releasePartitionOwnership($this->topic, $this->groupId, 342 | $partition)) { 343 | throw new \Exception ("failed to release partition ownership"); 344 | } else { 345 | $this->rkTopic->consumeStop($partition); 346 | } 347 | } 348 | $this->zkUtils->deleteConsumer($this->topic, $this->groupId, $this->consumerId); 349 | $this->currentPartitions = array(); 350 | } 351 | 352 | /** 353 | * stop consuming messages 354 | */ 355 | public static function stop() { 356 | self::$running = false; 357 | } 358 | 359 | private function consume($callback_func) { 360 | if ($this->needRebalance()) { 361 | $this->rebalance(); 362 | } 363 | $this->checkOwner(); 364 | 365 | //consume message in partitions 366 | foreach ($this->currentPartitions as $partition) { 367 | $offset = $this->offsets[$partition]; 368 | $cnt = 0; 369 | while ($cnt++ < $this->maxMessage) { 370 | $msg = $this->rkTopic->consume($partition, $this->consumeTimeout); 371 | $this->rk->poll(0); 372 | if ($msg !== null && $msg->err === RD_KAFKA_RESP_ERR_NO_ERROR) { 373 | call_user_func($callback_func, $msg); 374 | $offset = $msg->offset + 1; 375 | } 376 | else if ($msg === null || $msg->err === RD_KAFKA_RESP_ERR__PARTITION_EOF) { 377 | if ($this->eofHandler != NULL) { 378 | call_user_func($this->eofHandler, $msg); 379 | } 380 | break; 381 | } else if($msg->err === RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT || 382 | $msg->err === RD_KAFKA_RESP_ERR__FAIL || 383 | $msg->err === RD_KAFKA_RESP_ERR__TRANSPORT) { 384 | call_user_func($this->errHandler, $msg); 385 | } else { 386 | throw new \Exception($msg->errstr()); 387 | } 388 | 389 | if (!($msg === null || $msg->err == RD_KAFKA_RESP_ERR__PARTITION_EOF)) { 390 | self::$__serverDownTimes = 0; 391 | } 392 | } 393 | //commit offset to zookeeper when interval time is reached 394 | $this->offsets[$partition] = $offset; 395 | if ($this->getTime() - $this->lastCommitTime > $this->commitInterval) { 396 | $this->commitOffset($this->currentPartitions); 397 | $this->lastCommitTime = $this->getTime(); 398 | } 399 | } 400 | } 401 | } 402 | -------------------------------------------------------------------------------- /src/ZkUtils.php: -------------------------------------------------------------------------------- 1 | 0x1f, 'scheme' => 'world', 'id' => 'anyone') 20 | ); 21 | 22 | private $zookeeper; 23 | 24 | public function __construct($address, $sessionTimeout = 30000, $chroot = '') { 25 | $this->zookeeper = new \Zookeeper($address, null, $sessionTimeout); 26 | if (!empty($chroot)) { 27 | $this->consumer_dir = $chroot.$this->consumer_dir; 28 | $this->broker_topics_dir = $chroot.$this->broker_topics_dir; 29 | $this->brokers_dir = $chroot.$this->brokers_dir; 30 | } 31 | } 32 | 33 | public static function filterEmpty($e) { 34 | return $e !=false || $e === "0" || $e === 0; 35 | } 36 | 37 | /** 38 | * make persistent path recursive 39 | * 40 | * @var String $path path 41 | * @var String $value value 42 | * @var Int $flags flags to indicates which type of node to create 43 | * $return true or false 44 | */ 45 | private function makeNode($path, $value, $flags=null) { 46 | return $this->zookeeper->create($path, $value, self::$acl, $flags) != null ? true : false; 47 | } 48 | 49 | 50 | /** 51 | * make persistent path recursive 52 | * 53 | * @var String $path path 54 | * @var String $value value 55 | */ 56 | private function makePath($path, $value = '') { 57 | $parts = explode('/', $path); 58 | $parts = array_filter($parts, 'self::filterEmpty'); 59 | $subpath = ''; 60 | while (count($parts) > 1) { 61 | $subpath .= '/' . array_shift($parts); 62 | if (!$this->zookeeper->exists($subpath)) { 63 | $this->makeNode($subpath, $value); 64 | } 65 | } 66 | } 67 | 68 | /** 69 | * set a persistent node 70 | * 71 | * @var String $path path 72 | * @var String $value value 73 | * $return true or false 74 | */ 75 | public function set($path, $value) { 76 | if (!$this->zookeeper->exists($path)) { 77 | $this->makePath($path); 78 | return $this->makeNode($path, $value); 79 | } 80 | return $this->zookeeper->set($path, $value); 81 | } 82 | 83 | /** 84 | * set an ephemeral node 85 | * 86 | * @var String $path path 87 | * @var String $value value 88 | * $return true or false 89 | */ 90 | public function setEphemeral($path, $value) { 91 | if (!$this->zookeeper->exists($path)) { 92 | $this->makePath($path); 93 | return $this->makeNode($path, $value, self::EPHEMERAL); 94 | } 95 | return false; 96 | } 97 | 98 | /** 99 | * get the node on the path 100 | * 101 | * @var String $path path 102 | * $return the node name 103 | */ 104 | public function get($path) { 105 | if (!$this->zookeeper->exists($path)) { 106 | return null; 107 | } 108 | return $this->zookeeper->get($path); 109 | } 110 | 111 | /** 112 | * delete the path 113 | * 114 | * @var String $path path 115 | * $return true or false 116 | */ 117 | public function delete($path) { 118 | if (!$this->zookeeper->exists($path)) { 119 | return true; 120 | } 121 | return $this->zookeeper->delete($path); 122 | } 123 | 124 | /** 125 | * get the child nodes under the path 126 | * 127 | * @var String $path path 128 | * $return an array of strings about nodes' name 129 | */ 130 | public function getChildren($path) { 131 | if (strlen($path) > 1 && preg_match('@/$@', $path)) { 132 | $path = substr($path, 0, -1); 133 | } 134 | if (!$this->zookeeper->exists($path)) { 135 | return null; 136 | } 137 | return $this->zookeeper->getChildren($path); 138 | } 139 | 140 | /** 141 | * commit offset to zookeeper 142 | * 143 | * @var String $topic topic 144 | * @var String $groupId consumer group Id 145 | * @var Int $partition partition 146 | * @var Int $offset consumed msg offset in one partition 147 | * $return true or false 148 | */ 149 | public function commitOffset($topic, $groupId, $partition, $offset) { 150 | $path = $this->$consumer_dir."/$groupId/offsets/$topic/$partition"; 151 | return $this->set($path, $offset); 152 | } 153 | 154 | /** 155 | * Get a partition's message offset from zookeeper 156 | * 157 | * @var String $topic topic 158 | * @var String $groupId consumer group Id 159 | * @var Int $partition partition 160 | * $return Int offset 161 | */ 162 | public function getOffset($topic, $groupId, $partition) { 163 | $path = $this->consumer_dir."/$groupId/offsets/$topic/$partition"; 164 | if($this->zookeeper->exists($path)) { 165 | return $this->zookeeper->get($path); 166 | } else { 167 | //if offset not found, return -1 and consumer will reset offset base on autoReset 168 | return -1; 169 | } 170 | } 171 | 172 | /** 173 | * register a consumer for the group 174 | * 175 | * @var String $topic topic 176 | * @var String $groupId consumer group Id 177 | * @var String $consumerId consumer Id 178 | * $return ture or false 179 | */ 180 | public function registerConsumer($topic, $groupId, $consumerId) { 181 | $path = $this->consumer_dir."/$groupId/ids/$consumerId"; 182 | return $this->setEphemeral($path, ''); 183 | } 184 | 185 | /** 186 | * delete a consumer for the group 187 | * 188 | * @var String $topic topic 189 | * @var String $groupId consumer group Id 190 | * @var String $consumerId consumer Id 191 | * $return ture or false 192 | */ 193 | public function deleteConsumer($topic, $groupId, $consumerId) { 194 | $path = $this->consumer_dir."/$groupId/ids/$consumerId"; 195 | return $this->delete($path); 196 | } 197 | 198 | /** 199 | * get partition list for the topic 200 | * 201 | * @var String $topic topic 202 | * $return an array of strings about partition number 203 | */ 204 | public function getPartitions($topic) { 205 | $path = $this->broker_topics_dir."/$topic/partitions"; 206 | return $this->getChildren($path); 207 | } 208 | 209 | /** 210 | * get consumer list for the group 211 | * 212 | * @var String $groupId consumer group Id 213 | * $return an array of strings about consumer name 214 | */ 215 | public function getConsumers($groupId) { 216 | $path = $this->consumer_dir."/$groupId/ids"; 217 | return $this->getChildren($path); 218 | } 219 | 220 | /** 221 | * register the partition's owner 222 | * 223 | * @var String $topic topic 224 | * @var String $groupId consumer group Id 225 | * @var Int $partition partition 226 | * @var String $consumerId consumer Id 227 | * $return ture or false 228 | */ 229 | public function registerOwner($topic, $groupId, $partition, $consumerId) { 230 | $path = $this->consumer_dir."/$groupId/owners/$topic/$partition"; 231 | return $this->setEphemeral($path, $consumerId); 232 | } 233 | 234 | /** 235 | * release the partition's owner 236 | * 237 | * @var String $topic topic 238 | * @var String $groupId consumer group Id 239 | * @var Int $partition partition 240 | * $return ture or false 241 | */ 242 | public function releasePartitionOwnership($topic, $groupId, $partition) { 243 | $path = $this->consumer_dir."/$groupId/owners/$topic/$partition"; 244 | if ($this->zookeeper->exists($path)) { 245 | return $this->delete($path); 246 | } 247 | return true; 248 | } 249 | 250 | /** 251 | * get broker list for this cluster 252 | * 253 | * $return a string about broker list, elements that including ip adress and prot are splitted by ',' . 254 | */ 255 | public function getBrokerList() { 256 | $path = $this->brokers_dir; 257 | $brokers = $this->getChildren($path); 258 | $brokerList = ""; 259 | foreach($brokers as $broker) { 260 | $info = $this->get($this->brokers_dir."/$broker"); 261 | if ($info != null && $info !=false) { 262 | $json_decode = json_decode($info, true); 263 | if ($brokerList !== '') { 264 | $brokerList .= ","; 265 | } 266 | $brokerList .= $json_decode['host'] . ":" . $json_decode['port']; 267 | } 268 | } 269 | return $brokerList; 270 | } 271 | } 272 | -------------------------------------------------------------------------------- /tests/README.md: -------------------------------------------------------------------------------- 1 | ## 说明 2 | 3 | ### zkUtilsTest.sh 4 | zkUtils的测试 5 | 6 | ### consumerTest.sh 7 | 该程序用来测试consumer中的rebalance以及自动管理offset功能是否正常。 8 | 9 | 因为涉及到测试消费涉及到创建topic以及写入messages，所以需要先生成一个partition数小于等于13的topic，名为php-test，并写入消息。 10 | 11 | -------------------------------------------------------------------------------- /tests/ZkUtilsTest.php: -------------------------------------------------------------------------------- 1 | delete(self::TEST_PATH_PREFIX); 21 | } 22 | 23 | public function testSet() { 24 | $result = self::$zk_Utils->set(self::TEST_PATH, self::TEST_VALUE); 25 | $this->assertTrue($result); 26 | } 27 | 28 | /* 29 | * @depends testSet 30 | */ 31 | public function testGet() { 32 | $result = self::$zk_Utils->get(self::TEST_PATH); 33 | $this->assertEquals(self::TEST_VALUE, $result); 34 | } 35 | 36 | public function testDeleteEmpty() { 37 | $this->assertTrue(self::$zk_Utils->delete(self::TEST_NO_EXITS)); 38 | } 39 | 40 | /* 41 | * @depends testSet 42 | */ 43 | public function testDelete() { 44 | $this->assertTrue(self::$zk_Utils->delete(self::TEST_PATH)); 45 | } 46 | 47 | /* 48 | * @depends testDelete 49 | */ 50 | public function testSetEphemeral() { 51 | $this->assertTrue(self::$zk_Utils->setEphemeral(self::TEST_PATH, self::TEST_VALUE)); 52 | $this->assertFalse(self::$zk_Utils->setEphemeral(self::TEST_PATH, self::TEST_VALUE)); 53 | self::$zk_Utils->delete(self::TEST_PATH); 54 | } 55 | 56 | /* 57 | * @depends testSet 58 | */ 59 | public function testGetChildrenEmpty() { 60 | self::$zk_Utils->set(self::TEST_GET_CHILDREN,''); 61 | $result = self::$zk_Utils->getChildren(self::TEST_GET_CHILDREN); 62 | $this->assertEmpty($result); 63 | } 64 | 65 | /* 66 | * @depends testSet 67 | * @depends testDelete 68 | */ 69 | public function testGetChildren() { 70 | $count = 10; 71 | 72 | for($i = 0; $i < $count; ++$i) { 73 | self::$zk_Utils->set(self::TEST_GET_CHILDREN ."/$i", $i); 74 | } 75 | $result = self::$zk_Utils->getChildren(self::TEST_GET_CHILDREN ); 76 | $this->assertCount($count, $result); 77 | 78 | for($i = 0; $i < $count; ++$i) { 79 | self::$zk_Utils->delete(self::TEST_GET_CHILDREN."/$i", $i); 80 | } 81 | self::$zk_Utils->delete(self::TEST_GET_CHILDREN); 82 | } 83 | 84 | /* 85 | * @depends testSet 86 | * @depends testGet 87 | * @depends testDelete 88 | */ 89 | public function testCommitAndGetOffset() { 90 | $offset = rand(); 91 | $partition = 0; 92 | 93 | $this->assertEquals(-1,self::$zk_Utils->getOffset(self::TEST_TOPIC, self::TEST_GROUPID, $partition)); 94 | 95 | $result = self::$zk_Utils->commitOffset(self::TEST_TOPIC, self::TEST_GROUPID, $partition, $offset); 96 | $this->assertTrue($result); 97 | 98 | $result = self::$zk_Utils->getOffset(self::TEST_TOPIC, self::TEST_GROUPID, $partition); 99 | $this->assertEquals($offset, $result); 100 | $path = "/consumers/".self::TEST_GROUPID."/offsets/".self::TEST_TOPIC."/$partition"; 101 | self::$zk_Utils->delete($path); 102 | // TODO: 递归删除路径 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /tests/consumerTest.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ -f "consumerTestTempFile" ]; then 4 | rm consumerTestTempFile; 5 | fi 6 | 7 | php runExample.php 0 & 8 | sleep 2 9 | 10 | php runExample.php 1 & 11 | sleep 0.5 12 | echo "-" >> consumerTestTempFile 13 | sleep 1.5 14 | 15 | php runExample.php 2 & 16 | sleep 0.5 17 | echo "-" >> consumerTestTempFile 18 | sleep 1.5 19 | 20 | ps aux | grep "php runExample.php 2" | awk '{print $2}' | xargs kill -2 21 | sleep 0.5 22 | echo "-" >> consumerTestTempFile 23 | sleep 1.5 24 | 25 | ps aux | grep "php runExample.php 1" | awk '{print $2}' | xargs kill -2 26 | sleep 0.5 27 | echo "-" >> consumerTestTempFile 28 | sleep 1.5 29 | 30 | ps aux | grep "php runExample.php 0" | awk '{print $2}' | xargs kill -2 31 | 32 | a=1 33 | num=1 34 | flag=true 35 | offsets=(0 0 0 0 0 0 0 0 0 0 0 0 0) 36 | count=0 37 | 38 | while read line 39 | do 40 | if [ "$line" == "-" ] 41 | then 42 | num=`expr $num + $a` 43 | if [ $num -eq 3 ] 44 | then 45 | a=-1 46 | fi 47 | count=0 48 | continue 49 | else 50 | count=`expr $count + 1` 51 | consumer=`echo $line | awk '{print $1}'` 52 | partition=`echo $line | awk '{print $2}'` 53 | offset=`echo $line | awk '{print $3}'` 54 | if [ ${offsets[$partition]} -eq 0 ] 55 | then 56 | offsets[$partition]=$offset 57 | else 58 | if [ `expr ${offsets[$partition]} + 1` -ne $offset ] 59 | then 60 | flag=false 61 | break 62 | else 63 | offsets[$partition]=$offset 64 | fi 65 | fi 66 | 67 | if [ $count -lt 50 ] 68 | then 69 | if [ `expr $partition % $num` -ne $consumer ] 70 | then 71 | flag=false 72 | break 73 | fi 74 | fi 75 | fi 76 | done < consumerTestTempFile 77 | 78 | if [ "$flag" = true ] 79 | then 80 | echo "test success !" 81 | else 82 | echo "test fail !" 83 | fi 84 | 85 | rm consumerTestTempFile 86 | -------------------------------------------------------------------------------- /tests/phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /tests/runExample.php: -------------------------------------------------------------------------------- 1 | partition $msg->offset\n"); 21 | usleep(20*1000); 22 | fclose($file); 23 | } 24 | 25 | declare(ticks=1); 26 | 27 | $zkAddress = "localhost:2181"; 28 | $topic = "php-test"; 29 | $groupId = "group-test-1"; 30 | $maxMessage = 1; 31 | 32 | $consumer = New \MTKafka\Consumer($zkAddress); 33 | $consumer->setGroupId($groupId); 34 | $consumer->setTopic($topic); 35 | $consumer->setMaxMessage($maxMessage); 36 | $consumer->setClientId("test"); 37 | $consumer->setConsumeTimeout(1000); 38 | $consumer->setWatchInterval(10); 39 | 40 | pcntl_signal(SIGHUP, "sig_handler"); 41 | pcntl_signal(SIGINT, "sig_handler"); 42 | pcntl_signal(SIGQUIT, "sig_handler"); 43 | pcntl_signal(SIGTERM, "sig_handler"); 44 | 45 | $GLOBALS['consumerId'] = $argv[1]; 46 | $consumer->start("echo_message"); 47 | --------------------------------------------------------------------------------