├── .gitignore ├── LICENSE ├── README.md ├── composer.json └── src ├── examples ├── autoloader.php ├── consume.php ├── produce.php ├── produce_gzip.php └── zkconsumer.php ├── lib └── Kafka │ ├── Encoder.php │ ├── Exception.php │ ├── Exception │ ├── EmptyQueue.php │ ├── InvalidFetchSize.php │ ├── InvalidMessage.php │ ├── InvalidTopic.php │ ├── NotSupported.php │ ├── OffsetOutOfRange.php │ ├── OutOfRange.php │ ├── Socket.php │ ├── Socket │ │ ├── Connection.php │ │ ├── EOF.php │ │ └── Timeout.php │ ├── WrongPartition.php │ └── ZookeeperConnection.php │ ├── FetchRequest.php │ ├── Message.php │ ├── MessageSet.php │ ├── MessageSetInternalIterator.php │ ├── OffsetRequest.php │ ├── Producer.php │ ├── Registry │ ├── Broker.php │ ├── Offset.php │ └── Topic.php │ ├── Request.php │ ├── RequestKeys.php │ ├── Response.php │ ├── SimpleConsumer.php │ ├── Socket.php │ └── ZookeeperConsumer.php └── tests ├── Kafka ├── EncoderTest.php ├── FetchRequestTest.php ├── MessageSetTest.php ├── MessageTest.php ├── ProducerTest.php ├── RequestTest.php ├── ResponseTest.php ├── SimpleConsumerTest.php └── SocketTest.php ├── bootstrap.php └── phpunit.xml /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2012 Lorenzo Alberton 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-php 2 | kafka-php allows you to produce messages to the [Apache Kafka](http://kafka.apache.org/) distributed publish/subscribe messaging service. 3 | 4 | ## Requirements 5 | 6 | * Minimum PHP version: 5.3.3. 7 | * Apache Kafka 0.6.x or 0.7.x. 8 | * You need to have access to your Kafka instance and be able to connect through TCP. You can obtain a copy and instructions on how to setup kafka at https://github.com/kafka-dev/kafka 9 | * The [PHP Zookeeper extension](https://github.com/andreiz/php-zookeeper) is required if you want to use the Zookeeper-based consumer. 10 | 11 | ## Installation 12 | Add the lib directory to the PHP include_path and use an autoloader like the one in the examples directory (the code follows the PEAR/Zend one-class-per-file convention). 13 | 14 | ## Usage 15 | The examples directory contains an example of a Producer and a simple Consumer, and an example of the Zookeeper-based Consumer. 16 | 17 | Example Producer: 18 | 19 | ```php 20 | $producer = new Kafka_Producer('localhost', 9092, Kafka_Encoder::COMPRESSION_NONE); 21 | $messages = array('some', 'messages', 'here'); 22 | $topic = 'test'; 23 | $bytes = $producer->send($messages, $topic); 24 | ``` 25 | 26 | Example Consumer: 27 | 28 | ```php 29 | $topic = 'test'; 30 | $partition = 0; 31 | $offset = 0; 32 | $maxSize = 1000000; 33 | $socketTimeout = 5; 34 | 35 | while (true) { 36 | $consumer = new Kafka_SimpleConsumer('localhost', 9092, $socketTimeout, $maxSize); 37 | $fetchRequest = new Kafka_FetchRequest($topic, $partition, $offset, $maxSize); 38 | $messages = $consumer->fetch($fetchRequest); 39 | foreach ($messages as $msg) { 40 | echo "\nMessage: " . $msg->payload(); 41 | } 42 | //advance the offset after consuming each MessageSet 43 | $offset += $messages->validBytes(); 44 | unset($fetchRequest); 45 | } 46 | ``` 47 | 48 | 49 | ## TODO 50 | 51 | - support for Snappy compression 52 | 53 | ## Contact for questions 54 | 55 | Lorenzo Alberton 56 | 57 | l.alberton at(@) quipo.it 58 | 59 | http://twitter.com/lorenzoalberton 60 | 61 | http://alberton.info/ 62 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "quipo/kafka-php", 3 | "description": "PHP client library for Apache Kafka 0.7", 4 | "license": "Apache-2.0", 5 | "authors": [ 6 | { 7 | "name": "Lorenzo Alberton", 8 | "email": "l.alberton@quipo.it" 9 | } 10 | ], 11 | "require": {}, 12 | "autoload": { 13 | "classmap": ["src/lib/"] 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/examples/autoloader.php: -------------------------------------------------------------------------------- 1 | file association */ 33 | if (($file !== false) && ($file !== null)) { 34 | include $file; 35 | return; 36 | } 37 | 38 | throw new RuntimeException($className. ' not found'); 39 | }); 40 | -------------------------------------------------------------------------------- /src/examples/consume.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | fetch($fetchRequest); 47 | foreach ($messages as $msg) { 48 | ++$nMessages; 49 | echo "\nconsumed[$offset][$partialOffset][msg #{$nMessages}]: " . $msg->payload(); 50 | $partialOffset = $messages->validBytes(); 51 | } 52 | //advance the offset after consuming each message 53 | $offset += $messages->validBytes(); 54 | //echo "\n---[Advancing offset to $offset]------(".date('H:i:s').")"; 55 | unset($fetchRequest); 56 | //sleep(2); 57 | } catch (Exception $e) { 58 | // probably consumed all items in the queue. 59 | echo "\nERROR: " . get_class($e) . ': ' . $e->getMessage()."\n".$e->getTraceAsString()."\n"; 60 | sleep(2); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/examples/produce.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | send($messages, $topic); 41 | printf("\nSuccessfully sent %d messages (%d bytes)\n\n", count($messages), $bytes); 42 | } 43 | -------------------------------------------------------------------------------- /src/examples/produce_gzip.php: -------------------------------------------------------------------------------- 1 | #!/usr/bin/php 2 | send($messages, $topic); 41 | printf("\nSuccessfully sent %d messages (%d bytes)\n\n", count($messages), $bytes); 42 | } 43 | -------------------------------------------------------------------------------- /src/examples/zkconsumer.php: -------------------------------------------------------------------------------- 1 | getReadBytes() >= $maxBatchSize) { 61 | break; 62 | } 63 | } 64 | } catch (Kafka_Exception_OffsetOutOfRange $exception) { 65 | // if we haven't received any messages, resync the offsets for the next time, then bomb out 66 | if ($zkconsumer->getReadBytes() == 0) { 67 | $zkconsumer->resyncOffsets(); 68 | die($exception->getMessage()); 69 | } 70 | // if we did receive some messages before the exception, carry on. 71 | } catch (Kafka_Exception_Socket_Connection $exception) { 72 | // deal with it below 73 | } catch (Kafka_Exception $exception) { 74 | // deal with it below 75 | } 76 | 77 | if (null !== $exception) { 78 | // if we haven't received any messages, bomb out 79 | if ($zkconsumer->getReadBytes() == 0) { 80 | die($exception->getMessage()); 81 | } 82 | // otherwise log the error, commit the offsets for the messages read so far and return the data 83 | } 84 | 85 | // process the data in batches, wait for ACK 86 | 87 | $success = doSomethingWithTheMessages($messages); 88 | 89 | // Once the data is processed successfully, commit the byte offsets. 90 | if ($success) { 91 | $zkconsumer->commitOffsets(); 92 | } 93 | 94 | // get an approximate figure on the size of the queue 95 | try { 96 | echo "\nRemaining bytes in queue: " . $consumer->getRemainingSize(); 97 | } catch (Kafka_Exception_Socket_Connection $exception) { 98 | die($exception->getMessage()); 99 | } catch (Kafka_Exception $exception) { 100 | die($exception->getMessage()); 101 | } 102 | 103 | -------------------------------------------------------------------------------- /src/lib/Kafka/Encoder.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Encode messages and messages sets into the kafka protocol 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_Encoder 24 | { 25 | /** 26 | * 1 byte "magic" identifier to allow format changes 27 | * 28 | * @const integer 29 | */ 30 | const CURRENT_MAGIC_VALUE = 1; 31 | 32 | const COMPRESSION_NONE = 0; 33 | const COMPRESSION_GZIP = 1; 34 | const COMPRESSION_SNAPPY = 2; 35 | 36 | /** 37 | * Encode a message. The format of an N byte message is the following: 38 | * - 1 byte: "magic" identifier to allow format changes 39 | * - 1 byte: "compression-attributes" for compression alogrithm 40 | * - 4 bytes: CRC32 of the payload 41 | * - (N - 6) bytes: payload 42 | * 43 | * @param string $msg Message to encode 44 | * 45 | * @return string 46 | * @throws Kafka_Exception 47 | */ 48 | static public function encode_message($msg, $compression = self::COMPRESSION_NONE) { 49 | $compressed = self::compress($msg, $compression); 50 | // 51 | return pack('CCN', self::CURRENT_MAGIC_VALUE, $compression, crc32($compressed)) 52 | . $compressed; 53 | } 54 | 55 | /** 56 | * Compress a message 57 | * 58 | * @param string $msg Message 59 | * @param integer $compression 0=none, 1=gzip, 2=snappy 60 | * 61 | * @return string 62 | * @throws Kafka_Exception 63 | */ 64 | static public function compress($msg, $compression) { 65 | switch ($compression) { 66 | case self::COMPRESSION_NONE: 67 | return $msg; 68 | case self::COMPRESSION_GZIP: 69 | return gzencode($msg); 70 | case self::COMPRESSION_SNAPPY: 71 | throw new Kafka_Exception_NotSupported('SNAPPY compression not yet implemented'); 72 | default: 73 | throw new Kafka_Exception_NotSupported('Unknown compression flag: ' . $compression); 74 | } 75 | } 76 | 77 | /** 78 | * Decompress a message 79 | * 80 | * @param string $msg Message 81 | * @param integer $compression 0=none, 1=gzip, 2=snappy 82 | * 83 | * @return string 84 | * @throws Kafka_Exception 85 | */ 86 | static public function decompress($msg, $compression) { 87 | switch ($compression) { 88 | case self::COMPRESSION_NONE: 89 | return $msg; 90 | case self::COMPRESSION_GZIP: 91 | // NB: this is really a MessageSet, not just a single message 92 | // although I'm not sure this is the best way to handle the inner offsets, 93 | // as the symmetry with the outer collection iteration is broken. 94 | // @see https://issues.apache.org/jira/browse/KAFKA-406 95 | $stream = fopen('php://temp', 'w+b'); 96 | fwrite($stream, gzinflate(substr($msg, 10))); 97 | rewind($stream); 98 | $socket = Kafka_Socket::createFromStream($stream); 99 | return new Kafka_MessageSetInternalIterator($socket, 0, 0); 100 | case self::COMPRESSION_SNAPPY: 101 | throw new Kafka_Exception_NotSupported('SNAPPY decompression not yet implemented'); 102 | default: 103 | throw new Kafka_Exception_NotSupported('Unknown compression flag: ' . $compression); 104 | } 105 | } 106 | 107 | /** 108 | * Encode a complete request 109 | * 110 | * @param string $topic Topic 111 | * @param integer $partition Partition number 112 | * @param array $messages Array of messages to send 113 | * @param integer $compression flag for type of compression 114 | * 115 | * @return string 116 | * @throws Kafka_Exception 117 | */ 118 | static public function encode_produce_request($topic, $partition, array $messages, $compression = self::COMPRESSION_NONE) { 119 | // not sure I agree this is the best design for compressed messages 120 | // @see https://issues.apache.org/jira/browse/KAFKA-406 121 | $compress = ($compression !== self::COMPRESSION_NONE); 122 | $message_set = ''; 123 | foreach ($messages as $message) { 124 | $encoded = self::encode_message($message, self::COMPRESSION_NONE); 125 | // encode messages as 126 | $message_set .= pack('N', strlen($encoded)) . $encoded; 127 | } 128 | if ($compress) { 129 | $encoded = self::encode_message($message_set, $compression); 130 | $message_set = pack('N', strlen($encoded)) . $encoded; 131 | } 132 | 133 | // create the request as 134 | $data = pack('n', Kafka_RequestKeys::PRODUCE) . 135 | pack('n', strlen($topic)) . $topic . 136 | pack('N', $partition) . 137 | pack('N', strlen($message_set)) . $message_set; 138 | return pack('N', strlen($data)) . $data; 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Base exception class 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_Exception extends RuntimeException 24 | { 25 | 26 | } -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/EmptyQueue.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Exception class. Thrown when there's no new data to read from the queue 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_EmptyQueue extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/InvalidFetchSize.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_InvalidFetchSize extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/InvalidMessage.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_InvalidMessage extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/InvalidTopic.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_InvalidTopic extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/NotSupported.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_NotSupported extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/OffsetOutOfRange.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_OffsetOutOfRange extends Kafka_Exception 23 | { 24 | 25 | } -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/OutOfRange.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_OutOfRange extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/Socket.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base Socket exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_Socket extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/Socket/Connection.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Socket EndOfFile exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_Socket_Connection extends Kafka_Exception_Socket 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/Socket/EOF.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Socket EndOfFile exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_Socket_EOF extends Kafka_Exception_Socket 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/Socket/Timeout.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Socket Timeout exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_Socket_Timeout extends Kafka_Exception_Socket 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/WrongPartition.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_WrongPartition extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/Exception/ZookeeperConnection.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Base exception class 15 | * 16 | * @category Libraries 17 | * @package Kafka 18 | * @author Lorenzo Alberton 19 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 20 | * @link http://sna-projects.com/kafka/ 21 | */ 22 | class Kafka_Exception_ZookeeperConnection extends Kafka_Exception 23 | { 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/lib/Kafka/FetchRequest.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Represents a request object 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_FetchRequest extends Kafka_Request 24 | { 25 | /** 26 | * @var integer 27 | */ 28 | private $offset; 29 | 30 | /** 31 | * @var integer 32 | */ 33 | private $maxSize; 34 | 35 | /** 36 | * @param string $topic Topic 37 | * @param integer $partition Partition 38 | * @param integer $offset Offset 39 | * @param integer $maxSize Max buffer size 40 | */ 41 | public function __construct($topic, $partition = 0, $offset = 0, $maxSize = 1000000) { 42 | $this->id = Kafka_RequestKeys::FETCH; 43 | $this->topic = $topic; 44 | $this->partition = $partition; 45 | $this->offset = $offset; 46 | $this->maxSize = $maxSize; 47 | } 48 | 49 | /** 50 | * Write the request to the output stream 51 | * 52 | * @param resource $stream Output stream 53 | * 54 | * @return void 55 | */ 56 | public function writeTo(Kafka_Socket $socket) { 57 | $this->writeRequestHeader($socket); 58 | 59 | // OFFSET (long) 60 | $socket->write(self::packLong64bigendian($this->offset)); 61 | // MAX_SIZE (int) 62 | $socket->write(pack('N', $this->maxSize)); 63 | } 64 | 65 | /** 66 | * Get request size in bytes 67 | * 68 | * @return integer 69 | */ 70 | public function sizeInBytes() { 71 | // + + + + 72 | return 2 + strlen($this->topic) + 4 + 8 + 4; 73 | } 74 | 75 | /** 76 | * Get current offset 77 | * 78 | * @return integer 79 | */ 80 | public function getOffset() { 81 | return $this->offset; 82 | } 83 | 84 | /** 85 | * Get topic 86 | * 87 | * @return string 88 | */ 89 | public function getTopic() { 90 | return $this->topic; 91 | } 92 | 93 | /** 94 | * Get partition 95 | * 96 | * @return integer 97 | */ 98 | public function getPartition() { 99 | return $this->partition; 100 | } 101 | 102 | /** 103 | * String representation of the Fetch Request 104 | * 105 | * @return string 106 | */ 107 | public function __toString() { 108 | return 'topic:' . $this->topic . ', part:' . $this->partition . ' offset:' . $this->offset . ' maxSize:' . $this->maxSize; 109 | } 110 | } 111 | -------------------------------------------------------------------------------- /src/lib/Kafka/Message.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * A message. The format of an N byte message is the following: 16 | * 1 byte "magic" identifier to allow format changes 17 | * 1 byte compression-attribute (missing if magic=0) 18 | * 4 byte CRC32 of the payload 19 | * N - 6 byte payload (N-5 if magic=0) 20 | * 21 | * @category Libraries 22 | * @package Kafka 23 | * @author Lorenzo Alberton 24 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 25 | * @link http://sna-projects.com/kafka/ 26 | */ 27 | class Kafka_Message 28 | { 29 | /** 30 | * Wire format (0=without compression attribute, 1=with) 31 | * @var integer 32 | */ 33 | private $magic = Kafka_Encoder::CURRENT_MAGIC_VALUE; 34 | 35 | /** 36 | * @var string 37 | */ 38 | private $payload = null; 39 | 40 | /** 41 | * @var integer 42 | */ 43 | private $size = 0; 44 | 45 | /** 46 | * @var integer 47 | */ 48 | private $compression = Kafka_Encoder::COMPRESSION_NONE; 49 | 50 | /** 51 | * @var string 52 | */ 53 | private $crc = false; 54 | 55 | /** 56 | * Constructor 57 | * 58 | * @param string $data Message payload 59 | */ 60 | public function __construct($data) { 61 | $unpack = unpack('C', substr($data, 0, 1)); 62 | $this->magic = array_shift($unpack); 63 | if ($this->magic == 0) { 64 | $this->crc = array_shift(unpack('N', substr($data, 1, 4))); 65 | $this->payload = substr($data, 5); 66 | } else { 67 | $compression = unpack('C', substr($data, 1, 1)); 68 | $this->compression = array_shift($compression); 69 | $crc = unpack('N', substr($data, 2, 4)); 70 | $this->crc = array_shift($crc); 71 | $this->payload = substr($data, 6); 72 | } 73 | $this->size = strlen($this->payload); 74 | } 75 | 76 | 77 | /** 78 | * Return the compression flag 79 | * 80 | * @return integer 81 | */ 82 | public function compression() { 83 | return $this->compression; 84 | } 85 | 86 | 87 | /** 88 | * Encode a message 89 | * 90 | * @return string 91 | */ 92 | public function encode() { 93 | return Kafka_Encoder::encode_message($this->payload); 94 | } 95 | 96 | /** 97 | * Get the message size 98 | * 99 | * @return integer 100 | */ 101 | public function size() { 102 | return $this->size; 103 | } 104 | 105 | /** 106 | * Get the magic value 107 | * 108 | * @return integer 109 | */ 110 | public function magic() { 111 | return $this->magic; 112 | } 113 | 114 | /** 115 | * Get the message checksum 116 | * 117 | * @return integer 118 | */ 119 | public function checksum() { 120 | return $this->crc; 121 | } 122 | 123 | /** 124 | * Get the message payload 125 | * 126 | * @return string|Kafka_MessageSetInternalIterator 127 | */ 128 | public function payload() { 129 | return Kafka_Encoder::decompress($this->payload, $this->compression); 130 | } 131 | 132 | /** 133 | * Verify the message against the checksum 134 | * 135 | * @return boolean 136 | */ 137 | public function isValid() { 138 | return ($this->crc === crc32($this->payload)); 139 | } 140 | 141 | /** 142 | * Debug message 143 | * 144 | * @return string 145 | */ 146 | public function __toString() { 147 | try { 148 | $payload = $this->payload(); 149 | } catch (Exception $e) { 150 | $payload = 'ERROR decoding payload: ' . $e->getMessage(); 151 | } 152 | if (!is_string($payload)) { 153 | $payload = 'COMPRESSED-CONTENT'; 154 | } 155 | return 'message(' 156 | . 'magic = ' . $this->magic 157 | . ', compression = ' . $this->compression 158 | . ', size = ' . $this->size() 159 | . ', crc = ' . $this->crc 160 | . ', valid = ' . ($this->isValid() ? 'true' : 'false') 161 | . ', payload = ' . $payload 162 | . ')'; 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /src/lib/Kafka/MessageSet.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * A sequence of messages stored in a byte buffer 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_MessageSet implements Iterator 24 | { 25 | /** 26 | * @var Kafka_Socket 27 | */ 28 | protected $socket = null; 29 | 30 | /** 31 | * @var integer 32 | */ 33 | protected $initialOffset = 0; 34 | 35 | /** 36 | * @var integer 37 | */ 38 | protected $validByteCount = 0; 39 | 40 | /** 41 | * @var boolean 42 | */ 43 | private $valid = false; 44 | 45 | /** 46 | * @var Kafka_Message 47 | */ 48 | private $msg; 49 | 50 | /** 51 | * @var Kafka_MessageSetInternalIterator 52 | */ 53 | private $internalIterator = null; 54 | 55 | /** 56 | * Constructor 57 | * 58 | * @param Kafka_Socket $socket Stream resource 59 | * @param integer $initialOffset Initial offset 60 | */ 61 | public function __construct(Kafka_Socket $socket, $initialOffset = 0) { 62 | $this->socket = $socket; 63 | $this->initialOffset = $initialOffset; 64 | } 65 | 66 | /** 67 | * Read the size of the next message (4 bytes) 68 | * 69 | * @return integer Size of the response buffer in bytes 70 | * @throws Kafka_Exception when size is <=0 or >= $maxSize 71 | */ 72 | protected function getMessageSize() { 73 | $data = $this->socket->read(4, true); 74 | $unpack = unpack('N', $data); 75 | $size = array_shift($unpack); 76 | if ($size <= 0) { 77 | throw new Kafka_Exception_OutOfRange($size . ' is not a valid message size'); 78 | } 79 | // TODO check if $size is too large 80 | return $size; 81 | } 82 | 83 | /** 84 | * Read the next message 85 | * 86 | * @return string Message (raw) 87 | * @throws Kafka_Exception when the message cannot be read from the stream buffer 88 | */ 89 | protected function getMessage() { 90 | try { 91 | $size = $this->getMessageSize(); 92 | $msg = $this->socket->read($size, true); 93 | } catch (Kafka_Exception_Socket_EOF $e) { 94 | $size = isset($size) ? $size : 'enough'; 95 | $logMsg = 'Cannot read ' . $size . ' bytes, the message is likely bigger than the buffer - original exception: ' . $e->getMessage(); 96 | throw new Kafka_Exception_OutOfRange($logMsg); 97 | } 98 | $this->validByteCount += 4 + $size; 99 | return $msg; 100 | } 101 | 102 | /** 103 | * Get message set size in bytes 104 | * 105 | * @return integer 106 | */ 107 | public function validBytes() { 108 | return $this->validByteCount; 109 | } 110 | 111 | /** 112 | * Get message set size in bytes 113 | * 114 | * @return integer 115 | */ 116 | public function sizeInBytes() { 117 | return $this->validBytes(); 118 | } 119 | 120 | /** 121 | * next 122 | * 123 | * @return void 124 | */ 125 | public function next() { 126 | if (null !== $this->internalIterator) { 127 | $this->internalIterator->next(); 128 | if ($this->internalIterator->valid()) { 129 | return; 130 | } 131 | } 132 | $this->internalIterator = null; 133 | $this->preloadNextMessage(); 134 | } 135 | 136 | /** 137 | * valid 138 | * 139 | * @return boolean 140 | */ 141 | public function valid() { 142 | return $this->valid; 143 | } 144 | 145 | /** 146 | * key 147 | * 148 | * @return integer 149 | */ 150 | public function key() { 151 | return $this->validByteCount; 152 | } 153 | 154 | /** 155 | * current 156 | * 157 | * @return Kafka_Message 158 | */ 159 | public function current() { 160 | if (null !== $this->internalIterator && $this->internalIterator->valid()) { 161 | return $this->internalIterator->current(); 162 | } 163 | return $this->msg; 164 | } 165 | 166 | /** 167 | * rewind - Cannot use fseek() 168 | * 169 | * @return void 170 | */ 171 | public function rewind() { 172 | $this->internalIterator = null; 173 | $this->validByteCount = 0; 174 | $this->preloadNextMessage(); 175 | } 176 | 177 | /** 178 | * Preload the next message 179 | * 180 | * @return void 181 | */ 182 | private function preloadNextMessage() { 183 | try { 184 | $this->msg = new Kafka_Message($this->getMessage()); 185 | if ($this->msg->compression() != Kafka_Encoder::COMPRESSION_NONE) { 186 | $this->internalIterator = $this->msg->payload(); 187 | $this->internalIterator->rewind(); 188 | $this->msg = null; 189 | } else { 190 | $this->internalIterator = null; 191 | } 192 | $this->valid = TRUE; 193 | } catch (Kafka_Exception_OutOfRange $e) { 194 | $this->valid = FALSE; 195 | } 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /src/lib/Kafka/MessageSetInternalIterator.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * A sequence of messages stored in a byte buffer 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_MessageSetInternalIterator extends Kafka_MessageSet 24 | { 25 | /** 26 | * Read the next message. 27 | * Override the parent method: we don't want to increment the byte offset 28 | * 29 | * @return string Message (raw) 30 | * @throws Kafka_Exception when the message cannot be read from the stream buffer 31 | */ 32 | protected function getMessage() { 33 | $msg = parent::getMessage(); 34 | // do not increment the offset for internal iterators 35 | $this->validByteCount = 0; 36 | return $msg; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/lib/Kafka/OffsetRequest.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Represents a request object 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_OffsetRequest extends Kafka_Request 24 | { 25 | /** 26 | * @var integer 27 | */ 28 | private $time; 29 | 30 | /** 31 | * @var integer 32 | */ 33 | private $maxSize; 34 | 35 | /** 36 | * @param string $topic Topic 37 | * @param integer $partition Partition 38 | * @param integer $time Time in millisecs 39 | * (-1, from the latest offset available, 40 | * -2 from the smallest offset available) 41 | * @param integer $maxNumOffsets Max number of offsets to return 42 | */ 43 | public function __construct($topic, $partition, $time, $maxNumOffsets) { 44 | $this->id = Kafka_RequestKeys::OFFSETS; 45 | $this->topic = $topic; 46 | $this->partition = $partition; 47 | $this->time = $time; 48 | $this->maxNumOffsets = $maxNumOffsets; 49 | } 50 | 51 | /** 52 | * Write the request to the output stream 53 | * 54 | * @param Kafka_Socket $socket Output stream 55 | * 56 | * @return void 57 | */ 58 | public function writeTo(Kafka_Socket $socket) { 59 | $this->writeRequestHeader($socket); 60 | 61 | // TIMESTAMP (long) 62 | $socket->write(self::packLong64bigendian($this->time)); 63 | // N_OFFSETS (int) 64 | $socket->write(pack('N', $this->maxNumOffsets)); 65 | } 66 | 67 | /** 68 | * Get request size in bytes 69 | * 70 | * @return integer 71 | */ 72 | public function sizeInBytes() { 73 | return 2 + strlen($this->topic) + 4 + 8 + 4; 74 | } 75 | 76 | /** 77 | * Get time 78 | * 79 | * @return integer 80 | */ 81 | public function getTime() { 82 | return $this->time; 83 | } 84 | 85 | /** 86 | * Get maxSize 87 | * 88 | * @return integer 89 | */ 90 | public function getMaxSize() { 91 | return $this->maxSize; 92 | } 93 | 94 | /** 95 | * Get partition 96 | * 97 | * @return integer 98 | */ 99 | public function getPartition() { 100 | return $this->partition; 101 | } 102 | 103 | /** 104 | * Parse the response and return the array of offsets 105 | * 106 | * @param Kafka_Socket $socket Socket handle 107 | * 108 | * @return array 109 | */ 110 | static public function deserializeOffsetArray(Kafka_Socket $socket) { 111 | $unpack = unpack('N', $socket->read(4)); 112 | $nOffsets = array_shift($unpack); 113 | if ($nOffsets < 0) { 114 | throw new Kafka_Exception_OutOfRange($nOffsets . ' is not a valid number of offsets'); 115 | } 116 | $offsets = array(); 117 | for ($i=0; $i < $nOffsets; ++$i) { 118 | $offsets[] = self::unpackLong64bigendian($socket->read(8)); 119 | } 120 | return $offsets; 121 | } 122 | 123 | /** 124 | * String representation of the Fetch Request 125 | * 126 | * @return string 127 | */ 128 | public function __toString() { 129 | return 'topic:' . $this->topic . ', part:' . $this->partition . ' offset:' . $this->offset . ' maxSize:' . $this->maxSize; 130 | } 131 | } 132 | -------------------------------------------------------------------------------- /src/lib/Kafka/Producer.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Simple Kafka Producer 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_Producer 24 | { 25 | /** 26 | * @var integer 27 | */ 28 | protected $request_key; 29 | 30 | /** 31 | * @var Kafka_Socket 32 | */ 33 | protected $socket; 34 | 35 | /** 36 | * @var string 37 | */ 38 | protected $host; 39 | 40 | /** 41 | * @var integer 42 | */ 43 | protected $port; 44 | 45 | /** 46 | * Compression: 0=none; 1=gzip; 2=snappy 47 | * 48 | * @var integer 49 | */ 50 | protected $compression; 51 | 52 | /** 53 | * Send timeout in seconds. 54 | * 55 | * Combined with sendTimeoutUsec this is used for send timeouts. 56 | * 57 | * @var int 58 | */ 59 | private $sendTimeoutSec = 0; 60 | 61 | /** 62 | * Send timeout in microseconds. 63 | * 64 | * Combined with sendTimeoutSec this is used for send timeouts. 65 | * 66 | * @var int 67 | */ 68 | private $sendTimeoutUsec = 100000; 69 | 70 | /** 71 | * Recv timeout in seconds 72 | * 73 | * Combined with recvTimeoutUsec this is used for recv timeouts. 74 | * 75 | * @var int 76 | */ 77 | private $recvTimeoutSec = 0; 78 | 79 | /** 80 | * Recv timeout in microseconds 81 | * 82 | * Combined with recvTimeoutSec this is used for recv timeouts. 83 | * 84 | * @var int 85 | */ 86 | private $recvTimeoutUsec = 750000; 87 | 88 | /** 89 | * Constructor 90 | * 91 | * @param integer $host Host 92 | * @param integer $port Port 93 | */ 94 | public function __construct($host, $port, $compression = Kafka_Encoder::COMPRESSION_GZIP, $recvTimeoutSec = 0, $recvTimeoutUsec = 750000, $sendTimeoutSec = 0, $sendTimeoutUsec = 100000) { 95 | $this->request_key = Kafka_RequestKeys::PRODUCE; 96 | $this->host = $host; 97 | $this->port = $port; 98 | $this->compression = $compression; 99 | $this->recvTimeoutSec = $recvTimeoutSec; 100 | $this->recvTimeoutUsec = $recvTimeoutUsec; 101 | $this->sendTimeoutSec = $sendTimeoutSec; 102 | $this->sendTimeoutUsec = $sendTimeoutUsec; 103 | } 104 | 105 | /** 106 | * Connect to Kafka via a socket 107 | * 108 | * @return void 109 | * @throws Kafka_Exception 110 | */ 111 | public function connect() { 112 | if (null === $this->socket) { 113 | $this->socket = new Kafka_Socket($this->host, $this->port, $this->recvTimeoutSec, $this->recvTimeoutUsec, $this->sendTimeoutSec, $this->sendTimeoutUsec); 114 | } 115 | $this->socket->connect(); 116 | } 117 | 118 | /** 119 | * Close the socket 120 | * 121 | * @return void 122 | */ 123 | public function close() { 124 | if (null !== $this->socket) { 125 | $this->socket->close(); 126 | } 127 | } 128 | 129 | /** 130 | * Send messages to Kafka 131 | * 132 | * @param array $messages Messages to send 133 | * @param string $topic Topic 134 | * @param integer $partition Partition 135 | * 136 | * @return boolean 137 | */ 138 | public function send(array $messages, $topic, $partition = 0xFFFFFFFF) { 139 | $this->connect(); 140 | return $this->socket->write(Kafka_Encoder::encode_produce_request($topic, $partition, $messages, $this->compression)); 141 | } 142 | 143 | /** 144 | * When serializing, close the socket and save the connection parameters 145 | * so it can connect again 146 | * 147 | * @return array Properties to save 148 | */ 149 | public function __sleep() { 150 | $this->close(); 151 | return array('request_key', 'host', 'port', 'compression'); 152 | } 153 | 154 | /** 155 | * Restore parameters on unserialize 156 | * 157 | * @return void 158 | */ 159 | public function __wakeup() { 160 | 161 | } 162 | } 163 | -------------------------------------------------------------------------------- /src/lib/Kafka/Registry/Broker.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Nick Telford 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * A Registry for Kafka brokers and the partitions they manage. 15 | * 16 | * The primary use of this is a facade API on top of ZooKeeper, providing a 17 | * more friendly interface to some common operations. 18 | * 19 | * @category Libraries 20 | * @package Kafka 21 | * @author Nick Telford 22 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 23 | * @link http://sna-projects.com/kafka/ 24 | */ 25 | class Kafka_Registry_Broker 26 | { 27 | const BROKER_PATH = '/brokers/ids/%d'; 28 | 29 | /** 30 | * Zookeeper client 31 | * 32 | * @var Zookeeper 33 | */ 34 | private $zookeeper; 35 | 36 | /** 37 | * Create a Broker Registry instance backed by the given Zookeeper quorum. 38 | * 39 | * @param Zookeeper a client for contacting the backing Zookeeper quorum 40 | */ 41 | public function __construct(Zookeeper $zookeeper) { 42 | $this->zookeeper = $zookeeper; 43 | } 44 | 45 | /** 46 | * Get the hostname and port of a broker. 47 | * 48 | * @param int the id of the brother to get the address of 49 | * 50 | * @return string the hostname and port of the broker, separated by a colon: host:port 51 | */ 52 | public function address($broker) { 53 | $data = sprintf(self::BROKER_PATH, (int) $broker); 54 | $result = $this->zookeeper->get($data); 55 | 56 | if (empty($result)) { 57 | $result = null; 58 | } else { 59 | $parts = explode(":", $result); 60 | $result = $parts[1] . ':' . $parts[2]; 61 | } 62 | 63 | return $result; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/lib/Kafka/Registry/Offset.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * A Registry for Kafka Consumer offsets. 15 | * 16 | * The primary use of this is a facade API on top of ZooKeeper, providing a 17 | * more friendly interface to some common operations. 18 | * 19 | * @category Libraries 20 | * @package Kafka 21 | * @author Lorenzo Alberton 22 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 23 | * @link http://sna-projects.com/kafka/ 24 | */ 25 | class Kafka_Registry_Offset 26 | { 27 | const OFFSETS_PATH = '/consumers/%s/offsets/%s'; 28 | const OFFSET_PATH = '/consumers/%s/offsets/%s/%d-%d'; 29 | 30 | /** 31 | * @var Zookeeper 32 | */ 33 | private $zookeeper; 34 | 35 | /** 36 | * @var string 37 | */ 38 | private $group; 39 | 40 | /** 41 | * Create a new Offset Registry for the given group. 42 | * 43 | * @param Zookeeper $zookeeper a Zookeeper instance to back this OffsetRegistry 44 | * @param string $group the consumer group to create this OffsetRegistry for 45 | */ 46 | public function __construct(Zookeeper $zookeeper, $group) { 47 | $this->zookeeper = $zookeeper; 48 | $this->group = (string) $group; 49 | } 50 | 51 | /** 52 | * Commits the given offset for the given partition 53 | * 54 | * @param string $topic the topic the partition belongs to 55 | * @param int $broker the broker holding the partition 56 | * @param int $partition the partition on the broker 57 | * @param int $offset the offset to commit 58 | */ 59 | public function commit($topic, $broker, $partition, $offset) { 60 | $path = sprintf(self::OFFSET_PATH, $this->group, $topic, (int) $broker, (int) $partition); 61 | if (!$this->zookeeper->exists($path)) { 62 | $this->makeZkPath($path); 63 | $this->makeZkNode($path, (int)$offset); 64 | } else { 65 | $this->zookeeper->set($path, (int) $offset); 66 | } 67 | } 68 | 69 | /** 70 | * Equivalent of "mkdir -p" on ZooKeeper 71 | * 72 | * @param string $path The path to the node 73 | * @param mixed $value The value to assign to each new node along the path 74 | * 75 | * @return bool 76 | */ 77 | protected function makeZkPath($path, $value = 0) { 78 | $parts = explode('/', $path); 79 | $parts = array_filter($parts); 80 | $subpath = ''; 81 | while (count($parts) > 1) { 82 | $subpath .= '/' . array_shift($parts); 83 | if (!$this->zookeeper->exists($subpath)) { 84 | $this->makeZkNode($subpath, $value); 85 | } 86 | } 87 | } 88 | 89 | /** 90 | * Create a node on ZooKeeper at the given path 91 | * 92 | * @param string $path The path to the node 93 | * @param mixed $value The value to assign to the new node 94 | * 95 | * @return bool 96 | */ 97 | protected function makeZkNode($path, $value) { 98 | $params = array( 99 | array( 100 | 'perms' => Zookeeper::PERM_ALL, 101 | 'scheme' => 'world', 102 | 'id' => 'anyone', 103 | ) 104 | ); 105 | return $this->zookeeper->create($path, $value, $params); 106 | } 107 | 108 | /** 109 | * Get the current offset for the specified partition of a topic on a broker. 110 | * 111 | * @param string $topic the topic the partition belongs to 112 | * @param int $broker the broker holding the partition 113 | * @param int $partition the partition on the broker 114 | * 115 | * @return int the byte offset for the cursor in the partition 116 | */ 117 | public function offset($topic, $broker, $partition) { 118 | $path = sprintf(self::OFFSET_PATH, $this->group, $topic, (int) $broker, (int) $partition); 119 | if (!$this->zookeeper->exists($path)) { 120 | return 0; 121 | } 122 | 123 | $result = $this->zookeeper->get($path); 124 | return empty($result) ? 0 : $result; 125 | } 126 | 127 | /** 128 | * Gets the current offsets for all partitions of a topic. 129 | * 130 | * @param string $topic the topic to get the offsets for 131 | * 132 | * @return array a map of partition (broker + partition ID) to the byte offset offset (int). 133 | */ 134 | public function offsets($topic) { //: Map[(Int, Int), Long] == Map[(Broker, Partition), Offset] 135 | $offsets = array(); 136 | foreach ($this->partitions($topic) as $broker => $partition) { 137 | if (!isset($offsets[$broker])) { 138 | $offsets[$broker] = array(); 139 | } 140 | 141 | $offsets[$broker][$partition] = $this->offset($topic, $broker, $partition); 142 | } 143 | return $offsets; 144 | } 145 | 146 | /** 147 | * Gets all the partitions for a given topic. 148 | * 149 | * @param string $topic the topic to get the partitions for 150 | * 151 | * @return array an associative array of the broker (int) to the number of partitions (int) 152 | */ 153 | public function partitions($topic) { 154 | $offsetsPath = sprintf(self::OFFSETS_PATH, $this->group, $topic); 155 | if (!$this->zookeeper->exists($offsetsPath)) { 156 | return array(); 157 | } 158 | 159 | $children = $this->zookeeper->getChildren($offsetsPath); 160 | $partitions = array(); 161 | foreach ($children as $child) { 162 | list($broker, $partition) = explode('-', str_replace($offsetsPath, '', $child), 2); 163 | $partitions[intval($broker)] = intval($partition); 164 | } 165 | return $partitions; 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /src/lib/Kafka/Registry/Topic.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Nick Telford 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * A Registry for Kafka brokers and the partitions they manage. 15 | * 16 | * The primary use of this is a facade API on top of ZooKeeper, providing a 17 | * more friendly interface to some common operations. 18 | * 19 | * @category Libraries 20 | * @package Kafka 21 | * @author Nick Telford 22 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 23 | * @link http://sna-projects.com/kafka/ 24 | */ 25 | class Kafka_Registry_Topic 26 | { 27 | const TOPIC_PATH = "/brokers/topics/%s"; 28 | const BROKER_PATH = "/brokers/topics/%s/%d"; 29 | 30 | /** 31 | * @var Zookeeper 32 | */ 33 | private $zookeeper; 34 | 35 | /** 36 | * Create a new Topic Reigstry. 37 | * 38 | * @param Zookeeper $zookeeper the Zookeeper instance to back this TopicRegistry with. 39 | */ 40 | public function __construct(Zookeeper $zookeeper) 41 | { 42 | $this->zookeeper = $zookeeper; 43 | } 44 | 45 | /** 46 | * Get the partitions on a particular broker for a specific topic. 47 | * 48 | * @param string $topic the topic to get the partitions of 49 | * @param int $broker the broker to get the partitions from 50 | * 51 | * @return int the number of the topics partitions on the broker 52 | */ 53 | public function partitionsForBroker($topic, $broker) 54 | { 55 | $data = sprintf(self::BROKER_PATH, $topic, (int) $broker); 56 | $result = $this->zookeeper->get($data); 57 | return empty($result) ? 0 : (int) $result; 58 | } 59 | 60 | /** 61 | * Get the partitions for a particular topic, grouped by broker. 62 | * 63 | * @param string $topic the topic to get the partitions of 64 | * 65 | * @return array the partitions as a map of broker to number of partitions (int -> int) 66 | */ 67 | public function partitions($topic) 68 | { 69 | $results = array(); 70 | foreach ($this->brokers($topic) as $broker) { 71 | $results[$broker] = $this->partitionsForBroker($topic, $broker); 72 | } 73 | return $results; 74 | } 75 | 76 | /** 77 | * Get the currently active brokers participating in a particular topic. 78 | * 79 | * @param string $topic the topic to get the brokers for 80 | * 81 | * @return array an array of brokers (int) that are participating in the topic 82 | */ 83 | public function brokers($topic) 84 | { 85 | $topicPath = sprintf(self::TOPIC_PATH, $topic); 86 | if (!@$this->zookeeper->exists($topicPath)) { 87 | if ($this->zookeeper->getState() != Zookeeper::CONNECTED_STATE) { 88 | $msg = 'Cannot connect to Zookeeper to fetch brokers for topic ' . $topic; 89 | throw new Kafka_Exception_ZookeeperConnection($msg); 90 | } 91 | return array(); 92 | } 93 | $children = $this->zookeeper->getChildren($topicPath); 94 | if (empty($children)) { 95 | return array(); 96 | } 97 | 98 | $results = array(); 99 | foreach ($children as $child) { 100 | $results[] = intval(str_replace($topicPath, '', $child)); 101 | } 102 | return $results; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/lib/Kafka/Request.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Abstract Request class 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | abstract class Kafka_Request 24 | { 25 | /** 26 | * @var integer 27 | */ 28 | public $id; 29 | 30 | /** 31 | * @var string 32 | */ 33 | protected $topic; 34 | 35 | /** 36 | * @var integer 37 | */ 38 | protected $partition; 39 | 40 | /** 41 | * Write the request to the output stream 42 | * 43 | * @param Kafka_Socket $socket Output stream 44 | * 45 | * @return void 46 | */ 47 | abstract public function writeTo(Kafka_Socket $socket); 48 | 49 | /** 50 | * Get request size in bytes 51 | * 52 | * @return integer 53 | */ 54 | abstract public function sizeInBytes(); 55 | 56 | /** 57 | * Write the Request Header 58 | * + + + + 59 | * 60 | * @param Kafka_Socket $socket Socket 61 | * 62 | * @return void 63 | */ 64 | protected function writeRequestHeader(Kafka_Socket $socket) { 65 | // REQUEST_LENGTH (int) + REQUEST_TYPE (short) 66 | $socket->write(pack('N', $this->sizeInBytes() + 2)); 67 | $socket->write(pack('n', $this->id)); 68 | 69 | // TOPIC_SIZE (short) + TOPIC (bytes) 70 | $socket->write(pack('n', strlen($this->topic)) . $this->topic); 71 | // PARTITION (int) 72 | $socket->write(pack('N', $this->partition)); 73 | } 74 | 75 | /** 76 | * Pack a 64bit integer as big endian long 77 | * 78 | * @param integer $big Big int 79 | * 80 | * @return bytes 81 | */ 82 | static public function packLong64bigendian($big) { 83 | $left = 0xffffffff00000000; 84 | $right = 0x00000000ffffffff; 85 | 86 | $l = ($big & $left) >> 32; 87 | $r = $big & $right; 88 | 89 | return pack('NN', $l, $r); 90 | } 91 | 92 | /** 93 | * Pack a 64bit integer as big endian long 94 | * 95 | * @param integer $big Big int 96 | * 97 | * @return integer 98 | */ 99 | static public function unpackLong64bigendian($bytes) { 100 | $set = unpack('N2', $bytes); 101 | return $original = ($set[1] & 0xFFFFFFFF) << 32 | ($set[2] & 0xFFFFFFFF); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/lib/Kafka/RequestKeys.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Some constants for request keys 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_RequestKeys 24 | { 25 | const PRODUCE = 0; 26 | const FETCH = 1; 27 | const MULTIFETCH = 2; 28 | const MULTIPRODUCE = 3; 29 | const OFFSETS = 4; 30 | } 31 | -------------------------------------------------------------------------------- /src/lib/Kafka/Response.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Response class 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_Response 24 | { 25 | /** 26 | * Validate the error code from the response 27 | * 28 | * @param integer $errorCode Error code 29 | * 30 | * @return void 31 | * @throws Kafka_Exception 32 | */ 33 | static public function validateErrorCode($errorCode) { 34 | switch ($errorCode) { 35 | case 0: break; //success 36 | case 1: throw new Kafka_Exception_OffsetOutOfRange('OffsetOutOfRange reading response errorCode'); 37 | case 2: throw new Kafka_Exception_InvalidMessage('InvalidMessage reading response errorCode'); 38 | case 3: throw new Kafka_Exception_WrongPartition('WrongPartition reading response errorCode'); 39 | case 4: throw new Kafka_Exception_InvalidFetchSize('InvalidFetchSize reading response errorCode'); 40 | default: throw new Kafka_Exception('Unknown error reading response errorCode (' . $errorCode . ')'); 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/lib/Kafka/SimpleConsumer.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Simple Kafka Consumer 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_SimpleConsumer 24 | { 25 | /** 26 | * Latest offset available 27 | * 28 | * @const int 29 | */ 30 | const OFFSET_LAST = -1; 31 | 32 | /** 33 | * Smallest offset available 34 | * 35 | * @const int 36 | */ 37 | const OFFSET_FIRST = -2; 38 | 39 | /** 40 | * @var string 41 | */ 42 | protected $host = 'localhost'; 43 | 44 | /** 45 | * @var integer 46 | */ 47 | protected $port = 9092; 48 | 49 | /** 50 | * @var Kafka_Socket 51 | */ 52 | protected $socket = null; 53 | 54 | /** 55 | * Send timeout in seconds. 56 | * 57 | * Combined with sendTimeoutUsec this is used for send timeouts. 58 | * 59 | * @var int 60 | */ 61 | private $sendTimeoutSec = 0; 62 | 63 | /** 64 | * Send timeout in microseconds. 65 | * 66 | * Combined with sendTimeoutSec this is used for send timeouts. 67 | * 68 | * @var int 69 | */ 70 | private $sendTimeoutUsec = 100000; 71 | 72 | /** 73 | * Recv timeout in seconds 74 | * 75 | * Combined with recvTimeoutUsec this is used for recv timeouts. 76 | * 77 | * @var int 78 | */ 79 | private $recvTimeoutSec = 0; 80 | 81 | /** 82 | * Recv timeout in microseconds 83 | * 84 | * Combined with recvTimeoutSec this is used for recv timeouts. 85 | * 86 | * @var int 87 | */ 88 | private $recvTimeoutUsec = 250000; 89 | 90 | /** 91 | * @var integer 92 | */ 93 | protected $socketTimeout = 10; 94 | 95 | /** 96 | * @var integer 97 | */ 98 | protected $socketBufferSize = 1000000; 99 | 100 | /** 101 | * @var integer 102 | */ 103 | protected $lastResponseSize = 0; 104 | 105 | /** 106 | * Constructor 107 | * 108 | * @param integer $host Kafka Hostname 109 | * @param integer $port Port 110 | * @param integer $socketTimeout Socket timeout 111 | * @param integer $socketBufferSize Socket max buffer size 112 | */ 113 | public function __construct($host, $port, $socketTimeout, $socketBufferSize) { 114 | $this->host = $host; 115 | $this->port = $port; 116 | $this->recvTimeoutSec = $socketTimeout; 117 | $this->sendTimeoutSec = $socketTimeout; 118 | $this->socketBufferSize = $socketBufferSize; 119 | } 120 | 121 | /** 122 | * Set recv/send socket timeouts (in seconds and microseconds) 123 | * 124 | * @param integer $recvTimeoutSec Recv timeout in seconds 125 | * @param integer $recvTimeoutUsec Recv timeout in microseconds 126 | * @param integer $sendTimeoutSec Send timeout in seconds 127 | * @param integer $sendTimeoutUsec Send timeout in microseconds 128 | * 129 | * @return 130 | */ 131 | public function setSocketTimeouts($recvTimeoutSec = 0, $recvTimeoutUsec = 250000, $sendTimeoutSec = 0, $sendTimeoutUsec = 100000) { 132 | $this->recvTimeoutSec = (int) $recvTimeoutSec; 133 | $this->recvTimeoutUsec = (int) $recvTimeoutUsec; 134 | $this->sendTimeoutSec = (int) $sendTimeoutSec; 135 | $this->sendTimeoutUsec = (int) $sendTimeoutUsec; 136 | } 137 | 138 | /** 139 | * Connect to Kafka via socket 140 | * 141 | * @return void 142 | */ 143 | public function connect() { 144 | if (null === $this->socket) { 145 | $this->socket = new Kafka_Socket( 146 | $this->host, 147 | $this->port, 148 | $this->recvTimeoutSec, 149 | $this->recvTimeoutUsec, 150 | $this->sendTimeoutSec, 151 | $this->sendTimeoutUsec 152 | ); 153 | } 154 | $this->socket->connect(); 155 | } 156 | 157 | /** 158 | * Close the connection 159 | * 160 | * @return void 161 | */ 162 | public function close() { 163 | if (null !== $this->socket) { 164 | $this->socket->close(); 165 | } 166 | } 167 | 168 | /** 169 | * Send a request and fetch the response 170 | * 171 | * @param Kafka_Request $req Request 172 | * 173 | * @return Kafka_MessageSet $messages 174 | * @throws Kafka_Exception 175 | */ 176 | public function fetch(Kafka_Request $req) { 177 | $this->connect(); 178 | // send request 179 | $req->writeTo($this->socket); 180 | 181 | // get response 182 | $this->lastResponseSize = $this->getResponseSize(); 183 | $responseCode = $this->getResponseCode(); 184 | $initialOffset = 6; 185 | 186 | // validate response 187 | Kafka_Response::validateErrorCode($responseCode); 188 | if ($this->lastResponseSize == 2) { 189 | throw new Kafka_Exception_EmptyQueue(); 190 | } 191 | 192 | return new Kafka_MessageSet($this->socket, $initialOffset); 193 | } 194 | 195 | /** 196 | * Get the last response size 197 | * 198 | * @return integer 199 | */ 200 | public function getLastResponseSize() { 201 | return $this->lastResponseSize; 202 | } 203 | 204 | /** 205 | * Read the request size (4 bytes) if not read yet 206 | * 207 | * @param resource $stream Stream resource 208 | * 209 | * @return integer Size of the response buffer in bytes 210 | * @throws Kafka_Exception_Socket_EOF 211 | * @throws Kafka_Exception_Socket_Timeout 212 | * @throws Kafka_Exception when size is <=0 or >= $maxSize 213 | */ 214 | protected function getResponseSize() { 215 | $this->connect(); 216 | $size = $this->socket->read(4, true); 217 | $unpack = unpack('N', $size); 218 | $size = array_shift($unpack); 219 | if ($size <= 0) { 220 | throw new Kafka_Exception_OutOfRange($size . ' is not a valid response size'); 221 | } 222 | return $size; 223 | } 224 | 225 | /** 226 | * Read the response error code 227 | * 228 | * @return integer Error code 229 | */ 230 | protected function getResponseCode() { 231 | $this->connect(); 232 | $data = $this->socket->read(2, true); 233 | $unpack = unpack('n', $data); 234 | return array_shift($unpack); 235 | } 236 | 237 | /** 238 | * Get a list of valid offsets (up to maxSize) before the given time. 239 | * The result is a list of offsets, in descending order. 240 | * 241 | * @param time: time in millisecs (-1 from the latest offset available, -2 from the smallest offset available) 242 | * 243 | * @return an array of offsets 244 | */ 245 | public function getOffsetsBefore($topic, $partition, $time, $maxNumOffsets) { 246 | $req = new Kafka_OffsetRequest($topic, $partition, $time, $maxNumOffsets); 247 | try { 248 | $this->connect(); 249 | // send request 250 | $req->writeTo($this->socket); 251 | //echo "\nRequest sent: ".(string)$req."\n"; 252 | } catch (Kafka_Socket_Exception_EOF $e) { 253 | //echo "\nReconnect in get offetset request due to socket error: " . $e->getMessage(); 254 | // retry once 255 | $this->connect(); 256 | $req->writeTo($this->socket); 257 | } 258 | $size = $this->getResponseSize(); 259 | $errorCode = $this->getResponseCode(); 260 | Kafka_Response::validateErrorCode($errorCode); 261 | 262 | return Kafka_OffsetRequest::deserializeOffsetArray($this->socket); 263 | } 264 | 265 | /** 266 | * Close the socket connection if still open 267 | * 268 | * @return vpopmail_del_domain(domain) 269 | */ 270 | public function __destruct() { 271 | $this->close(); 272 | } 273 | } 274 | -------------------------------------------------------------------------------- /src/lib/Kafka/Socket.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @version $Revision: $ 11 | * @link http://sna-projects.com/kafka/ 12 | */ 13 | 14 | /** 15 | * Class to read/write to a socket 16 | * 17 | * @category Libraries 18 | * @package Kafka 19 | * @author Lorenzo Alberton 20 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 21 | * @link http://sna-projects.com/kafka/ 22 | */ 23 | class Kafka_Socket 24 | { 25 | /** 26 | * Send timeout in seconds. 27 | * 28 | * Combined with sendTimeoutUsec this is used for send timeouts. 29 | * 30 | * @var int 31 | */ 32 | private $sendTimeoutSec = 0; 33 | 34 | /** 35 | * Send timeout in microseconds. 36 | * 37 | * Combined with sendTimeoutSec this is used for send timeouts. 38 | * 39 | * @var int 40 | */ 41 | private $sendTimeoutUsec = 100000; 42 | 43 | /** 44 | * Recv timeout in seconds 45 | * 46 | * Combined with recvTimeoutUsec this is used for recv timeouts. 47 | * 48 | * @var int 49 | */ 50 | private $recvTimeoutSec = 0; 51 | 52 | /** 53 | * Recv timeout in microseconds 54 | * 55 | * Combined with recvTimeoutSec this is used for recv timeouts. 56 | * 57 | * @var int 58 | */ 59 | private $recvTimeoutUsec = 750000; 60 | 61 | /** 62 | * Stream resource 63 | * 64 | * @var resource 65 | */ 66 | private $stream = null; 67 | 68 | /** 69 | * Socket host 70 | * 71 | * @var string 72 | */ 73 | private $host = null; 74 | 75 | /** 76 | * Socket port 77 | * 78 | * @var int 79 | */ 80 | private $port = -1; 81 | 82 | /** 83 | * Constructor 84 | * 85 | * @param string $host Host 86 | * @param int $port Port 87 | * @param int $recvTimeoutSec Recv timeout in seconds 88 | * @param int $recvTimeoutUsec Recv timeout in microseconds 89 | * @param int $sendTimeoutSec Send timeout in seconds 90 | * @param int $sendTimeoutUsec Send timeout in microseconds 91 | */ 92 | public function __construct($host, $port, $recvTimeoutSec = 0, $recvTimeoutUsec = 750000, $sendTimeoutSec = 0, $sendTimeoutUsec = 100000) { 93 | $this->host = $host; 94 | $this->port = (int)$port; 95 | $this->recvTimeoutSec = $recvTimeoutSec; 96 | $this->recvTimeoutUsec = $recvTimeoutUsec; 97 | $this->sendTimeoutSec = $sendTimeoutSec; 98 | $this->sendTimeoutUsec = $sendTimeoutUsec; 99 | } 100 | 101 | /** 102 | * Optional method to set the internal stream handle 103 | * 104 | * @param resource $stream File handle 105 | * 106 | * @return Kafka_Socket 107 | */ 108 | static public function createFromStream($stream) { 109 | $socket = new self('localhost', 0); 110 | $socket->setStream($stream); 111 | return $socket; 112 | } 113 | 114 | /** 115 | * Optional method to set the internal stream handle 116 | * 117 | * @param resource $stream File handle 118 | * 119 | * @return void 120 | */ 121 | public function setStream($stream) { 122 | $this->stream = $stream; 123 | } 124 | 125 | /** 126 | * Connects the socket 127 | * 128 | * @return bool|null 129 | * @throws Kafka_Exception_Socket_Connection 130 | */ 131 | public function connect() { 132 | if (is_resource($this->stream)) { 133 | return true; 134 | } 135 | 136 | if (empty($this->host)) { 137 | throw new Kafka_Exception_Socket_Connection('Cannot open null host'); 138 | } 139 | if ($this->port <= 0) { 140 | throw new Kafka_Exception_Socket_Connection('Cannot open without port'); 141 | } 142 | 143 | $this->stream = @fsockopen( 144 | $this->host, 145 | $this->port, 146 | $errno, 147 | $errstr, 148 | $this->sendTimeoutSec + ($this->sendTimeoutUsec / 1000000) 149 | ); 150 | 151 | // Connect failed? 152 | if ($this->stream === FALSE) { 153 | $error = 'Could not connect to '.$this->host.':'.$this->port.' ('.$errstr.' ['.$errno.'])'; 154 | throw new Kafka_Exception_Socket_Connection($error); 155 | } 156 | 157 | // Set to blocking mode when keeping a persistent connection, 158 | // otherwise leave it as non-blocking when polling 159 | @stream_set_blocking($this->stream, 0); 160 | //socket_set_option($this->stream, SOL_TCP, TCP_NODELAY, 1); 161 | //socket_set_option($this->stream, SOL_TCP, SO_KEEPALIVE, 1); 162 | } 163 | 164 | /** 165 | * Closes the socket 166 | * 167 | * @return void 168 | */ 169 | public function close() { 170 | if (is_resource($this->stream)) { 171 | fclose($this->stream); 172 | } 173 | } 174 | 175 | /** 176 | * Read from the socket at most $len bytes. 177 | * 178 | * This method will not wait for all the requested data, it will return as 179 | * soon as any data is received. 180 | * 181 | * @param int $len Maximum number of bytes to read. 182 | * @param bool $verifyExactLength Throw an exception if the number of read bytes is less than $len 183 | * 184 | * @return string Binary data 185 | * @throws Kafka_Exception_Socket 186 | */ 187 | public function read($len, $verifyExactLength = false) { 188 | $null = null; 189 | $read = array($this->stream); 190 | $readable = @stream_select($read, $null, $null, $this->recvTimeoutSec, $this->recvTimeoutUsec); 191 | if ($readable > 0) { 192 | $remainingBytes = $len; 193 | $data = $chunk = ''; 194 | while ($remainingBytes > 0) { 195 | $chunk = fread($this->stream, $remainingBytes); 196 | if ($chunk === false) { 197 | $this->close(); 198 | throw new Kafka_Exception_Socket_EOF('Could not read '.$len.' bytes from stream (no data)'); 199 | } 200 | if (strlen($chunk) === 0) { 201 | // Zero bytes because of EOF? 202 | if (feof($this->stream)) { 203 | $this->close(); 204 | throw new Kafka_Exception_Socket_EOF('Unexpected EOF while reading '.$len.' bytes from stream (no data)'); 205 | } 206 | // Otherwise wait for bytes 207 | $readable = @stream_select($read, $null, $null, $this->recvTimeoutSec, $this->recvTimeoutUsec); 208 | if ($readable !== 1) { 209 | throw new Kafka_Exception_Socket_Timeout('Timed out reading socket while reading ' . $len . ' bytes with ' . $remainingBytes . ' bytes to go'); 210 | } 211 | continue; // attempt another read 212 | } 213 | $data .= $chunk; 214 | $remainingBytes -= strlen($chunk); 215 | } 216 | if ($len === $remainingBytes || ($verifyExactLength && $len !== strlen($data))) { 217 | // couldn't read anything at all OR reached EOF sooner than expected 218 | $this->close(); 219 | throw new Kafka_Exception_Socket_EOF('Read ' . strlen($data) . ' bytes instead of the requested ' . $len . ' bytes'); 220 | } 221 | 222 | return $data; 223 | } 224 | if (false !== $readable) { 225 | $res = stream_get_meta_data($this->stream); 226 | if (!empty($res['timed_out'])) { 227 | $this->close(); 228 | throw new Kafka_Exception_Socket_Timeout('Timed out reading '.$len.' bytes from stream'); 229 | } 230 | } 231 | $this->close(); 232 | throw new Kafka_Exception_Socket_EOF('Could not read '.$len.' bytes from stream (not readable)'); 233 | } 234 | 235 | /** 236 | * Write to the socket. 237 | * 238 | * @param string $buf The data to write 239 | * 240 | * @return int 241 | * @throws Kafka_Exception_Socket 242 | */ 243 | public function write($buf) { 244 | $null = null; 245 | $write = array($this->stream); 246 | 247 | // fwrite to a socket may be partial, so loop until we 248 | // are done with the entire buffer 249 | $written = 0; 250 | $buflen = strlen($buf); 251 | while ( $written < $buflen ) { 252 | // wait for stream to become available for writing 253 | $writable = @stream_select($null, $write, $null, $this->sendTimeoutSec, $this->sendTimeoutUsec); 254 | if ($writable > 0) { 255 | // Set a temporary error handler to watch for Broken pipes 256 | set_error_handler(function ($type, $msg, $file, $line) use ($buflen, &$written) { 257 | if (strpos($msg, 'Broken pipe') !== false) { 258 | throw new \Kafka_Exception_Socket( 259 | sprintf('Connection broken while writing %d bytes to stream, completed writing only %d bytes', $buflen, $written) 260 | ); 261 | } 262 | return false; // Allow normal error handling to continue 263 | }); 264 | try { 265 | // write remaining buffer bytes to stream 266 | $wrote = fwrite($this->stream, substr($buf, $written)); 267 | } finally { 268 | restore_error_handler(); 269 | } 270 | if ($wrote === -1 || $wrote === false) { 271 | throw new Kafka_Exception_Socket('Could not write ' . $buflen . ' bytes to stream, completed writing only ' . $written . ' bytes'); 272 | } 273 | $written += $wrote; 274 | continue; 275 | } 276 | if (false !== $writable) { 277 | $res = stream_get_meta_data($this->stream); 278 | if (!empty($res['timed_out'])) { 279 | throw new Kafka_Exception_Socket_Timeout('Timed out writing ' . $buflen . ' bytes to stream after writing ' . $written . ' bytes'); 280 | } 281 | } 282 | throw new Kafka_Exception_Socket('Could not write ' . $buflen . ' bytes to stream'); 283 | } 284 | return $written; 285 | } 286 | 287 | /** 288 | * Rewind the stream 289 | * 290 | * @return void 291 | */ 292 | public function rewind() { 293 | if (is_resource($this->stream)) { 294 | rewind($this->stream); 295 | } 296 | } 297 | } 298 | -------------------------------------------------------------------------------- /src/lib/Kafka/ZookeeperConsumer.php: -------------------------------------------------------------------------------- 1 | 8 | * @copyright 2012 Lorenzo Alberton 9 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 10 | * @link http://sna-projects.com/kafka/ 11 | */ 12 | 13 | /** 14 | * Zookeeper-based Kafka Consumer 15 | * 16 | * This is a sample implementation, there can be different strategies on how to consume 17 | * data from different brokers/partitions. Here the strategy is to read up to MAX_BATCH_SIZE 18 | * bytes from each partition before moving to the next. The order of brokers/partitions is 19 | * randomised in each loop to consume data from all queues in a more-or-less fair way. 20 | * An alternative strategy would be to round-robin the brokers/partitions, reading one message 21 | * from each; this strategy would be fairer, but way less efficient. 22 | * 23 | * @category Libraries 24 | * @package Kafka 25 | * @author Lorenzo Alberton 26 | * @license http://www.apache.org/licenses/LICENSE-2.0 Apache License, Version 2.0 27 | * @link http://sna-projects.com/kafka/ 28 | */ 29 | class Kafka_ZookeeperConsumer implements Iterator 30 | { 31 | /** 32 | * @var Kafka_Registry_Topic 33 | */ 34 | protected $topicRegistry; 35 | 36 | /** 37 | * @var Kafka_Registry_Broker 38 | */ 39 | protected $brokerRegistry; 40 | 41 | /** 42 | * @var Kafka_Registry_Offset 43 | */ 44 | protected $offsetRegistry; 45 | 46 | /** 47 | * @var string 48 | */ 49 | protected $topic; 50 | 51 | /** 52 | * @var integer 53 | */ 54 | protected $readBytes = 0; 55 | 56 | /** 57 | * @var integer 58 | */ 59 | protected $socketTimeout = 0; 60 | 61 | /** 62 | * @var integer 63 | */ 64 | protected $maxBatchSize = 20000000; 65 | 66 | /** 67 | * @var array 68 | */ 69 | protected $iterators = array(); 70 | 71 | /** 72 | * @var integer 73 | */ 74 | protected $idx = 0; 75 | 76 | /** 77 | * @var integer 78 | */ 79 | protected $nIterators = 0; 80 | 81 | /** 82 | * @var boolean 83 | */ 84 | protected $hasMore = false; 85 | 86 | /** 87 | * Create a new BatchedConsumer for a topic using the given TopicReigstry and OffsetRegistry. 88 | * 89 | * @param Kafka_Registry_Topic $topicRegistry a registry for the discovery of topic partitions 90 | * @param Kafka_Registry_Broker $brokerRegistry a registry for the tracking of brokers 91 | * @param Kafka_Registry_Offset $offsetRegistry a registry for the tracking of the consumer offsets 92 | * @param string $topic the topic to consume from 93 | * @param integer $maxBatchSize maximum batch size (in bytes) 94 | */ 95 | public function __construct( 96 | Kafka_Registry_Topic $topicRegistry, 97 | Kafka_Registry_Broker $brokerRegistry, 98 | Kafka_Registry_Offset $offsetRegistry, 99 | $topic, 100 | $maxBatchSize = 20000000 101 | ) { 102 | $this->topicRegistry = $topicRegistry; 103 | $this->brokerRegistry = $brokerRegistry; 104 | $this->offsetRegistry = $offsetRegistry; 105 | $this->topic = $topic; 106 | $this->maxBatchSize = $maxBatchSize; 107 | } 108 | 109 | /** 110 | * Shuffle the internal iterators for each broker/partition 111 | * 112 | * @return void 113 | */ 114 | public function shuffle() { 115 | shuffle($this->iterators); 116 | } 117 | 118 | /** 119 | * Advance the iterator's pointer 120 | * 121 | * @return void 122 | */ 123 | public function next() { 124 | return $this->iterators[$this->idx]->messages->next(); 125 | } 126 | 127 | /** 128 | * Get the key for this item 129 | * 130 | * @return integer 131 | */ 132 | public function key() { 133 | return $this->iterators[$this->idx]->messages->key(); 134 | } 135 | 136 | /** 137 | * Get the current message 138 | * 139 | * @return mixed 140 | */ 141 | public function current() { 142 | return $this->iterators[$this->idx]->messages->current()->payload(); 143 | } 144 | 145 | /** 146 | * Check whether we have a valid iterator 147 | * 148 | * @return boolean 149 | */ 150 | public function valid() { 151 | while ($this->idx < $this->nIterators) { 152 | $it = $this->iterators[$this->idx]; 153 | try { 154 | if (null === $it->messages) { 155 | $it->consumer = new Kafka_SimpleConsumer($it->host, $it->port, $this->socketTimeout, $this->maxBatchSize); 156 | $newOffset = $it->offset + $it->uncommittedOffset; 157 | $request = new Kafka_FetchRequest($this->topic, $it->partition, $newOffset, $this->maxBatchSize); 158 | $it->messages = $it->consumer->fetch($request); 159 | $it->messages->rewind(); 160 | } 161 | if ($it->messages->valid()) { 162 | $this->hasMore = true; 163 | return true; 164 | } 165 | // we're done with the current broker/partition, count how much we've read so far and update the offsets 166 | $this->readBytes += $it->messages->validBytes(); 167 | $it->uncommittedOffset += $it->messages->validBytes(); 168 | } catch (Kafka_Exception_EmptyQueue $e) { 169 | // no new data from this broker/partition 170 | } 171 | // reset the MessageSet iterator and move to the next 172 | $it->messages = null; 173 | $it->consumer->close(); 174 | ++$this->idx; 175 | if ($this->idx === $this->nIterators) { 176 | $this->idx = 0; 177 | // if we looped through all brokers/partitions and we did get data 178 | // from at least one of them, reset the iterator and do another loop 179 | if ($this->hasMore) { 180 | $this->hasMore = false; 181 | } else if ($this->getRemainingSize() > 1048576) { 182 | // we often get stuck, i.e. we fetch 0 bytes even if there is more data... keep trying 183 | } else { 184 | return false; 185 | } 186 | } 187 | } 188 | return false; 189 | } 190 | 191 | /** 192 | * Return the number of bytes read so far 193 | * 194 | * @return integer 195 | */ 196 | public function getReadBytes() { 197 | if (0 == $this->nIterators) { 198 | return 0; 199 | } 200 | $it = $this->iterators[$this->idx]; 201 | $readInCurrentPartition = isset($it->messages) ? $it->messages->validBytes() : 0; 202 | return $this->readBytes + $readInCurrentPartition; 203 | } 204 | 205 | /** 206 | * Commit the kafka offsets for each broker/partition in ZooKeeper 207 | * 208 | * @return integer 209 | */ 210 | public function commitOffsets() { 211 | foreach ($this->iterators as $it) { 212 | $readBytes = $it->uncommittedOffset; 213 | if (null !== $it->messages) { 214 | $readBytes += $it->messages->validBytes(); 215 | } 216 | if ($readBytes > 0) { 217 | $this->offsetRegistry->commit($this->topic, $it->broker, $it->partition, $it->offset + $readBytes); 218 | $it->uncommittedOffset = 0; 219 | $it->offset += $readBytes; 220 | } 221 | } 222 | } 223 | 224 | /** 225 | * Resync invalid offsets to the first valid position 226 | * 227 | * @return integer Number of partitions/broker resync'ed 228 | */ 229 | public function resyncOffsets() { 230 | $nReset = 0; 231 | foreach ($this->iterators as $it) { 232 | $consumer = new Kafka_SimpleConsumer($it->host, $it->port, $this->socketTimeout, $this->maxBatchSize); 233 | try { 234 | $newOffset = $it->offset + $it->uncommittedOffset; 235 | $request = new Kafka_FetchRequest($this->topic, $it->partition, $newOffset, $this->maxBatchSize); 236 | $it->messages = $consumer->fetch($request); 237 | } catch (Kafka_Exception_OffsetOutOfRange $e) { 238 | $offsets = $consumer->getOffsetsBefore($this->topic, $it->partition, Kafka_SimpleConsumer::OFFSET_FIRST, 1); 239 | if (count($offsets) > 0) { 240 | $newOffset = $offsets[0]; 241 | $this->offsetRegistry->commit($this->topic, $it->broker, $it->partition, $newOffset); 242 | $it->uncommittedOffset = 0; 243 | $it->offset = $newOffset; 244 | ++$nReset; 245 | } 246 | } 247 | } 248 | return $nReset; 249 | } 250 | 251 | /** 252 | * Get an approximate measure of the amount of data still to be consumed 253 | * 254 | * @return integer 255 | */ 256 | public function getRemainingSize() { 257 | try { 258 | if (0 == $this->nIterators) { 259 | $this->rewind(); // initialise simple consumers 260 | } 261 | } catch (Kafka_Exception_InvalidTopic $e) { 262 | $logMsg = 'Invalid topic from ZookeeperConsumer::rewind(): Most likely cause is no topic yet as there is no data'; 263 | error_log($logMsg); 264 | } 265 | $totalSize = 0; 266 | foreach ($this->iterators as $it) { 267 | $readBytes = $it->offset + $it->uncommittedOffset; 268 | if (null !== $it->messages) { 269 | $readBytes += $it->messages->validBytes(); 270 | } 271 | $consumer = new Kafka_SimpleConsumer($it->host, $it->port, $this->socketTimeout, $this->maxBatchSize); 272 | $offsets = $consumer->getOffsetsBefore($this->topic, $it->partition, Kafka_SimpleConsumer::OFFSET_LAST, 1); 273 | if (count($offsets) > 0) { 274 | $remaining = $offsets[0] - $readBytes; // remaining bytes for this broker/partition 275 | if ($remaining > 0) { 276 | $totalSize += $remaining; 277 | } 278 | } 279 | $consumer->close(); 280 | } 281 | return $totalSize; 282 | } 283 | 284 | /** 285 | * Rewind the iterator 286 | * 287 | * @return void 288 | */ 289 | public function rewind() { 290 | $this->iterators = array(); 291 | $this->nIterators = 0; 292 | foreach ($this->topicRegistry->partitions($this->topic) as $broker => $nPartitions) { 293 | for ($partition = 0; $partition < $nPartitions; ++$partition) { 294 | list($host, $port) = explode(':', $this->brokerRegistry->address($broker)); 295 | $offset = $this->offsetRegistry->offset($this->topic, $broker, $partition); 296 | $this->iterators[] = (object) array( 297 | 'consumer' => null, 298 | 'host' => $host, 299 | 'port' => $port, 300 | 'broker' => $broker, 301 | 'partition' => $partition, 302 | 'offset' => $offset, 303 | 'uncommittedOffset' => 0, 304 | 'messages' => null, 305 | ); 306 | ++$this->nIterators; 307 | } 308 | } 309 | if (0 == count($this->iterators)) { 310 | throw new Kafka_Exception_InvalidTopic('Cannot find topic ' . $this->topic); 311 | } 312 | // get a random broker/partition every time 313 | $this->shuffle(); 314 | } 315 | } 316 | -------------------------------------------------------------------------------- /src/tests/Kafka/EncoderTest.php: -------------------------------------------------------------------------------- 1 | 29 | */ 30 | class Kafka_EncoderTest extends PHPUnit_Framework_TestCase 31 | { 32 | public function testEncodedMessageLength() { 33 | $test = 'a sample string'; 34 | $encoded = Kafka_Encoder::encode_message($test); 35 | $this->assertEquals(6 + strlen($test), strlen($encoded)); 36 | } 37 | 38 | public function testByteArrayContainsString() { 39 | $test = 'a sample string'; 40 | $encoded = Kafka_Encoder::encode_message($test); 41 | $this->assertContains($test, $encoded); 42 | } 43 | 44 | public function testEncodedMessages() { 45 | $topic = 'sample topic'; 46 | $partition = 1; 47 | $messages = array( 48 | 'test 1', 49 | 'test 2 abcde', 50 | 'test 3', 51 | ); 52 | $encoded = Kafka_Encoder::encode_produce_request($topic, $partition, $messages, Kafka_Encoder::COMPRESSION_NONE); 53 | $this->assertContains($topic, $encoded); 54 | $this->assertContains($partition, $encoded); 55 | foreach ($messages as $msg) { 56 | $this->assertContains($msg, $encoded); 57 | } 58 | $size = 4 + 2 + 2 + strlen($topic) + 4 + 4; 59 | foreach ($messages as $msg) { 60 | $size += 10 + strlen($msg); 61 | } 62 | $this->assertEquals($size, strlen($encoded)); 63 | } 64 | 65 | public function testCompressNone() { 66 | $msg = 'test message'; 67 | $this->assertEquals($msg, Kafka_Encoder::compress($msg, Kafka_Encoder::COMPRESSION_NONE)); 68 | } 69 | 70 | public function testCompressGzip() { 71 | $msg = 'test message'; 72 | $this->assertEquals($msg, gzdecode(Kafka_Encoder::compress($msg, Kafka_Encoder::COMPRESSION_GZIP))); 73 | } 74 | 75 | /** 76 | * @expectedException Kafka_Exception_NotSupported 77 | */ 78 | public function testCompressSnappy() { 79 | $msg = 'test message'; 80 | Kafka_Encoder::compress($msg, Kafka_Encoder::COMPRESSION_SNAPPY); 81 | $this->fail('The above call should fail until SNAPPY support is added'); 82 | } 83 | 84 | /** 85 | * @expectedException Kafka_Exception_NotSupported 86 | */ 87 | public function testCompressUnknown() { 88 | $msg = 'test message'; 89 | Kafka_Encoder::compress($msg, 15); 90 | $this->fail('The above call should fail'); 91 | } 92 | 93 | public function testDecompressNone() { 94 | $msg = 'test message'; 95 | $this->assertEquals($msg, Kafka_Encoder::decompress($msg, Kafka_Encoder::COMPRESSION_NONE)); 96 | } 97 | 98 | /** 99 | * @expectedException Kafka_Exception_NotSupported 100 | */ 101 | public function testDecompressSnappy() { 102 | $msg = 'test message'; 103 | Kafka_Encoder::decompress($msg, Kafka_Encoder::COMPRESSION_SNAPPY); 104 | $this->fail('The above call should fail until SNAPPY support is added'); 105 | } 106 | 107 | /** 108 | * @expectedException Kafka_Exception_NotSupported 109 | */ 110 | public function testDecompressUnknown() { 111 | $msg = 'test message'; 112 | Kafka_Encoder::decompress($msg, 15); 113 | $this->fail('The above call should fail'); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/tests/Kafka/FetchRequestTest.php: -------------------------------------------------------------------------------- 1 | 25 | */ 26 | class Kafka_FetchRequestTest extends PHPUnit_Framework_TestCase 27 | { 28 | private $topic; 29 | private $partition; 30 | private $offset; 31 | private $maxSize; 32 | 33 | /** 34 | * @var Kafka_FetchRequest 35 | */ 36 | private $req; 37 | 38 | public function setUp() { 39 | $this->topic = 'testtopic'; 40 | $this->partition = 0; 41 | $this->offset = 0; 42 | $this->maxSize = 10000; 43 | $this->req = new Kafka_FetchRequest($this->topic, $this->partition, $this->offset, $this->maxSize); 44 | } 45 | 46 | public function testRequestSize() { 47 | $this->assertEquals(18 + strlen($this->topic) , $this->req->sizeInBytes()); 48 | } 49 | 50 | public function testGetters() { 51 | $this->assertEquals($this->topic, $this->req->getTopic()); 52 | $this->assertEquals($this->offset, $this->req->getOffset()); 53 | $this->assertEquals($this->partition, $this->req->getPartition()); 54 | } 55 | 56 | public function testWriteTo() { 57 | $stream = fopen('php://temp', 'w+b'); 58 | $socket = Kafka_Socket::createFromStream($stream); 59 | $this->req->writeTo($socket); 60 | rewind($stream); 61 | $data = stream_get_contents($stream); 62 | fclose($stream); 63 | $expected_len = strlen($data) - 6; //6 Bytes of headers + data 64 | $this->assertEquals($expected_len, $this->req->sizeInBytes()); 65 | $this->assertContains($this->topic, $data); 66 | $this->assertContains($this->partition, $data); 67 | } 68 | 69 | public function testWriteToOffset() { 70 | $this->offset = 14; 71 | $this->req = new Kafka_FetchRequest($this->topic, $this->partition, $this->offset, $this->maxSize); 72 | $stream = fopen('php://temp', 'w+b'); 73 | $socket = Kafka_Socket::createFromStream($stream); 74 | $this->req->writeTo($socket); 75 | rewind($stream); 76 | //read it back 77 | $headers = fread($stream, 6); 78 | $topicLen = array_shift(unpack('n', fread($stream, 2))); 79 | $this->assertEquals(strlen($this->topic), $topicLen); 80 | $this->assertEquals($this->topic, fread($stream, $topicLen)); 81 | $this->assertEquals($this->partition, array_shift(unpack('N', fread($stream, 4)))); 82 | $int64bit = unpack('N2', fread($stream, 8)); 83 | $this->assertEquals($this->offset, $int64bit[2]); 84 | $this->assertEquals($this->maxSize, array_shift(unpack('N', fread($stream, 4)))); 85 | } 86 | 87 | public function testToString() { 88 | $this->assertContains('topic:' . $this->topic, (string)$this->req); 89 | $this->assertContains('part:' . $this->partition, (string)$this->req); 90 | $this->assertContains('offset:' . $this->offset, (string)$this->req); 91 | $this->assertContains('maxSize:' . $this->maxSize, (string)$this->req); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/tests/Kafka/MessageSetTest.php: -------------------------------------------------------------------------------- 1 | 21 | */ 22 | class Kafka_MessageSetTest extends PHPUnit_Framework_TestCase 23 | { 24 | private function getMessageSetBuffer(array $messages) { 25 | $message_set = ''; 26 | foreach ($messages as $message) { 27 | $encoded = Kafka_Encoder::encode_message($message, Kafka_Encoder::COMPRESSION_NONE); 28 | // encode messages as 29 | $message_set .= pack('N', strlen($encoded)) . $encoded; 30 | } 31 | return $message_set; 32 | } 33 | 34 | private function writeDummyMessageSet($stream, array $messages) { 35 | return fwrite($stream, $this->getMessageSetBuffer($messages)); 36 | } 37 | 38 | private function writeDummyCompressedMessageSet($stream, array $messages, $compression) { 39 | $encoded = Kafka_Encoder::encode_message($this->getMessageSetBuffer($messages), $compression); 40 | return fwrite($stream, pack('N', strlen($encoded)) . $encoded); 41 | } 42 | 43 | public function testIterator() { 44 | $stream = fopen('php://temp', 'w+b'); 45 | $messages = array('message #1', 'message #2', 'message #3'); 46 | $this->writeDummyMessageSet($stream, $messages); 47 | rewind($stream); 48 | $socket = Kafka_Socket::createFromStream($stream); 49 | $set = new Kafka_MessageSet($socket, 0, 0); 50 | $idx = 0; 51 | foreach ($set as $offset => $msg) { 52 | $this->assertEquals($messages[$idx++], $msg->payload()); 53 | } 54 | $this->assertEquals(count($messages), $idx); 55 | 56 | // test new offset 57 | $readBytes = $set->validBytes(); 58 | $this->assertEquals(60, $readBytes); 59 | $readBytes = $set->sizeInBytes(); 60 | $this->assertEquals(60, $readBytes); 61 | 62 | // no more data 63 | $set = new Kafka_MessageSet($socket, $readBytes, 0); 64 | $cnt = 0; 65 | foreach ($set as $offset => $msg) { 66 | $cnt++; 67 | } 68 | $this->assertEquals(0, $cnt); 69 | 70 | fclose($stream); 71 | } 72 | 73 | public function testIteratorInvalidLastMessage() { 74 | $stream1 = fopen('php://temp', 'w+b'); 75 | $messages = array('message #1', 'message #2', 'message #3'); 76 | $size = $this->writeDummyMessageSet($stream1, $messages); 77 | rewind($stream1); 78 | $stream = fopen('php://temp', 'w+b'); 79 | fwrite($stream, fread($stream1, $size - 2)); // copy partial stream buffer 80 | rewind($stream); 81 | 82 | $socket = Kafka_Socket::createFromStream($stream); 83 | $set = new Kafka_MessageSet($socket, 0, 0); 84 | $idx = 0; 85 | foreach ($set as $offset => $msg) { 86 | $this->assertEquals($messages[$idx++], $msg->payload()); 87 | } 88 | $this->assertEquals(count($messages) - 1, $idx); // the last message should NOT be returned 89 | fclose($stream); 90 | 91 | // test new offset 92 | $readBytes = $set->validBytes(); 93 | $this->assertEquals(40, $readBytes); 94 | } 95 | 96 | public function testOffset() { 97 | $stream = fopen('php://temp', 'w+b'); 98 | $messages = array('message #1', 'message #2', 'message #3'); 99 | $this->writeDummyMessageSet($stream, $messages); 100 | $offsetOfSecondMessage = 20; // manually calculated 101 | fseek($stream, $offsetOfSecondMessage, SEEK_SET); 102 | 103 | $socket = Kafka_Socket::createFromStream($stream); 104 | $set = new Kafka_MessageSet($socket, $offsetOfSecondMessage, 0); 105 | 106 | $cnt = 0; 107 | $idx = 1; 108 | foreach ($set as $offset => $msg) { 109 | $cnt++; 110 | $this->assertEquals($messages[$idx++], $msg->payload()); 111 | } 112 | $this->assertEquals(2, $cnt); 113 | fclose($stream); 114 | 115 | // test new offset 116 | $readBytes = $set->validBytes(); 117 | $this->assertEquals(40, $readBytes); 118 | } 119 | 120 | public function testOffset2() { 121 | $stream = fopen('php://temp', 'w+b'); 122 | $messages = array('message #1', 'message #2', 'message #3'); 123 | $this->writeDummyMessageSet($stream, $messages); 124 | $offsetOfThirdMessage = 40; // manually calculated 125 | 126 | fseek($stream, $offsetOfThirdMessage, SEEK_SET); 127 | $socket = Kafka_Socket::createFromStream($stream); 128 | $set = new Kafka_MessageSet($socket, $offsetOfThirdMessage, 0); 129 | 130 | $cnt = 0; 131 | foreach ($set as $offset => $msg) { 132 | $cnt++; 133 | $this->assertEquals($messages[2], $msg->payload()); 134 | } 135 | $this->assertEquals(1, $cnt); 136 | fclose($stream); 137 | 138 | // test new offset 139 | $readBytes = $set->validBytes(); 140 | $this->assertEquals(20, $readBytes); 141 | } 142 | 143 | public function testCompressedMessages() { 144 | $stream = fopen('php://temp', 'w+b'); 145 | $messages = array('message #1', 'message #2', 'message #3'); 146 | $this->writeDummyCompressedMessageSet($stream, $messages, Kafka_Encoder::COMPRESSION_GZIP); 147 | rewind($stream); 148 | $socket = Kafka_Socket::createFromStream($stream); 149 | $set = new Kafka_MessageSet($socket, 0, 0); 150 | $idx = 0; 151 | foreach ($set as $offset => $msg) { 152 | $this->assertEquals($messages[$idx++], $msg->payload()); 153 | } 154 | $this->assertEquals(count($messages), $idx); 155 | 156 | // test new offset 157 | $readBytes = $set->validBytes(); 158 | $this->assertEquals(69, $readBytes); 159 | 160 | // no more data 161 | $set = new Kafka_MessageSet($socket, $readBytes, 0); 162 | $cnt = 0; 163 | foreach ($set as $offset => $msg) { 164 | $cnt++; 165 | } 166 | $this->assertEquals(0, $cnt); 167 | 168 | fclose($stream); 169 | } 170 | 171 | public function testMixedMessages() { 172 | $stream = fopen('php://temp', 'w+b'); 173 | $messages = array('message #1', 'message #2', 'message #3'); 174 | $this->writeDummyCompressedMessageSet($stream, $messages, Kafka_Encoder::COMPRESSION_GZIP); 175 | $messages2 = array('message #4', 'message #5', 'message #6'); 176 | $this->writeDummyMessageSet($stream, $messages2, Kafka_Encoder::COMPRESSION_NONE); 177 | $this->writeDummyCompressedMessageSet($stream, $messages, Kafka_Encoder::COMPRESSION_GZIP); 178 | rewind($stream); 179 | 180 | $allMessages = $messages; 181 | foreach ($messages2 as $msg) { 182 | $allMessages[] = $msg; 183 | } 184 | foreach ($messages as $msg) { 185 | $allMessages[] = $msg; 186 | } 187 | 188 | $socket = Kafka_Socket::createFromStream($stream); 189 | $set = new Kafka_MessageSet($socket, 0, 0); 190 | $idx = 0; 191 | foreach ($set as $offset => $msg) { 192 | $this->assertEquals($allMessages[$idx++], $msg->payload()); 193 | } 194 | $this->assertEquals(count($allMessages), $idx); 195 | 196 | // test new offset 197 | $readBytes = $set->validBytes(); 198 | $this->assertEquals(198, $readBytes); 199 | 200 | // no more data 201 | $set = new Kafka_MessageSet($socket, $readBytes, 0); 202 | $cnt = 0; 203 | foreach ($set as $offset => $msg) { 204 | $cnt++; 205 | } 206 | $this->assertEquals(0, $cnt); 207 | 208 | fclose($stream); 209 | } 210 | } 211 | -------------------------------------------------------------------------------- /src/tests/Kafka/MessageTest.php: -------------------------------------------------------------------------------- 1 | 22 | */ 23 | class Kafka_MessageTest extends PHPUnit_Framework_TestCase 24 | { 25 | private $test; 26 | private $encoded; 27 | private $msg; 28 | public function setUp() { 29 | $this->test = 'a sample string'; 30 | $this->encoded = Kafka_Encoder::encode_message($this->test); 31 | $this->msg = new Kafka_Message($this->encoded); 32 | 33 | } 34 | 35 | public function testPayload() { 36 | $this->assertEquals($this->test, $this->msg->payload()); 37 | } 38 | 39 | public function testValid() { 40 | $this->assertTrue($this->msg->isValid()); 41 | } 42 | 43 | public function testEncode() { 44 | $this->assertEquals($this->encoded, $this->msg->encode()); 45 | } 46 | 47 | public function testChecksum() { 48 | $this->assertInternalType('integer', $this->msg->checksum()); 49 | } 50 | 51 | public function testSize() { 52 | $this->assertEquals(strlen($this->test), $this->msg->size()); 53 | } 54 | 55 | public function testToString() { 56 | $this->assertInternalType('string', $this->msg->__toString()); 57 | } 58 | 59 | public function testMagic() { 60 | $this->assertInternalType('integer', $this->msg->magic()); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/tests/Kafka/ProducerTest.php: -------------------------------------------------------------------------------- 1 | 25 | */ 26 | class Kafka_ProducerMock extends Kafka_Producer { 27 | public function connect() { 28 | if (null === $this->socket) { 29 | $this->socket = Kafka_Socket::createFromStream(fopen('php://temp', 'w+b')); 30 | } 31 | } 32 | 33 | public function getData() { 34 | $this->connect(); 35 | $this->socket->rewind(); 36 | return $this->socket->read(10000000); 37 | } 38 | 39 | public function getHost() { 40 | return $this->host; 41 | } 42 | 43 | public function getPort() { 44 | return $this->port; 45 | } 46 | 47 | public function getCompression() { 48 | return $this->compression; 49 | } 50 | } 51 | 52 | /** 53 | * Description of ProducerTest 54 | * 55 | * @author Lorenzo Alberton 56 | */ 57 | class Kafka_ProducerTest extends PHPUnit_Framework_TestCase 58 | { 59 | /** 60 | * @var Kafka_Producer 61 | */ 62 | private $producer; 63 | 64 | public function setUp() { 65 | $this->producer = new Kafka_ProducerMock('localhost', 1234, Kafka_Encoder::COMPRESSION_NONE); 66 | } 67 | 68 | public function tearDown() { 69 | $this->producer->close(); 70 | unset($this->producer); 71 | } 72 | 73 | public function testProducer() { 74 | $messages = array( 75 | 'test 1', 76 | 'test 2 abc', 77 | ); 78 | $topic = 'a topic'; 79 | $partition = 3; 80 | $this->producer->send($messages, $topic, $partition); 81 | $sent = $this->producer->getData(); 82 | $this->assertContains($topic, $sent); 83 | $this->assertContains($partition, $sent); 84 | foreach ($messages as $msg) { 85 | $this->assertContains($msg, $sent); 86 | } 87 | } 88 | 89 | /** 90 | * @expectedException Kafka_Exception_Socket 91 | */ 92 | public function testConnectFailure() { 93 | $producer = new Kafka_Producer('invalid-host-name', 1234567890, Kafka_Encoder::COMPRESSION_NONE); 94 | $producer->connect(); 95 | $this->fail('The above call should throw an exception'); 96 | } 97 | 98 | public function testSerialize() { 99 | $producer = new Kafka_ProducerMock('host', 1234, Kafka_Encoder::COMPRESSION_SNAPPY); 100 | $serialized = serialize($producer); 101 | $unserialized = unserialize($serialized); 102 | $this->assertEquals('host', $unserialized->getHost()); 103 | $this->assertEquals(1234, $unserialized->getPort()); 104 | $this->assertEquals(Kafka_Encoder::COMPRESSION_SNAPPY, $unserialized->getCompression()); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /src/tests/Kafka/RequestTest.php: -------------------------------------------------------------------------------- 1 | 23 | */ 24 | class Kafka_RequestTest extends PHPUnit_Framework_TestCase 25 | { 26 | public function testEncodeDecode64bitShortUnsigned() { 27 | $short = 3; 28 | $encoded = Kafka_Request::packLong64bigendian($short); 29 | $this->assertEquals($short, Kafka_Request::unpackLong64bigendian($encoded)); 30 | } 31 | 32 | public function testEncodeDecode64bitShortSigned() { 33 | $short = -3; 34 | $encoded = Kafka_Request::packLong64bigendian($short); 35 | $this->assertEquals($short, Kafka_Request::unpackLong64bigendian($encoded)); 36 | } 37 | 38 | public function testEncodeDecode64bitIntUnsigned() { 39 | $int = 32767; 40 | $encoded = Kafka_Request::packLong64bigendian($int); 41 | $this->assertEquals($int, Kafka_Request::unpackLong64bigendian($encoded)); 42 | 43 | $int = 32768; 44 | $encoded = Kafka_Request::packLong64bigendian($int); 45 | $this->assertEquals($int, Kafka_Request::unpackLong64bigendian($encoded)); 46 | } 47 | 48 | public function testEncodeDecode64bitIntSigned() { 49 | $int = -32768; 50 | $encoded = Kafka_Request::packLong64bigendian($int); 51 | $this->assertEquals($int, Kafka_Request::unpackLong64bigendian($encoded)); 52 | 53 | $int = -32769; 54 | $encoded = Kafka_Request::packLong64bigendian($int); 55 | $this->assertEquals($int, Kafka_Request::unpackLong64bigendian($encoded)); 56 | } 57 | 58 | public function testEncodeDecode64bitLongUnsigned() { 59 | $long = 2147483647; 60 | $encoded = Kafka_Request::packLong64bigendian($long); 61 | $this->assertEquals($long, Kafka_Request::unpackLong64bigendian($encoded)); 62 | 63 | $long = 4294967295; 64 | $encoded = Kafka_Request::packLong64bigendian($long); 65 | $this->assertEquals($long, Kafka_Request::unpackLong64bigendian($encoded)); 66 | } 67 | 68 | public function testEncodeDecode64bitLongUnsignedLargerThan2_32() { 69 | $long = pow(2, 35); 70 | $encoded = Kafka_Request::packLong64bigendian($long); 71 | $this->assertEquals($long, Kafka_Request::unpackLong64bigendian($encoded)); 72 | 73 | $long = pow(2, 37); 74 | $encoded = Kafka_Request::packLong64bigendian($long); 75 | $this->assertEquals($long, Kafka_Request::unpackLong64bigendian($encoded)); 76 | } 77 | 78 | public function testEncodeDecode64bitLongSigned() { 79 | $long = -2147483648; 80 | $encoded = Kafka_Request::packLong64bigendian($long); 81 | $this->assertEquals($long, Kafka_Request::unpackLong64bigendian($encoded)); 82 | 83 | $long = -2147483649; 84 | $encoded = Kafka_Request::packLong64bigendian($long); 85 | $this->assertEquals($long, Kafka_Request::unpackLong64bigendian($encoded)); 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /src/tests/Kafka/ResponseTest.php: -------------------------------------------------------------------------------- 1 | 22 | */ 23 | class Kafka_ResponseTest extends PHPUnit_Framework_TestCase 24 | { 25 | /** 26 | * @expectedException Kafka_Exception_OffsetOutOfRange 27 | */ 28 | public function testErrorCodeValidationOffsetOutOfRange() { 29 | Kafka_Response::validateErrorCode(1); 30 | $this->fail('the line above should throw an exception'); 31 | } 32 | 33 | /** 34 | * @expectedException Kafka_Exception_InvalidMessage 35 | */ 36 | public function testErrorCodeValidationInvalidMessage() { 37 | Kafka_Response::validateErrorCode(2); 38 | $this->fail('the line above should throw an exception'); 39 | } 40 | 41 | /** 42 | * @expectedException Kafka_Exception_WrongPartition 43 | */ 44 | public function testErrorCodeValidationWrongPartition() { 45 | Kafka_Response::validateErrorCode(3); 46 | $this->fail('the line above should throw an exception'); 47 | } 48 | 49 | /** 50 | * @expectedException Kafka_Exception_InvalidFetchSize 51 | */ 52 | public function testErrorCodeValidationInvalidFetchSize() { 53 | Kafka_Response::validateErrorCode(4); 54 | $this->fail('the line above should throw an exception'); 55 | } 56 | 57 | /** 58 | * @expectedException Kafka_Exception 59 | */ 60 | public function testErrorCodeValidationUnknown() { 61 | Kafka_Response::validateErrorCode(20); 62 | $this->fail('the line above should throw an exception'); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/tests/Kafka/SimpleConsumerTest.php: -------------------------------------------------------------------------------- 1 | 25 | */ 26 | class Kafka_ConsumerMock extends Kafka_SimpleConsumer { 27 | public function connect() { 28 | if (null === $this->socket) { 29 | $this->socket = Kafka_Socket::createFromStream(fopen('php://temp', 'w+b')); 30 | } 31 | } 32 | 33 | public function writeInt4($n) { 34 | $this->socket->write(pack('N', $n)); 35 | } 36 | 37 | public function writeInt2($n) { 38 | $this->socket->write(pack('n', $n)); 39 | } 40 | 41 | public function rewind() { 42 | $this->socket->rewind(); 43 | } 44 | 45 | public function getResponseSize() { 46 | return parent::getResponseSize(); 47 | } 48 | 49 | public function getResponseCode() { 50 | return parent::getResponseCode(); 51 | } 52 | } 53 | 54 | /** 55 | * Description of ProducerTest 56 | * 57 | * @author Lorenzo Alberton 58 | */ 59 | class Kafka_SimpleConsumerTest extends PHPUnit_Framework_TestCase 60 | { 61 | /** 62 | * @var Kafka_Producer 63 | */ 64 | private $consumer; 65 | 66 | public function setUp() { 67 | $this->consumer = new Kafka_ConsumerMock('localhost', 1234, 10, 100000); 68 | } 69 | 70 | public function tearDown() { 71 | $this->consumer->close(); 72 | unset($this->consumer); 73 | } 74 | 75 | /** 76 | * @expectedException Kafka_Exception_OutOfRange 77 | */ 78 | public function testInvalidMessageSize() { 79 | $this->consumer->connect(); 80 | $this->consumer->writeInt4(0); 81 | $this->consumer->rewind(); 82 | $this->consumer->getResponseSize(); 83 | $this->fail('The above call should throw an exception'); 84 | } 85 | 86 | public function testMessageSize() { 87 | $this->consumer->connect(); 88 | $this->consumer->writeInt4(10); 89 | $this->consumer->rewind(); 90 | $this->assertEquals(10, $this->consumer->getResponseSize()); 91 | } 92 | 93 | public function testMessageCode() { 94 | $this->consumer->connect(); 95 | $this->consumer->writeInt2(1); 96 | $this->consumer->rewind(); 97 | $this->assertEquals(1, $this->consumer->getResponseCode()); 98 | } 99 | 100 | /** 101 | * @expectedException Kafka_Exception_Socket_EOF 102 | */ 103 | public function testMessageSizeFailure() { 104 | $this->consumer->close(); 105 | $this->consumer->getResponseSize(); 106 | $this->fail('The above call should throw an exception'); 107 | } 108 | 109 | /** 110 | * @expectedException Kafka_Exception_Socket 111 | */ 112 | public function testConnectFailure() { 113 | $consumer = new Kafka_SimpleConsumer('invalid-host-name', 1234567890, 10, 1000000); 114 | $consumer->connect(); 115 | $this->fail('The above call should throw an exception'); 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /src/tests/Kafka/SocketTest.php: -------------------------------------------------------------------------------- 1 | 25 | */ 26 | class Kafka_SocketTest extends PHPUnit_Framework_TestCase 27 | { 28 | /** 29 | * @expectedException Kafka_Exception_Socket_Connection 30 | */ 31 | public function testConnectNoHost() { 32 | $socket = new Kafka_Socket(null, 80); 33 | $socket->connect(); 34 | $this->fail('The above connect() call should fail on a null host'); 35 | } 36 | 37 | /** 38 | * @expectedException Kafka_Exception_Socket_Connection 39 | */ 40 | public function testConnectInvalidPort() { 41 | $socket = new Kafka_Socket('localhost', 80); 42 | $socket->connect(); 43 | $this->fail('The above connect() call should fail on an invalid port'); 44 | } 45 | 46 | /** 47 | * @expectedException Kafka_Exception_Socket_Connection 48 | */ 49 | public function testConnectInvalidHost() { 50 | $socket = new Kafka_Socket('invalid-host', 80); 51 | $socket->connect(); 52 | $this->fail('The above connect() call should fail on an invalid host'); 53 | } 54 | 55 | /** 56 | * @expectedException Kafka_Exception_Socket 57 | */ 58 | public function testWriteNoStream() { 59 | $socket = Kafka_Socket::createFromStream(null); 60 | $socket->write('test'); 61 | //$socket->rewind(); 62 | //var_dump($socket->read(4)); 63 | $this->fail('The above write() call should fail on a null socket'); 64 | } 65 | 66 | /** 67 | * @expectedException Kafka_Exception_Socket 68 | */ 69 | public function testWriteReadOnlySocket() { 70 | $roStream = fopen('php://temp', 'r'); 71 | $socket = Kafka_Socket::createFromStream($roStream); 72 | $socket->write('test'); 73 | //$socket->rewind(); 74 | //var_dump($socket->read(4)); 75 | $this->fail('The above write() call should fail on a read-only socket'); 76 | } 77 | 78 | /** 79 | * @expectedException Kafka_Exception_Socket_Timeout 80 | */ 81 | public function testWriteTimeout() { 82 | $this->markTestSkipped('find a better way of testing socket timeouts'); 83 | $stream = fopen('php://temp', 'w+b'); 84 | $socket = new Kafka_Socket('localhost', 0, 0, 0, -1, -1); 85 | $socket->setStream($stream); 86 | $socket->write('short timeout'); 87 | //$socket->rewind(); 88 | //var_dump($socket->read(4)); 89 | $this->fail('The above write() call should fail on a socket with timeout = -1'); 90 | } 91 | 92 | public function testWrite() { 93 | $socket = Kafka_Socket::createFromStream(fopen('php://temp', 'w+b')); 94 | $written = $socket->write('test'); 95 | $this->assertEquals(4, $written); 96 | } 97 | 98 | public function testWriteAndRead() { 99 | $socket = Kafka_Socket::createFromStream(fopen('php://temp', 'w+b')); 100 | $written = $socket->write('test'); 101 | $socket->rewind(); 102 | $this->assertEquals('test', $socket->read(4)); 103 | } 104 | 105 | public function testRead() { 106 | $stream = fopen('php://temp', 'w+b'); 107 | fwrite($stream, 'test'); 108 | fseek($stream, 0, SEEK_SET); 109 | $socket = Kafka_Socket::createFromStream($stream); 110 | $this->assertEquals('test', $socket->read(4)); 111 | } 112 | 113 | public function testReadFewerBytes() { 114 | $stream = fopen('php://temp', 'w+b'); 115 | fwrite($stream, 'tes'); 116 | fseek($stream, 0, SEEK_SET); 117 | $socket = Kafka_Socket::createFromStream($stream); 118 | $this->assertEquals('tes', $socket->read(4)); 119 | } 120 | 121 | /** 122 | * 123 | * @expectedException Kafka_Exception_Socket_EOF 124 | */ 125 | public function testReadFewerBytesVerifyLength() { 126 | $stream = fopen('php://temp', 'w+b'); 127 | fwrite($stream, 'tes'); 128 | fseek($stream, 0, SEEK_SET); 129 | $socket = Kafka_Socket::createFromStream($stream); 130 | $this->assertEquals('tes', $socket->read(4, true)); 131 | $this->fail('The above call shoud throw an exception because the socket had fewer bytes than requested'); 132 | } 133 | 134 | public function testReadMultiple() { 135 | $stream = fopen('php://temp', 'w+b'); 136 | fwrite($stream, 'test1test2'); 137 | fseek($stream, 0, SEEK_SET); 138 | $socket = Kafka_Socket::createFromStream($stream); 139 | $this->assertEquals('test1', $socket->read(5)); 140 | $this->assertEquals('test2', $socket->read(5)); 141 | } 142 | 143 | /** 144 | * @expectedException Kafka_Exception_Socket 145 | */ 146 | public function testReadAfterClose() { 147 | $stream = fopen('php://temp', 'w+b'); 148 | fwrite($stream, 'test'); 149 | fseek($stream, 0, SEEK_SET); 150 | $socket = Kafka_Socket::createFromStream($stream); 151 | $socket->close(); 152 | $socket->read(4); 153 | $this->fail('The above read() call should fail on a closed socket'); 154 | } 155 | 156 | /** 157 | * @expectedException Kafka_Exception_Socket 158 | */ 159 | public function testWriteAfterClose() { 160 | $stream = fopen('php://temp', 'w+b'); 161 | $socket = Kafka_Socket::createFromStream($stream); 162 | $socket->close(); 163 | $socket->write('test'); 164 | $this->fail('The above write() call should fail on a closed socket'); 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/tests/bootstrap.php: -------------------------------------------------------------------------------- 1 | file association */ 34 | if (($file !== false) && ($file !== null)) { 35 | include $file; 36 | return; 37 | } 38 | 39 | throw new RuntimeException($className. ' not found'); 40 | } 41 | 42 | // register the autoloader 43 | spl_autoload_register('test_autoload'); 44 | 45 | set_include_path( 46 | implode(PATH_SEPARATOR, array( 47 | realpath(__DIR__ . '/../lib'), 48 | get_include_path(), 49 | )) 50 | ); 51 | 52 | date_default_timezone_set('Europe/London'); 53 | -------------------------------------------------------------------------------- /src/tests/phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | ./Kafka 10 | 11 | 12 | 13 | 14 | ./ 15 | 16 | 17 | 18 | --------------------------------------------------------------------------------