├── Producer.class.php ├── Configuration.md ├── README.md ├── Consumer.class.php └── Rdkafka.class.php /Producer.class.php: -------------------------------------------------------------------------------- 1 | rk = new Rdkafka($config); 19 | $this->rkConf = $this->rk->getConf(); 20 | $this->config = $this->rk->getConfig(); 21 | $this->brokerConfig = $this->rk->getBrokerConfig(); 22 | } 23 | 24 | public function setBrokerServer($ip = NULL) 25 | { 26 | $this->producer = new \RdKafka\Producer($this->rkConf); 27 | $this->producer->setLogLevel(LOG_DEBUG); 28 | $this->producer->addBrokers($ip ? : $this->config['ip']); 29 | 30 | return $this; 31 | } 32 | 33 | public function setProducerTopic($topicName) 34 | { 35 | $this->producerTopicConf = new \RdKafka\TopicConf(); 36 | // -1必须等所有brokers确认 1当前服务器确认 0不确认，这里如果是0回调里的offset无返回，如果是1和-1会返回offset 37 | $this->producerTopicConf->set('request.required.acks', $this->brokerConfig['request.required.acks']); 38 | $this->producerTopic = $this->producer->newTopic($topicName, $this->producerTopicConf); 39 | 40 | return $this; 41 | } 42 | 43 | public function producer($msg, $option) 44 | { 45 | $this->producerTopic->produce(RD_KAFKA_PARTITION_UA, 0, $msg, $option); 46 | 47 | $len = $this->producer->getOutQLen(); 48 | while ($len > 0) { 49 | $len = $this->producer->getOutQLen(); 50 | $this->producer->poll(50); 51 | } 52 | 53 | return true; 54 | } 55 | } -------------------------------------------------------------------------------- /Configuration.md: -------------------------------------------------------------------------------- 1 | Kafka-php Configuration 2 | ================== 3 | 4 | | Property | C/P | Range | Default | Desc | 5 | | -- | -- | -- | -- | -- | 6 | | brokerVersion | C/P | 0.8.0 | 0.10.1.0 | User supplied broker version | 7 | | clientId | C/P | | kafka-php | This is a user supplied identifier for the client application | 8 | | messageMaxBytes | C/P | 1000 .. 1000000000 | 1000000 | Maximum transmit message size. | 9 | | metadataBrokerList | C/P | | | Kafka Broker server list | 10 | | metadataMaxAgeMs | C/P | 1 .. 86400000 | -1 | Metadata cache max age. Defaults to metadata.refresh.interval.ms * 3 | 11 | | metadataRefreshIntervalMs | C/P | 10 .. 3600000 | 300000 | Topic metadata refresh interval in milliseconds. The metadata is automatically refreshed on error and connect. Use -1 to disable the intervalled refresh. | 12 | | metadataRequestTimeoutMs | C/P | 10 .. 900000 | 60000 | Non-topic request timeout in milliseconds. This is for metadata requests, etc. | 13 | | sslEnable | C/P | true/false | false | Whether enable ssl connect or not | 14 | | sslCafile | C/P | | | Location of Certificate Authority file on local filesystem which should be used with the verify_peer context option to authenticate the identity of the remote peer.| 15 | | sslLocalCert | C/P | File path | | Path to local certificate file on filesystem. | 16 | | sslLocalPk | C/P | File path | | Path to local private key file on filesystem in case of separate files for certificate (local_cert) and private key. | 17 | | sslPassphrase | C/P | | | Passphrase with which your local_cert file was encoded. | 18 | | sslPeerName | C/P | | | Peer name to be used. If this value is not set, then the name is guessed based on the hostname used when opening the stream. | 19 | | sslVerifyPeer | C/P | true/false | false | Require verification of SSL certificate used. | 20 | | offsetReset | C | latest,earliest | latest | Action to take when there is no initial offset in offset store or the desired offset is out of range | 21 | | groupId | C | | | Client group id string. All clients sharing the same group.id belong to the same group. | 22 | | maxBytes | C | | 65536 | Maximum bytes to fetch. | 23 | | maxWaitTime | C | | 100 | Maximum time in ms to wait for the response | 24 | | sessionTimeout | C | 1 .. 3600000 | 30000 | Client group session and failure detection timeout. | 25 | | rebalanceTimeout | C | 1 .. 3600000 | 30000 | rebalance join wait timeout | 26 | | topics | C | | | Want consumer topics | 27 | | isAsyn | P | true, false | false | Whether to use asynchronous production messages | 28 | | produceInterval | P | 1 .. 900000 | 100 | The time interval at which requests for production messages are executed when the message is produced asynchronously | 29 | | requestTimeout | P | 1 .. 900000 | 6000 | The total timeout of the production message, which must be greater than the timeout config parameter | 30 | | requiredAck | P | -1 .. 1000 | 1 | This field indicates how many acknowledgements the leader broker must receive from ISR brokers before responding to the request: 0=Broker does not send any response/ack to client, 1=Only the leader broker will need to ack the message, -1 or all=broker will block until message is committed by all in sync replicas (ISRs) or broker\'s in.sync.replicas setting before sending response. | 31 | | timeout | P | 1 .. 900000 | 5000 | Producer request timeout | 32 | 33 | #### Note 34 | 35 | All of the above parameters are set by setXxx, for example, to set the `clientId` parameter 36 | 37 | ```php 38 | setClientId('test'); 42 | ``` 43 | 44 | Whether it is a consumer module or a production module, if the parameter settings do not match the rules will throw `\Kafka\Exception\Config` exception -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## php-rdkafka-class 2 | rdkafka的下php-rdkafka的php类库 3 | 4 | [![kafka version support](https://img.shields.io/badge/kafka-0.8%200.9%201.0%201.1%20or%201.1%2B-brightgreen.svg)](#) [![php version support](https://img.shields.io/badge/php-5.3%2B-green.svg)](#) [![librdkafka version support](https://img.shields.io/badge/librdkafka-3.0.5%2B-yellowgreen.svg)](#) [![php-librdkafka](https://img.shields.io/badge/php--librdkafka-3.0.5%2B-orange.svg)](#) 5 | 6 | ## 目录 7 | 8 | 1. [安装](#安装) 9 | 2. [使用](#使用) 10 | * [消费](#消费) 11 | * [生产](#生产) 12 | 3. [更多配置](#更多配置) 13 | 14 | ## 安装 15 | > 具体查看librdkafk和php-rdkafka 16 | 17 | ## 使用 18 | ### 消费 19 | ``` 20 | # setTopic('qkl01', 0, $offset) 不设置，从最后一次服务器记录一次消费开始消费 21 | $offset = 86; //开始消费点 22 | $consumer = new \Vendors\Queue\Msg\Kafka\Consumer(['ip'=>'192.168.216.122']); 23 | $consumer->setConsumerGroup('test-110-sxx1') 24 | ->setBrokerServer('192.168.216.122') 25 | ->setConsumerTopic() 26 | ->setTopic('qkl01', 0, $offset) 27 | ->subscribe(['qkl01']) 28 | ->consumer(function($msg){ 29 | var_dump($msg); 30 | }); 31 | ``` 32 | 33 | 34 | ### 生产 35 | ``` 36 | $config = [ 37 | 'ip'=>'192.168.216.122', 38 | 'dr_msg_cb' => function($kafka, $message) { 39 | var_dump((array)$message); 40 | //todo 41 | //do biz something, don't exit() or die() 42 | } 43 | ]; 44 | $producer = new \Vendors\Queue\Msg\Kafka\Producer($config); 45 | $rst = $producer->setBrokerServer() 46 | ->setProducerTopic('qkl01') 47 | ->producer('qkl037', 90); 48 | 49 | var_dump($rst); 50 | ``` 51 | 52 | ## 更多配置 53 | ``` 54 | $defaultConfig = [ 55 | 'ip'=>'127.0.0.1', #默认服务器地址 56 | 'log_path'=> sys_get_temp_dir(), #日志默认地址 57 | 'dr_msg_cb' => [$this, 'defaultDrMsg'], #生产的dr回调 58 | 'error_cb' => [$this, 'defaultErrorCb'], #错误回调 59 | 'rebalance_cb' => [$this, 'defaultRebalance'] #负载回调，你可以用匿名方法自定义 60 | ]; 61 | 62 | # broker相关配置，你可以参考Configuration.md 63 | $brokerConfig = [ 64 | 'request.required.acks'=> -1, 65 | 'auto.commit.enable'=> 1, 66 | 'auto.commit.interval.ms'=> 100, 67 | 'offset.store.method'=> 'broker', 68 | 'offset.store.path'=> sys_get_temp_dir(), 69 | 'auto.offset.reset'=> 'smallest', 70 | ]; 71 | ``` 72 | 73 | ### defaultDrMsg 74 | ``` 75 | function defaultDrMsg($kafka, $message) { 76 | file_put_contents($this->config['log_path'] . "/dr_cb.log", var_export($message, true).PHP_EOL, FILE_APPEND); 77 | } 78 | ``` 79 | 80 | ### defaultErrorCb 81 | ``` 82 | function defaultErrorCb($kafka, $err, $reason) { 83 | file_put_contents($this->config['log_path'] . "/err_cb.log", sprintf("Kafka error: %s (reason: %s)", rd_kafka_err2str($err), $reason).PHP_EOL, FILE_APPEND); 84 | } 85 | ``` 86 | 87 | 88 | ### defaultRebalance 89 | ``` 90 | function defaultRebalance(\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) 91 | { 92 | switch ($err) { 93 | case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: 94 | echo "Assign: "; 95 | if (is_null($this->getCurrentTopic())) { 96 | $kafka->assign(); 97 | } else { 98 | $kafka->assign([ 99 | new \RdKafka\TopicPartition( $this->getCurrentTopic(), $this->getPartition($this->getCurrentTopic()), $this->getOffset($this->getCurrentTopic()) ) 100 | ]); 101 | } 102 | break; 103 | 104 | case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: 105 | echo "Revoke: "; 106 | var_dump($partitions); 107 | $kafka->assign(NULL); 108 | break; 109 | 110 | default: 111 | throw new \Exception($err); 112 | } 113 | } 114 | ``` 115 | -------------------------------------------------------------------------------- /Consumer.class.php: -------------------------------------------------------------------------------- 1 | rk = new Rdkafka($config); 18 | $this->rkConf = $this->rk->getConf(); 19 | $this->config = $this->rk->getConfig(); 20 | $this->brokerConfig = $this->rk->getBrokerConfig(); 21 | } 22 | 23 | /** 24 | * 设置消费组 25 | * @param $groupName 26 | */ 27 | public function setConsumerGroup($groupName) 28 | { 29 | $this->rkConf->set('group.id', $groupName); 30 | return $this; 31 | } 32 | 33 | /** 34 | * 设置服务broker 35 | * $broker: 127.0.0.1|127.0.0.1:9092|127.0.0.1:9092,127.0.0.1:9093 36 | * @param $groupName 37 | */ 38 | public function setBrokerServer($broker) 39 | { 40 | $this->rkConf->set('metadata.broker.list', $broker); 41 | return $this; 42 | } 43 | 44 | /** 45 | * 设置服务broker 46 | * $broker: 127.0.0.1|127.0.0.1:9092|127.0.0.1:9092,127.0.0.1:9093 47 | * @param $groupName 48 | */ 49 | public function setTopic($topicName, $partition = 0, $offset = 0) 50 | { 51 | $this->rk->setTopic($topicName, $partition, $offset); 52 | return $this; 53 | } 54 | 55 | public function setConsumerTopic() 56 | { 57 | $this->topicConf = new \RdKafka\TopicConf(); 58 | 59 | $this->topicConf->set('request.required.acks', $this->brokerConfig['request.required.acks']); 60 | //在interval.ms的时间内自动提交确认、建议不要启动 61 | $this->topicConf->set('auto.commit.enable', $this->brokerConfig['auto.commit.enable']); 62 | if ($this->brokerConfig['auto.commit.enable']) { 63 | $this->topicConf->set('auto.commit.interval.ms', $this->brokerConfig['auto.commit.interval.ms']); 64 | } 65 | 66 | // 设置offset的存储为file 67 | // $this->topicConf->set('offset.store.method', 'file'); 68 | // $this->topicConf->set('offset.store.path', __DIR__); 69 | // 设置offset的存储为broker 70 | // $this->topicConf->set('offset.store.method', 'broker'); 71 | $this->topicConf->set('offset.store.method', $this->brokerConfig['offset.store.method']); 72 | if ($this->brokerConfig['offset.store.method'] == 'file') { 73 | $this->topicConf->set('offset.store.path', $this->brokerConfig['offset.store.path']); 74 | } 75 | 76 | // Set where to start consuming messages when there is no initial offset in 77 | // offset store or the desired offset is out of range. 78 | // 'smallest': start from the beginning 79 | $this->topicConf->set('auto.offset.reset', 'smallest'); 80 | $this->topicConf->set('auto.offset.reset', $this->brokerConfig['auto.offset.reset']); 81 | 82 | //设置默认话题配置 83 | $this->rkConf->setDefaultTopicConf($this->topicConf); 84 | 85 | return $this; 86 | } 87 | 88 | public function getConsumerTopic() 89 | { 90 | return $this->topicConf; 91 | } 92 | 93 | public function subscribe($topicNames) 94 | { 95 | $this->consumer = new \RdKafka\KafkaConsumer($this->rkConf); 96 | $this->consumer->subscribe($topicNames); 97 | return $this; 98 | } 99 | 100 | public function consumer(\Closure $handle) 101 | { 102 | while (true) { 103 | $message = $this->consumer->consume(120*1000); 104 | switch ($message->err) { 105 | case RD_KAFKA_RESP_ERR_NO_ERROR: 106 | $handle($message); 107 | break; 108 | case RD_KAFKA_RESP_ERR__PARTITION_EOF: 109 | echo "No more messages; will wait for more\n"; 110 | break; 111 | case RD_KAFKA_RESP_ERR__TIMED_OUT: 112 | echo "Timed out\n"; 113 | break; 114 | default: 115 | throw new \Exception($message->errstr(), $message->err); 116 | break; 117 | } 118 | } 119 | } 120 | 121 | public function consumer2(\Closure $callback) 122 | { 123 | //参数1表示消费分区，这里是分区0 124 | //参数2表示同步阻塞多久 125 | $message = $this->consumerTopic->consume(0, 12 * 1000); 126 | var_dump($message); 127 | switch ($message->err) { 128 | case RD_KAFKA_RESP_ERR_NO_ERROR: 129 | //todo 消费 130 | $callback($message); 131 | break; 132 | case RD_KAFKA_RESP_ERR__PARTITION_EOF: 133 | echo "No more messages; will wait for more\n"; 134 | break; 135 | case RD_KAFKA_RESP_ERR__TIMED_OUT: 136 | echo "Timed out\n"; 137 | break; 138 | default: 139 | echo $message->err . ":" . $message->errstr; 140 | // throw new \Exception($message->errstr(), $message->err); 141 | break; 142 | } 143 | // while (true) { 144 | // //参数1表示消费分区，这里是分区0 145 | // //参数2表示同步阻塞多久 146 | // $message = $this->consumerTopic->consume(0, 12 * 1000); 147 | // switch ($message->err) { 148 | // case RD_KAFKA_RESP_ERR_NO_ERROR: 149 | // //todo 消费 150 | // $callback($message); 151 | // break; 152 | // case RD_KAFKA_RESP_ERR__PARTITION_EOF: 153 | // echo "No more messages; will wait for more\n"; 154 | // break; 155 | // case RD_KAFKA_RESP_ERR__TIMED_OUT: 156 | // echo "Timed out\n"; 157 | // break; 158 | // default: 159 | // throw new \Exception($message->errstr(), $message->err); 160 | // break; 161 | // } 162 | // } 163 | } 164 | } -------------------------------------------------------------------------------- /Rdkafka.class.php: -------------------------------------------------------------------------------- 1 | '127.0.0.1', 21 | 'log_path'=> sys_get_temp_dir(), 22 | 'dr_msg_cb' => [$this, 'defaultDrMsg'], 23 | 'error_cb' => [$this, 'defaultErrorCb'], 24 | 'rebalance_cb' => [$this, 'defaultRebalance'] 25 | ]; 26 | 27 | $brokerConfig = [ 28 | 'request.required.acks'=> -1, 29 | 'auto.commit.enable'=> 1, 30 | 'auto.commit.interval.ms'=> 100, 31 | 'offset.store.method'=> 'broker', 32 | 'offset.store.path'=> sys_get_temp_dir(), 33 | 'auto.offset.reset'=> 'smallest', 34 | ]; 35 | 36 | $this->config = array_merge($defaultConfig, $config); 37 | $this->brokerConfig = array_merge($brokerConfig, isset($config['broker']) ? $config['broker'] : []); 38 | 39 | $this->rkConf = new \RdKafka\Conf(); 40 | 41 | $this->setDrMsgCb($this->config['dr_msg_cb']); 42 | $this->setErrorCb($this->config['error_cb']); 43 | $this->setRebalanceCb($this->config['rebalance_cb']); 44 | } 45 | 46 | public function getConf() 47 | { 48 | return $this->rkConf; 49 | } 50 | 51 | public function getConfig() 52 | { 53 | return $this->config; 54 | } 55 | 56 | public function getBrokerConfig() 57 | { 58 | return $this->brokerConfig; 59 | } 60 | 61 | /** 62 | * 设置话题、partition、offset 63 | * @param $topicName 64 | * @param int $offset 65 | */ 66 | public function setTopic($topicName, $partition = 0, $offset = 0) 67 | { 68 | $this->topics[$topicName] = $topicName; 69 | $this->partitions[$topicName] = $partition; 70 | $this->offsets[$topicName] = $offset; 71 | 72 | $this->topic = $topicName; 73 | } 74 | 75 | /** 76 | * 获取话题 77 | * @return mixed 78 | */ 79 | public function getCurrentTopic() 80 | { 81 | return $this->topic; 82 | } 83 | 84 | /** 85 | * 获取话题的offset 86 | * @param $topicName 87 | * @return mixed 88 | */ 89 | public function getPartition($topicName) 90 | { 91 | return $this->partitions[$topicName]; 92 | } 93 | 94 | /** 95 | * 获取话题的offset 96 | * @param $topicName 97 | * @return mixed 98 | */ 99 | public function getOffset($topicName) 100 | { 101 | return $this->offsets[$topicName]; 102 | } 103 | 104 | public function setDrMsgCb($callback) 105 | { 106 | // $this->rkConf->setDrMsgCb(function ($kafka, $message) { 107 | // file_put_contents("./dr_cb.log", var_export($message, true).PHP_EOL, FILE_APPEND); 108 | // }); 109 | 110 | if (is_null($callback)) { 111 | return ; 112 | } 113 | 114 | $this->rkConf->setDrMsgCb(function ($kafka, $message) use ($callback) { 115 | call_user_func_array($callback, [$kafka, $message]); 116 | }); 117 | } 118 | 119 | public function setErrorCb($callback) 120 | { 121 | // $this->rkConf->setErrorCb(function ($kafka, $err, $reason) { 122 | // file_put_contents("./err_cb.log", sprintf("Kafka error: %s (reason: %s)", rd_kafka_err2str($err), $reason).PHP_EOL, FILE_APPEND); 123 | // }); 124 | 125 | if (is_null($callback)) { 126 | return ; 127 | } 128 | 129 | $this->rkConf->setErrorCb(function ($kafka, $err, $reason) use ($callback) { 130 | call_user_func_array($callback, [$kafka, $err, $reason]); 131 | }); 132 | } 133 | 134 | public function setRebalanceCb($callback) 135 | { 136 | // $this->rkConf->setErrorCb(function (\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) use ($topic){ 137 | // global $offset; 138 | // switch ($err) { 139 | // case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: 140 | // echo "Assign: "; 141 | //// var_dump($partitions); 142 | // if (is_null($topic)) { 143 | // $kafka->assign(); 144 | // } else { 145 | // $kafka->assign([new \RdKafka\TopicPartition("qkl01", 0, $offset)]); 146 | // } 147 | // break; 148 | // 149 | // case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: 150 | // echo "Revoke: "; 151 | // var_dump($partitions); 152 | // $kafka->assign(NULL); 153 | // break; 154 | // 155 | // default: 156 | // throw new \Exception($err); 157 | // } 158 | // }); 159 | 160 | 161 | if (is_null($callback)) { 162 | return ; 163 | } 164 | 165 | $this->rkConf->setRebalanceCb(function ($kafka, $err, $partitions) use ($callback) { 166 | call_user_func_array($callback, [$kafka, $err, $partitions]); 167 | }); 168 | } 169 | 170 | private function defaultDrMsg($kafka, $message) { 171 | file_put_contents($this->config['log_path'] . "/dr_cb.log", var_export($message, true).PHP_EOL, FILE_APPEND); 172 | } 173 | 174 | private function defaultErrorCb($kafka, $err, $reason) { 175 | file_put_contents($this->config['log_path'] . "/err_cb.log", sprintf("Kafka error: %s (reason: %s)", rd_kafka_err2str($err), $reason).PHP_EOL, FILE_APPEND); 176 | } 177 | 178 | private function defaultRebalance(\RdKafka\KafkaConsumer $kafka, $err, array $partitions = null) 179 | { 180 | switch ($err) { 181 | case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: 182 | echo "Assign: "; 183 | if (is_null($this->getCurrentTopic())) { 184 | $kafka->assign(); 185 | } else { 186 | $kafka->assign([ 187 | new \RdKafka\TopicPartition( $this->getCurrentTopic(), $this->getPartition($this->getCurrentTopic()), $this->getOffset($this->getCurrentTopic()) ) 188 | ]); 189 | } 190 | break; 191 | 192 | case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: 193 | echo "Revoke: "; 194 | var_dump($partitions); 195 | $kafka->assign(NULL); 196 | break; 197 | 198 | default: 199 | throw new \Exception($err); 200 | } 201 | } 202 | } --------------------------------------------------------------------------------