├── .gitignore ├── README.md ├── composer.json ├── config └── kafka.php └── src ├── Exceptions └── QueueKafkaException.php ├── LaravelQueueKafkaServiceProvider.php ├── LumenQueueKafkaServiceProvider.php └── Queue ├── Jobs └── KafkaJob.php ├── KafkaConnector.php └── KafkaQueue.php /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /vendor 3 | composer.lock 4 | .phpstorm.meta.php 5 | phpunit.xml 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Laravel Kafka队列 2 | 3 | 4 | 5 | #### 安装 6 | 7 | 8 | 1. 安装PHP依赖库 9 | 10 | ```bash 11 | pecl install rdkafka 12 | ``` 13 | 2. 在PHP的初始化文件php.ini中添加以下一行语句以开启Kafka扩展。 14 | 15 | ```bash 16 | extension=rdkafka.so 17 | ``` 18 | 19 | b. 检车 rdkafka 是否安装成功 20 | 21 | 注意：如果你想在 php-fpm 上运行它，请先重启你的 php-fpm 22 | 23 | php -i | grep rdkafka 24 | 25 | 你的输出应该是这样的 26 | 27 | rdkafka 28 | rdkafka support => enabled 29 | librdkafka version (runtime) => 1.6.1-38-g7f0929-dirty 30 | librdkafka version (build) => 1.6.1.255 31 | 32 | 3. 通过 Composer 安装这个包 33 | 34 | composer require phpkafka/laravel-kafka 35 | 36 | 4. 将 LaravelQueueKafkaServiceProvider 添加到providers数组中config/app.php 37 | 38 | phpkafka\LaravelQueueKafka\LaravelQueueKafkaServiceProvider::class, 39 | 40 | 如果您使用 Lumen，请将其放入 bootstrap/app.php 41 | 42 | $app->register(phpkafka\LaravelQueueKafka\LumenQueueKafkaServiceProvider::class); 43 | 44 | 5.要使用 kafka 队列驱动程序，需要在 config/queue.php 配置文件中配置一个 kafka 连接。 45 | 46 | 'kafka' => [ 47 | 'driver' => 'kafka', 48 | 'queue' => env('KAFKA_QUEUE', 'default'), 49 | 'consumer_group_id' => env('KAFKA_CONSUMER_GROUP_ID', 'laravel_queue'), 50 | 'brokers' => env('KAFKA_BROKERS', 'localhost'), 51 | 'sleep_on_error' => env('KAFKA_ERROR_SLEEP', 5), 52 | 'sleep_on_deadlock' => env('KAFKA_DEADLOCK_SLEEP', 2), 53 | ] 54 | 55 | 56 | 6. 将这些属性添加到.env 文件中 57 | 58 | QUEUE_CONNECTION=kafka 59 | 60 | 选择配置 61 | KAFKA_BROKERS=127.0.0.1:9092 #kafka地址，多个用,隔开 62 | KAFKA_ERROR_SLEEP=5 #确定的秒数睡眠与kafka交流如果有一个错误(秒) 63 | KAFKA_DEADLOCK_SLEEP=2 #睡眠时检测到死锁(秒) 64 | KAFKA_QUEUE=default #默认队列名 65 | KAFKA_CONSUMER_GROUP_ID=test #默认分组 66 | 67 | 7. 如果你想为特定的消费者Group运行队列 68 | 69 | export KAFKA_CONSUMER_GROUP_ID="testgroup" && php artisan queue:work --sleep=3 --tries=3 70 | 8. 多消费者 71 | 1629095586676.jpg![1629095586676](https://user-images.githubusercontent.com/9024302/129521025-59821ce5-2d4b-43f1-871c-88dc9f207cee.jpg) 72 | 73 | 74 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "phpkafka/laravel-kafka", 3 | "description": "laravel-kafka", 4 | "license": "MIT", 5 | "authors": [ 6 | { 7 | "name": "陈鹏", 8 | "email": "476451758@qq.com" 9 | } 10 | ], 11 | "require": { 12 | "php": ">=7.1", 13 | "illuminate/database": ">=5.8", 14 | "illuminate/support": ">=5.8", 15 | "illuminate/queue": ">=5.8", 16 | "ext-rdkafka": "*" 17 | }, 18 | "autoload": { 19 | "psr-4": { 20 | "phpkafka\\LaravelQueueKafka\\": "src/" 21 | } 22 | }, 23 | "extra": { 24 | "laravel": { 25 | "providers": [ 26 | "phpkafka\\LaravelQueueKafka\\LaravelQueueKafkaServiceProvider" 27 | ] 28 | } 29 | }, 30 | "minimum-stability": "dev", 31 | "prefer-stable": true 32 | } 33 | -------------------------------------------------------------------------------- /config/kafka.php: -------------------------------------------------------------------------------- 1 | 'kafka', 13 | 14 | /* 15 | * 默认队列的名称 16 | */ 17 | 'queue' => env('KAFKA_QUEUE', 'default'), 18 | 19 | /* 20 | * 默认消费者所在的组 21 | */ 22 | 'consumer_group_id' => env('KAFKA_CONSUMER_GROUP_ID', 'laravel_queue'), 23 | 24 | /* 25 | * 地址配置，多个用","分割 26 | */ 27 | 'brokers' => env('KAFKA_BROKERS', 'localhost'), 28 | 29 | /* 30 | * 确定的秒数睡眠与卡夫卡交流如果有一个错误 31 | * 如果设置为false,它会抛出一个异常,而不是做X秒的睡眠 32 | */ 33 | 'sleep_on_error' => env('KAFKA_ERROR_SLEEP', 5), 34 | 35 | /* 36 | * 睡眠时检测到死锁 37 | */ 38 | 'sleep_on_deadlock' => env('KAFKA_DEADLOCK_SLEEP', 2), 39 | 40 | ]; 41 | -------------------------------------------------------------------------------- /src/Exceptions/QueueKafkaException.php: -------------------------------------------------------------------------------- 1 | mergeConfigFrom( 24 | __DIR__ . '/../config/kafka.php', 'queue.connections.kafka' 25 | ); 26 | 27 | $this->registerDependencies(); 28 | } 29 | 30 | /** 31 | * 注册应用程序的事件监听器 32 | */ 33 | public function boot() 34 | { 35 | 36 | $queue = $this->app['queue']; 37 | $connector = new KafkaConnector($this->app); 38 | 39 | $queue->addConnector('kafka', function () use ($connector) { 40 | return $connector; 41 | }); 42 | } 43 | 44 | /** 45 | * 适配器注册依赖项的容器 46 | */ 47 | protected function registerDependencies() 48 | { 49 | 50 | $this->app->bind('queue.kafka.conf', function () { 51 | return new \RdKafka\Conf(); 52 | }); 53 | 54 | $this->app->bind('queue.kafka.producer', function ($app,$parameters) { 55 | return new \RdKafka\Producer($parameters['conf']); 56 | }); 57 | 58 | $this->app->bind('queue.kafka.consumer', function ($app, $parameters) { 59 | return new \RdKafka\KafkaConsumer($parameters['conf']); 60 | }); 61 | 62 | } 63 | 64 | /** 65 | * 提供的服务提供者 66 | * 67 | * @return array 68 | */ 69 | public function provides() 70 | { 71 | return [ 72 | 'queue.kafka.producer', 73 | 'queue.kafka.consumer', 74 | 'queue.kafka.conf', 75 | ]; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/LumenQueueKafkaServiceProvider.php: -------------------------------------------------------------------------------- 1 | container = $container; 53 | $this->connection = $connection; 54 | $this->message = $message; 55 | $this->connectionName = $connectionName; 56 | $this->queue = $queue; 57 | } 58 | 59 | /** 60 | * Fire the job 61 | * 62 | * @throws Exception 63 | */ 64 | public function fire() 65 | { 66 | try { 67 | $payload = $this->payload(); 68 | list($class, $method) = JobName::parse($payload['job']); 69 | 70 | with($this->instance = $this->resolve($class))->{$method}($this, $payload['data']); 71 | } catch (Exception $exception) { 72 | if ( 73 | $this->causedByDeadlock($exception) || 74 | Str::contains($exception->getMessage(), ['detected deadlock']) 75 | ) { 76 | sleep($this->connection->getConfig()['sleep_on_deadlock']); 77 | $this->fire(); 78 | 79 | return; 80 | } 81 | 82 | throw $exception; 83 | } 84 | } 85 | 86 | /** 87 | * 获得一个任务 88 | * 89 | * @return int 90 | */ 91 | public function attempts() 92 | { 93 | return (int) ($this->payload()['attempts']) + 1; 94 | } 95 | 96 | /** 97 | * 获取消息payload 98 | * 99 | * @return string 100 | */ 101 | public function getRawBody() 102 | { 103 | return $this->message->payload; 104 | } 105 | 106 | /** 107 | * 从队列中删除工作 108 | */ 109 | public function delete() 110 | { 111 | try { 112 | parent::delete(); 113 | $this->connection->getConsumer()->commitAsync($this->message); 114 | } catch (\RdKafka\Exception $exception) { 115 | throw new QueueKafkaException('Could not delete job from the queue', 0, $exception); 116 | } 117 | } 118 | 119 | /** 120 | * 释放工作回到队列中 121 | * 122 | * @param int $delay 123 | * 124 | * @throws Exception 125 | */ 126 | public function release($delay = 0) 127 | { 128 | parent::release($delay); 129 | 130 | $this->delete(); 131 | 132 | $body = $this->payload(); 133 | 134 | if (isset($body['data']['command']) === true) { 135 | $job = $this->unserialize($body); 136 | } else { 137 | $job = $this->getName(); 138 | } 139 | 140 | $data = $body['data']; 141 | 142 | if ($delay > 0) { 143 | $this->connection->later($delay, $job, $data, $this->getQueue()); 144 | } else { 145 | $this->connection->push($job, $data, $this->getQueue()); 146 | } 147 | } 148 | 149 | /** 150 | * 设置队列ID 151 | * 152 | * @param string $id 153 | */ 154 | public function setJobId($id) 155 | { 156 | $this->connection->setCorrelationId($id); 157 | } 158 | 159 | /** 160 | * 得到消息队列ID 161 | * 162 | * @return string 163 | */ 164 | public function getJobId() 165 | { 166 | return $this->message->key; 167 | } 168 | 169 | /** 170 | * 反序列化队列 171 | * 172 | * @param array $body 173 | * 174 | * @return mixed 175 | * @throws Exception 176 | * 177 | */ 178 | private function unserialize(array $body) 179 | { 180 | try { 181 | return unserialize($body['data']['command']); 182 | } catch (Exception $exception) { 183 | if ( 184 | $this->causedByDeadlock($exception) 185 | || Str::contains($exception->getMessage(), ['detected deadlock']) 186 | ) { 187 | sleep($this->connection->getConfig()['sleep_on_deadlock']); 188 | 189 | return $this->unserialize($body); 190 | } 191 | 192 | throw $exception; 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /src/Queue/KafkaConnector.php: -------------------------------------------------------------------------------- 1 | container = $container; 32 | } 33 | 34 | /** 35 | * 建立队列连接 36 | * 37 | * @param array $config 38 | * 39 | * @return \Illuminate\Contracts\Queue\Queue 40 | */ 41 | public function connect(array $config) 42 | { 43 | 44 | /** 初始化生产者 */ 45 | $producerConf = $this->container->makeWith('queue.kafka.conf', []); 46 | $producerConf->set('bootstrap.servers', $config['brokers']); 47 | $producer = $this->container->makeWith('queue.kafka.producer', ['conf' => $producerConf]); 48 | $producer->addBrokers($config['brokers']); 49 | 50 | /** 初始化配置 */ 51 | $conf = $this->container->makeWith('queue.kafka.conf', []); 52 | $conf->set('group.id', array_get($config, 'consumer_group_id', 'php-pubsub')); 53 | $conf->set('metadata.broker.list', $config['brokers']); 54 | $conf->set('enable.auto.commit', 'false'); 55 | /** 56 | * earliest 当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，从头开始消费 57 | * latest 当各分区下有已提交的offset时，从提交的offset开始消费；无提交的offset时，消费新产生的该分区下的数据 58 | */ 59 | $conf->set('auto.offset.reset', 'largest'); 60 | 61 | /** 初始化消费者 */ 62 | $consumer = $this->container->makeWith('queue.kafka.consumer', ['conf' => $conf]); 63 | 64 | return new KafkaQueue( 65 | $producer, 66 | $consumer, 67 | $config 68 | ); 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/Queue/KafkaQueue.php: -------------------------------------------------------------------------------- 1 | defaultQueue = $config['queue']; 65 | $this->sleepOnError = isset($config['sleep_on_error']) ? $config['sleep_on_error'] : 5; 66 | $this->producer = $producer; 67 | $this->consumer = $consumer; 68 | $this->config = $config; 69 | } 70 | 71 | /** 72 | * 获取队列的大小 73 | * 74 | * @param string $queue 75 | * 76 | * @return int 77 | */ 78 | public function size($queue = null) 79 | { 80 | //由于kafka是无限队列我们不能计算队列的大小 81 | return 1; 82 | } 83 | 84 | /** 85 | * 推送一个新工作在队列中 86 | * 87 | * @param string $job 88 | * @param mixed $data 89 | * @param string $queue 90 | * 91 | * @return bool 92 | */ 93 | public function push($job, $data = '', $queue = null) 94 | { 95 | return $this->pushRaw($this->createPayload($job, $data), $queue, []); 96 | } 97 | 98 | /** 99 | * 把原始任务压入队列 100 | * 101 | * @param string $payload 102 | * @param string $queue 103 | * @param array $options 104 | * 105 | * @throws QueueKafkaException 106 | * 107 | * @return mixed 108 | */ 109 | public function pushRaw($payload, $queue = null, array $options = []) 110 | { 111 | try { 112 | $topic = $this->getTopic($queue); 113 | 114 | $pushRawCorrelationId = $this->getCorrelationId(); 115 | 116 | $topic->produce(RD_KAFKA_PARTITION_UA, 0, $payload, $pushRawCorrelationId); 117 | $this->producer->flush(-1); 118 | return $pushRawCorrelationId; 119 | } catch (ErrorException $exception) { 120 | $this->reportConnectionError('pushRaw', $exception); 121 | } 122 | } 123 | 124 | /** 125 | * 推动新工作后到队列延迟 126 | * 127 | * @param \DateTime|int $delay 128 | * @param string $job 129 | * @param mixed $data 130 | * @param string $queue 131 | * 132 | * @throws QueueKafkaException 133 | * 134 | * @return mixed 135 | */ 136 | public function later($delay, $job, $data = '', $queue = null) 137 | { 138 | //延迟队列,没有实现 139 | throw new QueueKafkaException('kafka 延迟队列未实现'); 140 | } 141 | 142 | /** 143 | * 获取队列的下一份工作了 144 | * 145 | * @param string|null $queue 146 | * 147 | * @throws QueueKafkaException 148 | * 149 | * @return \Illuminate\Queue\Jobs\Job|null 150 | */ 151 | public function pop($queue = null) 152 | { 153 | try { 154 | $queue = $this->getQueueName($queue); 155 | if (!in_array($queue, $this->subscribedQueueNames)) { 156 | $this->subscribedQueueNames[] = $queue; 157 | $this->consumer->subscribe($this->subscribedQueueNames); 158 | } 159 | //消费消息并触发回调,超时（毫秒） 160 | $message = $this->consumer->consume(30 * 1000); 161 | if ($message === null) { 162 | return null; 163 | } 164 | switch ($message->err) { 165 | case RD_KAFKA_RESP_ERR_NO_ERROR: 166 | return new KafkaJob( 167 | $this->container, $this, $message, 168 | $this->connectionName, $queue ?: $this->defaultQueue 169 | ); 170 | case RD_KAFKA_RESP_ERR__PARTITION_EOF: 171 | case RD_KAFKA_RESP_ERR__TIMED_OUT: 172 | break; 173 | default: 174 | throw new QueueKafkaException($message->errstr(), $message->err); 175 | } 176 | } catch (\RdKafka\Exception $exception) { 177 | throw new QueueKafkaException('不能从队列中获取任务', 0, $exception); 178 | } 179 | } 180 | 181 | /** 182 | * @param string $queue 183 | * 184 | * @return string 185 | */ 186 | private function getQueueName($queue) 187 | { 188 | return $queue ?: $this->defaultQueue; 189 | } 190 | 191 | /** 192 | * 获取一个kafka主题 193 | * 194 | * @param $queue 195 | * 196 | * @return \RdKafka\ProducerTopic 197 | */ 198 | private function getTopic($queue) 199 | { 200 | return $this->producer->newTopic($this->getQueueName($queue)); 201 | } 202 | 203 | /** 204 | * 集相关id的消息公布 205 | * 206 | * @param string $id 207 | */ 208 | public function setCorrelationId($id) 209 | { 210 | $this->correlationId = $id; 211 | } 212 | 213 | /** 214 | * 检索相关id,或者一个惟一的id 215 | * 216 | * @return string 217 | */ 218 | public function getCorrelationId() 219 | { 220 | return $this->correlationId ?: uniqid('', true); 221 | } 222 | 223 | /** 224 | * @return array 225 | */ 226 | public function getConfig() 227 | { 228 | return $this->config; 229 | } 230 | 231 | /** 232 | * 创建一个从给定的工作负载阵列和数据 233 | * 234 | * @param string $job 235 | * @param mixed $data 236 | * @param string $queue 237 | * 238 | * @return array 239 | */ 240 | protected function createPayloadArray($job, $data = '', $queue = null) 241 | { 242 | return array_merge(parent::createPayloadArray($job, $data), [ 243 | 'id' => $this->getCorrelationId(), 244 | 'attempts' => 0, 245 | ]); 246 | } 247 | 248 | /** 249 | * @param string $action 250 | * @param Exception $e 251 | * 252 | * @throws QueueKafkaException 253 | */ 254 | protected function reportConnectionError($action, Exception $e) 255 | { 256 | Log::error('Kafka error while attempting ' . $action . ': ' . $e->getMessage()); 257 | 258 | // 如果设置为false,抛出一个错误,而不是等待 259 | if ($this->sleepOnError === false) { 260 | throw new QueueKafkaException('Error writing data to the connection with Kafka'); 261 | } 262 | 263 | // 睡眠 264 | sleep($this->sleepOnError); 265 | } 266 | 267 | /** 268 | * 获取消费者 269 | * 270 | * @return \RdKafka\KafkaConsumer 271 | */ 272 | public function getConsumer() 273 | { 274 | return $this->consumer; 275 | } 276 | 277 | } 278 | --------------------------------------------------------------------------------