├── .gitattributes ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── composer.json ├── docker-compose-test.yaml ├── readme.md └── src ├── Commit ├── BatchCommitter.php ├── Committer.php ├── CommitterFactory.php ├── KafkaCommitter.php ├── NativeSleeper.php ├── RetryableCommitter.php ├── Sleeper.php └── VoidCommitter.php ├── Consumer.php ├── ConsumerBuilder.php ├── Contracts └── Consumer.php ├── Entities ├── Config.php └── Config │ └── Sasl.php ├── Exceptions ├── InvalidCommitException.php ├── InvalidConsumerException.php └── KafkaConsumerException.php ├── Laravel ├── Console │ └── Commands │ │ ├── PhpKafkaConsumer │ │ └── Options.php │ │ └── PhpKafkaConsumerCommand.php └── Providers │ └── PhpKafkaConsumerProvider.php ├── Log └── Logger.php ├── MessageCounter.php ├── MessageHandler └── CallableConsumer.php ├── Retry └── Retryable.php └── Validators └── Commands └── PhpKafkaConsumer └── Validator.php /.gitattributes: -------------------------------------------------------------------------------- 1 | /.github export-ignore 2 | /build export-ignore 3 | /dev export-ignore 4 | /tests export-ignore 5 | Dockefile export-ignore 6 | docker-compose.yaml export-ignore 7 | Makefile export-ignore 8 | phpunit.xml export-ignore 9 | start.sh export-ignore 10 | wait-for-it.sh export-ignore -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | vendor/ 3 | composer.lock 4 | .env 5 | coverage/ 6 | laravel-test/ -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to php-kafka-consumer 2 | 3 | We welcome all kinds of contributions. If you intend on becoming part of our 4 | community, please keep the following points in mind. 5 | 6 | ## Conduct 7 | 8 | In general, be good to each other and leave the world a better place than it was 9 | before your contribution. 10 | 11 | ## Issues and discussions 12 | 13 | We use Github to track bugs, feature requests, and other discussions. 14 | 15 | Please be clear when reporting a bug and add all relevant information. 16 | Maintainers may request additional info as necessary. 17 | 18 | You are encouraged to take part in all discussions about new features and 19 | improvements. 20 | 21 | ## Code and documentation 22 | 23 | When submitting a contribution, please fork the project and make a pull request 24 | against the master branch. Make sure all tests are passing and add relevant 25 | tests as necessary. A maintainer will review your contributions and make a new 26 | release as necessary. 27 | 28 | If your contributions address a complex issue, it might be a good idea to 29 | discuss it first, please make sure an issue is open and the community is settled 30 | on a solution. Pull requests may be denied if the maintainers are not sure that 31 | the changes address a real problem in the best way possible. 32 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2020, Arquivei 2 | 3 | Redistribution and use in source and binary forms, with or without modification, 4 | are permitted provided that the following conditions are met: 5 | 6 | 1. Redistributions of source code must retain the above copyright notice, this 7 | list of conditions and the following disclaimer. 8 | 9 | 2. Redistributions in binary form must reproduce the above copyright notice, 10 | this list of conditions and the following disclaimer in the documentation and/or 11 | other materials provided with the distribution. 12 | 13 | 3. Neither the name of the copyright holder nor the names of its contributors 14 | may be used to endorse or promote products derived from this software without 15 | specific prior written permission. 16 | 17 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 18 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 19 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 20 | IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 21 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 22 | BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 24 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE 25 | OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED 26 | OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "arquivei/php-kafka-consumer", 3 | "description": "A consumer of Kafka in PHP", 4 | "keywords": [ 5 | "php", 6 | "kafka", 7 | "consumer" 8 | ], 9 | "license": "MIT", 10 | "type": "project", 11 | "require": { 12 | "php": "~7.2 || ~7.3 || ~7.4 || ^8.0", 13 | "monolog/monolog": "~1 || ~2 || ~3", 14 | "illuminate/console": "~6 || ~7 || ~8 || ~9 || ~10 || ~11", 15 | "ext-rdkafka": "~3.0 || ~3.1 || ~4.0 || ~5.0 || ~6.0" 16 | }, 17 | "require-dev": { 18 | "phpunit/phpunit": "~7 || ~8 || ~9 || ~10 || ~11" 19 | }, 20 | "autoload": { 21 | "psr-4": { 22 | "Kafka\\Consumer\\": "src/" 23 | } 24 | }, 25 | "autoload-dev": { 26 | "psr-4": { 27 | "Kafka\\Consumer\\Tests\\": "tests/" 28 | } 29 | }, 30 | "archive": { 31 | "exclude": [ 32 | "/.github", 33 | "/build", 34 | "/dev", 35 | "/tests", 36 | "Dockefile", 37 | "docker-compose.yaml", 38 | "Makefile", 39 | "phpunit.xml", 40 | "start.sh", 41 | "wait-for-it.sh" 42 | ] 43 | }, 44 | "config": { 45 | "preferred-install": "dist", 46 | "sort-packages": true, 47 | "optimize-autoloader": true 48 | }, 49 | "extra": { 50 | "laravel": { 51 | "providers": [ 52 | "Kafka\\Consumer\\Laravel\\Providers\\PhpKafkaConsumerProvider" 53 | ] 54 | } 55 | }, 56 | "minimum-stability": "dev", 57 | "prefer-stable": true 58 | } 59 | -------------------------------------------------------------------------------- /docker-compose-test.yaml: -------------------------------------------------------------------------------- 1 | version: "3.7" 2 | networks: 3 | app-test: 4 | driver: bridge 5 | services: 6 | zookeeper-test: 7 | image: 'bitnami/zookeeper:latest' 8 | restart: always 9 | networks: 10 | - app-test 11 | ports: 12 | - '2182:2181' 13 | environment: 14 | - ALLOW_ANONYMOUS_LOGIN=yes 15 | 16 | kafka-test: 17 | image: 'bitnami/kafka:latest' 18 | restart: always 19 | ports: 20 | - '9094:9092' 21 | - '9095:9093' 22 | environment: 23 | - KAFKA_BROKER_ID=1 24 | - KAFKA_LISTENERS=CLIENT://:9092 25 | - KAFKA_CFG_LISTENERS=PLAINTEXT://:9092 26 | - KAFKA_ADVERTISED_LISTENERS=CLIENT://kafka-test:9092 27 | - KAFKA_ZOOKEEPER_CONNECT=zookeeper-test:2181 28 | - ALLOW_PLAINTEXT_LISTENER=yes 29 | - KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP=CLIENT:PLAINTEXT,EXTERNAL:PLAINTEXT 30 | - KAFKA_CFG_LISTENERS=CLIENT://:9092,EXTERNAL://:9093 31 | - KAFKA_CFG_ADVERTISED_LISTENERS=CLIENT://kafka-test:9092,EXTERNAL://localhost:9093 32 | - KAFKA_CFG_AUTO_CREATE_TOPICS_ENABLE=true 33 | - KAFKA_INTER_BROKER_LISTENER_NAME=CLIENT 34 | depends_on: 35 | - zookeeper-test 36 | networks: 37 | - app-test 38 | 39 | test: 40 | build: 41 | context: . 42 | dockerfile: build/test/Dockerfile 43 | args: 44 | TAG: 8.0-v1.6.1-5.0.0-8 45 | LARAVEL_VERSION: 8 46 | entrypoint: /application/php-kafka-consumer/start.sh 47 | networks: 48 | - app-test 49 | environment: 50 | KAFKA_BROKERS: kafka-test:9092 51 | depends_on: 52 | - kafka-test 53 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # php-kafka-consumer 2 | 3 | [![Latest Stable Version](https://poser.pugx.org/arquivei/php-kafka-consumer/v/stable)](https://packagist.org/packages/arquivei/php-kafka-consumer) [![Total Downloads](https://poser.pugx.org/arquivei/php-kafka-consumer/downloads)](https://packagist.org/packages/arquivei/php-kafka-consumer) ![Tests](https://github.com/arquivei/php-kafka-consumer/workflows/Test/badge.svg) ![Dependency coverage](https://github.com/arquivei/php-kafka-consumer/workflows/Version%20test/badge.svg) 4 | 5 | An Apache Kafka consumer in PHP. Subscribe to topics and define callbacks to handle the messages. 6 | 7 | ## Requirements 8 | 9 | In order to use this library, you'll need the [php-rdkafka](https://github.com/arnaud-lb/php-rdkafka) PECL extension. 10 | Please notice that the extension requires the [librdkafka](https://github.com/edenhill/librdkafka) C library. 11 | 12 | Minimum requirements: 13 | 14 | | Dependency | version | 15 | |-------------|---------| 16 | | librdkafka | v1.5.3 | 17 | | PHP | 7.4 + | 18 | | ext-rdkafka | 3.0 + | 19 | | Laravel | 6 + | 20 | 21 | ## Install 22 | 23 | Using composer: 24 | 25 | `composer require arquivei/php-kafka-consumer` 26 | 27 | ## Usage 28 | 29 | ```php 30 | withSasl(new Sasl('username', 'pasword', 'mechanisms')) 49 | ->withCommitBatchSize(1) 50 | ->withSecurityProtocol('security-protocol') 51 | ->withHandler(new DefaultConsumer()) // or any callable 52 | ->build(); 53 | 54 | $consumer->consume(); 55 | ``` 56 | 57 | Or by using the legacy API: 58 | 59 | ```php 60 | consume(); 97 | ``` 98 | 99 | ## Usage with Laravel 100 | 101 | You need to add the `php-kafka-consig.php` in `config` path: 102 | 103 | ```php 104 | 'topic', 108 | 'broker' => 'broker', 109 | 'groupId' => 'group-id', 110 | 'securityProtocol' => 'security-protocol', 111 | 'sasl' => [ 112 | 'mechanisms' => 'mechanisms', 113 | 'username' => 'username', 114 | 'password' => 'password', 115 | ], 116 | ]; 117 | 118 | ``` 119 | 120 | Use the command to execute the consumer: 121 | 122 | ```bash 123 | $ php artisan arquivei:php-kafka-consumer --consumer="App\Consumers\YourConsumer" --commit=1 124 | ``` 125 | 126 | ### Middlewares 127 | 128 | Middlewares are simple callables that receive two arguments: the message being handled and the 129 | next handler. Some possible use cases for middlewares: message transformation, filtering, logging stuff, 130 | or even transaction handling, your imagination is the limit. 131 | 132 | ```php 133 | withHandler(function ($message) {/** ... */}) 139 | // You may add any number of middlewares, they will be executed in the order provided 140 | ->withMiddleware(function (string $rawMessage, callable $next): void { 141 | $decoded = json_decode($rawMessage, true); 142 | $next($decoded); 143 | }) 144 | ->withMiddleware(function (array $message, callable $next): void { 145 | if (! isset($message['foo'])) { 146 | return; 147 | } 148 | $next($message); 149 | }) 150 | ->build(); 151 | 152 | $consumer->consume(); 153 | ``` 154 | 155 | ## Build and test 156 | 157 | If you want to contribute, there are a few utilities that will help. 158 | 159 | First create a container: 160 | 161 | `docker compose up -d --build` 162 | 163 | If you have make, you can use pre defined commands in the Makefile 164 | 165 | `make build` 166 | 167 | Then install the dependencies: 168 | 169 | `docker compose exec php-fpm composer install` 170 | 171 | or with make: 172 | 173 | `make composer install` 174 | 175 | You can run tests locally: 176 | 177 | `docker compose exec php-fpm ./vendor/phpunit/phpunit/phpunit tests` 178 | 179 | or with make: 180 | 181 | `make test` 182 | 183 | and check for coverage: 184 | 185 | `docker compose exec php-fpm phpdbg -qrr ./vendor/bin/phpunit --whitelist src/ --coverage-html coverage/` 186 | 187 | or with make: 188 | 189 | `make coverage` 190 | -------------------------------------------------------------------------------- /src/Commit/BatchCommitter.php: -------------------------------------------------------------------------------- 1 | committer = $committer; 25 | $this->messageCounter = $messageCounter; 26 | $this->batchSize = $batchSize; 27 | } 28 | 29 | public function commitMessage(): void 30 | { 31 | $this->commits++; 32 | if ($this->isMaxMessage() || $this->commits >= $this->batchSize) { 33 | $this->committer->commitMessage(); 34 | $this->commits = 0; 35 | } 36 | } 37 | 38 | private function isMaxMessage(): bool 39 | { 40 | return $this->messageCounter->isMaxMessage(); 41 | } 42 | 43 | public function commitDlq(): void 44 | { 45 | $this->committer->commitDlq(); 46 | $this->commits = 0; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Commit/Committer.php: -------------------------------------------------------------------------------- 1 | messageCounter = $messageCounter; 19 | } 20 | 21 | public function make(KafkaConsumer $kafkaConsumer, Config $config): Committer 22 | { 23 | if ($config->isAutoCommit()) { 24 | return new VoidCommitter(); 25 | } 26 | 27 | return new BatchCommitter( 28 | new RetryableCommitter( 29 | new KafkaCommitter( 30 | $kafkaConsumer 31 | ), 32 | new NativeSleeper(), 33 | $config->getMaxCommitRetries() 34 | ), 35 | $this->messageCounter, 36 | $config->getCommit() 37 | ); 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Commit/KafkaCommitter.php: -------------------------------------------------------------------------------- 1 | consumer = $consumer; 21 | } 22 | 23 | public function commitMessage(): void 24 | { 25 | $this->consumer->commit(); 26 | } 27 | 28 | public function commitDlq(): void 29 | { 30 | $this->consumer->commit(); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Commit/NativeSleeper.php: -------------------------------------------------------------------------------- 1 | committer = $committer; 26 | $this->retryable = new Retryable($sleeper, $maximumRetries, self::RETRYABLE_ERRORS); 27 | } 28 | 29 | public function commitMessage(): void 30 | { 31 | $this->retryable->retry(function () { 32 | $this->committer->commitMessage(); 33 | }); 34 | } 35 | 36 | public function commitDlq(): void 37 | { 38 | $this->retryable->retry(function () { 39 | $this->committer->commitDlq(); 40 | }); 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Commit/Sleeper.php: -------------------------------------------------------------------------------- 1 | config = $config; 45 | $this->logger = new Logger(); 46 | $this->messageCounter = new MessageCounter($config->getMaxMessages()); 47 | $this->retryable = new Retryable(new NativeSleeper(), 6, self::TIMEOUT_ERRORS); 48 | $this->committerFactory = new CommitterFactory( 49 | $this->messageCounter 50 | ); 51 | } 52 | 53 | public function consume(): void 54 | { 55 | if($this->config->getPrintConfigs()) { 56 | $this->printConsumerConfigs(); 57 | } 58 | 59 | $this->consumer = new KafkaConsumer($this->setConf($this->config->getConsumerOptions())); 60 | $this->producer = new Producer($this->setConf($this->config->getProducerOptions())); 61 | 62 | $this->committer = $this->committerFactory->make($this->consumer, $this->config); 63 | 64 | $this->consumer->subscribe($this->config->getTopics()); 65 | 66 | do { 67 | $this->retryable->retry(function () { 68 | $this->doConsume(); 69 | }); 70 | } while (!$this->isMaxMessage()); 71 | } 72 | 73 | private function printConsumerConfigs() { 74 | echo PHP_EOL; 75 | echo "\e[0;30;42m ++++++++++++++++++ CONSUMER CONFIGS ++++++++++++++++++\e[0m\n"; 76 | echo PHP_EOL; 77 | 78 | $mask = "\e[0;32m%26s | %s \e[0m\n"; 79 | printf($mask, 'CONFIG', 'VALUE'); 80 | 81 | $mask = "%26s | %s \n"; 82 | printf($mask, 'topics', implode(', ', $this->config->getTopics())); 83 | printf($mask, 'dlq', $this->config->getDlq()); 84 | printf($mask, 'commit', $this->config->getCommit()); 85 | printf($mask, 'maxCommitRetries', $this->config->getMaxCommitRetries()); 86 | printf($mask, 'maxMessages', $this->config->getMaxMessages()); 87 | 88 | foreach ($this->config->getConsumerOptions() as $key => $value) { 89 | if($key !== 'sasl.username' && $key !== 'sasl.password') { 90 | printf($mask, $key, $value); 91 | } 92 | } 93 | 94 | echo PHP_EOL; 95 | } 96 | 97 | private function doConsume() 98 | { 99 | $message = $this->consumer->consume(120000); 100 | $this->handleMessage($message); 101 | } 102 | 103 | public function setConf(array $options): Conf 104 | { 105 | $conf = new Conf(); 106 | 107 | foreach ($options as $key => $value) { 108 | $conf->set($key, $value); 109 | } 110 | 111 | return $conf; 112 | } 113 | 114 | private function executeMessage(Message $message): void 115 | { 116 | try { 117 | $this->config->getConsumer()->handle($message->payload); 118 | $success = true; 119 | } catch (Throwable $throwable) { 120 | $this->logger->error($message, $throwable); 121 | $success = $this->handleException($throwable, $message); 122 | } 123 | 124 | $this->commit($message, $success); 125 | } 126 | 127 | private function handleException( 128 | Throwable $exception, 129 | Message $message 130 | ): bool 131 | { 132 | try { 133 | $this->config->getConsumer()->failed( 134 | $message->payload, 135 | $this->config->getTopics()[0], 136 | $exception 137 | ); 138 | return true; 139 | } catch (Throwable $throwable) { 140 | if ($exception !== $throwable) { 141 | $this->logger->error($message, $throwable, 'HANDLER_EXCEPTION'); 142 | } 143 | return false; 144 | } 145 | } 146 | 147 | private function sendToDlq(Message $message): void 148 | { 149 | $topic = $this->producer->newTopic($this->config->getDlq()); 150 | $topic->produce( 151 | RD_KAFKA_PARTITION_UA, 152 | 0, 153 | $message->payload, 154 | $this->config->getConsumer()->producerKey($message->payload) 155 | ); 156 | 157 | if (method_exists($this->producer, 'flush')) { 158 | $this->producer->flush(12000); 159 | } 160 | } 161 | 162 | private function commit(Message $message, bool $success): void 163 | { 164 | try { 165 | if (!$success && !is_null($this->config->getDlq())) { 166 | $this->sendToDlq($message); 167 | $this->committer->commitDlq(); 168 | return; 169 | } 170 | 171 | $this->committer->commitMessage(); 172 | } catch (Throwable $throwable) { 173 | if (!in_array($throwable->getCode(), self::IGNORABLE_COMMIT_ERRORS)) { 174 | $this->logger->error($message, $throwable, 'MESSAGE_COMMIT'); 175 | throw $throwable; 176 | } 177 | } 178 | } 179 | 180 | private function isMaxMessage(): bool 181 | { 182 | return $this->messageCounter->isMaxMessage(); 183 | } 184 | 185 | private function handleMessage(Message $message): void 186 | { 187 | if (RD_KAFKA_RESP_ERR_NO_ERROR === $message->err) { 188 | $this->messageCounter->add(); 189 | $this->executeMessage($message); 190 | return; 191 | } 192 | 193 | if (!in_array($message->err, self::IGNORABLE_CONSUME_ERRORS)) { 194 | $this->logger->error($message, null, 'CONSUMER'); 195 | throw new KafkaConsumerException($message->errstr(), $message->err); 196 | } 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /src/ConsumerBuilder.php: -------------------------------------------------------------------------------- 1 | brokers = $brokers; 39 | $this->groupId = $groupId; 40 | $this->topics = $topics; 41 | 42 | $this->commit = 1; 43 | $this->handler = function () { 44 | }; 45 | $this->maxMessages = -1; 46 | $this->maxCommitRetries = 6; 47 | $this->middlewares = []; 48 | $this->securityProtocol = 'PLAINTEXT'; 49 | $this->autoCommit = false; 50 | $this->options = []; 51 | $this->printConfigs = false; 52 | } 53 | 54 | public static function create(string $brokers, $groupId, array $topics): self 55 | { 56 | return new ConsumerBuilder($brokers, $groupId, $topics); 57 | } 58 | 59 | public function withCommitBatchSize(int $size): self 60 | { 61 | $this->commit = $size; 62 | return $this; 63 | } 64 | 65 | /** 66 | * The function that will handle the incoming messages 67 | * 68 | * @param callable(mixed $message): void $handler 69 | */ 70 | public function withHandler(callable $handler): self 71 | { 72 | $this->handler = Closure::fromCallable($handler); 73 | return $this; 74 | } 75 | 76 | public function withMaxMessages(int $maxMessages): self 77 | { 78 | $this->maxMessages = $maxMessages; 79 | return $this; 80 | } 81 | 82 | public function withMaxCommitRetries(int $maxCommitRetries): self 83 | { 84 | $this->maxCommitRetries = $maxCommitRetries; 85 | return $this; 86 | } 87 | 88 | public function withDlq(?string $dlqTopic = null): self 89 | { 90 | if (null === $dlqTopic) { 91 | $dlqTopic = $this->topics[0] . '-dlq'; 92 | } 93 | 94 | $this->dlq = $dlqTopic; 95 | 96 | return $this; 97 | } 98 | 99 | public function withSasl(Sasl $saslConfig): self 100 | { 101 | $this->saslConfig = $saslConfig; 102 | return $this; 103 | } 104 | 105 | /** 106 | * The middlewares get executed in the order they are defined. 107 | * 108 | * The middleware is a callable in which the first argument is the message itself and the second is the next handler 109 | * 110 | * @param callable(mixed, callable): void $middleware 111 | * @return $this 112 | */ 113 | public function withMiddleware(callable $middleware): self 114 | { 115 | $this->middlewares[] = $middleware; 116 | return $this; 117 | } 118 | 119 | public function withSecurityProtocol(string $securityProtocol): self 120 | { 121 | $this->securityProtocol = $securityProtocol; 122 | return $this; 123 | } 124 | 125 | public function withAutoCommit(): self 126 | { 127 | $this->autoCommit = true; 128 | return $this; 129 | } 130 | 131 | public function withOptions(array $options): self 132 | { 133 | foreach ($options as $name => $value) { 134 | $this->withOption($name, $value); 135 | } 136 | 137 | return $this; 138 | } 139 | 140 | public function withOption(string $name, string $value): self 141 | { 142 | $this->options[$name] = $value; 143 | return $this; 144 | } 145 | 146 | public function withPrintConfigs(bool $printConfigs): self 147 | { 148 | $this->printConfigs = $printConfigs; 149 | return $this; 150 | } 151 | 152 | public function build(): Consumer 153 | { 154 | $config = new Config( 155 | $this->saslConfig, 156 | $this->topics, 157 | $this->brokers, 158 | $this->commit, 159 | $this->groupId, 160 | new CallableConsumer($this->handler, $this->middlewares), 161 | $this->securityProtocol, 162 | $this->dlq, 163 | $this->maxMessages, 164 | $this->maxCommitRetries, 165 | $this->autoCommit, 166 | $this->options, 167 | $this->printConfigs 168 | ); 169 | 170 | return new Consumer( 171 | $config 172 | ); 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /src/Contracts/Consumer.php: -------------------------------------------------------------------------------- 1 | dlq = $dlq; 40 | $this->sasl = $sasl; 41 | $this->topics = $topics; 42 | $this->broker = $broker; 43 | $this->commit = $commit; 44 | $this->groupId = $groupId; 45 | $this->consumer = $consumer; 46 | $this->maxMessages = $maxMessages; 47 | $this->securityProtocol = $securityProtocol; 48 | $this->maxCommitRetries = $maxCommitRetries; 49 | $this->autoCommit = $autoCommit; 50 | $this->customOptions = $customOptions; 51 | $this->printConfigs = $printConfigs; 52 | } 53 | 54 | public function getCommit(): int 55 | { 56 | return $this->commit; 57 | } 58 | 59 | public function getMaxCommitRetries(): int 60 | { 61 | return $this->maxCommitRetries; 62 | } 63 | 64 | public function getTopics(): array 65 | { 66 | return $this->topics; 67 | } 68 | 69 | public function getConsumer(): Consumer 70 | { 71 | return $this->consumer; 72 | } 73 | 74 | public function getDlq(): ?string 75 | { 76 | return $this->dlq; 77 | } 78 | 79 | public function getMaxMessages(): int 80 | { 81 | return $this->maxMessages; 82 | } 83 | 84 | public function isAutoCommit(): bool 85 | { 86 | return $this->autoCommit; 87 | } 88 | 89 | public function getConsumerOptions(): array 90 | { 91 | $options = [ 92 | 'auto.offset.reset' => 'smallest', 93 | 'queued.max.messages.kbytes' => '10000', 94 | 'enable.auto.commit' => 'false', 95 | 'compression.codec' => 'gzip', 96 | 'max.poll.interval.ms' => '86400000', 97 | 'group.id' => $this->groupId, 98 | 'bootstrap.servers' => $this->broker, 99 | 'security.protocol' => $this->securityProtocol, 100 | ]; 101 | 102 | if ($this->autoCommit) { 103 | $options['enable.auto.commit'] = 'true'; 104 | } 105 | 106 | return array_merge($options, $this->getSaslOptions(), $this->customOptions); 107 | } 108 | 109 | public function getProducerOptions(): array 110 | { 111 | $config = [ 112 | 'compression.codec' => 'gzip', 113 | 'bootstrap.servers' => $this->broker, 114 | 'security.protocol' => $this->securityProtocol, 115 | ]; 116 | 117 | return array_merge($config, $this->getSaslOptions()); 118 | } 119 | 120 | public function getPrintConfigs(): bool 121 | { 122 | return $this->printConfigs; 123 | } 124 | 125 | private function getSaslOptions(): array 126 | { 127 | if ($this->isPlainText() && $this->sasl !== null) { 128 | return [ 129 | 'sasl.username' => $this->sasl->getUsername(), 130 | 'sasl.password' => $this->sasl->getPassword(), 131 | 'sasl.mechanisms' => $this->sasl->getMechanisms(), 132 | ]; 133 | } 134 | 135 | return []; 136 | } 137 | 138 | private function isPlainText(): bool 139 | { 140 | return $this->securityProtocol == 'SASL_PLAINTEXT'; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/Entities/Config/Sasl.php: -------------------------------------------------------------------------------- 1 | username = $username; 14 | $this->password = $password; 15 | $this->mechanisms = $mechanisms; 16 | } 17 | 18 | public function getUsername(): string 19 | { 20 | return $this->username; 21 | } 22 | 23 | public function getPassword(): string 24 | { 25 | return $this->password; 26 | } 27 | 28 | public function getMechanisms(): string 29 | { 30 | return $this->mechanisms; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Exceptions/InvalidCommitException.php: -------------------------------------------------------------------------------- 1 | topics = $options['topic']; 21 | $this->consumer = $options['consumer']; 22 | $this->groupId = $options['groupId']; 23 | $this->commit = $options['commit']; 24 | $this->dlq = $options['dlq']; 25 | $this->maxMessage = $options['maxMessage']; 26 | } 27 | 28 | public function getTopics(): array 29 | { 30 | return (is_array($this->topics) && !empty($this->topics)) ? $this->topics : []; 31 | } 32 | 33 | public function getConsumer(): ?string 34 | { 35 | return $this->consumer; 36 | } 37 | 38 | public function getGroupId(): string 39 | { 40 | return $this->groupId; 41 | } 42 | 43 | public function getCommit(): ?string 44 | { 45 | return $this->commit; 46 | } 47 | 48 | public function getDlq(): ?string 49 | { 50 | return (is_string($this->dlq) && strlen($this->dlq) > 1) ? $this->dlq : null; 51 | } 52 | 53 | public function getMaxMessage(): int 54 | { 55 | return (is_int($this->maxMessage) && $this->maxMessage >= 1) ? $this->maxMessage : -1; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Laravel/Console/Commands/PhpKafkaConsumerCommand.php: -------------------------------------------------------------------------------- 1 | config = config('php-kafka-consumer'); 20 | } 21 | 22 | public function handle() 23 | { 24 | (new Validator())->validateOptions($this->options()); 25 | $options = $this->options(); 26 | $options['groupId'] = $options['groupId'] ?? $this->config['groupId']; 27 | $options = new Options($options); 28 | 29 | $consumer = $options->getConsumer(); 30 | $config = new \Kafka\Consumer\Entities\Config( 31 | new \Kafka\Consumer\Entities\Config\Sasl( 32 | $this->config['sasl']['username'], 33 | $this->config['sasl']['password'], 34 | $this->config['sasl']['mechanisms'] 35 | ), 36 | $options->getTopics(), 37 | $this->config['broker'], 38 | $options->getCommit(), 39 | $options->getGroupId(), 40 | new $consumer(), 41 | $this->config['securityProtocol'], 42 | $options->getDlq(), 43 | $options->getMaxMessage() 44 | ); 45 | 46 | (new \Kafka\Consumer\Consumer($config))->consume(); 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Laravel/Providers/PhpKafkaConsumerProvider.php: -------------------------------------------------------------------------------- 1 | commands([ 18 | PhpKafkaConsumerCommand::class 19 | ]); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Log/Logger.php: -------------------------------------------------------------------------------- 1 | setFormatter(new JsonFormatter()) 17 | ->pushProcessor(new UidProcessor(32)); 18 | $this->logger = new \Monolog\Logger('PHP-KAFKA-CONSUMER-ERROR'); 19 | $this->logger->pushHandler($handler); 20 | $this->logger->pushProcessor(function ($record) { 21 | if (is_array($record)) { 22 | $record['datetime'] = $record['datetime']->format('c'); 23 | } 24 | 25 | return $record; 26 | }); 27 | } 28 | 29 | public function error(\RdKafka\Message $message, \Throwable $exception = null, string $prefix = 'ERROR'): void 30 | { 31 | $this->logger->error("[$prefix] Error to consume message", [ 32 | 'message' => $message, 33 | 'throwable' => $exception 34 | ]); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/MessageCounter.php: -------------------------------------------------------------------------------- 1 | maxMessages = $maxMessages; 15 | } 16 | 17 | public function add(): void 18 | { 19 | $this->messageCount++; 20 | } 21 | 22 | public function isMaxMessage(): bool 23 | { 24 | return $this->messageCount === $this->maxMessages; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /src/MessageHandler/CallableConsumer.php: -------------------------------------------------------------------------------- 1 | handler = Closure::fromCallable($handler); 18 | $this->middlewares = array_map([$this, 'wrapMiddleware'], $middlewares); 19 | $this->middlewares[] = $this->wrapMiddleware(function ($message, callable $next) { 20 | $next($message); 21 | }); 22 | } 23 | 24 | public function handle(string $message): void 25 | { 26 | $middlewares = array_reverse($this->middlewares); 27 | $handler = array_shift($middlewares)($this->handler); 28 | 29 | foreach ($middlewares as $middleware) { 30 | $handler = $middleware($handler); 31 | } 32 | 33 | $handler($message); 34 | } 35 | 36 | private function wrapMiddleware(callable $middleware): callable 37 | { 38 | return function (callable $handler) use ($middleware) { 39 | return function ($message) use ($handler, $middleware) { 40 | $middleware($message, $handler); 41 | }; 42 | }; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Retry/Retryable.php: -------------------------------------------------------------------------------- 1 | sleeper = $sleeper; 17 | $this->maximumRetries = $maximumRetries; 18 | $this->retryableErrors = $retryableErrors; 19 | } 20 | 21 | public function retry( 22 | callable $function, 23 | int $currentRetries = 0, 24 | int $delayInSeconds = 1, 25 | bool $exponentially = true 26 | ) { 27 | try { 28 | $function(); 29 | } catch (Exception $exception) { 30 | if (in_array($exception->getCode(), $this->retryableErrors) && $currentRetries < $this->maximumRetries) { 31 | $this->sleeper->sleep((int)($delayInSeconds * 1e6)); 32 | $this->retry( 33 | $function, 34 | ++$currentRetries, 35 | $exponentially == true ? $delayInSeconds * 2 : $delayInSeconds 36 | ); 37 | return; 38 | } 39 | 40 | throw $exception; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /src/Validators/Commands/PhpKafkaConsumer/Validator.php: -------------------------------------------------------------------------------- 1 | validateCommit($options['commit']); 14 | $this->validateConsumer($options['consumer']); 15 | } 16 | 17 | private function validateCommit(?string $commit): void 18 | { 19 | if (is_null($commit) || $commit < 1) { 20 | throw new InvalidCommitException(); 21 | } 22 | } 23 | 24 | private function validateConsumer(?string $consumer): void 25 | { 26 | if (! class_exists($consumer) || !is_subclass_of($consumer, Consumer::class)) { 27 | throw new InvalidConsumerException(); 28 | } 29 | } 30 | } 31 | --------------------------------------------------------------------------------