├── .github ├── FUNDING.yml ├── ISSUE_TEMPLATE │ ├── custom.md │ ├── ask-a-question.md │ ├── feature_request.md │ └── bug_report.md ├── workflows │ ├── pull-requests.yml │ └── run-tests.yml └── CONTRIBUTING.md ├── docs ├── testing │ ├── _index.md │ ├── fake.md │ ├── assert-nothing-published.md │ ├── assert-published-times.md │ ├── assert-published-on-times.md │ ├── assert-published.md │ ├── assert-published-on.md │ └── mocking-your-kafka-consumer.md ├── advanced-usage │ ├── _index.md │ ├── sending-multiple-messages-with-the-same-producer.md │ ├── setting-global-configuration.md │ ├── stop-consumer-after-last-message.md │ ├── middlewares.md │ ├── replacing-default-serializer.md │ ├── stopping-a-consumer.md │ ├── before-callbacks.md │ ├── custom-loggers.md │ ├── graceful-shutdown.md │ └── sasl-authentication.md ├── consuming-messages │ ├── _index.md │ ├── consuming-messages.md │ ├── subscribing-to-kafka-topics.md │ ├── using-regex-to-subscribe-to-kafka-topics.md │ ├── creating-consumer.md │ ├── assigning-partitions.md │ ├── consuming-from-specific-offsets.md │ ├── queueable-handlers.md │ ├── custom-deserializers.md │ ├── consumer-groups.md │ └── class-structure.md ├── producing-messages │ ├── _index.md │ ├── publishing-to-kafka.md │ ├── producing-messages.md │ ├── configuring-producers.md │ ├── configuring-message-payload.md │ └── custom-serializers.md ├── _index.md ├── changelog.md ├── requirements.md ├── questions-and-issues.md ├── introduction.md ├── installation-and-setup.md └── upgrade-guide.md ├── art ├── sponsor.png ├── laravel-kafka.png └── donation-qr-code.png ├── .gitignore ├── src ├── Exceptions │ ├── LaravelKafkaException.php │ ├── SchemaRegistryException.php │ ├── ConsumerException.php │ ├── MessageIdNotSet.php │ ├── Serializers │ │ └── AvroSerializerException.php │ ├── ContextAwareException.php │ ├── Transactions │ │ ├── TransactionShouldBeRetriedException.php │ │ ├── TransactionShouldBeAbortedException.php │ │ └── TransactionFatalErrorException.php │ └── CouldNotPublishMessage.php ├── Contracts │ ├── Middleware.php │ ├── Handler.php │ ├── MessageSerializer.php │ ├── MessageDeserializer.php │ ├── AvroMessageSerializer.php │ ├── AvroMessageDeserializer.php │ ├── Sleeper.php │ ├── ConsumerMessage.php │ ├── Logger.php │ ├── ContextAware.php │ ├── CommitterFactory.php │ ├── KafkaMessage.php │ ├── KafkaAvroSchemaRegistry.php │ ├── RetryStrategy.php │ ├── Consumer.php │ ├── ProducerMessage.php │ ├── InteractsWithConfigCallbacks.php │ ├── Manager.php │ ├── AvroSchemaRegistry.php │ ├── Producer.php │ ├── Committer.php │ ├── MessageConsumer.php │ └── MessageProducer.php ├── Support │ ├── InfiniteTimer.php │ ├── Timer.php │ └── Testing │ │ └── Fakes │ │ ├── ProducerFake.php │ │ ├── BuilderFake.php │ │ └── ConsumerFake.php ├── Commit │ ├── NativeSleeper.php │ ├── VoidCommitter.php │ ├── DefaultCommitterFactory.php │ ├── SeekToCurrentErrorCommitter.php │ ├── BatchCommitter.php │ ├── RetryableCommitter.php │ └── Committer.php ├── Events │ ├── CouldNotPublishMessage.php │ ├── MessageConsumed.php │ ├── MessagePublished.php │ ├── PublishingMessage.php │ ├── StartedConsumingMessage.php │ └── MessageSentToDLQ.php ├── Handlers │ ├── RetryStrategies │ │ └── DefaultRetryStrategy.php │ └── RetryableHandler.php ├── Message │ ├── Serializers │ │ ├── JsonSerializer.php │ │ └── AvroSerializer.php │ ├── KafkaAvroSchema.php │ ├── ConsumedMessage.php │ ├── Deserializers │ │ ├── JsonDeserializer.php │ │ └── AvroDeserializer.php │ ├── Message.php │ └── Registry │ │ └── AvroSchemaRegistry.php ├── MessageCounter.php ├── Console │ └── Commands │ │ ├── RestartConsumersCommand.php │ │ ├── KafkaConsumer │ │ └── Options.php │ │ └── ConsumerCommand.php ├── Config │ ├── Sasl.php │ └── RebalanceStrategy.php ├── Concerns │ ├── HandleConsumedMessage.php │ ├── PrepareMiddlewares.php │ ├── InteractsWithConfigCallbacks.php │ └── ManagesTransactions.php ├── Consumers │ ├── DispatchQueuedHandler.php │ └── CallableConsumer.php ├── Logger.php ├── Retryable.php ├── AbstractMessage.php ├── Facades │ └── Kafka.php ├── Providers │ └── LaravelKafkaServiceProvider.php ├── Factory.php └── Producers │ └── Producer.php ├── tests ├── Fakes │ ├── FakeSleeper.php │ ├── FakeConsumer.php │ └── FakeHandler.php ├── Consumers │ ├── SimpleQueueableHandler.php │ └── CallableConsumerTest.php ├── FailingHandler.php ├── Producers │ └── ProducerTest.php ├── Config │ └── SaslTest.php ├── Message │ ├── Deserializers │ │ └── JsonDeserializerTest.php │ ├── Serializers │ │ └── JsonSerializerTest.php │ ├── KafkaAvroSchemaTest.php │ └── MessageTest.php ├── Commit │ ├── BatchCommitterTest.php │ ├── SeekToCurrentErrorCommitterTest.php │ ├── RetryableCommitterTest.php │ └── CommitterFactoryTest.php ├── Console │ └── Consumers │ │ ├── KafkaConsumerCommandTest.php │ │ └── OptionsTest.php ├── FailingCommitter.php └── Handlers │ └── RetryableHandlerTest.php ├── rector.php ├── phpunit.xml ├── phpunit.xml.bak ├── LICENSE ├── composer.json ├── README.md ├── pint.json └── config └── kafka.php /.github/FUNDING.yml: -------------------------------------------------------------------------------- 1 | github: mateusjunges 2 | -------------------------------------------------------------------------------- /docs/testing/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Testing 3 | weight: 5 4 | --- -------------------------------------------------------------------------------- /docs/advanced-usage/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Advanced usage 3 | weight: 4 4 | --- 5 | -------------------------------------------------------------------------------- /docs/consuming-messages/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Consuming messages 3 | weight: 3 4 | --- -------------------------------------------------------------------------------- /docs/producing-messages/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Producing messages 3 | weight: 2 4 | --- -------------------------------------------------------------------------------- /art/sponsor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mateusjunges/laravel-kafka/HEAD/art/sponsor.png -------------------------------------------------------------------------------- /art/laravel-kafka.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mateusjunges/laravel-kafka/HEAD/art/laravel-kafka.png -------------------------------------------------------------------------------- /art/donation-qr-code.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/mateusjunges/laravel-kafka/HEAD/art/donation-qr-code.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | node_modules/ 3 | /.vagrant 4 | .phpunit.result.cache 5 | .idea 6 | tests/reports 7 | .php-cs-fixer.cache 8 | composer.lock 9 | .phpunit.cache/ 10 | .claude/ -------------------------------------------------------------------------------- /docs/_index.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: v2.9 3 | slogan: Use Kafka Producers and Consumers in your laravel app with ease! 4 | githubUrl: https://github.com/mateusjunges/laravel-kafka 5 | branch: master 6 | --- 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/custom.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Custom issue template 3 | about: Describe this issue template's purpose here. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /src/Exceptions/LaravelKafkaException.php: -------------------------------------------------------------------------------- 1 | 10 | ``` -------------------------------------------------------------------------------- /src/Contracts/Sleeper.php: -------------------------------------------------------------------------------- 1 | 13 | */ 14 | public function getContext(): array; 15 | } 16 | -------------------------------------------------------------------------------- /docs/requirements.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Requirements 3 | weight: 2 4 | --- 5 | 6 | Laravel Kafka requires **PHP 8.1+** and **Laravel 9+** 7 | 8 | This package also requires the `rdkafka` php extension, which you can install by following [this documentation](https://github.com/edenhill/librdkafka#installation) 9 | 10 | ```+parse 11 | 12 | ``` -------------------------------------------------------------------------------- /src/Contracts/CommitterFactory.php: -------------------------------------------------------------------------------- 1 | message->getMessageIdentifier(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Contracts/KafkaMessage.php: -------------------------------------------------------------------------------- 1 | message->getMessageIdentifier(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Events/PublishingMessage.php: -------------------------------------------------------------------------------- 1 | message->getMessageIdentifier(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Contracts/KafkaAvroSchemaRegistry.php: -------------------------------------------------------------------------------- 1 | message->getMessageIdentifier(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Contracts/RetryStrategy.php: -------------------------------------------------------------------------------- 1 | sleeps[] = $timeInMicroseconds; 14 | } 15 | 16 | public function getSleeps(): array 17 | { 18 | return $this->sleeps; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /tests/Fakes/FakeConsumer.php: -------------------------------------------------------------------------------- 1 | message = $message; 14 | } 15 | 16 | public function getMessage(): ?ConsumerMessage 17 | { 18 | return $this->message ?? null; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /docs/questions-and-issues.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Questions and issues 3 | weight: 4 4 | --- 5 | 6 | Find yourself stuck using the package? found a bug? Do you have general questions or suggestion to improve laravel kafka? Feel free to [create an issue in GitHub](https://github.com/mateusjunges/laravel-kafka/issues/new/choose) and i'll try to address it as soon as possible. 7 | 8 | If you have found a bug regarding security, please email me at [mateus@junges.dev](mailto:mateus@junges.dev) instead of using the issue tracker. -------------------------------------------------------------------------------- /src/Exceptions/Serializers/AvroSerializerException.php: -------------------------------------------------------------------------------- 1 | build(); 10 | ``` 11 | 12 | ### Consuming the kafka messages 13 | 14 | After building the consumer, you must call the `consume` method to consume the messages: 15 | 16 | ```php 17 | $consumer->consume(); 18 | ``` 19 | 20 | ```+parse 21 | 22 | ``` -------------------------------------------------------------------------------- /src/Commit/VoidCommitter.php: -------------------------------------------------------------------------------- 1 | messageIdentifier; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Handlers/RetryStrategies/DefaultRetryStrategy.php: -------------------------------------------------------------------------------- 1 | getBody(), JSON_THROW_ON_ERROR); 15 | 16 | return $message->withBody($body); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/Contracts/Consumer.php: -------------------------------------------------------------------------------- 1 | key ?? null; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Contracts/ProducerMessage.php: -------------------------------------------------------------------------------- 1 | context; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/MessageCounter.php: -------------------------------------------------------------------------------- 1 | messageCount++; 14 | 15 | return $this; 16 | } 17 | 18 | public function messagesCounted(): int 19 | { 20 | return $this->messageCount; 21 | } 22 | 23 | public function maxMessagesLimitReached(): bool 24 | { 25 | return $this->maxMessages === $this->messageCount; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/Fakes/FakeHandler.php: -------------------------------------------------------------------------------- 1 | lastMessage; 16 | } 17 | 18 | public function handle(ConsumerMessage $message, MessageConsumer $consumer): void 19 | { 20 | $this->lastMessage = $message; 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/Contracts/InteractsWithConfigCallbacks.php: -------------------------------------------------------------------------------- 1 | 12 | ``` 13 | 14 | The Kafka facade also provides methods to perform assertions over published messages, such as `assertPublished`, `assertPublishedOn` and `assertNothingPublished`. 15 | 16 | -------------------------------------------------------------------------------- /rector.php: -------------------------------------------------------------------------------- 1 | paths([ 9 | __DIR__.'/config', 10 | __DIR__.'/dev', 11 | __DIR__.'/src', 12 | __DIR__.'/tests', 13 | ]); 14 | 15 | // register a single rule 16 | $rectorConfig->rule(InlineConstructorDefaultToPropertyRector::class); 17 | 18 | // define sets of rules 19 | $rectorConfig->sets([ 20 | LevelSetList::UP_TO_PHP_81, 21 | ]); 22 | }; 23 | -------------------------------------------------------------------------------- /tests/FailingHandler.php: -------------------------------------------------------------------------------- 1 | timesInvoked++ < $this->timesToFail) { 17 | throw $this->exception; 18 | } 19 | } 20 | 21 | public function getTimesInvoked(): int 22 | { 23 | return $this->timesInvoked; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /docs/advanced-usage/sending-multiple-messages-with-the-same-producer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Sending multiple messages with the same producer 3 | weight: 9 4 | --- 5 | 6 | Sometimes you may want to send multiple messages without having to create the consumer 7 | 8 | 9 | ```php 10 | // In a service provider: 11 | 12 | \Junges\Kafka\Facades\Kafka::macro('myProducer', function () { 13 | return $this->publish('broker') 14 | ->onTopic('my-awesome-topic') 15 | ->withConfigOption('key', 'value'); 16 | }); 17 | ``` 18 | 19 | Now, you can call `\Junges\Kafka\Facades\Kafka::myProducer()`, which will always apply the configs you defined in your service provider. 20 | 21 | 22 | ```+parse 23 | 24 | ``` -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: "" 5 | labels: "feature request" 6 | assignees: mateusjunges 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /src/Contracts/Manager.php: -------------------------------------------------------------------------------- 1 | $messages */ 19 | public function shouldReceiveMessages(array $messages): self; 20 | } 21 | -------------------------------------------------------------------------------- /src/Exceptions/Transactions/TransactionShouldBeRetriedException.php: -------------------------------------------------------------------------------- 1 | getMessage()), 16 | $baseException->getCode(), 17 | $baseException 18 | ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /docs/advanced-usage/setting-global-configuration.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Setting global configurations 3 | weight: 8 4 | --- 5 | 6 | At this moment, there is no way of setting global configuration for producers/consumers, but you can use laravel `macro` functionality 7 | to achieve that. Here's an example: 8 | 9 | 10 | ```php 11 | // In a service provider: 12 | 13 | \Junges\Kafka\Facades\Kafka::macro('myProducer', function () { 14 | return $this->publish('broker') 15 | ->onTopic('my-awesome-topic') 16 | ->withConfigOption('key', 'value'); 17 | }); 18 | ``` 19 | 20 | Now, you can call `\Junges\Kafka\Facades\Kafka::myProducer()`, which will always apply the configs you defined in your service provider. 21 | 22 | 23 | ```+parse 24 | 25 | ``` -------------------------------------------------------------------------------- /docs/advanced-usage/stop-consumer-after-last-message.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Stop consumer after last messages 3 | weight: 6 4 | --- 5 | 6 | Stopping consumers after the last received message is useful if you want to consume all messages from a given 7 | topic and stop your consumer when the last message arrives. 8 | 9 | You can do it by adding a call to `stopAfterLastMessage` method when creating your consumer: 10 | 11 | This is particularly useful when using signal handlers. 12 | 13 | ```php 14 | $consumer = \Junges\Kafka\Facades\Kafka::consumer(['topic']) 15 | ->withConsumerGroupId('group') 16 | ->stopAfterLastMessage() 17 | ->withHandler(new Handler) 18 | ->build(); 19 | 20 | $consumer->consume(); 21 | ``` 22 | 23 | ```+parse 24 | 25 | ``` -------------------------------------------------------------------------------- /src/Console/Commands/RestartConsumersCommand.php: -------------------------------------------------------------------------------- 1 | forever('laravel-kafka:consumer:restart', $this->currentTime()); 22 | $this->info('Kafka consumers restart signal sent.'); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Exceptions/Transactions/TransactionShouldBeAbortedException.php: -------------------------------------------------------------------------------- 1 | getMessage()), 16 | $baseException->getCode(), 17 | $baseException 18 | ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Exceptions/Transactions/TransactionFatalErrorException.php: -------------------------------------------------------------------------------- 1 | getMessage()), 16 | $baseException->getCode(), 17 | $baseException 18 | ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /docs/consuming-messages/subscribing-to-kafka-topics.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Subscribing to kafka topics 3 | weight: 2 4 | --- 5 | 6 | ```+parse 7 | 8 | ``` 9 | 10 | With a consumer created, you can subscribe to a kafka topic using the `subscribe` method: 11 | 12 | ```php 13 | use Junges\Kafka\Facades\Kafka; 14 | 15 | $consumer = Kafka::consumer()->subscribe('topic'); 16 | ``` 17 | 18 | Of course, you can subscribe to more than one topic at once, either using an array of topics or specifying one by one: 19 | 20 | ```php 21 | use Junges\Kafka\Facades\Kafka; 22 | 23 | $consumer = Kafka::consumer()->subscribe('topic-1', 'topic-2', 'topic-n'); 24 | 25 | // Or, using array: 26 | $consumer = Kafka::consumer()->subscribe([ 27 | 'topic-1', 28 | 'topic-2', 29 | 'topic-n' 30 | ]); 31 | ``` -------------------------------------------------------------------------------- /docs/consuming-messages/using-regex-to-subscribe-to-kafka-topics.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Using regex to subscribe to kafka topics 3 | weight: 2 4 | --- 5 | 6 | Kafka allows you to subscribe to topics using regex, and regex pattern matching is automatically performed for topics prefixed with `^` (e.g. `^myPfx[0-9]_.*`). 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | The consumer will see the new topics on its next periodic metadata refresh which is controlled by the `topic.metadata.refresh.interval.ms` 13 | 14 | To subscribe to topics using regex, you can simply pass the regex you want to use to the `subscribe` method: 15 | 16 | ```php 17 | \Junges\Kafka\Facades\Kafka::consumer() 18 | ->subscribe('^myPfx_.*') 19 | ->withHandler(...) 20 | ``` 21 | 22 | This pattern will match any topics that starts with `myPfx_`. -------------------------------------------------------------------------------- /src/Config/Sasl.php: -------------------------------------------------------------------------------- 1 | username; 17 | } 18 | 19 | public function getPassword(): string 20 | { 21 | return $this->password; 22 | } 23 | 24 | public function getMechanisms(): string 25 | { 26 | return $this->mechanisms; 27 | } 28 | 29 | public function getSecurityProtocol(): string 30 | { 31 | return $this->securityProtocol; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /docs/producing-messages/publishing-to-kafka.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Publishing to kafka 3 | weight: 5 4 | --- 5 | 6 | After configuring all your message options, you must use the send method, to send the message to kafka. 7 | 8 | ```php 9 | use Junges\Kafka\Facades\Kafka; 10 | 11 | /** @var \Junges\Kafka\Producers\Builder $producer */ 12 | $producer = Kafka::publish('broker') 13 | ->onTopic('topic') 14 | ->withConfigOptions(['key' => 'value']) 15 | ->withKafkaKey('kafka-key') 16 | ->withHeaders(['header-key' => 'header-value']); 17 | 18 | $producer->send(); 19 | ``` 20 | 21 | If you want to send multiple messages, consider using the async producer instead. The default `send` method is recommended for low-throughput systems only, as it 22 | flushes the producer after every message that is sent. 23 | 24 | ```+parse 25 | 26 | ``` -------------------------------------------------------------------------------- /src/Support/Timer.php: -------------------------------------------------------------------------------- 1 | startTime >= $this->timeoutInMilliseconds / 1000; 17 | } 18 | 19 | /** Starts a timer, Captures a start time and Captures a timeout in milliseconds */ 20 | public function start(int $timeoutInMilliseconds): void 21 | { 22 | $this->startTime = microtime(true); 23 | $this->timeoutInMilliseconds = $timeoutInMilliseconds; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /docs/advanced-usage/middlewares.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Middlewares 3 | weight: 5 4 | --- 5 | 6 | Middlewares provides a convenient way to filter and inspecting your Kafka messages. To write a middleware in this package, you can use the `withMiddleware` method. The middleware is a callable in which the first argument is the message itself and the second one is the next handler. The middlewares get executed in the order they are defined: 7 | 8 | ```php 9 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 10 | ->withMiddleware(function(\Junges\Kafka\Message\ConsumedMessage $message, callable $next) { 11 | // Perform some work here 12 | return $next($message); 13 | }); 14 | ``` 15 | 16 | You can add as many middlewares as you need, so you can divide different tasks into different middlewares. 17 | 18 | ```+parse 19 | 20 | ``` -------------------------------------------------------------------------------- /docs/advanced-usage/replacing-default-serializer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Replacing the default serializer/deserializer 3 | weight: 1 4 | --- 5 | 6 | The default Serializer is resolved using the `MessageSerializer` and `MessageDeserializer` contracts. Out of the box, the `Json` serializers are used. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | To set the default serializer you can bind the `MessageSerializer` and `MessageDeserializer` contracts to any class which implements this interfaces. 13 | 14 | Open your `AppServiceProvider` class and add this lines to the `register` method: 15 | 16 | ```php 17 | $this->app->bind(\Junges\Kafka\Contracts\MessageSerializer::class, function () { 18 | return new MyCustomSerializer(); 19 | }); 20 | 21 | $this->app->bind(\Junges\Kafka\Contracts\MessageDeserializer::class, function() { 22 | return new MyCustomDeserializer(); 23 | }); 24 | ``` -------------------------------------------------------------------------------- /docs/testing/assert-nothing-published.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Assert nothing published 3 | weight: 4 4 | --- 5 | 6 | You can assert that nothing was published at all, using the `assertNothingPublished`: 7 | 8 | ```php 9 | use PHPUnit\Framework\TestCase; 10 | use Junges\Kafka\Facades\Kafka; 11 | use Junges\Kafka\Message\Message; 12 | 13 | class MyTest extends TestCase 14 | { 15 | public function testWithSpecificTopic() 16 | { 17 | Kafka::fake(); 18 | 19 | if (false) { 20 | $producer = Kafka::publish('broker') 21 | ->onTopic('some-kafka-topic') 22 | ->withHeaders(['key' => 'value']) 23 | ->withBodyKey('key', 'value'); 24 | 25 | $producer->send(); 26 | } 27 | 28 | Kafka::assertNothingPublished(); 29 | } 30 | } 31 | ``` 32 | 33 | ```+parse 34 | 35 | ``` 36 | -------------------------------------------------------------------------------- /src/Handlers/RetryableHandler.php: -------------------------------------------------------------------------------- 1 | sleeper, $this->retryStrategy->getMaximumRetries(), null); 18 | $retryable->retry( 19 | fn () => ($this->handler)($message), 20 | 0, 21 | $this->retryStrategy->getInitialDelay(), 22 | $this->retryStrategy->useExponentialBackoff() 23 | ); 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /docs/consuming-messages/creating-consumer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Creating a kafka consumer 3 | weight: 1 4 | --- 5 | 6 | If your application needs to read messages from a Kafka topic, you must create a consumer object, subscribe to the appropriate topic and start receiving messages. 7 | 8 | To create a consumer using this package you can use the `consumer` method, on Kafka facade: 9 | 10 | ```php 11 | use Junges\Kafka\Facades\Kafka; 12 | 13 | $consumer = Kafka::consumer(); 14 | ``` 15 | 16 | This method also allows you to specify the `topics` it should consume, the `broker` and the consumer `group id`: 17 | 18 | ```php 19 | use Junges\Kafka\Facades\Kafka; 20 | 21 | $consumer = Kafka::consumer(['topic-1', 'topic-2'], 'group-id', 'broker'); 22 | ``` 23 | 24 | This method returns a `Junges\Kafka\Consumers\ConsumerBuilder::class` instance, and you can use it to configure your consumer. 25 | 26 | ```+parse 27 | 28 | ``` -------------------------------------------------------------------------------- /src/Concerns/HandleConsumedMessage.php: -------------------------------------------------------------------------------- 1 | $this->wrapMiddleware($middleware, $consumer), $middlewares); 20 | $middlewares = array_reverse($middlewares); 21 | 22 | foreach ($middlewares as $middleware) { 23 | $handler = $middleware($handler); 24 | } 25 | 26 | $handler($message, $consumer); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | tests 13 | 14 | 15 | 16 | 17 | 18 | 19 | src/ 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /src/Message/KafkaAvroSchema.php: -------------------------------------------------------------------------------- 1 | schemaName; 19 | } 20 | 21 | public function getVersion(): int 22 | { 23 | return $this->version; 24 | } 25 | 26 | public function setDefinition(AvroSchema $definition): void 27 | { 28 | $this->definition = $definition; 29 | } 30 | 31 | public function getDefinition(): ?AvroSchema 32 | { 33 | return $this->definition; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /docs/advanced-usage/stopping-a-consumer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Stop consumer on demand 3 | weight: 6 4 | --- 5 | 6 | Sometimes, you may want to stop your consumer based on a given message or any other condition. 7 | 8 | You can do it by adding a calling `stopConsuming()` method on the `MessageConsumer` instance that is passed as the 9 | second argument of your message handler: 10 | 11 | ```php 12 | $consumer = \Junges\Kafka\Facades\Kafka::consumer(['topic']) 13 | ->withConsumerGroupId('group') 14 | ->stopAfterLastMessage() 15 | ->withHandler(static function (\Junges\Kafka\Contracts\ConsumerMessage $message, \Junges\Kafka\Contracts\MessageConsumer $consumer) { 16 | if ($someCondition) { 17 | $consumer->stopConsuming(); 18 | } 19 | }) 20 | ->build(); 21 | 22 | $consumer->consume(); 23 | ``` 24 | 25 | The `onStopConsuming` callback will be executed before stopping your consumer. 26 | 27 | ```+parse 28 | 29 | ``` -------------------------------------------------------------------------------- /docs/testing/assert-published-times.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Assert published times 3 | weight: 5 4 | --- 5 | 6 | Sometimes, you need to assert that Kafka has published a given number of messages. For that, you can use the `assertPublishedTimes` method: 7 | 8 | ```php 9 | use PHPUnit\Framework\TestCase; 10 | use Junges\Kafka\Facades\Kafka; 11 | use Junges\Kafka\Message\Message; 12 | 13 | class MyTest extends TestCase 14 | { 15 | public function testWithSpecificTopic() 16 | { 17 | Kafka::fake(); 18 | 19 | Kafka::publish('broker') 20 | ->onTopic('topic') 21 | ->withHeaders(['key' => 'value']) 22 | ->withBodyKey('key', 'value'); 23 | 24 | Kafka::publish('broker') 25 | ->onTopic('topic') 26 | ->withHeaders(['key' => 'value']) 27 | ->withBodyKey('key', 'value'); 28 | 29 | Kafka::assertPublishedTimes(2); 30 | } 31 | } 32 | ``` 33 | 34 | ```+parse 35 | 36 | ``` -------------------------------------------------------------------------------- /src/Commit/DefaultCommitterFactory.php: -------------------------------------------------------------------------------- 1 | getMaxCommitRetries() 24 | ), 25 | $this->messageCounter, 26 | $config->getCommit() 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /docs/testing/assert-published-on-times.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Assert published on times 3 | weight: 6 4 | --- 5 | 6 | To assert that messages were published on a given topic a given number of times, you can use the `assertPublishedOnTimes` method: 7 | ```php 8 | use PHPUnit\Framework\TestCase; 9 | use Junges\Kafka\Facades\Kafka; 10 | use Junges\Kafka\Message\Message; 11 | 12 | class MyTest extends TestCase 13 | { 14 | public function testWithSpecificTopic() 15 | { 16 | Kafka::fake(); 17 | 18 | Kafka::publish('broker') 19 | ->onTopic('topic') 20 | ->withHeaders(['key' => 'value']) 21 | ->withBodyKey('key', 'value'); 22 | 23 | Kafka::publish('broker') 24 | ->onTopic('topic') 25 | ->withHeaders(['key' => 'value']) 26 | ->withBodyKey('key', 'value'); 27 | 28 | Kafka::assertPublishedOnTimes('some-kafka-topic', 2); 29 | } 30 | } 31 | ``` 32 | 33 | ```+parse 34 | 35 | ``` -------------------------------------------------------------------------------- /src/Concerns/PrepareMiddlewares.php: -------------------------------------------------------------------------------- 1 | new $middleware, 17 | $middleware instanceof Middleware => $middleware, 18 | is_callable($middleware) => $middleware, 19 | default => throw new LogicException('Invalid middleware.') 20 | }; 21 | 22 | return static fn (callable $handler) => static fn ($message) => $middleware($message, fn ($message) => $handler($message, $consumer)); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Message/ConsumedMessage.php: -------------------------------------------------------------------------------- 1 | topicName, 21 | $this->partition, 22 | $this->headers, 23 | $this->body, 24 | $this->key 25 | ); 26 | } 27 | 28 | public function getOffset(): ?int 29 | { 30 | return $this->offset; 31 | } 32 | 33 | public function getTimestamp(): ?int 34 | { 35 | return $this->timestamp; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: "" 5 | labels: "bug" 6 | assignees: mateusjunges 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Steps to reproduce the behavior: 15 | 1. Go to '...' 16 | 2. Click on '....' 17 | 3. Scroll down to '....' 18 | 4. See error 19 | 20 | **Expected behavior** 21 | A clear and concise description of what you expected to happen. 22 | 23 | **Screenshots** 24 | If applicable, add screenshots to help explain your problem. 25 | 26 | **Desktop (please complete the following information):** 27 | - OS: [e.g. iOS] 28 | - Browser [e.g. chrome, safari] 29 | - Version [e.g. 22] 30 | 31 | **Smartphone (please complete the following information):** 32 | - Device: [e.g. iPhone6] 33 | - OS: [e.g. iOS8.1] 34 | - Browser [e.g. stock browser, safari] 35 | - Version [e.g. 22] 36 | 37 | **Additional context** 38 | Add any other context about the problem here. 39 | -------------------------------------------------------------------------------- /src/Exceptions/CouldNotPublishMessage.php: -------------------------------------------------------------------------------- 1 | setErrorCode($code); 21 | 22 | return $exception; 23 | } 24 | 25 | public function setErrorCode(int $code): self 26 | { 27 | $this->kafkaErrorCode = $code; 28 | 29 | return $this; 30 | } 31 | 32 | public function getKafkaErrorCode(): int 33 | { 34 | return $this->kafkaErrorCode; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /tests/Producers/ProducerTest.php: -------------------------------------------------------------------------------- 1 | mockKafkaProducer(); 18 | $producer = new Producer(new Config('broker', ['test-topic']), new JsonSerializer); 19 | $payload = ['key' => 'value']; 20 | 21 | $message = new Message( 22 | body: $payload, 23 | ); 24 | $message->onTopic('test-topic'); 25 | $producer->produce($message); 26 | $producer->produce($message); 27 | 28 | $this->assertSame($payload, $message->getBody()); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/Message/Deserializers/JsonDeserializer.php: -------------------------------------------------------------------------------- 1 | getBody() !== null 16 | ? json_decode((string) $message->getBody(), true, 512, JSON_THROW_ON_ERROR) 17 | : null; 18 | 19 | return new ConsumedMessage( 20 | topicName: $message->getTopicName(), 21 | partition: $message->getPartition(), 22 | headers: $message->getHeaders(), 23 | body: $body, 24 | key: $message->getKey(), 25 | offset: $message->getOffset(), 26 | timestamp: $message->getTimestamp() 27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /phpunit.xml.bak: -------------------------------------------------------------------------------- 1 | 2 | 11 | 12 | 13 | src/ 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | tests 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /docs/advanced-usage/before-callbacks.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Before and after callbacks 3 | weight: 8 4 | --- 5 | 6 | You can call pre-defined callbacks **Before** and **After** consuming messages. As an example, you can use this to make your consumer to wait while in maintenance mode. 7 | The callbacks get executed in the order they are defined, and they receive a `\Junges\Kafka\Contracts\MessageConsumer` as argument: 8 | 9 | ```php 10 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 11 | ->beforeConsuming(function(\Junges\Kafka\Contracts\MessageConsumer $consumer) { 12 | while (app()->isDownForMaintenance()) { 13 | sleep(1); 14 | } 15 | }) 16 | ->afterConsuming(function (\Junges\Kafka\Contracts\MessageConsumer $consumer) { 17 | // Runs after consuming the message 18 | }) 19 | ``` 20 | 21 | These callbacks are not middlewares, so you can not interact with the consumed message. 22 | You can add as many callback as you need, so you can divide different tasks into 23 | different callbacks. 24 | 25 | ```+parse 26 | 27 | ``` -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Mateus Junges 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /docs/advanced-usage/custom-loggers.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Writing custom loggers 3 | weight: 7 4 | --- 5 | 6 | Sometimes you need more control over your logging setup. From `v1.10.1` of this package, you can define your own `Logger` implementation. This means that you have the flexibility to log to different types of storage, such as file or cloud-based logging service. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | This can be useful for organizations that need to comply with data privacy regulations, such as the General Data Protection Regulation (GDPR). For example, if an exception occurs and gets logged, it might contain sensitive information such as personally identifiable information (PII). Implementing a custom logger, you can now configure it to automatically redact this information before it gets written to the log. 13 | 14 | A `Logger` is any class that implements the `\Junges\Kafka\Contracts\Logger` interface, and it only require that you define a `error` method. 15 | 16 | After creating your Logger, you need to [bind it to the Laravel container](https://laravel.com/docs/9.x/container#binding-basics): 17 | 18 | ```php 19 | $this->app->bind(Logger::class, function ($app) { 20 | return new MyCustomLogger(); 21 | }); 22 | ``` -------------------------------------------------------------------------------- /src/Contracts/AvroSchemaRegistry.php: -------------------------------------------------------------------------------- 1 | */ 20 | public function getTopicSchemaMapping(): array; 21 | 22 | /** @throws SchemaRegistryException */ 23 | public function getBodySchemaForTopic(string $topicName): KafkaAvroSchemaRegistry; 24 | 25 | /** @throws SchemaRegistryException */ 26 | public function getKeySchemaForTopic(string $topicName): KafkaAvroSchemaRegistry; 27 | 28 | /** @throws SchemaRegistryException */ 29 | public function hasBodySchemaForTopic(string $topicName): bool; 30 | 31 | /** @throws SchemaRegistryException */ 32 | public function hasKeySchemaForTopic(string $topicName): bool; 33 | } 34 | -------------------------------------------------------------------------------- /src/Support/Testing/Fakes/ProducerFake.php: -------------------------------------------------------------------------------- 1 | producerCallback = $callback; 30 | 31 | return $this; 32 | } 33 | 34 | public function produce(ProducerMessage $message): bool 35 | { 36 | if ($this->producerCallback !== null) { 37 | $callback = $this->producerCallback; 38 | $callback($message); 39 | } 40 | 41 | return true; 42 | } 43 | 44 | public function flush(): int 45 | { 46 | return 1; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Consumers/DispatchQueuedHandler.php: -------------------------------------------------------------------------------- 1 | handleConsumedMessage( 32 | message: $this->message, 33 | handler: $this->handler, 34 | middlewares: $this->middlewares 35 | ); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /tests/Config/SaslTest.php: -------------------------------------------------------------------------------- 1 | assertEquals('username', $sasl->getUsername()); 21 | } 22 | 23 | #[Test] 24 | public function get_password(): void 25 | { 26 | $sasl = new Sasl( 27 | username: 'username', 28 | password: 'password', 29 | mechanisms: 'mechanisms' 30 | ); 31 | 32 | $this->assertEquals('password', $sasl->getPassword()); 33 | } 34 | 35 | #[Test] 36 | public function get_mechanisms(): void 37 | { 38 | $sasl = new Sasl( 39 | username: 'username', 40 | password: 'password', 41 | mechanisms: 'mechanisms' 42 | ); 43 | 44 | $this->assertEquals('mechanisms', $sasl->getMechanisms()); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Commit/SeekToCurrentErrorCommitter.php: -------------------------------------------------------------------------------- 1 | committer->commitMessage($message, $success); 17 | 18 | return; 19 | } 20 | 21 | $currentSubscriptions = $this->consumer->getSubscription(); 22 | $this->consumer->unsubscribe(); 23 | $this->consumer->subscribe($currentSubscriptions); 24 | } 25 | 26 | public function commitDlq(Message $message): void 27 | { 28 | $this->committer->commitDlq($message); 29 | } 30 | 31 | public function commit(mixed $messageOrOffsets = null): void 32 | { 33 | $this->committer->commit($messageOrOffsets); 34 | } 35 | 36 | public function commitAsync(mixed $messageOrOffsets = null): void 37 | { 38 | $this->committer->commitAsync($messageOrOffsets); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /docs/advanced-usage/graceful-shutdown.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Graceful shutdown 3 | weight: 2 4 | --- 5 | 6 | Stopping consumers is very useful if you want to ensure you don't kill a process halfway through processing a consumed message. 7 | 8 | Consumers automatically listen to the `SIGTERM`, `SIGINT` and `SIQUIT` signals, which means you can easily stop your consumers using those signals. 9 | 10 | ```+parse 11 | 12 | ``` 13 | 14 | ### Running callbacks when the consumer stops 15 | If your app requires that you run sum sort of processing when the consumers stop processing messages, you can use the `onStopConsume` method, available on the `\Junges\Kafka\Contracts\CanConsumeMessages` interface. This method accepts a `Closure` that will run once your consumer stops consuming. 16 | 17 | ```php 18 | use Junges\Kafka\Facades\Kafka; 19 | 20 | $consumer = Kafka::consumer(['topic']) 21 | ->withConsumerGroupId('group') 22 | ->withHandler(new Handler) 23 | ->onStopConsuming(static function () { 24 | // Do something when the consumer stop consuming messages 25 | }) 26 | ->build(); 27 | 28 | $consumer->consume(); 29 | ``` 30 | 31 | > This features requires [ Process Control Extension ](https://www.php.net/manual/en/book.pcntl.php) to be installed. 32 | -------------------------------------------------------------------------------- /src/Logger.php: -------------------------------------------------------------------------------- 1 | setFormatter(new JsonFormatter); 22 | $handler->pushProcessor(new UidProcessor(32)); 23 | 24 | $this->logger = new MonologLogger('PHP-KAFKA-CONSUMER-ERROR'); 25 | $this->logger->pushHandler($handler); 26 | $this->logger->pushProcessor(function ($record) { 27 | $record['datetime']->format('c'); 28 | 29 | return $record; 30 | }); 31 | } 32 | 33 | /** Log an error message. */ 34 | public function error(Message $message, ?Throwable $e = null, string $prefix = 'ERROR'): void 35 | { 36 | $this->logger->error("[{$prefix}] Error to consume message", [ 37 | 'message' => $message, 38 | 'throwable' => $e, 39 | ]); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Retryable.php: -------------------------------------------------------------------------------- 1 | maximumRetries 28 | && (is_null($this->retryableErrors) || in_array($exception->getCode(), $this->retryableErrors)) 29 | ) { 30 | $this->sleeper->sleep((int) ($delayInSeconds * 1e6)); 31 | $this->retry( 32 | $function, 33 | ++$currentRetries, 34 | $exponentially === true ? $delayInSeconds * 2 : $delayInSeconds 35 | ); 36 | 37 | return; 38 | } 39 | 40 | throw $exception; 41 | } 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /.github/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Contributions are **welcome**! 4 | 5 | We accept contributions via Pull Requests on [Github](https://github.com/mateusjunges/laravel-kafka). 6 | 7 | ## Pull Requests 8 | 9 | - **[PSR-2 Coding Standard](https://github.com/php-fig/fig-standards/blob/master/accepted/PSR-2-coding-style-guide.md)** - The easiest way to apply the conventions is to install [PHP Code Sniffer](http://pear.php.net/package/PHP_CodeSniffer). 10 | 11 | - **Document any change in behaviour** - Make sure the `README.md` and any other relevant documentation are kept up to date. 12 | 13 | - **Add tests** - Your patch won't be accepted if it doesn't have tests. 14 | 15 | - **DO NOT use conventional commits** - Your pull request will be immediatelly closed if you prefix your commits with `feat:`, `chore:`, `fix:` or anything like that. 16 | 17 | - **Create feature branches** - Pull requests from your master branch won't be accepted. 18 | 19 | - **One pull request per feature** - If you want to do more than one thing, create multiple pull requests. 20 | 21 | - **Send coherent history** - Make sure each individual commit in your pull request is meaningful. If you had to make multiple intermediate commits while developing, please [squash them](http://www.git-scm.com/book/en/v2/Git-Tools-Rewriting-History#Changing-Multiple-Commit-Messages) before submitting. 22 | -------------------------------------------------------------------------------- /tests/Message/Deserializers/JsonDeserializerTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(ConsumerMessage::class); 17 | $message->expects($this->exactly(2))->method('getBody')->willReturn('{"name":"foo"}'); 18 | $deserializer = new JsonDeserializer; 19 | $result = $deserializer->deserialize($message); 20 | 21 | $this->assertInstanceOf(ConsumerMessage::class, $result); 22 | $this->assertEquals(['name' => 'foo'], $result->getBody()); 23 | } 24 | 25 | #[Test] 26 | public function deserialize_non_json(): void 27 | { 28 | $message = $this->getMockForAbstractClass(ConsumerMessage::class); 29 | $message->expects($this->exactly(2))->method('getBody')->willReturn('test'); 30 | $deserializer = new JsonDeserializer; 31 | 32 | $this->expectException(JsonException::class); 33 | 34 | $deserializer->deserialize($message); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /.github/workflows/run-tests.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: ['push', 'pull_request'] 4 | 5 | jobs: 6 | ci: 7 | runs-on: ubuntu-22.04 8 | 9 | strategy: 10 | matrix: 11 | php: [8.4, 8.3, 8.2] 12 | laravel: [12.*, 11.*, 10.*] 13 | dependency-version: [prefer-stable] 14 | include: 15 | - laravel: 10.* 16 | testbench: 8.* 17 | - laravel: 11.* 18 | testbench: 9.* 19 | - laravel: 12.* 20 | testbench: 10.* 21 | 22 | name: CI - PHP ${{ matrix.php }} - Laravel ${{ matrix.laravel }} - Testbench ${{ matrix.testbench }} (${{ matrix.dependency-version }}) 23 | 24 | steps: 25 | 26 | - name: Checkout 27 | uses: actions/checkout@v2 28 | 29 | - name: Setup PHP 30 | uses: shivammathur/setup-php@v2 31 | with: 32 | php-version: ${{ matrix.php }} 33 | extensions: mbstring, zip, rdkafka 34 | tools: prestissimo 35 | coverage: pcov 36 | 37 | - name: Install Composer dependencies 38 | run: | 39 | composer require "laravel/framework:${{ matrix.laravel }}" "orchestra/testbench:${{ matrix.testbench }}" --no-interaction --no-update 40 | composer update --${{ matrix.dependency-version }} --no-interaction --prefer-dist 41 | 42 | - name: PHPUnit Testing 43 | run: vendor/bin/phpunit -------------------------------------------------------------------------------- /tests/Message/Serializers/JsonSerializerTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(ProducerMessage::class); 17 | $message->expects($this->once())->method('getBody')->willReturn(['name' => 'foo']); 18 | $message->expects($this->once())->method('withBody')->with('{"name":"foo"}')->willReturn($message); 19 | 20 | $serializer = $this->getMockForAbstractClass(JsonSerializer::class); 21 | 22 | $this->assertSame($message, $serializer->serialize($message)); 23 | } 24 | 25 | #[Test] 26 | public function serialize_throws_exception(): void 27 | { 28 | $message = $this->getMockForAbstractClass(ProducerMessage::class); 29 | $message->expects($this->once())->method('getBody')->willReturn(chr(255)); 30 | 31 | $serializer = $this->getMockForAbstractClass(JsonSerializer::class); 32 | 33 | $this->expectException(JsonException::class); 34 | 35 | $serializer->serialize($message); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /docs/testing/assert-published.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Assert Published 3 | weight: 2 4 | --- 5 | 6 | When you want to assert that a message was published into kafka, you can make use of the `assertPublished` method: 7 | 8 | ```php 9 | use Junges\Kafka\Facades\Kafka; 10 | use PHPUnit\Framework\TestCase; 11 | 12 | class MyTest extends TestCase 13 | { 14 | public function testMyAwesomeApp() 15 | { 16 | Kafka::fake(); 17 | 18 | $producer = Kafka::publish('broker') 19 | ->onTopic('topic') 20 | ->withHeaders(['key' => 'value']) 21 | ->withBodyKey('foo', 'bar'); 22 | 23 | $producer->send(); 24 | 25 | Kafka::assertPublished($producer->getMessage()); 26 | } 27 | } 28 | ``` 29 | 30 | You can also use `assertPublished` without passing the message argument: 31 | 32 | ```php 33 | use Junges\Kafka\Facades\Kafka; 34 | use PHPUnit\Framework\TestCase; 35 | 36 | class MyTest extends TestCase 37 | { 38 | public function testMyAwesomeApp() 39 | { 40 | Kafka::fake(); 41 | 42 | Kafka::publish('broker') 43 | ->onTopic('topic') 44 | ->withHeaders(['key' => 'value']) 45 | ->withBodyKey('foo', 'bar'); 46 | 47 | 48 | Kafka::assertPublished(); 49 | } 50 | } 51 | ``` 52 | 53 | ```+parse 54 | 55 | ``` -------------------------------------------------------------------------------- /docs/introduction.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Introduction 3 | weight: 1 4 | --- 5 | 6 | Do you use Kafka in your laravel projects? Every package I've seen until today, including some built by myself, does not provide a nice syntax usage syntax or, when it does, the test process with these packages are very painful. 7 | 8 | This package provides a nice way of producing and consuming kafka messages in your Laravel projects. 9 | 10 | ## We have badges! 11 | 12 |
13 | Latest version on packagist 14 | Total downloads 15 | License 16 | Continuous integration 17 | PHP Version 18 |
19 | 20 | 21 | ```+parse 22 | 23 | ``` 24 | 25 | -------------------------------------------------------------------------------- /src/Commit/BatchCommitter.php: -------------------------------------------------------------------------------- 1 | commits++; 22 | 23 | if ($this->maxMessagesLimitReached() || $this->commits >= $this->batchSize) { 24 | $this->committer->commitMessage($message, $success); 25 | $this->commits = 0; 26 | } 27 | } 28 | 29 | public function commitDlq(Message $message): void 30 | { 31 | $this->committer->commitDlq($message); 32 | $this->commits = 0; 33 | } 34 | 35 | public function commit(mixed $messageOrOffsets = null): void 36 | { 37 | $this->committer->commit($messageOrOffsets); 38 | } 39 | 40 | public function commitAsync(mixed $messageOrOffsets = null): void 41 | { 42 | $this->committer->commitAsync($messageOrOffsets); 43 | } 44 | 45 | private function maxMessagesLimitReached(): bool 46 | { 47 | return $this->messageCounter->maxMessagesLimitReached(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Config/RebalanceStrategy.php: -------------------------------------------------------------------------------- 1 | */ 32 | public static function values(): array 33 | { 34 | return array_map(fn (self $case) => $case->value, self::cases()); 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/AbstractMessage.php: -------------------------------------------------------------------------------- 1 | topicName = $topic; 21 | 22 | return $this; 23 | } 24 | 25 | public function getTopicName(): ?string 26 | { 27 | return $this->topicName; 28 | } 29 | 30 | public function getPartition(): ?int 31 | { 32 | return $this->partition; 33 | } 34 | 35 | public function getBody() 36 | { 37 | return $this->body; 38 | } 39 | 40 | public function getHeaders(): ?array 41 | { 42 | return $this->headers; 43 | } 44 | 45 | public function getKey(): mixed 46 | { 47 | return $this->key; 48 | } 49 | 50 | /** @throws MessageIdNotSet */ 51 | public function getMessageIdentifier(): string 52 | { 53 | if (! is_string($this->getHeaders()[config('kafka.message_id_key')])) { 54 | throw new MessageIdNotSet; 55 | } 56 | 57 | return $this->getHeaders()[config('kafka.message_id_key')]; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /docs/producing-messages/producing-messages.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Producing messages 3 | weight: 1 4 | --- 5 | 6 | To publish your messages to Kafka, you can use the `publish` method, of `Junges\Kafka\Facades\Kafka` class: 7 | 8 | ```php 9 | use Junges\Kafka\Facades\Kafka; 10 | 11 | Kafka::publish('broker')->onTopic('topic-name') 12 | ``` 13 | 14 | This method returns a `ProducerBuilder` instance, which contains a few methods to configure your kafka producer. 15 | The following lines describes these methods. 16 | 17 | If you are going to produce a lot of messages to different topics, please use the `asyncPublish` method on the `Junges\Kafka\Facades\Kafka` class: 18 | 19 | ```php 20 | use Junges\Kafka\Facades\Kafka; 21 | 22 | Kafka::asyncPublish('broker')->onTopic('topic-name') 23 | ``` 24 | 25 | The main difference is that the Async Producer is a singleton and will only flush the producer when the application is shutting down, instead of after each send. 26 | This reduces the overhead when you want to send a lot of messages in your request handlers. 27 | 28 | ```+parse 29 | 30 | ``` 31 | 32 | When doing async publishing, the builder is stored in memory during the entire request. If you need to use a fresh producer, you may use the `fresh` method 33 | available on the `Kafka` facade (added in v2.2.0). This method will return a fresh Kafka Manager, which you can use to produce messages with a newly created producer builder. 34 | 35 | 36 | ```php 37 | use Junges\Kafka\Facades\Kafka; 38 | 39 | Kafka::fresh() 40 | ->asyncPublish('broker') 41 | ->onTopic('topic-name') 42 | ``` -------------------------------------------------------------------------------- /tests/Commit/BatchCommitterTest.php: -------------------------------------------------------------------------------- 1 | createMock(Committer::class); 18 | $committer 19 | ->expects($this->exactly(2)) 20 | ->method('commitMessage'); 21 | 22 | $batchSize = 3; 23 | $messageCounter = new MessageCounter(42); 24 | $batchCommitter = new BatchCommitter($committer, $messageCounter, $batchSize); 25 | 26 | for ($i = 0; $i < 7; $i++) { 27 | $batchCommitter->commitMessage(new Message, true); 28 | } 29 | } 30 | 31 | #[Test] 32 | public function should_always_commit_dlq(): void 33 | { 34 | $committer = $this->createMock(Committer::class); 35 | $committer 36 | ->expects($this->exactly(2)) 37 | ->method('commitDlq'); 38 | 39 | $batchSize = 3; 40 | 41 | $messageCounter = new MessageCounter(42); 42 | $batchCommitter = new BatchCommitter($committer, $messageCounter, $batchSize); 43 | 44 | $batchCommitter->commitDlq(new Message); 45 | $batchCommitter->commitDlq(new Message); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Contracts/Producer.php: -------------------------------------------------------------------------------- 1 | 10 | ``` 11 | 12 | ### Defining configuration options 13 | 14 | The `withConfigOption` method sets a `\RdKafka\Conf::class` option. You can check all available options [here][rdkafka_config]. 15 | This method sets one config per call, and you can use `withConfigOptions` passing an array of config name and config value 16 | as argument. Here's an example: 17 | 18 | ```php 19 | use Junges\Kafka\Facades\Kafka; 20 | 21 | Kafka::publish('broker') 22 | ->onTopic('topic') 23 | ->withConfigOption('property-name', 'property-value') 24 | ->withConfigOptions([ 25 | 'property-name' => 'property-value' 26 | ]); 27 | ``` 28 | 29 | While you are developing your application, you can enable debug with the `withDebugEnabled` method. 30 | To disable debug mode, you can use `->withDebugEnabled(false)`, or `withDebugDisabled` methods. 31 | 32 | ```php 33 | use Junges\Kafka\Facades\Kafka; 34 | 35 | Kafka::publish('broker') 36 | ->onTopic('topic') 37 | ->withConfigOption('property-name', 'property-value') 38 | ->withConfigOptions([ 39 | 'property-name' => 'property-value' 40 | ]) 41 | ->withDebugEnabled() // To enable debug mode 42 | ->withDebugDisabled() // To disable debug mode 43 | ->withDebugEnabled(false) // Also to disable debug mode 44 | ``` 45 | 46 | [rdkafka_config]:https://github.com/confluentinc/librdkafka/blob/master/CONFIGURATION.md -------------------------------------------------------------------------------- /src/Contracts/Committer.php: -------------------------------------------------------------------------------- 1 | payload = 22 | <<<'JSON' 23 | {"foo": "bar"} 24 | JSON; 25 | $message->key = Str::uuid()->toString(); 26 | $message->topic_name = 'test-topic'; 27 | $message->partition = 1; 28 | $message->headers = []; 29 | $message->offset = 0; 30 | 31 | $messageConsumerMock = m::mock(MessageConsumer::class); 32 | 33 | $consumer = new CallableConsumer($this->handleMessage(...), [ 34 | function (ConsumerMessage $message, callable $next): void { 35 | $decoded = json_decode($message->getBody()); 36 | $next($decoded); 37 | }, 38 | function (stdClass $message, callable $next): void { 39 | $decoded = (array) $message; 40 | $next($decoded); 41 | }, 42 | ]); 43 | 44 | $consumer->handle($this->getConsumerMessage($message), $messageConsumerMock); 45 | } 46 | 47 | public function handleMessage(array $data): void 48 | { 49 | $this->assertEquals([ 50 | 'foo' => 'bar', 51 | ], $data); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/Commit/RetryableCommitter.php: -------------------------------------------------------------------------------- 1 | retryable = new Retryable($sleeper, $maximumRetries, self::RETRYABLE_ERRORS); 25 | } 26 | 27 | /** @throws \Carbon\Exceptions\Exception */ 28 | public function commitMessage(Message $message, bool $success): void 29 | { 30 | $this->retryable->retry(fn () => $this->committer->commitMessage($message, $success)); 31 | } 32 | 33 | /** @throws \Carbon\Exceptions\Exception */ 34 | public function commitDlq(Message $message): void 35 | { 36 | $this->retryable->retry(fn () => $this->committer->commitDlq($message)); 37 | } 38 | 39 | /** @throws \Carbon\Exceptions\Exception */ 40 | public function commit(mixed $messageOrOffsets = null): void 41 | { 42 | $this->retryable->retry(fn () => $this->committer->commit($messageOrOffsets)); 43 | } 44 | 45 | /** @throws \Carbon\Exceptions\Exception */ 46 | public function commitAsync(mixed $messageOrOffsets = null): void 47 | { 48 | $this->retryable->retry(fn () => $this->committer->commitAsync($messageOrOffsets)); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /docs/producing-messages/configuring-message-payload.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Configuring message payload 3 | weight: 3 4 | --- 5 | 6 | In kafka, you can configure your payload with a message, message headers and message key. All these configurations are available within ProducerBuilder class. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | ### Configuring message headers 13 | To configure the message headers, use the `withHeaders` method: 14 | 15 | ```php 16 | use Junges\Kafka\Facades\Kafka; 17 | 18 | Kafka::publish('broker') 19 | ->onTopic('topic') 20 | ->withHeaders([ 21 | 'header-key' => 'header-value' 22 | ]) 23 | ``` 24 | 25 | ### Configure the message body 26 | You can configure the message with the `withMessage` or `withBodyKey` methods. 27 | 28 | The `withMessage` sets the entire message, and it accepts a `Junges\Kafka\Message\Message::class` instance as argument. 29 | 30 | This is how you should use it: 31 | 32 | ```php 33 | use Junges\Kafka\Facades\Kafka; 34 | use Junges\Kafka\Message\Message; 35 | 36 | $message = new Message( 37 | headers: ['header-key' => 'header-value'], 38 | body: ['key' => 'value'], 39 | key: 'kafka key here' 40 | ) 41 | 42 | Kafka::publish('broker')->onTopic('topic')->withMessage($message); 43 | ``` 44 | 45 | The `withBodyKey` method sets only a key in your message. 46 | 47 | ```php 48 | use Junges\Kafka\Facades\Kafka; 49 | 50 | Kafka::publish('broker')->onTopic('topic')->withBodyKey('key', 'value'); 51 | ``` 52 | 53 | ### Using Kafka Keys 54 | In Kafka, keys are used to determine the partition within a log to which a message get's appended to. 55 | If you want to use a key in your message, you should use the `withKafkaKey` method: 56 | 57 | ```php 58 | use Junges\Kafka\Facades\Kafka; 59 | 60 | Kafka::publish('broker')->onTopic('topic')->withKafkaKey('your-kafka-key'); 61 | ``` -------------------------------------------------------------------------------- /src/Concerns/InteractsWithConfigCallbacks.php: -------------------------------------------------------------------------------- 1 | callbacks['setErrorCb'] = $callback; 13 | 14 | return $this; 15 | } 16 | 17 | /** Sets the delivery report callback. */ 18 | public function withDrMsgCb(callable $callback): self 19 | { 20 | $this->callbacks['setDrMsgCb'] = $callback; 21 | 22 | return $this; 23 | } 24 | 25 | /** Set consume callback to use with poll. */ 26 | public function withConsumeCb(callable $callback): self 27 | { 28 | $this->callbacks['setConsumeCb'] = $callback; 29 | 30 | return $this; 31 | } 32 | 33 | /** Set the log callback. */ 34 | public function withLogCb(callable $callback): self 35 | { 36 | $this->callbacks['setLogCb'] = $callback; 37 | 38 | return $this; 39 | } 40 | 41 | /** Set offset commit callback to use with consumer groups. */ 42 | public function withOffsetCommitCb(callable $callback): self 43 | { 44 | $this->callbacks['setOffsetCommitCb'] = $callback; 45 | 46 | return $this; 47 | } 48 | 49 | /** Set rebalance callback for use with coordinated consumer group balancing. */ 50 | public function withRebalanceCb(callable $callback): self 51 | { 52 | $this->callbacks['setRebalanceCb'] = $callback; 53 | 54 | return $this; 55 | } 56 | 57 | /** Set statistics callback. */ 58 | public function withStatsCb(callable $callback): self 59 | { 60 | $this->callbacks['setStatsCb'] = $callback; 61 | 62 | return $this; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tests/Message/KafkaAvroSchemaTest.php: -------------------------------------------------------------------------------- 1 | getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 17 | 18 | $schemaName = 'testSchema'; 19 | $version = 9; 20 | 21 | $avroSchema = new KafkaAvroSchema($schemaName, $version, $definition); 22 | 23 | $this->assertEquals($schemaName, $avroSchema->getName()); 24 | $this->assertEquals($version, $avroSchema->getVersion()); 25 | $this->assertEquals($definition, $avroSchema->getDefinition()); 26 | } 27 | 28 | #[Test] 29 | public function setters(): void 30 | { 31 | $definition = $this->getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 32 | 33 | $schemaName = 'testSchema'; 34 | 35 | $avroSchema = new KafkaAvroSchema($schemaName); 36 | 37 | $avroSchema->setDefinition($definition); 38 | 39 | $this->assertEquals($definition, $avroSchema->getDefinition()); 40 | } 41 | 42 | #[Test] 43 | public function avro_schema_with_just_name(): void 44 | { 45 | $schemaName = 'testSchema'; 46 | 47 | $avroSchema = new KafkaAvroSchema($schemaName); 48 | 49 | $this->assertEquals($schemaName, $avroSchema->getName()); 50 | $this->assertEquals(KafkaAvroSchemaRegistry::LATEST_VERSION, $avroSchema->getVersion()); 51 | $this->assertNull($avroSchema->getDefinition()); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /docs/advanced-usage/sasl-authentication.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: SASL Authentication 3 | weight: 3 4 | --- 5 | 6 | ```+parse 7 | 8 | ``` 9 | 10 | SASL allows your producers and your consumers to authenticate to your Kafka cluster, which verifies their identity. 11 | It's also a secure way to enable your clients to endorse an identity. To provide SASL configuration, you can use the `withSasl` method, 12 | passing a `Junges\Kafka\Config\Sasl` instance as the argument: 13 | 14 | ```php 15 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 16 | ->withSasl( 17 | password: 'password', 18 | username: 'username', 19 | mechanisms: 'authentication mechanism' 20 | ); 21 | ``` 22 | 23 | You can also set the security protocol used with sasl. It's optional and by default `SASL_PLAINTEXT` is used, but you can set it to `SASL_SSL`: 24 | 25 | ```php 26 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 27 | ->withSasl( 28 | password: 'password', 29 | username: 'username', 30 | mechanisms: 'authentication mechanism', 31 | securityProtocol: 'SASL_SSL', 32 | ); 33 | ``` 34 | 35 | ```+parse 36 | 37 | When using the `withSasl` method, the securityProtocol set in this method takes priority over `withSecurityProtocol` method. 38 | 39 | ``` 40 | 41 | ### TLS Authentication 42 | 43 | For using TLS authentication with Laravel Kafka you can configure your client using the following options: 44 | 45 | ```php 46 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 47 | ->withOptions([ 48 | 'ssl.ca.location' => '/some/location/kafka.crt', 49 | 'ssl.certificate.location' => '/some/location/client.crt', 50 | 'ssl.key.location' => '/some/location/client.key', 51 | 'ssl.endpoint.identification.algorithm' => 'none' 52 | ]); 53 | ``` -------------------------------------------------------------------------------- /src/Facades/Kafka.php: -------------------------------------------------------------------------------- 1 | shouldFake() 32 | )); 33 | 34 | return $fake; 35 | } 36 | 37 | public static function getFacadeAccessor(): string 38 | { 39 | return \Junges\Kafka\Factory::class; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /tests/Console/Consumers/KafkaConsumerCommandTest.php: -------------------------------------------------------------------------------- 1 | mockProducer(); 20 | 21 | $fakeHandler = new FakeHandler; 22 | 23 | $message = new Message; 24 | $message->err = 0; 25 | $message->key = 'key'; 26 | $message->topic_name = 'test-topic'; 27 | $message->payload = '{"body": "message payload"}'; 28 | $message->offset = 0; 29 | $message->partition = 1; 30 | $message->headers = []; 31 | 32 | $this->mockConsumerWithMessage($message); 33 | 34 | $config = new Config( 35 | broker: 'broker', 36 | topics: ['test-topic'], 37 | securityProtocol: 'security', 38 | commit: 1, 39 | groupId: 'group', 40 | consumer: $fakeHandler, 41 | sasl: null, 42 | dlq: null, 43 | maxMessages: 1, 44 | maxCommitRetries: 1 45 | ); 46 | 47 | $consumer = new Consumer($config, new JsonDeserializer); 48 | 49 | $this->app->bind(Consumer::class, fn () => $consumer); 50 | 51 | $this->artisan('kafka:consume --topics=test-topic --consumer=\\\\Junges\\\\Kafka\\\\Tests\\\\Fakes\\\\FakeHandler'); 52 | 53 | $this->assertInstanceOf(ConsumedMessage::class, $fakeHandler->lastMessage()); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Commit/Committer.php: -------------------------------------------------------------------------------- 1 | consumer->commit($message); 19 | } 20 | 21 | /** @throws \RdKafka\Exception */ 22 | public function commitDlq(Message $message): void 23 | { 24 | $this->consumer->commit($message); 25 | } 26 | 27 | /** @throws \RdKafka\Exception */ 28 | public function commit(mixed $messageOrOffsets = null): void 29 | { 30 | if ($messageOrOffsets instanceof ConsumerMessage) { 31 | $topicPartition = new TopicPartition( 32 | $messageOrOffsets->getTopicName(), 33 | $messageOrOffsets->getPartition(), 34 | $messageOrOffsets->getOffset() + 1 35 | ); 36 | $messageOrOffsets = [$topicPartition]; 37 | } 38 | 39 | $this->consumer->commit($messageOrOffsets); 40 | } 41 | 42 | /** @throws \RdKafka\Exception */ 43 | public function commitAsync(mixed $messageOrOffsets = null): void 44 | { 45 | if ($messageOrOffsets instanceof ConsumerMessage) { 46 | $topicPartition = new TopicPartition( 47 | $messageOrOffsets->getTopicName(), 48 | $messageOrOffsets->getPartition(), 49 | $messageOrOffsets->getOffset() + 1 50 | ); 51 | $messageOrOffsets = [$topicPartition]; 52 | } 53 | 54 | $this->consumer->commitAsync($messageOrOffsets); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Providers/LaravelKafkaServiceProvider.php: -------------------------------------------------------------------------------- 1 | publishesConfiguration(); 26 | 27 | if ($this->app->runningInConsole()) { 28 | $this->commands([ 29 | ConsumerCommand::class, 30 | RestartConsumersCommand::class, 31 | ]); 32 | } 33 | } 34 | 35 | public function register(): void 36 | { 37 | $this->app->bind(MessageSerializer::class, fn () => new JsonSerializer); 38 | 39 | $this->app->bind(MessageDeserializer::class, fn () => new JsonDeserializer); 40 | 41 | $this->app->bind(ProducerMessage::class, fn () => new Message('')); 42 | 43 | $this->app->bind(ConsumerMessage::class, ConsumedMessage::class); 44 | 45 | $this->app->bind(Manager::class, Factory::class); 46 | 47 | $this->app->singleton(LoggerContract::class, Logger::class); 48 | } 49 | 50 | private function publishesConfiguration(): void 51 | { 52 | $this->publishes([ 53 | __DIR__.'/../../config/kafka.php' => config_path('kafka.php'), 54 | ], 'laravel-kafka-config'); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel Kafka 2 | ![art/laravel-kafka.png](art/laravel-kafka.png) 3 | 4 | ![Packagist Version](https://img.shields.io/packagist/v/mateusjunges/laravel-kafka?label=Latest%20version%20on%20Packagist) 5 | ![Packagist Downloads](https://img.shields.io/packagist/dt/mateusjunges/laravel-kafka?style=flat&label=Total%20Downloads) 6 | [![MIT Licensed](https://img.shields.io/badge/license-MIT-brightgreen.svg?style=flat)](LICENSE) 7 | [![Continuous Integration](https://github.com/mateusjunges/laravel-kafka/actions/workflows/run-tests.yml/badge.svg)](https://github.com/mateusjunges/laravel-kafka/actions/workflows/run-tests.yml) 8 | 9 | Do you use Kafka in your laravel projects? All packages I've seen until today, including some built by myself, does not provide a nice usage syntax or, if it does, the test process with these packages are very painful. 10 | 11 | This package provides a nice way of producing and consuming kafka messages in your Laravel projects. 12 | 13 | # Sponsor my work! 14 | If you think this package helped you in any way, you can sponsor me on GitHub! 15 | 16 | [![Sponsor Me](art/sponsor.png)](https://github.com/sponsors/mateusjunges) 17 | 18 | - My personal website: https://mateusjunges.com 19 | - Follow me on Twitter: https://twitter.com/mateusjungess 20 | - Follow me on Bluesky: https://bsky.app/profile/mateusjunges.com 21 | 22 | # Documentation 23 | You can [find the documentations for this package here](https://laravelkafka.com/) 24 | 25 | # Testing 26 | Run `composer test` to test this package. 27 | 28 | # Contributing 29 | Thank you for considering contributing for the Laravel Kafka package! The contribution guide can be found [here][contributing]. 30 | 31 | # Credits 32 | - [Mateus Junges](https://twitter.com/mateusjungess) 33 | - [Arquivei](https://github.com/arquivei) 34 | 35 | # License 36 | The Laravel Kafka package is open-sourced software licenced under the [MIT][mit] License. Please see the [License File][license] for more information. 37 | 38 | [contributing]: .github/CONTRIBUTING.md 39 | [license]: LICENSE 40 | [mit]: https://opensource.org/licenses/MIT 41 | -------------------------------------------------------------------------------- /docs/testing/assert-published-on.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Assert published On 3 | weight: 3 4 | --- 5 | 6 | ```+parse 7 | 8 | ``` 9 | 10 | If you want to assert that a message was published in a specific kafka topic, you can use the `assertPublishedOn` method: 11 | 12 | ```php 13 | use PHPUnit\Framework\TestCase; 14 | use Junges\Kafka\Facades\Kafka; 15 | 16 | class MyTest extends TestCase 17 | { 18 | public function testWithSpecificTopic() 19 | { 20 | Kafka::fake(); 21 | 22 | $producer = Kafka::publish('broker') 23 | ->onTopic('some-kafka-topic') 24 | ->withHeaders(['key' => 'value']) 25 | ->withBodyKey('key', 'value'); 26 | 27 | $producer->send(); 28 | 29 | Kafka::assertPublishedOn('some-kafka-topic', $producer->getMessage()); 30 | 31 | // Or: 32 | Kafka::assertPublishedOn('some-kafka-topic'); 33 | 34 | } 35 | } 36 | ``` 37 | 38 | You can also use a callback function to perform assertions within the message using a callback in which the argument is the published message 39 | itself. 40 | 41 | ```php 42 | use PHPUnit\Framework\TestCase; 43 | use Junges\Kafka\Facades\Kafka; 44 | use Junges\Kafka\Message\Message; 45 | 46 | class MyTest extends TestCase 47 | { 48 | public function testWithSpecificTopic() 49 | { 50 | Kafka::fake(); 51 | 52 | $producer = Kafka::publish('broker') 53 | ->onTopic('some-kafka-topic') 54 | ->withHeaders(['key' => 'value']) 55 | ->withBodyKey('key', 'value'); 56 | 57 | $producer->send(); 58 | 59 | Kafka::assertPublishedOn('some-kafka-topic', $producer->getMessage(), function(Message $message) { 60 | return $message->getHeaders()['key'] === 'value'; 61 | }); 62 | 63 | // Or: 64 | Kafka::assertPublishedOn('some-kafka-topic', null, function(Message $message) { 65 | return $message->getHeaders()['key'] === 'value'; 66 | }); 67 | } 68 | } 69 | ``` -------------------------------------------------------------------------------- /docs/consuming-messages/assigning-partitions.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Assigning consumers to a topic partition 3 | weight: 3 4 | --- 5 | 6 | Kafka clients allows you to implement your own partition assignment strategies for consumers. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | If you have a topic with multiple consumers and want to assign a consumer to a specific partition topic, you can 13 | use the `assignPartitions` method, available on the `ConsumerBuilder` instance: 14 | 15 | ```php 16 | $partition = 1; // The partition number you want to assign 17 | 18 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 19 | ->assignPartitions([ 20 | new \RdKafka\TopicPartition('your-topic-name', $partition) 21 | ]); 22 | ``` 23 | 24 | The `assignPartitions` method accepts an array of `\RdKafka\TopicPartition` objects. You can assign multiple partitions to the same consumer 25 | by adding more entries to the `assignPartitions` parameter: 26 | 27 | ```php 28 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 29 | ->assignPartitions([ 30 | new \RdKafka\TopicPartition('your-topic-name', 1), 31 | new \RdKafka\TopicPartition('your-topic-name', 2), 32 | new \RdKafka\TopicPartition('your-topic-name', 3) 33 | ]); 34 | ``` 35 | 36 | ## Dynamic Partition Discovery 37 | 38 | If you don't know the partition numbers in advance (which is common when using consumer groups), you can use the partition discovery features: 39 | 40 | ```php 41 | $consumer = \Junges\Kafka\Facades\Kafka::consumer(['your-topic-name'], 'your-group') 42 | ->withPartitionAssignmentCallback(function ($partitions) { 43 | echo "Assigned " . count($partitions) . " partitions\n"; 44 | 45 | foreach ($partitions as $partition) { 46 | echo "Partition: {$partition->getPartition()}\n"; 47 | } 48 | }) 49 | ->withHandler(function ($message) { 50 | // Handle message 51 | }); 52 | ``` 53 | 54 | For more advanced partition discovery and dynamic offset assignment, see the [Partition Discovery documentation](partition-discovery.md). -------------------------------------------------------------------------------- /src/Support/Testing/Fakes/BuilderFake.php: -------------------------------------------------------------------------------- 1 | messages = $messages; 30 | 31 | return $this; 32 | } 33 | 34 | /** Build the Kafka consumer. */ 35 | public function build(): MessageConsumer 36 | { 37 | $config = new Config( 38 | broker: $this->brokers, 39 | topics: $this->topics, 40 | securityProtocol: $this->getSecurityProtocol(), 41 | commit: $this->commit, 42 | groupId: $this->groupId, 43 | consumer: new CallableConsumer($this->handler, $this->middlewares), 44 | sasl: $this->saslConfig, 45 | dlq: $this->dlq, 46 | maxMessages: $this->maxMessages, 47 | maxCommitRetries: $this->maxCommitRetries, 48 | autoCommit: $this->autoCommit, 49 | customOptions: $this->options, 50 | stopAfterLastMessage: $this->stopAfterLastMessage, 51 | callbacks: $this->callbacks, 52 | whenStopConsuming: $this->onStopConsuming, 53 | ); 54 | 55 | return new ConsumerFake( 56 | $config, 57 | $this->messages 58 | ); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /tests/FailingCommitter.php: -------------------------------------------------------------------------------- 1 | failure = $failure; 24 | $this->timesToFail = $timesToFail; 25 | } 26 | 27 | /** 28 | * @throws Exception 29 | */ 30 | public function commitMessage(?Message $message = null, ?bool $success = null): void 31 | { 32 | $this->timesTriedToCommitMessage++; 33 | $this->doCommit(); 34 | } 35 | 36 | /** 37 | * @throws Exception 38 | */ 39 | public function commitDlq(Message $message): void 40 | { 41 | $this->timesTriedToCommitDlq++; 42 | $this->doCommit(); 43 | } 44 | 45 | public function getTimesTriedToCommitMessage(): int 46 | { 47 | return $this->timesTriedToCommitMessage; 48 | } 49 | 50 | public function getTimesTriedToCommitDlq(): int 51 | { 52 | return $this->timesTriedToCommitDlq; 53 | } 54 | 55 | /** 56 | * @throws Exception 57 | */ 58 | public function commit(mixed $messageOrOffsets = null): void 59 | { 60 | $this->doCommit(); 61 | } 62 | 63 | /** 64 | * @throws Exception 65 | */ 66 | public function commitAsync(mixed $messageOrOffsets = null): void 67 | { 68 | $this->doCommit(); 69 | } 70 | 71 | /** 72 | * @throws Exception 73 | */ 74 | private function doCommit(): void 75 | { 76 | $this->commitCount++; 77 | 78 | if ($this->commitCount > $this->timesToFail) { 79 | $this->commitCount = 0; 80 | 81 | return; 82 | } 83 | 84 | throw $this->failure; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /docs/consuming-messages/consuming-from-specific-offsets.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Consuming messages from specific offsets 3 | weight: 3 4 | --- 5 | 6 | Kafka clients allows you to implement your own partition assignment strategies for consumers, and you can also consume messages from specific offsets. 7 | 8 | If you have a topic with multiple consumers and want to assign a consumer to a specific partition offset, you can 9 | use the `assignPartitions` method, available on the `ConsumerBuilder` instance: 10 | 11 | ```php 12 | $partition = 1; // The partition number you want to assign. 13 | $offset = 0; // The offset you want to start consuming messages from. 14 | 15 | $consumer = \Junges\Kafka\Facades\Kafka::consumer() 16 | ->assignPartitions([ 17 | new \RdKafka\TopicPartition('your-topic-name', $partition, $offset) 18 | ]); 19 | ``` 20 | 21 | ```+parse 22 | 23 | ``` 24 | 25 | ## Dynamic Offset Assignment 26 | 27 | If you need to assign offsets dynamically based on partition assignments (useful when you don't know partition numbers in advance), you can use the `assignPartitionsWithOffsets` method: 28 | 29 | ```php 30 | $consumer = \Junges\Kafka\Facades\Kafka::consumer(['your-topic-name'], 'your-group') 31 | ->assignPartitionsWithOffsets(function ($partitions) { 32 | $partitionsWithOffsets = []; 33 | 34 | foreach ($partitions as $partition) { 35 | // Set different offsets based on partition or other logic 36 | if ($partition->getPartition() === 0) { 37 | $partition->setOffset(0); // Start from beginning 38 | } else { 39 | $partition->setOffset(RD_KAFKA_OFFSET_END); // Start from end 40 | } 41 | 42 | $partitionsWithOffsets[] = $partition; 43 | } 44 | 45 | return $partitionsWithOffsets; 46 | }) 47 | ->withHandler(function ($message) { 48 | // Handle message 49 | }); 50 | ``` 51 | 52 | For more information about partition discovery and advanced offset management, see the [Partition Discovery documentation](partition-discovery.md). -------------------------------------------------------------------------------- /pint.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "laravel", 3 | "notPath": [ 4 | "tests/TestCase.php" 5 | ], 6 | "rules": { 7 | "array_push": true, 8 | "backtick_to_shell_exec": true, 9 | "blank_line_after_opening_tag": false, 10 | "linebreak_after_opening_tag": false, 11 | "date_time_immutable": true, 12 | "declare_strict_types": true, 13 | "lowercase_keywords": true, 14 | "lowercase_static_reference": true, 15 | "fully_qualified_strict_types": true, 16 | "global_namespace_import": { 17 | "import_classes": true, 18 | "import_constants": true, 19 | "import_functions": true 20 | }, 21 | "mb_str_functions": true, 22 | "modernize_types_casting": true, 23 | "new_with_parentheses": false, 24 | "no_superfluous_elseif": true, 25 | "no_useless_else": true, 26 | "no_multiple_statements_per_line": true, 27 | "ordered_class_elements": { 28 | "order": [ 29 | "use_trait", 30 | "case", 31 | "constant", 32 | "constant_public", 33 | "constant_protected", 34 | "constant_private", 35 | "property_public", 36 | "property_protected", 37 | "property_private", 38 | "construct", 39 | "destruct", 40 | "magic", 41 | "phpunit", 42 | "method_abstract", 43 | "method_public_static", 44 | "method_public", 45 | "method_protected_static", 46 | "method_protected", 47 | "method_private_static", 48 | "method_private" 49 | ], 50 | "sort_algorithm": "none" 51 | }, 52 | "ordered_interfaces": true, 53 | "ordered_traits": true, 54 | "protected_to_private": true, 55 | "self_accessor": true, 56 | "self_static_accessor": true, 57 | "strict_comparison": true, 58 | "visibility_required": true 59 | } 60 | } -------------------------------------------------------------------------------- /docs/consuming-messages/queueable-handlers.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Queueable handlers 3 | weight: 11 4 | --- 5 | 6 | Queueable handlers allow you to handle your kafka messages in a queue. This will put a job into the Laravel queue system for each message received by your Kafka consumer. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | This only requires you to implements the `Illuminate\Contracts\Queue\ShouldQueue` interface in your Handler. 13 | 14 | This is how a queueable handler looks like: 15 | 16 | ```php 17 | use Illuminate\Contracts\Queue\ShouldQueue; 18 | use Junges\Kafka\Contracts\Handler as HandlerContract; 19 | use Junges\Kafka\Contracts\KafkaConsumerMessage; 20 | 21 | class Handler implements HandlerContract, ShouldQueue 22 | { 23 | public function __invoke(KafkaConsumerMessage $message): void 24 | { 25 | // Handle the consumed message. 26 | } 27 | } 28 | ``` 29 | 30 | As you can see on the `__invoke` method, queued handlers does not have access to a `MessageConsumer` instance when handling the message, 31 | because it's running on a laravel queue and there are no actions that can be performed asynchronously on Kafka message consumer. 32 | 33 | You can specify which queue connection and queue name to use for your handler by implementing the `onConnection` and `onQueue` methods: 34 | 35 | ```php 36 | use Illuminate\Contracts\Queue\ShouldQueue; 37 | use Junges\Kafka\Contracts\Handler as HandlerContract; 38 | use Junges\Kafka\Contracts\KafkaConsumerMessage; 39 | 40 | class Handler implements HandlerContract, ShouldQueue 41 | { 42 | public function __invoke(KafkaConsumerMessage $message): void 43 | { 44 | // Handle the consumed message. 45 | } 46 | 47 | public function onConnection(): string 48 | { 49 | return 'sqs'; // Specify your queue connection 50 | } 51 | 52 | public function onQueue(): string 53 | { 54 | return 'kafka-handlers'; // Specify your queue name 55 | } 56 | } 57 | ``` 58 | 59 | After creating your handler class, you can use it just as a normal handler, and `laravel-kafka` will know how to handle it under the hoods 😄. 60 | 61 | 62 | -------------------------------------------------------------------------------- /docs/testing/mocking-your-kafka-consumer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Mocking your kafka consumer 3 | weight: 7 4 | --- 5 | 6 | If you want to test that your consumers are working correctly, you can mock and execute the consumer to 7 | ensure that everything works as expected. 8 | 9 | ```+parse 10 | 11 | ``` 12 | 13 | 14 | You just need to tell kafka which messages the consumer should receive and then start your consumer. This package will 15 | run all the specified messages through the consumer and stop after the last message, so you can perform whatever 16 | assertions you want to. 17 | 18 | For example, let's say you want to test that a simple blog post was published after consuming a `post-published` message: 19 | 20 | ```php 21 | public function test_post_is_marked_as_published() 22 | { 23 | // First, you use the fake method: 24 | \Junges\Kafka\Facades\Kafka::fake(); 25 | 26 | // Then, tells Kafka what messages the consumer should receive: 27 | \Junges\Kafka\Facades\Kafka::shouldReceiveMessages([ 28 | new \Junges\Kafka\Message\ConsumedMessage( 29 | topicName: 'mark-post-as-published-topic', 30 | partition: 0, 31 | headers: [], 32 | body: ['post_id' => 1], 33 | key: null, 34 | offset: 0, 35 | timestamp: 0 36 | ), 37 | ]); 38 | 39 | // Now, instantiate your consumer and start consuming messages. It will consume only the messages 40 | // specified in `shouldReceiveMessages` method: 41 | $consumer = \Junges\Kafka\Facades\Kafka::consumer(['mark-post-as-published-topic']) 42 | ->withHandler(function (\Junges\Kafka\Contracts\ConsumerMessage $message) use (&$posts) { 43 | $post = Post::find($message->getBody()['post_id']); 44 | 45 | $post->update(['published_at' => now()->format("Y-m-d H:i:s")]); 46 | 47 | return 0; 48 | 49 | })->build(); 50 | 51 | $consumer->consume(); 52 | 53 | // Now, you can test if the post published_at field is not empty, or anything else you want to test: 54 | 55 | $this->assertNotNull($post->refresh()->published_at); 56 | } 57 | ``` 58 | 59 | -------------------------------------------------------------------------------- /src/Contracts/MessageConsumer.php: -------------------------------------------------------------------------------- 1 | createMock(ConsumerMessage::class); 23 | $handler($messageMock); 24 | 25 | $this->assertSame(1, $failingHandler->getTimesInvoked()); 26 | } 27 | 28 | #[Test] 29 | public function it_does_retries_on_exception(): void 30 | { 31 | $failingHandler = new FailingHandler(4, new RuntimeException('test')); 32 | $sleeper = new FakeSleeper; 33 | $handler = new RetryableHandler($failingHandler(...), new DefaultRetryStrategy, $sleeper); 34 | 35 | $messageMock = $this->createMock(ConsumerMessage::class); 36 | $handler($messageMock); 37 | 38 | $this->assertSame(5, $failingHandler->getTimesInvoked()); 39 | $this->assertEquals([1e6, 2e6, 4e6, 8e6], $sleeper->getSleeps()); 40 | } 41 | 42 | #[Test] 43 | public function it_bubbles_exception_when_retries_exceeded(): void 44 | { 45 | $failingHandler = new FailingHandler(100, new RuntimeException('test')); 46 | $sleeper = new FakeSleeper; 47 | $handler = new RetryableHandler($failingHandler(...), new DefaultRetryStrategy, $sleeper); 48 | 49 | $messageMock = $this->createMock(ConsumerMessage::class); 50 | 51 | try { 52 | $handler($messageMock); 53 | 54 | $this->fail('Handler passed but a \RuntimeException is expected.'); 55 | } catch (RuntimeException) { 56 | $this->assertSame(7, $failingHandler->getTimesInvoked()); 57 | $this->assertEquals([1e6, 2e6, 4e6, 8e6, 16e6, 32e6], $sleeper->getSleeps()); 58 | } 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Message/Deserializers/AvroDeserializer.php: -------------------------------------------------------------------------------- 1 | registry; 21 | } 22 | 23 | public function deserialize(ConsumerMessage $message): ConsumerMessage 24 | { 25 | return new ConsumedMessage( 26 | topicName: $message->getTopicName(), 27 | partition: $message->getPartition(), 28 | headers: $message->getHeaders(), 29 | body: $this->decodeBody($message), 30 | key: $this->decodeKey($message), 31 | offset: $message->getOffset(), 32 | timestamp: $message->getTimestamp() 33 | ); 34 | } 35 | 36 | private function decodeBody(ConsumerMessage $message) 37 | { 38 | $body = $message->getBody(); 39 | $topicName = $message->getTopicName(); 40 | 41 | if ($body === null) { 42 | return null; 43 | } 44 | 45 | if ($this->registry->hasBodySchemaForTopic($topicName) === false) { 46 | return $body; 47 | } 48 | 49 | $avroSchema = $this->registry->getBodySchemaForTopic($topicName); 50 | $schemaDefinition = $avroSchema->getDefinition(); 51 | 52 | return $this->recordSerializer->decodeMessage($body, $schemaDefinition); 53 | } 54 | 55 | private function decodeKey(ConsumerMessage $message) 56 | { 57 | $key = $message->getKey(); 58 | $topicName = $message->getTopicName(); 59 | 60 | if ($key === null) { 61 | return null; 62 | } 63 | 64 | if ($this->registry->hasKeySchemaForTopic($topicName) === false) { 65 | return $key; 66 | } 67 | 68 | $avroSchema = $this->registry->getKeySchemaForTopic($topicName); 69 | $schemaDefinition = $avroSchema->getDefinition(); 70 | 71 | return $this->recordSerializer->decodeMessage($key, $schemaDefinition); 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /docs/consuming-messages/custom-deserializers.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Custom deserializers 3 | weight: 7 4 | --- 5 | 6 | To create a custom deserializer, you need to create a class that implements the `\Junges\Kafka\Contracts\MessageDeserializer` contract. 7 | This interface force you to declare the `deserialize` method. 8 | 9 | ```+parse 10 | 11 | ``` 12 | 13 | To set the deserializer you want to use, use the `usingDeserializer` method: 14 | 15 | ```php 16 | $consumer = \Junges\Kafka\Facades\Kafka::consumer()->usingDeserializer(new MyCustomDeserializer()); 17 | ``` 18 | 19 | ```+parse 20 | 21 | The deserializer class must use the same algorithm as the serializer used to produce this message. 22 | 23 | ``` 24 | 25 | 26 | ### Using AVRO deserializer 27 | To use the AVRO deserializer on your consumer, add the Avro deserializer: 28 | 29 | ```php 30 | use FlixTech\AvroSerializer\Objects\RecordSerializer; 31 | use FlixTech\SchemaRegistryApi\Registry\CachedRegistry; 32 | use FlixTech\SchemaRegistryApi\Registry\BlockingRegistry; 33 | use FlixTech\SchemaRegistryApi\Registry\PromisingRegistry; 34 | use FlixTech\SchemaRegistryApi\Registry\Cache\AvroObjectCacheAdapter; 35 | use GuzzleHttp\Client; 36 | 37 | 38 | $cachedRegistry = new CachedRegistry( 39 | new BlockingRegistry( 40 | new PromisingRegistry( 41 | new Client(['base_uri' => 'kafka-schema-registry:9081']) 42 | ) 43 | ), 44 | new AvroObjectCacheAdapter() 45 | ); 46 | 47 | $registry = new \Junges\Kafka\Message\Registry\AvroSchemaRegistry($cachedRegistry); 48 | $recordSerializer = new RecordSerializer($cachedRegistry); 49 | 50 | //if no version is defined, latest version will be used 51 | //if no schema definition is defined, the appropriate version will be fetched form the registry 52 | $registry->addBodySchemaMappingForTopic( 53 | 'test-topic', 54 | new \Junges\Kafka\Message\KafkaAvroSchema('bodySchema' , 9 /* , AvroSchema $definition */) 55 | ); 56 | $registry->addKeySchemaMappingForTopic( 57 | 'test-topic', 58 | new \Junges\Kafka\Message\KafkaAvroSchema('keySchema' , 9 /* , AvroSchema $definition */) 59 | ); 60 | 61 | // if you are only decoding key or value, you can pass that mode as additional third argument 62 | // per default both key and body will get decoded 63 | $deserializer = new \Junges\Kafka\Message\Deserializers\AvroDeserializer($registry, $recordSerializer /*, AvroDecoderInterface::DECODE_BODY */); 64 | 65 | $consumer = \Junges\Kafka\Facades\Kafka::consumer()->usingDeserializer($deserializer); 66 | ``` 67 | -------------------------------------------------------------------------------- /src/Contracts/MessageProducer.php: -------------------------------------------------------------------------------- 1 | createMock(KafkaConsumer::class); 18 | 19 | $mockedCommitter = $this->createMock(Committer::class); 20 | $mockedCommitter->expects($this->once()) 21 | ->method('commitMessage') 22 | ->with($this->isInstanceOf(Message::class), true); 23 | 24 | $seekToCurrentErrorCommitter = new SeekToCurrentErrorCommitter($mockedKafkaConsumer, $mockedCommitter); 25 | 26 | $seekToCurrentErrorCommitter->commitMessage(new Message, true); 27 | } 28 | 29 | #[Test] 30 | public function it_should_not_commit_and_resubscribe_on_error(): void 31 | { 32 | $mockedKafkaConsumer = $this->createMock(KafkaConsumer::class); 33 | $mockedKafkaConsumer->expects($this->once()) 34 | ->method('getSubscription') 35 | ->willReturn(['test-topic']); 36 | $mockedKafkaConsumer->expects($this->once()) 37 | ->method('unsubscribe'); 38 | $mockedKafkaConsumer->expects($this->once()) 39 | ->method('subscribe') 40 | ->with(['test-topic']); 41 | 42 | $mockedCommitter = $this->createMock(Committer::class); 43 | $mockedCommitter->expects($this->never()) 44 | ->method('commitMessage'); 45 | 46 | $seekToCurrentErrorCommitter = new SeekToCurrentErrorCommitter($mockedKafkaConsumer, $mockedCommitter); 47 | 48 | $seekToCurrentErrorCommitter->commitMessage(new Message, false); 49 | } 50 | 51 | #[Test] 52 | public function it_passes_dlq_commits(): void 53 | { 54 | $mockedKafkaConsumer = $this->createMock(KafkaConsumer::class); 55 | 56 | $mockedCommitter = $this->createMock(Committer::class); 57 | $mockedCommitter->expects($this->once()) 58 | ->method('commitDlq') 59 | ->with($this->isInstanceOf(Message::class)); 60 | 61 | $seekToCurrentErrorCommitter = new SeekToCurrentErrorCommitter($mockedKafkaConsumer, $mockedCommitter); 62 | 63 | $seekToCurrentErrorCommitter->commitDlq(new Message, true); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Message/Message.php: -------------------------------------------------------------------------------- 1 | body[$key] = $message; 25 | 26 | return $this; 27 | } 28 | 29 | /** Unset a key in the message array. */ 30 | public function forgetBodyKey(string $key): self 31 | { 32 | unset($this->body[$key]); 33 | 34 | return $this; 35 | } 36 | 37 | /** Set the message headers. */ 38 | public function withHeaders(array $headers = []): self 39 | { 40 | $this->headers = $headers; 41 | 42 | return $this; 43 | } 44 | 45 | public function onTopic(string $topic): self 46 | { 47 | $this->topicName = $topic; 48 | 49 | return $this; 50 | } 51 | 52 | /** Set the kafka message key. */ 53 | public function withKey(mixed $key): self 54 | { 55 | $this->key = $key; 56 | 57 | return $this; 58 | } 59 | 60 | #[ArrayShape(['payload' => 'array', 'key' => 'null|string', 'headers' => 'array'])] 61 | public function toArray(): array 62 | { 63 | return [ 64 | 'payload' => $this->body, 65 | 'key' => $this->key, 66 | 'headers' => $this->headers, 67 | ]; 68 | } 69 | 70 | public function withBody(mixed $body): ProducerMessage 71 | { 72 | $this->body = $body; 73 | 74 | return $this; 75 | } 76 | 77 | public function withHeader(string $key, string|int|float $value): ProducerMessage 78 | { 79 | $this->headers[$key] = $value; 80 | 81 | return $this; 82 | } 83 | 84 | public function getHeaders(): ?array 85 | { 86 | // Here we insert an uuid to be used to uniquely identify this message. If the 87 | // id is already set, then array_merge will override it. It's safe to do it 88 | // here because this class is used only when we produce a new message. 89 | return array_merge(parent::getHeaders(), [ 90 | config('kafka.message_id_key') => Str::uuid()->toString(), 91 | ]); 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /docs/installation-and-setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Installation and Setup 3 | weight: 3 4 | --- 5 | 6 | You can install this package using composer: 7 | 8 | ```bash 9 | composer require mateusjunges/laravel-kafka 10 | ``` 11 | 12 | You need to publish the configuration file using 13 | 14 | ```bash 15 | php artisan vendor:publish --tag=laravel-kafka-config 16 | ``` 17 | 18 | ```+parse 19 | 20 | ``` 21 | 22 | This is the default content of the configuration file: 23 | 24 | ```php 25 | env('KAFKA_BROKERS', 'localhost:9092'), 32 | 33 | /* 34 | | Kafka consumers belonging to the same consumer group share a group id. 35 | | The consumers in a group then divides the topic partitions as fairly amongst themselves as possible by 36 | | establishing that each partition is only consumed by a single consumer from the group. 37 | | This config defines the consumer group id you want to use for your project. 38 | */ 39 | 'consumer_group_id' => env('KAFKA_CONSUMER_GROUP_ID', 'group'), 40 | 41 | 'consumer_timeout_ms' => env("KAFKA_CONSUMER_DEFAULT_TIMEOUT", 2000), 42 | 43 | /* 44 | | After the consumer receives its assignment from the coordinator, 45 | | it must determine the initial position for each assigned partition. 46 | | When the group is first created, before any messages have been consumed, the position is set according to a configurable 47 | | offset reset policy (auto.offset.reset). Typically, consumption starts either at the earliest offset or the latest offset. 48 | | You can choose between "latest", "earliest" or "none". 49 | */ 50 | 'offset_reset' => env('KAFKA_OFFSET_RESET', 'latest'), 51 | 52 | /* 53 | | If you set enable.auto.commit (which is the default), then the consumer will automatically commit offsets periodically at the 54 | | interval set by auto.commit.interval.ms. 55 | */ 56 | 'auto_commit' => env('KAFKA_AUTO_COMMIT', true), 57 | 58 | 'sleep_on_error' => env('KAFKA_ERROR_SLEEP', 5), 59 | 60 | 'partition' => env('KAFKA_PARTITION', 0), 61 | 62 | /* 63 | | Kafka supports 4 compression codecs: none , gzip , lz4 and snappy 64 | */ 65 | 'compression' => env('KAFKA_COMPRESSION_TYPE', 'snappy'), 66 | 67 | /* 68 | | Choose if debug is enabled or not. 69 | */ 70 | 'debug' => env('KAFKA_DEBUG', false), 71 | 72 | 73 | /* 74 | | The sleep time in milliseconds that will be used when retrying flush 75 | */ 76 | 'flush_retry_sleep_in_ms' => 100, 77 | 78 | /* 79 | | The cache driver that will be used 80 | */ 81 | 'cache_driver' => env('KAFKA_CACHE_DRIVER', env('CACHE_DRIVER', 'file')), 82 | ]; 83 | 84 | ``` 85 | -------------------------------------------------------------------------------- /src/Concerns/ManagesTransactions.php: -------------------------------------------------------------------------------- 1 | transactionInitialized) { 22 | $this->producer->initTransactions($timeoutInMilliseconds); 23 | $this->transactionInitialized = true; 24 | } 25 | 26 | $this->producer->beginTransaction(); 27 | } catch (KafkaErrorException $exception) { 28 | $this->handleTransactionException($exception); 29 | } 30 | } 31 | 32 | /** 33 | * @throws TransactionShouldBeRetriedException 34 | * @throws TransactionFatalErrorException 35 | * @throws TransactionShouldBeAbortedException 36 | */ 37 | public function abortTransaction(int $timeoutInMilliseconds = 1000): void 38 | { 39 | try { 40 | $this->producer->abortTransaction($timeoutInMilliseconds); 41 | } catch (KafkaErrorException $exception) { 42 | $this->handleTransactionException($exception); 43 | } 44 | } 45 | 46 | /** 47 | * @throws TransactionShouldBeRetriedException 48 | * @throws TransactionFatalErrorException 49 | * @throws TransactionShouldBeAbortedException 50 | */ 51 | public function commitTransaction(int $timeoutInMilliseconds = 1000): void 52 | { 53 | try { 54 | $this->producer->commitTransaction($timeoutInMilliseconds); 55 | } catch (KafkaErrorException $exception) { 56 | $this->handleTransactionException($exception); 57 | } 58 | } 59 | 60 | /** 61 | * @throws TransactionShouldBeRetriedException 62 | * @throws TransactionShouldBeAbortedException 63 | * @throws TransactionFatalErrorException 64 | */ 65 | private function handleTransactionException(KafkaErrorException $exception): void 66 | { 67 | if ($exception->isRetriable() === true) { 68 | throw TransactionShouldBeRetriedException::new($exception); 69 | } 70 | 71 | if ($exception->transactionRequiresAbort() === true) { 72 | throw TransactionShouldBeAbortedException::new($exception); 73 | } 74 | 75 | $this->transactionInitialized = false; 76 | 77 | throw TransactionFatalErrorException::new($exception); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /docs/producing-messages/custom-serializers.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Custom serializers 3 | weight: 4 4 | --- 5 | 6 | Serialization is the process of converting messages to bytes. Deserialization is the inverse process - converting a stream of bytes into and object. In a nutshell, it transforms the content into readable and interpretable information. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | Basically, in order to prepare the message for transmission from the producer we use serializers. This package supports three serializers out of the box: 13 | 14 | - NullSerializer / NullDeserializer 15 | - JsonSerializer / JsonDeserializer 16 | - AvroSerializer / JsonDeserializer 17 | 18 | If the default `JsonSerializer` does not fulfill your needs, you can make use of custom serializers. 19 | 20 | To create a custom serializer, you need to create a class that implements the `\Junges\Kafka\Contracts\MessageSerializer` contract. This interface force you to declare the serialize method. 21 | 22 | You can inform your producer which serializer should be used with the `usingSerializer` method: 23 | 24 | ```php 25 | $producer = \Junges\Kafka\Facades\Kafka::publish('broker')->onTopic('topic')->usingSerializer(new MyCustomSerializer()); 26 | ``` 27 | 28 | To create a custom serializer, you need to create a class that implements the `\Junges\Kafka\Contracts\MessageSerializer` contract. 29 | This interface force you to declare the `serialize` method. 30 | 31 | ### Using AVRO serializer 32 | To use the AVRO serializer, add the AVRO serializer: 33 | 34 | ```php 35 | use FlixTech\AvroSerializer\Objects\RecordSerializer; 36 | use FlixTech\SchemaRegistryApi\Registry\CachedRegistry; 37 | use FlixTech\SchemaRegistryApi\Registry\BlockingRegistry; 38 | use FlixTech\SchemaRegistryApi\Registry\PromisingRegistry; 39 | use FlixTech\SchemaRegistryApi\Registry\Cache\AvroObjectCacheAdapter; 40 | use GuzzleHttp\Client; 41 | 42 | $cachedRegistry = new CachedRegistry( 43 | new BlockingRegistry( 44 | new PromisingRegistry( 45 | new Client(['base_uri' => 'kafka-schema-registry:9081']) 46 | ) 47 | ), 48 | new AvroObjectCacheAdapter() 49 | ); 50 | 51 | $registry = new AvroSchemaRegistry($cachedRegistry); 52 | $recordSerializer = new RecordSerializer($cachedRegistry); 53 | 54 | //if no version is defined, latest version will be used 55 | //if no schema definition is defined, the appropriate version will be fetched form the registry 56 | $registry->addBodySchemaMappingForTopic( 57 | 'test-topic', 58 | new \Junges\Kafka\Message\KafkaAvroSchema('bodySchemaName' /*, int $version, AvroSchema $definition */) 59 | ); 60 | $registry->addKeySchemaMappingForTopic( 61 | 'test-topic', 62 | new \Junges\Kafka\Message\KafkaAvroSchema('keySchemaName' /*, int $version, AvroSchema $definition */) 63 | ); 64 | 65 | $serializer = new \Junges\Kafka\Message\Serializers\AvroSerializer($registry, $recordSerializer /*, AvroEncoderInterface::ENCODE_BODY */); 66 | 67 | $producer = \Junges\Kafka\Facades\Kafka::publish('broker')->onTopic('topic')->usingSerializer($serializer); 68 | ``` -------------------------------------------------------------------------------- /src/Consumers/CallableConsumer.php: -------------------------------------------------------------------------------- 1 | handler = $this->handler instanceof Handler 26 | ? $handler 27 | : $handler(...); 28 | 29 | $this->dispatcher = App::make(Dispatcher::class); 30 | } 31 | 32 | /** Handle the received message. */ 33 | public function handle(ConsumerMessage $message, MessageConsumer $consumer): void 34 | { 35 | // If the message handler should be queued, we will dispatch a job to handle this message. 36 | // Otherwise, the message will be handled synchronously. 37 | if ($this->shouldQueueHandler()) { 38 | $this->queueHandler($this->handler, $message, $this->middlewares); 39 | 40 | return; 41 | } 42 | 43 | $this->handleMessageSynchronously($message, $consumer); 44 | } 45 | 46 | private function shouldQueueHandler(): bool 47 | { 48 | return $this->handler instanceof ShouldQueue; 49 | } 50 | 51 | private function handleMessageSynchronously(ConsumerMessage $message, MessageConsumer $consumer): void 52 | { 53 | $this->handleConsumedMessage($message, $this->handler, $consumer, $this->middlewares); 54 | } 55 | 56 | /** 57 | * This method dispatches a job to handle the consumed message. You can customize the connection and 58 | * queue in which it will be dispatched using the onConnection and onQueue methods. If this 59 | * methods doesn't exist in the handler class, we will use the default configuration accordingly to 60 | * your queue.php config file. 61 | */ 62 | private function queueHandler(Handler $handler, ConsumerMessage $message, array $middlewares): void 63 | { 64 | $connection = config('queue.default'); 65 | 66 | if (method_exists($handler, 'onConnection')) { 67 | $connection = $handler->onConnection(); 68 | } 69 | 70 | $queue = config("queue.$connection.queue", 'default'); 71 | 72 | if (method_exists($handler, 'onQueue')) { 73 | $queue = $handler->onQueue(); 74 | } 75 | 76 | $this->dispatcher->dispatch( 77 | (new DispatchQueuedHandler($handler, $message, $middlewares)) 78 | ->onQueue($queue) 79 | ->onConnection($connection) 80 | ); 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /tests/Console/Consumers/OptionsTest.php: -------------------------------------------------------------------------------- 1 | config = [ 19 | 'brokers' => config('kafka.brokers'), 20 | 'groupId' => config('kafka.group_id'), 21 | 'securityProtocol' => config('kafka.securityProtocol'), 22 | 'sasl' => [ 23 | 'mechanisms' => config('kafka.sasl.mechanisms'), 24 | 'username' => config('kafka.sasl.username'), 25 | 'password' => config('kafka.sasl.password'), 26 | ], 27 | ]; 28 | } 29 | 30 | #[Test] 31 | public function it_instantiate_the_class_with_correct_options(): void 32 | { 33 | $commandLineOptions = [ 34 | 'topics' => 'test-topic,test-topic-1', 35 | 'consumer' => FakeHandler::class, 36 | 'groupId' => 'test', 37 | 'commit' => 1, 38 | 'dlq' => 'test-dlq', 39 | 'maxMessages' => 2, 40 | 'securityProtocol' => 'plaintext', 41 | ]; 42 | 43 | $options = new Options($commandLineOptions, $this->config); 44 | 45 | $this->assertEquals('localhost:9092', $options->getBroker()); 46 | $this->assertEquals(['test-topic', 'test-topic-1'], $options->getTopics()); 47 | $this->assertEquals(FakeHandler::class, $options->getConsumer()); 48 | $this->assertEquals('test', $options->getGroupId()); 49 | $this->assertEquals(1, $options->getCommit()); 50 | $this->assertEquals('test-dlq', $options->getDlq()); 51 | $this->assertEquals(2, $options->getMaxMessages()); 52 | $this->assertEquals('plaintext', $options->getSecurityProtocol()); 53 | $this->assertNull($options->getSasl()); 54 | } 55 | 56 | #[Test] 57 | public function it_instantiates_using_only_required_options(): void 58 | { 59 | $options = [ 60 | 'topics' => 'test-topic,test-topic-1', 61 | 'consumer' => FakeHandler::class, 62 | ]; 63 | 64 | $options = new Options($options, $this->config); 65 | 66 | $this->assertEquals('localhost:9092', $options->getBroker()); 67 | $this->assertEquals(['test-topic', 'test-topic-1'], $options->getTopics()); 68 | $this->assertEquals(FakeHandler::class, $options->getConsumer()); 69 | $this->assertNull($options->getGroupId()); 70 | $this->assertEquals(1, $options->getCommit()); 71 | $this->assertNull($options->getDlq()); 72 | $this->assertEquals(-1, $options->getMaxMessages()); 73 | $this->assertEquals('plaintext', $options->getSecurityProtocol()); 74 | $this->assertNull($options->getSasl()); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /config/kafka.php: -------------------------------------------------------------------------------- 1 | env('KAFKA_BROKERS', 'localhost:9092'), 8 | 9 | /* 10 | | Default security protocol 11 | */ 12 | 'securityProtocol' => env('KAFKA_SECURITY_PROTOCOL', 'PLAINTEXT'), 13 | 14 | /* 15 | | Default sasl configuration 16 | */ 17 | 'sasl' => [ 18 | 'mechanisms' => env('KAFKA_MECHANISMS', 'PLAINTEXT'), 19 | 'username' => env('KAFKA_USERNAME', null), 20 | 'password' => env('KAFKA_PASSWORD', null), 21 | ], 22 | 23 | /* 24 | | Kafka consumers belonging to the same consumer group share a group id. 25 | | The consumers in a group then divides the topic partitions as fairly amongst themselves as possible by 26 | | establishing that each partition is only consumed by a single consumer from the group. 27 | | This config defines the consumer group id you want to use for your project. 28 | */ 29 | 'consumer_group_id' => env('KAFKA_CONSUMER_GROUP_ID', 'group'), 30 | 31 | 'consumer_timeout_ms' => env('KAFKA_CONSUMER_DEFAULT_TIMEOUT', 2000), 32 | 33 | /* 34 | | After the consumer receives its assignment from the coordinator, 35 | | it must determine the initial position for each assigned partition. 36 | | When the group is first created, before any messages have been consumed, the position is set according to a configurable 37 | | offset reset policy (auto.offset.reset). Typically, consumption starts either at the earliest offset or the latest offset. 38 | | You can choose between "latest", "earliest" or "none". 39 | */ 40 | 'offset_reset' => env('KAFKA_OFFSET_RESET', 'latest'), 41 | 42 | /* 43 | | If you set enable.auto.commit (which is the default), then the consumer will automatically commit offsets periodically at the 44 | | interval set by auto.commit.interval.ms. 45 | */ 46 | 'auto_commit' => env('KAFKA_AUTO_COMMIT', true), 47 | 48 | 'sleep_on_error' => env('KAFKA_ERROR_SLEEP', 5), 49 | 50 | 'partition' => env('KAFKA_PARTITION', 0), 51 | 52 | /* 53 | | Kafka supports 4 compression codecs: none , gzip , lz4 and snappy 54 | */ 55 | 'compression' => env('KAFKA_COMPRESSION_TYPE', 'snappy'), 56 | 57 | /* 58 | | Choose if debug is enabled or not. 59 | */ 60 | 'debug' => env('KAFKA_DEBUG', false), 61 | 62 | /* 63 | | The sleep time in milliseconds that will be used when retrying flush 64 | */ 65 | 'flush_retry_sleep_in_ms' => 100, 66 | 67 | /* 68 | * The number of retries that will be used when flushing the producer 69 | */ 70 | 'flush_retries' => 10, 71 | 72 | /** 73 | * The flush timeout in milliseconds 74 | */ 75 | 'flush_timeout_in_ms' => 1000, 76 | 77 | /* 78 | | The cache driver that will be used 79 | */ 80 | 'cache_driver' => env('KAFKA_CACHE_DRIVER', env('CACHE_DRIVER', env('CACHE_STORE', 'database'))), 81 | 82 | /* 83 | | Kafka message id key name 84 | */ 85 | 'message_id_key' => env('MESSAGE_ID_KEY', 'laravel-kafka::message-id'), 86 | ]; 87 | -------------------------------------------------------------------------------- /docs/consuming-messages/consumer-groups.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Consumer groups 3 | weight: 4 4 | --- 5 | 6 | Kafka consumers belonging to the same consumer group share a group id. The consumers in a group divides the topic partitions as fairly amongst themselves as possible by establishing that each partition is only consumed by a single consumer from the group. 7 | 8 | ```+parse 9 | 10 | ``` 11 | 12 | To attach your consumer to a consumer group, you can use the method `withConsumerGroupId` to specify the consumer group id: 13 | 14 | ```php 15 | use Junges\Kafka\Facades\Kafka; 16 | 17 | $consumer = Kafka::consumer()->withConsumerGroupId('foo'); 18 | ``` 19 | 20 | ### Kafka Consumer Group Rebalancing 21 | 22 | Watch how Kafka automatically redistributes partitions among consumers when the consumer group changes. Add or remove consumers to see the rebalancing process in action. 23 | 24 | ```+parse 25 | 26 | ``` 27 | 28 | ### Partition Assignment Strategies 29 | 30 | You can configure how Kafka assigns partitions to consumers in your consumer group by specifying a rebalance strategy: 31 | 32 | ```php 33 | use Junges\Kafka\Facades\Kafka; 34 | use Junges\Kafka\Config\RebalanceStrategy; 35 | 36 | $consumer = Kafka::consumer() 37 | ->withConsumerGroupId('my-group') 38 | ->withRebalanceStrategy(RebalanceStrategy::ROUND_ROBIN); 39 | ``` 40 | 41 | #### Available Strategies 42 | 43 | - **Range** (`RebalanceStrategy::RANGE`): Default strategy. Assigns partitions on a per-topic basis by dividing partitions evenly among consumers. 44 | - **Round Robin** (`RebalanceStrategy::ROUND_ROBIN`): Distributes partitions evenly across all consumers in a round-robin fashion. 45 | - **Sticky** (`RebalanceStrategy::STICKY`): Maintains balanced assignments while preserving existing assignments during rebalancing. 46 | - **Cooperative Sticky** (`RebalanceStrategy::COOPERATIVE_STICKY`): Same as sticky but allows cooperative rebalancing for reduced downtime. 47 | 48 | #### Examples 49 | 50 | ```php 51 | // Using Range strategy (default) 52 | $consumer = Kafka::consumer() 53 | ->withConsumerGroupId('my-group') 54 | ->withRebalanceStrategy(RebalanceStrategy::RANGE); 55 | 56 | // Using Round Robin for better distribution 57 | $consumer = Kafka::consumer() 58 | ->withConsumerGroupId('my-group') 59 | ->withRebalanceStrategy(RebalanceStrategy::ROUND_ROBIN); 60 | 61 | // Using Sticky for minimal disruption during rebalancing 62 | $consumer = Kafka::consumer() 63 | ->withConsumerGroupId('my-group') 64 | ->withRebalanceStrategy(RebalanceStrategy::STICKY); 65 | 66 | // Using Cooperative Sticky for minimal downtime 67 | $consumer = Kafka::consumer() 68 | ->withConsumerGroupId('my-group') 69 | ->withRebalanceStrategy(RebalanceStrategy::COOPERATIVE_STICKY); 70 | ``` 71 | 72 | You can also pass strategy names as strings: 73 | 74 | ```php 75 | $consumer = Kafka::consumer() 76 | ->withConsumerGroupId('my-group') 77 | ->withRebalanceStrategy('sticky'); 78 | ``` 79 | 80 | Or set the strategy using raw options: 81 | 82 | ```php 83 | $consumer = Kafka::consumer() 84 | ->withConsumerGroupId('my-group') 85 | ->withOption('partition.assignment.strategy', 'sticky'); 86 | ``` -------------------------------------------------------------------------------- /src/Message/Serializers/AvroSerializer.php: -------------------------------------------------------------------------------- 1 | registry; 23 | } 24 | 25 | public function serialize(ProducerMessage $message): ProducerMessage 26 | { 27 | $message = $this->encodeBody($message); 28 | 29 | return $this->encodeKey($message); 30 | } 31 | 32 | private function encodeBody(ProducerMessage $producerMessage): ProducerMessage 33 | { 34 | $topicName = $producerMessage->getTopicName(); 35 | $body = $producerMessage->getBody(); 36 | 37 | if ($body === null) { 38 | return $producerMessage; 39 | } 40 | 41 | if ($this->registry->hasBodySchemaForTopic($topicName) === false) { 42 | return $producerMessage; 43 | } 44 | 45 | $avroSchema = $this->registry->getBodySchemaForTopic($topicName); 46 | 47 | $encodedBody = $this->recordSerializer->encodeRecord( 48 | $avroSchema->getName(), 49 | $this->getAvroSchemaDefinition($avroSchema), 50 | $body 51 | ); 52 | 53 | return $producerMessage->withBody($encodedBody); 54 | } 55 | 56 | private function encodeKey(ProducerMessage $producerMessage): ProducerMessage 57 | { 58 | $topicName = $producerMessage->getTopicName(); 59 | $key = $producerMessage->getKey(); 60 | 61 | if ($key === null) { 62 | return $producerMessage; 63 | } 64 | 65 | if ($this->registry->hasKeySchemaForTopic($topicName) === false) { 66 | return $producerMessage; 67 | } 68 | 69 | $avroSchema = $this->registry->getKeySchemaForTopic($topicName); 70 | 71 | $encodedKey = $this->recordSerializer->encodeRecord( 72 | $avroSchema->getName(), 73 | $this->getAvroSchemaDefinition($avroSchema), 74 | $key 75 | ); 76 | 77 | return $producerMessage->withKey($encodedKey); 78 | } 79 | 80 | private function getAvroSchemaDefinition(KafkaAvroSchemaRegistry $avroSchema): AvroSchema 81 | { 82 | $schemaDefinition = $avroSchema->getDefinition(); 83 | 84 | if ($schemaDefinition === null) { 85 | throw new AvroSerializerException( 86 | sprintf( 87 | AvroSerializerException::UNABLE_TO_LOAD_DEFINITION_MESSAGE, 88 | $avroSchema->getName() 89 | ) 90 | ); 91 | } 92 | 93 | return $schemaDefinition; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /tests/Commit/RetryableCommitterTest.php: -------------------------------------------------------------------------------- 1 | commitMessage(new Message, false); 23 | $retryableCommitter->commitDlq(new Message); 24 | 25 | $this->assertEquals(4, $failingCommitter->getTimesTriedToCommitMessage()); 26 | $this->assertEquals(4, $failingCommitter->getTimesTriedToCommitDlq()); 27 | } 28 | 29 | #[Test] 30 | public function it_should_retry_only_up_to_the_maximum_number_of_retries(): void 31 | { 32 | $expectedException = new RdKafkaException('Something went wrong', RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT); 33 | $failingCommitter = new FailingCommitter($expectedException, 99); 34 | $retryableCommitter = new RetryableCommitter($failingCommitter, new FakeSleeper, 4); 35 | 36 | $commitMessageException = null; 37 | 38 | try { 39 | $retryableCommitter->commitMessage(new Message, false); 40 | } catch (RdKafkaException $exception) { 41 | $commitMessageException = $exception; 42 | } 43 | 44 | $commitDlqException = null; 45 | 46 | try { 47 | $retryableCommitter->commitDlq(new Message); 48 | } catch (RdKafkaException $exception) { 49 | $commitDlqException = $exception; 50 | } 51 | 52 | // first execution + 4 retries = 5 executions 53 | $this->assertEquals(5, $failingCommitter->getTimesTriedToCommitMessage()); 54 | $this->assertSame($expectedException, $commitMessageException); 55 | 56 | $this->assertEquals(5, $failingCommitter->getTimesTriedToCommitDlq()); 57 | $this->assertSame($expectedException, $commitDlqException); 58 | } 59 | 60 | #[Test] 61 | public function it_should_progressively_wait_for_the_next_retry(): void 62 | { 63 | $expectedException = new RdKafkaException('Something went wrong', RD_KAFKA_RESP_ERR_REQUEST_TIMED_OUT); 64 | 65 | $sleeper = new FakeSleeper; 66 | $failingCommitter = new FailingCommitter($expectedException, 99); 67 | $retryableCommitter = new RetryableCommitter($failingCommitter, $sleeper, 6); 68 | 69 | try { 70 | $retryableCommitter->commitMessage(new Message, true); 71 | } catch (RdKafkaException $exception) { 72 | } 73 | 74 | $expectedSleeps = [1e6, 2e6, 4e6, 8e6, 16e6, 32e6]; 75 | $this->assertEquals($expectedSleeps, $sleeper->getSleeps()); 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /tests/Commit/CommitterFactoryTest.php: -------------------------------------------------------------------------------- 1 | createMock(Consumer::class), 29 | sasl: null, 30 | dlq: null, 31 | maxMessages: -1, 32 | maxCommitRetries: 6, 33 | autoCommit: false 34 | ); 35 | 36 | $consumer = $this->createMock(KafkaConsumer::class); 37 | 38 | $messageCounter = new MessageCounter(6); 39 | 40 | $factory = new DefaultCommitterFactory($messageCounter); 41 | 42 | $committer = $factory->make($consumer, $config); 43 | 44 | $expectedCommitter = new BatchCommitter( 45 | new RetryableCommitter( 46 | new Committer( 47 | $consumer 48 | ), 49 | new NativeSleeper, 50 | $config->getMaxCommitRetries() 51 | ), 52 | $messageCounter, 53 | $config->getCommit() 54 | ); 55 | 56 | $this->assertEquals($expectedCommitter, $committer); 57 | } 58 | 59 | #[Test] 60 | public function should_build_a_retryable_batch_committer_when_auto_commit_is_enabled(): void 61 | { 62 | $config = new Config( 63 | broker: 'broker', 64 | topics: ['topic'], 65 | securityProtocol: 'security', 66 | commit: 1, 67 | groupId: 'group', 68 | consumer: $this->createMock(Consumer::class), 69 | sasl: null, 70 | dlq: null, 71 | maxMessages: 6, 72 | maxCommitRetries: 6, 73 | autoCommit: true 74 | ); 75 | 76 | $consumer = $this->createMock(KafkaConsumer::class); 77 | 78 | $messageCounter = new MessageCounter(6); 79 | 80 | $factory = new DefaultCommitterFactory($messageCounter); 81 | 82 | $committer = $factory->make($consumer, $config); 83 | 84 | $expectedCommitter = new BatchCommitter( 85 | new RetryableCommitter( 86 | new Committer( 87 | $consumer 88 | ), 89 | new NativeSleeper, 90 | $config->getMaxCommitRetries() 91 | ), 92 | $messageCounter, 93 | $config->getCommit() 94 | ); 95 | 96 | $this->assertEquals($expectedCommitter, $committer); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /docs/consuming-messages/class-structure.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Class structure 3 | weight: 10 4 | --- 5 | 6 | Consumer classes are very simple, and it is basically a Laravel Command class. To get started, let's take a look at an example consumer. 7 | 8 | ```php 9 | withBrokers('localhost:8092') 28 | ->withAutoCommit() 29 | ->withHandler(function(ConsumerMessage $message, MessageConsumer $consumer) { 30 | // Handle your message here 31 | // For manual commit control, use ->withManualCommit() and call $consumer->commit($message) 32 | }) 33 | ->build(); 34 | 35 | $consumer->consume(); 36 | } 37 | } 38 | ``` 39 | 40 | Now, to keep this consumer process running permanently in the background, you should use a process monitor such as [supervisor](http://supervisord.org/) to ensure that the consumer does not stop running. 41 | 42 | ```+parse 43 | 44 | ``` 45 | 46 | ## Supervisor configuration 47 | In production, you need a way to keep your consumer processes running. For this reason, you need to configure a process monitor that can detect when your consumer processes exit and automatically restart them. In addition, process monitors can allow you to specify how many consumer processes you would like to run concurrently. Supervisor is a process monitor commonly used in Linux environments and we will discuss how to configure it in the following documentation. 48 | 49 | ### Installing supervisor 50 | To install supervisor on Ubuntu, you may use the following command: 51 | ```bash 52 | sudo apt-get install supervisor 53 | ``` 54 | 55 | On mac, you can use homebrew: 56 | 57 | ```bash 58 | brew install supervisor 59 | ``` 60 | 61 | ### Configuring supervisor 62 | Supervisor configuration files are typically stored in the `/etc/supervisor/conf.d` directory. Within this directory, you may create any number of configuration files that instruct supervisor how your processes should be monitored. For example, let's create a `my-topic-consumer.conf` file that starts and monitors our Consumer: 63 | 64 | ```text 65 | [program:my-topic-consumer] 66 | directory=/var/www/html 67 | process_name=%(program_name)s_%(process_num)02d 68 | command=php artisan consume:my-topic 69 | autostart=true 70 | autorestart=true 71 | redirect_stderr=true 72 | stdout_logfile=/var/log/supervisor-laravel-worker.log 73 | stopwaitsecs=3600 74 | ``` 75 | 76 | #### Starting Supervisor 77 | Onnce the configuration file has been created, you may update Supervisor configuration and start the processes using the following commands: 78 | 79 | ```bash 80 | sudo supervisorctl reread 81 | 82 | sudo supervisorctl update 83 | 84 | sudo supervisorctl start my-topic-consumer:* 85 | ``` 86 | -------------------------------------------------------------------------------- /src/Message/Registry/AvroSchemaRegistry.php: -------------------------------------------------------------------------------- 1 | */ 14 | private array $schemaMapping = [ 15 | self::BODY_IDX => [], 16 | self::KEY_IDX => [], 17 | ]; 18 | 19 | /** AvroSchemaRegistry constructor. */ 20 | public function __construct(private readonly Registry $registry) {} 21 | 22 | public function addBodySchemaMappingForTopic(string $topicName, KafkaAvroSchemaRegistry $avroSchema): void 23 | { 24 | $this->schemaMapping[self::BODY_IDX][$topicName] = $avroSchema; 25 | } 26 | 27 | public function addKeySchemaMappingForTopic(string $topicName, KafkaAvroSchemaRegistry $avroSchema): void 28 | { 29 | $this->schemaMapping[self::KEY_IDX][$topicName] = $avroSchema; 30 | } 31 | 32 | public function getBodySchemaForTopic(string $topicName): KafkaAvroSchemaRegistry 33 | { 34 | return $this->getSchemaForTopicAndType($topicName, self::BODY_IDX); 35 | } 36 | 37 | public function getKeySchemaForTopic(string $topicName): KafkaAvroSchemaRegistry 38 | { 39 | return $this->getSchemaForTopicAndType($topicName, self::KEY_IDX); 40 | } 41 | 42 | public function hasBodySchemaForTopic(string $topicName): bool 43 | { 44 | return isset($this->schemaMapping[self::BODY_IDX][$topicName]); 45 | } 46 | 47 | public function hasKeySchemaForTopic(string $topicName): bool 48 | { 49 | return isset($this->schemaMapping[self::KEY_IDX][$topicName]); 50 | } 51 | 52 | /** @return array */ 53 | public function getTopicSchemaMapping(): array 54 | { 55 | return $this->schemaMapping; 56 | } 57 | 58 | private function getSchemaForTopicAndType(string $topicName, string $type): KafkaAvroSchemaRegistry 59 | { 60 | if (isset($this->schemaMapping[$type][$topicName]) === false) { 61 | throw new SchemaRegistryException( 62 | sprintf( 63 | SchemaRegistryException::SCHEMA_MAPPING_NOT_FOUND, 64 | $topicName, 65 | $type 66 | ) 67 | ); 68 | } 69 | 70 | $avroSchema = $this->schemaMapping[$type][$topicName]; 71 | 72 | if ($avroSchema->getDefinition() !== null) { 73 | return $avroSchema; 74 | } 75 | 76 | $avroSchema->setDefinition($this->getSchemaDefinition($avroSchema)); 77 | $this->schemaMapping[$type][$topicName] = $avroSchema; 78 | 79 | return $avroSchema; 80 | } 81 | 82 | private function getSchemaDefinition(KafkaAvroSchemaRegistry $avroSchema): AvroSchema 83 | { 84 | if ($avroSchema->getVersion() === KafkaAvroSchemaRegistry::LATEST_VERSION) { 85 | return $this->registry->latestVersion($avroSchema->getName()); 86 | } 87 | 88 | return $this->registry->schemaForSubjectAndVersion($avroSchema->getName(), $avroSchema->getVersion()); 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/Console/Commands/KafkaConsumer/Options.php: -------------------------------------------------------------------------------- 1 | $value) { 40 | $this->{$option} = $value; 41 | } 42 | 43 | $this->saslPassword = $config['sasl']['password']; 44 | $this->saslUsername = $config['sasl']['username']; 45 | $this->saslMechanisms = $config['sasl']['mechanisms']; 46 | } 47 | 48 | public function getTopics(): array 49 | { 50 | return ! empty($this->topics) ? $this->topics : []; 51 | } 52 | 53 | public function getConsumer(): ?string 54 | { 55 | return $this->consumer; 56 | } 57 | 58 | public function getDeserializer(): ?string 59 | { 60 | return $this->deserializer; 61 | } 62 | 63 | public function getGroupId(): ?string 64 | { 65 | return mb_strlen((string) $this->groupId) > 1 ? $this->groupId : $this->config['groupId']; 66 | } 67 | 68 | public function getCommit(): int 69 | { 70 | return $this->commit; 71 | } 72 | 73 | public function getDlq(): ?string 74 | { 75 | return mb_strlen((string) $this->dlq) > 1 ? $this->dlq : null; 76 | } 77 | 78 | public function getMaxMessages(): int 79 | { 80 | return $this->maxMessages >= 1 ? $this->maxMessages : -1; 81 | } 82 | 83 | public function getMaxTime(): int 84 | { 85 | return $this->maxTime; 86 | } 87 | 88 | #[Pure] 89 | public function getSasl(): ?Sasl 90 | { 91 | if (is_null($this->saslMechanisms) || is_null($this->saslPassword) || is_null($this->saslUsername)) { 92 | return null; 93 | } 94 | 95 | return new Sasl( 96 | username: $this->saslUsername, 97 | password: $this->saslPassword, 98 | mechanisms: $this->saslMechanisms, 99 | securityProtocol: $this->getSecurityProtocol() 100 | ); 101 | } 102 | 103 | public function getSecurityProtocol(): ?string 104 | { 105 | $securityProtocol = mb_strlen($this->securityProtocol ?? '') > 1 106 | ? $this->securityProtocol 107 | : $this->config['securityProtocol']; 108 | 109 | return $securityProtocol ?? 'plaintext'; 110 | } 111 | 112 | public function getBroker() 113 | { 114 | return $this->config['brokers']; 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /tests/Message/MessageTest.php: -------------------------------------------------------------------------------- 1 | message = new Message; 18 | } 19 | 20 | #[Test] 21 | public function it_can_set_a_message_key(): void 22 | { 23 | $this->message->withBodyKey('foo', 'bar'); 24 | 25 | $expected = new Message( 26 | body: ['foo' => 'bar'] 27 | ); 28 | 29 | $this->assertEquals($expected, $this->message); 30 | } 31 | 32 | #[Test] 33 | public function it_can_forget_a_message_key(): void 34 | { 35 | $this->message->withBodyKey('foo', 'bar'); 36 | $this->message->withBodyKey('bar', 'foo'); 37 | 38 | $expected = new Message( 39 | body: ['bar' => 'foo'] 40 | ); 41 | 42 | $this->message->forgetBodyKey('foo'); 43 | 44 | $this->assertEquals($expected, $this->message); 45 | } 46 | 47 | #[Test] 48 | public function it_can_set_message_headers(): void 49 | { 50 | $this->message->withHeaders([ 51 | 'foo' => 'bar', 52 | ]); 53 | 54 | $expected = new Message( 55 | headers: ['foo' => 'bar'] 56 | ); 57 | 58 | $this->assertEquals($expected, $this->message); 59 | } 60 | 61 | #[Test] 62 | public function it_can_set_the_message_key(): void 63 | { 64 | $this->message->withKey($uuid = Str::uuid()->toString()); 65 | 66 | $expected = new Message( 67 | key: $uuid 68 | ); 69 | 70 | $this->assertEquals($expected, $this->message); 71 | } 72 | 73 | #[Test] 74 | public function it_can_get_the_message_payload(): void 75 | { 76 | $this->message->withBodyKey('foo', 'bar'); 77 | $this->message->withBodyKey('bar', 'foo'); 78 | 79 | $expectedMessage = new Message( 80 | body: $array = ['foo' => 'bar', 'bar' => 'foo'] 81 | ); 82 | 83 | $this->assertEquals($expectedMessage, $this->message); 84 | 85 | $expectedPayload = $array; 86 | 87 | $this->assertEquals($expectedPayload, $this->message->getBody()); 88 | } 89 | 90 | #[Test] 91 | public function it_can_transform_a_message_in_array(): void 92 | { 93 | $this->message->withBodyKey('foo', 'bar'); 94 | $this->message->withBodyKey('bar', 'foo'); 95 | $this->message->withKey($uuid = Str::uuid()->toString()); 96 | $this->message->withHeaders($headers = ['foo' => 'bar']); 97 | 98 | $expectedMessage = new Message( 99 | headers: $headers, 100 | body: $array = ['foo' => 'bar', 'bar' => 'foo'], 101 | key: $uuid 102 | ); 103 | 104 | $expectedArray = [ 105 | 'payload' => $array, 106 | 'key' => $uuid, 107 | 'headers' => $headers, 108 | ]; 109 | 110 | $this->assertEquals($expectedMessage, $this->message); 111 | $this->assertEquals($expectedArray, $this->message->toArray()); 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/Factory.php: -------------------------------------------------------------------------------- 1 | This array is passed to the underlying consumer when faking macroed consumers. */ 20 | private array $fakeMessages = []; 21 | 22 | private ?ProducerBuilder $builder = null; 23 | 24 | /** Creates a new ProducerBuilder instance, setting brokers and topic. */ 25 | public function publish(?string $broker = null): MessageProducer 26 | { 27 | if ($this->shouldFake) { 28 | return Kafka::fake()->publish($broker); 29 | } 30 | 31 | return new ProducerBuilder( 32 | broker: $broker ?? config('kafka.brokers') 33 | ); 34 | } 35 | 36 | /** Returns a fresh factory instance. */ 37 | public function fresh(): self 38 | { 39 | return new self; 40 | } 41 | 42 | /** 43 | * Creates a new ProducerBuilder instance, optionally setting the brokers. 44 | * The producer will be flushed only when the application terminates, 45 | * and doing SEND does not mean that the message was flushed! 46 | */ 47 | public function asyncPublish(?string $broker = null): MessageProducer 48 | { 49 | if ($this->shouldFake) { 50 | return Kafka::fake()->publish($broker); 51 | } 52 | 53 | if ($this->builder instanceof ProducerBuilder) { 54 | return $this->builder; 55 | } 56 | 57 | $this->builder = new ProducerBuilder( 58 | broker: $broker ?? config('kafka.brokers'), 59 | asyncProducer: true 60 | ); 61 | 62 | return $this->builder; 63 | } 64 | 65 | /** This is an alias for the asyncPublish method. */ 66 | public function publishAsync(?string $broker = null): MessageProducer 67 | { 68 | return $this->asyncPublish($broker); 69 | } 70 | 71 | /** Return a ConsumerBuilder instance. */ 72 | public function consumer(array $topics = [], ?string $groupId = null, ?string $brokers = null): ConsumerBuilder 73 | { 74 | if ($this->shouldFake) { 75 | return Kafka::fake()->consumer( 76 | $topics, 77 | $groupId, 78 | $brokers 79 | )->setMessages($this->fakeMessages); 80 | } 81 | 82 | return ConsumerBuilder::create( 83 | brokers: $brokers ?? config('kafka.brokers'), 84 | topics: $topics, 85 | groupId: $groupId ?? config('kafka.consumer_group_id') 86 | ); 87 | } 88 | 89 | public function shouldFake(): self 90 | { 91 | $this->shouldFake = true; 92 | 93 | return $this; 94 | } 95 | 96 | /** @param array $messages */ 97 | public function shouldReceiveMessages(array $messages): self 98 | { 99 | $this->fakeMessages = $messages; 100 | 101 | return $this; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /src/Console/Commands/ConsumerCommand.php: -------------------------------------------------------------------------------- 1 | config = [ 36 | 'brokers' => config('kafka.brokers'), 37 | 'groupId' => config('kafka.consumer_group_id'), 38 | 'securityProtocol' => config('kafka.securityProtocol'), 39 | 'sasl' => [ 40 | 'mechanisms' => config('kafka.sasl.mechanisms'), 41 | 'username' => config('kafka.sasl.username'), 42 | 'password' => config('kafka.sasl.password'), 43 | ], 44 | ]; 45 | } 46 | 47 | public function handle(): int 48 | { 49 | if (empty($this->option('consumer'))) { 50 | $this->error('The [--consumer] option is required.'); 51 | 52 | return SymfonyCommand::SUCCESS; 53 | } 54 | 55 | if (empty($this->option('topics'))) { 56 | $this->error('The [--topics option is required.'); 57 | 58 | return SymfonyCommand::SUCCESS; 59 | } 60 | 61 | $parsedOptions = array_map($this->parseOptions(...), $this->options()); 62 | 63 | $options = new Options($parsedOptions, $this->config); 64 | 65 | $consumer = $options->getConsumer(); 66 | $deserializer = $options->getDeserializer(); 67 | 68 | $config = new Config( 69 | broker: $options->getBroker(), 70 | topics: $options->getTopics(), 71 | securityProtocol: $options->getSecurityProtocol(), 72 | commit: $options->getCommit(), 73 | groupId: $options->getGroupId(), 74 | consumer: app($consumer), 75 | sasl: $options->getSasl(), 76 | dlq: $options->getDlq(), 77 | maxMessages: $options->getMaxMessages(), 78 | maxTime: $options->getMaxTime(), 79 | ); 80 | 81 | /** @var Consumer $consumer */ 82 | $consumer = app(Consumer::class, [ 83 | 'config' => $config, 84 | 'deserializer' => app($deserializer ?? MessageDeserializer::class), 85 | ]); 86 | 87 | $consumer->consume(); 88 | 89 | return SymfonyCommand::SUCCESS; 90 | } 91 | 92 | private function parseOptions(int|string|null $option): int|string|null 93 | { 94 | if ($option === '?') { 95 | return null; 96 | } 97 | 98 | if (is_numeric($option)) { 99 | return (int) $option; 100 | } 101 | 102 | return $option; 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /docs/upgrade-guide.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: Upgrade guide 3 | weight: 6 4 | --- 5 | 6 | ## Upgrade to v2.9 from v2.8 7 | 8 | - **BREAKING CHANGE**: Deprecated producer batch messages feature has been removed (`MessageBatch`, `sendBatch`, `produceBatch`). Use `Kafka::asyncPublish()` instead for better performance 9 | - **BREAKING CHANGE**: Deprecated consumer batch messages feature has been removed (`enableBatching()`, `withBatchSizeLimit()`, `withBatchReleaseInterval()`). Process messages individually in your consumer handler 10 | - Removed classes: `BatchMessageConsumer`, `HandlesBatchConfiguration`, `BatchConfig`, `NullBatchConfig`, `CallableBatchConsumer`, etc. 11 | - Removed events: `BatchMessagePublished`, `MessageBatchPublished`, `PublishingMessageBatch` 12 | 13 | ## Upgrade to v2.8 from v2.x 14 | The only breaking change in this version was the change in the `Junges\Kafka\Contracts\Handler` contract signature. 15 | 16 | The `handle` method now requires a second parameter of type `Junges\Kafka\Contracts\MessageConsumer`. 17 | 18 | Here's the updated signature: 19 | ```diff 20 | class MyHandler implements Handler { 21 | - public function __invoke(ConsumerMessage $message): void { 22 | + public function __invoke(ConsumerMessage $message, MessageConsumer $consumer): void { 23 | // Process message here... 24 | } 25 | } 26 | ``` 27 | 28 | If you are handling your messages using a closure, no changes are needed as the closure signature already supports the second parameter. 29 | 30 | ## Upgrade to v2.x from v1.13.x 31 | 32 | ## High impact changes 33 | - The `\Junges\Kafka\Contracts\CanProduceMessages` contract was renamed to `\Junges\Kafka\Contracts\MessageProducer` 34 | - The `\Junges\Kafka\Contracts\KafkaProducerMessage` contract was renamed to `\Junges\Kafka\Contracts\ProducerMessage` 35 | - The `\Junges\Kafka\Contracts\CanConsumeMessages` was renamed to `\Junges\Kafka\Contracts\MessageConsumer` 36 | - The `\Junges\Kafka\Contracts\KafkaConsumerMessage` was renamed to `\Junges\Kafka\Contracts\ConsumerMessage` 37 | - The `\Junges\Kafka\Contracts\CanPublishMessagesToKafka` contract was removed. 38 | - The `\Junges\Kafka\Contracts\CanConsumeMessagesFromKafka` was removed. 39 | - The `\Junges\Kafka\Contracts\CanConsumeBatchMessages` contract was renamed to `\Junges\Kafka\Contracts\BatchMessageConsumer` 40 | - The `\Junges\Kafka\Contracts\CanConsumeMessages` contract was renamed to `\Junges\Kafka\Contracts\MessageConsumer` 41 | - Introduced a new `\Junges\Kafka\Contracts\Manager` used by `\Junges\Kafka\Factory` class 42 | 43 | ### The `withSasl` method signature was changed. 44 | 45 | The `withSasl` method now accepts all `SASL` parameters instead of a `Sasl` object. 46 | ```php 47 | public function withSasl(string $username, string $password, string $mechanisms, string $securityProtocol = 'SASL_PLAINTEXT'); 48 | ``` 49 | 50 | ### Handler functions require a second parameter 51 | 52 | In v2 handler functions and handler classes require a `\Junges\Kafka\Contracts\MessageConsumer` as a second argument. 53 | 54 | ```diff 55 | $consumer = Kafka::consumer(['topic']) 56 | ->withConsumerGroupId('group') 57 | - ->withHandler(function(ConsumerMessage $message) { 58 | + ->withHandler(function(ConsumerMessage $message, MessageConsumer $consumer) { 59 | // 60 | }) 61 | ``` 62 | 63 | ### Renamed `createConsumer` method 64 | The `Kafka::createConsumer` method has been renamed to just `consumer` 65 | 66 | ### Renamed `publishOn` method 67 | The `Kafka::publishOn` method has been renamed to `publish`, and it does not accept the `$topics` parameter anymore. 68 | 69 | Please chain a call to `onTopic` to specify in which topic the message should be published. 70 | 71 | ```php 72 | \Junges\Kafka\Facades\Kafka::publish('broker')->onTopic('topic-name'); 73 | ``` 74 | 75 | ### Setting `onStopConsuming` callbacks 76 | 77 | To set `onStopConsuming` callbacks you need to define them while building the consumer, instead of after calling the `build` method as in `v1.13.x`: 78 | 79 | ```diff 80 | $consumer = Kafka::consumer(['topic']) 81 | ->withConsumerGroupId('group') 82 | ->withHandler(new Handler) 83 | + ->onStopConsuming(static function () { 84 | + // Do something when the consumer stop consuming messages 85 | + }) 86 | ->build() 87 | - ->onStopConsuming(static function () { 88 | - // Do something when the consumer stop consuming messages 89 | - }) 90 | ``` 91 | 92 | 93 | ### Updating dependencies 94 | **PHP 8.2 Required** 95 | 96 | This package now requires PHP 8.2 or higher. 97 | 98 | You can use tools such as [rector](https://github.com/rectorphp/rector) to upgrade your app to PHP 8.2. 99 | -------------------------------------------------------------------------------- /src/Producers/Producer.php: -------------------------------------------------------------------------------- 1 | producer = app(KafkaProducer::class, [ 36 | 'conf' => $this->getConf($this->config->getProducerOptions()), 37 | ]); 38 | $this->dispatcher = App::make(Dispatcher::class); 39 | } 40 | 41 | public function __destruct() 42 | { 43 | if ($this->async) { 44 | $this->flush(); 45 | } 46 | } 47 | 48 | /** {@inheritDoc} */ 49 | public function produce(ProducerMessage $message): bool 50 | { 51 | $this->dispatcher->dispatch(new PublishingMessage($message)); 52 | 53 | $topic = $this->producer->newTopic($message->getTopicName()); 54 | 55 | $message = clone $message; 56 | 57 | $message = $this->serializer->serialize($message); 58 | 59 | $this->produceMessage($topic, $message); 60 | 61 | $this->producer->poll(0); 62 | 63 | if ($this->async) { 64 | return true; 65 | } 66 | 67 | return $this->flush(); 68 | } 69 | 70 | /** 71 | * @throws CouldNotPublishMessage 72 | * @throws Exception 73 | */ 74 | public function flush(): mixed 75 | { 76 | // Here we define the flush callback that is called shutting down a consumer. 77 | // This is called after every single message sent using Producer::send 78 | $flush = function () { 79 | $sleepMilliseconds = config('kafka.flush_retry_sleep_in_ms', 100); 80 | $retries = $this->config->flushRetries ?? config('kafka.flush_retries', 10); 81 | $timeout = $this->config->flushTimeoutInMs ?? config('kafka.flush_timeout_in_ms', 1000); 82 | 83 | try { 84 | return retry($retries, function () use ($timeout) { 85 | $result = $this->producer->flush($timeout); 86 | 87 | if ($result === RD_KAFKA_RESP_ERR_NO_ERROR) { 88 | return true; 89 | } 90 | 91 | $message = rd_kafka_err2str($result); 92 | 93 | throw CouldNotPublishMessage::withMessage($message, $result); 94 | }, $sleepMilliseconds); 95 | } catch (CouldNotPublishMessage $exception) { 96 | $this->dispatcher->dispatch(new \Junges\Kafka\Events\CouldNotPublishMessage( 97 | $exception->getKafkaErrorCode(), 98 | $exception->getMessage(), 99 | $exception, 100 | )); 101 | 102 | throw $exception; 103 | } 104 | }; 105 | 106 | return $flush(); 107 | } 108 | 109 | /** Set the Kafka Configuration. */ 110 | private function getConf(array $options): Conf 111 | { 112 | $conf = new Conf; 113 | 114 | foreach ($options as $key => $value) { 115 | $conf->set($key, (string) $value); 116 | } 117 | 118 | foreach ($this->config->getConfigCallbacks() as $method => $callback) { 119 | $conf->{$method}($callback); 120 | } 121 | 122 | return $conf; 123 | } 124 | 125 | private function produceMessage(ProducerTopic $topic, ProducerMessage $message): void 126 | { 127 | $topic->producev( 128 | partition: $message->getPartition(), 129 | msgflags: RD_KAFKA_MSG_F_BLOCK, 130 | payload: $message->getBody(), 131 | key: $message->getKey(), 132 | headers: $message->getHeaders() 133 | ); 134 | 135 | $this->dispatcher->dispatch(new MessagePublished($message)); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /src/Support/Testing/Fakes/ConsumerFake.php: -------------------------------------------------------------------------------- 1 | messageCounter = new MessageCounter($config->getMaxMessages()); 25 | $this->whenStopConsuming = $this->config->getWhenStopConsumingCallback(); 26 | } 27 | 28 | /** Consume messages from a kafka topic in loop. */ 29 | public function consume(): void 30 | { 31 | $this->doConsume(); 32 | 33 | if ($this->shouldRunStopConsumingCallback()) { 34 | $callback = $this->whenStopConsuming; 35 | $callback(...)(); 36 | } 37 | } 38 | 39 | /** {@inheritdoc} */ 40 | public function stopConsuming(): void 41 | { 42 | $this->stopRequested = true; 43 | } 44 | 45 | /** Will cancel the stopConsume request initiated by calling the stopConsume method */ 46 | public function cancelStopConsume(): void 47 | { 48 | $this->stopRequested = false; 49 | $this->whenStopConsuming = null; 50 | } 51 | 52 | /** Count the number of messages consumed by this consumer */ 53 | public function consumedMessagesCount(): int 54 | { 55 | return $this->messageCounter->messagesCounted(); 56 | } 57 | 58 | /** {@inheritdoc} */ 59 | public function commit(mixed $messageOrOffsets = null): void 60 | { 61 | // 62 | } 63 | 64 | /** {@inheritdoc} */ 65 | public function commitAsync(mixed $message_or_offsets = null): void 66 | { 67 | // 68 | } 69 | 70 | /** Get the current partition assignment for this consumer */ 71 | public function getAssignedPartitions(): array 72 | { 73 | return []; 74 | } 75 | 76 | /** Set the consumer configuration. */ 77 | public function setConf(array $options = []): Conf 78 | { 79 | return new Conf; 80 | } 81 | 82 | /** 83 | * Consume messages 84 | */ 85 | public function doConsume(): void 86 | { 87 | foreach ($this->messages as $message) { 88 | if ($this->shouldStopConsuming()) { 89 | break; 90 | } 91 | 92 | $this->handleMessage($message); 93 | } 94 | } 95 | 96 | private function shouldRunStopConsumingCallback(): bool 97 | { 98 | return $this->whenStopConsuming !== null; 99 | } 100 | 101 | /** Determine if the max message limit is reached. */ 102 | private function maxMessagesLimitReached(): bool 103 | { 104 | return $this->messageCounter->maxMessagesLimitReached(); 105 | } 106 | 107 | /** Return if the consumer should stop consuming messages. */ 108 | private function shouldStopConsuming(): bool 109 | { 110 | return $this->maxMessagesLimitReached() || $this->stopRequested; 111 | } 112 | 113 | /** Handle the message. */ 114 | private function handleMessage(ConsumerMessage $message): void 115 | { 116 | $this->config->getConsumer()->handle($message, $this); 117 | $this->messageCounter->add(); 118 | } 119 | 120 | private function getRdKafkaMessage(ConsumerMessage $message): Message 121 | { 122 | $rdKafkaMessage = new Message; 123 | $rdKafkaMessage->err = 0; 124 | $rdKafkaMessage->topic_name = $message->getTopicName(); 125 | $rdKafkaMessage->partition = $message->getPartition(); 126 | $rdKafkaMessage->headers = $message->getHeaders() ?? []; 127 | $rdKafkaMessage->payload = serialize($message->getBody()); 128 | $rdKafkaMessage->key = $message->getKey(); 129 | $rdKafkaMessage->offset = $message->getOffset(); 130 | $rdKafkaMessage->timestamp = $message->getTimestamp(); 131 | 132 | return $rdKafkaMessage; 133 | } 134 | 135 | private function getConsumerMessage(Message $message): ConsumerMessage 136 | { 137 | return app(ConsumerMessage::class, [ 138 | 'topicName' => $message->topic_name, 139 | 'partition' => $message->partition, 140 | 'headers' => $message->headers ?? [], 141 | 'body' => unserialize($message->payload), 142 | 'key' => $message->key, 143 | 'offset' => $message->offset, 144 | 'timestamp' => $message->timestamp, 145 | ]); 146 | } 147 | } 148 | --------------------------------------------------------------------------------