├── .circleci └── config.yml ├── .github └── CODEOWNERS ├── .gitignore ├── LICENSE ├── MIGRATION.md ├── Makefile ├── README.md ├── UPGRADE.md ├── composer.json ├── docker ├── .env ├── dev │ └── php │ │ ├── Dockerfile │ │ └── files │ │ ├── bin │ │ ├── php-ext-disable │ │ └── php-ext-enable │ │ └── php │ │ └── 20-pcov.ini ├── docker-compose.ci.yml └── docker-compose.yml ├── infection.json ├── phpcs.xml ├── phpstan.neon ├── phpunit.xml ├── src ├── Callback │ ├── KafkaConsumerRebalanceCallback.php │ ├── KafkaErrorCallback.php │ └── KafkaProducerDeliveryReportCallback.php ├── Conf │ └── KafkaConfiguration.php ├── Consumer │ ├── AbstractKafkaConsumer.php │ ├── KafkaConsumerBuilder.php │ ├── KafkaConsumerBuilderInterface.php │ ├── KafkaConsumerInterface.php │ ├── KafkaHighLevelConsumer.php │ ├── KafkaHighLevelConsumerInterface.php │ ├── KafkaLowLevelConsumer.php │ ├── KafkaLowLevelConsumerInterface.php │ ├── TopicSubscription.php │ └── TopicSubscriptionInterface.php ├── Exception │ ├── AvroEncoderException.php │ ├── AvroSchemaRegistryException.php │ ├── KafkaBrokerException.php │ ├── KafkaConsumerAssignmentException.php │ ├── KafkaConsumerBuilderException.php │ ├── KafkaConsumerCommitException.php │ ├── KafkaConsumerConsumeException.php │ ├── KafkaConsumerEndOfPartitionException.php │ ├── KafkaConsumerRequestException.php │ ├── KafkaConsumerSubscriptionException.php │ ├── KafkaConsumerTimeoutException.php │ ├── KafkaMessageException.php │ ├── KafkaProducerException.php │ ├── KafkaProducerTransactionAbortException.php │ ├── KafkaProducerTransactionFatalException.php │ ├── KafkaProducerTransactionRetryException.php │ └── KafkaRebalanceException.php ├── Message │ ├── AbstractKafkaMessage.php │ ├── Decoder │ │ ├── AvroDecoder.php │ │ ├── AvroDecoderInterface.php │ │ ├── DecoderInterface.php │ │ ├── JsonDecoder.php │ │ └── NullDecoder.php │ ├── Encoder │ │ ├── AvroEncoder.php │ │ ├── AvroEncoderInterface.php │ │ ├── EncoderInterface.php │ │ ├── JsonEncoder.php │ │ └── NullEncoder.php │ ├── KafkaAvroSchema.php │ ├── KafkaAvroSchemaInterface.php │ ├── KafkaConsumerMessage.php │ ├── KafkaConsumerMessageInterface.php │ ├── KafkaMessageInterface.php │ ├── KafkaProducerMessage.php │ ├── KafkaProducerMessageInterface.php │ └── Registry │ │ ├── AvroSchemaRegistry.php │ │ └── AvroSchemaRegistryInterface.php └── Producer │ ├── KafkaProducer.php │ ├── KafkaProducerBuilder.php │ ├── KafkaProducerBuilderInterface.php │ └── KafkaProducerInterface.php └── tests ├── Unit ├── Callback │ ├── KafkaConsumerRebalanceCallbackTest.php │ ├── KafkaErrorCallbackTest.php │ └── KafkaProducerDeliveryReportCallbackTest.php ├── Conf │ └── KafkaConfigurationTest.php ├── Consumer │ ├── KafkaConsumerBuilderTest.php │ ├── KafkaHighLevelConsumerTest.php │ ├── KafkaLowLevelConsumerTest.php │ └── TopicSubscriptionTest.php ├── Exception │ └── KafkaConsumerConsumeExceptionTest.php ├── Message │ ├── Decoder │ │ ├── AvroDecoderTest.php │ │ ├── JsonDecoderTest.php │ │ └── NullDecoderTest.php │ ├── Encoder │ │ ├── AvroEncoderTest.php │ │ ├── JsonEncoderTest.php │ │ └── NullEncoderTest.php │ ├── KafkaAvroSchemaTest.php │ ├── KafkaConsumerMessageTest.php │ ├── KafkaProducerMessageTest.php │ └── Registry │ │ └── AvroSchemaRegistryTest.php └── Producer │ ├── KafkaProducerBuilderTest.php │ └── KafkaProducerTest.php └── bootstrap.php /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | orbs: 4 | ci-caching: jobcloud/ci-caching@3.1 5 | ci-php: jobcloud/ci-php@2.7 6 | 7 | workflows: 8 | test-php-kafka-lib: 9 | jobs: 10 | - ci-caching/build-docker-images: 11 | context: dockerhub-credentials 12 | dockerComposeFile: "./docker/docker-compose.yml" 13 | - ci-php/install-dependencies: 14 | context: dockerhub-credentials 15 | dockerComposeFile: "./docker/docker-compose.yml" 16 | dependencyCheckSumFile: "./composer.json" 17 | requires: 18 | - ci-caching/build-docker-images 19 | - coverage: 20 | context: dockerhub-credentials 21 | requires: 22 | - ci-php/install-dependencies 23 | - ci-php/code-style: 24 | context: dockerhub-credentials 25 | dockerComposeFile: "./docker/docker-compose.yml" 26 | dependencyCheckSumFile: "./composer.json" 27 | requires: 28 | - ci-php/install-dependencies 29 | - ci-php/static-analysis: 30 | context: dockerhub-credentials 31 | dockerComposeFile: "./docker/docker-compose.yml" 32 | dependencyCheckSumFile: "./composer.json" 33 | requires: 34 | - ci-php/install-dependencies 35 | - ci-php/infection-testing: 36 | context: dockerhub-credentials 37 | dockerComposeFile: "./docker/docker-compose.yml" 38 | dependencyCheckSumFile: "./composer.json" 39 | requires: 40 | - ci-php/install-dependencies 41 | - ci-php/todo-checker: 42 | name: todo-check 43 | context: 44 | - todochecker 45 | daily: 46 | triggers: 47 | - schedule: 48 | cron: "0 7 * * *" 49 | filters: 50 | branches: 51 | only: 52 | - main 53 | jobs: 54 | - ci-caching/build-docker-images: 55 | context: dockerhub-credentials 56 | name: build-docker-images 57 | - ci-php/install-dependencies: 58 | context: dockerhub-credentials 59 | name: install-dependencies 60 | dependencyCheckSumFile: "./composer.json" 61 | requires: 62 | - build-docker-images 63 | - ci-php/composer-audit: 64 | context: dockerhub-credentials 65 | dependencyCheckSumFile: "./composer.json" 66 | requires: 67 | - build-docker-images 68 | - install-dependencies 69 | 70 | jobs: 71 | coverage: 72 | machine: true 73 | steps: 74 | - ci-php/coverage-command: 75 | dockerComposeFile: "./docker/docker-compose.yml" 76 | dependencyCheckSumFile: "./composer.json" 77 | - run: 78 | name: Download cc-test-reporter 79 | command: | 80 | mkdir -p tmp/ 81 | curl -L https://codeclimate.com/downloads/test-reporter/test-reporter-latest-linux-amd64 > ./tmp/cc-test-reporter 82 | chmod +x ./tmp/cc-test-reporter 83 | - run: 84 | name: Upload coverage results to Code Climate 85 | command: | 86 | ./tmp/cc-test-reporter after-build -p /var/www/html --coverage-input-type clover --exit-code $? 87 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @jobcloud/publication-team-be 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /build/ 2 | /vendor/ 3 | /.idea 4 | /composer.symlink 5 | composer.lock 6 | .phpunit.result.cache 7 | clover.xml 8 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 JobCloud AG 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MIGRATION.md: -------------------------------------------------------------------------------- 1 | # Migration from messaging-lib to php-kafka-lib 2 | 3 | Internally not much has changed, we have mostly gotten rid of, 4 | the general interfaces, since we won't implement support for other 5 | messaging systems than Kafka. 6 | 7 | In most cases you can just: 8 | 1. `composer remove jobcloud/messaging-lib` 9 | 2. `composer require jobcloud/php-kafka-lib ~0.1` (after migration, consider switching to the most current release) 10 | 3. Replace namespace `Jobcloud\Messaging\Kafka` with `Jobcloud\Kafka` 11 | 4. Replace the following: 12 | - `ConsumerException` with `KafkaConsumerConsumeException` 13 | - `MessageInterface` with `KafkaMessageInterface` or depending on your use case with `KafkaConsumerMessageInterface` and `KafkaProducerMessageInterface` 14 | - `ProducerInterface` with `KafkaProducerInterface` 15 | - `ConsumerInterface` with `KafkaConsumerInterface` 16 | - `ProducerPool` is not supported anymore 17 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: clean code-style coverage help test static-analysis install-dependencies install-dependencies-lowest update-dependencies pcov-enable pcov-disable infection-testing 2 | .DEFAULT_GOAL := test 3 | 4 | PHPUNIT = ./vendor/bin/phpunit -c ./phpunit.xml 5 | PHPSTAN = ./vendor/bin/phpstan 6 | PHPCS = ./vendor/bin/phpcs --extensions=php 7 | CONSOLE = ./bin/console 8 | INFECTION = ./vendor/bin/infection 9 | 10 | clean: 11 | rm -rf ./build ./vendor 12 | 13 | code-style: pcov-disable 14 | mkdir -p build/logs/phpcs 15 | ${PHPCS} --report-full --report-gitblame --standard=PSR12 ./src --exclude=Generic.Commenting.Todo --report-junit=build/logs/phpcs/junit.xml 16 | 17 | coverage: pcov-enable 18 | ${PHPUNIT} && ./vendor/bin/coverage-check clover.xml 100 19 | 20 | test: pcov-disable 21 | ${PHPUNIT} 22 | 23 | static-analysis: pcov-disable 24 | mkdir -p build/logs/phpstan 25 | ${PHPSTAN} analyse --no-progress 26 | 27 | update-dependencies: 28 | composer update 29 | 30 | install-dependencies: 31 | composer install 32 | 33 | install-dependencies-lowest: 34 | composer install --prefer-lowest 35 | 36 | infection-testing: 37 | make coverage 38 | cp -f build/logs/phpunit/junit.xml build/logs/phpunit/coverage/junit.xml 39 | sudo php-ext-disable pcov 40 | ${INFECTION} --coverage=build/logs/phpunit/coverage --min-msi=91 --threads=`nproc` 41 | sudo php-ext-enable pcov 42 | 43 | pcov-enable: 44 | sudo php-ext-enable pcov 45 | 46 | pcov-disable: 47 | sudo php-ext-disable pcov 48 | 49 | help: 50 | # Usage: 51 | # make [OPTION=value] 52 | # 53 | # Targets: 54 | # clean Cleans the coverage and the vendor directory 55 | # code-style Check codestyle using phpcs 56 | # coverage Generate code coverage (html, clover) 57 | # help You're looking at it! 58 | # test (default) Run all the tests with phpunit 59 | # static-analysis Run static analysis using phpstan 60 | # infection-testing Run infection/mutation testing 61 | # install-dependencies Run composer install 62 | # install-dependencies-lowest Run composer install with --prefer-lowest 63 | # update-dependencies Run composer update 64 | # pcov-enable Enable pcov 65 | # pcov-disable Disable pcov 66 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # php-kafka-lib 2 | 3 | [![CircleCI](https://circleci.com/gh/jobcloud/php-kafka-lib.svg?style=shield)](https://circleci.com/gh/jobcloud/php-kafka-lib) 4 | [![Maintainability](https://api.codeclimate.com/v1/badges/beae5fe991d080cbad8c/maintainability)](https://codeclimate.com/github/jobcloud/php-kafka-lib/maintainability) 5 | [![Test Coverage](https://api.codeclimate.com/v1/badges/beae5fe991d080cbad8c/test_coverage)](https://codeclimate.com/github/jobcloud/php-kafka-lib/test_coverage) 6 | [![Latest Stable Version](https://poser.pugx.org/jobcloud/php-kafka-lib/v/stable)](https://packagist.org/packages/jobcloud/php-kafka-lib) 7 | [![Latest Unstable Version](https://poser.pugx.org/jobcloud/php-kafka-lib/v/unstable)](https://packagist.org/packages/jobcloud/php-kafka-lib) 8 | 9 | ## Description 10 | This is a library that makes it easier to use Kafka in your PHP project. 11 | 12 | This library relies on [arnaud-lb/php-rdkafka](https://github.com/arnaud-lb/php-rdkafka) 13 | Avro support relies on [flix-tech/avro-serde-php](https://github.com/flix-tech/avro-serde-php) 14 | The [documentation](https://arnaud.le-blanc.net/php-rdkafka/phpdoc/book.rdkafka.html) of the php extension, 15 | can help out to understand the internals of this library. 16 | 17 | ## Requirements 18 | - php: ^8.0 19 | - ext-rdkafka: >=4.0.0 20 | - librdkafka: >=0.11.6 (if you use `=4.1.0 24 | - librdkafka: >=1.4 25 | 26 | ## Installation 27 | ``` 28 | composer require jobcloud/php-kafka-lib "~1.0" 29 | ``` 30 | 31 | ### Enable Avro support 32 | If you need Avro support, run: 33 | ``` 34 | composer require flix-tech/avro-serde-php "~1.4" 35 | ``` 36 | 37 | ## Usage 38 | 39 | ### Producer 40 | 41 | #### Kafka 42 | 43 | ##### Simple example 44 | ```php 45 | withAdditionalBroker('localhost:9092') 52 | ->build(); 53 | 54 | $message = KafkaProducerMessage::create('test-topic', 0) 55 | ->withKey('asdf-asdf-asfd-asdf') 56 | ->withBody('some test message payload') 57 | ->withHeaders([ 'key' => 'value' ]); 58 | 59 | $producer->produce($message); 60 | 61 | // Shutdown producer, flush messages that are in queue. Give up after 20s 62 | $result = $producer->flush(20000); 63 | ``` 64 | 65 | ##### Transactional producer (needs >=php-rdkafka:4.1 and >=librdkafka:1.4) 66 | ```php 67 | withAdditionalBroker('localhost:9092') 77 | ->build(); 78 | 79 | $message = KafkaProducerMessage::create('test-topic', 0) 80 | ->withKey('asdf-asdf-asfd-asdf') 81 | ->withBody('some test message payload') 82 | ->withHeaders([ 'key' => 'value' ]); 83 | try { 84 | $producer->beginTransaction(10000); 85 | $producer->produce($message); 86 | $producer->commitTransaction(10000); 87 | } catch (KafkaProducerTransactionRetryException $e) { 88 | // something went wrong but you can retry the failed call (either beginTransaction or commitTransaction) 89 | } catch (KafkaProducerTransactionAbortException $e) { 90 | // you need to call $producer->abortTransaction(10000); and try again 91 | } catch (KafkaProducerTransactionFatalException $e) { 92 | // something went very wrong, re-create your producer, otherwise you could jeopardize the idempotency guarantees 93 | } 94 | 95 | // Shutdown producer, flush messages that are in queue. Give up after 20s 96 | $result = $producer->flush(20000); 97 | ``` 98 | 99 | ##### Avro Producer 100 | To create an avro prodcuer add the avro encoder. 101 | 102 | ```php 103 | 'jobcloud-kafka-schema-registry:9081']) 121 | ) 122 | ), 123 | new AvroObjectCacheAdapter() 124 | ); 125 | 126 | $registry = new AvroSchemaRegistry($cachedRegistry); 127 | $recordSerializer = new RecordSerializer($cachedRegistry); 128 | 129 | //if no version is defined, latest version will be used 130 | //if no schema definition is defined, the appropriate version will be fetched form the registry 131 | $registry->addBodySchemaMappingForTopic( 132 | 'test-topic', 133 | new KafkaAvroSchema('bodySchemaName' /*, int $version, AvroSchema $definition */) 134 | ); 135 | $registry->addKeySchemaMappingForTopic( 136 | 'test-topic', 137 | new KafkaAvroSchema('keySchemaName' /*, int $version, AvroSchema $definition */) 138 | ); 139 | 140 | // if you are only encoding key or value, you can pass that mode as additional third argument 141 | // per default both key and body will get encoded 142 | $encoder = new AvroEncoder($registry, $recordSerializer /*, AvroEncoderInterface::ENCODE_BODY */); 143 | 144 | $producer = KafkaProducerBuilder::create() 145 | ->withAdditionalBroker('kafka:9092') 146 | ->withEncoder($encoder) 147 | ->build(); 148 | 149 | $schemaName = 'testSchema'; 150 | $version = 1; 151 | $message = KafkaProducerMessage::create('test-topic', 0) 152 | ->withKey('asdf-asdf-asfd-asdf') 153 | ->withBody(['name' => 'someName']) 154 | ->withHeaders([ 'key' => 'value' ]); 155 | 156 | $producer->produce($message); 157 | 158 | // Shutdown producer, flush messages that are in queue. Give up after 20s 159 | $result = $producer->flush(20000); 160 | ``` 161 | 162 | **NOTE:** To improve producer latency you can install the `pcntl` extension. 163 | The php-kafka-lib already has code in place, similarly described here: 164 | https://github.com/arnaud-lb/php-rdkafka#performance--low-latency-settings 165 | 166 | ### Consumer 167 | 168 | #### Kafka High Level 169 | 170 | ```php 171 | withAdditionalConfig( 180 | [ 181 | 'compression.codec' => 'lz4', 182 | 'auto.commit.interval.ms' => 500 183 | ] 184 | ) 185 | ->withAdditionalBroker('kafka:9092') 186 | ->withConsumerGroup('testGroup') 187 | ->withAdditionalSubscription('test-topic') 188 | ->build(); 189 | 190 | $consumer->subscribe(); 191 | 192 | while (true) { 193 | try { 194 | $message = $consumer->consume(); 195 | // your business logic 196 | $consumer->commit($message); 197 | } catch (KafkaConsumerTimeoutException $e) { 198 | //no messages were read in a given time 199 | } catch (KafkaConsumerEndOfPartitionException $e) { 200 | //only occurs if enable.partition.eof is true (default: false) 201 | } catch (KafkaConsumerConsumeException $e) { 202 | // Failed 203 | } 204 | } 205 | ``` 206 | 207 | #### Kafka Low Level 208 | 209 | ```php 210 | withAdditionalConfig( 219 | [ 220 | 'compression.codec' => 'lz4', 221 | 'auto.commit.interval.ms' => 500 222 | ] 223 | ) 224 | ->withAdditionalBroker('kafka:9092') 225 | ->withConsumerGroup('testGroup') 226 | ->withAdditionalSubscription('test-topic') 227 | ->withConsumerType(KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL) 228 | ->build(); 229 | 230 | $consumer->subscribe(); 231 | 232 | while (true) { 233 | try { 234 | $message = $consumer->consume(); 235 | // your business logic 236 | $consumer->commit($message); 237 | } catch (KafkaConsumerTimeoutException $e) { 238 | //no messages were read in a given time 239 | } catch (KafkaConsumerEndOfPartitionException $e) { 240 | //only occurs if enable.partition.eof is true (default: false) 241 | } catch (KafkaConsumerConsumeException $e) { 242 | // Failed 243 | } 244 | } 245 | ``` 246 | 247 | #### Avro Consumer 248 | To create an avro consumer add the avro decoder. 249 | 250 | ```php 251 | 'jobcloud-kafka-schema-registry:9081']) 271 | ) 272 | ), 273 | new AvroObjectCacheAdapter() 274 | ); 275 | 276 | $registry = new AvroSchemaRegistry($cachedRegistry); 277 | $recordSerializer = new RecordSerializer($cachedRegistry); 278 | 279 | //if no version is defined, latest version will be used 280 | //if no schema definition is defined, the appropriate version will be fetched form the registry 281 | $registry->addBodySchemaMappingForTopic( 282 | 'test-topic', 283 | new KafkaAvroSchema('bodySchema' , 9 /* , AvroSchema $definition */) 284 | ); 285 | $registry->addKeySchemaMappingForTopic( 286 | 'test-topic', 287 | new KafkaAvroSchema('keySchema' , 9 /* , AvroSchema $definition */) 288 | ); 289 | 290 | // If you are only encoding / decoding key or value, only register the schema(s) you need. 291 | // It is advised against doing that though, some tools might not play 292 | // nice if you don't fully encode your message 293 | $decoder = new AvroDecoder($registry, $recordSerializer); 294 | 295 | $consumer = KafkaConsumerBuilder::create() 296 | ->withAdditionalConfig( 297 | [ 298 | 'compression.codec' => 'lz4', 299 | 'auto.commit.interval.ms' => 500 300 | ] 301 | ) 302 | ->withDecoder($decoder) 303 | ->withAdditionalBroker('kafka:9092') 304 | ->withConsumerGroup('testGroup') 305 | ->withAdditionalSubscription('test-topic') 306 | ->build(); 307 | 308 | $consumer->subscribe(); 309 | 310 | while (true) { 311 | try { 312 | $message = $consumer->consume(); 313 | // your business logic 314 | $consumer->commit($message); 315 | } catch (KafkaConsumerTimeoutException $e) { 316 | //no messages were read in a given time 317 | } catch (KafkaConsumerEndOfPartitionException $e) { 318 | //only occurs if enable.partition.eof is true (default: false) 319 | } catch (KafkaConsumerConsumeException $e) { 320 | // Failed 321 | } 322 | } 323 | ``` 324 | 325 | ## Additional information 326 | Replaces [messaging-lib](https://github.com/jobcloud/messaging-lib) 327 | Check [Migration.md](MIGRATION.md) for help to migrate. 328 | -------------------------------------------------------------------------------- /UPGRADE.md: -------------------------------------------------------------------------------- 1 | # Upgrade to 1.0 2 | 3 | ## Avro encoding / decoding 4 | Has been removed as a fixed dependency. If you rely on it you need to run 5 | the following in your project: 6 | ``` 7 | composer require flix-tech/avro-serde-php "~1.4" 8 | ``` 9 | 10 | ## Producer improvements 11 | Producer used to poll all events after `produce`, now per default only one 12 | non-blocking poll call will be triggered after `produce`. 13 | This improvement was made so higher throughput can be achieved if needed. 14 | This affects the following classes: 15 | - KafkaProducer 16 | - Added `syncProduce` - waits indefinitely for an event to be polled 17 | - Added `poll` - gives you the ability to poll, especially useful if you disable auto poll for `produce` 18 | - Added `pollUntilQueueSizeReached` - polls until the poll queue has reached a certain size 19 | - Changed `produce`, changed behaviour (see above), has new optional parameters `$autoPoll` and `$pollTimeoutMs` 20 | 21 | To achieve the previous default behaviour change (e.g. suited for REST API applications which trigger a message on call and are not using long running processes like Swoole): 22 | ``` 23 | $producer->produce($message); 24 | ``` 25 | to 26 | ``` 27 | $producer->produce($message, false); 28 | $producer->pollUntilQueueSizeReached(); // defaults should work fine 29 | ``` 30 | 31 | ## Possibility to decode message later (Consumer) 32 | Consume has now a second optional parameter `consume(int $timeoutMs = 10000, bool $autoDecode = true)`. 33 | If set to false, you must decode your message later using `$consumer->decodeMessage($message)`. 34 | For high throughput, you don't need to decode immediately, some decisions can be made 35 | relying on the message headers alone. This helps to leverage that. 36 | 37 | ## Remove timout from builder / configuration, added timeout parameter to functions (Consumer / Producer) 38 | You were able to set timeout in the builder, but some methods 39 | still had `timeout` as parameter. To avoid confusion, every method 40 | needing a timeout, will now have it as a parameter with a sane default value. 41 | This affects the following classes: 42 | - KafkaConfiguration 43 | - Removed timeout parameter from `__construct()` 44 | - Removed `getTimeout()` 45 | - KafkaConsumerBuilder / KafkaConsumerBuilderInterface 46 | - Removed `withTimeout()` 47 | - KafkaProducerBuilder / KafkaProducerBuilderInterface 48 | - Removed `withPollTimeout()` 49 | - KafkaHighLevelConsumer / KafkaLowLevelConsumer / KafkaConsumerInterface 50 | - Added `$timeoutMs` to `consume()`, default is 10s 51 | - Added `$timeoutMs` to `getMetadataForTopic()`, default is 10s 52 | - KafkaProducer / KafkaProducerInterface 53 | - Added `pollTimeoutMs` to `produce()`, default is 10s 54 | - Added `$timeoutMs` to `getMetadataForTopic()`, default is 10s 55 | 56 | ## Possibility to avro encode / decode both key and body of a message 57 | The previous default behaviour was to only encode the body of a message. 58 | Now you are also able to encode / decode message keys. 59 | This affects the following classes: 60 | - AvroSchemaRegistry 61 | - Removed `addSchemaMappingForTopic` 62 | - Removed `getSchemaForTopic` 63 | - Added `addBodySchemaMappingForTopic` 64 | - Added `addKeySchemaMappingForTopic` 65 | - Added `getBodySchemaForTopic` 66 | - Added `getKeySchemaForTopic` 67 | - Behaviour change: Trying to get a schema for which no mapping was registered will throw `AvroSchemaRegistryException` 68 | - `getTopicSchemaMapping` will still return an array with mappings but with an additional first key `body` or `key` for the type of the schema 69 | - KafkaMessageInterface 70 | - type of `key` is now mixed 71 | 72 | ## Default error callback 73 | The default error callback now only throws exceptions for fatal errors. 74 | Other errors will be retried by librdkafka and are only informational. 75 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jobcloud/php-kafka-lib", 3 | "description": "Jobcloud PHP Kafka library", 4 | "license": [ 5 | "MIT" 6 | ], 7 | "authors": [ 8 | { 9 | "name": "Jobcloud AG", 10 | "homepage": "https://careers.jobcloud.ch/en/our-team/?term=133" 11 | } 12 | ], 13 | "keywords": [ 14 | "php", 15 | "kafka", 16 | "messaging", 17 | "rdkafka" 18 | ], 19 | "require": { 20 | "php": "^8.0", 21 | "ext-rdkafka": "^4.0|^5.0|^6.0", 22 | "ext-json": "*" 23 | }, 24 | "require-dev": { 25 | "phpunit/phpunit": "^9.4", 26 | "squizlabs/php_codesniffer": "^3.5.4", 27 | "phpstan/phpstan": "^1.8", 28 | "php-mock/php-mock-phpunit": "^2.6", 29 | "kwn/php-rdkafka-stubs": "^2.0.0", 30 | "rregeer/phpunit-coverage-check": "^0.3.1", 31 | "johnkary/phpunit-speedtrap": "^3.1", 32 | "flix-tech/avro-serde-php": "^1.4", 33 | "infection/infection": "^0.26" 34 | }, 35 | "autoload": { 36 | "psr-4": { 37 | "Jobcloud\\Kafka\\": "src/" 38 | } 39 | }, 40 | "suggest": { 41 | "flix-tech/avro-serde-php": "Is needed for Avro support" 42 | }, 43 | "extra": { 44 | "branch-alias": { 45 | "dev-master": "2.0-dev" 46 | } 47 | }, 48 | "config": { 49 | "allow-plugins": { 50 | "infection/extension-installer": false 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /docker/.env: -------------------------------------------------------------------------------- 1 | COMPOSE_PROJECT_NAME=php-kafka-lib 2 | -------------------------------------------------------------------------------- /docker/dev/php/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM php:8.0-cli-alpine3.16 2 | 3 | ARG HOST_USER_ID 4 | ARG HOST_USER 5 | 6 | # PHP: Copy configuration files & remove dist files 7 | RUN mkdir /phpIni 8 | COPY files/bin/ /usr/local/bin/ 9 | COPY files/php/ /phpIni 10 | 11 | # SYS: Install required packages 12 | RUN apk --no-cache upgrade && \ 13 | apk --no-cache add bash git sudo openssh autoconf gcc g++ make librdkafka librdkafka-dev 14 | 15 | # we need support for users with ID higher than 65k, so instead of using this: 16 | #RUN adduser -u $HOST_USER_ID -D -H $HOST_USER 17 | # we do it manually 18 | RUN echo "$HOST_USER:x:$HOST_USER_ID:82:Linux User,,,:/home/$HOST_USER:" >> /etc/passwd && \ 19 | echo "$HOST_USER:!:$(($(date +%s) / 60 / 60 / 24)):0:99999:7:::" >> /etc/shadow && \ 20 | echo "$HOST_USER:x:$HOST_USER_ID:" >> /etc/group && \ 21 | mkdir /home/$HOST_USER && \ 22 | chown $HOST_USER:$HOST_USER /home/$HOST_USER && \ 23 | echo "ALL ALL=NOPASSWD: ALL" >> /etc/sudoers && \ 24 | addgroup $HOST_USER www-data 25 | 26 | # COMPOSER: install binary and prestissimo 27 | RUN curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer --version=2.5.1 28 | 29 | # PHP: Install php extensions 30 | RUN pecl channel-update pecl.php.net && \ 31 | pecl install rdkafka-6.0.3 pcov && \ 32 | docker-php-ext-install pcntl && \ 33 | php-ext-enable rdkafka pcntl pcov 34 | 35 | USER $HOST_USER 36 | 37 | WORKDIR /var/www/html 38 | 39 | -------------------------------------------------------------------------------- /docker/dev/php/files/bin/php-ext-disable: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | if [ $# -eq 0 ] 3 | then 4 | echo "Please pass a php module name" 5 | exit 6 | fi 7 | 8 | for phpmod in "$@" 9 | do 10 | rm -f /usr/local/etc/php/conf.d/*$phpmod*.ini 11 | done 12 | -------------------------------------------------------------------------------- /docker/dev/php/files/bin/php-ext-enable: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [[ ${#} -eq 0 ]] ; then 4 | echo -e "\\nPHP module name required!\\n" 5 | exit 1 6 | fi 7 | 8 | for phpmod in "${@}" ; do 9 | 10 | files=($(find /phpIni -type f -iname "*${phpmod}*.ini" -exec ls -1 '{}' +)) 11 | 12 | for i in "${files[@]}" ; do 13 | ln -s "${i}" /usr/local/etc/php/conf.d 14 | done 15 | 16 | if [[ ${#files[@]} -eq 0 ]] ; then 17 | docker-php-ext-enable "${phpmod}" 18 | fi 19 | 20 | done 21 | -------------------------------------------------------------------------------- /docker/dev/php/files/php/20-pcov.ini: -------------------------------------------------------------------------------- 1 | extension=pcov.so -------------------------------------------------------------------------------- /docker/docker-compose.ci.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | php: 4 | build: 5 | context: ./dev/php 6 | args: 7 | HOST_USER: ${USER} 8 | HOST_USER_ID: ${USER_ID} 9 | container_name: php-kafka-lib-php 10 | tty: true 11 | volumes: 12 | - ../:/var/www/html 13 | - ../../php-rdkafka:/var/www/rdkafka 14 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.2' 2 | services: 3 | php: 4 | build: 5 | context: ./dev/php 6 | args: 7 | HOST_USER: ${USER} 8 | HOST_USER_ID: ${USER_ID} 9 | container_name: php-kafka-lib 10 | hostname: php-kafka-lib 11 | tty: true 12 | volumes: 13 | - ../:/var/www/html 14 | - ../../php-rdkafka:/var/www/rdkafka 15 | -------------------------------------------------------------------------------- /infection.json: -------------------------------------------------------------------------------- 1 | { 2 | "timeout": 10, 3 | "source": { 4 | "directories": [ 5 | "src" 6 | ] 7 | }, 8 | "logs": { 9 | "text": "build\/logs\/infection\/infection.log", 10 | "summary": "build\/logs\/infection\/infection-summary.log" 11 | }, 12 | "mutators": { 13 | "@default": true 14 | }, 15 | "phpUnit": { 16 | "customPath": "vendor/bin/phpunit" 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /phpcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | src 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /phpstan.neon: -------------------------------------------------------------------------------- 1 | parameters: 2 | level: 8 3 | paths: [ src ] 4 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 14 | 15 | 16 | src 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | ./tests/Unit 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /src/Callback/KafkaConsumerRebalanceCallback.php: -------------------------------------------------------------------------------- 1 | assign($partitions); 31 | break; 32 | 33 | default: 34 | $consumer->assign(null); 35 | break; 36 | } 37 | } catch (RdKafkaException $e) { 38 | throw new KafkaRebalanceException($e->getMessage(), $e->getCode(), $e); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Callback/KafkaErrorCallback.php: -------------------------------------------------------------------------------- 1 | err) { 27 | return; 28 | } 29 | 30 | throw new KafkaProducerException( 31 | $message->errstr(), 32 | $message->err 33 | ); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Conf/KafkaConfiguration.php: -------------------------------------------------------------------------------- 1 | 31 | */ 32 | private $lowLevelTopicSettings = [ 33 | 'auto.commit.interval.ms' => 1, 34 | 'auto.offset.reset' => 1, 35 | ]; 36 | 37 | /** 38 | * @param string[] $brokers 39 | * @param array|TopicSubscription[] $topicSubscriptions 40 | * @param mixed[] $config 41 | * @param string $type 42 | */ 43 | public function __construct(array $brokers, array $topicSubscriptions = [], array $config = [], string $type = '') 44 | { 45 | parent::__construct(); 46 | 47 | $this->brokers = $brokers; 48 | $this->topicSubscriptions = $topicSubscriptions; 49 | $this->type = $type; 50 | $this->initializeConfig($config); 51 | } 52 | 53 | /** 54 | * @return string[] 55 | */ 56 | public function getBrokers(): array 57 | { 58 | return $this->brokers; 59 | } 60 | 61 | /** 62 | * @return array|TopicSubscription[] 63 | */ 64 | public function getTopicSubscriptions(): array 65 | { 66 | return $this->topicSubscriptions; 67 | } 68 | 69 | /** 70 | * @return string[] 71 | */ 72 | public function getConfiguration(): array 73 | { 74 | return $this->dump(); 75 | } 76 | 77 | /** 78 | * @param mixed[] $config 79 | * @return void 80 | */ 81 | protected function initializeConfig(array $config = []): void 82 | { 83 | $topicConf = new RdKafkaTopicConf(); 84 | 85 | foreach ($config as $name => $value) { 86 | if (false === is_scalar($value)) { 87 | continue; 88 | } 89 | 90 | if ( 91 | KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL === $this->type 92 | && true === $this->isLowLevelTopicConfSetting($name) 93 | ) { 94 | $topicConf->set($name, (string) $value); 95 | $this->setDefaultTopicConf($topicConf); 96 | } 97 | 98 | if (true === is_bool($value)) { 99 | $value = true === $value ? 'true' : 'false'; 100 | } 101 | 102 | $this->set($name, (string) $value); 103 | } 104 | 105 | $this->set('metadata.broker.list', implode(',', $this->getBrokers())); 106 | } 107 | 108 | /** 109 | * @param string $settingName 110 | * @return bool 111 | */ 112 | private function isLowLevelTopicConfSetting(string $settingName): bool 113 | { 114 | return true === isset($this->lowLevelTopicSettings[$settingName]); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /src/Consumer/AbstractKafkaConsumer.php: -------------------------------------------------------------------------------- 1 | consumer = $consumer; 54 | $this->kafkaConfiguration = $kafkaConfiguration; 55 | $this->decoder = $decoder; 56 | } 57 | 58 | /** 59 | * Returns true if the consumer has subscribed to its topics, otherwise false 60 | * It is mandatory to call `subscribe` before `consume` 61 | * 62 | * @return boolean 63 | */ 64 | public function isSubscribed(): bool 65 | { 66 | return $this->subscribed; 67 | } 68 | 69 | /** 70 | * Returns the configuration settings for this consumer instance as array 71 | * 72 | * @return string[] 73 | */ 74 | public function getConfiguration(): array 75 | { 76 | return $this->kafkaConfiguration->dump(); 77 | } 78 | 79 | /** 80 | * Consumes a message and returns it 81 | * In cases of errors / timeouts an exception is thrown 82 | * 83 | * @param integer $timeoutMs 84 | * @param boolean $autoDecode 85 | * @return KafkaConsumerMessageInterface 86 | * @throws KafkaConsumerConsumeException 87 | * @throws KafkaConsumerEndOfPartitionException 88 | * @throws KafkaConsumerTimeoutException 89 | */ 90 | public function consume(int $timeoutMs = 10000, bool $autoDecode = true): KafkaConsumerMessageInterface 91 | { 92 | if (false === $this->isSubscribed()) { 93 | throw new KafkaConsumerConsumeException(KafkaConsumerConsumeException::NOT_SUBSCRIBED_EXCEPTION_MESSAGE); 94 | } 95 | 96 | if (null === $rdKafkaMessage = $this->kafkaConsume($timeoutMs)) { 97 | throw new KafkaConsumerEndOfPartitionException( 98 | rd_kafka_err2str(RD_KAFKA_RESP_ERR__PARTITION_EOF), 99 | RD_KAFKA_RESP_ERR__PARTITION_EOF 100 | ); 101 | } 102 | 103 | if (RD_KAFKA_RESP_ERR__PARTITION_EOF === $rdKafkaMessage->err) { 104 | throw new KafkaConsumerEndOfPartitionException($rdKafkaMessage->errstr(), $rdKafkaMessage->err); 105 | } elseif (RD_KAFKA_RESP_ERR__TIMED_OUT === $rdKafkaMessage->err) { 106 | throw new KafkaConsumerTimeoutException($rdKafkaMessage->errstr(), $rdKafkaMessage->err); 107 | } 108 | 109 | $message = $this->getConsumerMessage($rdKafkaMessage); 110 | 111 | if (RD_KAFKA_RESP_ERR_NO_ERROR !== $rdKafkaMessage->err) { 112 | throw new KafkaConsumerConsumeException($rdKafkaMessage->errstr(), $rdKafkaMessage->err, $message); 113 | } 114 | 115 | if (true === $autoDecode) { 116 | return $this->decoder->decode($message); 117 | } 118 | 119 | return $message; 120 | } 121 | 122 | /** 123 | * Decode consumer message 124 | * 125 | * @param KafkaConsumerMessageInterface $message 126 | * @return KafkaConsumerMessageInterface 127 | */ 128 | public function decodeMessage(KafkaConsumerMessageInterface $message): KafkaConsumerMessageInterface 129 | { 130 | return $this->decoder->decode($message); 131 | } 132 | 133 | /** 134 | * Queries the broker for metadata on a certain topic 135 | * 136 | * @param string $topicName 137 | * @param integer $timeoutMs 138 | * @return RdKafkaMetadataTopic 139 | * @throws RdKafkaException 140 | */ 141 | public function getMetadataForTopic(string $topicName, int $timeoutMs = 10000): RdKafkaMetadataTopic 142 | { 143 | $topic = $this->consumer->newTopic($topicName); 144 | return $this->consumer 145 | ->getMetadata( 146 | false, 147 | $topic, 148 | $timeoutMs 149 | ) 150 | ->getTopics() 151 | ->current(); 152 | } 153 | 154 | /** 155 | * Get the earliest offset for a certain timestamp for topic partitions 156 | * 157 | * @param array|RdKafkaTopicPartition[] $topicPartitions 158 | * @param integer $timeoutMs 159 | * @return array|RdKafkaTopicPartition[] 160 | */ 161 | public function offsetsForTimes(array $topicPartitions, int $timeoutMs): array 162 | { 163 | return $this->consumer->offsetsForTimes($topicPartitions, $timeoutMs); 164 | } 165 | 166 | /** 167 | * Queries the broker for the first offset of a given topic and partition 168 | * 169 | * @param string $topic 170 | * @param integer $partition 171 | * @param integer $timeoutMs 172 | * @return integer 173 | */ 174 | public function getFirstOffsetForTopicPartition(string $topic, int $partition, int $timeoutMs): int 175 | { 176 | $lowOffset = 0; 177 | $highOffset = 0; 178 | 179 | $this->consumer->queryWatermarkOffsets($topic, $partition, $lowOffset, $highOffset, $timeoutMs); 180 | 181 | return $lowOffset; 182 | } 183 | 184 | /** 185 | * Queries the broker for the last offset of a given topic and partition 186 | * 187 | * @param string $topic 188 | * @param integer $partition 189 | * @param integer $timeoutMs 190 | * @return integer 191 | */ 192 | public function getLastOffsetForTopicPartition(string $topic, int $partition, int $timeoutMs): int 193 | { 194 | $lowOffset = 0; 195 | $highOffset = 0; 196 | 197 | $this->consumer->queryWatermarkOffsets($topic, $partition, $lowOffset, $highOffset, $timeoutMs); 198 | 199 | return $highOffset; 200 | } 201 | 202 | /** 203 | * @param string $topic 204 | * @return int[] 205 | * @throws RdKafkaException 206 | */ 207 | protected function getAllTopicPartitions(string $topic): array 208 | { 209 | 210 | $partitions = []; 211 | $topicMetadata = $this->getMetadataForTopic($topic); 212 | 213 | foreach ($topicMetadata->getPartitions() as $partition) { 214 | $partitions[] = $partition->getId(); 215 | } 216 | 217 | return $partitions; 218 | } 219 | 220 | /** 221 | * @param RdKafkaMessage $message 222 | * @return KafkaConsumerMessageInterface 223 | */ 224 | protected function getConsumerMessage(RdKafkaMessage $message): KafkaConsumerMessageInterface 225 | { 226 | return new KafkaConsumerMessage( 227 | (string) $message->topic_name, 228 | (int) $message->partition, 229 | (int) $message->offset, 230 | (int) $message->timestamp, 231 | $message->key, 232 | $message->payload, 233 | (array) $message->headers 234 | ); 235 | } 236 | 237 | /** 238 | * @return array 239 | */ 240 | public function getTopicSubscriptions(): array 241 | { 242 | return $this->kafkaConfiguration->getTopicSubscriptions(); 243 | } 244 | 245 | /** 246 | * @param integer $timeoutMs 247 | * @return null|RdKafkaMessage 248 | */ 249 | abstract protected function kafkaConsume(int $timeoutMs): ?RdKafkaMessage; 250 | } 251 | -------------------------------------------------------------------------------- /src/Consumer/KafkaConsumerBuilder.php: -------------------------------------------------------------------------------- 1 | 27 | */ 28 | private $config = [ 29 | 'enable.auto.offset.store' => false, 30 | 'enable.auto.commit' => false, 31 | 'auto.offset.reset' => 'earliest' 32 | ]; 33 | 34 | /** 35 | * @var array|TopicSubscription[] 36 | */ 37 | private $topics = []; 38 | 39 | /** 40 | * @var string 41 | */ 42 | private $consumerGroup = 'default'; 43 | 44 | /** 45 | * @var string 46 | */ 47 | private $consumerType = self::CONSUMER_TYPE_HIGH_LEVEL; 48 | 49 | /** 50 | * @var callable 51 | */ 52 | private $errorCallback; 53 | 54 | /** 55 | * @var callable 56 | */ 57 | private $rebalanceCallback; 58 | 59 | /** 60 | * @var callable 61 | */ 62 | private $consumeCallback; 63 | 64 | /** 65 | * @var callable 66 | */ 67 | private $logCallback; 68 | 69 | /** 70 | * @var callable 71 | */ 72 | private $offsetCommitCallback; 73 | 74 | /** 75 | * @var DecoderInterface 76 | */ 77 | private $decoder; 78 | 79 | /** 80 | * KafkaConsumerBuilder constructor. 81 | */ 82 | private function __construct() 83 | { 84 | $this->errorCallback = new KafkaErrorCallback(); 85 | $this->decoder = new NullDecoder(); 86 | } 87 | 88 | /** 89 | * Returns the builder 90 | * 91 | * @return KafkaConsumerBuilder 92 | */ 93 | public static function create(): self 94 | { 95 | return new self(); 96 | } 97 | 98 | /** 99 | * Adds a broker from which you want to consume 100 | * 101 | * @param string $broker 102 | * @return KafkaConsumerBuilderInterface 103 | */ 104 | public function withAdditionalBroker(string $broker): KafkaConsumerBuilderInterface 105 | { 106 | $that = clone $this; 107 | 108 | $that->brokers[] = $broker; 109 | 110 | return $that; 111 | } 112 | 113 | /** 114 | * Add topic name(s) (and additionally partitions and offsets) to subscribe to 115 | * 116 | * @param string $topicName 117 | * @param int[] $partitions 118 | * @param integer $offset 119 | * @return KafkaConsumerBuilderInterface 120 | */ 121 | public function withAdditionalSubscription( 122 | string $topicName, 123 | array $partitions = [], 124 | int $offset = self::OFFSET_STORED 125 | ): KafkaConsumerBuilderInterface { 126 | $that = clone $this; 127 | 128 | $that->topics[] = new TopicSubscription($topicName, $partitions, $offset); 129 | 130 | return $that; 131 | } 132 | 133 | /** 134 | * Replaces all topic names previously configured with a topic and additionally partitions and an offset to 135 | * subscribe to 136 | * 137 | * @param string $topicName 138 | * @param int[] $partitions 139 | * @param integer $offset 140 | * @return KafkaConsumerBuilderInterface 141 | */ 142 | public function withSubscription( 143 | string $topicName, 144 | array $partitions = [], 145 | int $offset = self::OFFSET_STORED 146 | ): KafkaConsumerBuilderInterface { 147 | $that = clone $this; 148 | 149 | $that->topics = [new TopicSubscription($topicName, $partitions, $offset)]; 150 | 151 | return $that; 152 | } 153 | 154 | /** 155 | * Add configuration settings, otherwise the kafka defaults apply 156 | * 157 | * @param string[] $config 158 | * @return KafkaConsumerBuilderInterface 159 | */ 160 | public function withAdditionalConfig(array $config): KafkaConsumerBuilderInterface 161 | { 162 | $that = clone $this; 163 | $that->config = $config + $this->config; 164 | 165 | return $that; 166 | } 167 | 168 | /** 169 | * Set the consumer group 170 | * 171 | * @param string $consumerGroup 172 | * @return KafkaConsumerBuilderInterface 173 | */ 174 | public function withConsumerGroup(string $consumerGroup): KafkaConsumerBuilderInterface 175 | { 176 | $that = clone $this; 177 | $that->consumerGroup = $consumerGroup; 178 | 179 | return $that; 180 | } 181 | 182 | /** 183 | * Set the consumer type, can be either CONSUMER_TYPE_LOW_LEVEL or CONSUMER_TYPE_HIGH_LEVEL 184 | * 185 | * @param string $consumerType 186 | * @return KafkaConsumerBuilderInterface 187 | */ 188 | public function withConsumerType(string $consumerType): KafkaConsumerBuilderInterface 189 | { 190 | $that = clone $this; 191 | $that->consumerType = $consumerType; 192 | 193 | return $that; 194 | } 195 | 196 | /** 197 | * Set a callback to be called on errors. 198 | * The default callback will throw an exception for every error 199 | * 200 | * @param callable $errorCallback 201 | * @return KafkaConsumerBuilderInterface 202 | */ 203 | public function withErrorCallback(callable $errorCallback): KafkaConsumerBuilderInterface 204 | { 205 | $that = clone $this; 206 | $that->errorCallback = $errorCallback; 207 | 208 | return $that; 209 | } 210 | 211 | /** 212 | * Set a callback to be called on consumer rebalance 213 | * 214 | * @param callable $rebalanceCallback 215 | * @return KafkaConsumerBuilderInterface 216 | */ 217 | public function withRebalanceCallback(callable $rebalanceCallback): KafkaConsumerBuilderInterface 218 | { 219 | $that = clone $this; 220 | $that->rebalanceCallback = $rebalanceCallback; 221 | 222 | return $that; 223 | } 224 | 225 | /** 226 | * Only applicable for the high level consumer 227 | * Callback that is going to be called when you call consume 228 | * 229 | * @param callable $consumeCallback 230 | * @return KafkaConsumerBuilderInterface 231 | */ 232 | public function withConsumeCallback(callable $consumeCallback): KafkaConsumerBuilderInterface 233 | { 234 | $that = clone $this; 235 | $that->consumeCallback = $consumeCallback; 236 | 237 | return $that; 238 | } 239 | 240 | /** 241 | * Callback for log related events 242 | * 243 | * @param callable $logCallback 244 | * @return KafkaConsumerBuilderInterface 245 | */ 246 | public function withLogCallback(callable $logCallback): KafkaConsumerBuilderInterface 247 | { 248 | $that = clone $this; 249 | $that->logCallback = $logCallback; 250 | 251 | return $that; 252 | } 253 | 254 | /** 255 | * Set callback that is being called on offset commits 256 | * 257 | * @param callable $offsetCommitCallback 258 | * @return KafkaConsumerBuilderInterface 259 | */ 260 | public function withOffsetCommitCallback(callable $offsetCommitCallback): KafkaConsumerBuilderInterface 261 | { 262 | $that = clone $this; 263 | $that->offsetCommitCallback = $offsetCommitCallback; 264 | 265 | return $that; 266 | } 267 | 268 | /** 269 | * Lets you set a custom decoder for the consumed message 270 | * 271 | * @param DecoderInterface $decoder 272 | * @return KafkaConsumerBuilderInterface 273 | */ 274 | public function withDecoder(DecoderInterface $decoder): KafkaConsumerBuilderInterface 275 | { 276 | $that = clone $this; 277 | $that->decoder = $decoder; 278 | 279 | return $that; 280 | } 281 | 282 | /** 283 | * Returns your consumer instance 284 | * 285 | * @return KafkaConsumerInterface 286 | * @throws KafkaConsumerBuilderException 287 | */ 288 | public function build(): KafkaConsumerInterface 289 | { 290 | if ([] === $this->brokers) { 291 | throw new KafkaConsumerBuilderException(KafkaConsumerBuilderException::NO_BROKER_EXCEPTION_MESSAGE); 292 | } 293 | 294 | //set additional config 295 | $this->config['group.id'] = $this->consumerGroup; 296 | 297 | //create config 298 | $kafkaConfig = new KafkaConfiguration( 299 | $this->brokers, 300 | $this->topics, 301 | $this->config, 302 | $this->consumerType 303 | ); 304 | 305 | //set consumer callbacks 306 | $this->registerCallbacks($kafkaConfig); 307 | 308 | //create RdConsumer 309 | if (self::CONSUMER_TYPE_LOW_LEVEL === $this->consumerType) { 310 | if (null !== $this->consumeCallback) { 311 | throw new KafkaConsumerBuilderException( 312 | sprintf( 313 | KafkaConsumerBuilderException::UNSUPPORTED_CALLBACK_EXCEPTION_MESSAGE, 314 | 'consumerCallback', 315 | KafkaLowLevelConsumer::class 316 | ) 317 | ); 318 | } 319 | 320 | $rdKafkaConsumer = new RdKafkaLowLevelConsumer($kafkaConfig); 321 | 322 | return new KafkaLowLevelConsumer( 323 | $rdKafkaConsumer, 324 | $kafkaConfig, 325 | $this->decoder 326 | ); 327 | } 328 | 329 | $rdKafkaConsumer = new RdKafkaHighLevelConsumer($kafkaConfig); 330 | 331 | return new KafkaHighLevelConsumer($rdKafkaConsumer, $kafkaConfig, $this->decoder); 332 | } 333 | 334 | /** 335 | * @param KafkaConfiguration $conf 336 | * @return void 337 | */ 338 | private function registerCallbacks(KafkaConfiguration $conf): void 339 | { 340 | $conf->setErrorCb($this->errorCallback); 341 | 342 | if (null !== $this->rebalanceCallback) { 343 | $conf->setRebalanceCb($this->rebalanceCallback); 344 | } 345 | 346 | if (null !== $this->consumeCallback) { 347 | $conf->setConsumeCb($this->consumeCallback); 348 | } 349 | 350 | if (null !== $this->logCallback) { 351 | $conf->setLogCb($this->logCallback); 352 | } 353 | 354 | if (null !== $this->offsetCommitCallback) { 355 | $conf->setOffsetCommitCb($this->offsetCommitCallback); 356 | } 357 | } 358 | } 359 | -------------------------------------------------------------------------------- /src/Consumer/KafkaConsumerBuilderInterface.php: -------------------------------------------------------------------------------- 1 | 122 | */ 123 | //public function getTopicSubscriptions(): array; 124 | } 125 | -------------------------------------------------------------------------------- /src/Consumer/KafkaHighLevelConsumer.php: -------------------------------------------------------------------------------- 1 | $topicSubscriptions 45 | * @throws KafkaConsumerSubscriptionException 46 | * @return void 47 | */ 48 | public function subscribe(array $topicSubscriptions = []): void 49 | { 50 | $subscriptions = $this->getTopicSubscriptionNames($topicSubscriptions); 51 | $assignments = $this->getTopicAssignments($topicSubscriptions); 52 | 53 | if ([] !== $subscriptions && [] !== $assignments) { 54 | throw new KafkaConsumerSubscriptionException( 55 | KafkaConsumerSubscriptionException::MIXED_SUBSCRIPTION_EXCEPTION_MESSAGE 56 | ); 57 | } 58 | 59 | try { 60 | if ([] !== $subscriptions) { 61 | $this->consumer->subscribe($subscriptions); 62 | } else { 63 | $this->consumer->assign($assignments); 64 | } 65 | $this->subscribed = true; 66 | } catch (RdKafkaException $e) { 67 | throw new KafkaConsumerSubscriptionException($e->getMessage(), $e->getCode(), $e); 68 | } 69 | } 70 | 71 | /** 72 | * Unsubscribes from the current subscription / assignment 73 | * 74 | * @throws KafkaConsumerSubscriptionException 75 | * @return void 76 | */ 77 | public function unsubscribe(): void 78 | { 79 | try { 80 | $this->consumer->unsubscribe(); 81 | $this->subscribed = false; 82 | } catch (RdKafkaException $e) { 83 | throw new KafkaConsumerSubscriptionException($e->getMessage(), $e->getCode(), $e); 84 | } 85 | } 86 | 87 | /** 88 | * Commits the offset to the broker for the given message(s) 89 | * This is a blocking function, checkout out commitAsync if you want to commit in a non blocking manner 90 | * 91 | * @param KafkaConsumerMessageInterface|KafkaConsumerMessageInterface[] $messages 92 | * @return void 93 | * @throws KafkaConsumerCommitException 94 | */ 95 | public function commit($messages): void 96 | { 97 | $this->commitMessages($messages); 98 | } 99 | 100 | /** 101 | * Assigns a consumer to the given TopicPartition(s) 102 | * 103 | * @param RdKafkaTopicPartition[] $topicPartitions 104 | * @throws KafkaConsumerAssignmentException 105 | * @return void 106 | */ 107 | public function assign(array $topicPartitions): void 108 | { 109 | try { 110 | $this->consumer->assign($topicPartitions); 111 | } catch (RdKafkaException $e) { 112 | throw new KafkaConsumerAssignmentException($e->getMessage(), $e->getCode()); 113 | } 114 | } 115 | 116 | /** 117 | * Asynchronous version of commit (non blocking) 118 | * 119 | * @param KafkaConsumerMessageInterface|KafkaConsumerMessageInterface[] $messages 120 | * @return void 121 | * @throws KafkaConsumerCommitException 122 | */ 123 | public function commitAsync($messages): void 124 | { 125 | $this->commitMessages($messages, true); 126 | } 127 | 128 | /** 129 | * Gets the current assignment for the consumer 130 | * 131 | * @return array|RdKafkaTopicPartition[] 132 | * @throws KafkaConsumerAssignmentException 133 | */ 134 | public function getAssignment(): array 135 | { 136 | try { 137 | return $this->consumer->getAssignment(); 138 | } catch (RdKafkaException $e) { 139 | throw new KafkaConsumerAssignmentException($e->getMessage(), $e->getCode()); 140 | } 141 | } 142 | 143 | /** 144 | * Gets the commited offset for a TopicPartition for the configured consumer group 145 | * 146 | * @param array|RdKafkaTopicPartition[] $topicPartitions 147 | * @param integer $timeoutMs 148 | * @return array|RdKafkaTopicPartition[] 149 | * @throws KafkaConsumerRequestException 150 | */ 151 | public function getCommittedOffsets(array $topicPartitions, int $timeoutMs): array 152 | { 153 | try { 154 | return $this->consumer->getCommittedOffsets($topicPartitions, $timeoutMs); 155 | } catch (RdKafkaException $e) { 156 | throw new KafkaConsumerRequestException($e->getMessage(), $e->getCode()); 157 | } 158 | } 159 | 160 | /** 161 | * Get current offset positions of the consumer 162 | * 163 | * @param array|RdKafkaTopicPartition[] $topicPartitions 164 | * @return array|RdKafkaTopicPartition[] 165 | */ 166 | public function getOffsetPositions(array $topicPartitions): array 167 | { 168 | return $this->consumer->getOffsetPositions($topicPartitions); 169 | } 170 | 171 | /** 172 | * Close the consumer connection 173 | * 174 | * @return void; 175 | */ 176 | public function close(): void 177 | { 178 | $this->consumer->close(); 179 | } 180 | 181 | /** 182 | * @param integer $timeoutMs 183 | * @return RdKafkaMessage|null 184 | * @throws RdKafkaException 185 | */ 186 | protected function kafkaConsume(int $timeoutMs): ?RdKafkaMessage 187 | { 188 | return $this->consumer->consume($timeoutMs); 189 | } 190 | 191 | /** 192 | * @param KafkaConsumerMessageInterface|KafkaConsumerMessageInterface[] $messages 193 | * @param boolean $asAsync 194 | * @return void 195 | * @throws KafkaConsumerCommitException 196 | */ 197 | private function commitMessages($messages, bool $asAsync = false): void 198 | { 199 | $messages = is_array($messages) ? $messages : [$messages]; 200 | 201 | $offsetsToCommit = $this->getOffsetsToCommitForMessages($messages); 202 | 203 | try { 204 | if (true === $asAsync) { 205 | $this->consumer->commitAsync($offsetsToCommit); 206 | } else { 207 | $this->consumer->commit($offsetsToCommit); 208 | } 209 | } catch (RdKafkaException $e) { 210 | throw new KafkaConsumerCommitException($e->getMessage(), $e->getCode()); 211 | } 212 | } 213 | 214 | /** 215 | * @param array|KafkaConsumerMessageInterface[] $messages 216 | * @return array|RdKafkaTopicPartition[] 217 | */ 218 | private function getOffsetsToCommitForMessages(array $messages): array 219 | { 220 | $offsetsToCommit = []; 221 | 222 | foreach ($messages as $message) { 223 | $topicPartition = sprintf('%s-%s', $message->getTopicName(), $message->getPartition()); 224 | 225 | if (true === isset($offsetsToCommit[$topicPartition])) { 226 | if ($message->getOffset() + 1 > $offsetsToCommit[$topicPartition]->getOffset()) { 227 | $offsetsToCommit[$topicPartition]->setOffset($message->getOffset() + 1); 228 | } 229 | continue; 230 | } 231 | 232 | $offsetsToCommit[$topicPartition] = new RdKafkaTopicPartition( 233 | $message->getTopicName(), 234 | $message->getPartition(), 235 | $message->getOffset() + 1 236 | ); 237 | } 238 | 239 | return $offsetsToCommit; 240 | } 241 | 242 | /** 243 | * @param array $topicSubscriptions 244 | * @return array|string[] 245 | */ 246 | private function getTopicSubscriptionNames(array $topicSubscriptions = []): array 247 | { 248 | $subscriptions = []; 249 | 250 | if ([] === $topicSubscriptions) { 251 | $topicSubscriptions = $this->kafkaConfiguration->getTopicSubscriptions(); 252 | } 253 | 254 | foreach ($topicSubscriptions as $topicSubscription) { 255 | if ( 256 | [] !== $topicSubscription->getPartitions() 257 | || KafkaConsumerBuilderInterface::OFFSET_STORED !== $topicSubscription->getOffset() 258 | ) { 259 | continue; 260 | } 261 | $subscriptions[] = $topicSubscription->getTopicName(); 262 | } 263 | 264 | return $subscriptions; 265 | } 266 | 267 | /** 268 | * @param array $topicSubscriptions 269 | * @return array|RdKafkaTopicPartition[] 270 | */ 271 | private function getTopicAssignments(array $topicSubscriptions = []): array 272 | { 273 | $assignments = []; 274 | 275 | if ([] === $topicSubscriptions) { 276 | $topicSubscriptions = $this->kafkaConfiguration->getTopicSubscriptions(); 277 | } 278 | 279 | foreach ($topicSubscriptions as $topicSubscription) { 280 | if ( 281 | [] === $topicSubscription->getPartitions() 282 | && KafkaConsumerBuilderInterface::OFFSET_STORED === $topicSubscription->getOffset() 283 | ) { 284 | continue; 285 | } 286 | 287 | $offset = $topicSubscription->getOffset(); 288 | $partitions = $topicSubscription->getPartitions(); 289 | 290 | if ([] === $partitions) { 291 | $partitions = $this->getAllTopicPartitions($topicSubscription->getTopicName()); 292 | } 293 | 294 | foreach ($partitions as $partitionId) { 295 | $assignments[] = new RdKafkaTopicPartition( 296 | $topicSubscription->getTopicName(), 297 | $partitionId, 298 | $offset 299 | ); 300 | } 301 | } 302 | 303 | return $assignments; 304 | } 305 | } 306 | -------------------------------------------------------------------------------- /src/Consumer/KafkaHighLevelConsumerInterface.php: -------------------------------------------------------------------------------- 1 | queue = $consumer->newQueue(); 49 | } 50 | 51 | /** 52 | * Subcribes to all defined topics, if no partitions were set, subscribes to all partitions. 53 | * If partition(s) (and optionally offset(s)) were set, subscribes accordingly 54 | * 55 | * @return void 56 | * @throws KafkaConsumerSubscriptionException 57 | */ 58 | public function subscribe(): void 59 | { 60 | if (true === $this->isSubscribed()) { 61 | return; 62 | } 63 | 64 | try { 65 | $topicSubscriptions = $this->kafkaConfiguration->getTopicSubscriptions(); 66 | foreach ($topicSubscriptions as $topicSubscription) { 67 | $topicName = $topicSubscription->getTopicName(); 68 | $offset = $topicSubscription->getOffset(); 69 | 70 | if (false === isset($this->topics[$topicName])) { 71 | $this->topics[$topicName] = $topic = $this->consumer->newTopic($topicName); 72 | } else { 73 | $topic = $this->topics[$topicName]; 74 | } 75 | 76 | $partitions = $topicSubscription->getPartitions(); 77 | 78 | if ([] === $partitions) { 79 | $topicSubscription->setPartitions($this->getAllTopicPartitions($topicName)); 80 | $partitions = $topicSubscription->getPartitions(); 81 | } 82 | 83 | foreach ($partitions as $partitionId) { 84 | $topic->consumeQueueStart($partitionId, $offset, $this->queue); 85 | } 86 | } 87 | 88 | $this->subscribed = true; 89 | } catch (RdKafkaException $e) { 90 | throw new KafkaConsumerSubscriptionException($e->getMessage(), $e->getCode(), $e); 91 | } 92 | } 93 | 94 | /** 95 | * Commits the offset to the broker for the given message(s). This is a blocking function 96 | * 97 | * @param mixed $messages 98 | * @return void 99 | * @throws KafkaConsumerCommitException 100 | */ 101 | public function commit($messages): void 102 | { 103 | $messages = is_array($messages) ? $messages : [$messages]; 104 | 105 | foreach ($messages as $i => $message) { 106 | if (false === $message instanceof KafkaConsumerMessageInterface) { 107 | throw new KafkaConsumerCommitException( 108 | sprintf('Provided message (index: %d) is not an instance of "%s"', $i, KafkaConsumerMessage::class) 109 | ); 110 | } 111 | 112 | $this->topics[$message->getTopicName()]->offsetStore( 113 | $message->getPartition(), 114 | $message->getOffset() 115 | ); 116 | } 117 | } 118 | 119 | /** 120 | * Unsubscribes from the current subscription 121 | * 122 | * @return void 123 | */ 124 | public function unsubscribe(): void 125 | { 126 | if (false === $this->isSubscribed()) { 127 | return; 128 | } 129 | 130 | $topicSubscriptions = $this->kafkaConfiguration->getTopicSubscriptions(); 131 | 132 | /** @var TopicSubscription $topicSubscription */ 133 | foreach ($topicSubscriptions as $topicSubscription) { 134 | foreach ($topicSubscription->getPartitions() as $partitionId) { 135 | $this->topics[$topicSubscription->getTopicName()]->consumeStop($partitionId); 136 | } 137 | } 138 | 139 | $this->subscribed = false; 140 | } 141 | 142 | /** 143 | * @param integer $timeoutMs 144 | * @return null|RdKafkaMessage 145 | */ 146 | protected function kafkaConsume(int $timeoutMs): ?RdKafkaMessage 147 | { 148 | return $this->queue->consume($timeoutMs); 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /src/Consumer/KafkaLowLevelConsumerInterface.php: -------------------------------------------------------------------------------- 1 | topicName = $topicName; 35 | $this->partitions = $partitions; 36 | $this->offset = $offset; 37 | } 38 | 39 | /** 40 | * @return string 41 | */ 42 | public function getTopicName(): string 43 | { 44 | return $this->topicName; 45 | } 46 | 47 | /** 48 | * @param int[] $partitions 49 | * @return void 50 | */ 51 | public function setPartitions(array $partitions): void 52 | { 53 | $this->partitions = $partitions; 54 | } 55 | 56 | /** 57 | * @return int[] 58 | */ 59 | public function getPartitions(): array 60 | { 61 | return $this->partitions; 62 | } 63 | 64 | /** 65 | * @return integer 66 | */ 67 | public function getOffset(): int 68 | { 69 | return $this->offset ?? RD_KAFKA_OFFSET_STORED; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/Consumer/TopicSubscriptionInterface.php: -------------------------------------------------------------------------------- 1 | kafkaMessage = $kafkaMessage; 33 | } 34 | 35 | /** 36 | * @return null|KafkaConsumerMessageInterface 37 | */ 38 | public function getKafkaMessage(): ?KafkaConsumerMessageInterface 39 | { 40 | return $this->kafkaMessage; 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Exception/KafkaConsumerEndOfPartitionException.php: -------------------------------------------------------------------------------- 1 | |null 31 | */ 32 | protected $headers; 33 | 34 | /** 35 | * @return mixed 36 | */ 37 | public function getKey() 38 | { 39 | return $this->key; 40 | } 41 | 42 | /** 43 | * @return mixed 44 | */ 45 | public function getBody() 46 | { 47 | return $this->body; 48 | } 49 | 50 | /** 51 | * @return string 52 | */ 53 | public function getTopicName(): string 54 | { 55 | return $this->topicName; 56 | } 57 | 58 | /** 59 | * @return integer 60 | */ 61 | public function getPartition(): int 62 | { 63 | return $this->partition; 64 | } 65 | 66 | /** 67 | * @return array|null 68 | */ 69 | public function getHeaders(): ?array 70 | { 71 | return $this->headers; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/Message/Decoder/AvroDecoder.php: -------------------------------------------------------------------------------- 1 | recordSerializer = $recordSerializer; 35 | $this->registry = $registry; 36 | } 37 | 38 | /** 39 | * @param KafkaConsumerMessageInterface $consumerMessage 40 | * @return KafkaConsumerMessageInterface 41 | * @throws SchemaRegistryException 42 | */ 43 | public function decode(KafkaConsumerMessageInterface $consumerMessage): KafkaConsumerMessageInterface 44 | { 45 | return new KafkaConsumerMessage( 46 | $consumerMessage->getTopicName(), 47 | $consumerMessage->getPartition(), 48 | $consumerMessage->getOffset(), 49 | $consumerMessage->getTimestamp(), 50 | $this->decodeKey($consumerMessage), 51 | $this->decodeBody($consumerMessage), 52 | $consumerMessage->getHeaders() 53 | ); 54 | } 55 | 56 | /** 57 | * @param KafkaConsumerMessageInterface $consumerMessage 58 | * @return mixed 59 | * @throws SchemaRegistryException 60 | */ 61 | private function decodeBody(KafkaConsumerMessageInterface $consumerMessage) 62 | { 63 | $body = $consumerMessage->getBody(); 64 | $topicName = $consumerMessage->getTopicName(); 65 | 66 | if (null === $body) { 67 | return null; 68 | } 69 | 70 | if (false === $this->registry->hasBodySchemaForTopic($topicName)) { 71 | return $body; 72 | } 73 | 74 | $avroSchema = $this->registry->getBodySchemaForTopic($topicName); 75 | $schemaDefinition = $avroSchema->getDefinition(); 76 | 77 | return $this->recordSerializer->decodeMessage($body, $schemaDefinition); 78 | } 79 | 80 | /** 81 | * @param KafkaConsumerMessageInterface $consumerMessage 82 | * @return mixed 83 | * @throws SchemaRegistryException 84 | */ 85 | private function decodeKey(KafkaConsumerMessageInterface $consumerMessage) 86 | { 87 | $key = $consumerMessage->getKey(); 88 | $topicName = $consumerMessage->getTopicName(); 89 | 90 | if (null === $key) { 91 | return null; 92 | } 93 | 94 | if (false === $this->registry->hasKeySchemaForTopic($topicName)) { 95 | return $key; 96 | } 97 | 98 | $avroSchema = $this->registry->getKeySchemaForTopic($topicName); 99 | $schemaDefinition = $avroSchema->getDefinition(); 100 | 101 | return $this->recordSerializer->decodeMessage($key, $schemaDefinition); 102 | } 103 | 104 | /** 105 | * @return AvroSchemaRegistryInterface 106 | */ 107 | public function getRegistry(): AvroSchemaRegistryInterface 108 | { 109 | return $this->registry; 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /src/Message/Decoder/AvroDecoderInterface.php: -------------------------------------------------------------------------------- 1 | getBody(), true, 512, JSON_THROW_ON_ERROR); 19 | 20 | return new KafkaConsumerMessage( 21 | $consumerMessage->getTopicName(), 22 | $consumerMessage->getPartition(), 23 | $consumerMessage->getOffset(), 24 | $consumerMessage->getTimestamp(), 25 | $consumerMessage->getKey(), 26 | $body, 27 | $consumerMessage->getHeaders() 28 | ); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/Message/Decoder/NullDecoder.php: -------------------------------------------------------------------------------- 1 | recordSerializer = $recordSerializer; 36 | $this->registry = $registry; 37 | } 38 | 39 | /** 40 | * @param KafkaProducerMessageInterface $producerMessage 41 | * @return KafkaProducerMessageInterface 42 | * @throws SchemaRegistryException 43 | * @throws AvroEncoderException 44 | */ 45 | public function encode(KafkaProducerMessageInterface $producerMessage): KafkaProducerMessageInterface 46 | { 47 | $producerMessage = $this->encodeBody($producerMessage); 48 | 49 | return $this->encodeKey($producerMessage); 50 | } 51 | 52 | /** 53 | * @param KafkaProducerMessageInterface $producerMessage 54 | * @return KafkaProducerMessageInterface 55 | * @throws SchemaRegistryException 56 | */ 57 | private function encodeBody(KafkaProducerMessageInterface $producerMessage): KafkaProducerMessageInterface 58 | { 59 | $topicName = $producerMessage->getTopicName(); 60 | $body = $producerMessage->getBody(); 61 | 62 | if (null === $body) { 63 | return $producerMessage; 64 | } 65 | 66 | if (false === $this->registry->hasBodySchemaForTopic($topicName)) { 67 | return $producerMessage; 68 | } 69 | 70 | $avroSchema = $this->registry->getBodySchemaForTopic($topicName); 71 | 72 | $encodedBody = $this->recordSerializer->encodeRecord( 73 | $avroSchema->getName(), 74 | $this->getAvroSchemaDefinition($avroSchema), 75 | $body 76 | ); 77 | 78 | return $producerMessage->withBody($encodedBody); 79 | } 80 | 81 | /** 82 | * @param KafkaProducerMessageInterface $producerMessage 83 | * @return KafkaProducerMessageInterface 84 | * @throws SchemaRegistryException 85 | */ 86 | private function encodeKey(KafkaProducerMessageInterface $producerMessage): KafkaProducerMessageInterface 87 | { 88 | $topicName = $producerMessage->getTopicName(); 89 | $key = $producerMessage->getKey(); 90 | 91 | if (null === $key) { 92 | return $producerMessage; 93 | } 94 | 95 | if (false === $this->registry->hasKeySchemaForTopic($topicName)) { 96 | return $producerMessage; 97 | } 98 | 99 | $avroSchema = $this->registry->getKeySchemaForTopic($topicName); 100 | 101 | $encodedKey = $this->recordSerializer->encodeRecord( 102 | $avroSchema->getName(), 103 | $this->getAvroSchemaDefinition($avroSchema), 104 | $key 105 | ); 106 | 107 | return $producerMessage->withKey($encodedKey); 108 | } 109 | 110 | private function getAvroSchemaDefinition(KafkaAvroSchemaInterface $avroSchema): AvroSchema 111 | { 112 | $schemaDefinition = $avroSchema->getDefinition(); 113 | 114 | if (null === $schemaDefinition) { 115 | throw new AvroEncoderException( 116 | sprintf( 117 | AvroEncoderException::UNABLE_TO_LOAD_DEFINITION_MESSAGE, 118 | $avroSchema->getName() 119 | ) 120 | ); 121 | } 122 | 123 | return $schemaDefinition; 124 | } 125 | 126 | /** 127 | * @return AvroSchemaRegistryInterface 128 | */ 129 | public function getRegistry(): AvroSchemaRegistryInterface 130 | { 131 | return $this->registry; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/Message/Encoder/AvroEncoderInterface.php: -------------------------------------------------------------------------------- 1 | getBody(), JSON_THROW_ON_ERROR); 18 | 19 | return $producerMessage->withBody($body); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/Message/Encoder/NullEncoder.php: -------------------------------------------------------------------------------- 1 | name = $schemaName; 36 | $this->version = $version; 37 | $this->definition = $definition; 38 | } 39 | 40 | /** 41 | * @return string 42 | */ 43 | public function getName(): string 44 | { 45 | return $this->name; 46 | } 47 | 48 | /** 49 | * @return integer 50 | */ 51 | public function getVersion(): int 52 | { 53 | return $this->version; 54 | } 55 | 56 | /** 57 | * @param \AvroSchema $definition 58 | * @return void 59 | */ 60 | public function setDefinition(\AvroSchema $definition): void 61 | { 62 | $this->definition = $definition; 63 | } 64 | 65 | /** 66 | * @return \AvroSchema|null 67 | */ 68 | public function getDefinition(): ?\AvroSchema 69 | { 70 | return $this->definition; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Message/KafkaAvroSchemaInterface.php: -------------------------------------------------------------------------------- 1 | topicName = $topicName; 39 | $this->partition = $partition; 40 | $this->offset = $offset; 41 | $this->timestamp = $timestamp; 42 | $this->key = $key; 43 | $this->body = $body; 44 | $this->headers = $headers; 45 | } 46 | 47 | /** 48 | * @return integer 49 | */ 50 | public function getOffset(): int 51 | { 52 | return $this->offset; 53 | } 54 | 55 | /** 56 | * @return integer 57 | */ 58 | public function getTimestamp(): int 59 | { 60 | return $this->timestamp; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Message/KafkaConsumerMessageInterface.php: -------------------------------------------------------------------------------- 1 | topicName = $topicName; 16 | $this->partition = $partition; 17 | } 18 | 19 | /** 20 | * @param string $topicName 21 | * @param integer $partition 22 | * @return KafkaProducerMessageInterface 23 | */ 24 | public static function create(string $topicName, int $partition): KafkaProducerMessageInterface 25 | { 26 | return new self($topicName, $partition); 27 | } 28 | 29 | /** 30 | * @param string|null $key 31 | * @return KafkaProducerMessageInterface 32 | */ 33 | public function withKey(?string $key): KafkaProducerMessageInterface 34 | { 35 | $new = clone $this; 36 | 37 | $new->key = $key; 38 | 39 | return $new; 40 | } 41 | 42 | /** 43 | * @param mixed $body 44 | * @return KafkaProducerMessageInterface 45 | */ 46 | public function withBody($body): KafkaProducerMessageInterface 47 | { 48 | $new = clone $this; 49 | 50 | $new->body = $body; 51 | 52 | return $new; 53 | } 54 | 55 | /** 56 | * @param string[]|null $headers 57 | * @return KafkaProducerMessageInterface 58 | */ 59 | public function withHeaders(?array $headers): KafkaProducerMessageInterface 60 | { 61 | $new = clone $this; 62 | 63 | $new->headers = $headers; 64 | 65 | return $new; 66 | } 67 | 68 | /** 69 | * @param string $key 70 | * @param string|integer $value 71 | * @return KafkaProducerMessageInterface 72 | */ 73 | public function withHeader(string $key, $value): KafkaProducerMessageInterface 74 | { 75 | $new = clone $this; 76 | 77 | $new->headers[$key] = $value; 78 | 79 | return $new; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/Message/KafkaProducerMessageInterface.php: -------------------------------------------------------------------------------- 1 | 21 | */ 22 | private $schemaMapping = [ 23 | self::BODY_IDX => [], 24 | self::KEY_IDX => [], 25 | ]; 26 | 27 | /** 28 | * AvroSchemaRegistry constructor. 29 | * @param Registry $registry 30 | */ 31 | public function __construct(Registry $registry) 32 | { 33 | $this->registry = $registry; 34 | } 35 | 36 | /** 37 | * @param string $topicName 38 | * @param KafkaAvroSchemaInterface $avroSchema 39 | * @return void 40 | */ 41 | public function addBodySchemaMappingForTopic(string $topicName, KafkaAvroSchemaInterface $avroSchema): void 42 | { 43 | $this->schemaMapping[self::BODY_IDX][$topicName] = $avroSchema; 44 | } 45 | 46 | /** 47 | * @param string $topicName 48 | * @param KafkaAvroSchemaInterface $avroSchema 49 | * @return void 50 | */ 51 | public function addKeySchemaMappingForTopic(string $topicName, KafkaAvroSchemaInterface $avroSchema): void 52 | { 53 | $this->schemaMapping[self::KEY_IDX][$topicName] = $avroSchema; 54 | } 55 | 56 | /** 57 | * @param string $topicName 58 | * @return KafkaAvroSchemaInterface 59 | * @throws SchemaRegistryException 60 | */ 61 | public function getBodySchemaForTopic(string $topicName): KafkaAvroSchemaInterface 62 | { 63 | return $this->getSchemaForTopicAndType($topicName, self::BODY_IDX); 64 | } 65 | 66 | /** 67 | * @param string $topicName 68 | * @return KafkaAvroSchemaInterface 69 | * @throws SchemaRegistryException 70 | */ 71 | public function getKeySchemaForTopic(string $topicName): KafkaAvroSchemaInterface 72 | { 73 | return $this->getSchemaForTopicAndType($topicName, self::KEY_IDX); 74 | } 75 | 76 | /** 77 | * @param string $topicName 78 | * @return boolean 79 | * @throws SchemaRegistryException 80 | */ 81 | public function hasBodySchemaForTopic(string $topicName): bool 82 | { 83 | return isset($this->schemaMapping[self::BODY_IDX][$topicName]); 84 | } 85 | 86 | /** 87 | * @param string $topicName 88 | * @return boolean 89 | * @throws SchemaRegistryException 90 | */ 91 | public function hasKeySchemaForTopic(string $topicName): bool 92 | { 93 | return isset($this->schemaMapping[self::KEY_IDX][$topicName]); 94 | } 95 | 96 | /** 97 | * @param string $topicName 98 | * @param string $type 99 | * @return KafkaAvroSchemaInterface 100 | * @throws SchemaRegistryException|AvroSchemaRegistryException 101 | */ 102 | private function getSchemaForTopicAndType(string $topicName, string $type): KafkaAvroSchemaInterface 103 | { 104 | if (false === isset($this->schemaMapping[$type][$topicName])) { 105 | throw new AvroSchemaRegistryException( 106 | sprintf( 107 | AvroSchemaRegistryException::SCHEMA_MAPPING_NOT_FOUND, 108 | $topicName, 109 | $type 110 | ) 111 | ); 112 | } 113 | 114 | $avroSchema = $this->schemaMapping[$type][$topicName]; 115 | 116 | if (null !== $avroSchema->getDefinition()) { 117 | return $avroSchema; 118 | } 119 | 120 | $avroSchema->setDefinition($this->getSchemaDefinition($avroSchema)); 121 | $this->schemaMapping[$type][$topicName] = $avroSchema; 122 | 123 | return $avroSchema; 124 | } 125 | 126 | /** 127 | * @param KafkaAvroSchemaInterface $avroSchema 128 | * @return \AvroSchema 129 | * @throws SchemaRegistryException 130 | */ 131 | private function getSchemaDefinition(KafkaAvroSchemaInterface $avroSchema): \AvroSchema 132 | { 133 | if (KafkaAvroSchemaInterface::LATEST_VERSION === $avroSchema->getVersion()) { 134 | return $this->registry->latestVersion($avroSchema->getName()); 135 | } 136 | 137 | return $this->registry->schemaForSubjectAndVersion($avroSchema->getName(), $avroSchema->getVersion()); 138 | } 139 | 140 | /** 141 | * @return array 142 | */ 143 | public function getTopicSchemaMapping(): array 144 | { 145 | return $this->schemaMapping; 146 | } 147 | } 148 | -------------------------------------------------------------------------------- /src/Message/Registry/AvroSchemaRegistryInterface.php: -------------------------------------------------------------------------------- 1 | 34 | */ 35 | public function getTopicSchemaMapping(): array; 36 | 37 | /** 38 | * @param string $topicName 39 | * @return KafkaAvroSchemaInterface 40 | * @throws SchemaRegistryException 41 | */ 42 | public function getBodySchemaForTopic(string $topicName): KafkaAvroSchemaInterface; 43 | 44 | /** 45 | * @param string $topicName 46 | * @return KafkaAvroSchemaInterface 47 | * @throws SchemaRegistryException 48 | */ 49 | public function getKeySchemaForTopic(string $topicName): KafkaAvroSchemaInterface; 50 | 51 | /** 52 | * @param string $topicName 53 | * @return boolean 54 | * @throws SchemaRegistryException 55 | */ 56 | public function hasBodySchemaForTopic(string $topicName): bool; 57 | 58 | /** 59 | * @param string $topicName 60 | * @return boolean 61 | * @throws SchemaRegistryException 62 | */ 63 | public function hasKeySchemaForTopic(string $topicName): bool; 64 | } 65 | -------------------------------------------------------------------------------- /src/Producer/KafkaProducer.php: -------------------------------------------------------------------------------- 1 | producer = $producer; 58 | $this->kafkaConfiguration = $kafkaConfiguration; 59 | $this->encoder = $encoder; 60 | } 61 | 62 | /** 63 | * Produces a message to the topic and partition defined in the message 64 | * If a schema name was given, the message body will be avro serialized. 65 | * 66 | * @param KafkaProducerMessageInterface $message 67 | * @param boolean $autoPoll 68 | * @param integer $pollTimeoutMs 69 | * @return void 70 | */ 71 | public function produce(KafkaProducerMessageInterface $message, bool $autoPoll = true, int $pollTimeoutMs = 0): void 72 | { 73 | $message = $this->encoder->encode($message); 74 | 75 | $topicProducer = $this->getProducerTopicForTopic($message->getTopicName()); 76 | 77 | $topicProducer->producev( 78 | $message->getPartition(), 79 | RD_KAFKA_MSG_F_BLOCK, 80 | $message->getBody(), 81 | $message->getKey(), 82 | $message->getHeaders() 83 | ); 84 | 85 | if (true === $autoPoll) { 86 | $this->producer->poll($pollTimeoutMs); 87 | } 88 | } 89 | 90 | /** 91 | * Produces a message to the topic and partition defined in the message 92 | * If a schema name was given, the message body will be avro serialized. 93 | * Wait for an event to arrive before continuing (blocking) 94 | * 95 | * @param KafkaProducerMessageInterface $message 96 | * @return void 97 | */ 98 | public function syncProduce(KafkaProducerMessageInterface $message): void 99 | { 100 | $this->produce($message, true, -1); 101 | } 102 | 103 | /** 104 | * Poll for producer event, pass 0 for non-blocking, pass -1 to block until an event arrives 105 | * 106 | * @param integer $timeoutMs 107 | * @return void 108 | */ 109 | public function poll(int $timeoutMs = 0): void 110 | { 111 | $this->producer->poll($timeoutMs); 112 | } 113 | 114 | /** 115 | * Poll for producer events until the number of $queueSize events remain 116 | * 117 | * @param integer $timeoutMs 118 | * @param integer $queueSize 119 | * @return void 120 | */ 121 | public function pollUntilQueueSizeReached(int $timeoutMs = 0, int $queueSize = 0): void 122 | { 123 | while ($this->producer->getOutQLen() > $queueSize) { 124 | $this->producer->poll($timeoutMs); 125 | } 126 | } 127 | 128 | /** 129 | * Purge producer messages that are in flight 130 | * 131 | * @param integer $purgeFlags 132 | * @return integer 133 | */ 134 | public function purge(int $purgeFlags): int 135 | { 136 | return $this->producer->purge($purgeFlags); 137 | } 138 | 139 | /** 140 | * Wait until all outstanding produce requests are completed 141 | * 142 | * @param integer $timeoutMs 143 | * @return integer 144 | */ 145 | public function flush(int $timeoutMs): int 146 | { 147 | return $this->producer->flush($timeoutMs); 148 | } 149 | 150 | /** 151 | * Queries the broker for metadata on a certain topic 152 | * 153 | * @param string $topicName 154 | * @param integer $timeoutMs 155 | * @return RdKafkaMetadataTopic 156 | * @throws RdKafkaException 157 | */ 158 | public function getMetadataForTopic(string $topicName, int $timeoutMs = 10000): RdKafkaMetadataTopic 159 | { 160 | $topic = $this->producer->newTopic($topicName); 161 | return $this->producer 162 | ->getMetadata( 163 | false, 164 | $topic, 165 | $timeoutMs 166 | ) 167 | ->getTopics() 168 | ->current(); 169 | } 170 | 171 | /** 172 | * Start a producer transaction 173 | * 174 | * @param int $timeoutMs 175 | * @return void 176 | * 177 | * @throws KafkaProducerTransactionAbortException 178 | * @throws KafkaProducerTransactionFatalException 179 | * @throws KafkaProducerTransactionRetryException 180 | */ 181 | public function beginTransaction(int $timeoutMs): void 182 | { 183 | try { 184 | if (false === $this->transactionInitialized) { 185 | $this->producer->initTransactions($timeoutMs); 186 | $this->transactionInitialized = true; 187 | } 188 | 189 | $this->producer->beginTransaction(); 190 | } catch (RdKafkaErrorException $e) { 191 | $this->handleTransactionError($e); 192 | } 193 | } 194 | 195 | /** 196 | * Commit the current producer transaction 197 | * 198 | * @param int $timeoutMs 199 | * @return void 200 | * 201 | * @throws KafkaProducerTransactionAbortException 202 | * @throws KafkaProducerTransactionFatalException 203 | * @throws KafkaProducerTransactionRetryException 204 | */ 205 | public function commitTransaction(int $timeoutMs): void 206 | { 207 | try { 208 | $this->producer->commitTransaction($timeoutMs); 209 | } catch (RdKafkaErrorException $e) { 210 | $this->handleTransactionError($e); 211 | } 212 | } 213 | 214 | /** 215 | * Abort the current producer transaction 216 | * 217 | * @param int $timeoutMs 218 | * @return void 219 | * 220 | * @throws KafkaProducerTransactionAbortException 221 | * @throws KafkaProducerTransactionFatalException 222 | * @throws KafkaProducerTransactionRetryException 223 | */ 224 | public function abortTransaction(int $timeoutMs): void 225 | { 226 | try { 227 | $this->producer->abortTransaction($timeoutMs); 228 | } catch (RdKafkaErrorException $e) { 229 | $this->handleTransactionError($e); 230 | } 231 | } 232 | 233 | /** 234 | * @param string $topic 235 | * @return RdKafkaProducerTopic 236 | */ 237 | private function getProducerTopicForTopic(string $topic): RdKafkaProducerTopic 238 | { 239 | if (!isset($this->producerTopics[$topic])) { 240 | $this->producerTopics[$topic] = $this->producer->newTopic($topic); 241 | } 242 | 243 | return $this->producerTopics[$topic]; 244 | } 245 | 246 | /** 247 | * @param RdKafkaErrorException $e 248 | * 249 | * @throws KafkaProducerTransactionAbortException 250 | * @throws KafkaProducerTransactionFatalException 251 | * @throws KafkaProducerTransactionRetryException 252 | */ 253 | private function handleTransactionError(RdKafkaErrorException $e): void 254 | { 255 | if (true === $e->isRetriable()) { 256 | throw new KafkaProducerTransactionRetryException( 257 | sprintf( 258 | KafkaProducerTransactionRetryException::RETRIABLE_TRANSACTION_EXCEPTION_MESSAGE, 259 | $e->getMessage() 260 | ), 261 | $e->getCode(), 262 | $e 263 | ); 264 | } elseif (true === $e->transactionRequiresAbort()) { 265 | throw new KafkaProducerTransactionAbortException( 266 | sprintf( 267 | KafkaProducerTransactionAbortException::TRANSACTION_REQUIRES_ABORT_EXCEPTION_MESSAGE, 268 | $e->getMessage() 269 | ), 270 | $e->getCode(), 271 | $e 272 | ); 273 | } else { 274 | $this->transactionInitialized = false; 275 | // according to librdkafka documentation, everything that is not retriable, abortable or fatal is fatal 276 | // fatal errors (so stated), need the producer to be destroyed 277 | throw new KafkaProducerTransactionFatalException( 278 | sprintf( 279 | KafkaProducerTransactionFatalException::FATAL_TRANSACTION_EXCEPTION_MESSAGE, 280 | $e->getMessage() 281 | ), 282 | $e->getCode(), 283 | $e 284 | ); 285 | } 286 | } 287 | } 288 | -------------------------------------------------------------------------------- /src/Producer/KafkaProducerBuilder.php: -------------------------------------------------------------------------------- 1 | deliverReportCallback = new KafkaProducerDeliveryReportCallback(); 53 | $this->errorCallback = new KafkaErrorCallback(); 54 | $this->encoder = new NullEncoder(); 55 | } 56 | 57 | /** 58 | * Returns the producer builder 59 | * 60 | * @return KafkaProducerBuilderInterface 61 | */ 62 | public static function create(): KafkaProducerBuilderInterface 63 | { 64 | return new self(); 65 | } 66 | 67 | /** 68 | * Adds a broker to which you want to produce 69 | * 70 | * @param string $broker 71 | * @return KafkaProducerBuilderInterface 72 | */ 73 | public function withAdditionalBroker(string $broker): KafkaProducerBuilderInterface 74 | { 75 | $this->brokers[] = $broker; 76 | 77 | return $this; 78 | } 79 | 80 | /** 81 | * Add configuration settings, otherwise the kafka defaults apply 82 | * 83 | * @param string[] $config 84 | * @return KafkaProducerBuilderInterface 85 | */ 86 | public function withAdditionalConfig(array $config): KafkaProducerBuilderInterface 87 | { 88 | $this->config = $config + $this->config; 89 | 90 | return $this; 91 | } 92 | 93 | /** 94 | * Sets callback for the delivery report. The broker will send a delivery 95 | * report for every message which describes if the delivery was successful or not 96 | * 97 | * @param callable $deliveryReportCallback 98 | * @return KafkaProducerBuilderInterface 99 | */ 100 | public function withDeliveryReportCallback(callable $deliveryReportCallback): KafkaProducerBuilderInterface 101 | { 102 | $this->deliverReportCallback = $deliveryReportCallback; 103 | 104 | return $this; 105 | } 106 | 107 | /** 108 | * Set a callback to be called on errors. 109 | * The default callback will throw an exception for every error 110 | * 111 | * @param callable $errorCallback 112 | * @return KafkaProducerBuilderInterface 113 | */ 114 | public function withErrorCallback(callable $errorCallback): KafkaProducerBuilderInterface 115 | { 116 | $this->errorCallback = $errorCallback; 117 | 118 | return $this; 119 | } 120 | 121 | /** 122 | * Callback for log related events 123 | * 124 | * @param callable $logCallback 125 | * @return KafkaProducerBuilderInterface 126 | */ 127 | public function withLogCallback(callable $logCallback): KafkaProducerBuilderInterface 128 | { 129 | $this->logCallback = $logCallback; 130 | 131 | return $this; 132 | } 133 | 134 | /** 135 | * Lets you set a custom encoder for produce message 136 | * 137 | * @param EncoderInterface $encoder 138 | * @return KafkaProducerBuilderInterface 139 | */ 140 | public function withEncoder(EncoderInterface $encoder): KafkaProducerBuilderInterface 141 | { 142 | $this->encoder = $encoder; 143 | 144 | return $this; 145 | } 146 | 147 | /** 148 | * Returns your producer instance 149 | * 150 | * @return KafkaProducerInterface 151 | * @throws KafkaProducerException 152 | */ 153 | public function build(): KafkaProducerInterface 154 | { 155 | if ([] === $this->brokers) { 156 | throw new KafkaProducerException(KafkaProducerException::NO_BROKER_EXCEPTION_MESSAGE); 157 | } 158 | 159 | //Thread termination improvement (https://github.com/arnaud-lb/php-rdkafka#performance--low-latency-settings) 160 | $this->config['socket.timeout.ms'] = '50'; 161 | $this->config['queue.buffering.max.ms'] = '1'; 162 | 163 | if (function_exists('pcntl_sigprocmask')) { 164 | pcntl_sigprocmask(SIG_BLOCK, array(SIGIO)); 165 | $this->config['internal.termination.signal'] = (string) SIGIO; 166 | unset($this->config['queue.buffering.max.ms']); 167 | } 168 | 169 | $kafkaConfig = new KafkaConfiguration($this->brokers, [], $this->config); 170 | 171 | //set producer callbacks 172 | $this->registerCallbacks($kafkaConfig); 173 | 174 | $rdKafkaProducer = new RdKafkaProducer($kafkaConfig); 175 | 176 | return new KafkaProducer($rdKafkaProducer, $kafkaConfig, $this->encoder); 177 | } 178 | 179 | /** 180 | * @param KafkaConfiguration $conf 181 | * @return void 182 | */ 183 | private function registerCallbacks(KafkaConfiguration $conf): void 184 | { 185 | $conf->setDrMsgCb($this->deliverReportCallback); 186 | $conf->setErrorCb($this->errorCallback); 187 | 188 | if (null !== $this->logCallback) { 189 | $conf->setLogCb($this->logCallback); 190 | } 191 | } 192 | } 193 | -------------------------------------------------------------------------------- /src/Producer/KafkaProducerBuilderInterface.php: -------------------------------------------------------------------------------- 1 | getConsumerMock(); 27 | 28 | $consumer 29 | ->expects(self::once()) 30 | ->method('assign') 31 | ->with(null) 32 | ->willThrowException(new RdKafkaException($exceptionMessage, $exceptionCode)); 33 | 34 | call_user_func(new KafkaConsumerRebalanceCallback(), $consumer, 1, []); 35 | } 36 | 37 | public function testInvokeAssign() 38 | { 39 | $partitions = [1, 2, 3]; 40 | 41 | $consumer = $this->getConsumerMock(); 42 | 43 | $consumer 44 | ->expects(self::once()) 45 | ->method('assign') 46 | ->with($partitions) 47 | ->willReturn(null); 48 | 49 | 50 | call_user_func( 51 | new KafkaConsumerRebalanceCallback(), 52 | $consumer, 53 | RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS, 54 | $partitions 55 | ); 56 | } 57 | 58 | public function testInvokeRevoke() 59 | { 60 | $consumer = $this->getConsumerMock(); 61 | 62 | $consumer 63 | ->expects(self::once()) 64 | ->method('assign') 65 | ->with(null) 66 | ->willReturn(null); 67 | 68 | call_user_func(new KafkaConsumerRebalanceCallback(), $consumer, RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS); 69 | } 70 | 71 | /** 72 | * @return MockObject|RdKafkaConsumer 73 | */ 74 | private function getConsumerMock() 75 | { 76 | //create mock to assign topics 77 | $consumerMock = $this->getMockBuilder(RdKafkaConsumer::class) 78 | ->disableOriginalConstructor() 79 | ->onlyMethods(['assign', 'unsubscribe', 'getSubscription']) 80 | ->getMock(); 81 | 82 | $consumerMock 83 | ->expects(self::any()) 84 | ->method('unsubscribe') 85 | ->willReturn(null); 86 | 87 | $consumerMock 88 | ->expects(self::any()) 89 | ->method('getSubscription') 90 | ->willReturn([]); 91 | 92 | return $consumerMock; 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /tests/Unit/Callback/KafkaErrorCallbackTest.php: -------------------------------------------------------------------------------- 1 | getMockBuilder(RdKafkaProducer::class) 20 | ->disableOriginalConstructor() 21 | ->getMock(); 22 | } 23 | 24 | public function testInvokeDefault() 25 | { 26 | self::expectException(KafkaProducerException::class); 27 | 28 | $message = new Message(); 29 | $message->err = -1; 30 | 31 | call_user_func(new KafkaProducerDeliveryReportCallback(), $this->getProducerMock(), $message); 32 | } 33 | 34 | public function testInvokeTimeout() 35 | { 36 | self::expectException(KafkaProducerException::class); 37 | 38 | $message = new Message(); 39 | $message->err = RD_KAFKA_RESP_ERR__MSG_TIMED_OUT; 40 | 41 | call_user_func(new KafkaProducerDeliveryReportCallback(), $this->getProducerMock(), $message); 42 | } 43 | 44 | public function testInvokeNoError() 45 | { 46 | $message = new Message(); 47 | $message->err = RD_KAFKA_RESP_ERR_NO_ERROR; 48 | 49 | $result = call_user_func(new KafkaProducerDeliveryReportCallback(), $this->getProducerMock(), $message); 50 | 51 | self::assertNull($result); 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tests/Unit/Conf/KafkaConfigurationTest.php: -------------------------------------------------------------------------------- 1 | getBrokers()); 52 | self::assertEquals($topicSubscriptions, $kafkaConfiguration->getTopicSubscriptions()); 53 | } 54 | 55 | /** 56 | * @dataProvider kafkaConfigurationDataProvider 57 | * @param array $brokers 58 | * @param array $topicSubscriptions 59 | * @return void 60 | */ 61 | public function testGetConfiguration(array $brokers, array $topicSubscriptions): void 62 | { 63 | $kafkaConfiguration = new KafkaConfiguration($brokers, $topicSubscriptions); 64 | 65 | self::assertEquals($kafkaConfiguration->dump(), $kafkaConfiguration->getConfiguration()); 66 | } 67 | 68 | /** 69 | * @return array 70 | */ 71 | public function configValuesProvider(): array 72 | { 73 | return [ 74 | [ 1, '1' ], 75 | [ -1, '-1' ], 76 | [ 1.123333, '1.123333' ], 77 | [ -0.99999, '-0.99999' ], 78 | [ true, 'true' ], 79 | [ false, 'false' ], 80 | [ ' ', ' ' ], 81 | [ [], null ], 82 | [ new stdClass(), null ], 83 | ]; 84 | } 85 | 86 | /** 87 | * @dataProvider configValuesProvider 88 | * @param mixed $inputValue 89 | * @param mixed $expectedValue 90 | */ 91 | public function testConfigValues($inputValue, $expectedValue): void 92 | { 93 | $kafkaConfiguration = new KafkaConfiguration( 94 | ['localhost'], 95 | [new TopicSubscription('test-topic')], 96 | [ 97 | 'group.id' => $inputValue, 98 | 'auto.commit.interval.ms' => 100 99 | ], 100 | KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL 101 | ); 102 | 103 | $config = $kafkaConfiguration->getConfiguration(); 104 | 105 | if (null === $expectedValue) { 106 | self::assertArrayNotHasKey('group.id', $config); 107 | return; 108 | } 109 | 110 | self::assertEquals($config['metadata.broker.list'], 'localhost'); 111 | self::assertEquals($expectedValue, $config['group.id']); 112 | self::assertEquals('100', $config['auto.commit.interval.ms']); 113 | self::assertArrayHasKey('default_topic_conf', $config); 114 | self::assertIsString($config['default_topic_conf']); 115 | } 116 | 117 | public function testMethodVisibility(): void 118 | { 119 | $reflectionClass = new \ReflectionClass(KafkaConfiguration::class); 120 | 121 | $methodInitializedConfig = $reflectionClass->getMethod('initializeConfig'); 122 | $methodInitializedConfig->setAccessible(true); 123 | 124 | $this->assertTrue($methodInitializedConfig->isProtected()); 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /tests/Unit/Consumer/KafkaConsumerBuilderTest.php: -------------------------------------------------------------------------------- 1 | kafkaConsumerBuilder = KafkaConsumerBuilder::create(); 31 | } 32 | 33 | /** 34 | * @return void 35 | */ 36 | public function testCreate(): void 37 | { 38 | self::assertInstanceOf(KafkaConsumerBuilder::class, KafkaConsumerBuilder::create()); 39 | } 40 | 41 | /** 42 | * @return void 43 | * @throws \ReflectionException 44 | */ 45 | public function testAddBroker(): void 46 | { 47 | self::assertNotSame( 48 | $this->kafkaConsumerBuilder, 49 | $clone = $this->kafkaConsumerBuilder->withAdditionalBroker('localhost') 50 | ); 51 | 52 | $reflectionProperty = new \ReflectionProperty($clone, 'brokers'); 53 | $reflectionProperty->setAccessible(true); 54 | 55 | self::assertSame(['localhost'], $reflectionProperty->getValue($clone)); 56 | } 57 | 58 | /** 59 | * @return void 60 | * @throws \ReflectionException 61 | */ 62 | public function testSubscribeToTopic(): void 63 | { 64 | self::assertNotSame( 65 | $this->kafkaConsumerBuilder, 66 | $clone = $this->kafkaConsumerBuilder->withAdditionalSubscription('test-topic') 67 | ); 68 | 69 | $reflectionProperty = new \ReflectionProperty($clone, 'topics'); 70 | $reflectionProperty->setAccessible(true); 71 | 72 | self::isInstanceOf(TopicSubscription::class, $reflectionProperty->getValue($clone)); 73 | } 74 | 75 | /** 76 | * @return void 77 | * @throws \ReflectionException 78 | */ 79 | public function testReplaceSubscribedToTopics(): void 80 | { 81 | self::assertNotSame( 82 | $this->kafkaConsumerBuilder, 83 | $clone = $this->kafkaConsumerBuilder->withSubscription('new-topic') 84 | ); 85 | 86 | $reflectionProperty = new \ReflectionProperty($clone, 'topics'); 87 | $reflectionProperty->setAccessible(true); 88 | 89 | $topicSubscription = $reflectionProperty->getValue($clone); 90 | self::assertCount(1, $topicSubscription); 91 | self::isInstanceOf(TopicSubscription::class, $topicSubscription[0]); 92 | self::assertSame('new-topic', $topicSubscription[0]->getTopicName()); 93 | } 94 | 95 | /** 96 | * @return void 97 | * @throws \ReflectionException 98 | */ 99 | public function testAddConfig(): void 100 | { 101 | $intialConfig = ['group.id' => 'test-group', 'enable.auto.offset.store' => true]; 102 | $newConfig = ['offset.store.sync.interval.ms' => 60e3]; 103 | $clone = $this->kafkaConsumerBuilder->withAdditionalConfig($intialConfig); 104 | $clone = $clone->withAdditionalConfig($newConfig); 105 | 106 | $reflectionProperty = new \ReflectionProperty($clone, 'config'); 107 | $reflectionProperty->setAccessible(true); 108 | 109 | self::assertSame( 110 | [ 111 | 'offset.store.sync.interval.ms' => 60e3, 112 | 'group.id' => 'test-group', 113 | 'enable.auto.offset.store' => true, 114 | 'enable.auto.commit' => false, 115 | 'auto.offset.reset' => 'earliest' 116 | ], 117 | $reflectionProperty->getValue($clone) 118 | ); 119 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 120 | } 121 | 122 | /** 123 | * @return void 124 | * @throws \ReflectionException 125 | */ 126 | public function testSetDecoder(): void 127 | { 128 | $decoder = $this->getMockForAbstractClass(DecoderInterface::class); 129 | 130 | $clone = $this->kafkaConsumerBuilder->withDecoder($decoder); 131 | 132 | $reflectionProperty = new \ReflectionProperty($clone, 'decoder'); 133 | $reflectionProperty->setAccessible(true); 134 | 135 | self::assertInstanceOf(DecoderInterface::class, $reflectionProperty->getValue($clone)); 136 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 137 | } 138 | 139 | /** 140 | * @return void 141 | * @throws \ReflectionException 142 | */ 143 | public function testSetConsumerGroup(): void 144 | { 145 | $clone = $this->kafkaConsumerBuilder->withConsumerGroup('test-consumer'); 146 | 147 | $reflectionProperty = new \ReflectionProperty($clone, 'consumerGroup'); 148 | $reflectionProperty->setAccessible(true); 149 | 150 | self::assertSame('test-consumer', $reflectionProperty->getValue($clone)); 151 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 152 | } 153 | 154 | /** 155 | * @return void 156 | * @throws \ReflectionException 157 | */ 158 | public function testSetConsumerTypeLow(): void 159 | { 160 | $clone = $this->kafkaConsumerBuilder->withConsumerType(KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL); 161 | 162 | $actualConsumerType = new \ReflectionProperty($clone, 'consumerType'); 163 | $actualConsumerType->setAccessible(true); 164 | 165 | self::assertSame(KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL, $actualConsumerType->getValue($clone)); 166 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 167 | } 168 | 169 | /** 170 | * @return void 171 | * @throws \ReflectionException 172 | */ 173 | public function testSetConsumerTypeHigh(): void 174 | { 175 | $clone = $this->kafkaConsumerBuilder->withConsumerType(KafkaConsumerBuilder::CONSUMER_TYPE_HIGH_LEVEL); 176 | 177 | $actualConsumerType = new \ReflectionProperty($clone, 'consumerType'); 178 | $actualConsumerType->setAccessible(true); 179 | 180 | self::assertSame(KafkaConsumerBuilder::CONSUMER_TYPE_HIGH_LEVEL, $actualConsumerType->getValue($clone)); 181 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 182 | } 183 | 184 | /** 185 | * @return void 186 | * @throws \ReflectionException 187 | */ 188 | public function testSetErrorCallback(): void 189 | { 190 | $callback = function () { 191 | // Anonymous test method, no logic required 192 | }; 193 | 194 | $clone = $this->kafkaConsumerBuilder->withErrorCallback($callback); 195 | 196 | $reflectionProperty = new \ReflectionProperty($clone, 'errorCallback'); 197 | $reflectionProperty->setAccessible(true); 198 | 199 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 200 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 201 | } 202 | 203 | /** 204 | * @return void 205 | * @throws \ReflectionException 206 | */ 207 | public function testSetRebalanceCallback(): void 208 | { 209 | $callback = function () { 210 | // Anonymous test method, no logic required 211 | }; 212 | 213 | $clone = $this->kafkaConsumerBuilder->withRebalanceCallback($callback); 214 | 215 | $reflectionProperty = new \ReflectionProperty($clone, 'rebalanceCallback'); 216 | $reflectionProperty->setAccessible(true); 217 | 218 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 219 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 220 | } 221 | 222 | /** 223 | * @return void 224 | * @throws \ReflectionException 225 | */ 226 | public function testSetConsumeCallback(): void 227 | { 228 | $callback = function () { 229 | // Anonymous test method, no logic required 230 | }; 231 | 232 | $clone = $this->kafkaConsumerBuilder->withConsumeCallback($callback); 233 | 234 | $reflectionProperty = new \ReflectionProperty($clone, 'consumeCallback'); 235 | $reflectionProperty->setAccessible(true); 236 | 237 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 238 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 239 | } 240 | 241 | /** 242 | * @return void 243 | * @throws \ReflectionException 244 | */ 245 | public function testSetOffsetCommitCallback(): void 246 | { 247 | $callback = function () { 248 | // Anonymous test method, no logic required 249 | }; 250 | 251 | $clone = $this->kafkaConsumerBuilder->withOffsetCommitCallback($callback); 252 | 253 | $reflectionProperty = new \ReflectionProperty($clone, 'offsetCommitCallback'); 254 | $reflectionProperty->setAccessible(true); 255 | 256 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 257 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 258 | } 259 | 260 | /** 261 | * @return void 262 | * @throws \ReflectionException 263 | */ 264 | public function testSetLogCallback(): void 265 | { 266 | $callback = function () { 267 | // Anonymous test method, no logic required 268 | }; 269 | 270 | $clone = $this->kafkaConsumerBuilder->withLogCallback($callback); 271 | 272 | $reflectionProperty = new \ReflectionProperty($clone, 'logCallback'); 273 | $reflectionProperty->setAccessible(true); 274 | 275 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 276 | self::assertNotSame($clone, $this->kafkaConsumerBuilder); 277 | } 278 | 279 | /** 280 | * @return void 281 | * @throws KafkaConsumerBuilderException 282 | */ 283 | public function testBuildFailMissingBrokers(): void 284 | { 285 | self::expectException(KafkaConsumerBuilderException::class); 286 | self::expectExceptionMessage(KafkaConsumerBuilderException::NO_BROKER_EXCEPTION_MESSAGE); 287 | 288 | $this->kafkaConsumerBuilder->build(); 289 | } 290 | 291 | /** 292 | * @return void 293 | */ 294 | public function testBuildSuccess(): void 295 | { 296 | $callback = function ($kafka, $errId, $msg) { 297 | // Anonymous test method, no logic required 298 | }; 299 | 300 | /** @var $consumer KafkaLowLevelConsumer */ 301 | $consumer = $this->kafkaConsumerBuilder 302 | ->withAdditionalBroker('localhost') 303 | ->withAdditionalSubscription('test-topic') 304 | ->withRebalanceCallback($callback) 305 | ->withOffsetCommitCallback($callback) 306 | ->withConsumeCallback($callback) 307 | ->withErrorCallback($callback) 308 | ->withLogCallback($callback) 309 | ->build(); 310 | 311 | self::assertInstanceOf(KafkaConsumerInterface::class, $consumer); 312 | self::assertInstanceOf(KafkaHighLevelConsumer::class, $consumer); 313 | } 314 | 315 | /** 316 | * @return void 317 | */ 318 | public function testBuildLowLevelSuccess(): void 319 | { 320 | $callback = function ($kafka, $errId, $msg) { 321 | // Anonymous test method, no logic required 322 | }; 323 | /** @var $consumer KafkaLowLevelConsumer */ 324 | $consumer = $this->kafkaConsumerBuilder 325 | ->withAdditionalBroker('localhost') 326 | ->withAdditionalSubscription('test-topic') 327 | ->withRebalanceCallback($callback) 328 | ->withErrorCallback($callback) 329 | ->withLogCallback($callback) 330 | ->withConsumerType(KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL) 331 | ->build(); 332 | 333 | $conf = $consumer->getConfiguration(); 334 | 335 | self::assertInstanceOf(KafkaConsumerInterface::class, $consumer); 336 | self::assertInstanceOf(KafkaLowLevelConsumerInterface::class, $consumer); 337 | self::assertArrayHasKey('enable.auto.offset.store', $conf); 338 | self::assertEquals($conf['enable.auto.offset.store'], 'false'); 339 | } 340 | 341 | /** 342 | * @return void 343 | */ 344 | public function testBuildLowLevelFailureOnUnsupportedCallback(): void 345 | { 346 | $callback = function ($kafka, $errId, $msg) { 347 | // Anonymous test method, no logic required 348 | }; 349 | 350 | self::expectException(KafkaConsumerBuilderException::class); 351 | self::expectExceptionMessage( 352 | sprintf( 353 | KafkaConsumerBuilderException::UNSUPPORTED_CALLBACK_EXCEPTION_MESSAGE, 354 | 'consumerCallback', 355 | KafkaLowLevelConsumer::class 356 | ) 357 | ); 358 | 359 | $this->kafkaConsumerBuilder 360 | ->withAdditionalBroker('localhost') 361 | ->withAdditionalSubscription('test-topic') 362 | ->withConsumeCallback($callback) 363 | ->withConsumerType(KafkaConsumerBuilder::CONSUMER_TYPE_LOW_LEVEL) 364 | ->build(); 365 | } 366 | 367 | /** 368 | * @return void 369 | */ 370 | public function testBuildHighLevelSuccess(): void 371 | { 372 | $callback = function ($kafka, $errId, $msg) { 373 | // Anonymous test method, no logic required 374 | }; 375 | 376 | /** @var $consumer KafkaHighLevelConsumer */ 377 | $consumer = $this->kafkaConsumerBuilder 378 | ->withAdditionalBroker('localhost') 379 | ->withAdditionalSubscription('test-topic') 380 | ->withRebalanceCallback($callback) 381 | ->withErrorCallback($callback) 382 | ->withLogCallback($callback) 383 | ->build(); 384 | 385 | $conf = $consumer->getConfiguration(); 386 | 387 | self::assertInstanceOf(KafkaConsumerInterface::class, $consumer); 388 | self::assertInstanceOf(KafkaHighLevelConsumerInterface::class, $consumer); 389 | self::assertArrayHasKey('enable.auto.commit', $conf); 390 | self::assertEquals($conf['enable.auto.commit'], 'false'); 391 | } 392 | } 393 | -------------------------------------------------------------------------------- /tests/Unit/Consumer/TopicSubscriptionTest.php: -------------------------------------------------------------------------------- 1 | getTopicName()); 26 | self::assertEquals($partitions, $topicSubscription->getPartitions()); 27 | self::assertEquals($offset, $topicSubscription->getOffset()); 28 | 29 | $topicSubscription->setPartitions($newPartitions); 30 | 31 | self::assertEquals($newPartitions, $topicSubscription->getPartitions()); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /tests/Unit/Exception/KafkaConsumerConsumeExceptionTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaConsumerMessageInterface::class); 20 | 21 | $exception = new KafkaConsumerConsumeException('', 0, $message); 22 | 23 | self::assertSame($message, $exception->getKafkaMessage()); 24 | } 25 | 26 | public function testGetAndConstructOfKafkaConsumerConsumeExceptionWithNullAsMessage() 27 | { 28 | $exception = new KafkaConsumerConsumeException('test', 100, null); 29 | 30 | self::assertNull($exception->getKafkaMessage()); 31 | self::assertEquals('test', $exception->getMessage()); 32 | self::assertEquals(100, $exception->getCode()); 33 | } 34 | 35 | public function testGetDefaults() 36 | { 37 | $exception = new KafkaConsumerConsumeException(); 38 | 39 | self::assertNull($exception->getKafkaMessage()); 40 | self::assertEquals('', $exception->getMessage()); 41 | self::assertEquals(0, $exception->getCode()); 42 | self::assertNull($exception->getPrevious()); 43 | 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /tests/Unit/Message/Decoder/AvroDecoderTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaConsumerMessageInterface::class); 23 | $message->expects(self::once())->method('getBody')->willReturn(null); 24 | 25 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 26 | $registry->expects(self::never())->method('hasBodySchemaForTopic'); 27 | $registry->expects(self::never())->method('hasKeySchemaForTopic'); 28 | 29 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 30 | $recordSerializer->expects(self::never())->method('decodeMessage'); 31 | 32 | $decoder = new AvroDecoder($registry, $recordSerializer); 33 | 34 | $result = $decoder->decode($message); 35 | 36 | self::assertInstanceOf(KafkaConsumerMessageInterface::class, $result); 37 | self::assertNull($result->getBody()); 38 | } 39 | 40 | public function testDecodeWithSchema() 41 | { 42 | $schemaDefinition = $this->getMockBuilder(\AvroSchema::class)->disableOriginalConstructor()->getMock(); 43 | 44 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 45 | $avroSchema->expects(self::exactly(2))->method('getDefinition')->willReturn($schemaDefinition); 46 | 47 | $message = $this->getMockForAbstractClass(KafkaConsumerMessageInterface::class); 48 | $message->expects(self::exactly(3))->method('getTopicName')->willReturn('test-topic'); 49 | $message->expects(self::once())->method('getPartition')->willReturn(0); 50 | $message->expects(self::once())->method('getOffset')->willReturn(1); 51 | $message->expects(self::once())->method('getTimestamp')->willReturn(time()); 52 | $message->expects(self::exactly(2))->method('getKey')->willReturn('test-key'); 53 | $message->expects(self::exactly(2))->method('getBody')->willReturn('body'); 54 | $message->expects(self::once())->method('getHeaders')->willReturn([]); 55 | 56 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 57 | $registry->expects(self::once())->method('getBodySchemaForTopic')->willReturn($avroSchema); 58 | $registry->expects(self::once())->method('getKeySchemaForTopic')->willReturn($avroSchema); 59 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(true); 60 | $registry->expects(self::once())->method('hasKeySchemaForTopic')->willReturn(true); 61 | 62 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 63 | $recordSerializer->expects(self::exactly(2)) 64 | ->method('decodeMessage') 65 | ->withConsecutive( 66 | [$message->getKey(), $schemaDefinition], 67 | [$message->getBody(), $schemaDefinition], 68 | ) 69 | ->willReturnOnConsecutiveCalls('decoded-key', ['test']); 70 | 71 | $decoder = new AvroDecoder($registry, $recordSerializer); 72 | 73 | $result = $decoder->decode($message); 74 | 75 | self::assertInstanceOf(KafkaConsumerMessageInterface::class, $result); 76 | self::assertSame(['test'], $result->getBody()); 77 | self::assertSame('decoded-key', $result->getKey()); 78 | 79 | } 80 | 81 | public function testDecodeKeyMode() 82 | { 83 | $schemaDefinition = $this->getMockBuilder(\AvroSchema::class)->disableOriginalConstructor()->getMock(); 84 | 85 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 86 | $avroSchema->expects(self::once())->method('getDefinition')->willReturn($schemaDefinition); 87 | 88 | $message = $this->getMockForAbstractClass(KafkaConsumerMessageInterface::class); 89 | $message->expects(self::exactly(3))->method('getTopicName')->willReturn('test-topic'); 90 | $message->expects(self::once())->method('getPartition')->willReturn(0); 91 | $message->expects(self::once())->method('getOffset')->willReturn(1); 92 | $message->expects(self::once())->method('getTimestamp')->willReturn(time()); 93 | $message->expects(self::exactly(2))->method('getKey')->willReturn('test-key'); 94 | $message->expects(self::once())->method('getBody')->willReturn('body'); 95 | $message->expects(self::once())->method('getHeaders')->willReturn([]); 96 | 97 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 98 | $registry->expects(self::never())->method('getBodySchemaForTopic'); 99 | $registry->expects(self::once())->method('getKeySchemaForTopic')->willReturn($avroSchema); 100 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(false); 101 | $registry->expects(self::once())->method('hasKeySchemaForTopic')->willReturn(true); 102 | 103 | 104 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 105 | $recordSerializer->expects(self::once())->method('decodeMessage')->with($message->getKey(), $schemaDefinition)->willReturn('decoded-key'); 106 | 107 | $decoder = new AvroDecoder($registry, $recordSerializer); 108 | 109 | $result = $decoder->decode($message); 110 | 111 | self::assertInstanceOf(KafkaConsumerMessageInterface::class, $result); 112 | self::assertSame('decoded-key', $result->getKey()); 113 | self::assertSame('body', $result->getBody()); 114 | 115 | } 116 | 117 | public function testDecodeBodyMode() 118 | { 119 | $schemaDefinition = $this->getMockBuilder(\AvroSchema::class)->disableOriginalConstructor()->getMock(); 120 | 121 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 122 | $avroSchema->expects(self::once())->method('getDefinition')->willReturn($schemaDefinition); 123 | 124 | $message = $this->getMockForAbstractClass(KafkaConsumerMessageInterface::class); 125 | $message->expects(self::exactly(3))->method('getTopicName')->willReturn('test-topic'); 126 | $message->expects(self::once())->method('getPartition')->willReturn(0); 127 | $message->expects(self::once())->method('getOffset')->willReturn(1); 128 | $message->expects(self::once())->method('getTimestamp')->willReturn(time()); 129 | $message->expects(self::once())->method('getKey')->willReturn('test-key'); 130 | $message->expects(self::exactly(2))->method('getBody')->willReturn('body'); 131 | $message->expects(self::once())->method('getHeaders')->willReturn([]); 132 | 133 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 134 | $registry->expects(self::once())->method('getBodySchemaForTopic')->willReturn($avroSchema); 135 | $registry->expects(self::never())->method('getKeySchemaForTopic'); 136 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(true); 137 | $registry->expects(self::once())->method('hasKeySchemaForTopic')->willReturn(false); 138 | 139 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 140 | $recordSerializer->expects(self::once())->method('decodeMessage')->with($message->getBody(), $schemaDefinition)->willReturn(['test']); 141 | 142 | $decoder = new AvroDecoder($registry, $recordSerializer); 143 | 144 | $result = $decoder->decode($message); 145 | 146 | self::assertInstanceOf(KafkaConsumerMessageInterface::class, $result); 147 | self::assertSame('test-key', $result->getKey()); 148 | self::assertSame(['test'], $result->getBody()); 149 | 150 | } 151 | 152 | public function testGetRegistry() 153 | { 154 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 155 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 156 | 157 | $decoder = new AvroDecoder($registry, $recordSerializer); 158 | 159 | self::assertSame($registry, $decoder->getRegistry()); 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /tests/Unit/Message/Decoder/JsonDecoderTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaConsumerMessageInterface::class); 23 | $message->expects(self::once())->method('getBody')->willReturn('{"name":"foo"}'); 24 | $decoder = new JsonDecoder(); 25 | $result = $decoder->decode($message); 26 | 27 | self::assertInstanceOf(KafkaConsumerMessageInterface::class, $result); 28 | self::assertEquals(['name' => 'foo'], $result->getBody()); 29 | } 30 | 31 | /** 32 | * @return void 33 | */ 34 | public function testDecodeNonJson(): void 35 | { 36 | $message = $this->getMockForAbstractClass(KafkaConsumerMessageInterface::class); 37 | $message->expects(self::once())->method('getBody')->willReturn('test'); 38 | $decoder = new JsonDecoder(); 39 | 40 | self::expectException(\JsonException::class); 41 | 42 | $decoder->decode($message); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/Unit/Message/Decoder/NullDecoderTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaConsumerMessageInterface::class); 23 | 24 | self::assertSame($message, (new NullDecoder())->decode($message)); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/Unit/Message/Encoder/AvroEncoderTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaProducerMessageInterface::class); 23 | $producerMessage->expects(self::exactly(2))->method('getBody')->willReturn(null); 24 | 25 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 26 | $registry->expects(self::never())->method('hasBodySchemaForTopic'); 27 | $registry->expects(self::never())->method('hasKeySchemaForTopic'); 28 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 29 | $recordSerializer->expects(self::never())->method('encodeRecord'); 30 | $encoder = new AvroEncoder($registry, $recordSerializer); 31 | $result = $encoder->encode($producerMessage); 32 | 33 | self::assertInstanceOf(KafkaProducerMessageInterface::class, $result); 34 | self::assertSame($producerMessage, $result); 35 | self::assertNull($result->getBody()); 36 | } 37 | 38 | public function testEncodeWithoutSchemaDefinition() 39 | { 40 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 41 | $avroSchema->expects(self::once())->method('getDefinition')->willReturn(null); 42 | 43 | $producerMessage = $this->getMockForAbstractClass(KafkaProducerMessageInterface::class); 44 | $producerMessage->expects(self::once())->method('getTopicName')->willReturn('test'); 45 | $producerMessage->expects(self::once())->method('getBody')->willReturn('test'); 46 | 47 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 48 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(true); 49 | $registry->expects(self::once())->method('getBodySchemaForTopic')->willReturn($avroSchema); 50 | 51 | self::expectException(AvroEncoderException::class); 52 | self::expectExceptionMessage( 53 | sprintf( 54 | AvroEncoderException::UNABLE_TO_LOAD_DEFINITION_MESSAGE, 55 | $avroSchema->getName() 56 | ) 57 | ); 58 | 59 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 60 | 61 | $encoder = new AvroEncoder($registry, $recordSerializer); 62 | $encoder->encode($producerMessage); 63 | } 64 | 65 | public function testEncodeSuccessWithSchema() 66 | { 67 | $schemaDefinition = $this->getMockBuilder(\AvroSchema::class)->disableOriginalConstructor()->getMock(); 68 | 69 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 70 | $avroSchema->expects(self::exactly(4))->method('getName')->willReturn('schemaName'); 71 | $avroSchema->expects(self::never())->method('getVersion'); 72 | $avroSchema->expects(self::exactly(4))->method('getDefinition')->willReturn($schemaDefinition); 73 | 74 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 75 | $registry->expects(self::once())->method('getBodySchemaForTopic')->willReturn($avroSchema); 76 | $registry->expects(self::once())->method('getKeySchemaForTopic')->willReturn($avroSchema); 77 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(true); 78 | $registry->expects(self::once())->method('hasKeySchemaForTopic')->willReturn(true); 79 | 80 | $producerMessage = $this->getMockForAbstractClass(KafkaProducerMessageInterface::class); 81 | $producerMessage->expects(self::exactly(2))->method('getTopicName')->willReturn('test'); 82 | $producerMessage->expects(self::once())->method('getBody')->willReturn([]); 83 | $producerMessage->expects(self::once())->method('getKey')->willReturn('test-key'); 84 | $producerMessage->expects(self::once())->method('withBody')->with('encodedValue')->willReturn($producerMessage); 85 | $producerMessage->expects(self::once())->method('withKey')->with('encodedKey')->willReturn($producerMessage); 86 | 87 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 88 | $recordSerializer 89 | ->expects(self::exactly(2)) 90 | ->method('encodeRecord') 91 | ->withConsecutive( 92 | [$avroSchema->getName(), $avroSchema->getDefinition(), []], 93 | [$avroSchema->getName(), $avroSchema->getDefinition(), 'test-key'] 94 | )->willReturnOnConsecutiveCalls('encodedValue', 'encodedKey'); 95 | 96 | $encoder = new AvroEncoder($registry, $recordSerializer); 97 | 98 | self::assertSame($producerMessage, $encoder->encode($producerMessage)); 99 | } 100 | 101 | public function testEncodeKeyMode() 102 | { 103 | $schemaDefinition = $this->getMockBuilder(\AvroSchema::class)->disableOriginalConstructor()->getMock(); 104 | 105 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 106 | $avroSchema->expects(self::exactly(2))->method('getName')->willReturn('schemaName'); 107 | $avroSchema->expects(self::never())->method('getVersion'); 108 | $avroSchema->expects(self::exactly(2))->method('getDefinition')->willReturn($schemaDefinition); 109 | 110 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 111 | $registry->expects(self::never())->method('getBodySchemaForTopic'); 112 | $registry->expects(self::once())->method('getKeySchemaForTopic')->willReturn($avroSchema); 113 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(false); 114 | $registry->expects(self::once())->method('hasKeySchemaForTopic')->willReturn(true); 115 | 116 | $producerMessage = $this->getMockForAbstractClass(KafkaProducerMessageInterface::class); 117 | $producerMessage->expects(self::exactly(2))->method('getTopicName')->willReturn('test'); 118 | $producerMessage->expects(self::once())->method('getBody')->willReturn([]); 119 | $producerMessage->expects(self::once())->method('getKey')->willReturn('test-key'); 120 | $producerMessage->expects(self::never())->method('withBody'); 121 | $producerMessage->expects(self::once())->method('withKey')->with('encodedKey')->willReturn($producerMessage); 122 | 123 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 124 | $recordSerializer->expects(self::once())->method('encodeRecord')->with($avroSchema->getName(), $avroSchema->getDefinition(), 'test-key')->willReturn('encodedKey'); 125 | 126 | $encoder = new AvroEncoder($registry, $recordSerializer); 127 | 128 | self::assertSame($producerMessage, $encoder->encode($producerMessage)); 129 | } 130 | 131 | public function testEncodeBodyMode() 132 | { 133 | $schemaDefinition = $this->getMockBuilder(\AvroSchema::class)->disableOriginalConstructor()->getMock(); 134 | 135 | $avroSchema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 136 | $avroSchema->expects(self::exactly(2))->method('getName')->willReturn('schemaName'); 137 | $avroSchema->expects(self::never())->method('getVersion'); 138 | $avroSchema->expects(self::exactly(2))->method('getDefinition')->willReturn($schemaDefinition); 139 | 140 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 141 | $registry->expects(self::once())->method('getBodySchemaForTopic')->willReturn($avroSchema); 142 | $registry->expects(self::never())->method('getKeySchemaForTopic'); 143 | $registry->expects(self::once())->method('hasBodySchemaForTopic')->willReturn(true); 144 | $registry->expects(self::once())->method('hasKeySchemaForTopic')->willReturn(false); 145 | 146 | $producerMessage = $this->getMockForAbstractClass(KafkaProducerMessageInterface::class); 147 | $producerMessage->expects(self::exactly(2))->method('getTopicName')->willReturn('test'); 148 | $producerMessage->expects(self::once())->method('getBody')->willReturn([]); 149 | $producerMessage->expects(self::once())->method('getKey')->willReturn('test-key'); 150 | $producerMessage->expects(self::once())->method('withBody')->with('encodedBody')->willReturn($producerMessage); 151 | $producerMessage->expects(self::never())->method('withKey'); 152 | 153 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 154 | $recordSerializer->expects(self::once())->method('encodeRecord')->with($avroSchema->getName(), $avroSchema->getDefinition(), [])->willReturn('encodedBody'); 155 | 156 | $encoder = new AvroEncoder($registry, $recordSerializer); 157 | 158 | self::assertSame($producerMessage, $encoder->encode($producerMessage)); 159 | } 160 | 161 | public function testGetRegistry() 162 | { 163 | $registry = $this->getMockForAbstractClass(AvroSchemaRegistryInterface::class); 164 | 165 | $recordSerializer = $this->getMockBuilder(RecordSerializer::class)->disableOriginalConstructor()->getMock(); 166 | $encoder = new AvroEncoder($registry, $recordSerializer); 167 | 168 | self::assertSame($registry, $encoder->getRegistry()); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /tests/Unit/Message/Encoder/JsonEncoderTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaProducerMessageInterface::class); 22 | $message->expects(self::once())->method('getBody')->willReturn(['name' => 'foo']); 23 | $message->expects(self::once())->method('withBody')->with('{"name":"foo"}')->willReturn($message); 24 | 25 | $encoder = $this->getMockForAbstractClass(JsonEncoder::class); 26 | 27 | self::assertSame($message, $encoder->encode($message)); 28 | } 29 | 30 | /** 31 | * @return void 32 | */ 33 | public function testEncodeThrowsException(): void 34 | { 35 | $message = $this->getMockForAbstractClass(KafkaProducerMessageInterface::class); 36 | $message->expects(self::once())->method('getBody')->willReturn(chr(255)); 37 | 38 | $encoder = $this->getMockForAbstractClass(JsonEncoder::class); 39 | 40 | self::expectException(\JsonException::class); 41 | 42 | $encoder->encode($message); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/Unit/Message/Encoder/NullEncoderTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(KafkaProducerMessageInterface::class); 23 | 24 | $this->assertSame($message, (new NullEncoder())->encode($message)); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/Unit/Message/KafkaAvroSchemaTest.php: -------------------------------------------------------------------------------- 1 | getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 20 | 21 | $schemaName = 'testSchema'; 22 | $version = 9; 23 | 24 | $avroSchema = new KafkaAvroSchema($schemaName, $version, $definition); 25 | 26 | self::assertEquals($schemaName, $avroSchema->getName()); 27 | self::assertEquals($version, $avroSchema->getVersion()); 28 | self::assertEquals($definition, $avroSchema->getDefinition()); 29 | } 30 | 31 | public function testSetters() 32 | { 33 | $definition = $this->getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 34 | 35 | $schemaName = 'testSchema'; 36 | 37 | $avroSchema = new KafkaAvroSchema($schemaName); 38 | 39 | $avroSchema->setDefinition($definition); 40 | 41 | self::assertEquals($definition, $avroSchema->getDefinition()); 42 | } 43 | 44 | public function testAvroSchemaWithJustName() 45 | { 46 | $schemaName = 'testSchema'; 47 | 48 | $avroSchema = new KafkaAvroSchema($schemaName); 49 | 50 | self::assertEquals($schemaName, $avroSchema->getName()); 51 | self::assertEquals(KafkaAvroSchemaInterface::LATEST_VERSION, $avroSchema->getVersion()); 52 | self::assertNull($avroSchema->getDefinition()); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /tests/Unit/Message/KafkaConsumerMessageTest.php: -------------------------------------------------------------------------------- 1 | 'value' ]; 25 | 26 | $message = new KafkaConsumerMessage( 27 | $topic, 28 | $partition, 29 | $offset, 30 | $timestamp, 31 | $key, 32 | $body, 33 | $headers 34 | ); 35 | 36 | self::assertEquals($key, $message->getKey()); 37 | self::assertEquals($body, $message->getBody()); 38 | self::assertEquals($topic, $message->getTopicName()); 39 | self::assertEquals($offset, $message->getOffset()); 40 | self::assertEquals($partition, $message->getPartition()); 41 | self::assertEquals($timestamp, $message->getTimestamp()); 42 | self::assertEquals($headers, $message->getHeaders()); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/Unit/Message/KafkaProducerMessageTest.php: -------------------------------------------------------------------------------- 1 | 'value' ]; 23 | $expectedHeader = [ 24 | 'key' => 'value', 25 | 'anotherKey' => 1 26 | ]; 27 | 28 | $message = KafkaProducerMessage::create($topic, $partition) 29 | ->withKey($key) 30 | ->withBody($body) 31 | ->withHeaders($headers) 32 | ->withHeader('anotherKey', 1); 33 | 34 | self::assertEquals($key, $message->getKey()); 35 | self::assertEquals($body, $message->getBody()); 36 | self::assertEquals($topic, $message->getTopicName()); 37 | self::assertEquals($partition, $message->getPartition()); 38 | self::assertEquals($expectedHeader, $message->getHeaders()); 39 | } 40 | 41 | public function testClone() 42 | { 43 | $key = '1234-1234-1234'; 44 | $body = 'foo bar baz'; 45 | $topic = 'test'; 46 | $partition = 1; 47 | $headers = [ 'key' => 'value' ]; 48 | 49 | 50 | $origMessage = KafkaProducerMessage::create($topic, $partition); 51 | 52 | $message = $origMessage->withKey($key); 53 | self::assertNotSame($origMessage, $message); 54 | 55 | $message = $origMessage->withBody($body); 56 | self::assertNotSame($origMessage, $message); 57 | 58 | $message = $origMessage->withHeaders($headers); 59 | self::assertNotSame($origMessage, $message); 60 | 61 | $message = $origMessage->withHeader('anotherKey', 1); 62 | self::assertNotSame($origMessage, $message); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /tests/Unit/Message/Registry/AvroSchemaRegistryTest.php: -------------------------------------------------------------------------------- 1 | getMockForAbstractClass(Registry::class); 24 | 25 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 26 | 27 | $registry = new AvroSchemaRegistry($flixRegistry); 28 | 29 | $registry->addBodySchemaMappingForTopic('test', $schema); 30 | 31 | $reflectionProperty = new \ReflectionProperty($registry, 'schemaMapping'); 32 | $reflectionProperty->setAccessible(true); 33 | 34 | $schemaMapping = $reflectionProperty->getValue($registry); 35 | 36 | self::assertArrayHasKey(AvroSchemaRegistryInterface::BODY_IDX, $schemaMapping); 37 | self::assertArrayHasKey('test', $schemaMapping[AvroSchemaRegistryInterface::BODY_IDX]); 38 | self::assertSame($schema, $schemaMapping[AvroSchemaRegistryInterface::BODY_IDX]['test']); 39 | } 40 | 41 | public function testAddKeySchemaMappingForTopic() 42 | { 43 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 44 | 45 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 46 | 47 | $registry = new AvroSchemaRegistry($flixRegistry); 48 | 49 | $registry->addKeySchemaMappingForTopic('test2', $schema); 50 | 51 | $reflectionProperty = new \ReflectionProperty($registry, 'schemaMapping'); 52 | $reflectionProperty->setAccessible(true); 53 | 54 | $schemaMapping = $reflectionProperty->getValue($registry); 55 | 56 | self::assertArrayHasKey(AvroSchemaRegistryInterface::KEY_IDX, $schemaMapping); 57 | self::assertArrayHasKey('test2', $schemaMapping[AvroSchemaRegistryInterface::KEY_IDX]); 58 | self::assertSame($schema, $schemaMapping[AvroSchemaRegistryInterface::KEY_IDX]['test2']); 59 | } 60 | 61 | public function testHasBodySchemaMappingForTopic() 62 | { 63 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 64 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 65 | 66 | $registry = new AvroSchemaRegistry($flixRegistry); 67 | $registry->addBodySchemaMappingForTopic('test', $schema); 68 | 69 | self::assertTrue($registry->hasBodySchemaForTopic('test')); 70 | self::assertFalse($registry->hasBodySchemaForTopic('test2')); 71 | } 72 | 73 | public function testHasKeySchemaMappingForTopic() 74 | { 75 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 76 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 77 | 78 | $registry = new AvroSchemaRegistry($flixRegistry); 79 | $registry->addKeySchemaMappingForTopic('test', $schema); 80 | 81 | self::assertTrue($registry->hasKeySchemaForTopic('test')); 82 | self::assertFalse($registry->hasKeySchemaForTopic('test2')); 83 | } 84 | 85 | public function testGetBodySchemaForTopicWithNoMapping() 86 | { 87 | self::expectException(AvroSchemaRegistryException::class); 88 | self::expectExceptionMessage( 89 | sprintf( 90 | AvroSchemaRegistryException::SCHEMA_MAPPING_NOT_FOUND, 91 | 'test', 92 | AvroSchemaRegistryInterface::BODY_IDX 93 | ) 94 | ); 95 | 96 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 97 | 98 | $registry = new AvroSchemaRegistry($flixRegistry); 99 | 100 | $registry->getBodySchemaForTopic('test'); 101 | } 102 | 103 | public function testGetBodySchemaForTopicWithMappingWithDefinition() 104 | { 105 | $definition = $this->getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 106 | 107 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 108 | 109 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 110 | $schema->expects(self::once())->method('getDefinition')->willReturn($definition); 111 | 112 | $registry = new AvroSchemaRegistry($flixRegistry); 113 | 114 | $registry->addBodySchemaMappingForTopic('test', $schema); 115 | 116 | self::assertSame($schema, $registry->getBodySchemaForTopic('test')); 117 | } 118 | 119 | public function testGetKeySchemaForTopicWithMappingWithDefinition() 120 | { 121 | $definition = $this->getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 122 | 123 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 124 | 125 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 126 | $schema->expects(self::once())->method('getDefinition')->willReturn($definition); 127 | 128 | $registry = new AvroSchemaRegistry($flixRegistry); 129 | 130 | $registry->addKeySchemaMappingForTopic('test2', $schema); 131 | 132 | self::assertSame($schema, $registry->getKeySchemaForTopic('test2')); 133 | } 134 | 135 | public function testGetBodySchemaForTopicWithMappingWithoutDefinitionLatest() 136 | { 137 | $definition = $this->getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 138 | 139 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 140 | $flixRegistry->expects(self::once())->method('latestVersion')->with('test-schema')->willReturn($definition); 141 | 142 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 143 | $schema->expects(self::once())->method('getDefinition')->willReturn(null); 144 | $schema->expects(self::once())->method('getVersion')->willReturn(KafkaAvroSchemaInterface::LATEST_VERSION); 145 | $schema->expects(self::once())->method('getName')->willReturn('test-schema'); 146 | $schema->expects(self::once())->method('setDefinition')->with($definition); 147 | 148 | $registry = new AvroSchemaRegistry($flixRegistry); 149 | 150 | $registry->addBodySchemaMappingForTopic('test', $schema); 151 | 152 | $registry->getBodySchemaForTopic('test'); 153 | } 154 | 155 | public function testGetBodySchemaForTopicWithMappingWithoutDefinitionVersion() 156 | { 157 | $definition = $this->getMockBuilder(AvroSchema::class)->disableOriginalConstructor()->getMock(); 158 | 159 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 160 | $flixRegistry->expects(self::once())->method('schemaForSubjectAndVersion')->with('test-schema', 1)->willReturn($definition); 161 | 162 | $schema = $this->getMockForAbstractClass(KafkaAvroSchemaInterface::class); 163 | $schema->expects(self::once())->method('getDefinition')->willReturn(null); 164 | $schema->expects(self::exactly(2))->method('getVersion')->willReturn(1); 165 | $schema->expects(self::once())->method('getName')->willReturn('test-schema'); 166 | $schema->expects(self::once())->method('setDefinition')->with($definition); 167 | 168 | $registry = new AvroSchemaRegistry($flixRegistry); 169 | 170 | $registry->addBodySchemaMappingForTopic('test', $schema); 171 | 172 | $registry->getBodySchemaForTopic('test'); 173 | } 174 | 175 | public function testGetTopicSchemaMapping() 176 | { 177 | $flixRegistry = $this->getMockForAbstractClass(Registry::class); 178 | 179 | $registry = new AvroSchemaRegistry($flixRegistry); 180 | 181 | self::assertIsArray($registry->getTopicSchemaMapping()); 182 | } 183 | } 184 | -------------------------------------------------------------------------------- /tests/Unit/Producer/KafkaProducerBuilderTest.php: -------------------------------------------------------------------------------- 1 | kafkaProducerBuilder = KafkaProducerBuilder::create(); 26 | } 27 | 28 | /** 29 | * @return void 30 | * @throws \ReflectionException 31 | */ 32 | public function testAddConfig(): void 33 | { 34 | $config = ['auto.offset.reset' => 'earliest']; 35 | $clone = $this->kafkaProducerBuilder->withAdditionalConfig($config); 36 | $config = ['auto.offset.reset' => 'latest']; 37 | $clone = $clone->withAdditionalConfig($config); 38 | 39 | $reflectionProperty = new \ReflectionProperty($clone, 'config'); 40 | $reflectionProperty->setAccessible(true); 41 | 42 | self::assertSame($config, $reflectionProperty->getValue($clone)); 43 | } 44 | 45 | /** 46 | * @return void 47 | * @throws \ReflectionException 48 | */ 49 | public function testAddBroker(): void 50 | { 51 | $clone = $this->kafkaProducerBuilder->withAdditionalBroker('localhost'); 52 | 53 | $reflectionProperty = new \ReflectionProperty($clone, 'brokers'); 54 | $reflectionProperty->setAccessible(true); 55 | 56 | self::assertSame(['localhost'], $reflectionProperty->getValue($clone)); 57 | } 58 | 59 | /** 60 | * @return void 61 | * @throws \ReflectionException 62 | */ 63 | public function testSetEncoder(): void 64 | { 65 | $encoder = $this->getMockForAbstractClass(EncoderInterface::class); 66 | 67 | $clone = $this->kafkaProducerBuilder->withEncoder($encoder); 68 | 69 | $reflectionProperty = new \ReflectionProperty($clone, 'encoder'); 70 | $reflectionProperty->setAccessible(true); 71 | 72 | self::assertInstanceOf(EncoderInterface::class, $reflectionProperty->getValue($clone)); 73 | } 74 | 75 | /** 76 | * @return void 77 | * @throws \ReflectionException 78 | */ 79 | public function testSetDeliveryReportCallback(): void 80 | { 81 | $callback = function () { 82 | // Anonymous test method, no logic required 83 | }; 84 | 85 | $clone = $this->kafkaProducerBuilder->withDeliveryReportCallback($callback); 86 | 87 | $reflectionProperty = new \ReflectionProperty($clone, 'deliverReportCallback'); 88 | $reflectionProperty->setAccessible(true); 89 | 90 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 91 | } 92 | 93 | /** 94 | * @return void 95 | * @throws \ReflectionException 96 | */ 97 | public function testSetErrorCallback(): void 98 | { 99 | $callback = function () { 100 | // Anonymous test method, no logic required 101 | }; 102 | 103 | $clone = $this->kafkaProducerBuilder->withErrorCallback($callback); 104 | 105 | $reflectionProperty = new \ReflectionProperty($clone, 'errorCallback'); 106 | $reflectionProperty->setAccessible(true); 107 | 108 | self::assertSame($callback, $reflectionProperty->getValue($clone)); 109 | } 110 | 111 | /** 112 | * @throws KafkaProducerException 113 | */ 114 | public function testBuildNoBroker(): void 115 | { 116 | self::expectException(KafkaProducerException::class); 117 | 118 | $this->kafkaProducerBuilder->build(); 119 | } 120 | 121 | /** 122 | * @return void 123 | */ 124 | public function testBuild(): void 125 | { 126 | $callback = function ($kafka, $errId, $msg) { 127 | // Anonymous test method, no logic required 128 | }; 129 | 130 | $producer = $this->kafkaProducerBuilder 131 | ->withAdditionalBroker('localhost') 132 | ->withDeliveryReportCallback($callback) 133 | ->withErrorCallback($callback) 134 | ->withLogCallback($callback) 135 | ->build(); 136 | 137 | self::assertInstanceOf(KafkaProducerInterface::class, $producer); 138 | } 139 | 140 | /** 141 | * @return void 142 | * @throws \ReflectionException 143 | */ 144 | public function testKafkaProducerBuilderConfig(): void 145 | { 146 | $callback = function ($kafka, $errId, $msg) { 147 | // Anonymous test method, no logic required 148 | }; 149 | 150 | $producer = $this->kafkaProducerBuilder 151 | ->withAdditionalBroker('localhost') 152 | ->withDeliveryReportCallback($callback) 153 | ->withErrorCallback($callback) 154 | ->withLogCallback($callback) 155 | ->build(); 156 | 157 | $reflectionProperty = new \ReflectionProperty($this->kafkaProducerBuilder, 'config'); 158 | $reflectionProperty->setAccessible(true); 159 | 160 | self::assertSame( 161 | [ 162 | 'socket.timeout.ms' => '50', 163 | 'internal.termination.signal' => (string) SIGIO 164 | ], 165 | $reflectionProperty->getValue($this->kafkaProducerBuilder) 166 | ); 167 | 168 | self::assertInstanceOf(KafkaProducerInterface::class, $producer); 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /tests/bootstrap.php: -------------------------------------------------------------------------------- 1 | setPsr4('Jobcloud\\Messaging\\Tests\\', __DIR__); 6 | --------------------------------------------------------------------------------