├── src ├── Client │ ├── Contract │ │ ├── ClientInterface.php │ │ ├── CallableInterface.php │ │ ├── PartitionAwareProducerInterface.php │ │ ├── ProducerInterface.php │ │ └── ConsumerInterface.php │ ├── Consumer │ │ ├── Exception │ │ │ ├── NullMessageException.php │ │ │ ├── InvalidConsumerException.php │ │ │ └── RecoverableMessageException.php │ │ ├── ConsumerProvider.php │ │ ├── Message.php │ │ ├── Factory │ │ │ └── MessageFactory.php │ │ └── ConsumerClient.php │ ├── Producer │ │ ├── Exception │ │ │ └── InvalidProducerException.php │ │ ├── Message.php │ │ ├── ProducerProvider.php │ │ └── ProducerClient.php │ └── Traits │ │ └── CommitOffsetTrait.php ├── Configuration │ ├── Exception │ │ ├── InvalidClientException.php │ │ ├── InvalidConfigurationType.php │ │ └── InvalidConfigurationException.php │ ├── Contract │ │ ├── ConsumerConfigurationInterface.php │ │ ├── ProducerConfigurationInterface.php │ │ ├── KafkaConfigurationInterface.php │ │ ├── CastValueInterface.php │ │ └── ConfigurationInterface.php │ ├── Traits │ │ ├── SupportsConsumerTrait.php │ │ ├── SupportsProducerTrait.php │ │ ├── BooleanConfigurationTrait.php │ │ └── ObjectConfigurationTrait.php │ ├── Type │ │ ├── ProducerTopic.php │ │ ├── RetryMultiplier.php │ │ ├── RetryDelay.php │ │ ├── SchemaRegistry.php │ │ ├── RegisterMissingSchemas.php │ │ ├── RegisterMissingSubjects.php │ │ ├── MaxRetryDelay.php │ │ ├── Topics.php │ │ ├── EnableAutoOffsetStore.php │ │ ├── GroupId.php │ │ ├── ProducerPartition.php │ │ ├── Timeout.php │ │ ├── MaxRetries.php │ │ ├── EnableAutoCommit.php │ │ ├── AutoCommitIntervalMs.php │ │ ├── Validators.php │ │ ├── Denormalizer.php │ │ ├── Decoder.php │ │ ├── StatisticsIntervalMs.php │ │ ├── AutoOffsetReset.php │ │ ├── LogLevel.php │ │ ├── Brokers.php │ │ └── MaxPollIntervalMs.php │ ├── RawConfiguration.php │ ├── ResolvedConfiguration.php │ └── ConfigurationResolver.php ├── Denormalizer │ ├── Contract │ │ └── DenormalizerInterface.php │ └── PlainDenormalizer.php ├── StsGamingGroupKafkaBundle.php ├── Traits │ ├── CheckForRdKafkaExtensionTrait.php │ └── AddConfigurationsToCommandTrait.php ├── Event │ ├── PreMessageConsumedEvent.php │ ├── PostMessageConsumedEvent.php │ └── AbstractMessageConsumedEvent.php ├── Decoder │ ├── PlainDecoder.php │ ├── Contract │ │ └── DecoderInterface.php │ ├── JsonDecoder.php │ └── AvroDecoder.php ├── Validator │ ├── Contract │ │ └── ValidatorInterface.php │ ├── Type │ │ └── PlainValidator.php │ ├── Exception │ │ └── ValidationException.php │ └── Validator.php ├── Resources │ └── config │ │ ├── denormalizers.xml │ │ ├── validators.xml │ │ ├── rd_kafka.xml │ │ ├── configurations.xml │ │ ├── producers.xml │ │ ├── decoders.xml │ │ ├── commands.xml │ │ ├── consumers.xml │ │ └── configuration_types.xml ├── RdKafka │ ├── Callbacks.php │ ├── Context.php │ └── Factory │ │ └── KafkaConfigurationFactory.php ├── Command │ ├── ProducersDescribeCommand.php │ ├── ConsumersDescribeCommand.php │ ├── Traits │ │ └── DescribeTrait.php │ └── ConsumeCommand.php └── DependencyInjection │ ├── StsGamingGroupKafkaExtension.php │ └── Configuration.php ├── .gitignore ├── tests ├── Dummy │ ├── Denormalizer │ │ └── DummyDenormalizerOne.php │ ├── Decoder │ │ └── DummyDecoderOne.php │ ├── Client │ │ ├── Producer │ │ │ ├── DummyProducerOne.php │ │ │ └── DummyProducerTwo.php │ │ └── Consumer │ │ │ ├── DummyConsumerThree.php │ │ │ ├── DummyConsumerOne.php │ │ │ ├── DummyConsumerTwo.php │ │ │ └── DummyConsumerOneClone.php │ └── Validator │ │ └── DummyValidatorOne.php ├── Unit │ ├── Configuration │ │ ├── Type │ │ │ ├── EnableAutoCommitTest.php │ │ │ ├── AbstractBooleanConfigurationTest.php │ │ │ ├── EnableAutoOffsetStoreTest.php │ │ │ ├── RegisterMissingSchemasTest.php │ │ │ ├── RegisterMissingSubjectsTest.php │ │ │ ├── AbstractObjectConfigurationTest.php │ │ │ ├── DecoderTest.php │ │ │ ├── ValidatorsTest.php │ │ │ ├── DenormalizerTest.php │ │ │ ├── GroupIdTest.php │ │ │ ├── LogLevelTest.php │ │ │ ├── TimeoutTest.php │ │ │ ├── RetryDelayTest.php │ │ │ ├── MaxRetriesTest.php │ │ │ ├── ProducerTopicTest.php │ │ │ ├── MaxRetryDelayTest.php │ │ │ ├── SchemaRegistryTest.php │ │ │ ├── RetryMultiplierTest.php │ │ │ ├── ProducerPartitionTest.php │ │ │ ├── BrokersTest.php │ │ │ ├── TopicsTest.php │ │ │ ├── AutoCommitIntervalMsTest.php │ │ │ ├── AutoOffsetResetTest.php │ │ │ ├── MaxPollIntervalMsTest.php │ │ │ ├── StatisticsIntervalMsTest.php │ │ │ └── AbstractConfigurationTest.php │ │ ├── RawConfigurationTest.php │ │ ├── ResolvedConfigurationTest.php │ │ └── ConfigurationResolverTest.php │ ├── Decoder │ │ ├── PlainDecoderTest.php │ │ └── JsonDecoderTest.php │ └── Client │ │ └── Consumer │ │ ├── ConsumerClientTest.php │ │ └── ConsumerProviderTest.php ├── AppKernel.php ├── config │ ├── producers.json │ ├── consumers.json │ └── base.xml └── Functional │ ├── Command │ ├── ProducersDescribeCommandTest.php │ └── ConsumersDescribeCommandTest.php │ └── Client │ └── Consumer │ └── ConsumerProviderTest.php ├── LICENSE.md ├── phpunit.xml.dist ├── composer.json └── phpstan.neon.dist /src/Client/Contract/ClientInterface.php: -------------------------------------------------------------------------------- 1 | implementsInterface(ConsumerInterface::class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /src/Configuration/Traits/SupportsProducerTrait.php: -------------------------------------------------------------------------------- 1 | implementsInterface(ProducerInterface::class); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /tests/Unit/Configuration/Type/RegisterMissingSchemasTest.php: -------------------------------------------------------------------------------- 1 | payload = $payload; 15 | $this->key = $key; 16 | } 17 | 18 | public function getPayload(): string 19 | { 20 | return $this->payload; 21 | } 22 | 23 | public function getKey(): ?string 24 | { 25 | return $this->key; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /tests/Unit/Configuration/Type/AbstractObjectConfigurationTest.php: -------------------------------------------------------------------------------- 1 | getObject(), get_class($this->getObject())]; 14 | } 15 | 16 | protected function getInvalidValues(): array 17 | { 18 | return [1, 1.51, '1', [], null, new \stdClass(), false, true]; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Resources/config/denormalizers.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /src/Validator/Type/PlainValidator.php: -------------------------------------------------------------------------------- 1 | load(__DIR__ . '/config/base.xml'); 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /tests/Dummy/Client/Consumer/DummyConsumerThree.php: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/Unit/Configuration/Type/StatisticsIntervalMsTest.php: -------------------------------------------------------------------------------- 1 | consumedMessages = $consumedMessages; 17 | $this->consumptionTimeMs = $consumptionTimeMs; 18 | } 19 | 20 | public function getConsumedMessages(): int 21 | { 22 | return $this->consumedMessages; 23 | } 24 | 25 | public function getConsumptionTimeMs(): float 26 | { 27 | return $this->consumptionTimeMs; 28 | } 29 | 30 | abstract public static function getEventName(string $consumerName): string; 31 | } 32 | -------------------------------------------------------------------------------- /src/Resources/config/rd_kafka.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 10 | 11 | 12 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /tests/Unit/Decoder/PlainDecoderTest.php: -------------------------------------------------------------------------------- 1 | resolvedConfiguration = $this->createMock(ResolvedConfiguration::class); 21 | $this->decoder = new PlainDecoder(); 22 | } 23 | 24 | public function testTheSameMessage(): void 25 | { 26 | $message = 'abc'; 27 | 28 | $result = $this->decoder->decode($this->resolvedConfiguration, $message); 29 | 30 | $this->assertEquals('abc', $result); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Traits/AddConfigurationsToCommandTrait.php: -------------------------------------------------------------------------------- 1 | getConfigurations() as $configuration) { 23 | $this->addOption( 24 | $configuration->getName(), 25 | null, 26 | $configuration->getMode(), 27 | $configuration->getDescription() 28 | ); 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | Copyright (c) 2021 STS Gaming Group 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /src/Resources/config/configurations.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /src/Configuration/Type/ProducerTopic.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 32 | ); 33 | } 34 | 35 | public function isValueValid($value): bool 36 | { 37 | return is_int($value) && $value > 0; 38 | } 39 | 40 | public function getDefaultValue(): int 41 | { 42 | return 2; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Configuration/Type/RetryDelay.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 32 | ); 33 | } 34 | 35 | public function isValueValid($value): bool 36 | { 37 | return is_int($value) && $value >= 0; 38 | } 39 | 40 | public function getDefaultValue(): int 41 | { 42 | return 200; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Configuration/Type/SchemaRegistry.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 32 | ); 33 | } 34 | 35 | public function isValueValid($value): bool 36 | { 37 | return is_string($value) && '' !== $value; 38 | } 39 | 40 | public function getDefaultValue(): string 41 | { 42 | return '127.0.0.1'; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Client/Traits/CommitOffsetTrait.php: -------------------------------------------------------------------------------- 1 | getValue(EnableAutoCommit::NAME) === 'true') { 23 | throw new InvalidConfigurationException(sprintf( 24 | 'Unable to manually commit offset when %s configuration is set to `true`.', 25 | EnableAutoCommit::NAME 26 | )); 27 | } 28 | 29 | if ($async) { 30 | $context->getRdKafkaConsumer()->commitAsync($context->getRdKafkaMessage()); 31 | } else { 32 | $context->getRdKafkaConsumer()->commit($context->getRdKafkaMessage()); 33 | } 34 | 35 | return true; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /tests/Unit/Decoder/JsonDecoderTest.php: -------------------------------------------------------------------------------- 1 | resolvedConfiguration = $this->createMock(ResolvedConfiguration::class); 21 | $this->decoder = new JsonDecoder(); 22 | } 23 | 24 | public function testDecoded(): void 25 | { 26 | $message = '{"status": "ok"}'; 27 | 28 | $result = $this->decoder->decode($this->resolvedConfiguration, $message); 29 | 30 | $this->assertEquals(['status' => 'ok'], $result); 31 | } 32 | 33 | public function testInvalidMessage(): void 34 | { 35 | $message = '{"status: "ok"}'; 36 | 37 | $this->expectException(\JsonException::class); 38 | 39 | $this->decoder->decode($this->resolvedConfiguration, $message); 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Configuration/Type/RegisterMissingSchemas.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 32 | ); 33 | } 34 | 35 | public function getDefaultValue(): string 36 | { 37 | return 'false'; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Configuration/Type/RegisterMissingSubjects.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 32 | ); 33 | } 34 | 35 | public function getDefaultValue(): string 36 | { 37 | return 'false'; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /tests/Functional/Command/ProducersDescribeCommandTest.php: -------------------------------------------------------------------------------- 1 | find('kafka:producers:describe'); 23 | $this->commandTester = new CommandTester($command); 24 | } 25 | 26 | public function testConfigurationDisplayed(): void 27 | { 28 | $this->commandTester->execute([]); 29 | 30 | $output = $this->commandTester->getDisplay(); 31 | 32 | $this->assertStringContainsString(DummyProducerOne::class, $output); 33 | $this->assertStringContainsString(DummyProducerTwo::class, $output); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/Configuration/Traits/ObjectConfigurationTrait.php: -------------------------------------------------------------------------------- 1 | getInterface(); 14 | 15 | if (!is_array($value)) { 16 | return $this->doValidate($interface, $value); 17 | } 18 | 19 | if (empty($value)) { 20 | return false; 21 | } 22 | 23 | foreach ($value as $item) { 24 | if (!$this->doValidate($interface, $item)) { 25 | return false; 26 | } 27 | } 28 | 29 | return true; 30 | } 31 | 32 | /** 33 | * @param string $interface 34 | * @param mixed $item 35 | * @return bool 36 | */ 37 | private function doValidate(string $interface, $item): bool 38 | { 39 | if (is_object($item)) { 40 | return in_array($interface, class_implements($item), true); 41 | } 42 | 43 | if (is_string($item)) { 44 | return class_exists($item) && in_array($interface, class_implements($item), true); 45 | } 46 | 47 | return false; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | ./tests 21 | 22 | 23 | ./tests/Unit 24 | 25 | 26 | ./tests/Functional 27 | 28 | 29 | 30 | 31 | 32 | ./src 33 | 34 | 35 | 36 | -------------------------------------------------------------------------------- /src/Resources/config/producers.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 10 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | -------------------------------------------------------------------------------- /src/Validator/Exception/ValidationException.php: -------------------------------------------------------------------------------- 1 | validator = $validator; 28 | $this->failedReason = $failedReason; 29 | $this->data = $data; 30 | 31 | parent::__construct($message); 32 | } 33 | 34 | public function getFailedValidator(): ValidatorInterface 35 | { 36 | return $this->validator; 37 | } 38 | 39 | public function getFailedReason(): string 40 | { 41 | return $this->failedReason; 42 | } 43 | 44 | /** 45 | * @return mixed 46 | */ 47 | public function getData() 48 | { 49 | return $this->data; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sts-gaming-group/kafka-bundle", 3 | "description": "Bundle to consume and produce messages from/to Apache Kafka.", 4 | "type": "symfony-bundle", 5 | "license": "MIT", 6 | "authors": [ 7 | { 8 | "name": "Mariusz Jeruzal", 9 | "email": "mariusz.jeruzal@sts.pl" 10 | } 11 | ], 12 | "require": { 13 | "php": ">=8.1.0", 14 | "ext-rdkafka": "*", 15 | "ext-json": "*", 16 | "symfony/config": "^6.4|^7.0", 17 | "symfony/dependency-injection": "^6.4|^7.0", 18 | "symfony/http-kernel": "^6.4|^7.0", 19 | "symfony/console": "^6.4|^7.0", 20 | "symfony/options-resolver": "^6.4|^7.0", 21 | "flix-tech/avro-serde-php": "^2.0|^3.0" 22 | }, 23 | "autoload": { 24 | "psr-4": { 25 | "StsGamingGroup\\KafkaBundle\\": "src/" 26 | } 27 | }, 28 | "autoload-dev": { 29 | "psr-4": { 30 | "StsGamingGroup\\KafkaBundle\\Tests\\": "tests/" 31 | } 32 | }, 33 | "require-dev": { 34 | "symfony/phpunit-bridge": "^6.4|^7.0", 35 | "symfony/dotenv": "^6.4|^7.0", 36 | "phpstan/phpstan": "^0.12.75|^1.12", 37 | "phpstan/phpstan-symfony": "^0.12.18|^1.4", 38 | "phpstan/phpstan-phpunit": "^0.12.17|^1.4", 39 | "squizlabs/php_codesniffer": "^3.5", 40 | "phpstan/extension-installer": "^1.1", 41 | "symfony/framework-bundle": "^6.4|^7.0", 42 | "symfony/event-dispatcher": "^6.4|^7.0" 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Resources/config/decoders.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /src/Configuration/Type/MaxRetryDelay.php: -------------------------------------------------------------------------------- 1 | getDefaultValue()); 31 | } 32 | 33 | public function isValueValid($value): bool 34 | { 35 | return is_numeric($value) && strpos((string) $value, '.') === false && $value >= 0; 36 | } 37 | 38 | public function getDefaultValue(): int 39 | { 40 | return 2000; 41 | } 42 | 43 | public function cast($validatedValue): int 44 | { 45 | return (int) $validatedValue; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/RdKafka/Context.php: -------------------------------------------------------------------------------- 1 | configuration = $configuration; 25 | $this->consumer = $consumer; 26 | $this->message = $message; 27 | $this->retryNo = $retryNo; 28 | } 29 | 30 | /** 31 | * @param string $name 32 | * @return mixed 33 | */ 34 | public function getValue(string $name) 35 | { 36 | return $this->configuration->getValue($name); 37 | } 38 | 39 | public function getRdKafkaConsumer(): RdKafkaConsumer 40 | { 41 | return $this->consumer; 42 | } 43 | 44 | public function getRdKafkaMessage(): RdKafkaMessage 45 | { 46 | return $this->message; 47 | } 48 | 49 | public function getRetryNo(): int 50 | { 51 | return $this->retryNo; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/Configuration/Type/Topics.php: -------------------------------------------------------------------------------- 1 | = 0) || 39 | $value === $this->getDefaultValue(); 40 | } 41 | 42 | public function getDefaultValue(): int 43 | { 44 | return defined('RD_KAFKA_PARTITION_UA') ? RD_KAFKA_PARTITION_UA : -1; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Configuration/Type/Timeout.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 33 | ); 34 | } 35 | 36 | public function isValueValid($value): bool 37 | { 38 | return is_numeric($value) && strpos((string) $value, '.') === false && $value >= 0; 39 | } 40 | 41 | public function cast($validatedValue): int 42 | { 43 | return (int) $validatedValue; 44 | } 45 | 46 | public function getDefaultValue(): int 47 | { 48 | return 1000; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Configuration/Type/MaxRetries.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 33 | ); 34 | } 35 | 36 | public function isValueValid($value): bool 37 | { 38 | return is_numeric($value) && strpos((string) $value, '.') === false && $value >= 0; 39 | } 40 | 41 | public function getDefaultValue(): int 42 | { 43 | return 0; 44 | } 45 | 46 | public function cast($validatedValue): int 47 | { 48 | return (int) $validatedValue; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Client/Producer/ProducerProvider.php: -------------------------------------------------------------------------------- 1 | 14 | */ 15 | protected array $producers = []; 16 | 17 | public function addProducer(ProducerInterface $producer): self 18 | { 19 | $this->producers[] = $producer; 20 | 21 | return $this; 22 | } 23 | 24 | /** 25 | * @param mixed $data 26 | * @return ProducerInterface 27 | */ 28 | public function provide($data): ProducerInterface 29 | { 30 | $producers = []; 31 | 32 | foreach ($this->producers as $producer) { 33 | if ($producer->supports($data)) { 34 | $producers[] = $producer; 35 | } 36 | } 37 | 38 | if (count($producers) > 1) { 39 | throw new InvalidProducerException('Multiple producers found'); 40 | } 41 | 42 | if (!$producers) { 43 | throw new InvalidProducerException('There is no matching producer.'); 44 | } 45 | 46 | return $producers[0]; 47 | } 48 | 49 | /** 50 | * @return array 51 | */ 52 | public function getProducers(): array 53 | { 54 | return $this->producers; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Configuration/Type/EnableAutoCommit.php: -------------------------------------------------------------------------------- 1 | 14 | */ 15 | protected array $consumers = []; 16 | 17 | public function addConsumer(ConsumerInterface $consumer): self 18 | { 19 | $this->consumers[] = $consumer; 20 | 21 | return $this; 22 | } 23 | 24 | /** 25 | * @param string $name 26 | * @return ConsumerInterface 27 | */ 28 | public function provide(string $name): ConsumerInterface 29 | { 30 | $consumers = []; 31 | 32 | foreach ($this->consumers as $consumer) { 33 | if ($consumer->getName() === $name) { 34 | $consumers[] = $consumer; 35 | } 36 | } 37 | 38 | if (count($consumers) > 1) { 39 | throw new InvalidConsumerException(sprintf('Multiple consumers found with name %s', $name)); 40 | } 41 | 42 | if (!$consumers) { 43 | throw new InvalidConsumerException(sprintf('There is no matching consumer with name %s.', $name)); 44 | } 45 | 46 | return $consumers[0]; 47 | } 48 | 49 | /** 50 | * @return array 51 | */ 52 | public function getAll(): array 53 | { 54 | return $this->consumers; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Command/ProducersDescribeCommand.php: -------------------------------------------------------------------------------- 1 | producerProvider = $producerProvider; 31 | $this->configurationResolver = $configurationResolver; 32 | 33 | parent::__construct(); 34 | } 35 | 36 | protected function execute(InputInterface $input, OutputInterface $output): int 37 | { 38 | $producers = $this->producerProvider->getProducers(); 39 | 40 | foreach ($producers as $producer) { 41 | $this->describe($this->configurationResolver->resolve($producer), $output, $producer); 42 | } 43 | 44 | return self::SUCCESS; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /src/Configuration/Type/AutoCommitIntervalMs.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 41 | ); 42 | } 43 | 44 | public function isValueValid($value): bool 45 | { 46 | return is_numeric($value) && is_string($value); 47 | } 48 | 49 | public function getDefaultValue(): string 50 | { 51 | return '50'; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /src/Configuration/Type/Validators.php: -------------------------------------------------------------------------------- 1 | kafkaConfigurationFactory = $this->createMock(KafkaConfigurationFactory::class); 28 | $this->messageFactory = $this->createMock(MessageFactory::class); 29 | $this->configurationResolver = $this->createMock(ConfigurationResolver::class); 30 | $this->eventDispatcher = $this->createMock(EventDispatcherInterface::class); 31 | 32 | $this->client = new ConsumerClient( 33 | $this->kafkaConfigurationFactory, 34 | $this->messageFactory, 35 | $this->configurationResolver, 36 | $this->eventDispatcher 37 | ); 38 | } 39 | // TODO: maybe some tests 40 | } 41 | -------------------------------------------------------------------------------- /src/Configuration/RawConfiguration.php: -------------------------------------------------------------------------------- 1 | 14 | */ 15 | private array $configurations = []; 16 | 17 | /** 18 | * @param ConfigurationInterface $configuration 19 | * @return $this 20 | */ 21 | public function addConfiguration(ConfigurationInterface $configuration): self 22 | { 23 | $this->validateConfiguration($configuration); 24 | $this->configurations[$configuration->getName()] = $configuration; 25 | 26 | return $this; 27 | } 28 | 29 | /** 30 | * @return array 31 | */ 32 | public function getConfigurations(): array 33 | { 34 | return $this->configurations; 35 | } 36 | 37 | public function getConfigurationByName(string $name): ConfigurationInterface 38 | { 39 | return $this->configurations[$name]; 40 | } 41 | 42 | private function validateConfiguration(ConfigurationInterface $configuration): void 43 | { 44 | if (array_key_exists($configuration->getName(), $this->configurations)) { 45 | throw new InvalidConfigurationException( 46 | sprintf( 47 | 'Configuration with name `%s` has already been registered', 48 | $configuration->getName() 49 | ) 50 | ); 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /tests/Functional/Command/ConsumersDescribeCommandTest.php: -------------------------------------------------------------------------------- 1 | find('kafka:consumers:describe'); 23 | $this->commandTester = new CommandTester($command); 24 | } 25 | 26 | public function testConfigurationDisplayed(): void 27 | { 28 | $this->commandTester->execute([]); 29 | 30 | $output = $this->commandTester->getDisplay(); 31 | 32 | $this->assertStringContainsString(DummyConsumerOne::class, $output); 33 | $this->assertStringContainsString(DummyConsumerTwo::class, $output); 34 | } 35 | 36 | public function testConfigurationDisplayedByName(): void 37 | { 38 | $this->commandTester->execute(['--name' => 'dummy_consumer_one']); 39 | 40 | $output = $this->commandTester->getDisplay(); 41 | 42 | $this->assertStringContainsString(DummyConsumerOne::class, $output); 43 | $this->assertStringNotContainsString(DummyConsumerTwo::class, $output); 44 | } 45 | } 46 | -------------------------------------------------------------------------------- /src/Configuration/Type/Denormalizer.php: -------------------------------------------------------------------------------- 1 | configurationResolver = $configurationResolver; 19 | } 20 | 21 | public function create(ClientInterface $client, ?InputInterface $input = null): Conf 22 | { 23 | $configuration = $this->configurationResolver->resolve($client, $input); 24 | $conf = new Conf(); 25 | 26 | foreach ($configuration->getConfigurations(ResolvedConfiguration::KAFKA_TYPES) as $kafkaConfiguration) { 27 | $resolvedValue = $kafkaConfiguration['resolvedValue']; 28 | $value = is_array($resolvedValue) ? implode(',', $resolvedValue) : $resolvedValue; 29 | $conf->set( 30 | $kafkaConfiguration['configuration']->getKafkaProperty(), 31 | $value 32 | ); 33 | } 34 | 35 | if ($client instanceof CallableInterface) { 36 | $callbacks = $client->callbacks(); 37 | foreach ($callbacks as $name => $callback) { 38 | $conf->{$name}($callback); 39 | } 40 | } 41 | 42 | return $conf; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Client/Consumer/Message.php: -------------------------------------------------------------------------------- 1 | topicName = $topicName; 36 | $this->partition = $partition; 37 | $this->payload = $payload; 38 | $this->offset = $offset; 39 | $this->data = $data; 40 | $this->key = $key; 41 | } 42 | 43 | public function getTopicName(): string 44 | { 45 | return $this->topicName; 46 | } 47 | 48 | public function getPartition(): int 49 | { 50 | return $this->partition; 51 | } 52 | 53 | public function getPayload(): string 54 | { 55 | return $this->payload; 56 | } 57 | 58 | public function getOffset(): int 59 | { 60 | return $this->offset; 61 | } 62 | 63 | /** 64 | * @return mixed 65 | */ 66 | public function getData() 67 | { 68 | return $this->data; 69 | } 70 | 71 | public function getKey(): ?string 72 | { 73 | return $this->key; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/Configuration/Type/Decoder.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 42 | ); 43 | } 44 | 45 | public function getDefaultValue(): string 46 | { 47 | return AvroDecoder::class; 48 | } 49 | 50 | protected function getInterface(): string 51 | { 52 | return DecoderInterface::class; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Configuration/Type/StatisticsIntervalMs.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 42 | ); 43 | } 44 | 45 | public function isValueValid($value): bool 46 | { 47 | return is_numeric($value) && strpos((string) $value, '.') === false && $value >= 0; 48 | } 49 | 50 | public function getDefaultValue(): int 51 | { 52 | return 0; 53 | } 54 | 55 | public function cast($validatedValue): int 56 | { 57 | return (int) $validatedValue; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/Configuration/Type/AutoOffsetReset.php: -------------------------------------------------------------------------------- 1 | getConfiguration(); 26 | 27 | $this->assertTrue($configuration->isValueValid($value)); 28 | } 29 | 30 | /** 31 | * @dataProvider getInvalidValuesProvider 32 | * @param mixed $value 33 | */ 34 | public function testInvalidValue($value): void 35 | { 36 | $configuration = $this->getConfiguration(); 37 | 38 | $this->assertFalse($configuration->isValueValid($value)); 39 | } 40 | 41 | public function getValidValuesProvider(): array 42 | { 43 | $values = $this->getValidValues(); 44 | $provided = []; 45 | 46 | foreach ($values as $value) { 47 | $provided[] = [$value]; 48 | } 49 | 50 | return $provided; 51 | } 52 | 53 | public function getInvalidValuesProvider(): array 54 | { 55 | $values = $this->getInvalidValues(); 56 | $provided = []; 57 | 58 | foreach ($values as $value) { 59 | $provided[] = [$value]; 60 | } 61 | 62 | return $provided; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Resources/config/commands.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 17 | 18 | 19 | 20 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /src/Decoder/AvroDecoder.php: -------------------------------------------------------------------------------- 1 | cachedRegistry) { 26 | $client = new Client( 27 | ['base_uri' => $configuration->getValue(SchemaRegistry::NAME)] 28 | ); 29 | $this->cachedRegistry = new CachedRegistry( 30 | new PromisingRegistry($client), 31 | new AvroObjectCacheAdapter() 32 | ); 33 | } 34 | 35 | if (!$this->recordSerializer) { 36 | $this->recordSerializer = new RecordSerializer( 37 | $this->cachedRegistry, 38 | [ 39 | $configuration->getValue(RegisterMissingSchemas::NAME), 40 | $configuration->getValue(RegisterMissingSubjects::NAME), 41 | ] 42 | ); 43 | } 44 | 45 | return $this->recordSerializer->decodeMessage($message); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /tests/Unit/Configuration/RawConfigurationTest.php: -------------------------------------------------------------------------------- 1 | configurationOne = $this->createMock(ConfigurationInterface::class); 21 | $this->configurationTwo = $this->createMock(ConfigurationInterface::class); 22 | } 23 | 24 | public function testAddConfiguration(): void 25 | { 26 | $this->configurationOne->method('getName') 27 | ->willReturn('configuration_1'); 28 | 29 | $this->configurationTwo->method('getName') 30 | ->willReturn('configuration_2'); 31 | 32 | $rawConfiguration = new RawConfiguration(); 33 | $rawConfiguration->addConfiguration($this->configurationOne) 34 | ->addConfiguration($this->configurationTwo); 35 | 36 | $this->assertCount(2, $rawConfiguration->getConfigurations()); 37 | } 38 | 39 | public function testValidation(): void 40 | { 41 | $this->configurationOne->method('getName') 42 | ->willReturn('configuration_1'); 43 | 44 | $this->configurationTwo->method('getName') 45 | ->willReturn('configuration_1'); 46 | 47 | $this->expectException(InvalidConfigurationException::class); 48 | $rawConfiguration = new RawConfiguration(); 49 | $rawConfiguration->addConfiguration($this->configurationOne) 50 | ->addConfiguration($this->configurationTwo); 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Command/ConsumersDescribeCommand.php: -------------------------------------------------------------------------------- 1 | consumerProvider = $consumerProvider; 32 | $this->configurationResolver = $configurationResolver; 33 | 34 | parent::__construct(); 35 | } 36 | 37 | protected function configure(): void 38 | { 39 | $this->addOption( 40 | name: 'name', 41 | mode: InputOption::VALUE_REQUIRED, 42 | description: 'Shows specific consumer configuration.' 43 | ); 44 | } 45 | 46 | protected function execute(InputInterface $input, OutputInterface $output): int 47 | { 48 | $name = $input->getOption('name'); 49 | $consumers = $name ? [$this->consumerProvider->provide($name)] : $this->consumerProvider->getAll(); 50 | 51 | foreach ($consumers as $consumer) { 52 | $this->describe($this->configurationResolver->resolve($consumer), $output, $consumer); 53 | } 54 | 55 | return self::SUCCESS; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Resources/config/consumers.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | -------------------------------------------------------------------------------- /src/Command/Traits/DescribeTrait.php: -------------------------------------------------------------------------------- 1 | setHeaders(['configuration', 'value']); 23 | $table->setStyle('box'); 24 | $values['class'] = get_class($client); 25 | 26 | $configurationType = ResolvedConfiguration::ALL_TYPES; 27 | if ($client instanceof ConsumerInterface) { 28 | $values['name'] = $client->getName(); 29 | $configurationType = ResolvedConfiguration::CONSUMER_TYPES; 30 | } 31 | 32 | if ($client instanceof ProducerInterface) { 33 | $configurationType = ResolvedConfiguration::PRODUCER_TYPES; 34 | } 35 | 36 | foreach ($configuration->getConfigurations($configurationType) as $configuration) { 37 | $resolvedValue = $configuration['resolvedValue']; 38 | $name = $configuration['configuration']->getName(); 39 | if (is_array($resolvedValue)) { 40 | $values[$name] = implode(PHP_EOL, $resolvedValue); 41 | 42 | continue; 43 | } 44 | 45 | if ($resolvedValue === true || $resolvedValue === false) { 46 | $values[$name] = var_export($resolvedValue, true); 47 | 48 | continue; 49 | } 50 | $values[$name] = $resolvedValue; 51 | } 52 | 53 | foreach ($values as $name => $value) { 54 | $table->addRow([$name, $value]); 55 | } 56 | 57 | $table->render(); 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/Configuration/Type/LogLevel.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 41 | ); 42 | } 43 | 44 | public function isValueValid($value): bool 45 | { 46 | return is_numeric($value) && strpos((string) $value, '.') === false; 47 | } 48 | 49 | public function cast($validatedValue): int 50 | { 51 | return (int) $validatedValue; 52 | } 53 | 54 | public function getDefaultValue(): int 55 | { 56 | return LOG_ERR; 57 | } 58 | 59 | public function supportsClient(ClientInterface $client): bool 60 | { 61 | $clientRef = new ReflectionClass($client::class); 62 | 63 | return $clientRef->implementsInterface(ConsumerInterface::class) 64 | || $clientRef->implementsInterface(ProducerInterface::class); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /tests/Functional/Client/Consumer/ConsumerProviderTest.php: -------------------------------------------------------------------------------- 1 | getContainer(); 22 | 23 | $this->provider = $container->get('sts_gaming_group_kafka.client.consumer.consumer_provider'); 24 | } 25 | 26 | public function testConsumersRegistered(): void 27 | { 28 | $consumerOne = $this->provider->provide(DummyConsumerOne::NAME); 29 | $consumerTwo = $this->provider->provide(DummyConsumerTwo::NAME); 30 | 31 | $this->assertEquals(DummyConsumerOne::NAME, $consumerOne->getName()); 32 | $this->assertEquals(DummyConsumerTwo::NAME, $consumerTwo->getName()); 33 | } 34 | 35 | public function testMoreThanOneConsumerFound(): void 36 | { 37 | $this->provider->addConsumer(new DummyConsumerOneClone()); 38 | 39 | $this->expectException(InvalidConsumerException::class); 40 | $this->expectExceptionMessageMatches('/Multiple consumers found/'); 41 | 42 | $this->provider->provide(DummyConsumerOneClone::NAME); 43 | } 44 | 45 | public function testNoConsumersFound(): void 46 | { 47 | $this->expectException(InvalidConsumerException::class); 48 | $this->expectExceptionMessageMatches('/There is no matching consumer/'); 49 | 50 | $this->provider->provide('foo'); 51 | } 52 | 53 | public function testGetAllConsumers(): void 54 | { 55 | $this->assertCount(2, $this->provider->getAll()); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Configuration/Type/Brokers.php: -------------------------------------------------------------------------------- 1 | implementsInterface(ConsumerInterface::class) 65 | || $clientRef->implementsInterface(ProducerInterface::class); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/Client/Consumer/Factory/MessageFactory.php: -------------------------------------------------------------------------------- 1 | 20 | */ 21 | private array $decoders; 22 | 23 | /** 24 | * @var array 25 | */ 26 | private array $denormalizers; 27 | 28 | private Validator $validator; 29 | 30 | public function __construct(iterable $decoders, iterable $denormalizers, Validator $validator) 31 | { 32 | foreach ($decoders as $decoder) { 33 | $this->decoders[get_class($decoder)] = $decoder; 34 | } 35 | foreach ($denormalizers as $denormalizer) { 36 | $this->denormalizers[get_class($denormalizer)] = $denormalizer; 37 | } 38 | 39 | $this->validator = $validator; 40 | } 41 | 42 | public function create(RdKafkaMessage $rdKafkaMessage, ResolvedConfiguration $configuration): Message 43 | { 44 | $requiredDecoder = $configuration->getValue(Decoder::NAME); 45 | $decoded = $this->decoders[$requiredDecoder]->decode($configuration, $rdKafkaMessage->payload); 46 | 47 | $this->validator->validate($configuration, $decoded, Validator::PRE_DENORMALIZE_TYPE); 48 | 49 | $requiredDenormalizer = $configuration->getValue(Denormalizer::NAME); 50 | $denormalized = $this->denormalizers[$requiredDenormalizer]->denormalize($decoded); 51 | 52 | $this->validator->validate($configuration, $denormalized, Validator::POST_DENORMALIZE_TYPE); 53 | 54 | return new Message( 55 | $rdKafkaMessage->topic_name, 56 | $rdKafkaMessage->partition, 57 | $rdKafkaMessage->payload, 58 | $rdKafkaMessage->offset, 59 | $denormalized, 60 | $rdKafkaMessage->key 61 | ); 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Configuration/Type/MaxPollIntervalMs.php: -------------------------------------------------------------------------------- 1 | getDefaultValue() 48 | ); 49 | } 50 | 51 | public function isValueValid($value): bool 52 | { 53 | return is_numeric($value) && !str_contains((string)$value, '.') && $value >= 0; 54 | } 55 | 56 | public function getDefaultValue(): int 57 | { 58 | return 300000; 59 | } 60 | 61 | public function cast($validatedValue): int 62 | { 63 | return (int) $validatedValue; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /tests/config/consumers.json: -------------------------------------------------------------------------------- 1 | { 2 | "consumers": { 3 | "instances": { 4 | "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Client\\Consumer\\DummyConsumerOne": { 5 | "brokers": [ 6 | "127.0.0.1:9092", 7 | "127.0.0.2:9092", 8 | "127.0.0.3:9092" 9 | ], 10 | "topics": [ 11 | "dummy_topic_one", 12 | "dummy_topic_two" 13 | ], 14 | "group_id": "dummy_group_id_one", 15 | "schema_registry": "http://127.0.0.1:8081", 16 | "max_retries": 5, 17 | "max_retry_delay": 1000, 18 | "denormalizer": "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Denormalizer\\DummyDenormalizerOne", 19 | "validators": [ 20 | "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Validator\\DummyValidatorOne" 21 | ], 22 | "decoder": "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Decoder\\DummyDecoderOne", 23 | "auto_commit_interval_ms": "50", 24 | "auto_offset_reset": "smallest", 25 | "timeout": 1000, 26 | "enable_auto_offset_store": "true", 27 | "enable_auto_commit": "true", 28 | "register_missing_schemas": "false", 29 | "register_missing_subjects": "false", 30 | "retry_delay": 200, 31 | "retry_multiplier": 2, 32 | "log_level": 3 33 | }, 34 | "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Client\\Consumer\\DummyConsumerTwo": { 35 | "brokers": [ 36 | "127.0.0.4:9092", 37 | "127.0.0.5:9092", 38 | "127.0.0.6:9092" 39 | ], 40 | "topics": [ 41 | "dummy_topic_three", 42 | "dummy_topic_four" 43 | ], 44 | "group_id": "dummy_group_id_two", 45 | "schema_registry": "http://127.0.0.2:8081", 46 | "max_retries": 4, 47 | "max_retry_delay": 2000, 48 | "denormalizer": "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Denormalizer\\DummyDenormalizerOne", 49 | "validators": [ 50 | "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Validator\\DummyValidatorOne" 51 | ], 52 | "decoder": "StsGamingGroup\\KafkaBundle\\Tests\\Dummy\\Decoder\\DummyDecoderOne", 53 | "auto_commit_interval_ms": "40", 54 | "auto_offset_reset": "largest", 55 | "timeout": 2000, 56 | "enable_auto_offset_store": "false", 57 | "enable_auto_commit": "false", 58 | "register_missing_schemas": "true", 59 | "register_missing_subjects": "true", 60 | "retry_delay": 300, 61 | "retry_multiplier": 3, 62 | "log_level": 2 63 | } 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /tests/Unit/Client/Consumer/ConsumerProviderTest.php: -------------------------------------------------------------------------------- 1 | consumerOne = $this->createMock(ConsumerInterface::class); 22 | $this->consumerTwo = $this->createMock(ConsumerInterface::class); 23 | $this->consumerProvider = new ConsumerProvider(); 24 | } 25 | 26 | public function testAddAndProviderConsumer(): void 27 | { 28 | $this->consumerOne->method('getName') 29 | ->willReturn('consumer_1'); 30 | $this->consumerTwo->method('getName') 31 | ->willReturn('consumer_2'); 32 | 33 | $this->consumerProvider->addConsumer($this->consumerOne) 34 | ->addConsumer($this->consumerTwo); 35 | 36 | $this->assertInstanceOf(get_class($this->consumerTwo), $this->consumerProvider->provide('consumer_2')); 37 | } 38 | 39 | public function testMultipleConsumersFound(): void 40 | { 41 | $this->consumerOne->method('getName') 42 | ->willReturn('consumer_2'); 43 | $this->consumerTwo->method('getName') 44 | ->willReturn('consumer_2'); 45 | 46 | $this->consumerProvider->addConsumer($this->consumerOne) 47 | ->addConsumer($this->consumerTwo); 48 | 49 | $this->expectException(InvalidConsumerException::class); 50 | $this->expectExceptionMessageMatches('/Multiple consumers/'); 51 | $this->consumerProvider->provide('consumer_2'); 52 | } 53 | 54 | public function testNoConsumerFound(): void 55 | { 56 | $this->consumerOne->method('getName') 57 | ->willReturn('consumer_1'); 58 | $this->consumerTwo->method('getName') 59 | ->willReturn('consumer_2'); 60 | 61 | $this->consumerProvider->addConsumer($this->consumerOne) 62 | ->addConsumer($this->consumerTwo); 63 | 64 | $this->expectException(InvalidConsumerException::class); 65 | $this->expectExceptionMessageMatches('/no matching consumer/'); 66 | $this->consumerProvider->provide('consumer_3'); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/Configuration/ResolvedConfiguration.php: -------------------------------------------------------------------------------- 1 | configurations[$configuration->getName()] = [ 30 | 'configuration' => $configuration, 31 | 'resolvedValue' => $resolvedValue 32 | ]; 33 | 34 | return $this; 35 | } 36 | 37 | /** 38 | * @param string $type 39 | * @return array 40 | */ 41 | public function getConfigurations(string $type = self::ALL_TYPES): array 42 | { 43 | switch ($type) { 44 | case self::ALL_TYPES: 45 | $interface = ConfigurationInterface::class; 46 | break; 47 | case self::KAFKA_TYPES: 48 | $interface = KafkaConfigurationInterface::class; 49 | break; 50 | case self::CONSUMER_TYPES: 51 | $interface = ConsumerConfigurationInterface::class; 52 | break; 53 | case self::PRODUCER_TYPES: 54 | $interface = ProducerConfigurationInterface::class; 55 | break; 56 | default: 57 | throw new InvalidConfigurationType(sprintf('Unknown configuration type %s', $type)); 58 | } 59 | 60 | $configurations = []; 61 | foreach ($this->configurations as $configuration) { 62 | if ($configuration['configuration'] instanceof $interface) { 63 | $configurations[] = $configuration; 64 | } 65 | } 66 | 67 | return $configurations; 68 | } 69 | 70 | /** 71 | * @param string $name 72 | * @return mixed 73 | */ 74 | public function getValue(string $name) 75 | { 76 | return $this->configurations[$name]['resolvedValue']; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /src/Validator/Validator.php: -------------------------------------------------------------------------------- 1 | 19 | */ 20 | private array $preDenormalizeValidators = []; 21 | /** 22 | * @var array 23 | */ 24 | private array $postDenormalizeValidators = []; 25 | 26 | /** 27 | * @param iterable $validators 28 | */ 29 | public function __construct(iterable $validators) 30 | { 31 | foreach ($validators as $validator) { 32 | if ($validator->type() === self::PRE_DENORMALIZE_TYPE) { 33 | $this->preDenormalizeValidators[get_class($validator)] = $validator; 34 | } 35 | if ($validator->type() === self::POST_DENORMALIZE_TYPE) { 36 | $this->postDenormalizeValidators[get_class($validator)] = $validator; 37 | } 38 | } 39 | } 40 | 41 | /** 42 | * @param ResolvedConfiguration $configuration 43 | * @param mixed $data 44 | * @param string $type 45 | * @return bool 46 | */ 47 | public function validate(ResolvedConfiguration $configuration, $data, string $type): bool 48 | { 49 | if ($type !== self::PRE_DENORMALIZE_TYPE && $type !== self::POST_DENORMALIZE_TYPE) { 50 | throw new \RuntimeException(sprintf( 51 | 'Type must be either %s or %s.', 52 | self::PRE_DENORMALIZE_TYPE, 53 | self::POST_DENORMALIZE_TYPE 54 | )); 55 | } 56 | 57 | $validators = $type === self::PRE_DENORMALIZE_TYPE ? 58 | $this->preDenormalizeValidators : 59 | $this->postDenormalizeValidators; 60 | 61 | $requiredValidators = $configuration->getValue(Validators::NAME); 62 | foreach ($requiredValidators as $requiredValidator) { 63 | if (isset($validators[$requiredValidator]) && !$validators[$requiredValidator]->validate($data)) { 64 | throw new ValidationException( 65 | $validators[$requiredValidator], 66 | $validators[$requiredValidator]->failureReason($data), 67 | $data, 68 | sprintf('Validation not passed by %s', $requiredValidator) 69 | ); 70 | } 71 | } 72 | 73 | return true; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/Command/ConsumeCommand.php: -------------------------------------------------------------------------------- 1 | rawConfiguration = $rawConfiguration; 42 | $this->consumerProvider = $consumerProvider; 43 | $this->consumerClient = $consumerClient; 44 | $this->configurationResolver = $configurationResolver; 45 | 46 | parent::__construct(); 47 | } 48 | 49 | protected function configure(): void 50 | { 51 | $this 52 | ->addArgument(name: 'name', mode: InputArgument::REQUIRED, description: 'Name of the registered consumer.') 53 | ->addOption(name: 'describe', mode: InputOption::VALUE_NONE, description: 'Describes consumer'); 54 | 55 | $this->addConfigurations($this->rawConfiguration); 56 | } 57 | 58 | protected function execute(InputInterface $input, OutputInterface $output): int 59 | { 60 | $consumer = $this->consumerProvider->provide($input->getArgument('name')); 61 | 62 | if ($input->getOption('describe')) { 63 | $this->describe($this->configurationResolver->resolve($consumer, $input), $output, $consumer); 64 | 65 | return self::SUCCESS; 66 | } 67 | 68 | $this->consumerClient->consume($consumer, $input); 69 | 70 | return self::SUCCESS; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /phpstan.neon.dist: -------------------------------------------------------------------------------- 1 | parameters: 2 | bootstrapFiles: 3 | - vendor/bin/.phpunit/phpunit/vendor/autoload.php 4 | level: max 5 | checkMissingIterableValueType: false 6 | reportUnmatchedIgnoredErrors: false 7 | paths: 8 | - %currentWorkingDirectory% 9 | excludePaths: 10 | - vendor/* 11 | - var/* 12 | - bin/* 13 | - config/* 14 | ignoreErrors: 15 | - 16 | message: '/expects Symfony\\Component\\Config\\Definition\\ConfigurationInterface/' 17 | path: src/DependencyInjection/StsKafkaExtension.php 18 | - 19 | message: '/Conf::set\(\) expects string/' 20 | path: src/RdKafka/Factory/ConsumerFactory.php 21 | - 22 | message: '/RdKafka\\Message\|null given/' 23 | path: src/Client/Consumer/ConsumerClient.php 24 | - 25 | message: '/provide\(\) expects string/' 26 | path: src/Command/ConsumeCommand.php 27 | - 28 | message: '/MockObject given/' 29 | path: tests 30 | - 31 | message: '/cast\(\) should return bool/' 32 | path: src/Configuration/Traits/BooleanConfigurationTrait.php 33 | - 34 | message: '/RecordSerializer constructor expects array/' 35 | path: src/Decoder/AvroDecoder.php 36 | - 37 | message: '/should return array but returns RdKafka\\ProducerTopic/' 38 | path: src/RdKafka/Factory/ProducerBuilder.php 39 | - 40 | message: '/Cannot access an offset on RdKafka\\ProducerTopic/' 41 | path: src/RdKafka/Factory/ProducerBuilder.php 42 | - 43 | message: '/Else branch is unreachable because ternary operator condition is always true/' 44 | path: src/Configuration/ConfigurationResolver.php 45 | - 46 | message: '/ConsumerProvider::provide\(\) expects string/' 47 | path: src/Command/ConsumersDescribeCommand.php 48 | - 49 | message: '/Variable \$rdKafkaMessage might not be defined/' 50 | path: src/Client/Consumer/ConsumerClient.php 51 | - 52 | message: '/ConsumerProviderTest::\$provider/' 53 | path: tests/Functional/Consumer/ConsumerProviderTest.php 54 | - 55 | message: '/json_decode expects string/' 56 | path: tests/Unit/Configuration/ConfigurationResolverTest.php 57 | - 58 | message: '/assertInstanceOf\(\) expects class-string/' 59 | path: tests/Unit/Configuration/ResolvedConfigurationTest.php 60 | - 61 | message: '/in_array expects array/' 62 | path: src/Configuration/Traits/ObjectConfigurationTrait.php 63 | - 64 | message: '/does not accept object\|null/' 65 | path: tests/Functional/Client/Consumer/ConsumerProviderTest.php 66 | -------------------------------------------------------------------------------- /tests/Unit/Configuration/ResolvedConfigurationTest.php: -------------------------------------------------------------------------------- 1 | kafkaConfigurationOne = $this->createMock(KafkaConfigurationInterface::class); 28 | $this->kafkaConfigurationTwo = $this->createMock(KafkaConfigurationInterface::class); 29 | $this->consumerConfigurationOne = $this->createMock(ConsumerConfigurationInterface::class); 30 | $this->consumerConfigurationTwo = $this->createMock(ConsumerConfigurationInterface::class); 31 | $this->producerConfigurationOne = $this->createMock(ProducerConfigurationInterface::class); 32 | $this->producerConfigurationTwo = $this->createMock(ProducerConfigurationInterface::class); 33 | } 34 | 35 | /** 36 | * @dataProvider getConfigurationsByTypeProvider 37 | */ 38 | public function testGetConfigurationsByType(string $name, int $expectedCount, string $interface): void 39 | { 40 | $this->createDefaultExpectations(); 41 | $resolved = $this->createDefaultResolvedConfiguration(); 42 | $configurations = $resolved->getConfigurations($name); 43 | 44 | $this->assertCount($expectedCount, $configurations); 45 | 46 | foreach ($configurations as $configuration) { 47 | $this->assertInstanceOf($interface, $configuration['configuration']); 48 | } 49 | } 50 | 51 | public function getConfigurationsByTypeProvider(): array 52 | { 53 | return [ 54 | ['all', 6, ConfigurationInterface::class], 55 | ['kafka', 2, KafkaConfigurationInterface::class], 56 | ['consumer', 2, ConsumerConfigurationInterface::class], 57 | ['producer', 2, ProducerConfigurationInterface::class] 58 | ]; 59 | } 60 | 61 | public function testGetValue(): void 62 | { 63 | $this->createDefaultExpectations(); 64 | $resolved = $this->createDefaultResolvedConfiguration(); 65 | 66 | $this->assertEquals('faz', $resolved->getValue('cc_one')); 67 | } 68 | 69 | public function testUnknownConfigurationType(): void 70 | { 71 | $this->expectException(InvalidConfigurationType::class); 72 | 73 | $resolved = new ResolvedConfiguration(); 74 | 75 | $resolved->getConfigurations('foo'); 76 | } 77 | 78 | private function createDefaultExpectations(): void 79 | { 80 | $this->kafkaConfigurationOne 81 | ->method('getName') 82 | ->willReturn('kc_one'); 83 | 84 | $this->kafkaConfigurationTwo 85 | ->method('getName') 86 | ->willReturn('kc_two'); 87 | 88 | $this->consumerConfigurationOne 89 | ->method('getName') 90 | ->willReturn('cc_one'); 91 | 92 | $this->consumerConfigurationTwo 93 | ->method('getName') 94 | ->willReturn('cc_two'); 95 | 96 | $this->producerConfigurationOne 97 | ->method('getName') 98 | ->willReturn('pc_one'); 99 | 100 | $this->producerConfigurationTwo 101 | ->method('getName') 102 | ->willReturn('pc_two'); 103 | } 104 | 105 | private function createDefaultResolvedConfiguration(): ResolvedConfiguration 106 | { 107 | return (new ResolvedConfiguration()) 108 | ->addConfiguration($this->kafkaConfigurationOne, 'foo') 109 | ->addConfiguration($this->kafkaConfigurationTwo, 'boo') 110 | ->addConfiguration($this->consumerConfigurationOne, 'faz') 111 | ->addConfiguration($this->consumerConfigurationTwo, 'baz') 112 | ->addConfiguration($this->producerConfigurationOne, 'fee') 113 | ->addConfiguration($this->producerConfigurationTwo, 'bee'); 114 | } 115 | } 116 | -------------------------------------------------------------------------------- /src/Client/Producer/ProducerClient.php: -------------------------------------------------------------------------------- 1 | producerProvider = $producerProvider; 39 | $this->kafkaConfigurationFactory = $kafkaConfigurationFactory; 40 | $this->configurationResolver = $configurationResolver; 41 | } 42 | 43 | /** 44 | * @param mixed $data 45 | * @return $this 46 | */ 47 | public function produce($data): self 48 | { 49 | $this->isKafkaExtensionLoaded(); 50 | 51 | $producer = $this->producerProvider->provide($data); 52 | $rdKafkaConfig = $this->kafkaConfigurationFactory->create($producer); 53 | 54 | $producerClass = get_class($producer); 55 | if (!isset($this->rdKafkaProducers[$producerClass])) { 56 | $this->rdKafkaProducers[$producerClass] = new Producer($rdKafkaConfig); 57 | } 58 | 59 | $this->lastCalledProducer = $this->rdKafkaProducers[$producerClass]; 60 | $configuration = $this->configurationResolver->resolve($producer); 61 | $topic = $this->lastCalledProducer->newTopic($configuration->getValue(ProducerTopic::NAME)); 62 | 63 | $message = $producer->produce($data); 64 | $topic->produce( 65 | $this->getPartition($data, $producer, $configuration), 66 | 0, 67 | $message->getPayload(), 68 | $message->getKey() 69 | ); 70 | 71 | if ($this->lastCalledProducer->getOutQLen() % $this->pollingBatch === 0) { 72 | while ($this->lastCalledProducer->getOutQLen() > 0) { 73 | $this->lastCalledProducer->poll($this->pollingTimeoutMs); 74 | } 75 | } 76 | 77 | return $this; 78 | } 79 | 80 | public function setMaxFlushRetries(int $maxFlushRetries): self 81 | { 82 | $this->maxFlushRetries = $maxFlushRetries; 83 | 84 | return $this; 85 | } 86 | 87 | public function setFlushTimeoutMs(int $flushTimeoutMs): self 88 | { 89 | $this->flushTimeoutMs = $flushTimeoutMs; 90 | 91 | return $this; 92 | } 93 | 94 | public function setPollingBatch(int $pollingBatch): self 95 | { 96 | $this->pollingBatch = $pollingBatch; 97 | 98 | return $this; 99 | } 100 | 101 | public function setPollingTimeoutMs(int $pollingTimeoutMs): self 102 | { 103 | $this->pollingTimeoutMs = $pollingTimeoutMs; 104 | 105 | return $this; 106 | } 107 | 108 | public function flush(): void 109 | { 110 | if (!$this->lastCalledProducer) { 111 | throw new \RuntimeException('You have to call `produce` method first to be able to flush.'); 112 | } 113 | 114 | $result = RD_KAFKA_RESP_ERR_NO_ERROR; 115 | for ($flushRetries = 0; $flushRetries < $this->maxFlushRetries; $flushRetries++) { 116 | $result = $this->lastCalledProducer->flush($this->flushTimeoutMs); 117 | if (RD_KAFKA_RESP_ERR_NO_ERROR === $result) { 118 | break; 119 | } 120 | } 121 | 122 | if (RD_KAFKA_RESP_ERR_NO_ERROR !== $result) { 123 | throw new \RuntimeException('Unable to flush, messages might be lost.'); 124 | } 125 | } 126 | 127 | /** 128 | * @param mixed $data 129 | */ 130 | private function getPartition($data, ProducerInterface $producer, ResolvedConfiguration $configuration): int 131 | { 132 | if (!$producer instanceof PartitionAwareProducerInterface) { 133 | return $configuration->getValue(ProducerPartition::NAME); 134 | } 135 | 136 | return $producer->getPartition($data, $configuration); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/DependencyInjection/StsGamingGroupKafkaExtension.php: -------------------------------------------------------------------------------- 1 | load(sprintf($xmlFile . '.xml')); 40 | } 41 | 42 | $container->registerForAutoconfiguration(ConsumerInterface::class) 43 | ->addTag('sts_gaming_group_kafka.kafka.consumer'); 44 | 45 | $container->registerForAutoconfiguration(ProducerInterface::class) 46 | ->addTag('sts_gaming_group_kafka.kafka.producer'); 47 | 48 | $container->registerForAutoconfiguration(ConfigurationInterface::class) 49 | ->addTag('sts_gaming_group_kafka.configuration.type'); 50 | 51 | $container->registerForAutoconfiguration(DecoderInterface::class) 52 | ->addTag('sts_gaming_group_kafka.decoder'); 53 | 54 | $container->registerForAutoconfiguration(DenormalizerInterface::class) 55 | ->addTag('sts_gaming_group_kafka.denormalizer'); 56 | 57 | $container->registerForAutoconfiguration(ValidatorInterface::class) 58 | ->addTag('sts_gaming_group_kafka.validator'); 59 | 60 | $configurationResolver = $container->getDefinition('sts_gaming_group_kafka.configuration.configuration_resolver'); 61 | $configurationResolver->setArgument(1, $mergedConfig); 62 | } 63 | 64 | public function process(ContainerBuilder $container): void 65 | { 66 | $this->addConsumersAndProvider($container); 67 | $this->addProducersAndProvider($container); 68 | $this->addConfigurations($container); 69 | } 70 | 71 | private function addConsumersAndProvider(ContainerBuilder $container): void 72 | { 73 | $providerId = 'sts_gaming_group_kafka.client.consumer.consumer_provider'; 74 | if (!$container->has($providerId)) { 75 | throw new InvalidDefinitionException( 76 | sprintf('Unable to find any consumer provider. Looking for service id %s', $providerId) 77 | ); 78 | } 79 | 80 | $consumerProvider = $container->findDefinition($providerId); 81 | $consumers = $container->findTaggedServiceIds('sts_gaming_group_kafka.kafka.consumer'); 82 | foreach ($consumers as $id => $tags) { 83 | $consumerProvider->addMethodCall('addConsumer', [new Reference($id)]); 84 | } 85 | } 86 | 87 | private function addProducersAndProvider(ContainerBuilder $container): void 88 | { 89 | $providerId = 'sts_gaming_group_kafka.client.producer.producer_provider'; 90 | if (!$container->has($providerId)) { 91 | throw new InvalidDefinitionException( 92 | sprintf('Unable to find any producer provider. Looking for service id %s', $providerId) 93 | ); 94 | } 95 | 96 | $producerProvider = $container->findDefinition($providerId); 97 | $producers = $container->findTaggedServiceIds('sts_gaming_group_kafka.kafka.producer'); 98 | foreach ($producers as $id => $tags) { 99 | $producerProvider->addMethodCall('addProducer', [new Reference($id)]); 100 | } 101 | } 102 | 103 | private function addConfigurations(ContainerBuilder $container): void 104 | { 105 | $configurationsId = 'sts_gaming_group_kafka.configuration.raw_configuration'; 106 | if (!$container->has($configurationsId)) { 107 | throw new InvalidDefinitionException( 108 | sprintf('Unable to find configurations class. Looking for service id %s', $configurationsId) 109 | ); 110 | } 111 | 112 | $configurations = $container->findDefinition($configurationsId); 113 | $configurationTypes = $container->findTaggedServiceIds('sts_gaming_group_kafka.configuration.type'); 114 | foreach ($configurationTypes as $id => $tags) { 115 | $configurations->addMethodCall('addConfiguration', [new Reference($id)]); 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /src/Configuration/ConfigurationResolver.php: -------------------------------------------------------------------------------- 1 | rawConfiguration = $rawConfiguration; 24 | $this->yamlConfig = $yamlConfig; 25 | } 26 | 27 | /** 28 | * @param string|ClientInterface $clientClass 29 | * @param InputInterface|null $input 30 | * @return ResolvedConfiguration 31 | */ 32 | public function resolve($clientClass, ?InputInterface $input = null): ResolvedConfiguration 33 | { 34 | $configuration = new ResolvedConfiguration(); 35 | 36 | foreach ($this->rawConfiguration->getConfigurations() as $rawConfiguration) { 37 | if (is_object($clientClass) && !$rawConfiguration->supportsClient($clientClass)) { 38 | continue; 39 | } 40 | 41 | $resolvedValue = $this->getResolvedValue($rawConfiguration, $clientClass, $input); 42 | 43 | if ($rawConfiguration instanceof CastValueInterface) { 44 | $resolvedValue = $rawConfiguration->cast($resolvedValue); 45 | } 46 | 47 | $configuration->addConfiguration($rawConfiguration, $resolvedValue); 48 | } 49 | 50 | return $configuration; 51 | } 52 | 53 | /** 54 | * @param ConfigurationInterface $configuration 55 | * @param string|ClientInterface $clientClass 56 | * @param InputInterface|null $input 57 | * @return mixed 58 | */ 59 | private function getResolvedValue( 60 | ConfigurationInterface $configuration, 61 | $clientClass, 62 | ?InputInterface $input 63 | ) { 64 | $type = ''; 65 | if (is_a($clientClass, ConsumerInterface::class, true)) { 66 | $type = 'consumers'; 67 | } 68 | 69 | if (is_a($clientClass, ProducerInterface::class, true)) { 70 | $type = 'producers'; 71 | } 72 | 73 | if (!$type) { 74 | throw new InvalidClientException(sprintf( 75 | 'Object must implement %s or %s to properly resolve configuration.', 76 | ConsumerInterface::class, 77 | ProducerInterface::class 78 | )); 79 | } 80 | 81 | $name = $configuration->getName(); 82 | if ($input && $input->getParameterOption('--' . $name) !== false) { 83 | $resolvedValue = $input->getOption($name); 84 | $this->validateResolvedValue($configuration, $resolvedValue); 85 | 86 | return $resolvedValue; 87 | } 88 | 89 | $clientClass = is_string($clientClass) ? $clientClass : get_class($clientClass); 90 | if ($this->shouldResolveInstance($clientClass, $type, $configuration)) { 91 | $resolvedValue = $this->yamlConfig[$type]['instances'][$clientClass][$name]; 92 | $this->validateResolvedValue($configuration, $resolvedValue); 93 | 94 | return $resolvedValue; 95 | } 96 | 97 | $parentClass = $this->getParentClass($clientClass); 98 | if ($this->shouldResolveInstance($parentClass, $type, $configuration)) { 99 | $resolvedValue = $this->yamlConfig[$type]['instances'][$parentClass][$name]; 100 | $this->validateResolvedValue($configuration, $resolvedValue); 101 | 102 | return $resolvedValue; 103 | } 104 | 105 | return $configuration->getDefaultValue(); 106 | } 107 | 108 | /** 109 | * @param ConfigurationInterface $configuration 110 | * @param mixed $resolvedValue 111 | */ 112 | private function validateResolvedValue(ConfigurationInterface $configuration, $resolvedValue): void 113 | { 114 | if (!$configuration->isValueValid($resolvedValue)) { 115 | throw new InvalidConfigurationException(sprintf( 116 | 'Invalid option passed for %s. Passed value `%s`. Configuration description: %s', 117 | $configuration->getName(), 118 | is_array($resolvedValue) ? implode(', ', $resolvedValue) : $resolvedValue, 119 | $configuration->getDescription() 120 | )); 121 | } 122 | } 123 | 124 | /** 125 | * @param string|ClientInterface $clientClass 126 | * @return string 127 | */ 128 | private function getParentClass($clientClass): string 129 | { 130 | $parentClass = get_parent_class($clientClass); 131 | 132 | return $parentClass === false ? '' : $parentClass; 133 | } 134 | 135 | private function shouldResolveInstance(string $class, string $type, ConfigurationInterface $configuration): bool 136 | { 137 | $name = $configuration->getName(); 138 | 139 | return isset($this->yamlConfig[$type]['instances'][$class][$name]) && 140 | $this->yamlConfig[$type]['instances'][$class][$name]; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /src/Resources/config/configuration_types.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 22 | 23 | 24 | 26 | 27 | 28 | 30 | 31 | 32 | 34 | 35 | 36 | 37 | 38 | 39 | 41 | 42 | 43 | 45 | 46 | 47 | 49 | 50 | 51 | 52 | 53 | 54 | 56 | 57 | 58 | 60 | 61 | 62 | 64 | 65 | 66 | 68 | 69 | 70 | 72 | 73 | 74 | 76 | 77 | 78 | 80 | 81 | 82 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | -------------------------------------------------------------------------------- /src/Client/Consumer/ConsumerClient.php: -------------------------------------------------------------------------------- 1 | kafkaConfigurationFactory = $kafkaConfigurationFactory; 49 | $this->messageFactory = $messageFactory; 50 | $this->configurationResolver = $configurationResolver; 51 | $this->dispatcher = $dispatcher; 52 | } 53 | 54 | public function consume(ConsumerInterface $consumer, ?InputInterface $input = null): bool 55 | { 56 | $this->isKafkaExtensionLoaded(); 57 | 58 | $configuration = $this->configurationResolver->resolve($consumer, $input); 59 | 60 | $timeout = $configuration->getValue(Timeout::NAME); 61 | $maxRetries = $configuration->getValue(MaxRetries::NAME); 62 | $retryDelay = $configuration->getValue(RetryDelay::NAME); 63 | $maxRetryDelay = $configuration->getValue(MaxRetryDelay::NAME); 64 | $retryMultiplier = $configuration->getValue(RetryMultiplier::NAME); 65 | $topics = $configuration->getValue(Topics::NAME); 66 | $enableAutoCommit = $configuration->getValue(EnableAutoCommit::NAME); 67 | 68 | $rdKafkaConfig = $this->kafkaConfigurationFactory->create($consumer, $input); 69 | $rdKafkaConsumer = new RdKafkaConsumer($rdKafkaConfig); 70 | $rdKafkaConsumer->subscribe($topics); 71 | 72 | $consumptionStart = microtime(true); 73 | while (true) { 74 | try { 75 | $this->dispatch(PreMessageConsumedEvent::class, $consumer); 76 | 77 | $rdKafkaMessage = $rdKafkaConsumer->consume($timeout); 78 | $this->validateRdKafkaMessage($rdKafkaMessage); 79 | } catch (NullMessageException $exception) { 80 | $consumer->handleException( 81 | $exception, 82 | new Context($configuration, $rdKafkaConsumer, $rdKafkaMessage, 0) 83 | ); 84 | 85 | $this->setConsumptionTime($consumptionStart); 86 | 87 | continue; 88 | } 89 | 90 | for ($retry = 0; $retry <= $maxRetries; ++$retry) { 91 | $context = new Context($configuration, $rdKafkaConsumer, $rdKafkaMessage, $retry); 92 | try { 93 | $message = $this->messageFactory->create($rdKafkaMessage, $configuration); 94 | $consumer->consume($message, $context); 95 | } catch (ValidationException | RecoverableMessageException $exception) { 96 | $consumer->handleException($exception, $context); 97 | 98 | if ($exception instanceof ValidationException) { 99 | if ($enableAutoCommit === 'false') { 100 | $rdKafkaConsumer->commit($rdKafkaMessage); 101 | } 102 | 103 | break; 104 | } 105 | 106 | if ($exception instanceof RecoverableMessageException) { 107 | if ($retry !== $maxRetries) { 108 | $retryDelay *= $retryMultiplier; 109 | if ($retryDelay > $maxRetryDelay) { 110 | $retryDelay = $maxRetryDelay; 111 | } 112 | usleep($retryDelay * 1000); 113 | } 114 | 115 | continue; 116 | } 117 | } 118 | 119 | break; 120 | } 121 | 122 | $retryDelay = $configuration->getValue(RetryDelay::NAME); 123 | 124 | $this->increaseConsumedMessages(); 125 | $this->setConsumptionTime($consumptionStart); 126 | 127 | $this->dispatch(PostMessageConsumedEvent::class, $consumer); 128 | } 129 | } 130 | 131 | private function setConsumptionTime(float $consumptionStart): void 132 | { 133 | $this->consumptionTimeMs = microtime(true) - $consumptionStart; 134 | } 135 | 136 | private function increaseConsumedMessages(): void 137 | { 138 | ++$this->consumedMessages; 139 | } 140 | 141 | private function validateRdKafkaMessage(?RdKafkaMessage $message): void 142 | { 143 | if (null === $message || RD_KAFKA_RESP_ERR__PARTITION_EOF === $message->err) { 144 | throw new NullMessageException('Currently, there are no more messages.'); 145 | } 146 | 147 | if (RD_KAFKA_RESP_ERR__TIMED_OUT === $message->err) { 148 | throw new NullMessageException( 149 | 'Kafka brokers have timed out or there are no messages. Unable to differentiate the reason.' 150 | ); 151 | } 152 | 153 | if (null === $message->payload) { 154 | throw new NullMessageException('Null payload received in kafka message.'); 155 | } 156 | } 157 | 158 | private function dispatch(string $eventClass, ConsumerInterface $consumer): void 159 | { 160 | if (!$this->dispatcher) { 161 | return; 162 | } 163 | 164 | switch ($eventClass) { 165 | case PostMessageConsumedEvent::class: 166 | $event = new PostMessageConsumedEvent($this->consumedMessages, $this->consumptionTimeMs); 167 | break; 168 | case PreMessageConsumedEvent::class: 169 | $event = new PreMessageConsumedEvent($this->consumedMessages, $this->consumptionTimeMs); 170 | break; 171 | default: 172 | throw new \RuntimeException(sprintf('Event class %s does not exist', $eventClass)); 173 | } 174 | 175 | $this->dispatcher->dispatch($event, $event::getEventName($consumer->getName())); 176 | $this->dispatcher->dispatch($event); 177 | } 178 | } 179 | -------------------------------------------------------------------------------- /tests/Unit/Configuration/ConfigurationResolverTest.php: -------------------------------------------------------------------------------- 1 | configurationOne = $this->createMock(ConfigurationInterface::class); 30 | $this->configurationTwo = $this->createMock(ConfigurationInterface::class); 31 | $this->configurationThree = $this->createMock(CastValueInterface::class); 32 | $this->input = $this->createMock(Input::class); 33 | $this->yamlConfig = json_decode(file_get_contents(__DIR__ . '/../../config/consumers.json'), true); 34 | } 35 | 36 | public function testDefaultValue(): void 37 | { 38 | $this->configurationOne 39 | ->method('getName') 40 | ->willReturn('configuration_one'); 41 | 42 | $this->configurationOne 43 | ->method('getDefaultValue') 44 | ->willReturn('foo'); 45 | 46 | $this->configurationOne 47 | ->method('supportsClient') 48 | ->willReturn(true); 49 | 50 | $rawConfiguration = (new RawConfiguration()) 51 | ->addConfiguration($this->configurationOne); 52 | 53 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 54 | $resolved = $resolver->resolve(DummyConsumerOne::class); 55 | 56 | $this->assertEquals('foo', $resolved->getValue('configuration_one')); 57 | } 58 | 59 | public function testValueCasted(): void 60 | { 61 | $this->configurationThree 62 | ->method('getName') 63 | ->willReturn('configuration_three'); 64 | 65 | $this->configurationThree 66 | ->method('getDefaultValue') 67 | ->willReturn('1'); 68 | 69 | $this->configurationThree 70 | ->method('cast') 71 | ->willReturn(1); 72 | 73 | $this->configurationThree 74 | ->method('supportsClient') 75 | ->willReturn(true); 76 | 77 | $rawConfiguration = (new RawConfiguration()) 78 | ->addConfiguration($this->configurationThree); 79 | 80 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 81 | $resolved = $resolver->resolve(DummyConsumerOne::class); 82 | 83 | $this->assertEquals(1, $resolved->getValue('configuration_three')); 84 | } 85 | 86 | public function testWrongClientException(): void 87 | { 88 | $this->configurationOne 89 | ->method('supportsClient') 90 | ->willReturn(true); 91 | 92 | $rawConfiguration = (new RawConfiguration()) 93 | ->addConfiguration($this->configurationOne); 94 | 95 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 96 | 97 | $this->expectException(InvalidClientException::class); 98 | 99 | $resolver->resolve('foo'); 100 | } 101 | public function testInputValue(): void 102 | { 103 | $this->configurationOne 104 | ->method('getName') 105 | ->willReturn('configuration_one'); 106 | 107 | $this->configurationOne 108 | ->method('isValueValid') 109 | ->willReturn(true); 110 | 111 | $this->configurationOne 112 | ->method('supportsClient') 113 | ->willReturn(true); 114 | 115 | $this->input 116 | ->method('getParameterOption') 117 | ->willReturn('bar'); 118 | 119 | $this->input 120 | ->method('getOption') 121 | ->willReturn('bar'); 122 | 123 | $rawConfiguration = (new RawConfiguration()) 124 | ->addConfiguration($this->configurationOne); 125 | 126 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 127 | $resolved = $resolver->resolve(DummyConsumerOne::class, $this->input); 128 | 129 | $this->assertEquals('bar', $resolved->getValue('configuration_one')); 130 | } 131 | 132 | public function testInputValueInvalid(): void 133 | { 134 | $this->configurationOne 135 | ->method('getName') 136 | ->willReturn('configuration_one'); 137 | 138 | $this->configurationOne 139 | ->method('isValueValid') 140 | ->willReturn(false); 141 | 142 | $this->configurationOne 143 | ->method('supportsClient') 144 | ->willReturn(true); 145 | 146 | $this->input 147 | ->method('getParameterOption') 148 | ->willReturn('bar'); 149 | 150 | $this->input 151 | ->method('getOption') 152 | ->willReturn('bar'); 153 | 154 | $rawConfiguration = (new RawConfiguration()) 155 | ->addConfiguration($this->configurationOne); 156 | 157 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 158 | 159 | $this->expectException(InvalidConfigurationException::class); 160 | $this->expectExceptionMessageMatches('/configuration_one/'); 161 | 162 | $resolver->resolve(DummyConsumerOne::class, $this->input); 163 | } 164 | 165 | public function testYamlConfig(): void 166 | { 167 | $this->configurationOne 168 | ->method('getName') 169 | ->willReturn('group_id'); 170 | 171 | $this->configurationOne 172 | ->method('isValueValid') 173 | ->willReturn(true); 174 | 175 | $this->configurationOne 176 | ->method('supportsClient') 177 | ->willReturn(true); 178 | 179 | $rawConfiguration = (new RawConfiguration()) 180 | ->addConfiguration($this->configurationOne); 181 | 182 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 183 | $resolved = $resolver->resolve(DummyConsumerOne::class); 184 | 185 | $this->assertEquals('dummy_group_id_one', $resolved->getValue('group_id')); 186 | } 187 | 188 | public function testYamlConfigInvalid(): void 189 | { 190 | $this->configurationOne 191 | ->method('getName') 192 | ->willReturn('group_id'); 193 | 194 | $this->configurationOne 195 | ->method('isValueValid') 196 | ->willReturn(false); 197 | 198 | $this->configurationOne 199 | ->method('supportsClient') 200 | ->willReturn(true); 201 | 202 | $rawConfiguration = (new RawConfiguration()) 203 | ->addConfiguration($this->configurationOne); 204 | 205 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 206 | 207 | $this->expectException(InvalidConfigurationException::class); 208 | $this->expectExceptionMessageMatches('/group_id/'); 209 | 210 | $resolver->resolve(DummyConsumerOne::class); 211 | } 212 | 213 | public function testParentYamlConfig(): void 214 | { 215 | $this->configurationOne 216 | ->method('getName') 217 | ->willReturn('group_id'); 218 | 219 | $this->configurationOne 220 | ->method('isValueValid') 221 | ->willReturn(true); 222 | 223 | $this->configurationOne 224 | ->method('supportsClient') 225 | ->willReturn(true); 226 | 227 | $rawConfiguration = (new RawConfiguration()) 228 | ->addConfiguration($this->configurationOne); 229 | 230 | $resolver = new ConfigurationResolver($rawConfiguration, $this->yamlConfig); 231 | $resolved = $resolver->resolve(DummyConsumerThree::class); 232 | 233 | $this->assertEquals('dummy_group_id_two', $resolved->getValue('group_id')); 234 | } 235 | } 236 | -------------------------------------------------------------------------------- /src/DependencyInjection/Configuration.php: -------------------------------------------------------------------------------- 1 | getRootNode(); 41 | $builder = $rootNode->children(); 42 | $builder 43 | ->append($this->addConsumersNode()) 44 | ->append($this->addProducersNode()); 45 | 46 | return $treeBuilder; 47 | } 48 | 49 | /** 50 | * @return \Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition|\Symfony\Component\Config\Definition\Builder\NodeDefinition 51 | */ 52 | private function addConsumersNode() 53 | { 54 | $consumersTreeBuilder = new TreeBuilder('consumers'); 55 | $consumersNode = $consumersTreeBuilder->getRootNode(); 56 | $consumersBuilder = $consumersNode->children(); 57 | 58 | $instancesTreeBuilder = new TreeBuilder('instances'); 59 | $instancesNode = $instancesTreeBuilder->getRootNode(); 60 | $instancesBuilder = $instancesNode->arrayPrototype()->children(); 61 | $this->addConsumerConfigurations($instancesBuilder); 62 | 63 | $consumersBuilder->append($instancesNode); 64 | 65 | return $consumersNode; 66 | } 67 | 68 | /** 69 | * @return \Symfony\Component\Config\Definition\Builder\ArrayNodeDefinition|\Symfony\Component\Config\Definition\Builder\NodeDefinition 70 | */ 71 | private function addProducersNode() 72 | { 73 | $producersTreeBuilder = new TreeBuilder('producers'); 74 | $producersNode = $producersTreeBuilder->getRootNode(); 75 | $producersBuilder = $producersNode->children(); 76 | 77 | $instancesTreeBuilder = new TreeBuilder('instances'); 78 | $instancesNode = $instancesTreeBuilder->getRootNode(); 79 | $instancesBuilder = $instancesNode->arrayPrototype()->children(); 80 | $this->addProducerConfigurations($instancesBuilder); 81 | 82 | $producersBuilder->append($instancesNode); 83 | 84 | return $producersNode; 85 | } 86 | 87 | private function addProducerConfigurations(NodeBuilder $builder): void 88 | { 89 | $builder 90 | ->integerNode(ProducerPartition::NAME) 91 | ->defaultValue((new ProducerPartition())->getDefaultValue()) 92 | ->end() 93 | ->scalarNode(ProducerTopic::NAME) 94 | ->defaultValue((new ProducerTopic())->getDefaultValue()) 95 | ->cannotBeEmpty() 96 | ->end() 97 | ->integerNode(LogLevel::NAME) 98 | ->defaultValue((new LogLevel)->getDefaultValue()) 99 | ->end() 100 | ->integerNode(StatisticsIntervalMs::NAME) 101 | ->defaultValue((new StatisticsIntervalMs)->getDefaultValue()) 102 | ->end() 103 | ->arrayNode(Brokers::NAME) 104 | ->defaultValue((new Brokers)->getDefaultValue()) 105 | ->cannotBeEmpty() 106 | ->scalarPrototype() 107 | ->cannotBeEmpty() 108 | ->end(); 109 | } 110 | 111 | private function addConsumerConfigurations(NodeBuilder $builder): void 112 | { 113 | $builder 114 | ->scalarNode(SchemaRegistry::NAME) 115 | ->defaultValue((new SchemaRegistry)->getDefaultValue()) 116 | ->cannotBeEmpty() 117 | ->end() 118 | ->arrayNode(Validators::NAME) 119 | ->defaultValue((new Validators)->getDefaultValue()) 120 | ->cannotBeEmpty() 121 | ->scalarPrototype() 122 | ->cannotBeEmpty() 123 | ->end() 124 | ->end() 125 | ->arrayNode(Topics::NAME) 126 | ->defaultValue((new Topics)->getDefaultValue()) 127 | ->cannotBeEmpty() 128 | ->scalarPrototype() 129 | ->cannotBeEmpty() 130 | ->end() 131 | ->end() 132 | ->scalarNode(Decoder::NAME) 133 | ->defaultValue((new Decoder)->getDefaultValue()) 134 | ->cannotBeEmpty() 135 | ->end() 136 | ->scalarNode(Denormalizer::NAME) 137 | ->defaultValue((new Denormalizer)->getDefaultValue()) 138 | ->cannotBeEmpty() 139 | ->end() 140 | ->scalarNode(AutoCommitIntervalMs::NAME) 141 | ->defaultValue((new AutoCommitIntervalMs)->getDefaultValue()) 142 | ->cannotBeEmpty() 143 | ->end() 144 | ->integerNode(StatisticsIntervalMs::NAME) 145 | ->defaultValue((new StatisticsIntervalMs)->getDefaultValue()) 146 | ->end() 147 | ->integerNode(MaxPollIntervalMs::NAME) 148 | ->defaultValue((new MaxPollIntervalMs)->getDefaultValue()) 149 | ->end() 150 | ->scalarNode(AutoOffsetReset::NAME) 151 | ->defaultValue((new AutoOffsetReset)->getDefaultValue()) 152 | ->cannotBeEmpty() 153 | ->end() 154 | ->scalarNode(GroupId::NAME) 155 | ->defaultValue((new GroupId)->getDefaultValue()) 156 | ->cannotBeEmpty() 157 | ->end() 158 | ->integerNode(Timeout::NAME) 159 | ->defaultValue((new Timeout)->getDefaultValue()) 160 | ->end() 161 | ->scalarNode(EnableAutoOffsetStore::NAME) 162 | ->defaultValue((new EnableAutoOffsetStore)->getDefaultValue()) 163 | ->cannotBeEmpty() 164 | ->end() 165 | ->scalarNode(EnableAutoCommit::NAME) 166 | ->defaultValue((new EnableAutoCommit)->getDefaultValue()) 167 | ->cannotBeEmpty() 168 | ->end() 169 | ->booleanNode(RegisterMissingSchemas::NAME) 170 | ->defaultValue((new RegisterMissingSchemas)->getDefaultValue()) 171 | ->end() 172 | ->booleanNode(RegisterMissingSubjects::NAME) 173 | ->defaultValue((new RegisterMissingSubjects)->getDefaultValue()) 174 | ->end() 175 | ->integerNode(MaxRetries::NAME) 176 | ->defaultValue((new MaxRetries)->getDefaultValue()) 177 | ->end() 178 | ->integerNode(RetryDelay::NAME) 179 | ->defaultValue((new RetryDelay)->getDefaultValue()) 180 | ->end() 181 | ->integerNode(RetryMultiplier::NAME) 182 | ->defaultValue((new RetryMultiplier)->getDefaultValue()) 183 | ->end() 184 | ->integerNode(MaxRetryDelay::NAME) 185 | ->defaultValue((new MaxRetryDelay)->getDefaultValue()) 186 | ->end() 187 | ->integerNode(LogLevel::NAME) 188 | ->defaultValue((new LogLevel)->getDefaultValue()) 189 | ->end() 190 | ->arrayNode(Brokers::NAME) 191 | ->defaultValue((new Brokers)->getDefaultValue()) 192 | ->cannotBeEmpty() 193 | ->scalarPrototype() 194 | ->cannotBeEmpty() 195 | ->end(); 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /tests/config/base.xml: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 127.0.0.1:9092 58 | 127.0.0.2:9092 59 | 127.0.0.3:9092 60 | 61 | 62 | dummy_topic_one 63 | dummy_topic_two 64 | 65 | dummy_group_id_one 66 | http://127.0.0.1:8081 67 | 5 68 | 1000 69 | StsGamingGroup\KafkaBundle\Tests\Dummy\Denormalizer\DummyDenormalizerOne 70 | 71 | StsGamingGroup\KafkaBundle\Tests\Dummy\Validator\DummyValidatorOne 72 | 73 | StsGamingGroup\KafkaBundle\Tests\Dummy\Decoder\DummyDecoderOne 74 | 50 75 | smallest 76 | 1000 77 | true 78 | true 79 | false 80 | false 81 | 200 82 | 2 83 | 3 84 | 85 | 86 | 87 | 127.0.0.4:9092 88 | 127.0.0.5:9092 89 | 127.0.0.6:9092 90 | 91 | 92 | dummy_topic_three 93 | dummy_topic_four 94 | 95 | dummy_group_id_two 96 | http://127.0.0.2:8081 97 | 4 98 | 2000 99 | StsGamingGroup\KafkaBundle\Tests\Dummy\Denormalizer\DummyDenormalizerOne 100 | 101 | StsGamingGroup\KafkaBundle\Tests\Dummy\Validator\DummyValidatorOne 102 | 103 | StsGamingGroup\KafkaBundle\Tests\Dummy\Decoder\DummyDecoderOne 104 | 40 105 | largest 106 | 2000 107 | false 108 | false 109 | true 110 | true 111 | 300 112 | 3 113 | 2 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 127.0.0.1:9092 122 | 127.0.0.2:9092 123 | 127.0.0.3:9092 124 | 125 | dummy_topic_one 126 | 1 127 | 3 128 | 129 | 130 | 131 | 127.0.0.4:9092 132 | 127.0.0.5:9092 133 | 127.0.0.6:9092 134 | 135 | dummy_topic_two 136 | 2 137 | 2 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | --------------------------------------------------------------------------------