├── docker
├── custom.ini
└── Dockerfile
├── docs
├── logo.png
├── upgrade.md
├── CONTRIBUTING.md
└── CONTRIBUTING.pt.md
├── .gitignore
├── src
├── Authentication
│ ├── AuthenticationInterface.php
│ ├── NoAuthentication.php
│ ├── SSLAuthentication.php
│ ├── SASLAuthentication.php
│ └── Factory.php
├── Exceptions
│ ├── JsonException.php
│ ├── ConfigurationException.php
│ ├── ResponseErrorException.php
│ ├── AuthenticationException.php
│ ├── ResponseTimeoutException.php
│ └── ResponseWarningException.php
├── Console
│ ├── stubs
│ │ ├── producer.stub
│ │ ├── middleware.stub
│ │ └── consumer.stub
│ ├── ProducerMakeCommand.php
│ ├── MiddlewareMakeCommand.php
│ ├── ConsumerMakeCommand.php
│ ├── ConfigOptionsCommand.php
│ └── ConsumerCommand.php
├── Avro
│ ├── Serializer
│ │ ├── SchemaFormats.php
│ │ ├── Decoders
│ │ │ ├── DecoderInterface.php
│ │ │ ├── SchemaId.php
│ │ │ └── SchemaSubjectAndVersion.php
│ │ ├── Encoders
│ │ │ ├── EncoderInterface.php
│ │ │ ├── SchemaSubjectAndVersion.php
│ │ │ └── SchemaId.php
│ │ ├── MessageDecoder.php
│ │ └── MessageEncoder.php
│ ├── ClientFactory.php
│ ├── Schema.php
│ ├── Client.php
│ └── CachedSchemaRegistryClient.php
├── Consumers
│ ├── ConsumerInterface.php
│ ├── Runner.php
│ ├── HighLevel.php
│ └── LowLevel.php
├── TopicHandler
│ ├── ConfigOptions
│ │ ├── Auth
│ │ │ ├── AuthInterface.php
│ │ │ ├── EnumType.php
│ │ │ ├── None.php
│ │ │ ├── Ssl.php
│ │ │ └── SaslSsl.php
│ │ ├── Factories
│ │ │ ├── BrokerFactory.php
│ │ │ ├── AvroSchemaFactory.php
│ │ │ ├── AuthFactory.php
│ │ │ ├── ProducerFactory.php
│ │ │ └── ConsumerFactory.php
│ │ ├── Broker.php
│ │ ├── AvroSchema.php
│ │ ├── Producer.php
│ │ └── Consumer.php
│ ├── Producer
│ │ ├── HandleableResponseInterface.php
│ │ ├── HandlerInterface.php
│ │ ├── AbstractProducer.php
│ │ └── AbstractHandler.php
│ └── Consumer
│ │ ├── Handler.php
│ │ └── AbstractHandler.php
├── Middlewares
│ ├── Handler
│ │ ├── MiddlewareHandlerInterface.php
│ │ ├── Dispatcher.php
│ │ ├── AbstractMiddlewareHandler.php
│ │ ├── Iterator.php
│ │ ├── Consumer.php
│ │ └── Producer.php
│ ├── MiddlewareInterface.php
│ ├── JsonDecode.php
│ ├── Log.php
│ ├── AvroSchemaDecoder.php
│ └── AvroSchemaMixedEncoder.php
├── Facades
│ └── Metamorphosis.php
├── Connectors
│ ├── Consumer
│ │ ├── ConnectorInterface.php
│ │ ├── HighLevel.php
│ │ ├── LowLevel.php
│ │ ├── Factory.php
│ │ ├── Manager.php
│ │ └── Config.php
│ ├── Producer
│ │ ├── Config.php
│ │ └── Connector.php
│ └── AbstractConfig.php
├── Consumer.php
├── Record
│ ├── RecordInterface.php
│ ├── ProducerRecord.php
│ └── ConsumerRecord.php
├── MetamorphosisServiceProvider.php
├── Producer
│ └── Poll.php
└── Producer.php
├── tests
├── Unit
│ ├── Dummies
│ │ ├── MiddlewareDummy.php
│ │ ├── ConsumerHandlerDummy.php
│ │ ├── SecondProducerHandlerDummy.php
│ │ └── ProducerHandlerDummy.php
│ ├── Facades
│ │ └── MetamorphosisTest.php
│ ├── Console
│ │ ├── ConsumerMakeCommandTest.php
│ │ ├── MiddlewareMakeCommandTest.php
│ │ ├── ProducerMakeCommandTest.php
│ │ └── ConsumerCommandTest.php
│ ├── Authentication
│ │ ├── SASLAuthenticationTest.php
│ │ ├── SSLAuthenticationTest.php
│ │ └── FactoryTest.php
│ ├── Middlewares
│ │ ├── Handler
│ │ │ ├── DispatcherTest.php
│ │ │ ├── IteratorTest.php
│ │ │ ├── ConsumerTest.php
│ │ │ └── ProducerTest.php
│ │ ├── JsonDecodeTest.php
│ │ ├── LogTest.php
│ │ ├── AvroSchemaDecoderTest.php
│ │ └── AvroSchemaMixedEncoderTest.php
│ ├── TopicHandler
│ │ ├── ConfigOptions
│ │ │ ├── Factories
│ │ │ │ ├── BrokerFactoryTest.php
│ │ │ │ ├── AvroSchemaFactoryTest.php
│ │ │ │ ├── ProducerFactoryTest.php
│ │ │ │ └── ConsumerFactoryTest.php
│ │ │ ├── ProducerTest.php
│ │ │ └── ConsumerTest.php
│ │ ├── Consumer
│ │ │ └── AbstractHandlerTest.php
│ │ └── Producer
│ │ │ └── AbstractHandlerTest.php
│ ├── Connectors
│ │ ├── Consumer
│ │ │ ├── LowLevelTest.php
│ │ │ └── HighLevelTest.php
│ │ └── Producer
│ │ │ └── ConnectorTest.php
│ ├── Consumers
│ │ ├── LowLevelTest.php
│ │ ├── HighLevelTest.php
│ │ └── RunnerTest.php
│ ├── Record
│ │ └── ProducerRecordTest.php
│ ├── Avro
│ │ └── Serializer
│ │ │ └── MessageDecoderTest.php
│ └── Producer
│ │ └── PollTest.php
├── LaravelTestCase.php
└── Integration
│ ├── Dummies
│ ├── MessageConsumer.php
│ ├── MessageProducer.php
│ └── MessageProducerWithConfigOptions.php
│ ├── ConsumerTest.php
│ └── ProducerWithConfigOptionsTest.php
├── rector.php
├── .editorconfig
├── psalm.xml
├── docker-compose.yml
├── psalm-baseline.xml
├── grumphp.yml
├── config
├── service.php
└── kafka.php
├── CHANGELOG.md
├── phpunit.xml.dist
├── LICENSE
├── phpcs.xml
├── composer.json
└── .github
└── workflows
└── continuous-integration.yml
/docker/custom.ini:
--------------------------------------------------------------------------------
1 | xdebug.mode=coverage
2 |
--------------------------------------------------------------------------------
/docs/logo.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/leroy-merlin-br/metamorphosis/HEAD/docs/logo.png
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | build/
2 | vendor/
3 | coverage/
4 | storage/
5 | .idea/
6 | composer.lock
7 | .phpcs-cache
8 | .phpunit.result.cache
9 |
--------------------------------------------------------------------------------
/src/Authentication/AuthenticationInterface.php:
--------------------------------------------------------------------------------
1 | paths([__DIR__ . '/src', __DIR__ . '/tests']);
9 |
10 | $rectorConfig->sets([
11 | SetList::CODE_QUALITY,
12 | ]);
13 |
14 | $rectorConfig->phpVersion(PhpVersion::PHP_74);
15 | };
16 |
--------------------------------------------------------------------------------
/src/Middlewares/Handler/Dispatcher.php:
--------------------------------------------------------------------------------
1 | queue);
12 | $iterator = new Iterator($this->queue);
13 |
14 | return $iterator->handle($record);
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Factories/BrokerFactory.php:
--------------------------------------------------------------------------------
1 | handle($record);
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/tests/Unit/Dummies/SecondProducerHandlerDummy.php:
--------------------------------------------------------------------------------
1 | record = $record;
12 | $this->topic = $topic;
13 | $this->key = $key;
14 | $this->partition = $partition;
15 | }
16 | }
17 |
--------------------------------------------------------------------------------
/tests/Unit/Facades/MetamorphosisTest.php:
--------------------------------------------------------------------------------
1 | assertInstanceOf(Producer::class, $producer);
18 | }
19 | }
20 |
--------------------------------------------------------------------------------
/.editorconfig:
--------------------------------------------------------------------------------
1 | #
2 | # EditorConfig help us maintain consistent coding style between different editors.
3 | #
4 | # EditorConfig
5 | # http://editorconfig.org
6 | #
7 | root = true
8 |
9 | [*]
10 | indent_style = space
11 | indent_size = 4
12 | end_of_line = lf
13 | charset = utf-8
14 | trim_trailing_whitespace = true
15 | insert_final_newline = true
16 |
17 | [*.md]
18 | trim_trailing_whitespace = false
19 |
20 | [*.yml]
21 | indent_style = space
22 | indent_size = 2
23 | end_of_line = lf
24 | charset = utf-8
25 | trim_trailing_whitespace = true
26 | insert_final_newline = true
27 |
--------------------------------------------------------------------------------
/src/Console/stubs/consumer.stub:
--------------------------------------------------------------------------------
1 |
11 | */
12 | protected iterable $queue;
13 |
14 | /**
15 | * Handles the current entry in the middleware queue and advances.
16 | */
17 | abstract public function handle(RecordInterface $record);
18 |
19 | public function __construct(iterable $queue)
20 | {
21 | $this->queue = $queue;
22 | }
23 | }
24 |
--------------------------------------------------------------------------------
/src/Avro/Serializer/Encoders/EncoderInterface.php:
--------------------------------------------------------------------------------
1 |
2 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
1 | version: '3.1'
2 |
3 | services:
4 | php:
5 | build: docker
6 | volumes:
7 | - .:/var/www/html
8 | depends_on:
9 | - kafka
10 |
11 | zookeeper:
12 | image: bitnami/zookeeper
13 | environment:
14 | - ALLOW_ANONYMOUS_LOGIN=yes
15 |
16 | kafka:
17 | image: bitnami/kafka
18 | depends_on:
19 | - zookeeper
20 | environment:
21 | - KAFKA_LISTENERS=PLAINTEXT://:9092
22 | - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
23 | - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
24 | - ALLOW_PLAINTEXT_LISTENER=yes
25 |
--------------------------------------------------------------------------------
/src/Consumers/Runner.php:
--------------------------------------------------------------------------------
1 | manager = $manager;
14 | }
15 |
16 | public function run(?int $times = null): void
17 | {
18 | if ($times) {
19 | for ($i = 0; $i < $times; $i++) {
20 | $this->manager->handleMessage();
21 | }
22 |
23 | return;
24 | }
25 |
26 | while (true) {
27 | $this->manager->handleMessage();
28 | }
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/tests/Unit/Console/ConsumerMakeCommandTest.php:
--------------------------------------------------------------------------------
1 | Str::random(8),
17 | ];
18 |
19 | // Actions
20 | $statusCode = Artisan::call($command, $parameters);
21 |
22 | // Assertions
23 | $this->assertSame(0, $statusCode);
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/tests/Unit/Console/MiddlewareMakeCommandTest.php:
--------------------------------------------------------------------------------
1 | Str::random(8),
17 | ];
18 |
19 | // Actions
20 | $statusCode = Artisan::call($command, $parameters);
21 |
22 | // Assertions
23 | $this->assertSame(0, $statusCode);
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/tests/Unit/Console/ProducerMakeCommandTest.php:
--------------------------------------------------------------------------------
1 | Str::random(8),
17 | ];
18 |
19 | // Actions
20 | $statusCode = Artisan::call($command, $parameters);
21 |
22 | // Assertions
23 | $this->assertSame(0, $statusCode);
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/Middlewares/JsonDecode.php:
--------------------------------------------------------------------------------
1 | getPayload(), true);
14 |
15 | if (null === $payload && JSON_ERROR_NONE !== json_last_error()) {
16 | throw new Exception(
17 | 'Malformed JSON. Error: ' . json_last_error_msg()
18 | );
19 | }
20 |
21 | $record->setPayload($payload);
22 |
23 | return $next($record);
24 | }
25 | }
26 |
--------------------------------------------------------------------------------
/src/Middlewares/Handler/Iterator.php:
--------------------------------------------------------------------------------
1 | queue);
15 | $middleware = $entry;
16 | next($this->queue);
17 |
18 | if ($middleware instanceof MiddlewareInterface) {
19 | return $middleware->process($record, $closure);
20 | }
21 |
22 | return $record;
23 | }
24 | }
25 |
--------------------------------------------------------------------------------
/src/Middlewares/Log.php:
--------------------------------------------------------------------------------
1 | log = $log;
16 | }
17 |
18 | public function process(RecordInterface $record, Closure $next)
19 | {
20 | $this->log->info(
21 | 'Processing kafka record: ' . $record->getPayload(),
22 | [
23 | 'original' => (array) $record->getOriginal(),
24 | ]
25 | );
26 |
27 | return $next($record);
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/src/TopicHandler/Consumer/Handler.php:
--------------------------------------------------------------------------------
1 | registry = $registry;
17 | }
18 |
19 | public function decode(AvroStringIO $io)
20 | {
21 | $id = unpack('N', $io->read(4));
22 | $id = $id[1];
23 |
24 | $schema = $this->registry->getById($id);
25 | $reader = new AvroIODatumReader($schema->getAvroSchema());
26 |
27 | return $reader->read(new AvroIOBinaryDecoder($io));
28 | }
29 | }
30 |
--------------------------------------------------------------------------------
/tests/LaravelTestCase.php:
--------------------------------------------------------------------------------
1 | app->bind(
24 | $abstract,
25 | function () use ($instance) {
26 | return $instance;
27 | }
28 | );
29 |
30 | return $instance;
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/tests/Unit/Dummies/ProducerHandlerDummy.php:
--------------------------------------------------------------------------------
1 | record = $record;
14 | $this->topic = $topic;
15 | $this->key = $key;
16 | $this->partition = $partition;
17 | }
18 |
19 | public function success(Message $message): void
20 | {
21 | dump('success!');
22 | }
23 |
24 | public function failed(Message $message): void
25 | {
26 | dump('failed!');
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/psalm-baseline.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 | consumeStart
6 |
7 |
8 |
9 |
10 | mixed
11 |
12 |
13 |
14 |
15 | $this->queue
16 |
17 |
18 |
19 |
20 | $this->queue
21 | $this->queue
22 |
23 |
24 |
25 |
--------------------------------------------------------------------------------
/src/Middlewares/Handler/Consumer.php:
--------------------------------------------------------------------------------
1 | consumerTopicHandler = $consumerTopicHandler;
17 | }
18 |
19 | /**
20 | * @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter
21 | * @return mixed
22 | */
23 | public function process(RecordInterface $record, Closure $next)
24 | {
25 | $this->consumerTopicHandler->handle($record);
26 | }
27 | }
28 |
--------------------------------------------------------------------------------
/src/Consumers/HighLevel.php:
--------------------------------------------------------------------------------
1 | consumer = $consumer;
17 |
18 | $this->timeout = $timeout;
19 | }
20 |
21 | public function consume(): ?Message
22 | {
23 | return $this->consumer->consume($this->timeout);
24 | }
25 |
26 | public function commit(): void
27 | {
28 | $this->consumer->commit();
29 | }
30 |
31 | public function commitAsync(): void
32 | {
33 | $this->consumer->commitAsync();
34 | }
35 |
36 | public function canCommit(): bool
37 | {
38 | return true;
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/TopicHandler/Producer/HandlerInterface.php:
--------------------------------------------------------------------------------
1 | [
5 | 'url' => '',
6 | 'request_options' => [
7 | 'headers' => [
8 | 'Authorization' => [
9 | 'Basic' . base64_encode(
10 | env('AVRO_SCHEMA_USERNAME') . ':' . env(
11 | 'AVRO_SCHEMA_PASSWORD'
12 | )
13 | ),
14 | ],
15 | ],
16 | ],
17 | 'ssl_verify' => true,
18 | 'username' => 'USERNAME',
19 | 'password' => 'PASSWORD',
20 | ],
21 | 'broker' => [
22 | 'connections' => env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092'),
23 | 'auth' => [
24 | 'type' => 'ssl', // ssl and none
25 | 'ca' => storage_path('ca.pem'),
26 | 'certificate' => storage_path('kafka.cert'),
27 | 'key' => storage_path('kafka.key'),
28 | ],
29 | ],
30 | ];
31 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Factories/AvroSchemaFactory.php:
--------------------------------------------------------------------------------
1 | conf = $conf;
17 | $this->configOptions = $configOptions;
18 |
19 | $this->authenticate();
20 | }
21 |
22 | private function authenticate(): void
23 | {
24 | $this->conf->set('security.protocol', $this->configOptions->getType());
25 | $this->conf->set('ssl.ca.location', $this->configOptions->getCa());
26 | $this->conf->set(
27 | 'ssl.certificate.location',
28 | $this->configOptions->getCertificate()
29 | );
30 | $this->conf->set('ssl.key.location', $this->configOptions->getKey());
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | All notable changes to this project will be documented in this file.
4 |
5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
7 |
8 | ### Added
9 |
10 | - Added AvroSchemaMixedEncoderTest
11 | - Added AvroSchemaDecoderTest
12 | - Added ProducerWithConfigOptionsTest
13 | - Added ConfigOptionsCommand to run commands with ConfigOptions class
14 | - Added pt_BR contributing section
15 | - Added setup-dev script on composer
16 | - Added grumphp commit validation
17 |
18 | ### Fixed
19 |
20 | - Fixed parameters and options override on Consumer\Config class
21 | - Update instructions in the contributions section
22 | - Update project install section
23 |
24 | ### Changed
25 |
26 | - Updated ConfigManager class for ConfigOptions in unit tests and wherever any configuration requests are made
27 | - Consumer and Producer middlewares resolution
28 |
--------------------------------------------------------------------------------
/phpunit.xml.dist:
--------------------------------------------------------------------------------
1 |
2 |
7 |
8 |
9 | src/
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 | tests
19 |
20 |
21 |
22 |
23 |
--------------------------------------------------------------------------------
/tests/Integration/Dummies/MessageConsumer.php:
--------------------------------------------------------------------------------
1 | getPayload();
16 |
17 | Log::alert($priceUpdate);
18 | }
19 |
20 | public function warning(ResponseWarningException $exception): void
21 | {
22 | Log::debug('Something happened while handling kafka consumer.', [
23 | 'exception' => $exception,
24 | ]);
25 | }
26 |
27 | public function failed(Exception $exception): void
28 | {
29 | Log::error('Failed to handle kafka record for sku.', [
30 | 'exception' => $exception,
31 | ]);
32 | }
33 | }
34 |
--------------------------------------------------------------------------------
/src/Connectors/Producer/Config.php:
--------------------------------------------------------------------------------
1 | 'required',
14 | 'connections' => 'required|string',
15 | 'timeout' => 'int',
16 | 'is_async' => 'boolean',
17 | 'required_acknowledgment' => 'boolean',
18 | 'max_poll_records' => 'int',
19 | 'flush_attempts' => 'int',
20 | 'auth' => 'nullable|array',
21 | 'middlewares' => 'array',
22 | 'ssl_verify' => 'boolean',
23 | ];
24 |
25 | /**
26 | * @var mixed[]
27 | */
28 | protected array $default = [
29 | 'timeout' => 1000,
30 | 'is_async' => true,
31 | 'required_acknowledgment' => true,
32 | 'max_poll_records' => 500,
33 | 'flush_attempts' => 10,
34 | 'partition' => RD_KAFKA_PARTITION_UA,
35 | 'ssl_verify' => false,
36 | ];
37 | }
38 |
--------------------------------------------------------------------------------
/tests/Unit/Authentication/SASLAuthenticationTest.php:
--------------------------------------------------------------------------------
1 | 'sasl_ssl',
24 | 'sasl.username' => 'some-username',
25 | 'sasl.password' => 'some-password',
26 | 'sasl.mechanisms' => 'PLAIN',
27 | ];
28 |
29 | // Actions
30 | new SASLAuthentication($conf, $configSaslSsl);
31 |
32 | // Assertions
33 | $this->assertArraySubset($expected, $conf->dump());
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/tests/Unit/Authentication/SSLAuthenticationTest.php:
--------------------------------------------------------------------------------
1 | 'ssl',
23 | 'ssl.ca.location' => 'path/to/ca',
24 | 'ssl.certificate.location' => 'path/to/certificate',
25 | 'ssl.key.location' => 'path/to/key',
26 | ];
27 |
28 | // Actions
29 | new SSLAuthentication($conf, $configSsl);
30 |
31 | // Assertions
32 | $this->assertArraySubset($expected, $conf->dump());
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/Consumer.php:
--------------------------------------------------------------------------------
1 | consumer = Factory::getConsumer(true, $configOptions);
21 | $this->dispatcher = new Dispatcher($configOptions->getMiddlewares());
22 | }
23 |
24 | public function consume(): ?RecordInterface
25 | {
26 | if ($response = $this->consumer->consume()) {
27 | $record = app(ConsumerRecord::class, compact('response'));
28 |
29 | return $this->dispatcher->handle($record);
30 | }
31 |
32 | return null;
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/Consumers/LowLevel.php:
--------------------------------------------------------------------------------
1 | consumer = $consumer;
20 |
21 | $this->partition = $consumerConfigOptions->getPartition();
22 | $this->timeout = $consumerConfigOptions->getTimeout();
23 | }
24 |
25 | public function consume(): ?Message
26 | {
27 | return $this->consumer->consume($this->partition, $this->timeout);
28 | }
29 |
30 | /**
31 | * When running low level consumer, we dont need
32 | * to commit the messages as they've already been committed.
33 | */
34 | public function canCommit(): bool
35 | {
36 | return false;
37 | }
38 | }
39 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/Handler/DispatcherTest.php:
--------------------------------------------------------------------------------
1 | payload = 'original message';
24 | $kafkaMessage->err = RD_KAFKA_RESP_ERR_NO_ERROR;
25 |
26 | $record = new Record($kafkaMessage);
27 |
28 | // Expectations
29 | $middleware->expects('process')
30 | ->withSomeOfArgs($record);
31 |
32 | // Actions
33 | $dispatcher->handle($record);
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/tests/Integration/Dummies/MessageProducer.php:
--------------------------------------------------------------------------------
1 | $message->topic_name,
16 | 'payload' => $message->payload,
17 | 'key' => $message->key,
18 | 'partition' => $message->partition,
19 | ]);
20 | }
21 |
22 | public function failed(Message $message): void
23 | {
24 | Log::error('Unable to delivery record to broker.', [
25 | 'topic' => $message->topic_name,
26 | 'payload' => $message->payload,
27 | 'key' => $message->key,
28 | 'partition' => $message->partition,
29 | 'error' => $message->err,
30 | ]);
31 |
32 | throw new RuntimeException('error sending message!');
33 | }
34 | }
35 |
--------------------------------------------------------------------------------
/src/Avro/Serializer/Decoders/SchemaSubjectAndVersion.php:
--------------------------------------------------------------------------------
1 | registry = $registry;
17 | }
18 |
19 | public function decode(AvroStringIO $io)
20 | {
21 | $size = $io->read(4);
22 | $subjectSize = unpack('N', $size);
23 | $subjectBytes = unpack('C*', $io->read($subjectSize[1]));
24 | $subject = implode(array_map('chr', $subjectBytes));
25 |
26 | $version = unpack('N', $io->read(4));
27 | $version = $version[1];
28 |
29 | $schema = $this->registry->getBySubjectAndVersion($subject, $version);
30 |
31 | $reader = new AvroIODatumReader($schema->getAvroSchema());
32 |
33 | return $reader->read(new AvroIOBinaryDecoder($io));
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 Leroy Merlin Brasil
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Factories/AuthFactory.php:
--------------------------------------------------------------------------------
1 | SaslSsl::class,
16 | EnumType::SSL_TYPE => Ssl::class,
17 | EnumType::NONE_TYPE => None::class,
18 | ];
19 |
20 | public static function make(array $attributes = []): AuthInterface
21 | {
22 | if (!$attributes) {
23 | $attributes['type'] = EnumType::NONE_TYPE;
24 | }
25 |
26 | if (!isset(self::AUTH_MAP[$attributes['type']])) {
27 | throw new Exception('Invalid Auth Type on Broker Authentication.');
28 | }
29 |
30 | return app(self::AUTH_MAP[$attributes['type']], $attributes);
31 | }
32 | }
33 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/Handler/IteratorTest.php:
--------------------------------------------------------------------------------
1 | payload = 'original message';
25 | $kafkaMessage->err = RD_KAFKA_RESP_ERR_NO_ERROR;
26 |
27 | $record = new Record($kafkaMessage);
28 |
29 | // Expectations
30 | $middleware->expects()
31 | ->process($record, m::type(Closure::class));
32 |
33 | // Actions
34 | $iterator->handle($record);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/ConfigOptions/Factories/BrokerFactoryTest.php:
--------------------------------------------------------------------------------
1 | 'kafka:9092',
16 | 'auth' => [
17 | 'type' => 'ssl',
18 | 'ca' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/ca.pem',
19 | 'certificate' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.cert',
20 | 'key' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.key',
21 | ],
22 | ];
23 | // Actions
24 | $result = BrokerFactory::make($data);
25 |
26 | // Assertions
27 | $this->assertInstanceOf(Broker::class, $result);
28 | $this->assertEquals($data, $result->toArray());
29 | }
30 | }
31 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Auth/Ssl.php:
--------------------------------------------------------------------------------
1 | ca = $ca;
16 | $this->certificate = $certificate;
17 | $this->key = $key;
18 | }
19 |
20 | public function toArray(): array
21 | {
22 | return [
23 | 'type' => $this->getType(),
24 | 'ca' => $this->getCa(),
25 | 'certificate' => $this->getCertificate(),
26 | 'key' => $this->getKey(),
27 | ];
28 | }
29 |
30 | public function getCa(): string
31 | {
32 | return $this->ca;
33 | }
34 |
35 | public function getCertificate(): string
36 | {
37 | return $this->certificate;
38 | }
39 |
40 | public function getKey(): string
41 | {
42 | return $this->key;
43 | }
44 |
45 | public function getType(): string
46 | {
47 | return EnumType::SSL_TYPE;
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/Handler/ConsumerTest.php:
--------------------------------------------------------------------------------
1 | payload = 'original message';
24 | $kafkaMessage->err = RD_KAFKA_RESP_ERR_NO_ERROR;
25 |
26 | $record = new Record($kafkaMessage);
27 | $consumer = new Consumer($consumerTopicHandler);
28 |
29 | // Expectations
30 | $consumerTopicHandler->expects()
31 | ->handle($record);
32 |
33 | // Actions
34 | $consumer->process($record, $closure);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Broker.php:
--------------------------------------------------------------------------------
1 | connections = $connections;
26 | $this->auth = $auth;
27 | }
28 |
29 | public function getConnections(): string
30 | {
31 | return $this->connections;
32 | }
33 |
34 | public function getAuth(): AuthInterface
35 | {
36 | return $this->auth;
37 | }
38 |
39 | public function toArray(): array
40 | {
41 | return [
42 | 'connections' => $this->getConnections(),
43 | 'auth' => $this->getAuth()->toArray() ?: null,
44 | ];
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/Connectors/AbstractConfig.php:
--------------------------------------------------------------------------------
1 | rules);
38 |
39 | if (!$validator->errors()->isEmpty()) {
40 | throw new ConfigurationException($validator->errors()->toJson());
41 | }
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/src/TopicHandler/Consumer/AbstractHandler.php:
--------------------------------------------------------------------------------
1 | configOptions = $configOptions;
20 | }
21 |
22 | /**
23 | * @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter
24 | */
25 | public function warning(ResponseWarningException $exception): void
26 | {
27 | }
28 |
29 | /**
30 | * @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter
31 | */
32 | public function failed(Exception $exception): void
33 | {
34 | }
35 |
36 | public function finished(): void
37 | {
38 | }
39 |
40 | public function getConfigOptions(): ?ConsumerConfigOptions
41 | {
42 | return $this->configOptions;
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/tests/Integration/Dummies/MessageProducerWithConfigOptions.php:
--------------------------------------------------------------------------------
1 | $message->topic_name,
17 | 'payload' => $message->payload,
18 | 'key' => $message->key,
19 | 'partition' => $message->partition,
20 | ]);
21 | }
22 |
23 | public function failed(Message $message): void
24 | {
25 | Log::error('Unable to delivery record to broker.', [
26 | 'topic' => $message->topic_name,
27 | 'payload' => $message->payload,
28 | 'key' => $message->key,
29 | 'partition' => $message->partition,
30 | 'error' => $message->err,
31 | ]);
32 |
33 | throw new RuntimeException('error sending message!');
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/src/Authentication/SASLAuthentication.php:
--------------------------------------------------------------------------------
1 | conf = $conf;
17 | $this->configOptions = $configOptions;
18 |
19 | $this->authenticate();
20 | }
21 |
22 | private function authenticate(): void
23 | {
24 | $this->conf->set('security.protocol', $this->configOptions->getType());
25 |
26 | // The mechanisms key is optional when configuring this kind of authentication
27 | // If the user does not specify the mechanism, the default will be 'PLAIN'.
28 | // But, to make config more clear, we are asking the user every time.
29 | $this->conf->set(
30 | 'sasl.mechanisms',
31 | $this->configOptions->getMechanisms()
32 | );
33 | $this->conf->set('sasl.username', $this->configOptions->getUsername());
34 | $this->conf->set('sasl.password', $this->configOptions->getPassword());
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/Middlewares/AvroSchemaDecoder.php:
--------------------------------------------------------------------------------
1 | getAvroSchema()->getUrl()) {
19 | throw new ConfigurationException(
20 | "Avro schema url not found, it's required to use AvroSchemaDecoder Middleware"
21 | );
22 | }
23 |
24 | $this->decoder = new MessageDecoder(
25 | $factory->make($consumerConfigOptions->getAvroSchema())
26 | );
27 | }
28 |
29 | public function process(RecordInterface $record, Closure $next)
30 | {
31 | $record->setPayload(
32 | $this->decoder->decodeMessage($record->getPayload())
33 | );
34 |
35 | return $next($record);
36 | }
37 | }
38 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Auth/SaslSsl.php:
--------------------------------------------------------------------------------
1 | mechanisms = $mechanisms;
16 | $this->username = $username;
17 | $this->password = $password;
18 | }
19 |
20 | public function getPassword(): string
21 | {
22 | return $this->password;
23 | }
24 |
25 | public function getUsername(): string
26 | {
27 | return $this->username;
28 | }
29 |
30 | public function getMechanisms(): string
31 | {
32 | return $this->mechanisms;
33 | }
34 |
35 | public function toArray(): array
36 | {
37 | return [
38 | 'type' => $this->getType(),
39 | 'mechanisms' => $this->getMechanisms(),
40 | 'username' => $this->getUsername(),
41 | 'password' => $this->getPassword(),
42 | ];
43 | }
44 |
45 | public function getType(): string
46 | {
47 | return EnumType::SASL_SSL_TYPE;
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/Handler/ProducerTest.php:
--------------------------------------------------------------------------------
1 | 'original record']);
24 | $record = new ProducerRecord($record, 'topic_key');
25 |
26 | // Expectations
27 | $poll->expects()
28 | ->handleResponse();
29 |
30 | $poll->expects()
31 | ->flushMessage();
32 |
33 | $producerTopic->expects()
34 | ->produce(1, 0, $record->getPayload(), null);
35 |
36 | // Actions
37 | $producerHandler = new Producer($producerTopic, $poll, 1);
38 | $producerHandler->process($record, $closure);
39 | }
40 | }
41 |
--------------------------------------------------------------------------------
/src/Record/RecordInterface.php:
--------------------------------------------------------------------------------
1 | getOriginal()->payload.
11 | *
12 | * @param mixed $payload
13 | */
14 | public function setPayload($payload): void;
15 |
16 | /**
17 | * Get the record payload.
18 | * It can either be the original value sent to Kafka or
19 | * a version modified by a middleware.
20 | *
21 | * @return mixed
22 | */
23 | public function getPayload();
24 |
25 | /**
26 | * Get the topic name where the record was published.
27 | */
28 | public function getTopicName(): string;
29 |
30 | /**
31 | * Get the partition number where the record was published.
32 | *
33 | * @return int|null
34 | */
35 | public function getPartition();
36 |
37 | /**
38 | * Get the record key.
39 | *
40 | * @return string|null
41 | */
42 | public function getKey();
43 |
44 | /**
45 | * Get original record when manipulating the topic.
46 | * With this object, it is possible to get original payload.
47 | */
48 | public function getOriginal();
49 | }
50 |
--------------------------------------------------------------------------------
/tests/Unit/Connectors/Consumer/LowLevelTest.php:
--------------------------------------------------------------------------------
1 | getConsumer(true, $consumerConfigOptions);
32 |
33 | // Assertions
34 | $this->assertInstanceOf(LowLevelConsumer::class, $result);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/tests/Unit/Connectors/Consumer/HighLevelTest.php:
--------------------------------------------------------------------------------
1 | getConsumer(false, $consumerConfigOptions);
32 |
33 | // Assertions
34 | $this->assertInstanceOf(HighLevelConsumer::class, $result);
35 | }
36 | }
37 |
--------------------------------------------------------------------------------
/src/MetamorphosisServiceProvider.php:
--------------------------------------------------------------------------------
1 | publishes([
17 | __DIR__ . '/../config/kafka.php' => config_path('kafka.php'),
18 | __DIR__ . '/../config/service.php' => config_path('service.php'),
19 | ], 'config');
20 |
21 | $this->mergeConfigFrom(__DIR__ . '/../config/kafka.php', 'kafka');
22 | $this->mergeConfigFrom(__DIR__ . '/../config/service.php', 'service');
23 | }
24 |
25 | public function register()
26 | {
27 | $this->commands([
28 | ConsumerCommand::class,
29 | ConsumerMakeCommand::class,
30 | MiddlewareMakeCommand::class,
31 | ProducerMakeCommand::class,
32 | ConfigOptionsCommand::class,
33 | ]);
34 |
35 | $this->app->bind('metamorphosis', function ($app) {
36 | return $app->make(Producer::class);
37 | });
38 | }
39 | }
40 |
--------------------------------------------------------------------------------
/docker/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM php:8.0-fpm
2 | LABEL maintainer="boitata@leroymerlin.com.br"
3 |
4 | USER root:root
5 |
6 | COPY --from=composer:latest /usr/bin/composer /usr/local/bin/composer
7 |
8 | RUN apt-get update -qq \
9 | && apt-get install -qq --no-install-recommends \
10 | git zip unzip \
11 | libzip-dev libssl-dev \
12 | zlib1g-dev libicu-dev \
13 | && apt-get clean
14 |
15 | RUN pecl install xdebug-3.1.6 \
16 | && docker-php-ext-enable \
17 | xdebug \
18 | && docker-php-ext-configure \
19 | intl \
20 | && docker-php-ext-install \
21 | intl pcntl zip \
22 | && rm -rf /tmp/*
23 |
24 | RUN cd /tmp \
25 | && git clone https://github.com/edenhill/librdkafka.git \
26 | && cd librdkafka \
27 | && ./configure \
28 | && make \
29 | && make install \
30 | && rm -rf /tmp/*
31 |
32 | RUN pecl install rdkafka-6.0.3 \
33 | && docker-php-ext-enable \
34 | rdkafka \
35 | xdebug \
36 | && rm -rf /tmp/*
37 |
38 | ARG UID=1000
39 | ARG GID=1000
40 |
41 | RUN groupmod -g ${GID} www-data \
42 | && usermod -u ${UID} -g www-data www-data \
43 | && mkdir -p /var/www/html \
44 | && chown -hR www-data:www-data \
45 | /var/www \
46 | /usr/local/
47 |
48 | COPY custom.ini /usr/local/etc/php/conf.d/custom.ini
49 |
50 | USER www-data:www-data
51 | WORKDIR /var/www/html
52 | ENV PATH=$PATH:/var/www/.composer/vendor/bin
53 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/ConfigOptions/Factories/AvroSchemaFactoryTest.php:
--------------------------------------------------------------------------------
1 | 'http://avroschema',
16 | 'ssl_verify' => true,
17 | 'request_options' => [
18 | 'headers' => [
19 | 'Authorization' => [
20 | 'Basic Og==',
21 | ],
22 | ],
23 | ],
24 | ];
25 |
26 | // Actions
27 | $result = AvroSchemaFactory::make($data);
28 |
29 | // Assertions
30 | $this->assertInstanceOf(AvroSchema::class, $result);
31 | $this->assertEquals($data, $result->toArray());
32 | }
33 |
34 | public function testShouldNotMakeAvroSchemaWhenDataIsEmpty(): void
35 | {
36 | // Set
37 | $data = [];
38 |
39 | // Actions
40 | $result = AvroSchemaFactory::make($data);
41 |
42 | // Assertions
43 | $this->assertEmpty($result);
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/src/Authentication/Factory.php:
--------------------------------------------------------------------------------
1 | getType();
20 | switch ($type) {
21 | case null:
22 | case self::TYPE_NONE:
23 | app(NoAuthentication::class);
24 |
25 | break;
26 | case self::TYPE_SSL:
27 | app(
28 | SSLAuthentication::class,
29 | compact('conf', 'configOptions')
30 | );
31 |
32 | break;
33 | case self::TYPE_SASL_SSL:
34 | app(
35 | SASLAuthentication::class,
36 | compact('conf', 'configOptions')
37 | );
38 |
39 | break;
40 | default:
41 | throw new AuthenticationException(
42 | 'Invalid Protocol Configuration.'
43 | );
44 | }
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/Console/ProducerMakeCommand.php:
--------------------------------------------------------------------------------
1 | getGuzzleHttpClient($avroSchema);
15 |
16 | $client = app(Client::class, ['client' => $guzzleHttp]);
17 |
18 | return app(CachedSchemaRegistryClient::class, compact('client'));
19 | }
20 |
21 | private function getGuzzleHttpClient(AvroSchema $avroSchema): GuzzleClient
22 | {
23 | $config = $avroSchema->getRequestOptions();
24 | $config['timeout'] = self::REQUEST_TIMEOUT;
25 | $config['base_uri'] = $avroSchema->getUrl();
26 | $config['headers'] = array_merge(
27 | $this->getDefaultHeaders(),
28 | $config['headers'] ?? []
29 | );
30 | $config['verify'] = $avroSchema->getRequestOptions()['ssl_verify'] ?? false;
31 |
32 | return app(GuzzleClient::class, compact('config'));
33 | }
34 |
35 | private function getDefaultHeaders(): array
36 | {
37 | return [
38 | 'Accept' => 'application/vnd.schemaregistry.v1+json, application/vnd.schemaregistry+json, application/json',
39 | ];
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/src/Console/ConsumerMakeCommand.php:
--------------------------------------------------------------------------------
1 | 'topic-id',
31 | 'connections' => 'kafka:9092',
32 | 'auth' => null,
33 | 'timeout' => 4000,
34 | 'is_async' => false,
35 | 'partition' => RD_KAFKA_PARTITION_UA,
36 | 'middlewares' => [],
37 | 'required_acknowledgment' => true,
38 | 'max_poll_records' => 500,
39 | 'flush_attempts' => 10,
40 | ];
41 |
42 | // Actions
43 | $result = $configOptions->toArray();
44 |
45 | // Expectations
46 | $this->assertEquals($expected, $result);
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Factories/ProducerFactory.php:
--------------------------------------------------------------------------------
1 | topic = $topic;
22 | $this->poll = $poll;
23 | $this->partition = $partition;
24 | }
25 |
26 | public function __destruct()
27 | {
28 | $this->poll->flushMessage();
29 | }
30 |
31 | /**
32 | * @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter
33 | */
34 | public function process(RecordInterface $record, Closure $next): void
35 | {
36 | $this->topic->produce(
37 | $this->getPartition($record),
38 | 0,
39 | $record->getPayload(),
40 | $record->getKey()
41 | );
42 |
43 | $this->poll->handleResponse();
44 | }
45 |
46 | public function getPartition(RecordInterface $record): int
47 | {
48 | return is_null(
49 | $record->getPartition()
50 | )
51 | ? $this->partition
52 | : $record->getPartition();
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/src/Avro/Serializer/Encoders/SchemaSubjectAndVersion.php:
--------------------------------------------------------------------------------
1 | getVersion();
16 | $subject = $schema->getSubject();
17 |
18 | $writer = new AvroIODatumWriter($schema->getAvroSchema());
19 | $io = new AvroStringIO();
20 |
21 | // write the header
22 |
23 | // magic byte
24 | $io->write(pack('C', SchemaFormats::MAGIC_BYTE_SUBJECT_VERSION));
25 |
26 | // write the subject length in network byte order (big end)
27 | $io->write(pack('N', strlen($subject)));
28 |
29 | // then the subject
30 | foreach (str_split($subject) as $letter) {
31 | $io->write(pack('C', ord($letter)));
32 | }
33 |
34 | // and finally the version
35 | $io->write(pack('N', $version));
36 |
37 | // write the record to the rest of it
38 | // Create an encoder that we'll write to
39 | $encoder = new AvroIOBinaryEncoder($io);
40 |
41 | // write the object in 'obj' as Avro to the fake file...
42 | $writer->write($message, $encoder);
43 |
44 | return $io->string();
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/Avro/Serializer/Encoders/SchemaId.php:
--------------------------------------------------------------------------------
1 | registry = $registry;
19 | }
20 |
21 | public function encode(Schema $schema, $message): string
22 | {
23 | $schemaId = $schema->getSchemaId();
24 | $writer = new AvroIODatumWriter($schema->getAvroSchema());
25 | $io = new AvroStringIO();
26 |
27 | // write the header
28 |
29 | // magic byte
30 | $io->write(pack('C', SchemaFormats::MAGIC_BYTE_SCHEMAID));
31 |
32 | // write the schema ID in network byte order (big end)
33 | $io->write(pack('N', $schemaId));
34 |
35 | // write the record to the rest of it
36 | // Create an encoder that we'll write to
37 | $encoder = new AvroIOBinaryEncoder($io);
38 |
39 | // write the object in 'obj' as Avro to the fake file...
40 | $writer->write($message, $encoder);
41 |
42 | return $io->string();
43 | }
44 |
45 | public function getRegistry()
46 | {
47 | return $this->registry;
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/Consumer/AbstractHandlerTest.php:
--------------------------------------------------------------------------------
1 | warning(new ResponseWarningException());
25 |
26 | // Assertions
27 | $this->assertNull($result);
28 | }
29 |
30 | public function testItShouldHandleFailedConsumer(): void
31 | {
32 | // Set
33 | $consumerHandler = new class () extends AbstractHandler {
34 | /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */
35 | public function handle(RecordInterface $record): void
36 | {
37 | }
38 | };
39 |
40 | // Actions
41 | $result = $consumerHandler->failed(new Exception());
42 |
43 | // Assertions
44 | $this->assertNull($result);
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/phpcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 | config
14 | src
15 | tests
16 |
17 |
18 |
19 |
20 |
21 | config
22 |
23 |
24 | tests/*/Dummies/*.php
25 |
26 |
27 | src/Record/ConsumerRecord.php
28 | src/Record/ProducerRecord.php
29 | src/TopicHandler/Producer/AbstractHandler.php
30 | src/TopicHandler/Producer/AbstractProducer.php
31 |
32 |
33 | src/Producer/Poll.php
34 |
35 |
36 |
--------------------------------------------------------------------------------
/src/Avro/Schema.php:
--------------------------------------------------------------------------------
1 | setAvroSchema(AvroSchema::parse($schema));
20 | $this->setSchemaId($id);
21 | $this->setSubject($subject);
22 | $this->setVersion($version);
23 |
24 | return $this;
25 | }
26 |
27 | public function getSchemaId(): string
28 | {
29 | return $this->schemaId;
30 | }
31 |
32 | public function setSchemaId(string $schemaId): void
33 | {
34 | $this->schemaId = $schemaId;
35 | }
36 |
37 | public function getAvroSchema(): AvroSchema
38 | {
39 | return $this->avroSchema;
40 | }
41 |
42 | public function setAvroSchema(AvroSchema $avroSchema): void
43 | {
44 | $this->avroSchema = $avroSchema;
45 | }
46 |
47 | public function getVersion(): ?string
48 | {
49 | return $this->version;
50 | }
51 |
52 | public function setVersion(?string $version): void
53 | {
54 | $this->version = $version;
55 | }
56 |
57 | public function getSubject(): ?string
58 | {
59 | return $this->subject;
60 | }
61 |
62 | public function setSubject(?string $subject): void
63 | {
64 | $this->subject = $subject;
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/src/Connectors/Producer/Connector.php:
--------------------------------------------------------------------------------
1 | canHandleResponse($handler)) {
22 | $conf->setDrMsgCb(
23 | /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */
24 | function ($kafka, Message $message) use ($handler) {
25 | if ($message->err) {
26 | $handler->failed($message);
27 | } else {
28 | $handler->success($message);
29 | }
30 | }
31 | );
32 | }
33 |
34 | $broker = $producerConfigOptions->getBroker();
35 | $conf->set('metadata.broker.list', $broker->getConnections());
36 |
37 | Factory::authenticate($conf, $broker->getAuth());
38 |
39 | return app(KafkaProducer::class, compact('conf'));
40 | }
41 |
42 | private function canHandleResponse(HandlerInterface $handler): bool
43 | {
44 | return $handler instanceof HandleableResponseInterface;
45 | }
46 | }
47 |
--------------------------------------------------------------------------------
/src/Connectors/Consumer/HighLevel.php:
--------------------------------------------------------------------------------
1 | getConf($configOptions);
17 | $maxPollIntervalMs = $configOptions->getMaxPollInterval();
18 | $conf->set('group.id', $configOptions->getConsumerGroup());
19 | $conf->set('auto.offset.reset', $configOptions->getOffsetReset());
20 | if (!$autoCommit) {
21 | $conf->set('enable.auto.commit', 'false');
22 | }
23 | $conf->set(
24 | 'max.poll.interval.ms',
25 | $maxPollIntervalMs
26 | );
27 |
28 | $consumer = app(KafkaConsumer::class, ['conf' => $conf]);
29 | $consumer->subscribe([$configOptions->getTopicId()]);
30 | $timeout = $configOptions->getTimeout();
31 |
32 | return app(HighLevelConsumer::class, compact('consumer', 'timeout'));
33 | }
34 |
35 | protected function getConf(ConfigOptions $configOptions): Conf
36 | {
37 | $conf = resolve(Conf::class);
38 | $broker = $configOptions->getBroker();
39 | Factory::authenticate($conf, $broker->getAuth());
40 |
41 | $conf->set('metadata.broker.list', $broker->getConnections());
42 |
43 | return $conf;
44 | }
45 | }
46 |
--------------------------------------------------------------------------------
/tests/Unit/Consumers/LowLevelTest.php:
--------------------------------------------------------------------------------
1 | expects()
46 | ->consume($partition, $timeout)
47 | ->andReturn($message);
48 |
49 | // Actions
50 | $message = $lowLevelConsumer->consume();
51 |
52 | // Assertions
53 | $this->assertInstanceOf(Message::class, $message);
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/ConfigOptions/ConsumerTest.php:
--------------------------------------------------------------------------------
1 | 'topic-id',
33 | 'connections' => 'kafka:9092',
34 | 'auth' => null,
35 | 'timeout' => 200,
36 | 'handler' => 'Tests\Unit\Dummies\ProducerHandlerDummy',
37 | 'partition' => RD_KAFKA_PARTITION_UA,
38 | 'offset' => null,
39 | 'consumer_group' => 'some_consumer_group',
40 | 'middlewares' => [],
41 | 'auto_commit' => false,
42 | 'commit_async' => true,
43 | 'offset_reset' => 'smallest',
44 | 'max_poll_interval_ms' => 300000,
45 | ];
46 |
47 | // Actions
48 | $result = $configOptions->toArray();
49 |
50 | // Expectations
51 | $this->assertEquals($expected, $result);
52 | }
53 | }
54 |
--------------------------------------------------------------------------------
/tests/Unit/Consumers/HighLevelTest.php:
--------------------------------------------------------------------------------
1 | expects()
22 | ->consume(1000)
23 | ->andReturn($message);
24 |
25 | // Actions
26 | $message = $highLevelConsumer->consume();
27 |
28 | // Assertions
29 | $this->assertInstanceOf(Message::class, $message);
30 | }
31 |
32 | public function testItShouldCommit(): void
33 | {
34 | // Set
35 | $kafkaConsumer = m::mock(KafkaConsumer::class);
36 | $highLevelConsumer = new HighLevel($kafkaConsumer, 1000);
37 |
38 | // Expectations
39 | $kafkaConsumer->expects()
40 | ->commit()
41 | ->andReturn();
42 |
43 | // Actions
44 | $highLevelConsumer->commit();
45 | }
46 |
47 | public function testItShouldCommitAsynchronously(): void
48 | {
49 | // Set
50 | $kafkaConsumer = m::mock(KafkaConsumer::class);
51 | $highLevelConsumer = new HighLevel($kafkaConsumer, 1000);
52 |
53 | // Expectations
54 | $kafkaConsumer->expects()
55 | ->commitAsync()
56 | ->andReturn();
57 |
58 | // Actions
59 | $highLevelConsumer->commitAsync();
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/AvroSchema.php:
--------------------------------------------------------------------------------
1 | [
26 | * 'Authorization' => [
27 | * 'Basic AUTHENTICATION',
28 | * ],
29 | * ],
30 | * ],
31 | *
32 | * @var mixed[]
33 | */
34 | private array $requestOptions;
35 |
36 | public function __construct(string $url, array $requestOptions = [], bool $sslVerify = true)
37 | {
38 | $this->url = $url;
39 | $this->requestOptions = $requestOptions;
40 | $this->sslVerify = $sslVerify;
41 | }
42 |
43 | public function toArray(): array
44 | {
45 | return [
46 | 'url' => $this->getUrl(),
47 | 'request_options' => $this->getRequestOptions(),
48 | 'ssl_verify' => $this->isSslVerify(),
49 | ];
50 | }
51 |
52 | public function getUrl(): string
53 | {
54 | return $this->url;
55 | }
56 |
57 | public function isSslVerify(): bool
58 | {
59 | return $this->sslVerify;
60 | }
61 |
62 | public function getRequestOptions(): array
63 | {
64 | return $this->requestOptions;
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/src/Avro/Client.php:
--------------------------------------------------------------------------------
1 | client = $client;
15 | }
16 |
17 | public function get(string $url): array
18 | {
19 | $response = $this->client->get($url);
20 |
21 | return $this->parseResponse($response);
22 | }
23 |
24 | public function post(string $url, array $body = []): array
25 | {
26 | $response = $this->client->post($url, [
27 | 'headers' => $this->getContentTypeForPostRequest(),
28 | 'form_params' => $body,
29 | ]);
30 |
31 | return $this->parseResponse($response);
32 | }
33 |
34 | public function put(string $url, array $body = []): array
35 | {
36 | $response = $this->client->post($url, [
37 | 'headers' => $this->getContentTypeForPostRequest(),
38 | 'form_params' => $body,
39 | ]);
40 |
41 | return $this->parseResponse($response);
42 | }
43 |
44 | public function delete(string $url): array
45 | {
46 | $response = $this->client->delete($url);
47 |
48 | return $this->parseResponse($response);
49 | }
50 |
51 | private function parseResponse(ResponseInterface $response): array
52 | {
53 | return [$response->getStatusCode(), json_decode(
54 | $response->getBody(),
55 | true
56 | ),
57 | ];
58 | }
59 |
60 | private function getContentTypeForPostRequest(): array
61 | {
62 | return ['Content-Type' => 'application/vnd.schemaregistry.v1+json'];
63 | }
64 | }
65 |
--------------------------------------------------------------------------------
/src/Record/ProducerRecord.php:
--------------------------------------------------------------------------------
1 | payload = $payload;
41 | $this->original = $payload;
42 | $this->partition = $partition;
43 | $this->topic = $topic;
44 | $this->key = $key;
45 | }
46 |
47 | public function setPayload($payload): void
48 | {
49 | $this->payload = $payload;
50 | }
51 |
52 | public function getPayload()
53 | {
54 | return $this->payload;
55 | }
56 |
57 | public function getTopicName(): string
58 | {
59 | return $this->topic;
60 | }
61 |
62 | public function getPartition(): ?int
63 | {
64 | return $this->partition;
65 | }
66 |
67 | public function getKey(): ?string
68 | {
69 | return $this->key;
70 | }
71 |
72 | public function getOriginal()
73 | {
74 | return $this->original;
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/JsonDecodeTest.php:
--------------------------------------------------------------------------------
1 | 1392, 'member_name' => 'Jose']];
18 | $json = json_encode($data);
19 | $middleware = new JsonDecode();
20 | $kafkaMessage = new KafkaMessage();
21 | $kafkaMessage->payload = $json;
22 | $kafkaMessage->err = RD_KAFKA_RESP_ERR_NO_ERROR;
23 | $record = new ConsumerRecord($kafkaMessage);
24 | $closure = Closure::fromCallable(function ($record) {
25 | return $record;
26 | });
27 |
28 | // Actions
29 | $record = $middleware->process($record, $closure);
30 |
31 | // Assertions
32 | $this->assertSame($data, $record->getPayload());
33 | }
34 |
35 | public function testItShouldThrowAnExceptionOnInvalidJsonString(): void
36 | {
37 | // Set
38 | $json = "{'Organization': 'Metamorphosis Team'}";
39 | $closure = Closure::fromCallable(function () {
40 | });
41 | $middleware = new JsonDecode();
42 |
43 | $kafkaMessage = new KafkaMessage();
44 | $kafkaMessage->payload = $json;
45 | $kafkaMessage->err = RD_KAFKA_RESP_ERR_NO_ERROR;
46 |
47 | $record = new ConsumerRecord($kafkaMessage);
48 |
49 | $this->expectException(Exception::class);
50 | $this->expectExceptionMessage('Malformed JSON. Error: Syntax error');
51 |
52 | // Actions
53 | $middleware->process($record, $closure);
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/Producer/AbstractHandlerTest.php:
--------------------------------------------------------------------------------
1 | createRecord();
23 |
24 | // Assertions
25 | $this->assertInstanceOf(ProducerRecord::class, $result);
26 | $this->assertSame($key, $result->getKey());
27 | $this->assertSame($topic, $result->getTopicName());
28 | $this->assertSame($record, $result->getPayload());
29 | $this->assertSame($partition, $result->getPartition());
30 | }
31 |
32 | public function testShouldCreateEncodeJsonWhenRecordIsArray(): void
33 | {
34 | // Set
35 | $record = ['number' => 1, 'float' => 0.0];
36 | $topic = 'default';
37 | $key = 'default_1';
38 | $partition = 1;
39 | $handler = new class ($record, $topic, $key, $partition) extends AbstractHandler {
40 | };
41 |
42 | // Actions
43 | $result = $handler->createRecord();
44 |
45 | // Assertions
46 | $this->assertInstanceOf(ProducerRecord::class, $result);
47 | $this->assertSame($key, $result->getKey());
48 | $this->assertSame($topic, $result->getTopicName());
49 | $this->assertSame('{"number":1,"float":0.0}', $result->getPayload());
50 | $this->assertSame($partition, $result->getPartition());
51 | }
52 | }
53 |
--------------------------------------------------------------------------------
/src/Avro/Serializer/MessageDecoder.php:
--------------------------------------------------------------------------------
1 | [int Magic Byte => string Schema Decoder Class]
16 | */
17 | private array $decoders = [
18 | SchemaFormats::MAGIC_BYTE_SCHEMAID => SchemaId::class,
19 | SchemaFormats::MAGIC_BYTE_SUBJECT_VERSION => SchemaSubjectAndVersion::class,
20 | ];
21 |
22 | private CachedSchemaRegistryClient $registry;
23 |
24 | public function __construct(CachedSchemaRegistryClient $registry)
25 | {
26 | $this->registry = $registry;
27 | }
28 |
29 | /**
30 | * Decode a message from kafka that has been encoded for use with the schema registry.
31 | *
32 | * @throws \AvroIOException
33 | *
34 | * @return mixed
35 | */
36 | public function decodeMessage(string $message)
37 | {
38 | if (!$message) {
39 | throw new RuntimeException('Message is too small to decode');
40 | }
41 |
42 | $io = new AvroStringIO($message);
43 |
44 | if (!$decoder = $this->getDecoder($io)) {
45 | return $message;
46 | }
47 |
48 | return $decoder->decode($io);
49 | }
50 |
51 | private function getDecoder(AvroStringIO $io): ?DecoderInterface
52 | {
53 | $magicByte = unpack('C', $io->read(1));
54 | $magicByte = $magicByte[1];
55 |
56 | if (!$class = $this->decoders[$magicByte] ?? null) {
57 | return null;
58 | }
59 |
60 | return app($class, ['registry' => $this->registry]);
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/TopicHandler/Producer/AbstractProducer.php:
--------------------------------------------------------------------------------
1 | record = $record;
23 | $this->key = $key;
24 | $this->producer = $configOptions;
25 | }
26 |
27 | public function getConfigOptions(): Producer
28 | {
29 | return $this->producer;
30 | }
31 |
32 | public function getRecord()
33 | {
34 | return $this->record;
35 | }
36 |
37 | public function getKey(): ?string
38 | {
39 | return $this->key;
40 | }
41 |
42 | public function createRecord(): ProducerRecord
43 | {
44 | $record = $this->getRecord();
45 |
46 | if (is_array($record)) {
47 | $record = $this->encodeRecord($record);
48 | }
49 |
50 | $topic = $this->getConfigOptions()->getTopicId();
51 | $partition = $this->getConfigOptions()->getPartition();
52 | $key = $this->getKey();
53 |
54 | return new ProducerRecord($record, $topic, $partition, $key);
55 | }
56 |
57 | private function encodeRecord(array $record): string
58 | {
59 | $record = json_encode($record, JSON_PRESERVE_ZERO_FRACTION);
60 |
61 | if (JSON_ERROR_NONE !== json_last_error()) {
62 | throw new JsonException(
63 | 'Cannot convert data into a valid JSON. Reason: ' . json_last_error_msg()
64 | );
65 | }
66 |
67 | return $record;
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/src/Console/ConfigOptionsCommand.php:
--------------------------------------------------------------------------------
1 | argument('handler'));
35 |
36 | $configOptions = $consumerHandler->getConfigOptions();
37 |
38 | $this->writeStartingConsumer($configOptions);
39 |
40 | $manager = Factory::make($configOptions);
41 |
42 | $runner = app(Runner::class, compact('manager'));
43 | $runner->run($this->option('times'));
44 | }
45 |
46 | private function writeStartingConsumer(ConfigOptions $configOptions): void
47 | {
48 | $text = 'Starting consumer for topic: ' . $configOptions->getTopicId() . PHP_EOL;
49 | $text .= ' on consumer group: ' . $configOptions->getConsumerGroup() . PHP_EOL;
50 | $text .= 'Connecting in ' . $configOptions->getBroker()->getConnections() . PHP_EOL;
51 | $text .= 'Running consumer..';
52 |
53 | $this->output->writeln($text);
54 | }
55 | }
56 |
--------------------------------------------------------------------------------
/src/TopicHandler/Producer/AbstractHandler.php:
--------------------------------------------------------------------------------
1 | record = $record;
27 | $this->topic = $topic;
28 | $this->key = $key;
29 | $this->partition = $partition;
30 | }
31 |
32 | public function getRecord()
33 | {
34 | return $this->record;
35 | }
36 |
37 | public function getTopic(): string
38 | {
39 | return $this->topic;
40 | }
41 |
42 | public function getPartition(): ?int
43 | {
44 | return $this->partition;
45 | }
46 |
47 | public function getKey(): ?string
48 | {
49 | return $this->key;
50 | }
51 |
52 | public function createRecord(): ProducerRecord
53 | {
54 | $record = $this->getRecord();
55 |
56 | if (is_array($record)) {
57 | $record = $this->encodeRecord($record);
58 | }
59 |
60 | $topic = $this->getTopic();
61 | $partition = $this->getPartition();
62 | $key = $this->getKey();
63 |
64 | return new ProducerRecord($record, $topic, $partition, $key);
65 | }
66 |
67 | private function encodeRecord(array $record): string
68 | {
69 | $record = json_encode($record, JSON_PRESERVE_ZERO_FRACTION);
70 |
71 | if (JSON_ERROR_NONE !== json_last_error()) {
72 | throw new JsonException(
73 | 'Cannot convert data into a valid JSON. Reason: ' . json_last_error_msg()
74 | );
75 | }
76 |
77 | return $record;
78 | }
79 | }
80 |
--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "leroy-merlin-br/metamorphosis",
3 | "license": "MIT",
4 | "type": "library",
5 | "keywords": [
6 | "kafka"
7 | ],
8 | "description": "Kafka package for laravel applications",
9 | "require": {
10 | "ext-rdkafka": ">=4.0",
11 | "ext-json": "*",
12 | "guzzlehttp/guzzle": "^6.5.0 || ^7.0",
13 | "illuminate/support": "^9.0 || ^10.0",
14 | "illuminate/console": "^9.0 || ^10.0",
15 | "illuminate/config": "^9.0 || ^10.0",
16 | "php": "^8.0",
17 | "rg/avro-php": "^3.0"
18 | },
19 | "require-dev": {
20 | "leroy-merlin-br/coding-standard": "^v3.1.0",
21 | "phpunit/phpunit": "^9.6.10",
22 | "mockery/mockery": "^1.6.6",
23 | "kwn/php-rdkafka-stubs": "^2.2.1",
24 | "orchestra/testbench": "^7.0|^8.0",
25 | "dms/phpunit-arraysubset-asserts": "^0.2.1",
26 | "phpro/grumphp": "^1.16.0",
27 | "vimeo/psalm": "^4.30.0",
28 | "psalm/plugin-mockery": "^0.9.1",
29 | "rector/rector": "^0.13.10"
30 | },
31 | "autoload": {
32 | "psr-4": {
33 | "Metamorphosis\\": "src/"
34 | }
35 | },
36 | "autoload-dev": {
37 | "psr-4": {
38 | "Tests\\": "tests"
39 | }
40 | },
41 | "authors": [
42 | {
43 | "name": "Boitata Team",
44 | "email": "boitata@leroymerlin.com.br"
45 | }
46 | ],
47 | "extra": {
48 | "laravel": {
49 | "providers": [
50 | "Metamorphosis\\MetamorphosisServiceProvider"
51 | ]
52 | },
53 | "aliases": {
54 | "Metamorphosis": "Metamorphosis\\Facades\\Metamorphosis"
55 | }
56 | },
57 | "scripts": {
58 | "setup-dev": [
59 | "@composer install",
60 | "vendor/bin/grumphp git:init"
61 | ]
62 | },
63 | "config": {
64 | "allow-plugins": {
65 | "dealerdirect/phpcodesniffer-composer-installer": true,
66 | "phpro/grumphp": true
67 | }
68 | }
69 | }
70 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/LogTest.php:
--------------------------------------------------------------------------------
1 | payload = 'original record';
22 | $kafkaMessage->err = RD_KAFKA_RESP_ERR_NO_ERROR;
23 | $record = new Record($kafkaMessage);
24 | $closure = Closure::fromCallable(function ($record) {
25 | return $record;
26 | });
27 |
28 | // Expectations
29 | $log->expects()
30 | ->info('Processing kafka record: original record', m::on(
31 | static function (array $context): bool {
32 | $original = $context['original'];
33 | $expected = [
34 | 'err' => RD_KAFKA_RESP_ERR_NO_ERROR,
35 | 'topic_name' => null,
36 | 'timestamp' => null,
37 | 'payload' => 'original record',
38 | 'len' => null,
39 | 'key' => null,
40 | 'opaque' => null,
41 | ];
42 |
43 | foreach ($expected as $key => $expectedValue) {
44 | if (!array_key_exists($key, $original)) {
45 | return false;
46 | }
47 |
48 | if ($original[$key] !== $expectedValue) {
49 | return false;
50 | }
51 | }
52 |
53 | return true;
54 | }
55 | ));
56 |
57 | // Actions
58 | $middleware->process($record, $closure);
59 | }
60 | }
61 |
--------------------------------------------------------------------------------
/src/Producer/Poll.php:
--------------------------------------------------------------------------------
1 | isAsync = $producerConfigOptions->isAsync();
30 | $this->maxPollRecords = $producerConfigOptions->getMaxPollRecords();
31 | $this->requiredAcknowledgment = $producerConfigOptions->isRequiredAcknowledgment();
32 | $this->maxFlushAttempts = $producerConfigOptions->getFlushAttempts();
33 | $this->timeout = $producerConfigOptions->getTimeout();
34 |
35 | $this->producer = $producer;
36 | }
37 |
38 | public function handleResponse(): void
39 | {
40 | $this->producer->poll(self::NON_BLOCKING_POLL);
41 | $this->processedMessagesCount++;
42 |
43 | if (!$this->isAsync) {
44 | $this->flushMessage();
45 |
46 | return;
47 | }
48 |
49 | if (0 === ($this->processedMessagesCount % $this->maxPollRecords)) {
50 | $this->flushMessage();
51 | }
52 | }
53 |
54 | public function flushMessage(): void
55 | {
56 | if (!$this->requiredAcknowledgment) {
57 | return;
58 | }
59 |
60 | for ($flushAttempts = 0; $flushAttempts < $this->maxFlushAttempts; $flushAttempts++) {
61 | if (
62 | RD_KAFKA_RESP_ERR_NO_ERROR === $this->producer->flush(
63 | $this->timeout
64 | )
65 | ) {
66 | return;
67 | }
68 |
69 | sleep($this->timeout / 1000);
70 | }
71 |
72 | throw new RuntimeException('Unable to flush, messages might be lost!');
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Factories/ConsumerFactory.php:
--------------------------------------------------------------------------------
1 | getAvroSchema()->getUrl()) {
32 | throw new ConfigurationException(
33 | "Avro schema url not found, it's required to use AvroSchemaEncoder Middleware"
34 | );
35 | }
36 |
37 | $schemaRegistry = $factory->make(
38 | $producerConfigOptions->getAvroSchema()
39 | );
40 | $this->schemaIdEncoder = $schemaIdEncoder;
41 | $this->schemaRegistry = $schemaRegistry;
42 | $this->producerConfigOptions = $producerConfigOptions;
43 | }
44 |
45 | public function process(RecordInterface $record, Closure $next)
46 | {
47 | $topic = $this->producerConfigOptions->getTopicId();
48 | $schema = $this->schemaRegistry->getBySubjectAndVersion(
49 | "{$topic}-value",
50 | 'latest'
51 | );
52 | $arrayPayload = json_decode($record->getPayload(), true);
53 | $encodedPayload = $this->schemaIdEncoder->encode(
54 | $schema,
55 | $arrayPayload
56 | );
57 |
58 | $record->setPayload($encodedPayload);
59 |
60 | return $next($record);
61 | }
62 | }
63 |
--------------------------------------------------------------------------------
/src/Connectors/Consumer/LowLevel.php:
--------------------------------------------------------------------------------
1 | getConf();
18 | $maxPollIntervalMs = $configOptions->getMaxPollInterval();
19 | $conf->set(
20 | 'max.poll.interval.ms',
21 | $maxPollIntervalMs
22 | );
23 | $conf->set('group.id', $configOptions->getConsumerGroup());
24 | if (!$autoCommit) {
25 | $conf->set('enable.auto.commit', 'false');
26 | }
27 |
28 | $broker = $configOptions->getBroker();
29 | Factory::authenticate($conf, $broker->getAuth());
30 |
31 | $consumer = new Consumer($conf);
32 | $consumer->addBrokers($broker->getConnections());
33 |
34 | $topicConf = $this->getTopicConfigs($configOptions);
35 | $topicConsumer = $consumer->newTopic(
36 | $configOptions->getTopicId(),
37 | $topicConf
38 | );
39 |
40 | $topicConsumer->consumeStart(
41 | $configOptions->getPartition(),
42 | $configOptions->getOffset()
43 | );
44 |
45 | return new LowLevelConsumer($topicConsumer, $configOptions);
46 | }
47 |
48 | protected function getTopicConfigs(ConfigOptions $configOptions)
49 | {
50 | $topicConfig = new TopicConf();
51 |
52 | // Set where to start consuming messages when there is no initial offset in
53 | // offset store or the desired offset is out of range.
54 | // 'smallest': start from the beginning
55 | $topicConfig->set(
56 | 'auto.offset.reset',
57 | $configOptions->getOffsetReset()
58 | );
59 |
60 | return $topicConfig;
61 | }
62 |
63 | protected function getConf(): Conf
64 | {
65 | return resolve(Conf::class);
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/tests/Unit/Consumers/RunnerTest.php:
--------------------------------------------------------------------------------
1 | shouldReceive('handleMessage')
22 | ->times(4)
23 | ->andReturnUsing(function () use (&$count) {
24 | if (3 === $count) {
25 | $exception = new Exception('Error when consuming.');
26 |
27 | throw $exception;
28 | }
29 | $count++;
30 |
31 | return;
32 | });
33 |
34 | // Ensure that one message went through the middleware stack
35 | $this->expectException(Exception::class);
36 | $this->expectExceptionMessage('Error when consuming.');
37 |
38 | // Actions
39 | $runner->run();
40 | }
41 |
42 | public function testItShouldRunADeterminedNumberOfTimes(): void
43 | {
44 | // Set
45 | $manager = m::mock(Manager::class);
46 | $runner = new Runner($manager);
47 |
48 | // Expectations
49 | $manager->shouldReceive('handleMessage')
50 | ->times(3)
51 | ->andReturnUsing(function () {
52 | return;
53 | });
54 |
55 | // Actions
56 | $runner->run(3);
57 | }
58 |
59 | public function testItShouldRunADeterminedNumberOfTimesButStopsOnException(): void
60 | {
61 | // Set
62 | $manager = m::mock(Manager::class);
63 | $runner = new Runner($manager);
64 |
65 | // Expectations
66 | $manager->shouldReceive('handleMessage')
67 | ->times(1)
68 | ->andReturnUsing(function () {
69 | throw new Exception('Error when consuming.');
70 | });
71 |
72 | $this->expectException(Exception::class);
73 | $this->expectExceptionMessage('Error when consuming.');
74 |
75 | // Actions
76 | $runner->run(3);
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/src/Console/ConsumerCommand.php:
--------------------------------------------------------------------------------
1 | make($this->option(), $this->argument());
43 |
44 | $this->writeStartingConsumer($consumer);
45 |
46 | $manager = Factory::make($consumer);
47 |
48 | $runner = app(Runner::class, compact('manager'));
49 | $runner->run($this->option('times'));
50 | }
51 |
52 | private function writeStartingConsumer(Consumer $consumer): void
53 | {
54 | $text = 'Starting consumer for topic: ' . $consumer->getTopicId() . PHP_EOL;
55 | $text .= ' on consumer group: ' . $consumer->getConsumerGroup() . PHP_EOL;
56 | $text .= 'Connecting in ' . $consumer->getBroker()->getConnections() . PHP_EOL;
57 | $text .= 'Running consumer..';
58 |
59 | $this->output->writeln($text);
60 | }
61 | }
62 |
--------------------------------------------------------------------------------
/tests/Integration/ConsumerTest.php:
--------------------------------------------------------------------------------
1 | ['id' => 'MESSAGE_ID'],
54 | 'configOptions' => $producerConfigOptions,
55 | 'key' => 1,
56 | ]
57 | );
58 |
59 | $saleOrderDispatcher = Metamorphosis::build($messageProducer);
60 | $saleOrderDispatcher->handle($messageProducer->createRecord());
61 |
62 | $consumer = $this->app->make(
63 | Consumer::class,
64 | ['configOptions' => $consumerConfigOptions]
65 | );
66 | $expected = '{"id":"MESSAGE_ID"}';
67 |
68 | // Actions
69 | $result = $consumer->consume()->getPayload();
70 |
71 | // Assertions
72 | $this->assertSame($expected, $result);
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/docs/upgrade.md:
--------------------------------------------------------------------------------
1 |
2 | ## Upgrade guide
3 |
4 | To upgrade from version X.x to version X.y:
5 |
6 | Move your `avroschema` and `broker` section from old `config/kafka.php` file into a new file:
7 |
8 |
9 | ```php
10 | [
14 | 'this_is_your_topic_name' => [
15 | 'topic_id' => "this_is_your_topic_id",
16 | 'consumer' => [
17 | 'consumer_group' => 'your-consumer-group',
18 | 'offset_reset' => 'earliest',
19 | 'offset' => 0,
20 | 'partition' => 0,
21 | 'handler' => '\App\Kafka\Consumers\ConsumerExample',
22 | 'timeout' => 20000,
23 | 'auto_commit' => true,
24 | 'commit_async' => false,
25 | 'middlewares' => [],
26 | ],
27 |
28 | 'producer' => [
29 | 'required_acknowledgment' => true,
30 | 'is_async' => true,
31 | 'max_poll_records' => 500,
32 | 'flush_attempts' => 10,
33 | 'middlewares' => [],
34 | 'timeout' => 10000,
35 | 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1,
36 | ],
37 | ]
38 | ],
39 | ];
40 | ```
41 |
42 | Upgrade your topic configuration files:
43 |
44 | ```php
45 | [
49 | 'this_is_your_topic_name' => [
50 | 'topic_id' => "this_is_your_topic_id",
51 | 'consumer' => [
52 | 'consumer_group' => 'your-consumer-group',
53 | 'offset_reset' => 'earliest',
54 | 'offset' => 0,
55 | 'partition' => 0,
56 | 'handler' => '\App\Kafka\Consumers\ConsumerExample',
57 | 'timeout' => 20000,
58 | 'auto_commit' => true,
59 | 'commit_async' => false,
60 | 'middlewares' => [],
61 | ],
62 |
63 | 'producer' => [
64 | 'required_acknowledgment' => true,
65 | 'is_async' => true,
66 | 'max_poll_records' => 500,
67 | 'flush_attempts' => 10,
68 | 'middlewares' => [],
69 | 'timeout' => 10000,
70 | 'partition' => constant('RD_KAFKA_PARTITION_UA') ?? -1,
71 | ],
72 | ]
73 | ],
74 | ];
75 | ```
76 |
77 |
--------------------------------------------------------------------------------
/tests/Unit/Authentication/FactoryTest.php:
--------------------------------------------------------------------------------
1 | 'ssl',
27 | 'ssl.ca.location' => 'path/to/ca',
28 | 'ssl.certificate.location' => 'path/to/certificate',
29 | 'ssl.key.location' => 'path/to/key',
30 | ];
31 |
32 | // Actions
33 | Factory::authenticate($conf, $configOptionsSsl);
34 |
35 | // Assertions
36 | $this->assertArraySubset($expected, $conf->dump());
37 | }
38 |
39 | public function testItMakesSASLAuthenticationClass(): void
40 | {
41 | // Set
42 | $configOptionsSaslSsl = new SaslSsl(
43 | 'PLAIN',
44 | 'some-username',
45 | 'some-password'
46 | );
47 | $conf = new Conf();
48 | $expected = [
49 | 'security.protocol' => 'sasl_ssl',
50 | 'sasl.username' => 'some-username',
51 | 'sasl.password' => 'some-password',
52 | 'sasl.mechanisms' => 'PLAIN',
53 | ];
54 |
55 | // Actions
56 | Factory::authenticate($conf, $configOptionsSaslSsl);
57 |
58 | // Assertions
59 | $this->assertArraySubset($expected, $conf->dump());
60 | }
61 |
62 | public function testItThrowsExceptionWhenInvalidProtocolIsPassed(): void
63 | {
64 | // Set
65 | $invalidAuth = m::mock(AuthInterface::class);
66 | $conf = new Conf();
67 |
68 | // Expectations
69 | $invalidAuth->expects()
70 | ->getType()
71 | ->andReturn('some-invalid-type');
72 |
73 | $this->expectException(AuthenticationException::class);
74 |
75 | // Actions
76 | Factory::authenticate($conf, $invalidAuth);
77 | }
78 | }
79 |
--------------------------------------------------------------------------------
/src/Producer.php:
--------------------------------------------------------------------------------
1 | config = $config;
22 | $this->connector = $connector;
23 | }
24 |
25 | public function produce(HandlerInterface $producerHandler): void
26 | {
27 | $middlewareDispatcher = $this->build($producerHandler);
28 |
29 | $middlewareDispatcher->handle($producerHandler->createRecord());
30 | }
31 |
32 | public function build(HandlerInterface $producerHandler): Dispatcher
33 | {
34 | $producerConfigOptions = $producerHandler->getConfigOptions();
35 |
36 | $middlewares = $producerConfigOptions->getMiddlewares();
37 | foreach ($middlewares as &$middleware) {
38 | $middleware = is_string($middleware)
39 | ? app(
40 | $middleware,
41 | ['producerConfigOptions' => $producerConfigOptions]
42 | )
43 | : $middleware;
44 | }
45 |
46 | $middlewares[] = $this->getProducerMiddleware(
47 | $producerHandler,
48 | $producerConfigOptions
49 | );
50 |
51 | return new Dispatcher($middlewares);
52 | }
53 |
54 | public function getProducerMiddleware(
55 | HandlerInterface $producerHandler,
56 | ProducerConfigOptions $producerConfigOptions
57 | ): ProducerMiddleware {
58 | $producer = $this->connector->getProducerTopic(
59 | $producerHandler,
60 | $producerConfigOptions
61 | );
62 |
63 | $topic = $producer->newTopic($producerConfigOptions->getTopicId());
64 | $poll = app(
65 | Poll::class,
66 | ['producer' => $producer, 'producerConfigOptions' => $producerConfigOptions]
67 | );
68 | $partition = $producerConfigOptions->getPartition();
69 |
70 | return app(
71 | ProducerMiddleware::class,
72 | compact('topic', 'poll', 'partition')
73 | );
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/src/Connectors/Consumer/Factory.php:
--------------------------------------------------------------------------------
1 | isAutoCommit();
21 | $commitAsync = $configOptions->isCommitASync();
22 |
23 | $consumer = self::getConsumer($autoCommit, $configOptions);
24 |
25 | $handler = app($configOptions->getHandler());
26 |
27 | $middlewares = $configOptions->getMiddlewares();
28 | foreach ($middlewares as &$middleware) {
29 | $middleware = is_string($middleware)
30 | ? app(
31 | $middleware,
32 | ['consumerConfigOptions' => $configOptions]
33 | )
34 | : $middleware;
35 | }
36 |
37 | $middlewares[] = app(
38 | ConsumerMiddleware::class,
39 | ['consumerTopicHandler' => $handler]
40 | );
41 |
42 | $dispatcher = self::getMiddlewareDispatcher($middlewares);
43 |
44 | return new Manager(
45 | $consumer,
46 | $handler,
47 | $dispatcher,
48 | $autoCommit,
49 | $commitAsync
50 | );
51 | }
52 |
53 | public static function getConsumer(bool $autoCommit, ConsumerConfigOptions $configOptions): ConsumerInterface
54 | {
55 | if (self::requiresPartition($configOptions)) {
56 | return app(LowLevel::class)->getConsumer(
57 | $autoCommit,
58 | $configOptions
59 | );
60 | }
61 |
62 | return app(HighLevel::class)->getConsumer($autoCommit, $configOptions);
63 | }
64 |
65 | protected static function requiresPartition(ConsumerConfigOptions $configOptions): bool
66 | {
67 | $partition = $configOptions->getPartition();
68 |
69 | return !is_null($partition) && $partition >= 0;
70 | }
71 |
72 | private static function getMiddlewareDispatcher(array $middlewares): Dispatcher
73 | {
74 | return new Dispatcher($middlewares);
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/ConfigOptions/Factories/ProducerFactoryTest.php:
--------------------------------------------------------------------------------
1 | 'kafka:9092',
16 | 'auth' => [
17 | 'type' => 'ssl',
18 | 'ca' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/ca.pem',
19 | 'certificate' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.cert',
20 | 'key' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.key',
21 | ],
22 | ];
23 | $avroSchemaData = [];
24 | $topicData = [
25 | 'topic_id' => 'kafka-test',
26 | 'producer' => [
27 | 'timeout' => 10000,
28 | 'is_async' => true,
29 | 'partition' => RD_KAFKA_PARTITION_UA,
30 | 'required_acknowledgment' => true,
31 | 'max_poll_records' => 500,
32 | 'flush_attempts' => 10,
33 | 'middlewares' => [],
34 | ],
35 | ];
36 | $expected = [
37 | 'topic_id' => 'kafka-test',
38 | 'connections' => 'kafka:9092',
39 | 'auth' => [
40 | 'type' => 'ssl',
41 | 'ca' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/ca.pem',
42 | 'certificate' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.cert',
43 | 'key' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.key',
44 | ],
45 | 'timeout' => 10000,
46 | 'is_async' => true,
47 | 'partition' => RD_KAFKA_PARTITION_UA,
48 | 'required_acknowledgment' => true,
49 | 'max_poll_records' => 500,
50 | 'flush_attempts' => 10,
51 | 'middlewares' => [],
52 | ];
53 |
54 | // Actions
55 | $result = ProducerFactory::make(
56 | $brokerData,
57 | $topicData,
58 | $avroSchemaData
59 | );
60 |
61 | // Assertions
62 | $this->assertInstanceOf(Producer::class, $result);
63 | $this->assertEquals($expected, $result->toArray());
64 | }
65 | }
66 |
--------------------------------------------------------------------------------
/src/Avro/Serializer/MessageEncoder.php:
--------------------------------------------------------------------------------
1 | [int Magic Byte => string Schema Decoder Class]
16 | */
17 | private array $encoders = [
18 | SchemaFormats::MAGIC_BYTE_SCHEMAID => SchemaId::class,
19 | SchemaFormats::MAGIC_BYTE_SUBJECT_VERSION => SchemaSubjectAndVersion::class,
20 | ];
21 |
22 | private CachedSchemaRegistryClient $registry;
23 |
24 | private bool $registerMissingSchemas;
25 |
26 | private int $defaultEncodingFormat;
27 |
28 | public function __construct(CachedSchemaRegistryClient $registry, array $options = [])
29 | {
30 | $this->registry = $registry;
31 |
32 | $this->registerMissingSchemas = $options['register_missing_schemas'] ?? false;
33 | $this->defaultEncodingFormat = $options['default_encoding_format'] ?? SchemaFormats::MAGIC_BYTE_SCHEMAID;
34 | }
35 |
36 | /**
37 | * Given a parsed Avro schema, encode a record for the given topic.
38 | * The schema is registered with the subject of 'topic-value'
39 | *
40 | * @param string $topic Topic name
41 | * @param Schema $schema Avro Schema
42 | * @param mixed $message An message/record (object, array, string, etc) to serialize
43 | * @param bool $isKey If the record is a key
44 | * @param int|null $format Encoding Format
45 | *
46 | * @throws \AvroIOException
47 | *
48 | * @return string Encoded record with schema ID as bytes
49 | */
50 | public function encodeMessage(
51 | string $topic,
52 | Schema $schema,
53 | $message,
54 | bool $isKey = false,
55 | ?int $format = null
56 | ): string {
57 | $suffix = $isKey ? '-key' : '-value';
58 | $subject = $topic . $suffix;
59 | $format = $format ?? $this->defaultEncodingFormat;
60 |
61 | $encoder = $this->getEncoder($format);
62 |
63 | return $encoder->encode($schema, $message);
64 | }
65 |
66 | private function getEncoder(int $format): EncoderInterface
67 | {
68 | if (!$class = $this->encoders[$format] ?? null) {
69 | throw new RuntimeException('Unsuported format: ' . $format);
70 | }
71 |
72 | return app($class, ['registry' => $this->registry]);
73 | }
74 | }
75 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/AvroSchemaDecoderTest.php:
--------------------------------------------------------------------------------
1 | getAvroSchema();
36 | $avroSchema = new AvroSchema('string');
37 | $decoder = m::mock(Schema::class);
38 | $clientFactory = m::mock(ClientFactory::class);
39 | $cachedSchemaRegistryClient = m::mock(
40 | CachedSchemaRegistryClient::class
41 | );
42 | $expected = 'my awesome message';
43 |
44 | $message = new Message();
45 | $message->payload = "\x01\x00\x00\x00\fmy-topic-key\x00\x00\x00\x05\$my awesome message";
46 | $message->err = 0;
47 |
48 | $closure = Closure::fromCallable(function ($producerRecord) {
49 | return $producerRecord;
50 | });
51 |
52 | $consumerRecord = new ConsumerRecord($message);
53 |
54 | // Expectations
55 | $clientFactory->expects()
56 | ->make($avroSchemaConfigOptions)
57 | ->andReturn($cachedSchemaRegistryClient);
58 |
59 | $cachedSchemaRegistryClient->expects()
60 | ->getBySubjectAndVersion('my-topic-key', 5)
61 | ->andReturn($decoder);
62 |
63 | $decoder->expects()
64 | ->getAvroSchema()
65 | ->andReturn($avroSchema);
66 |
67 | $avroSchemaDecoder = new AvroSchemaDecoder(
68 | $clientFactory,
69 | $consumerConfigOptions
70 | );
71 |
72 | $result = $avroSchemaDecoder->process($consumerRecord, $closure);
73 |
74 | $this->assertSame($expected, $result->getPayload());
75 | }
76 | }
77 |
--------------------------------------------------------------------------------
/.github/workflows/continuous-integration.yml:
--------------------------------------------------------------------------------
1 | name: Continuous Integration
2 |
3 | on:
4 | - push
5 |
6 | concurrency:
7 | group: ${{ github.workflow }}-${{ github.ref }}
8 | cancel-in-progress: true
9 |
10 | jobs:
11 | testing:
12 | name: Testing and Code Quality for PHP
13 | runs-on: ubuntu-latest
14 | strategy:
15 | matrix:
16 | php:
17 | - '8.0'
18 | - '8.1'
19 | - '8.2'
20 | env:
21 | KAFKA_BROKER_CONNECTIONS: 'localhost:9092'
22 | services:
23 | zookeeper:
24 | image: bitnami/zookeeper
25 | env:
26 | ALLOW_ANONYMOUS_LOGIN: yes
27 | kafka:
28 | image: bitnami/kafka
29 | ports:
30 | - '9092:9092'
31 | env:
32 | KAFKA_LISTENERS: 'PLAINTEXT://:9092'
33 | KAFKA_CFG_ADVERTISED_LISTENERS: 'PLAINTEXT://localhost:9092'
34 | KAFKA_CFG_ZOOKEEPER_CONNECT: 'zookeeper:2181'
35 | ALLOW_PLAINTEXT_LISTENER: yes
36 | steps:
37 | - name: Checkout
38 | uses: actions/checkout@v3
39 | with:
40 | fetch-depth: 0
41 |
42 | - name: Setup PHP cache environment
43 | id: php-ext-cache
44 | uses: shivammathur/cache-extensions@v1
45 | with:
46 | php-version: ${{ matrix.php }}
47 | extensions: rdkafka-arnaud-lb/php-rdkafka@6.0.3
48 | key: metamorphosis-php-extensions-${{ matrix.php }}
49 |
50 | - name: Cache PHP extensions
51 | uses: actions/cache@v3
52 | with:
53 | path: ${{ steps.php-ext-cache.outputs.dir }}
54 | key: ${{ steps.php-ext-cache.outputs.key }}
55 | restore-keys: ${{ steps.php-ext-cache.outputs.key }}
56 |
57 | - name: Setup PHP
58 | uses: shivammathur/setup-php@v2
59 | env:
60 | RDKAFKA_LIBS: librdkafka-dev
61 | with:
62 | php-version: ${{ matrix.php }}
63 | extensions: rdkafka-arnaud-lb/php-rdkafka@6.0.3
64 | tools: cs2pr
65 |
66 | - name: Cache composer dependencies
67 | uses: actions/cache@v3
68 | with:
69 | path: vendor
70 | key: metamorphosis-vendor-${{ hashFiles('composer.lock') }}
71 |
72 | - name: Composer
73 | uses: ramsey/composer-install@v2
74 |
75 | - name: Run code standards
76 | run: vendor/bin/phpcs -q --report=checkstyle | cs2pr
77 |
78 | - name: Run psalm
79 | run: vendor/bin/psalm --php-version=${{ matrix.php }} --output-format=github
80 |
81 | - name: Run tests
82 | run: vendor/bin/phpunit
83 |
84 | - name: Report Coverage
85 | continue-on-error: true
86 | run: CODACY_PROJECT_TOKEN=${{ secrets.CODACY_PROJECT_TOKEN }} bash <(curl -Ls https://coverage.codacy.com/get.sh) report
87 |
--------------------------------------------------------------------------------
/docs/CONTRIBUTING.md:
--------------------------------------------------------------------------------
1 | ## Contributing
2 |
3 | ## Getting Started
4 |
5 | 1. [Clone](https://help.github.com/en/articles/cloning-a-repository) [metamorphosis](https://github.com/leroy-merlin-br/metamorphosis)
6 |
7 | 2. We've prepare a docker environment with all required dependencies for making the process more smoothie. Build
8 | the image and install the dependencies:
9 |
10 | ```bash
11 | $ docker-compose build
12 | $ docker-compose run php composer setup-dev
13 | ```
14 |
15 | 3. We follow PHP Standards Recommendations (PSRs) by [PHP Framework Interoperability Group](http://www.php-fig.org/). If you're not familiar with these standards, [familiarize yourself now](https://github.com/php-fig/fig-standards).
16 |
17 | ## Branches
18 |
19 | To collaborate, create a new feature branch from the develop branch.
20 | Use objective names for it. If it's hard to name the branch,
21 | it could be a sign that it does a lot of things.
22 | Evaluate the possibility of breaking your contribution into more objective branches.
23 |
24 | **Examples**: `feat/add-payment-method` `fix/update-loyalty-acceptance-tests`
25 |
26 | Some examples of prefixes we can use: `feat`, `fix`, `ref`, `doc`, `chore`
27 |
28 |
29 | ## Commits
30 |
31 | We use [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) as a guide for commit messages.
32 |
33 | Try to let your commit do only one thing in the code, while not breaking if it is rolled back.
34 |
35 | Ex: Imagine that you are going to change the return of a method that was `string`, but now you can also return `null`. It's interesting that at the same time you change the method signature, you also fix the test related to this change.
36 | That way, should you need to roll back that commit, you'll have confidence that the tests will continue to pass and nothing is affected.
37 |
38 |
39 | ## Description and PR comments by the author
40 |
41 | To make easier the review of a *Pull Request*, it's interesting to write a description.
42 |
43 | This description can explain why the change is being made and also help to understand the choices made during implementation.
44 |
45 | If the author realizes that a certain piece of code may generate doubts, he can write a comment on the code snippet, explaining the reasons that led to carrying out that implementation.
46 |
47 | **Source**: https://smartbear.com/learn/code-review/best-practices-for-peer-code-review/
48 |
49 |
50 | ## Tests
51 |
52 | Tests belong in the /tests directory. There are two tests types: Unit and Integration.
53 |
54 | To run only unit tests:
55 |
56 | ```bash
57 | $ docker-compose run -rm php vendor/bin/phpunit tests/Unit
58 | ```
59 |
60 | To run only integration tests:
61 |
62 | ```bash
63 | $ docker-compose run -rm php vendor/bin/phpunit tests/Integration
64 | ```
65 |
66 | To run all tests:
67 |
68 | ```bash
69 | $ docker-compose run -rm php vendor/bin/phpunit
70 | ```
71 |
--------------------------------------------------------------------------------
/docs/CONTRIBUTING.pt.md:
--------------------------------------------------------------------------------
1 | ## Contribuindo
2 |
3 | ## Começando
4 |
5 | 1. Faça um [clone](https://help.github.com/en/articles/cloning-a-repository) do [metamorphosis](https://github.com/leroy-merlin-br/metamorphosis)
6 |
7 | 2. Preparamos um ambiente docker com todas as dependências necessárias para tornar o processo mais suave. Compile
8 | a imagem e instale as dependencias:
9 |
10 | ```bash
11 | $ docker-compose build
12 | $ docker-compose run php composer setup-dev
13 | ```
14 |
15 | 3. Seguimos as recomendações de Padrões do PHP (PSRs) do [PHP Framework Interoperability Group](http://www.php-fig.org/). Se você não estiver familiarizado com esses padrões, [familiarize-se agora](https://github.com/php-fig/fig-standards).
16 |
17 |
18 | ## Branches
19 |
20 | Para colaborar, crie um branch a partir do branch develop.
21 | Use nomes objetivos para o branch. Se for difícil nomear um branch,
22 | avalie a possibilidade de dividir a contribuição em branches mais objetivos.
23 |
24 | **Exemplos**: `feat/add-payment-method` `fix/update-loyalty-acceptance-tests`
25 |
26 | Alguns exemplos de prefixos que podemos usar: `feat`, `fix`, `ref`, `doc`, `chore`
27 |
28 | ## Commits
29 |
30 | Usamos [Conventional Commits](https://www.conventionalcommits.org/en/v1.0.0/) como um guia para mensagens de commit.
31 |
32 | Seu commit deve fazer apenas uma coisa no código, sem quebrar caso ele seja revertido.
33 |
34 | Ex: Imagine que você vai alterar o retorno de um método que era `string`,
35 | mas agora pode retornar `null`.
36 | É interessante alterar a assinatura do método e o teste relacionado a esse metodo,
37 | para garantir a integridade da funcionalidade. Dessa forma, caso seja necessário reverter esse commit,
38 | os testes continuarão a passar a nada foi afetado.
39 |
40 |
41 | ## Descrição e comentários de PR do autor
42 |
43 | Para facilitar a revisão de um *Pull Request*, é interessante escrever uma descrição.
44 |
45 | Essa descrição deve explicar porque a mudança foi feita e ajuda a entender as escolhas feitas durante a implementação.
46 |
47 | Caso o autor perceba que um trecho de código pode gerar dúvidas,
48 | ele pode escrever um comentário sobre o trecho de código,
49 | explicando os motivos que o levaram a realizar aquela implementação.
50 |
51 | **Fonte**: https://smartbear.com/learn/code-review/best-practices-for-peer-code-review/
52 |
53 |
54 | ## Testes
55 |
56 | Os testes pertencem ao diretório /tests. Existem dois tipos de testes: Unitário e Integração.
57 |
58 | Para executar apenas testes de unidade:
59 |
60 | ```bash
61 | $ docker-compose run --rm php vendor/bin/phpunit tests/Unit
62 | ```
63 |
64 | Para executar apenas testes de integração:
65 |
66 | ```bash
67 | $ docker-compose run --rm php vendor/bin/phpunit tests/Integration
68 | ```
69 |
70 | Para executar todos os testes:
71 |
72 | ```bash
73 | $ docker-compose run --rm php vendor/bin/phpunit
74 | ```
75 |
76 |
--------------------------------------------------------------------------------
/tests/Unit/Record/ProducerRecordTest.php:
--------------------------------------------------------------------------------
1 | 'original record']);
14 | $topicName = 'some_topic';
15 |
16 | // Actions
17 | $record = new Record($message, $topicName);
18 |
19 | // Assertions
20 | $this->assertSame($message, $record->getPayload());
21 | }
22 |
23 | public function testItShouldGetOriginalMessage(): void
24 | {
25 | // Set
26 | $message = json_encode(['message' => 'original record']);
27 | $topicName = 'some_topic';
28 |
29 | // Actions
30 | $record = new Record($message, $topicName);
31 |
32 | // Assertions
33 | $this->assertSame($message, $record->getOriginal());
34 | }
35 |
36 | public function testItShouldGetTopicName(): void
37 | {
38 | // Set
39 | $message = json_encode(['message' => 'original record']);
40 | $topicName = 'some_topic';
41 |
42 | // Actions
43 | $record = new Record($message, $topicName);
44 |
45 | // Assertions
46 | $this->assertSame($topicName, $record->getTopicName());
47 | }
48 |
49 | public function testItShouldGetPartition(): void
50 | {
51 | // Set
52 | $message = json_encode(['message' => 'original record']);
53 | $topicName = 'some_topic';
54 | $partition = 0;
55 |
56 | // Actions
57 | $record = new Record($message, $topicName, $partition);
58 |
59 | // Assertions
60 | $this->assertSame($partition, $record->getPartition());
61 | }
62 |
63 | public function testItShouldGetKey(): void
64 | {
65 | // Set
66 | $message = json_encode(['message' => 'original record']);
67 | $topicName = 'some_topic';
68 | $partition = 0;
69 | $key = 'message-key';
70 |
71 | // Actions
72 | $record = new Record($message, $topicName, $partition, $key);
73 |
74 | // Assertions
75 | $this->assertSame($key, $record->getKey());
76 | }
77 |
78 | public function testItShouldOverridePayload(): void
79 | {
80 | // Set
81 | $originalMessage = json_encode(['message' => 'original record']);
82 | $changedMessage = json_encode(['message' => 'changed record']);
83 | $topicName = 'some_topic';
84 |
85 | // Actions
86 | $record = new Record($originalMessage, $topicName);
87 |
88 | $record->setPayload($changedMessage);
89 |
90 | // Assertions
91 | $this->assertSame($originalMessage, $record->getOriginal());
92 | $this->assertSame($changedMessage, $record->getPayload());
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/Connectors/Consumer/Manager.php:
--------------------------------------------------------------------------------
1 | consumer = $consumer;
37 | $this->consumerHandler = $consumerHandler;
38 | $this->dispatcher = $dispatcher;
39 | $this->autoCommit = $autoCommit;
40 | $this->commitAsync = $commitAsync;
41 | $this->lastResponse = $lastResponse;
42 | }
43 |
44 | public function getConsumer(): ConsumerInterface
45 | {
46 | return $this->consumer;
47 | }
48 |
49 | public function consume(): ?Message
50 | {
51 | return $this->getConsumer()->consume();
52 | }
53 |
54 | public function handleMessage(): void
55 | {
56 | try {
57 | if ($response = $this->consume()) {
58 | $record = app(ConsumerRecord::class, compact('response'));
59 | $this->dispatcher->handle($record);
60 | $this->commit();
61 | }
62 | } catch (ResponseTimeoutException $exception) {
63 | $response = null;
64 | } catch (ResponseWarningException $exception) {
65 | $this->consumerHandler->warning($exception);
66 |
67 | return;
68 | } catch (Exception $exception) {
69 | $this->consumerHandler->failed($exception);
70 |
71 | return;
72 | }
73 |
74 | $this->handleFinished($response);
75 | }
76 |
77 | private function commit(): void
78 | {
79 | if ($this->autoCommit || !$this->consumer->canCommit()) {
80 | return;
81 | }
82 |
83 | if ($this->commitAsync) {
84 | $this->consumer->commitAsync();
85 |
86 | return;
87 | }
88 |
89 | $this->consumer->commit();
90 | }
91 |
92 | private function handleFinished(?Message $response): void
93 | {
94 | if ($this->lastResponse && !$response) {
95 | $this->consumerHandler->finished();
96 | }
97 |
98 | $this->lastResponse = $response;
99 | }
100 | }
101 |
--------------------------------------------------------------------------------
/tests/Unit/Avro/Serializer/MessageDecoderTest.php:
--------------------------------------------------------------------------------
1 | decodeMessage($message);
25 |
26 | // Assertions
27 | $this->assertSame($expected, $result);
28 | }
29 |
30 | public function testShouldNotDecodeEmptyMessage(): void
31 | {
32 | // Set
33 | $registry = m::mock(CachedSchemaRegistryClient::class);
34 | $serializer = new MessageDecoder($registry);
35 | $message = '';
36 |
37 | // Expectations
38 | $this->expectException(RuntimeException::class);
39 | $this->expectExceptionMessage('Message is too small to decode');
40 |
41 | // Actions
42 | $serializer->decodeMessage($message);
43 | }
44 |
45 | public function testShouldDecodeMessageUsingSchemaId(): void
46 | {
47 | // Set
48 | $registry = m::mock(CachedSchemaRegistryClient::class);
49 | $serializer = new MessageDecoder($registry);
50 | $decoder = m::mock(Schema::class);
51 | $avroSchema = new AvroSchema('boolean');
52 | $message = "\x00\x00\x00\x00\x07\x00";
53 | $expected = false;
54 |
55 | // Expectations
56 | $registry->expects()
57 | ->getById(7)
58 | ->andReturn($decoder);
59 |
60 | $decoder->expects()
61 | ->getAvroSchema()
62 | ->andReturn($avroSchema);
63 |
64 | // Actions
65 | $result = $serializer->decodeMessage($message);
66 |
67 | // Assertions
68 | $this->assertSame($expected, $result);
69 | }
70 |
71 | public function testShouldDecodeMessageUsingSchemaSubjectAndVersion(): void
72 | {
73 | // Set
74 | $registry = m::mock(CachedSchemaRegistryClient::class);
75 | $serializer = new MessageDecoder($registry);
76 | $decoder = m::mock(Schema::class);
77 | $avroSchema = new AvroSchema('string');
78 | $message = "\x01\x00\x00\x00\fmy-topic-key\x00\x00\x00\x05\$my awesome message";
79 | $expected = 'my awesome message';
80 |
81 | // Expectations
82 | $registry->expects()
83 | ->getBySubjectAndVersion('my-topic-key', 5)
84 | ->andReturn($decoder);
85 |
86 | $decoder->expects()
87 | ->getAvroSchema()
88 | ->andReturn($avroSchema);
89 |
90 | // Actions
91 | $result = $serializer->decodeMessage($message);
92 |
93 | // Assertions
94 | $this->assertSame($expected, $result);
95 | }
96 | }
97 |
--------------------------------------------------------------------------------
/src/Avro/CachedSchemaRegistryClient.php:
--------------------------------------------------------------------------------
1 | client = $client;
26 | }
27 |
28 | /**
29 | * GET schemas/ids/{int: id}
30 | * Retrieve a parsed avro schema by id or None if not found
31 | *
32 | * @param string|int $schemaId
33 | */
34 | public function getById($schemaId): Schema
35 | {
36 | if (isset($this->idToSchema[$schemaId])) {
37 | return $this->idToSchema[$schemaId];
38 | }
39 |
40 | $schema = app(Schema::class);
41 | $url = sprintf('schemas/ids/%d', $schemaId);
42 | [$status, $response] = $this->client->get($url);
43 |
44 | if (404 === $status) {
45 | throw new RuntimeException('Schema not found');
46 | }
47 |
48 | if (!($status >= 200 && $status < 300)) {
49 | throw new RuntimeException(
50 | 'Unable to get schema for the specific ID: ' . $status
51 | );
52 | }
53 |
54 | $schema = $schema->parse($response['schema'], $schemaId);
55 |
56 | $this->cacheSchema($schema);
57 |
58 | return $this->idToSchema[$schemaId];
59 | }
60 |
61 | /**
62 | * @param string $subject
63 | * @param int|string $version Version number or 'latest'
64 | *
65 | * @throws AvroSchemaParseException
66 | * @throws RuntimeException
67 | */
68 | public function getBySubjectAndVersion($subject, $version): Schema
69 | {
70 | if (isset($this->subjectVersionToSchema[$subject][$version])) {
71 | return $this->subjectVersionToSchema[$subject][$version];
72 | }
73 | $schema = app(Schema::class);
74 |
75 | $version = 'latest' === $version ? 'latest' : (int) $version;
76 | $url = sprintf('subjects/%s/versions/%s', $subject, $version);
77 | [$status, $response] = $this->client->get($url);
78 |
79 | if (404 === $status) {
80 | throw new RuntimeException('Schema not found');
81 | }
82 |
83 | if (!($status >= 200 && $status < 300)) {
84 | throw new RuntimeException(
85 | 'Unable to get schema for the specific ID: ' . $status
86 | );
87 | }
88 |
89 | $schemaId = $response['id'];
90 | $schema = $schema->parse(
91 | $response['schema'],
92 | $schemaId,
93 | $subject,
94 | $version
95 | );
96 |
97 | $this->cacheSchema($schema);
98 |
99 | return $this->subjectVersionToSchema[$subject][$version];
100 | }
101 |
102 | private function cacheSchema(Schema $schema): void
103 | {
104 | if ($schema->getSubject() && $schema->getVersion()) {
105 | $this->subjectVersionToSchema[$schema->getSubject()][$schema->getVersion()] = $schema;
106 | }
107 |
108 | $this->idToSchema[$schema->getSchemaId()] = $schema;
109 | }
110 | }
111 |
--------------------------------------------------------------------------------
/tests/Unit/TopicHandler/ConfigOptions/Factories/ConsumerFactoryTest.php:
--------------------------------------------------------------------------------
1 | 'kafka:9092',
16 | 'auth' => [
17 | 'type' => 'ssl',
18 | 'ca' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/ca.pem',
19 | 'certificate' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.cert',
20 | 'key' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.key',
21 | ],
22 | ];
23 | $avroSchemaData = [
24 | 'url' => '',
25 | 'ssl_verify' => true,
26 | 'request_options' => [
27 | 'headers' => [
28 | 'Authorization' => [
29 | 'Basic Og==',
30 | ],
31 | ],
32 | ],
33 | ];
34 | $topicData = [
35 | 'topic_id' => 'kafka-test',
36 | 'consumer' => [
37 | 'consumer_group' => 'test-consumer-group',
38 | 'middlewares' => [],
39 | 'auto_commit' => true,
40 | 'commit_async' => true,
41 | 'offset_reset' => 'earliest',
42 | 'handler' => '\App\Kafka\Consumers\ConsumerExample',
43 | 'partition' => 0,
44 | 'offset' => 0,
45 | 'timeout' => 20000,
46 | ],
47 | ];
48 | $expected = [
49 | 'topic_id' => 'kafka-test',
50 | 'connections' => 'kafka:9092',
51 | 'auth' => [
52 | 'type' => 'ssl',
53 | 'ca' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/ca.pem',
54 | 'certificate' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.cert',
55 | 'key' => '/var/www/html/vendor/orchestra/testbench-core/laravel/storage/kafka.key',
56 | ],
57 | 'timeout' => 20000,
58 | 'handler' => '\App\Kafka\Consumers\ConsumerExample',
59 | 'partition' => 0,
60 | 'offset' => 0,
61 | 'consumer_group' => 'test-consumer-group',
62 | 'middlewares' => [],
63 | 'url' => '',
64 | 'ssl_verify' => true,
65 | 'request_options' => [
66 | 'headers' => [
67 | 'Authorization' => [
68 | 'Basic Og==',
69 | ],
70 | ],
71 | ],
72 | 'auto_commit' => true,
73 | 'commit_async' => true,
74 | 'offset_reset' => 'earliest',
75 | 'max_poll_interval_ms' => 300000,
76 | ];
77 | // Actions
78 | $result = ConsumerFactory::make(
79 | $brokerData,
80 | $topicData,
81 | $avroSchemaData
82 | );
83 |
84 | // Assertions
85 | $this->assertInstanceOf(Consumer::class, $result);
86 | $this->assertEquals($expected, $result->toArray());
87 | }
88 | }
89 |
--------------------------------------------------------------------------------
/tests/Unit/Producer/PollTest.php:
--------------------------------------------------------------------------------
1 | expects()
38 | ->poll(0);
39 |
40 | $kafkaProducer->shouldReceive('flush')
41 | ->never();
42 |
43 | // Actions
44 | $poll->handleResponse();
45 | }
46 |
47 | public function testShouldThrowExceptionWhenFlushFailed(): void
48 | {
49 | // Set
50 | $broker = new Broker('kafka:9092', new None());
51 | $producerConfigOptions = new ProducerConfigOptions(
52 | 'topic_name',
53 | $broker,
54 | null,
55 | new AvroSchemaConfigOptions('string', []),
56 | [],
57 | 100,
58 | false,
59 | true,
60 | 500,
61 | 10
62 | );
63 | $kafkaProducer = m::mock(KafkaProducer::class);
64 | $poll = new Poll($kafkaProducer, $producerConfigOptions);
65 |
66 | // Expectations
67 | $kafkaProducer->expects()
68 | ->poll(0);
69 |
70 | $kafkaProducer->expects()
71 | ->flush(100)
72 | ->times(10)
73 | ->andReturn(1);
74 |
75 | $this->expectException(RuntimeException::class);
76 |
77 | // Actions
78 | $poll->handleResponse();
79 | }
80 |
81 | public function testItShouldHandleResponseEveryTimeWhenAsyncModeIsTrue(): void
82 | {
83 | // Set
84 | $broker = new Broker('kafka:9092', new None());
85 | $producerConfigOptions = new ProducerConfigOptions(
86 | 'topic_name',
87 | $broker,
88 | null,
89 | new AvroSchemaConfigOptions('string', []),
90 | [],
91 | 4000,
92 | false,
93 | true,
94 | 10,
95 | 500
96 | );
97 |
98 | $kafkaProducer = m::mock(KafkaProducer::class);
99 | $poll = new Poll($kafkaProducer, $producerConfigOptions);
100 |
101 | // Expectations
102 | $kafkaProducer->expects()
103 | ->poll(0)
104 | ->times(3);
105 |
106 | $kafkaProducer->expects()
107 | ->flush(4000)
108 | ->times(3)
109 | ->andReturn(0);
110 |
111 | // Actions
112 | $poll->handleResponse();
113 | $poll->handleResponse();
114 | $poll->handleResponse();
115 | }
116 | }
117 |
--------------------------------------------------------------------------------
/src/Connectors/Consumer/Config.php:
--------------------------------------------------------------------------------
1 | 'required',
26 | 'broker' => 'required',
27 | 'offset_reset' => 'required', // latest, earliest, none
28 | 'offset' => 'required_with:partition|integer',
29 | 'partition' => 'integer',
30 | 'handler' => 'required|string',
31 | 'timeout' => 'required|integer',
32 | 'consumer_group' => 'required|string',
33 | 'connections' => 'required|string',
34 | 'url' => 'string',
35 | 'ssl_verify' => 'boolean',
36 | 'auth' => 'array',
37 | 'request_options' => 'array',
38 | 'auto_commit' => 'boolean',
39 | 'commit_async' => 'boolean',
40 | 'middlewares' => 'array',
41 | ];
42 |
43 | public function makeWithConfigOptions(string $handlerClass): ?Consumer
44 | {
45 | $handler = app($handlerClass);
46 | $configOptions = $handler->getConfigOptions();
47 | if (is_null($configOptions)) {
48 | throw new InvalidArgumentException('Handler class cannot be null');
49 | }
50 |
51 | return $configOptions;
52 | }
53 |
54 | public function make(array $options, array $arguments): Consumer
55 | {
56 | $configName = $options['config_name'] ?? 'kafka';
57 | $service = $options['service_name'] ?? 'service';
58 |
59 | $topicConfig = $this->getTopicConfig($configName, $arguments['topic']);
60 | $brokerConfig = $this->getBrokerConfig($service);
61 | $schemaConfig = $this->getSchemaConfig($service);
62 |
63 | if (isset($topicConfig['consumer'])) {
64 | if (isset($options['partition'])) {
65 | $topicConfig['consumer']['partition'] = $options['partition'];
66 | }
67 |
68 | if (isset($options['offset'])) {
69 | $topicConfig['consumer']['offset'] = $options['offset'];
70 | }
71 |
72 | if (isset($options['timeout'])) {
73 | $topicConfig['consumer']['timeout'] = $options['timeout'];
74 | }
75 | }
76 |
77 | return ConsumerFactory::make(
78 | $brokerConfig,
79 | $topicConfig,
80 | $schemaConfig
81 | );
82 | }
83 |
84 | /**
85 | * @psalm-suppress InvalidReturnStatement
86 | */
87 | private function getTopicConfig(string $configName, string $topicId): array
88 | {
89 | $topicConfig = config($configName . '.topics.' . $topicId);
90 | if (!$topicConfig) {
91 | throw new ConfigurationException("Topic '{$topicId}' not found");
92 | }
93 |
94 | $topicConfig['middlewares'] = config(
95 | 'kafka.middlewares.consumer',
96 | []
97 | );
98 |
99 | return $topicConfig;
100 | }
101 | }
102 |
--------------------------------------------------------------------------------
/src/Record/ConsumerRecord.php:
--------------------------------------------------------------------------------
1 | original = $response;
30 | $this->setPayload($response->payload);
31 |
32 | if ($this->hasError()) {
33 | $this->throwResponseErrorException();
34 | }
35 | }
36 |
37 | /**
38 | * Overwrite record payload.
39 | * It is possible to get the original payload
40 | * after overwriting it by calling: $record->getOriginal()->payload.
41 | *
42 | * @param mixed $payload
43 | */
44 | public function setPayload($payload): void
45 | {
46 | $this->payload = $payload;
47 | }
48 |
49 | /**
50 | * Get the record payload.
51 | * It can either be the original value sent to Kafka or
52 | * a version modified by a middleware.
53 | *
54 | * @return mixed
55 | */
56 | public function getPayload()
57 | {
58 | return $this->payload;
59 | }
60 |
61 | /**
62 | * Get original message returned when consuming the topic.
63 | * With this object, it is possible to get original payload.
64 | *
65 | * @see https://arnaud-lb.github.io/php-rdkafka/phpdoc/class.rdkafka-message.html
66 | */
67 | public function getOriginal(): Message
68 | {
69 | return $this->original;
70 | }
71 |
72 | /**
73 | * Get the topic name where the record was published.
74 | */
75 | public function getTopicName(): string
76 | {
77 | return $this->original->topic_name;
78 | }
79 |
80 | /**
81 | * Get the partition number where the record was published.
82 | */
83 | public function getPartition(): int
84 | {
85 | return $this->original->partition;
86 | }
87 |
88 | /**
89 | * Get the record key.
90 | */
91 | public function getKey(): string
92 | {
93 | return $this->original->key;
94 | }
95 |
96 | /**
97 | * Get the record offset.
98 | */
99 | public function getOffset(): int
100 | {
101 | return $this->original->offset;
102 | }
103 |
104 | private function hasError(): bool
105 | {
106 | return RD_KAFKA_RESP_ERR_NO_ERROR !== $this->original->err;
107 | }
108 |
109 | private function throwResponseErrorException(): void
110 | {
111 | if (RD_KAFKA_RESP_ERR__TIMED_OUT === $this->original->err) {
112 | throw new ResponseTimeoutException(
113 | 'Consumer finished to process or timed out: ' . $this->original->errstr(),
114 | $this->original->err
115 | );
116 | }
117 |
118 | if (in_array($this->original->err, self::KAFKA_ERROR_WHITELIST)) {
119 | throw new ResponseWarningException(
120 | 'Invalid response: ' . $this->original->errstr(),
121 | $this->original->err
122 | );
123 | }
124 |
125 | throw new ResponseErrorException(
126 | 'Error response: ' . $this->original->errstr(),
127 | $this->original->err
128 | );
129 | }
130 | }
131 |
--------------------------------------------------------------------------------
/tests/Integration/ProducerWithConfigOptionsTest.php:
--------------------------------------------------------------------------------
1 | haveAHandlerConfigured();
25 |
26 | // I Expect That
27 | $this->myMessagesHaveBeenProduced();
28 |
29 | // When I
30 | $this->haveSomeRandomMessageProduced();
31 |
32 | // I Expect That
33 | $this->myMessagesHaveBeenLogged();
34 | $this->expectNotToPerformAssertions();
35 |
36 | // When I
37 | $this->runTheConsumer();
38 | }
39 |
40 | protected function runTheConsumer(): void
41 | {
42 | $dummy = new MessageConsumer($this->consumerConfigOptions);
43 | $this->instance('\App\Kafka\Consumers\ConsumerOverride', $dummy);
44 |
45 | $this->artisan(
46 | 'kafka:consume-config-class',
47 | [
48 | 'handler' => '\\App\\Kafka\\Consumers\\ConsumerOverride',
49 | '--times' => 2,
50 | ]
51 | );
52 | }
53 |
54 | protected function haveAHandlerConfigured(): void
55 | {
56 | $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092');
57 | $broker = new Broker($connections, new None());
58 | $this->producerConfigOptions = new ProducerConfigOptions(
59 | 'sale_order_override',
60 | $broker,
61 | null,
62 | null,
63 | [],
64 | 20000,
65 | false,
66 | true,
67 | 10,
68 | 100
69 | );
70 |
71 | $this->consumerConfigOptions = new ConsumerConfigOptions(
72 | 'sale_order_override',
73 | $broker,
74 | '\App\Kafka\Consumers\ConsumerOverride',
75 | null,
76 | null,
77 | 'test-consumer-group',
78 | null,
79 | [],
80 | 20000,
81 | false,
82 | true
83 | );
84 | }
85 |
86 | private function haveSomeRandomMessageProduced(): void
87 | {
88 | $saleOrderProducer = app(
89 | MessageProducerWithConfigOptions::class,
90 | [
91 | 'record' => ['saleOrderId' => 'SALE_ORDER_ID'],
92 | 'configOptions' => $this->producerConfigOptions,
93 | 'key' => 1,
94 | ]
95 | );
96 |
97 | $saleOrderDispatcher = Metamorphosis::build($saleOrderProducer);
98 | $saleOrderDispatcher->handle($saleOrderProducer->createRecord());
99 | }
100 |
101 | private function myMessagesHaveBeenLogged()
102 | {
103 | Log::shouldReceive('alert')
104 | ->with('{"saleOrderId":"SALE_ORDER_ID"}');
105 | }
106 |
107 | private function myMessagesHaveBeenProduced()
108 | {
109 | Log::shouldReceive('info')
110 | ->with('Record successfully sent to broker.', [
111 | 'topic' => 'sale_order_override',
112 | 'payload' => '{"saleOrderId":"SALE_ORDER_ID"}',
113 | 'key' => '1',
114 | 'partition' => 0,
115 | ]);
116 | }
117 | }
118 |
--------------------------------------------------------------------------------
/config/kafka.php:
--------------------------------------------------------------------------------
1 | [
5 | // This is your topic "keyword" where you will put all configurations needed
6 | // on this specific topic.
7 | 'default' => [
8 | // The topic id is where you want to send or consume
9 | // your messages from kafka.
10 | 'topic_id' => 'kafka-test',
11 |
12 | //your consumer configurations
13 | 'consumer' => [
14 | 'consumer_group' => 'test-consumer-group',
15 | // Action to take when there is no initial
16 | // offset in offset store or the desired offset is out of range.
17 | // This config will be passed to 'auto.offset.reset'.
18 | // The valid options are: smallest, earliest, beginning, largest, latest, end, error.
19 | 'offset_reset' => 'earliest',
20 |
21 | // The offset at which to start consumption. This only applies if partition is set.
22 | // You can use a positive integer or any of the constants: RD_KAFKA_OFFSET_BEGINNING,
23 | // RD_KAFKA_OFFSET_END, RD_KAFKA_OFFSET_STORED.
24 | 'offset' => 0,
25 |
26 | // The partition to consume. It can be null,
27 | // if you don't wish do specify one.
28 | 'partition' => 0,
29 |
30 | // A consumer class that implements ConsumerTopicHandler
31 | 'handler' => '\App\Kafka\Consumers\ConsumerExample',
32 |
33 | // A Timeout to listen to a message. That means: how much
34 | // time we need to wait until receiving a message?
35 | 'timeout' => 20000,
36 |
37 | // Once you've enabled this, the Kafka consumer will commit the
38 | // offset of the last message received in response to its poll() call
39 | 'auto_commit' => true,
40 |
41 | // If commit_async is false process block until offsets are committed or the commit fails.
42 | // Only works when auto_commit is false
43 | 'commit_async' => false,
44 |
45 | // An array of middlewares applied only for this consumer_group
46 | 'middlewares' => [],
47 |
48 | // A max interval for consumer to make poll calls. That means: how much
49 | // time we need to wait for poll calls until consider the consumer has inactive.
50 | 'max_poll_interval_ms' => 300000,
51 | ],
52 |
53 | 'producer' => [
54 | // Sets to true if you want to know if a message was successfully posted.
55 | 'required_acknowledgment' => true,
56 |
57 | // Whether if you want to receive the response asynchronously.
58 | 'is_async' => true,
59 |
60 | // The amount of records to be sent in every iteration
61 | // That means that at each 500 messages we check if messages was sent.
62 | 'max_poll_records' => 500,
63 |
64 | // The amount of attempts we will try to run the flush.
65 | // There's no magic number here, it depends on any factor
66 | // Try yourself a good number.
67 | 'flush_attempts' => 10,
68 |
69 | // Middlewares specific for this producer.
70 | 'middlewares' => [],
71 |
72 | // We need to set a timeout when polling the messages.
73 | // That means: how long we'll wait a response from poll
74 | 'timeout' => 10000,
75 |
76 | // Here you can configure which partition you want to send the message
77 | // it can be -1 (RD_KAFKA_PARTITION_UA) to let Kafka decide, or an int with the partition number
78 | 'partition' => defined('RD_KAFKA_PARTITION_UA')
79 | ? constant('RD_KAFKA_PARTITION_UA')
80 | : -1,
81 | ],
82 | ],
83 | ],
84 | ];
85 |
--------------------------------------------------------------------------------
/tests/Unit/Middlewares/AvroSchemaMixedEncoderTest.php:
--------------------------------------------------------------------------------
1 | getSchemaFixture();
26 | $broker = new Broker('kafka:9092', new None());
27 | $producerConfigOptions = new ProducerConfigOptions(
28 | 'kafka-test',
29 | $broker,
30 | null,
31 | new AvroSchemaConfigOptions(
32 | 'subjects/kafka-test-value/versions/latest',
33 | []
34 | )
35 | );
36 | $avroSchemaConfigOptions = $producerConfigOptions->getAvroSchema();
37 |
38 | $clientFactory = m::mock(ClientFactory::class);
39 |
40 | $cachedSchemaRegistryClient = m::mock(
41 | CachedSchemaRegistryClient::class
42 | );
43 | $schemaIdEncoder = m::mock(
44 | SchemaId::class,
45 | [$cachedSchemaRegistryClient]
46 | );
47 |
48 | $schema = new Schema();
49 | $parsedSchema = $schema->parse(
50 | $avroSchema,
51 | '123',
52 | 'kafka-test-value',
53 | 'latest'
54 | );
55 | $record = $this->getRecord($parsedSchema->getAvroSchema());
56 | $producerRecord = new ProducerRecord($record, 'kafka-test');
57 |
58 | $closure = Closure::fromCallable(function ($producerRecord) {
59 | return $producerRecord;
60 | });
61 |
62 | $payload = json_decode($producerRecord->getPayload(), true);
63 | $encodedMessage = 'binary_message';
64 |
65 | // Expectations
66 | $clientFactory->expects()
67 | ->make($avroSchemaConfigOptions)
68 | ->andReturn($cachedSchemaRegistryClient);
69 |
70 | $cachedSchemaRegistryClient->expects()
71 | ->getBySubjectAndVersion('kafka-test-value', 'latest')
72 | ->andReturn($schema);
73 |
74 | $schemaIdEncoder->expects()
75 | ->encode($schema, $payload)
76 | ->andReturn($encodedMessage);
77 |
78 | // Actions
79 | $avroSchemaMixedEncoder = new AvroSchemaMixedEncoder(
80 | $schemaIdEncoder,
81 | $clientFactory,
82 | $producerConfigOptions
83 | );
84 | $result = $avroSchemaMixedEncoder->process($producerRecord, $closure);
85 |
86 | // Assertions
87 | $this->assertSame($record, $result->getOriginal());
88 | $this->assertSame($encodedMessage, $result->getPayload());
89 | }
90 |
91 | private function getRecord(AvroSchema $avroSchema): string
92 | {
93 | $defaultValues = [
94 | 'null' => null,
95 | 'boolean' => true,
96 | 'string' => 'abc',
97 | 'int' => 1,
98 | 'long' => 1.0,
99 | 'float' => 1.0,
100 | 'double' => 1.0,
101 | 'array' => [],
102 | ];
103 |
104 | $result = [];
105 | foreach ($avroSchema->fields() as $field) {
106 | $result[$field->name()] = $defaultValues[$field->type->type];
107 | }
108 |
109 | return json_encode($result);
110 | }
111 |
112 | private function getSchemaFixture(): string
113 | {
114 | return file_get_contents(
115 | __DIR__ . '/../fixtures/schemas/sales_price.avsc'
116 | );
117 | }
118 | }
119 |
--------------------------------------------------------------------------------
/tests/Unit/Connectors/Producer/ConnectorTest.php:
--------------------------------------------------------------------------------
1 | instance(
24 | Conf::class,
25 | m::mock(Conf::class)
26 | );
27 | $this->instance(
28 | KafkaProducer::class,
29 | m::mock(KafkaProducer::class)
30 | );
31 |
32 | $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092');
33 | $broker = new Broker($connections, new None());
34 | $producerConfigOptions = m::mock(ProducerConfigOptions::class);
35 |
36 | $connector = new Connector();
37 | $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandleableResponseInterface {
38 | /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */
39 | public function success(Message $message): void
40 | {
41 | }
42 |
43 | /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */
44 | public function failed(Message $message): void
45 | {
46 | }
47 | };
48 |
49 | // Expectations
50 | $conf->expects()
51 | ->setDrMsgCb()
52 | ->withAnyArgs();
53 |
54 | $conf->expects()
55 | ->set('metadata.broker.list', $connections);
56 |
57 | $producerConfigOptions->expects()
58 | ->getBroker()
59 | ->andReturn($broker);
60 |
61 | // Actions
62 | $result = $connector->getProducerTopic(
63 | $handler,
64 | $producerConfigOptions
65 | );
66 |
67 | // Assertions
68 | $this->assertInstanceOf(KafkaProducer::class, $result);
69 | }
70 |
71 | public function testItShouldMakeSetupWithoutHandleResponse(): void
72 | {
73 | // Set
74 | $conf = $this->instance(
75 | Conf::class,
76 | m::mock(Conf::class)
77 | );
78 | $this->instance(
79 | KafkaProducer::class,
80 | m::mock(KafkaProducer::class)
81 | );
82 |
83 | $connections = env('KAFKA_BROKER_CONNECTIONS', 'kafka:9092');
84 | $broker = new Broker($connections, new None());
85 | $producerConfigOptions = m::mock(ProducerConfigOptions::class);
86 |
87 | $connector = new Connector();
88 | $handler = new class ('record', $producerConfigOptions) extends AbstractProducer implements HandlerInterface {
89 | /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */
90 | public function success(Message $message): void
91 | {
92 | }
93 |
94 | /** @phpcsSuppress SlevomatCodingStandard.Functions.UnusedParameter.UnusedParameter */
95 | public function failed(Message $message): void
96 | {
97 | }
98 | };
99 |
100 | // Expectations
101 | $conf->shouldReceive('setDrMsgCb')
102 | ->never();
103 |
104 | $conf->expects()
105 | ->set('metadata.broker.list', $connections);
106 |
107 | $producerConfigOptions->expects()
108 | ->getBroker()
109 | ->andReturn($broker);
110 |
111 | // Actions
112 | $result = $connector->getProducerTopic(
113 | $handler,
114 | $producerConfigOptions
115 | );
116 |
117 | // Assertions
118 | $this->assertInstanceOf(KafkaProducer::class, $result);
119 | }
120 | }
121 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Producer.php:
--------------------------------------------------------------------------------
1 | broker = $broker;
69 | $this->middlewares = $middlewares;
70 | $this->timeout = $timeout;
71 | $this->isAsync = $isAsync;
72 | $this->requiredAcknowledgment = $requiredAcknowledgment;
73 | $this->maxPollRecords = $maxPollRecords;
74 | $this->flushAttempts = $flushAttempts;
75 | $this->topicId = $topicId;
76 | $this->avroSchema = $avroSchema;
77 | $this->partition = $partition;
78 | }
79 |
80 | public function getTimeout(): int
81 | {
82 | return $this->timeout;
83 | }
84 |
85 | public function isRequiredAcknowledgment(): bool
86 | {
87 | return $this->requiredAcknowledgment;
88 | }
89 |
90 | public function getMiddlewares(): array
91 | {
92 | return $this->middlewares;
93 | }
94 |
95 | public function getMaxPollRecords(): int
96 | {
97 | return $this->maxPollRecords;
98 | }
99 |
100 | public function isAsync(): bool
101 | {
102 | return $this->isAsync;
103 | }
104 |
105 | public function getFlushAttempts(): int
106 | {
107 | return $this->flushAttempts;
108 | }
109 |
110 | public function getBroker(): Broker
111 | {
112 | return $this->broker;
113 | }
114 |
115 | public function getTopicId(): string
116 | {
117 | return $this->topicId;
118 | }
119 |
120 | public function getAvroSchema(): ?AvroSchema
121 | {
122 | return $this->avroSchema;
123 | }
124 |
125 | public function toArray(): array
126 | {
127 | $data = [
128 | 'topic_id' => $this->getTopicId(),
129 | 'timeout' => $this->getTimeout(),
130 | 'partition' => $this->getPartition(),
131 | 'is_async' => $this->isAsync(),
132 | 'required_acknowledgment' => $this->isRequiredAcknowledgment(),
133 | 'max_poll_records' => $this->getMaxPollRecords(),
134 | 'flush_attempts' => $this->getFlushAttempts(),
135 | 'middlewares' => $this->getMiddlewares(),
136 | ];
137 |
138 | if ($avroSchema = $this->getAvroSchema()) {
139 | $data = array_merge($data, $avroSchema->toArray());
140 | }
141 |
142 | return array_merge($this->broker->toArray(), $data);
143 | }
144 |
145 | public function getPartition(): int
146 | {
147 | return $this->partition ?? RD_KAFKA_PARTITION_UA;
148 | }
149 | }
150 |
--------------------------------------------------------------------------------
/src/TopicHandler/ConfigOptions/Consumer.php:
--------------------------------------------------------------------------------
1 | broker = $broker;
59 | $this->middlewares = $middlewares;
60 | $this->timeout = $timeout;
61 | $this->partition = $partition;
62 | $this->offset = $offset;
63 | $this->topicId = $topicId;
64 | $this->avroSchema = $avroSchema;
65 | $this->consumerGroup = $consumerGroup;
66 | $this->handler = $handler;
67 | $this->autoCommit = $autoCommit;
68 | $this->commitASync = $commitASync;
69 | $this->offsetReset = $offsetReset;
70 | $this->maxPollInterval = $maxPollInterval;
71 | }
72 |
73 | public function getTimeout(): int
74 | {
75 | return $this->timeout;
76 | }
77 |
78 | public function getMiddlewares(): array
79 | {
80 | return $this->middlewares;
81 | }
82 |
83 | public function getBroker(): Broker
84 | {
85 | return $this->broker;
86 | }
87 |
88 | public function getPartition(): int
89 | {
90 | return $this->partition ?? RD_KAFKA_PARTITION_UA;
91 | }
92 |
93 | public function getTopicId(): string
94 | {
95 | return $this->topicId;
96 | }
97 |
98 | public function getAvroSchema(): ?AvroSchema
99 | {
100 | return $this->avroSchema;
101 | }
102 |
103 | public function getConsumerGroup(): string
104 | {
105 | return $this->consumerGroup;
106 | }
107 |
108 | public function getHandler(): ?string
109 | {
110 | return $this->handler;
111 | }
112 |
113 | public function isAutoCommit(): bool
114 | {
115 | return $this->autoCommit;
116 | }
117 |
118 | public function isCommitASync(): bool
119 | {
120 | return $this->commitASync;
121 | }
122 |
123 | public function getOffsetReset(): string
124 | {
125 | return $this->offsetReset;
126 | }
127 |
128 | public function toArray(): array
129 | {
130 | $data = [
131 | 'topic_id' => $this->getTopicId(),
132 | 'timeout' => $this->getTimeout(),
133 | 'handler' => $this->getHandler(),
134 | 'partition' => $this->getPartition(),
135 | 'offset' => $this->getOffset(),
136 | 'consumer_group' => $this->getConsumerGroup(),
137 | 'middlewares' => $this->getMiddlewares(),
138 | 'auto_commit' => $this->isAutoCommit(),
139 | 'commit_async' => $this->isCommitASync(),
140 | 'offset_reset' => $this->getOffsetReset(),
141 | 'max_poll_interval_ms' => $this->getMaxPollInterval(),
142 | ];
143 |
144 | if ($avroSchema = $this->getAvroSchema()) {
145 | $data = array_merge($data, $avroSchema->toArray());
146 | }
147 |
148 | return array_merge($this->broker->toArray(), $data);
149 | }
150 |
151 | public function getOffset(): ?int
152 | {
153 | return $this->offset;
154 | }
155 | public function getMaxPollInterval(): int
156 | {
157 | return $this->maxPollInterval;
158 | }
159 |
160 | public function setMaxPollInterval(int $maxPollInterval): void
161 | {
162 | $this->maxPollInterval = $maxPollInterval;
163 | }
164 | }
165 |
--------------------------------------------------------------------------------
/tests/Unit/Console/ConsumerCommandTest.php:
--------------------------------------------------------------------------------
1 | 'some_topic',
19 | ];
20 |
21 | // Expectations
22 | $this->expectException(ConfigurationException::class);
23 | $this->expectExceptionMessage('Topic \'some_topic\' not found');
24 |
25 | // Actions
26 | $this->artisan($command, $parameters);
27 | }
28 |
29 | public function testItCallsCommandWithOffsetWithoutPartition(): void
30 | {
31 | // Set
32 | $runner = $this->instance(Runner::class, m::mock(Runner::class));
33 | $command = 'kafka:consume';
34 | $parameters = [
35 | 'topic' => 'some_topic',
36 | '--offset' => 1,
37 | ];
38 |
39 | // Expectations
40 | $runner->expects()
41 | ->run()
42 | ->never();
43 |
44 | $this->expectException(ConfigurationException::class);
45 |
46 | // Actions
47 | $this->artisan($command, $parameters);
48 | }
49 |
50 | public function testItCallsWithHighLevelConsumer(): void
51 | {
52 | // Set
53 | $runner = $this->instance(Runner::class, m::mock(Runner::class));
54 | $command = 'kafka:consume';
55 | $parameters = [
56 | 'topic' => 'topic_key',
57 | 'consumer_group' => 'default',
58 | '--times' => 66,
59 | ];
60 |
61 | // Expectations
62 | $runner->expects()
63 | ->run(66)
64 | ->once();
65 |
66 | // Actions
67 | $this->artisan($command, $parameters);
68 | }
69 |
70 | public function testItCallsWithLowLevelConsumer(): void
71 | {
72 | // Set
73 | $runner = $this->instance(Runner::class, m::mock(Runner::class));
74 | $command = 'kafka:consume';
75 | $parameters = [
76 | 'topic' => 'topic_key',
77 | '--partition' => 1,
78 | '--offset' => 5,
79 | ];
80 |
81 | // Expectations
82 | $runner->expects()
83 | ->run(null)
84 | ->once();
85 |
86 | // Actions
87 | $this->artisan($command, $parameters);
88 | }
89 |
90 | public function testItAcceptsTimeoutWhenCallingCommand(): void
91 | {
92 | // Set
93 | $runner = $this->instance(Runner::class, m::mock(Runner::class));
94 | $command = 'kafka:consume';
95 | $parameters = [
96 | 'topic' => 'topic_key',
97 | '--timeout' => 1,
98 | ];
99 |
100 | // Expectations
101 | $runner->expects()
102 | ->run(null)
103 | ->once();
104 |
105 | // Actions
106 | $this->artisan($command, $parameters);
107 | }
108 |
109 | public function testItOverridesBrokerConnectionWhenCallingCommand(): void
110 | {
111 | // Set
112 | config([
113 | 'kafka.brokers.some-broker' => [
114 | 'connections' => '',
115 | 'auth' => [],
116 | ],
117 | ]);
118 |
119 | $runner = $this->instance(Runner::class, m::mock(Runner::class));
120 | $command = 'kafka:consume';
121 | $parameters = [
122 | 'topic' => 'topic_key',
123 | '--timeout' => 1,
124 | '--broker' => 'some-broker',
125 | ];
126 |
127 | // Expectations
128 | $runner->expects()
129 | ->run(null)
130 | ->once();
131 |
132 | $this->artisan($command, $parameters);
133 | }
134 |
135 | protected function setUp(): void
136 | {
137 | parent::setUp();
138 |
139 | config([
140 | 'kafka' => [
141 | 'topics' => [
142 | 'topic_key' => [
143 | 'topic_id' => 'topic_name',
144 | 'consumer' => [
145 | 'consumer_group' => 'default',
146 | 'offset_reset' => 'earliest',
147 | 'handler' => ConsumerHandlerDummy::class,
148 | 'timeout' => 123,
149 | 'max_poll_interval_ms' => 300000,
150 | ],
151 | ],
152 | ],
153 | ],
154 | 'service' => [
155 | 'broker' => [
156 | 'connections' => 'test_kafka:6680',
157 | 'auth' => [],
158 | ],
159 | ],
160 | ]);
161 | }
162 | }
163 |
--------------------------------------------------------------------------------