├── docs
    ├── qq_group.png
    ├── ch
    │   └── Configure.md
    ├── error.md
    ├── state.md
    └── Configure.md
├── Dockerfile
├── src
    ├── Exception.php
    ├── Exception
    │   ├── Config.php
    │   ├── Protocol.php
    │   ├── NotSupported.php
    │   ├── ConnectionException.php
    │   ├── InvalidRecordInSet.php
    │   └── Socket.php
    ├── Consumer
    │   ├── StopStrategy.php
    │   ├── StopStrategy
    │   │   ├── Delay.php
    │   │   └── Callback.php
    │   └── Assignment.php
    ├── SaslMechanism.php
    ├── SocketCoroutine.php
    ├── SingletonTrait.php
    ├── Sasl
    │   ├── Plain.php
    │   ├── Mechanism.php
    │   ├── Gssapi.php
    │   └── Scram.php
    ├── Producer
    │   ├── RecordValidator.php
    │   ├── CoroutineProcess.php
    │   ├── State.php
    │   └── Process.php
    ├── Protocol
    │   ├── LeaveGroup.php
    │   ├── ApiVersions.php
    │   ├── ListGroup.php
    │   ├── Heartbeat.php
    │   ├── GroupCoordinator.php
    │   ├── SaslHandShake.php
    │   ├── FetchOffset.php
    │   ├── Metadata.php
    │   ├── Offset.php
    │   ├── DescribeGroups.php
    │   ├── CommitOffset.php
    │   ├── SyncGroup.php
    │   ├── JoinGroup.php
    │   ├── Produce.php
    │   └── Fetch.php
    ├── SocketSync.php
    ├── Producer.php
    ├── Consumer.php
    ├── ProducerConfig.php
    ├── LoggerTrait.php
    ├── ConsumerConfig.php
    ├── Socket.php
    ├── SwooleSocket.php
    ├── Broker.php
    ├── kafka.dio
    └── Config.php
├── .coveralls.yml
├── docker-compose.yml
├── README_CH.md
├── README.md
└── LICENSE


/docs/qq_group.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/wjcgithub/kafka-php/HEAD/docs/qq_group.png


--------------------------------------------------------------------------------
/Dockerfile:
--------------------------------------------------------------------------------
1 | FROM php:7.1-alpine
2 | 
3 | WORKDIR /opt/kafka-php
4 | 
5 | CMD ["./vendor/bin/phpunit", "--testsuite", "functional"]
6 | 


--------------------------------------------------------------------------------
/src/Exception.php:
--------------------------------------------------------------------------------
1 | <?php
2 | declare(strict_types=1);
3 | 
4 | namespace Kafka;
5 | 
6 | class Exception extends \RuntimeException
7 | {
8 | }
9 | 


--------------------------------------------------------------------------------
/.coveralls.yml:
--------------------------------------------------------------------------------
1 | repo_token: ojc3Y2LcKW8VCM8CW2I0phJi1f7FJXzPm
2 | coverage_clover: build/logs/clover.xml
3 | json_path: build/logs/coveralls-upload.json
4 | 


--------------------------------------------------------------------------------
/src/Exception/Config.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Exception;
 5 | 
 6 | use Kafka\Exception;
 7 | 
 8 | class Config extends Exception
 9 | {
10 | }
11 | 


--------------------------------------------------------------------------------
/src/Exception/Protocol.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Exception;
 5 | 
 6 | use Kafka\Exception;
 7 | 
 8 | class Protocol extends Exception
 9 | {
10 | }
11 | 


--------------------------------------------------------------------------------
/src/Exception/NotSupported.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Exception;
 5 | 
 6 | use Kafka\Exception;
 7 | 
 8 | class NotSupported extends Exception
 9 | {
10 | }
11 | 


--------------------------------------------------------------------------------
/src/Consumer/StopStrategy.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Consumer;
 5 | 
 6 | use Kafka\Consumer;
 7 | 
 8 | interface StopStrategy
 9 | {
10 |     public function setup(Consumer $consumer): void;
11 | }
12 | 


--------------------------------------------------------------------------------
/src/SaslMechanism.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka;
 5 | 
 6 | interface SaslMechanism
 7 | {
 8 |     /**
 9 |      *
10 |      * sasl authenticate
11 |      *
12 |      * @access public
13 |      */
14 |     public function authenticate(CommonSocket $socket): void;
15 | }
16 | 


--------------------------------------------------------------------------------
/src/Exception/ConnectionException.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Exception;
 5 | 
 6 | use Kafka\Exception;
 7 | use function sprintf;
 8 | 
 9 | final class ConnectionException extends Exception
10 | {
11 |     public static function fromBrokerList(string $brokerList): self
12 |     {
13 |         return new self(
14 |             sprintf(
15 |                 'It was not possible to establish a connection for metadata with the brokers "%s"',
16 |                 $brokerList
17 |             )
18 |         );
19 |     }
20 | }
21 | 


--------------------------------------------------------------------------------
/src/SocketCoroutine.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka;
 5 | 
 6 | use function fclose;
 7 | use function is_resource;
 8 | use function rewind;
 9 | 
10 | class SocketCoroutine extends SwooleSocket
11 | {
12 |     public function connect(): void
13 |     {
14 |         if (is_resource($this->stream)) {
15 |             return;
16 |         }
17 | 
18 |         $this->createStream();
19 |     }
20 | 
21 |     public function close(): void
22 |     {
23 |         if (is_resource($this->stream)) {
24 |             fclose($this->stream);
25 |         }
26 |     }
27 | 
28 |     public function isResource(): bool
29 |     {
30 |         return is_resource($this->stream);
31 |     }
32 | }
33 | 


--------------------------------------------------------------------------------
/src/SingletonTrait.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka;
 5 | 
 6 | use Psr\Log\LoggerAwareTrait;
 7 | 
 8 | trait SingletonTrait
 9 | {
10 |     use LoggerAwareTrait;
11 |     use LoggerTrait;
12 | 
13 |     /**
14 |      * @var object
15 |      */
16 |     protected static $instance;
17 | 
18 |     /**
19 |      * Need to be compatible php 7.1.x, so this scene cannot be specified return type `object`
20 |      * @return object
21 |      */
22 |     public static function getInstance()
23 |     {
24 |         if (self::$instance === null) {
25 |             static::$instance = new static();
26 |         }
27 | 
28 |         return static::$instance;
29 |     }
30 | 
31 |     private function __construct()
32 |     {
33 |     }
34 | }
35 | 


--------------------------------------------------------------------------------
/src/Consumer/StopStrategy/Delay.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Consumer\StopStrategy;
 5 | 
 6 | use Amp\Loop;
 7 | use Kafka\Consumer;
 8 | use Kafka\Consumer\StopStrategy;
 9 | 
10 | final class Delay implements StopStrategy
11 | {
12 |     /**
13 |      * The amount of time, in milliseconds, to stop the consumer
14 |      *
15 |      * @var int
16 |      */
17 |     private $delay;
18 | 
19 |     public function __construct(int $delay)
20 |     {
21 |         $this->delay = $delay;
22 |     }
23 | 
24 |     public function setup(Consumer $consumer): void
25 |     {
26 |         Loop::delay(
27 |             $this->delay,
28 |             function () use ($consumer): void {
29 |                 $consumer->stop();
30 |             }
31 |         );
32 |     }
33 | }
34 | 


--------------------------------------------------------------------------------
/src/Exception/InvalidRecordInSet.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Exception;
 5 | 
 6 | use Kafka\Exception;
 7 | use function sprintf;
 8 | 
 9 | final class InvalidRecordInSet extends Exception
10 | {
11 |     public static function missingTopic(): self
12 |     {
13 |         return new self('You have to set "topic" to your message.');
14 |     }
15 | 
16 |     public static function nonExististingTopic(string $topic): self
17 |     {
18 |         return new self(sprintf('Requested topic "%s" does not exist. Did you forget to create it?', $topic));
19 |     }
20 | 
21 |     /**
22 |      * FIXME: kill with 🔥
23 |      */
24 |     public static function topicIsNotString(): self
25 |     {
26 |         return new self('Topic must be string.');
27 |     }
28 | 
29 |     public static function missingValue(): self
30 |     {
31 |         return new self('You have to set "value" to your message.');
32 |     }
33 | 
34 |     /**
35 |      * FIXME: kill with 🔥
36 |      */
37 |     public static function valueIsNotString(): self
38 |     {
39 |         return new self('Value must be string.');
40 |     }
41 | }
42 | 


--------------------------------------------------------------------------------
/src/Sasl/Plain.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Sasl;
 5 | 
 6 | use Kafka\CommonSocket;
 7 | use Kafka\Protocol\Protocol;
 8 | use function trim;
 9 | 
10 | class Plain extends Mechanism
11 | {
12 |     private const MECHANISM_NAME = 'PLAIN';
13 | 
14 |     /**
15 |      * @var string
16 |      */
17 |     private $username;
18 | 
19 |     /**
20 |      * @var string
21 |      */
22 |     private $password;
23 | 
24 | 
25 |     public function __construct(string $username, string $password)
26 |     {
27 |         $this->username = trim($username);
28 |         $this->password = trim($password);
29 |     }
30 | 
31 |     protected function performAuthentication(CommonSocket $socket): void
32 |     {
33 |         $split = Protocol::pack(Protocol::BIT_B8, '0');
34 | 
35 |         $data = Protocol::encodeString(
36 |             $split . $this->username . $split . $this->password,
37 |             Protocol::PACK_INT32
38 |         );
39 | 
40 |         $socket->writeBlocking($data);
41 |         $socket->readBlocking(4);
42 |     }
43 | 
44 |     public function getName(): string
45 |     {
46 |         return self::MECHANISM_NAME;
47 |     }
48 | }
49 | 


--------------------------------------------------------------------------------
/src/Exception/Socket.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Exception;
 5 | 
 6 | use Kafka\Exception;
 7 | use function sprintf;
 8 | 
 9 | final class Socket extends Exception
10 | {
11 |     public static function invalidLength(int $length, int $maxLength): self
12 |     {
13 |         return new self(sprintf('Invalid length %d given, it should be lesser than or equals to %d', $length, $maxLength));
14 |     }
15 | 
16 |     public static function notReadable(int $length): self
17 |     {
18 |         return new self(sprintf('Could not read %d bytes from stream (not readable)', $length));
19 |     }
20 | 
21 |     public static function timedOut(int $length): self
22 |     {
23 |         return new self(sprintf('Timed out reading %d bytes from stream', $length));
24 |     }
25 | 
26 |     public static function timedOutWithRemainingBytes(int $length, int $remainingBytes): self
27 |     {
28 |         return new self(sprintf('Timed out while reading %d bytes from stream, %d bytes are still needed', $length, $remainingBytes));
29 |     }
30 | 
31 |     public static function unexpectedEOF(int $length): self
32 |     {
33 |         return new self(sprintf('Unexpected EOF while reading %d bytes from stream (no data)', $length));
34 |     }
35 | }
36 | 


--------------------------------------------------------------------------------
/docker-compose.yml:
--------------------------------------------------------------------------------
 1 | version: '3.2'
 2 | 
 3 | services:
 4 |   zookeeper:
 5 |     image: zookeeper:3.4
 6 | 
 7 |   kafka-0.9.0.1:
 8 |     image: wurstmeister/kafka:0.9.0.1
 9 |     depends_on:
10 |       - zookeeper
11 |     environment:
12 |       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
13 |       KAFKA_ADVERTISED_HOST_NAME: kafka-0.9.0.1
14 |       KAFKA_ADVERTISED_PORT: 9092
15 |     ports:
16 |       - "9092"
17 | 
18 |   kafka-0.10.2.1:
19 |     image: wurstmeister/kafka:0.10.2.1
20 |     depends_on:
21 |       - zookeeper
22 |     environment:
23 |       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
24 |       KAFKA_ADVERTISED_HOST_NAME: kafka-0.10.2.1
25 |       KAFKA_ADVERTISED_PORT: 9092
26 |     ports:
27 |       - "9092"
28 | 
29 |   kafka-0.11.0.1:
30 |     image: wurstmeister/kafka:0.11.0.1
31 |     depends_on:
32 |       - zookeeper
33 |     environment:
34 |       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
35 |       KAFKA_ADVERTISED_HOST_NAME: kafka-0.11.0.1
36 |     ports:
37 |       - "9092"
38 | 
39 |   kafka-1.0.0:
40 |     image: wurstmeister/kafka:1.0.0
41 |     depends_on:
42 |       - zookeeper
43 |     environment:
44 |       KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
45 |       KAFKA_ADVERTISED_HOST_NAME: kafka-1.0.0
46 |     ports:
47 |       - "9092"
48 | 
49 |   test-runner:
50 |     build: .
51 |     volumes:
52 |        - .:/opt/kafka-php
53 | 


--------------------------------------------------------------------------------
/src/Consumer/StopStrategy/Callback.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Consumer\StopStrategy;
 5 | 
 6 | use Amp\Loop;
 7 | use Kafka\Consumer;
 8 | use Kafka\Consumer\StopStrategy;
 9 | 
10 | final class Callback implements StopStrategy
11 | {
12 |     private const DEFAULT_INTERVAL = 250;
13 | 
14 |     /**
15 |      * The verification callback that will be executed to check if the consumer must be stopped or not
16 |      *
17 |      * @var callable
18 |      */
19 |     private $callback;
20 | 
21 |     /**
22 |      * The time interval, in milliseconds, to wait between executions
23 |      *
24 |      * @var int
25 |      */
26 |     private $interval;
27 | 
28 |     public function __construct(callable $callback, int $interval = self::DEFAULT_INTERVAL)
29 |     {
30 |         $this->callback = $callback;
31 |         $this->interval = $interval;
32 |     }
33 | 
34 |     public function setup(Consumer $consumer): void
35 |     {
36 |         Loop::repeat(
37 |             $this->interval,
38 |             function (string $watcherId) use ($consumer): void {
39 |                 $shouldStop = (bool) ($this->callback)();
40 | 
41 |                 if (! $shouldStop) {
42 |                     return;
43 |                 }
44 | 
45 |                 $consumer->stop();
46 |                 Loop::cancel($watcherId);
47 |             }
48 |         );
49 |     }
50 | }
51 | 


--------------------------------------------------------------------------------
/src/Producer/RecordValidator.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Producer;
 5 | 
 6 | use Kafka\Exception;
 7 | use function is_string;
 8 | use function trim;
 9 | 
10 | /**
11 |  * @internal
12 |  */
13 | final class RecordValidator
14 | {
15 |     /**
16 |      * @param string[]  $record
17 |      * @param mixed[][] $topicList
18 |      */
19 |     public function validate(array $record, array $topicList): void
20 |     {
21 |         if (! isset($record['topic'])) {
22 |             throw Exception\InvalidRecordInSet::missingTopic();
23 |         }
24 | 
25 |         if (! is_string($record['topic'])) {
26 |             throw Exception\InvalidRecordInSet::topicIsNotString();
27 |         }
28 | 
29 |         if (trim($record['topic']) === '') {
30 |             throw Exception\InvalidRecordInSet::missingTopic();
31 |         }
32 | 
33 |         if (! isset($topicList[$record['topic']])) {
34 |             throw Exception\InvalidRecordInSet::nonExististingTopic($record['topic']);
35 |         }
36 | 
37 |         if (! isset($record['value'])) {
38 |             throw Exception\InvalidRecordInSet::missingValue();
39 |         }
40 | 
41 |         if (! is_string($record['value'])) {
42 |             throw Exception\InvalidRecordInSet::valueIsNotString();
43 |         }
44 | 
45 |         if (trim($record['value']) === '') {
46 |             throw Exception\InvalidRecordInSet::missingValue();
47 |         }
48 |     }
49 | }
50 | 


--------------------------------------------------------------------------------
/src/Protocol/LeaveGroup.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Protocol;
 5 | 
 6 | use Kafka\Exception\NotSupported;
 7 | use Kafka\Exception\Protocol as ProtocolException;
 8 | use function substr;
 9 | 
10 | class LeaveGroup extends Protocol
11 | {
12 |     /**
13 |      * @param mixed[] $payloads
14 |      *
15 |      * @throws NotSupported
16 |      * @throws ProtocolException
17 |      */
18 |     public function encode(array $payloads = []): string
19 |     {
20 |         if (! isset($payloads['group_id'])) {
21 |             throw new ProtocolException('given leave group data invalid. `group_id` is undefined.');
22 |         }
23 | 
24 |         if (! isset($payloads['member_id'])) {
25 |             throw new ProtocolException('given leave group data invalid. `member_id` is undefined.');
26 |         }
27 | 
28 |         $header = $this->requestHeader('kafka-php', self::LEAVE_GROUP_REQUEST, self::LEAVE_GROUP_REQUEST);
29 |         $data   = self::encodeString($payloads['group_id'], self::PACK_INT16);
30 |         $data  .= self::encodeString($payloads['member_id'], self::PACK_INT16);
31 | 
32 |         return self::encodeString($header . $data, self::PACK_INT32);
33 |     }
34 | 
35 |     /**
36 |      * @return mixed[]
37 |      */
38 |     public function decode(string $data): array
39 |     {
40 |         $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, 0, 2));
41 | 
42 |         return ['errorCode' => $errorCode];
43 |     }
44 | }
45 | 


--------------------------------------------------------------------------------
/src/SocketSync.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka;
 5 | 
 6 | use function fclose;
 7 | use function is_resource;
 8 | use function rewind;
 9 | use function stream_set_blocking;
10 | 
11 | class SocketSync extends CommonSocket
12 | {
13 |     public function connect(): void
14 |     {
15 |         if (is_resource($this->stream)) {
16 |             return;
17 |         }
18 | 
19 |         $this->createStream();
20 | 
21 |         stream_set_blocking($this->stream, false);
22 |     }
23 | 
24 |     public function close(): void
25 |     {
26 |         if (is_resource($this->stream)) {
27 |             fclose($this->stream);
28 |         }
29 |     }
30 | 
31 |     public function isResource(): bool
32 |     {
33 |         return is_resource($this->stream);
34 |     }
35 | 
36 |     /**
37 |      * @param string|int $data
38 |      *
39 |      * @throws \Kafka\Exception
40 |      */
41 |     public function read($data): string
42 |     {
43 |         return $this->readBlocking((int) $data);
44 |     }
45 | 
46 |     /**
47 |      * @throws \Kafka\Exception
48 |      */
49 |     public function write(?string $buffer = null): int
50 |     {
51 |         if ($buffer === null) {
52 |             throw new Exception('You must inform some data to be written');
53 |         }
54 | 
55 |         return $this->writeBlocking($buffer);
56 |     }
57 | 
58 |     public function rewind(): void
59 |     {
60 |         if (is_resource($this->stream)) {
61 |             rewind($this->stream);
62 |         }
63 |     }
64 | }
65 | 


--------------------------------------------------------------------------------
/src/Sasl/Mechanism.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Sasl;
 5 | 
 6 | use Kafka\CommonSocket;
 7 | use Kafka\Exception;
 8 | use Kafka\Protocol;
 9 | use Kafka\Protocol\Protocol as ProtocolTool;
10 | use Kafka\SaslMechanism;
11 | use function substr;
12 | 
13 | abstract class Mechanism implements SaslMechanism
14 | {
15 |     public function authenticate(CommonSocket $socket): void
16 |     {
17 |         $this->handShake($socket, $this->getName());
18 |         $this->performAuthentication($socket);
19 |     }
20 | 
21 |     /**
22 |      *
23 |      * sasl authenticate hand shake
24 |      *
25 |      * @access protected
26 |      */
27 |     protected function handShake(CommonSocket $socket, string $mechanism): void
28 |     {
29 |         $requestData = Protocol::encode(Protocol::SASL_HAND_SHAKE_REQUEST, [$mechanism]);
30 |         $socket->writeBlocking($requestData);
31 |         $dataLen = ProtocolTool::unpack(\Kafka\Protocol\Protocol::BIT_B32, $socket->readBlocking(4));
32 | 
33 |         $data          = $socket->readBlocking($dataLen);
34 |         $correlationId = ProtocolTool::unpack(\Kafka\Protocol\Protocol::BIT_B32, substr($data, 0, 4));
35 |         $result        = Protocol::decode(Protocol::SASL_HAND_SHAKE_REQUEST, substr($data, 4));
36 | 
37 |         if ($result['errorCode'] !== Protocol::NO_ERROR) {
38 |             throw new Exception(Protocol::getError($result['errorCode']));
39 |         }
40 |     }
41 | 
42 |     abstract protected function performAuthentication(CommonSocket $socket): void;
43 |     abstract public function getName(): string;
44 | }
45 | 


--------------------------------------------------------------------------------
/src/Producer.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka;
 5 | 
 6 | use Amp\Loop;
 7 | use Kafka\Producer\CoroutineProcess;
 8 | use Kafka\Producer\Process;
 9 | use Kafka\Producer\SyncProcess;
10 | use Psr\Log\LoggerAwareTrait;
11 | use function is_array;
12 | 
13 | class Producer
14 | {
15 |     use LoggerAwareTrait;
16 |     use LoggerTrait;
17 | 
18 |     /**
19 |      * @var Process|SyncProcess
20 |      */
21 |     private $process;
22 | 
23 |     public function __construct(?callable $producer = null)
24 |     {
25 |         $this->process = new CoroutineProcess();
26 |     }
27 | 
28 |     /**
29 |      * @param mixed[]|bool $data
30 |      *
31 |      * @return mixed[]|null
32 |      *
33 |      * @throws \Kafka\Exception
34 |      */
35 |     public function send($data = true): ?array
36 |     {
37 |         $ret = null;
38 |         if (is_array($data)) {
39 |             $ret = $this->process->send($data);
40 |         }
41 | 
42 |         return $ret;
43 |     }
44 | 
45 |     public function syncMeta(): void
46 |     {
47 |         $this->process->syncMeta();
48 |     }
49 | 
50 |     public function success(callable $success): void
51 |     {
52 |         if ($this->process instanceof SyncProcess) {
53 |             throw new Exception('Success callback can only be configured for asynchronous process');
54 |         }
55 | 
56 |         $this->process->setSuccess($success);
57 |     }
58 | 
59 |     public function error(callable $error): void
60 |     {
61 |         if ($this->process instanceof SyncProcess) {
62 |             throw new Exception('Error callback can only be configured for asynchronous process');
63 |         }
64 | 
65 |         $this->process->setError($error);
66 |     }
67 | }
68 | 


--------------------------------------------------------------------------------
/src/Protocol/ApiVersions.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Protocol;
 5 | 
 6 | use function substr;
 7 | 
 8 | class ApiVersions extends Protocol
 9 | {
10 |     /**
11 |      * @param mixed[] $payloads
12 |      */
13 |     public function encode(array $payloads = []): string
14 |     {
15 |         $header = $this->requestHeader('kafka-php', self::API_VERSIONS_REQUEST, self::API_VERSIONS_REQUEST);
16 | 
17 |         return self::encodeString($header, self::PACK_INT32);
18 |     }
19 | 
20 |     /**
21 |      * @return mixed[]
22 |      */
23 |     public function decode(string $data): array
24 |     {
25 |         $offset      = 0;
26 |         $errcode     = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
27 |         $offset     += 2;
28 |         $apiVersions = $this->decodeArray(substr($data, $offset), [$this, 'apiVersion']);
29 |         $offset     += $apiVersions['length'];
30 | 
31 |         return [
32 |             'apiVersions' => $apiVersions['data'],
33 |             'errorCode'   => $errcode,
34 |         ];
35 |     }
36 | 
37 |     /**
38 |      * @return mixed[]
39 |      */
40 |     protected function apiVersion(string $data): array
41 |     {
42 |         $offset     = 0;
43 |         $apiKey     = self::unpack(self::BIT_B16, substr($data, $offset, 2));
44 |         $offset    += 2;
45 |         $minVersion = self::unpack(self::BIT_B16, substr($data, $offset, 2));
46 |         $offset    += 2;
47 |         $maxVersion = self::unpack(self::BIT_B16, substr($data, $offset, 2));
48 |         $offset    += 2;
49 | 
50 |         return [
51 |             'length' => $offset,
52 |             'data'   => [$apiKey, $minVersion, $maxVersion],
53 |         ];
54 |     }
55 | }
56 | 


--------------------------------------------------------------------------------
/src/Protocol/ListGroup.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Protocol;
 5 | 
 6 | use Kafka\Exception\NotSupported;
 7 | use function substr;
 8 | 
 9 | class ListGroup extends Protocol
10 | {
11 |     /**
12 |      * @param mixed[] $payloads
13 |      *
14 |      * @throws NotSupported
15 |      */
16 |     public function encode(array $payloads = []): string
17 |     {
18 |         $header = $this->requestHeader('kafka-php', self::LIST_GROUPS_REQUEST, self::LIST_GROUPS_REQUEST);
19 | 
20 |         return self::encodeString($header, self::PACK_INT32);
21 |     }
22 | 
23 |     /**
24 |      * @return mixed[]
25 |      */
26 |     public function decode(string $data): array
27 |     {
28 |         $offset    = 0;
29 |         $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
30 |         $offset   += 2;
31 |         $groups    = $this->decodeArray(substr($data, $offset), [$this, 'decodeGroup']);
32 | 
33 |         return [
34 |             'errorCode' => $errorCode,
35 |             'groups'    => $groups['data'],
36 |         ];
37 |     }
38 | 
39 |     /**
40 |      * @return mixed[]
41 |      */
42 |     protected function decodeGroup(string $data): array
43 |     {
44 |         $offset       = 0;
45 |         $groupId      = $this->decodeString(substr($data, $offset), self::BIT_B16);
46 |         $offset      += $groupId['length'];
47 |         $protocolType = $this->decodeString(substr($data, $offset), self::BIT_B16);
48 |         $offset      += $protocolType['length'];
49 | 
50 |         return [
51 |             'length' => $offset,
52 |             'data'   => [
53 |                 'groupId'      => $groupId['data'],
54 |                 'protocolType' => $protocolType['data'],
55 |             ],
56 |         ];
57 |     }
58 | }
59 | 


--------------------------------------------------------------------------------
/src/Protocol/Heartbeat.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Protocol;
 5 | 
 6 | use Kafka\Exception\NotSupported;
 7 | use Kafka\Exception\Protocol as ProtocolException;
 8 | use function substr;
 9 | 
10 | class Heartbeat extends Protocol
11 | {
12 |     /**
13 |      * @param mixed[] $payloads
14 |      *
15 |      * @throws NotSupported
16 |      * @throws ProtocolException
17 |      */
18 |     public function encode(array $payloads = []): string
19 |     {
20 |         if (! isset($payloads['group_id'])) {
21 |             throw new ProtocolException('given heartbeat data invalid. `group_id` is undefined.');
22 |         }
23 | 
24 |         if (! isset($payloads['generation_id'])) {
25 |             throw new ProtocolException('given heartbeat data invalid. `generation_id` is undefined.');
26 |         }
27 | 
28 |         if (! isset($payloads['member_id'])) {
29 |             throw new ProtocolException('given heartbeat data invalid. `member_id` is undefined.');
30 |         }
31 | 
32 |         $header = $this->requestHeader('kafka-php', self::HEART_BEAT_REQUEST, self::HEART_BEAT_REQUEST);
33 |         $data   = self::encodeString($payloads['group_id'], self::PACK_INT16);
34 |         $data  .= self::pack(self::BIT_B32, (string) $payloads['generation_id']);
35 |         $data  .= self::encodeString($payloads['member_id'], self::PACK_INT16);
36 | 
37 |         $data = self::encodeString($header . $data, self::PACK_INT32);
38 | 
39 |         return $data;
40 |     }
41 | 
42 |     /**
43 |      * @return mixed[]
44 |      */
45 |     public function decode(string $data): array
46 |     {
47 |         $offset    = 0;
48 |         $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
49 |         $offset   += 2;
50 | 
51 |         return ['errorCode' => $errorCode];
52 |     }
53 | }
54 | 


--------------------------------------------------------------------------------
/src/Protocol/GroupCoordinator.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Protocol;
 5 | 
 6 | use Kafka\Exception\NotSupported;
 7 | use Kafka\Exception\Protocol as ProtocolException;
 8 | use function substr;
 9 | 
10 | class GroupCoordinator extends Protocol
11 | {
12 |     /**
13 |      * @param mixed[] $payloads
14 |      *
15 |      * @throws ProtocolException
16 |      * @throws NotSupported
17 |      */
18 |     public function encode(array $payloads = []): string
19 |     {
20 |         if (! isset($payloads['group_id'])) {
21 |             throw new ProtocolException('given group coordinator invalid. `group_id` is undefined.');
22 |         }
23 | 
24 |         $header = $this->requestHeader('kafka-php', self::GROUP_COORDINATOR_REQUEST, self::GROUP_COORDINATOR_REQUEST);
25 |         $data   = self::encodeString($payloads['group_id'], self::PACK_INT16);
26 |         $data   = self::encodeString($header . $data, self::PACK_INT32);
27 | 
28 |         return $data;
29 |     }
30 | 
31 |     /**
32 |      * @return mixed[]
33 |      */
34 |     public function decode(string $data): array
35 |     {
36 |         $offset          = 0;
37 |         $errorCode       = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
38 |         $offset         += 2;
39 |         $coordinatorId   = self::unpack(self::BIT_B32, substr($data, $offset, 4));
40 |         $offset         += 4;
41 |         $hosts           = $this->decodeString(substr($data, $offset), self::BIT_B16);
42 |         $offset         += $hosts['length'];
43 |         $coordinatorPort = self::unpack(self::BIT_B32, substr($data, $offset, 4));
44 |         $offset         += 4;
45 | 
46 |         return [
47 |             'errorCode'       => $errorCode,
48 |             'coordinatorId'   => $coordinatorId,
49 |             'coordinatorHost' => $hosts['data'],
50 |             'coordinatorPort' => $coordinatorPort,
51 |         ];
52 |     }
53 | }
54 | 


--------------------------------------------------------------------------------
/docs/ch/Configure.md:
--------------------------------------------------------------------------------
 1 | Kafka-php 配置参数
 2 | ==================
 3 | 
 4 | | Property	| C/P	| Range	| Default | Desc |
 5 | | ----  | ---- | ---- | ---- | ---- | 
 6 | | clientId | C/P | | kafka-php | 客户端标识 | 
 7 | | brokerVersion | C/P | 大于 0.8.0 | 0.10.1.0 | 为了计算 Kafka 请求的协议版本 |
 8 | | metadataBrokerList | C/P | | | 指定 Kafka Broker 列表，多个用逗号分隔 |
 9 | | messageMaxBytes | C/P | 1000 .. 1000000000 | 1000000 | 消息最大长度 |
10 | | metadataRequestTimeoutMs | C/P | 10 .. 900000 | 60000 | 获取 meta 信息超时时间 |
11 | | metadataRefreshIntervalMs | C/P | 10 .. 3600000  | 300000 | 获取同步 meta 信息的时间间隔 |
12 | | metadataMaxAgeMs | C/P | 1 .. 86400000 | -1 | meta 信息有效期
13 | | sslEnable | C/P | true/false | false | 是否开启 Ssl 连接 |
14 | | sslLocalCert | C/P | File path |  | 本地证书路径 |
15 | | sslLocalPk | C/P | File path |  | 如果使用独立的文件来存储证书（local_cert）和私钥， 那么使用此选项来指明私钥文件的路径。|
16 | | sslVerifyPeer | C/P | true/false | false | 是否需要验证 SSL 证书。|
17 | | sslPassphrase | C/P |  |  | local_cert 文件的密码。|
18 | | sslCafile | C/P |  |  | 当设置 sslVerifyPeer 为 true 时， 用来验证远端证书所用到的 CA 证书。 本选项值为 CA 证书在本地文件系统的全路径及文件名。|
19 | | sslPeerName | C/P |  |  | 要连接的服务器名称。如果未设置，那么服务器名称将根据打开 SSL 流的主机名称猜测得出。|
20 | | groupId | C |  | |  消费模块的分组 ID |
21 | | sessionTimeout | C | 1 .. 3600000 | 30000 | 分组中消费者的有效时间 |
22 | | rebalanceTimeout | C | 1 .. 3600000 | 30000 | 分组 rebalance 等待 join 时间 |
23 | | topics | C | | |  将要消费的 kafka topic 名称 | 
24 | | offsetReset | C | latest,earliest | latest | 如果消费 offset 失效的时候重置 offset 的策略 |
25 | | maxBytes | C |  | 65536 | 单次 FETCH 请求对于单个分区请求的最大字节数 |
26 | | maxWaitTime | C |  | 100 | 等待服务端响应 FETCH 请求的最大时间 |
27 | | requiredAck | P | -1 .. 1000 | 1 | 生产消息确认策略 |
28 | | timeout | P | 1 .. 900000 | 5000 | 生产消息请求超时时间 |
29 | | isAsyn | P | true, false | false | 是否采用异步生产消息 |
30 | | requestTimeout | P | 1 .. 900000 | 6000 |  消费消息整体超时时间, 该值一定要大于 timeout 参数 |
31 | | produceInterval | P | 1 .. 900000 | 100 | 在异步生产消息时执行生产消息的请求的时间间隔 |
32 | 
33 | #### 注意
34 | 
35 | 如上所有的参数均通过 setXxx 方式设置，例如要设置 `clientId` 参数
36 | 
37 | ```php
38 | <?php
39 | 
40 | $config = \Kafka\ConsumerConfig::getInstance();
41 | $config->setClientId('test');
42 | ```
43 | 
44 | 无论是消费模块还是生产模块，如果参数设置不符合规则时都会抛 `\Kafka\Exception\Config` 异常
45 | 
46 | 


--------------------------------------------------------------------------------
/src/Consumer.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka;
 5 | 
 6 | use Amp\Loop;
 7 | use Kafka\Consumer\Process;
 8 | use Kafka\Consumer\StopStrategy;
 9 | use Psr\Log\LoggerAwareTrait;
10 | 
11 | class Consumer
12 | {
13 |     use LoggerAwareTrait;
14 |     use LoggerTrait;
15 | 
16 |     /**
17 |      * @var StopStrategy|null
18 |      */
19 |     private $stopStrategy;
20 | 
21 |     /**
22 |      * @var Process|null
23 |      */
24 |     private $process;
25 | 
26 |     public function __construct(?StopStrategy $stopStrategy = null)
27 |     {
28 |         $this->stopStrategy = $stopStrategy;
29 |     }
30 | 
31 |     /**
32 |      * start consumer
33 |      *
34 |      * @access public
35 |      *
36 |      *
37 |      */
38 |     public function start(?callable $consumer = null): void
39 |     {
40 |         if ($this->process !== null) {
41 |             $this->error('Consumer is already being executed');
42 |             return;
43 |         }
44 | 
45 |         $this->setupStopStrategy();
46 | 
47 |         $this->process = $this->createProcess($consumer);
48 |         $this->process->start();
49 | 
50 |         Loop::run();
51 |     }
52 | 
53 |     /**
54 |      * FIXME: remove it when we implement dependency injection
55 |      *
56 |      * This is a very bad practice, but if we don't create this method
57 |      * this class will never be testable...
58 |      *
59 |      * @codeCoverageIgnore
60 |      */
61 |     protected function createProcess(?callable $consumer): Process
62 |     {
63 |         $process = new Process($consumer);
64 | 
65 |         if ($this->logger) {
66 |             $process->setLogger($this->logger);
67 |         }
68 | 
69 |         return $process;
70 |     }
71 | 
72 |     private function setupStopStrategy(): void
73 |     {
74 |         if ($this->stopStrategy === null) {
75 |             return;
76 |         }
77 | 
78 |         $this->stopStrategy->setup($this);
79 |     }
80 | 
81 |     public function stop(): void
82 |     {
83 |         if ($this->process === null) {
84 |             $this->error('Consumer is not running');
85 |             return;
86 |         }
87 | 
88 |         $this->process->stop();
89 |         $this->process = null;
90 | 
91 |         Loop::stop();
92 |     }
93 | }
94 | 


--------------------------------------------------------------------------------
/docs/error.md:
--------------------------------------------------------------------------------
  1 | 1. OFFSET_OUT_OF_RANGE 
  2 | // 将 offset 设置为当前分区的最开始或者最末端
  3 | // C
  4 | 
  5 | 2. INVALID_MESSAGE
  6 | // retry
  7 | // C/P
  8 | 
  9 | 3. UNKNOWN_TOPIC_OR_PARTITION
 10 | // retry
 11 | // 全部暂停重新开始
 12 | // C/P
 13 | 
 14 | 4. INVALID_MESSAGE_SIZE
 15 | // retry
 16 | // C/P
 17 | 
 18 | 5. LEADER_NOT_AVAILABLE
 19 | // retry
 20 | // C/P
 21 | 
 22 | 6. NOT_LEADER_FOR_PARTITION
 23 | // retry
 24 | // 全部暂停重新开始
 25 | // C/P
 26 | 
 27 | 7. REQUEST_TIMED_OUT
 28 | // retry
 29 | // C/P
 30 | 
 31 | 8. BROKER_NOT_AVAILABLE
 32 | // retry
 33 | // 全部暂停重新开始
 34 | // C/P
 35 | 
 36 | 9. REPLICA_NOT_AVAILABLE
 37 | // retry
 38 | // C/P
 39 | 
 40 | 10. MESSAGE_SIZE_TOO_LARGE
 41 | // retry
 42 | // C/P
 43 | 
 44 | 11. STALE_CONTROLLER_EPOCH
 45 | // retry
 46 | // C/P
 47 | 
 48 | 12. OFFSET_METADATA_TOO_LARGE
 49 | // retry
 50 | // C
 51 | 
 52 | 14. GROUP_LOAD_IN_PROGRESS
 53 | // retry
 54 | // 全部暂停重新开始
 55 | // C
 56 | 
 57 | 15. GROUP_COORDINATOR_NOT_AVAILABLE
 58 | // retry
 59 | // 全部暂停重新开始
 60 | // C
 61 | 
 62 | 16. NOT_COORDINATOR_FOR_GROUP
 63 | // retry
 64 | // 全部暂停重新开始
 65 | // C
 66 | 
 67 | 17. INVALID_TOPIC
 68 | // retry
 69 | // 全部暂停重新开始
 70 | // C/P
 71 | 
 72 | 18. RECORD_LIST_TOO_LARGE
 73 | // retry
 74 | // P
 75 | 
 76 | 19. NOT_ENOUGH_REPLICAS
 77 | // error
 78 | // retry
 79 | // P
 80 | 
 81 | 20. NOT_ENOUGH_REPLICAS_AFTER_APPEND
 82 | // error
 83 | // retry
 84 | // P
 85 | 
 86 | 21 INVALID_REQUIRED_ACKS
 87 | // error
 88 | // retry
 89 | // P
 90 | 
 91 | 22. ILLEGAL_GENERATION
 92 | // retry
 93 | // 全部暂停重新开始
 94 | // C
 95 | 
 96 | 23. INCONSISTENT_GROUP_PROTOCOL
 97 | // error exit
 98 | // 全部暂停重新开始
 99 | // C
100 | 
101 | 24. INVALID_GROUP_ID
102 | // exit
103 | // 全部暂停重新开始
104 | // C
105 | 
106 | 25. UNKNOWN_MEMBER_ID
107 | // retry
108 | // 全部暂停重新开始
109 | // C
110 | 
111 | 26. INVALID_SESSION_TIMEOUT
112 | // retry
113 | // 全部暂停重新开始
114 | // C
115 | 
116 | 27. REBALANCE_IN_PROGRESS
117 | // retry 仅仅可以运行 heartbeat
118 | // todo
119 | // C
120 | 
121 | 28. INVALID_COMMIT_OFFSET_SIZE
122 | // retry  error
123 | // C
124 | 
125 | 29. TOPIC_AUTHORIZATION_FAILED
126 | // error
127 | 
128 | 30. GROUP_AUTHORIZATION_FAILED
129 | // error
130 | 
131 | 31. CLUSTER_AUTHORIZATION_FAILED
132 | // error
133 | 
134 | 43. UNSUPPORTED_FOR_MESSAGE_FORMAT
135 | // error
136 | 


--------------------------------------------------------------------------------
/docs/state.md:
--------------------------------------------------------------------------------
 1 | <?php
 2 | 
 3 |     init
 4 |     REQUEST_METADATA =>  STATUS_LOOP 
 5 |     REQUEST_GETGROUP =>  STATUS_START
 6 |     REQUEST_JOINGROUP =>  STATUS_START
 7 |     REQUEST_SYNCGROUP =>  STATUS_START
 8 |     REQUEST_HEARTGROUP =>  STATUS_LOOP
 9 |     REQUEST_OFFSET =>  STATUS_LOOP
10 |     REQUEST_FETCH =>  STATUS_LOOP
11 |     REQUEST_FETCH_OFFSET => STATUS_LOOP
12 |     REQUEST_COMMIT_OFFSET =>  STATUS_LOOP
13 |     instance => empty
14 | 
15 |     run condition
16 |     REQUEST_METADATA => (STATUS_LOOP) && !STATUS_PROCESS
17 |         (run)  => status->STATUS_LOOP|STATUS_PROCESS
18 |         (succ) => status->STATUS_LOOP|STATUS_FINISH
19 |             (change) recover
20 |         (fail) => retry
21 |     REQUEST_GETGROUP => (STATUS_START) && (REQUEST_METADATA^STATUS_FINISH)
22 |         (run)  => status->STATUS_PROCESS
23 |         (succ) => status->STATUS_STOP|STATUS_FINISH
24 |         (fail) => recover
25 |     REQUEST_JOINGROUP => (STATUS_START) && (REQUEST_GETGROUP^STATUS_FINISH)
26 |         (run)  => status->STATUS_PROCESS
27 |         (succ) => status->STATUS_STOP|STATUS_FINISH
28 |         (fail) => recover
29 |     REQUEST_SYNCGROUP => (STATUS_START) && (REQUEST_JOINGROUP^STATUS_FINISH)
30 |         (run)  => status->STATUS_PROCESS
31 |         (succ) => status->STATUS_STOP|STATUS_FINISH
32 |         (fail) => recover
33 |     REQUEST_HEARTGROUP => (STATUS_LOOP) && (REQUEST_SYNCGROUP^STATUS_FINISH)
34 |         (run)  => status->STATUS_PROCESS
35 |         (succ) => status->STATUS_LOOP|STATUS_FINISH
36 |         (fail) => 
37 |                 27 => if !REQUEST_JOINGROUP^STATUS_PROCESS -> modify(REQUEST_JOINGROUP init)
38 |                 25 => if !REQUEST_JOINGROUP^STATUS_PROCESS -> modify(REQUEST_JOINGROUP init) empty member_id
39 |     
40 |     REQUEST_OFFSET => (STATUS_LOOP) && (REQUEST_METADATA^STATUS_FINISH)
41 |         (run)  => status->STATUS_PROCESS
42 |         (succ) => status->STATUS_LOOP|STATUS_FINISH 
43 |         (fail) => recover
44 | 
45 |     REQUEST_FETCH_OFFSET => (STATUS_LOOP) && (REQUEST_SYNCGROUP^STATUS_FINISH)
46 |         (run)  => status->STATUS_PROCESS
47 |         (succ) => status->STATUS_LOOP|STATUS_FINISH 
48 |         (fail) => recover
49 | 
50 |     REQUEST_FETCH => (STATUS_LOOP) && (REQUEST_FETCH_OFFSET^STATUS_FINISH)
51 |         (run)  => status->STATUS_PROCESS
52 |         (succ) => status->STATUS_LOOP|STATUS_FINISH 
53 |         (fail) => recover
54 | 
55 |     REQUEST_COMMIT_OFFSET => (STATUS_LOOP) && (REQUEST_FETCH^STATUS_FINISH)
56 |         (run)  => status->STATUS_PROCESS
57 |         (succ) => status->STATUS_LOOP|STATUS_FINISH 
58 |         (fail) => recover
59 | 
60 | 


--------------------------------------------------------------------------------
/src/Protocol/SaslHandShake.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | declare(strict_types=1);
 3 | 
 4 | namespace Kafka\Protocol;
 5 | 
 6 | use Kafka\Exception\NotSupported;
 7 | use Kafka\Exception\Protocol as ProtocolException;
 8 | use function array_shift;
 9 | use function implode;
10 | use function in_array;
11 | use function is_string;
12 | use function substr;
13 | 
14 | class SaslHandShake extends Protocol
15 | {
16 |     private const ALLOW_SASL_MECHANISMS = [
17 |         'GSSAPI',
18 |         'PLAIN',
19 |         'SCRAM-SHA-256',
20 |         'SCRAM-SHA-512',
21 |     ];
22 | 
23 |     /**
24 |      * @param mixed[] $payloads
25 |      *
26 |      * @throws NotSupported
27 |      * @throws ProtocolException
28 |      */
29 |     public function encode(array $payloads = []): string
30 |     {
31 |         $mechanism = array_shift($payloads);
32 | 
33 |         if (! is_string($mechanism)) {
34 |             throw new ProtocolException('Invalid request SASL hand shake mechanism given. ');
35 |         }
36 | 
37 |         if (! in_array($mechanism, self::ALLOW_SASL_MECHANISMS, true)) {
38 |             throw new ProtocolException(
39 |                 'Invalid request SASL hand shake mechanism given, it must be one of: ' . implode('|', self::ALLOW_SASL_MECHANISMS)
40 |             );
41 |         }
42 | 
43 |         $header = $this->requestHeader('kafka-php', self::SASL_HAND_SHAKE_REQUEST, self::SASL_HAND_SHAKE_REQUEST);
44 |         $data   = self::encodeString($mechanism, self::PACK_INT16);
45 |         $data   = self::encodeString($header . $data, self::PACK_INT32);
46 | 
47 |         return $data;
48 |     }
49 | 
50 |     /**
51 |      * @return mixed[]
52 |      */
53 |     public function decode(string $data): array
54 |     {
55 |         $offset            = 0;
56 |         $errcode           = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
57 |         $offset           += 2;
58 |         $enabledMechanisms = $this->decodeArray(substr($data, $offset), [$this, 'mechanism']);
59 |         $offset           += $enabledMechanisms['length'];
60 | 
61 |         return [
62 |             'mechanisms' => $enabledMechanisms['data'],
63 |             'errorCode'  => $errcode,
64 |         ];
65 |     }
66 | 
67 |     /**
68 |      * @return mixed[]
69 |      */
70 |     protected function mechanism(string $data): array
71 |     {
72 |         $offset        = 0;
73 |         $mechanismInfo = $this->decodeString(substr($data, $offset), self::BIT_B16);
74 |         $offset       += $mechanismInfo['length'];
75 | 
76 |         return [
77 |             'length' => $offset,
78 |             'data'   => $mechanismInfo['data'],
79 |         ];
80 |     }
81 | }
82 | 


--------------------------------------------------------------------------------
/src/ProducerConfig.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use Kafka\Protocol\Produce;
  7 | use function in_array;
  8 | 
  9 | /**
 10 |  * @method int getRequestTimeout()
 11 |  * @method int getProduceInterval()
 12 |  * @method int getTimeout()
 13 |  * @method int getRequiredAck()
 14 |  * @method bool getIsAsyn()
 15 |  * @method int getCompression()
 16 |  */
 17 | class ProducerConfig extends Config
 18 | {
 19 |     use SingletonTrait;
 20 | 
 21 |     private const COMPRESSION_OPTIONS = [
 22 |         Produce::COMPRESSION_NONE,
 23 |         Produce::COMPRESSION_GZIP,
 24 |         Produce::COMPRESSION_SNAPPY,
 25 |     ];
 26 | 
 27 |     /**
 28 |      * @var mixed[]
 29 |      */
 30 |     protected static $defaults = [
 31 |         'requiredAck'     => 1,
 32 |         'timeout'         => 5000,
 33 |         'isAsyn'          => false,
 34 |         'requestTimeout'  => 6000,
 35 |         'produceInterval' => 100,
 36 |         'compression'     => Protocol\Protocol::COMPRESSION_NONE,
 37 |     ];
 38 | 
 39 |     /**
 40 |      * @throws \Kafka\Exception\Config
 41 |      */
 42 |     public function setRequestTimeout(int $requestTimeout): void
 43 |     {
 44 |         if ($requestTimeout < 1 || $requestTimeout > 900000) {
 45 |             throw new Exception\Config('Set request timeout value is invalid, must set it 1 .. 900000');
 46 |         }
 47 | 
 48 |         static::$options['requestTimeout'] = $requestTimeout;
 49 |     }
 50 | 
 51 |     /**
 52 |      * @throws \Kafka\Exception\Config
 53 |      */
 54 |     public function setProduceInterval(int $produceInterval): void
 55 |     {
 56 |         if ($produceInterval < 1 || $produceInterval > 900000) {
 57 |             throw new Exception\Config('Set produce interval timeout value is invalid, must set it 1 .. 900000');
 58 |         }
 59 | 
 60 |         static::$options['produceInterval'] = $produceInterval;
 61 |     }
 62 | 
 63 |     /**
 64 |      * @throws \Kafka\Exception\Config
 65 |      */
 66 |     public function setTimeout(int $timeout): void
 67 |     {
 68 |         if ($timeout < 1 || $timeout > 900000) {
 69 |             throw new Exception\Config('Set timeout value is invalid, must set it 1 .. 900000');
 70 |         }
 71 | 
 72 |         static::$options['timeout'] = $timeout;
 73 |     }
 74 | 
 75 |     /**
 76 |      * @throws \Kafka\Exception\Config
 77 |      */
 78 |     public function setRequiredAck(int $requiredAck): void
 79 |     {
 80 |         if ($requiredAck < -1 || $requiredAck > 1000) {
 81 |             throw new Exception\Config('Set required ack value is invalid, must set it -1 .. 1000');
 82 |         }
 83 | 
 84 |         static::$options['requiredAck'] = $requiredAck;
 85 |     }
 86 | 
 87 |     public function setIsAsyn(bool $asyn): void
 88 |     {
 89 |         static::$options['isAsyn'] = $asyn;
 90 |     }
 91 | 
 92 |     public function setCompression(int $compression): void
 93 |     {
 94 |         if (! in_array($compression, self::COMPRESSION_OPTIONS, true)) {
 95 |             throw new Exception\Config('Compression must be one the Kafka\Protocol\Produce::COMPRESSION_* constants');
 96 |         }
 97 | 
 98 |         static::$options['compression'] = $compression;
 99 |     }
100 | }
101 | 


--------------------------------------------------------------------------------
/src/Sasl/Gssapi.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Sasl;
  5 | 
  6 | use GSSAPIContext;
  7 | use Kafka\CommonSocket;
  8 | use Kafka\Exception;
  9 | use Kafka\Protocol\Protocol as ProtocolTool;
 10 | use KRB5CCache;
 11 | use const GSS_C_INITIATE;
 12 | use function extension_loaded;
 13 | use function file_exists;
 14 | use function is_file;
 15 | use function is_readable;
 16 | 
 17 | class Gssapi extends Mechanism
 18 | {
 19 |     private const MECHANISM_NAME = 'GSSAPI';
 20 | 
 21 |     /**
 22 |      * @var string
 23 |      */
 24 |     private $principal;
 25 | 
 26 |     /**
 27 |      * @var GSSAPIContext
 28 |      */
 29 |     private $gssapi;
 30 | 
 31 |     /**
 32 |      * @var KRB5CCache
 33 |      */
 34 |     private static $ccache;
 35 | 
 36 |     public function __construct(GSSAPIContext $gssapi, string $principal)
 37 |     {
 38 |         $this->gssapi    = $gssapi;
 39 |         $this->principal = $principal;
 40 |     }
 41 | 
 42 |     public static function fromKeytab(string $keytab, string $principal): self
 43 |     {
 44 |         if (! extension_loaded('krb5')) {
 45 |             throw new Exception('Extension "krb5" is required for "GSSAPI" authentication');
 46 |         }
 47 | 
 48 |         if (! file_exists($keytab) || ! is_file($keytab)) {
 49 |             throw new Exception('Invalid keytab, keytab file not exists.');
 50 |         }
 51 | 
 52 |         if (! is_readable($keytab)) {
 53 |             throw new Exception('Invalid keytab, keytab file disable read.');
 54 |         }
 55 | 
 56 |         self::$ccache = new KRB5CCache();
 57 |         self::$ccache->initKeytab($principal, $keytab);
 58 | 
 59 |         $gssapi = new GSSAPIContext();
 60 |         $gssapi->acquireCredentials(self::$ccache, $principal, GSS_C_INITIATE);
 61 |         return new self($gssapi, $principal);
 62 |     }
 63 | 
 64 |     /**
 65 |      * @throws \Kafka\Exception\NotSupported
 66 |      * @throws \Kafka\Exception
 67 |      */
 68 |     protected function performAuthentication(CommonSocket $socket): void
 69 |     {
 70 |         $token = $this->initSecurityContext();
 71 | 
 72 |         // send token to server and get server token
 73 |         $data = ProtocolTool::encodeString($token, ProtocolTool::PACK_INT32);
 74 |         $socket->writeBlocking($data);
 75 | 
 76 |         $dataLen = ProtocolTool::unpack(ProtocolTool::BIT_B32, $socket->readBlocking(4));
 77 | 
 78 |         // wrap message use server token and send to server authenticate
 79 |         $data = ProtocolTool::encodeString(
 80 |             $this->wrapToken($socket->readBlocking($dataLen)),
 81 |             ProtocolTool::PACK_INT32
 82 |         );
 83 | 
 84 |         $socket->writeBlocking($data);
 85 |     }
 86 | 
 87 |     /**
 88 |      *
 89 |      * get sasl authenticate mechanism name
 90 |      *
 91 |      * @access public
 92 |      */
 93 |     public function getName(): string
 94 |     {
 95 |         return self::MECHANISM_NAME;
 96 |     }
 97 | 
 98 |     private function initSecurityContext(): string
 99 |     {
100 |         $token = '';
101 |         $ret   = $this->gssapi->initSecContext($this->principal, null, null, null, $token);
102 |         if (! $ret) {
103 |             throw new Exception('Init security context failure.');
104 |         }
105 |         return $token;
106 |     }
107 | 
108 |     private function wrapToken(string $token): string
109 |     {
110 |         $message = '';
111 |         $this->gssapi->wrap($token, $message);
112 |         return $message;
113 |     }
114 | }
115 | 


--------------------------------------------------------------------------------
/src/LoggerTrait.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use Psr\Log\LogLevel;
  7 | use Psr\Log\NullLogger;
  8 | 
  9 | trait LoggerTrait
 10 | {
 11 |     /**
 12 |      * System is unusable.
 13 |      *
 14 |      * @param mixed[] $context
 15 |      *
 16 |      */
 17 |     public function emergency(string $message, array $context = []): void
 18 |     {
 19 |         $this->log(LogLevel::EMERGENCY, $message, $context);
 20 |     }
 21 | 
 22 |     /**
 23 |      * Action must be taken immediately.
 24 |      *
 25 |      * Example: Entire website down, database unavailable, etc. This should
 26 |      * trigger the SMS alerts and wake you up.
 27 |      *
 28 |      * @param mixed[] $context
 29 |      *
 30 |      */
 31 |     public function alert(string $message, array $context = []): void
 32 |     {
 33 |         $this->log(LogLevel::ALERT, $message, $context);
 34 |     }
 35 | 
 36 |     /**
 37 |      * Critical conditions.
 38 |      *
 39 |      * Example: Application component unavailable, unexpected exception.
 40 |      *
 41 |      * @param mixed[] $context
 42 |      *
 43 |      */
 44 |     public function critical(string $message, array $context = []): void
 45 |     {
 46 |         $this->log(LogLevel::CRITICAL, $message, $context);
 47 |     }
 48 | 
 49 |     /**
 50 |      * Runtime errors that do not require immediate action but should typically
 51 |      * be logged and monitored.
 52 |      *
 53 |      * @param mixed[] $context
 54 |      *
 55 |      */
 56 |     public function error(string $message, array $context = []): void
 57 |     {
 58 |         $this->log(LogLevel::ERROR, $message, $context);
 59 |     }
 60 | 
 61 |     /**
 62 |      * Exceptional occurrences that are not errors.
 63 |      *
 64 |      * Example: Use of deprecated APIs, poor use of an API, undesirable things
 65 |      * that are not necessarily wrong.
 66 |      *
 67 |      * @param mixed[] $context
 68 |      *
 69 |      */
 70 |     public function warning(string $message, array $context = []): void
 71 |     {
 72 |         $this->log(LogLevel::WARNING, $message, $context);
 73 |     }
 74 | 
 75 |     /**
 76 |      * Normal but significant events.
 77 |      *
 78 |      * @param mixed[] $context
 79 |      *
 80 |      */
 81 |     public function notice(string $message, array $context = []): void
 82 |     {
 83 |         $this->log(LogLevel::NOTICE, $message, $context);
 84 |     }
 85 | 
 86 |     /**
 87 |      * Interesting events.
 88 |      *
 89 |      * Example: User logs in, SQL logs.
 90 |      *
 91 |      * @param mixed[] $context
 92 |      *
 93 |      */
 94 |     public function info(string $message, array $context = []): void
 95 |     {
 96 |         $this->log(LogLevel::INFO, $message, $context);
 97 |     }
 98 | 
 99 |     /**
100 |      * Detailed debug information.
101 |      *
102 |      * @param mixed[] $context
103 |      *
104 |      */
105 |     public function debug(string $message, array $context = []): void
106 |     {
107 |         $this->log(LogLevel::DEBUG, $message, $context);
108 |     }
109 | 
110 |     /**
111 |      * Logs with an arbitrary level.
112 |      *
113 |      * @param mixed   $level
114 |      * @param string  $message
115 |      * @param mixed[] $context
116 |      *
117 |      */
118 |     public function log($level, $message, array $context = []): void
119 |     {
120 |         if ($this->logger === null) {
121 |             $this->logger = new NullLogger();
122 |         }
123 |         $this->logger->log($level, $message, $context);
124 |     }
125 | }
126 | 


--------------------------------------------------------------------------------
/docs/Configure.md:
--------------------------------------------------------------------------------
 1 | Kafka-php Configuration
 2 | ==================
 3 | 
 4 | | Property	| C/P	| Range	| Default | Desc |
 5 | | --  | -- | -- | -- | -- |
 6 | | brokerVersion | C/P | 0.8.0 | 0.10.1.0 | User supplied broker version |
 7 | | clientId | C/P |  | kafka-php | This is a user supplied identifier for the client application | 
 8 | | messageMaxBytes | C/P | 1000 .. 1000000000 | 1000000 | Maximum transmit message size. |
 9 | | metadataBrokerList | C/P | | | Kafka Broker server list |
10 | | metadataMaxAgeMs | C/P | 1 .. 86400000 | -1 | Metadata cache max age. Defaults to metadata.refresh.interval.ms * 3 |
11 | | metadataRefreshIntervalMs | C/P | 10 .. 3600000  | 300000 | Topic metadata refresh interval in milliseconds. The metadata is automatically refreshed on error and connect. Use -1 to disable the intervalled refresh.  |
12 | | metadataRequestTimeoutMs | C/P | 10 .. 900000 | 60000 | Non-topic request timeout in milliseconds. This is for metadata requests, etc. |
13 | | sslEnable | C/P | true/false | false | Whether enable ssl connect or not |
14 | | sslCafile | C/P |  |  | Location of Certificate Authority file on local filesystem which should be used with the verify_peer context option to authenticate the identity of the remote peer.|
15 | | sslLocalCert | C/P | File path |  | Path to local certificate file on filesystem. |
16 | | sslLocalPk | C/P | File path |  | Path to local private key file on filesystem in case of separate files for certificate (local_cert) and private key. |
17 | | sslPassphrase | C/P |  |  | Passphrase with which your local_cert file was encoded. |
18 | | sslPeerName | C/P |  |  | Peer name to be used. If this value is not set, then the name is guessed based on the hostname used when opening the stream. |
19 | | sslVerifyPeer | C/P | true/false | false | Require verification of SSL certificate used. |
20 | | offsetReset | C | latest,earliest | latest | Action to take when there is no initial offset in offset store or the desired offset is out of range |
21 | | groupId | C |  | |  Client group id string. All clients sharing the same group.id belong to the same group. |
22 | | maxBytes | C |  | 65536 | Maximum bytes to fetch. |
23 | | maxWaitTime | C |  | 100 | Maximum time in ms to wait for the response |
24 | | sessionTimeout | C | 1 .. 3600000 | 30000 | Client group session and failure detection timeout.  |
25 | | rebalanceTimeout | C | 1 .. 3600000 | 30000 | rebalance join wait timeout |
26 | | topics | C | | |  Want consumer topics | 
27 | | isAsyn | P | true, false | false | Whether to use asynchronous production messages |
28 | | produceInterval | P | 1 .. 900000 | 100 | The time interval at which requests for production messages are executed when the message is produced asynchronously |
29 | | requestTimeout | P | 1 .. 900000 | 6000 |  The total timeout of the production message, which must be greater than the timeout config parameter |
30 | | requiredAck | P | -1 .. 1000 | 1 | This field indicates how many acknowledgements the leader broker must receive from ISR brokers before responding to the request: 0=Broker does not send any response/ack to client, 1=Only the leader broker will need to ack the message, -1 or all=broker will block until message is committed by all in sync replicas (ISRs) or broker\'s in.sync.replicas setting before sending response.  |
31 | | timeout | P | 1 .. 900000 | 5000 | Producer request timeout |
32 | 
33 | #### Note
34 | 
35 | All of the above parameters are set by setXxx, for example, to set the `clientId` parameter
36 | 
37 | ```php
38 | <?php
39 | 
40 | $config = \Kafka\ConsumerConfig::getInstance();
41 | $config->setClientId('test');
42 | ```
43 | 
44 | Whether it is a consumer module or a production module, if the parameter settings do not match the rules will throw `\Kafka\Exception\Config` exception
45 | 


--------------------------------------------------------------------------------
/src/Protocol/FetchOffset.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use Kafka\Exception\Protocol as ProtocolException;
  8 | use function substr;
  9 | 
 10 | class FetchOffset extends Protocol
 11 | {
 12 |     /**
 13 |      * @param mixed[] $payloads
 14 |      *
 15 |      * @throws NotSupported
 16 |      * @throws ProtocolException
 17 |      */
 18 |     public function encode(array $payloads = []): string
 19 |     {
 20 |         if (! isset($payloads['data'])) {
 21 |             throw new ProtocolException('given fetch offset data invalid. `data` is undefined.');
 22 |         }
 23 | 
 24 |         if (! isset($payloads['group_id'])) {
 25 |             throw new ProtocolException('given fetch offset data invalid. `group_id` is undefined.');
 26 |         }
 27 | 
 28 |         $header = $this->requestHeader('kafka-php', self::OFFSET_FETCH_REQUEST, self::OFFSET_FETCH_REQUEST);
 29 |         $data   = self::encodeString($payloads['group_id'], self::PACK_INT16);
 30 |         $data  .= self::encodeArray($payloads['data'], [$this, 'encodeOffsetTopic']);
 31 |         $data   = self::encodeString($header . $data, self::PACK_INT32);
 32 | 
 33 |         return $data;
 34 |     }
 35 | 
 36 |     /**
 37 |      * @return mixed[]
 38 |      */
 39 |     public function decode(string $data): array
 40 |     {
 41 |         $offset  = 0;
 42 |         $topics  = $this->decodeArray(substr($data, $offset), [$this, 'offsetTopic']);
 43 |         $offset += $topics['length'];
 44 | 
 45 |         return $topics['data'];
 46 |     }
 47 | 
 48 |     protected function encodeOffsetPartition(int $values): string
 49 |     {
 50 |         return self::pack(self::BIT_B32, (string) $values);
 51 |     }
 52 | 
 53 |     /**
 54 |      * @param mixed[] $values
 55 |      *
 56 |      * @throws NotSupported
 57 |      * @throws ProtocolException
 58 |      */
 59 |     protected function encodeOffsetTopic(array $values): string
 60 |     {
 61 |         if (! isset($values['topic_name'])) {
 62 |             throw new ProtocolException('given fetch offset data invalid. `topic_name` is undefined.');
 63 |         }
 64 | 
 65 |         if (! isset($values['partitions']) || empty($values['partitions'])) {
 66 |             throw new ProtocolException('given fetch offset data invalid. `partitions` is undefined.');
 67 |         }
 68 | 
 69 |         $topic      = self::encodeString($values['topic_name'], self::PACK_INT16);
 70 |         $partitions = self::encodeArray($values['partitions'], [$this, 'encodeOffsetPartition']);
 71 | 
 72 |         return $topic . $partitions;
 73 |     }
 74 | 
 75 |     /**
 76 |      * @return mixed[]
 77 |      * @throws ProtocolException
 78 |      */
 79 |     protected function offsetTopic(string $data): array
 80 |     {
 81 |         $offset    = 0;
 82 |         $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16);
 83 |         $offset   += $topicInfo['length'];
 84 | 
 85 |         $partitions = $this->decodeArray(substr($data, $offset), [$this, 'offsetPartition']);
 86 |         $offset    += $partitions['length'];
 87 | 
 88 |         return [
 89 |             'length' => $offset,
 90 |             'data'   => [
 91 |                 'topicName'  => $topicInfo['data'],
 92 |                 'partitions' => $partitions['data'],
 93 |             ],
 94 |         ];
 95 |     }
 96 | 
 97 |     /**
 98 |      * @return mixed[]
 99 |      * @throws ProtocolException
100 |      */
101 |     protected function offsetPartition(string $data): array
102 |     {
103 |         $offset = 0;
104 | 
105 |         $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4));
106 |         $offset     += 4;
107 | 
108 |         $roffset = self::unpack(self::BIT_B64, substr($data, $offset, 8));
109 |         $offset += 8;
110 | 
111 |         $metadata  = $this->decodeString(substr($data, $offset), self::BIT_B16);
112 |         $offset   += $metadata['length'];
113 |         $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
114 |         $offset   += 2;
115 | 
116 |         return [
117 |             'length' => $offset,
118 |             'data'   => [
119 |                 'partition' => $partitionId,
120 |                 'errorCode' => $errorCode,
121 |                 'metadata'  => $metadata['data'],
122 |                 'offset'    => $roffset,
123 |             ],
124 |         ];
125 |     }
126 | }
127 | 


--------------------------------------------------------------------------------
/src/Protocol/Metadata.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use Kafka\Exception\Protocol as ProtocolException;
  8 | use function is_string;
  9 | use function substr;
 10 | 
 11 | class Metadata extends Protocol
 12 | {
 13 |     /**
 14 |      * @param mixed[] $payloads
 15 |      *
 16 |      * @throws NotSupported
 17 |      * @throws ProtocolException
 18 |      */
 19 |     public function encode(array $payloads = []): string
 20 |     {
 21 |         foreach ($payloads as $topic) {
 22 |             if (! is_string($topic)) {
 23 |                 throw new ProtocolException('request metadata topic array have invalid value. ');
 24 |             }
 25 |         }
 26 | 
 27 |         $header = $this->requestHeader('kafka-php', self::METADATA_REQUEST, self::METADATA_REQUEST);
 28 |         $data   = self::encodeArray($payloads, [$this, 'encodeString'], self::PACK_INT16);
 29 |         $data   = self::encodeString($header . $data, self::PACK_INT32);
 30 | 
 31 |         return $data;
 32 |     }
 33 | 
 34 |     /**
 35 |      * @return mixed[]
 36 |      */
 37 |     public function decode(string $data): array
 38 |     {
 39 |         $offset       = 0;
 40 |         $brokerRet    = $this->decodeArray(substr($data, $offset), [$this, 'metaBroker']);
 41 |         $offset      += $brokerRet['length'];
 42 |         $topicMetaRet = $this->decodeArray(substr($data, $offset), [$this, 'metaTopicMetaData']);
 43 |         $offset      += $topicMetaRet['length'];
 44 | 
 45 |         $result = [
 46 |             'brokers' => $brokerRet['data'],
 47 |             'topics'  => $topicMetaRet['data'],
 48 |         ];
 49 | 
 50 |         return $result;
 51 |     }
 52 | 
 53 |     /**
 54 |      * @return mixed[]
 55 |      */
 56 |     protected function metaBroker(string $data): array
 57 |     {
 58 |         $offset       = 0;
 59 |         $nodeId       = self::unpack(self::BIT_B32, substr($data, $offset, 4));
 60 |         $offset      += 4;
 61 |         $hostNameInfo = $this->decodeString(substr($data, $offset), self::BIT_B16);
 62 |         $offset      += $hostNameInfo['length'];
 63 |         $port         = self::unpack(self::BIT_B32, substr($data, $offset, 4));
 64 |         $offset      += 4;
 65 | 
 66 |         return [
 67 |             'length' => $offset,
 68 |             'data'   => [
 69 |                 'host'   => $hostNameInfo['data'],
 70 |                 'port'   => $port,
 71 |                 'nodeId' => $nodeId,
 72 |             ],
 73 |         ];
 74 |     }
 75 | 
 76 |     /**
 77 |      * @return mixed[]
 78 |      */
 79 |     protected function metaTopicMetaData(string $data): array
 80 |     {
 81 |         $offset         = 0;
 82 |         $topicErrCode   = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
 83 |         $offset        += 2;
 84 |         $topicInfo      = $this->decodeString(substr($data, $offset), self::BIT_B16);
 85 |         $offset        += $topicInfo['length'];
 86 |         $partitionsMeta = $this->decodeArray(substr($data, $offset), [$this, 'metaPartitionMetaData']);
 87 |         $offset        += $partitionsMeta['length'];
 88 | 
 89 |         return [
 90 |             'length' => $offset,
 91 |             'data'   => [
 92 |                 'topicName'  => $topicInfo['data'],
 93 |                 'errorCode'  => $topicErrCode,
 94 |                 'partitions' => $partitionsMeta['data'],
 95 |             ],
 96 |         ];
 97 |     }
 98 | 
 99 |     /**
100 |      * @return mixed[]
101 |      */
102 |     protected function metaPartitionMetaData(string $data): array
103 |     {
104 |         $offset   = 0;
105 |         $errcode  = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
106 |         $offset  += 2;
107 |         $partId   = self::unpack(self::BIT_B32, substr($data, $offset, 4));
108 |         $offset  += 4;
109 |         $leader   = self::unpack(self::BIT_B32, substr($data, $offset, 4));
110 |         $offset  += 4;
111 |         $replicas = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32);
112 |         $offset  += $replicas['length'];
113 |         $isr      = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32);
114 |         $offset  += $isr['length'];
115 | 
116 |         return [
117 |             'length' => $offset,
118 |             'data'   => [
119 |                 'partitionId' => $partId,
120 |                 'errorCode'   => $errcode,
121 |                 'replicas'    => $replicas['data'],
122 |                 'leader'      => $leader,
123 |                 'isr'         => $isr['data'],
124 |             ],
125 |         ];
126 |     }
127 | }
128 | 


--------------------------------------------------------------------------------
/src/ConsumerConfig.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use function in_array;
  7 | use function trim;
  8 | 
  9 | /**
 10 |  * @method string|false ietGroupId()
 11 |  * @method array|false ietTopics()
 12 |  * @method int getSessionTimeout()
 13 |  * @method int getRebalanceTimeout()
 14 |  * @method string getOffsetReset()
 15 |  * @method int getMaxBytes()
 16 |  * @method int getMaxWaitTime()
 17 |  */
 18 | class ConsumerConfig extends Config
 19 | {
 20 |     use SingletonTrait;
 21 | 
 22 |     public const CONSUME_AFTER_COMMIT_OFFSET  = 1;
 23 |     public const CONSUME_BEFORE_COMMIT_OFFSET = 2;
 24 | 
 25 |     /**
 26 |      * @var mixed[]
 27 |      */
 28 |     protected $runtimeOptions = [
 29 |         'consume_mode' => self::CONSUME_AFTER_COMMIT_OFFSET,
 30 |     ];
 31 | 
 32 |     /**
 33 |      * @var mixed[]
 34 |      */
 35 |     protected static $defaults = [
 36 |         'groupId'          => '',
 37 |         'sessionTimeout'   => 30000,
 38 |         'rebalanceTimeout' => 30000,
 39 |         'topics'           => [],
 40 |         'offsetReset'      => 'latest', // earliest
 41 |         'maxBytes'         => 65536, // 64kb
 42 |         'maxWaitTime'      => 100,
 43 |     ];
 44 | 
 45 |     /**
 46 |      * @throws \Kafka\Exception\Config
 47 |      */
 48 |     public function getGroupId(): string
 49 |     {
 50 |         $groupId = trim($this->ietGroupId());
 51 | 
 52 |         if ($groupId === false || $groupId === '') {
 53 |             throw new Exception\Config('Get group id value is invalid, must set it not empty string');
 54 |         }
 55 | 
 56 |         return $groupId;
 57 |     }
 58 | 
 59 |     /**
 60 |      * @throws \Kafka\Exception\Config
 61 |      */
 62 |     public function setGroupId(string $groupId): void
 63 |     {
 64 |         $groupId = trim($groupId);
 65 | 
 66 |         if ($groupId === false || $groupId === '') {
 67 |             throw new Exception\Config('Set group id value is invalid, must set it not empty string');
 68 |         }
 69 | 
 70 |         static::$options['groupId'] = $groupId;
 71 |     }
 72 | 
 73 |     /**
 74 |      * @throws \Kafka\Exception\Config
 75 |      */
 76 |     public function setSessionTimeout(int $sessionTimeout): void
 77 |     {
 78 |         if ($sessionTimeout < 1 || $sessionTimeout > 3600000) {
 79 |             throw new Exception\Config('Set session timeout value is invalid, must set it 1 .. 3600000');
 80 |         }
 81 | 
 82 |         static::$options['sessionTimeout'] = $sessionTimeout;
 83 |     }
 84 | 
 85 |     /**
 86 |      * @throws \Kafka\Exception\Config
 87 |      */
 88 |     public function setRebalanceTimeout(int $rebalanceTimeout): void
 89 |     {
 90 |         if ($rebalanceTimeout < 1 || $rebalanceTimeout > 3600000) {
 91 |             throw new Exception\Config('Set rebalance timeout value is invalid, must set it 1 .. 3600000');
 92 |         }
 93 | 
 94 |         static::$options['rebalanceTimeout'] = $rebalanceTimeout;
 95 |     }
 96 | 
 97 |     /**
 98 |      * @throws \Kafka\Exception\Config
 99 |      */
100 |     public function setOffsetReset(string $offsetReset): void
101 |     {
102 |         if (! in_array($offsetReset, ['latest', 'earliest'], true)) {
103 |             throw new Exception\Config('Set offset reset value is invalid, must set it `latest` or `earliest`');
104 |         }
105 | 
106 |         static::$options['offsetReset'] = $offsetReset;
107 |     }
108 | 
109 |     /**
110 |      * @return string[]
111 |      *
112 |      * @throws \Kafka\Exception\Config
113 |      */
114 |     public function getTopics(): array
115 |     {
116 |         $topics = $this->ietTopics();
117 | 
118 |         if (empty($topics)) {
119 |             throw new Exception\Config('Get consumer topics value is invalid, must set it not empty');
120 |         }
121 | 
122 |         return $topics;
123 |     }
124 | 
125 |     /**
126 |      * @param string[] $topics
127 |      *
128 |      * @throws \Kafka\Exception\Config
129 |      */
130 |     public function setTopics(array $topics): void
131 |     {
132 |         if (empty($topics)) {
133 |             throw new Exception\Config('Set consumer topics value is invalid, must set it not empty array');
134 |         }
135 | 
136 |         static::$options['topics'] = $topics;
137 |     }
138 | 
139 |     public function setConsumeMode(int $mode): void
140 |     {
141 |         if (! in_array($mode, [self::CONSUME_AFTER_COMMIT_OFFSET, self::CONSUME_BEFORE_COMMIT_OFFSET], true)) {
142 |             throw new Exception\Config(
143 |                 'Invalid consume mode given, it must be either "ConsumerConfig::CONSUME_AFTER_COMMIT_OFFSET" or '
144 |                 . '"ConsumerConfig::CONSUME_BEFORE_COMMIT_OFFSET"'
145 |             );
146 |         }
147 | 
148 |         $this->runtimeOptions['consume_mode'] = $mode;
149 |     }
150 | 
151 |     public function getConsumeMode(): int
152 |     {
153 |         return $this->runtimeOptions['consume_mode'];
154 |     }
155 | }
156 | 


--------------------------------------------------------------------------------
/src/Protocol/Offset.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\Protocol as ProtocolException;
  7 | use function substr;
  8 | 
  9 | class Offset extends Protocol
 10 | {
 11 |     /**
 12 |      * @param mixed[] $payloads
 13 |      */
 14 |     public function encode(array $payloads = []): string
 15 |     {
 16 |         if (! isset($payloads['data'])) {
 17 |             throw new ProtocolException('given offset data invalid. `data` is undefined.');
 18 |         }
 19 | 
 20 |         if (! isset($payloads['replica_id'])) {
 21 |             $payloads['replica_id'] = -1;
 22 |         }
 23 | 
 24 |         $header = $this->requestHeader('kafka-php', self::OFFSET_REQUEST, self::OFFSET_REQUEST);
 25 |         $data   = self::pack(self::BIT_B32, (string) $payloads['replica_id']);
 26 |         $data  .= self::encodeArray($payloads['data'], [$this, 'encodeOffsetTopic']);
 27 |         $data   = self::encodeString($header . $data, self::PACK_INT32);
 28 | 
 29 |         return $data;
 30 |     }
 31 | 
 32 |     /**
 33 |      * @return mixed[]
 34 |      */
 35 |     public function decode(string $data): array
 36 |     {
 37 |         $offset = 0;
 38 | 
 39 |         $version = $this->getApiVersion(self::OFFSET_REQUEST);
 40 |         $topics  = $this->decodeArray(substr($data, $offset), [$this, 'offsetTopic'], $version);
 41 |         $offset += $topics['length'];
 42 | 
 43 |         return $topics['data'];
 44 |     }
 45 | 
 46 |     /**
 47 |      * @param mixed[] $values
 48 |      */
 49 |     protected function encodeOffsetPartition(array $values): string
 50 |     {
 51 |         if (! isset($values['partition_id'])) {
 52 |             throw new ProtocolException('given offset data invalid. `partition_id` is undefined.');
 53 |         }
 54 | 
 55 |         if (! isset($values['time'])) {
 56 |             $values['time'] = -1; // -1
 57 |         }
 58 | 
 59 |         if (! isset($values['max_offset'])) {
 60 |             $values['max_offset'] = 100000;
 61 |         }
 62 | 
 63 |         $data  = self::pack(self::BIT_B32, (string) $values['partition_id']);
 64 |         $data .= self::pack(self::BIT_B64, (string) $values['time']);
 65 | 
 66 |         if ($this->getApiVersion(self::OFFSET_REQUEST) === self::API_VERSION0) {
 67 |             $data .= self::pack(self::BIT_B32, (string) $values['max_offset']);
 68 |         }
 69 | 
 70 |         return $data;
 71 |     }
 72 | 
 73 |     /**
 74 |      * @param mixed[] $values
 75 |      */
 76 |     protected function encodeOffsetTopic(array $values): string
 77 |     {
 78 |         if (! isset($values['topic_name'])) {
 79 |             throw new ProtocolException('given offset data invalid. `topic_name` is undefined.');
 80 |         }
 81 | 
 82 |         if (! isset($values['partitions']) || empty($values['partitions'])) {
 83 |             throw new ProtocolException('given offset data invalid. `partitions` is undefined.');
 84 |         }
 85 | 
 86 |         $topic      = self::encodeString($values['topic_name'], self::PACK_INT16);
 87 |         $partitions = self::encodeArray($values['partitions'], [$this, 'encodeOffsetPartition']);
 88 | 
 89 |         return $topic . $partitions;
 90 |     }
 91 | 
 92 |     /**
 93 |      * @return mixed[]
 94 |      */
 95 |     protected function offsetTopic(string $data, int $version): array
 96 |     {
 97 |         $offset    = 0;
 98 |         $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16);
 99 |         $offset   += $topicInfo['length'];
100 | 
101 |         $partitions = $this->decodeArray(substr($data, $offset), [$this, 'offsetPartition'], $version);
102 |         $offset    += $partitions['length'];
103 | 
104 |         return [
105 |             'length' => $offset,
106 |             'data'   => [
107 |                 'topicName'  => $topicInfo['data'],
108 |                 'partitions' => $partitions['data'],
109 |             ],
110 |         ];
111 |     }
112 | 
113 |     /**
114 |      * @return mixed[]
115 |      */
116 |     protected function offsetPartition(string $data, int $version): array
117 |     {
118 |         $offset      = 0;
119 |         $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4));
120 |         $offset     += 4;
121 |         $errorCode   = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
122 |         $offset     += 2;
123 |         $timestamp   = 0;
124 | 
125 |         if ($version !== self::API_VERSION0) {
126 |             $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8));
127 |             $offset   += 8;
128 |         }
129 | 
130 |         $offsets = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B64);
131 |         $offset += $offsets['length'];
132 | 
133 |         return [
134 |             'length' => $offset,
135 |             'data'   => [
136 |                 'partition' => $partitionId,
137 |                 'errorCode' => $errorCode,
138 |                 'timestamp' => $timestamp,
139 |                 'offsets'   => $offsets['data'],
140 |             ],
141 |         ];
142 |     }
143 | }
144 | 


--------------------------------------------------------------------------------
/README_CH.md:
--------------------------------------------------------------------------------
  1 | Kafka-php
  2 | ==========
  3 | 
  4 | [English Document](README.md)
  5 | 
  6 | [![QQ Group](https://img.shields.io/badge/QQ%20Group-657517955-brightgreen.svg)]()
  7 | [![Build Status](https://travis-ci.org/weiboad/kafka-php.svg?branch=master)](https://travis-ci.org/weiboad/kafka-php)
  8 | [![Packagist](https://img.shields.io/packagist/dm/nmred/kafka-php.svg?style=plastic)]()
  9 | [![Packagist](https://img.shields.io/packagist/dd/nmred/kafka-php.svg?style=plastic)]()
 10 | [![Packagist](https://img.shields.io/packagist/dt/nmred/kafka-php.svg?style=plastic)]()
 11 | [![GitHub issues](https://img.shields.io/github/issues/weiboad/kafka-php.svg?style=plastic)](https://github.com/weiboad/kafka-php/issues)
 12 | [![GitHub forks](https://img.shields.io/github/forks/weiboad/kafka-php.svg?style=plastic)](https://github.com/weiboad/kafka-php/network)
 13 | [![GitHub stars](https://img.shields.io/github/stars/weiboad/kafka-php.svg?style=plastic)](https://github.com/weiboad/kafka-php/stargazers)
 14 | [![GitHub license](https://img.shields.io/badge/license-Apache%202-blue.svg?style=plastic)](https://raw.githubusercontent.com/weiboad/kafka-php/master/LICENSE)
 15 | 
 16 | Kafka-php 使用纯粹的PHP 编写的 kafka 客户端，目前支持 0.8.x 以上版本的 Kafka，该项目 v0.2.x 和 v0.1.x 不兼容，如果使用原有的 v0.1.x 的可以参照文档 [Kafka PHP v0.1.x Document](https://github.com/weiboad/kafka-php/blob/v0.1.6/README.md), 不过建议切换到 v0.2.x 上。v0.2.x 使用 PHP 异步执行的方式来和kafka broker 交互，较 v0.1.x 更加稳定高效, 由于使用 PHP 语言编写所以不用编译任何的扩展就可以使用，降低了接入与维护成本
 17 | 
 18 | 
 19 | ## 安装环境要求
 20 | 
 21 | * PHP 版本大于 5.5
 22 | * Kafka Server 版本大于 0.8.0
 23 | * 消费模块 Kafka Server 版本需要大于 0.9.0
 24 | 
 25 | ## Installation
 26 | 
 27 | ## 使用 Composer 安装
 28 | 
 29 | 添加 composer 依赖 `nmred/kafka-php` 到项目的 `composer.json` 文件中即可，如：
 30 | 
 31 | ```
 32 | {
 33 | 	"require": {
 34 | 		"nmred/kafka-php": "0.2.*"
 35 | 	}
 36 | }
 37 | ```
 38 | 
 39 | ## 配置
 40 | 
 41 | 配置参数见 [配置](docs/Configure.md)
 42 | 
 43 | ## Produce
 44 | 
 45 | ### 异步回调方式调用
 46 | 
 47 | ```php
 48 | <?php
 49 | require '../vendor/autoload.php';
 50 | date_default_timezone_set('PRC');
 51 | use Monolog\Logger;
 52 | use Monolog\Handler\StdoutHandler;
 53 | // Create the logger
 54 | $logger = new Logger('my_logger');
 55 | // Now add some handlers
 56 | $logger->pushHandler(new StdoutHandler());
 57 | 
 58 | $config = \Kafka\ProducerConfig::getInstance();
 59 | $config->setMetadataRefreshIntervalMs(10000);
 60 | $config->setMetadataBrokerList('10.13.4.159:9192');
 61 | $config->setBrokerVersion('0.9.0.1');
 62 | $config->setRequiredAck(1);
 63 | $config->setIsAsyn(false);
 64 | $config->setProduceInterval(500);
 65 | $producer = new \Kafka\Producer(function() {
 66 | 	return array(
 67 | 		array(
 68 | 			'topic' => 'test',
 69 | 			'value' => 'test....message.',
 70 | 			'key' => 'testkey',
 71 | 		),
 72 | 	);
 73 | });
 74 | $producer->setLogger($logger);
 75 | $producer->success(function($result) {
 76 | 	var_dump($result);
 77 | });
 78 | $producer->error(function($errorCode) {
 79 | 	var_dump($errorCode);
 80 | });
 81 | $producer->send(true);
 82 | ```
 83 | 
 84 | ### 同步方式调用生产者
 85 | 
 86 | ```
 87 | <?php
 88 | require '../vendor/autoload.php';
 89 | date_default_timezone_set('PRC');
 90 | use Monolog\Logger;
 91 | use Monolog\Handler\StdoutHandler;
 92 | // Create the logger
 93 | $logger = new Logger('my_logger');
 94 | // Now add some handlers
 95 | $logger->pushHandler(new StdoutHandler());
 96 | 
 97 | $config = \Kafka\ProducerConfig::getInstance();
 98 | $config->setMetadataRefreshIntervalMs(10000);
 99 | $config->setMetadataBrokerList('127.0.0.1:9192');
100 | $config->setBrokerVersion('0.9.0.1');
101 | $config->setRequiredAck(1);
102 | $config->setIsAsyn(false);
103 | $config->setProduceInterval(500);
104 | $producer = new \Kafka\Producer();
105 | $producer->setLogger($logger);
106 | 
107 | for($i = 0; $i < 100; $i++) {
108 |         $result = $producer->send(array(
109 |                 array(
110 |                         'topic' => 'test1',
111 |                         'value' => 'test1....message.',
112 |                         'key' => '',
113 |                 ),
114 |         ));
115 |         var_dump($result);
116 | }
117 | ```
118 | 
119 | ## Consumer
120 | 
121 | ```php
122 | <?php
123 | require '../vendor/autoload.php';
124 | date_default_timezone_set('PRC');
125 | use Monolog\Logger;
126 | use Monolog\Handler\StdoutHandler;
127 | // Create the logger
128 | $logger = new Logger('my_logger');
129 | // Now add some handlers
130 | $logger->pushHandler(new StdoutHandler());
131 | 
132 | $config = \Kafka\ConsumerConfig::getInstance();
133 | $config->setMetadataRefreshIntervalMs(10000);
134 | $config->setMetadataBrokerList('10.13.4.159:9192');
135 | $config->setGroupId('test');
136 | $config->setBrokerVersion('0.9.0.1');
137 | $config->setTopics(array('test'));
138 | //$config->setOffsetReset('earliest');
139 | $consumer = new \Kafka\Consumer();
140 | $consumer->setLogger($logger);
141 | $consumer->start(function($topic, $part, $message) {
142 | 	var_dump($message);
143 | });
144 | ```
145 | 
146 | ## Basic Protocol
147 | 
148 | 基础协议 API 调用方式见 [Example](https://github.com/weiboad/kafka-php/tree/master/example)
149 | 
150 | ## QQ 群号 
151 | 
152 | 群一： 531522091 （已满）
153 | 群二： 657517955
154 | ![QQ Group](docs/qq_group.png)
155 | 


--------------------------------------------------------------------------------
/src/Socket.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use Amp\Loop;
  7 | use Kafka\Protocol\Protocol;
  8 | use function fclose;
  9 | use function feof;
 10 | use function fread;
 11 | use function fwrite;
 12 | use function is_resource;
 13 | use function stream_set_blocking;
 14 | use function stream_set_read_buffer;
 15 | use function strlen;
 16 | use function substr;
 17 | 
 18 | class Socket extends CommonSocket
 19 | {
 20 |     /**
 21 |      * @var string
 22 |      */
 23 |     private $readWatcherId = '';
 24 | 
 25 |     /**
 26 |      * @var string
 27 |      */
 28 |     private $writeWatcherId = '';
 29 | 
 30 |     /**
 31 |      * @var string
 32 |      */
 33 |     private $writeBuffer = '';
 34 | 
 35 |     /**
 36 |      * @var string
 37 |      */
 38 |     private $readBuffer = '';
 39 | 
 40 |     /**
 41 |      * @var int
 42 |      */
 43 |     private $readNeedLength = 0;
 44 | 
 45 |     /**
 46 |      * @var callable|null
 47 |      */
 48 |     private $onReadable;
 49 | 
 50 |     public function connect(): void
 51 |     {
 52 |         if (! $this->isSocketDead()) {
 53 |             return;
 54 |         }
 55 | 
 56 |         $this->createStream();
 57 | 
 58 |         stream_set_blocking($this->stream, false);
 59 |         stream_set_read_buffer($this->stream, 0);
 60 | 
 61 |         $this->readWatcherId = Loop::onReadable(
 62 |             $this->stream,
 63 |             function (): void {
 64 |                 do {
 65 |                     if ($this->isSocketDead()) {
 66 |                         $this->reconnect();
 67 |                         return;
 68 |                     }
 69 | 
 70 |                     $newData = @fread($this->stream, self::READ_MAX_LENGTH);
 71 | 
 72 |                     if ($newData) {
 73 |                         $this->read($newData);
 74 |                     }
 75 |                 } while ($newData);
 76 |             }
 77 |         );
 78 | 
 79 |         $this->writeWatcherId = Loop::onWritable(
 80 |             $this->stream,
 81 |             function (): void {
 82 |                 $this->write();
 83 |             },
 84 |             ['enable' => false] // <-- let's initialize the watcher as "disabled"
 85 |         );
 86 |     }
 87 | 
 88 |     public function reconnect(): void
 89 |     {
 90 |         $this->close();
 91 |         $this->connect();
 92 |     }
 93 | 
 94 |     public function setOnReadable(callable $read): void
 95 |     {
 96 |         $this->onReadable = $read;
 97 |     }
 98 | 
 99 |     public function close(): void
100 |     {
101 |         Loop::cancel($this->readWatcherId);
102 |         Loop::cancel($this->writeWatcherId);
103 | 
104 |         if (is_resource($this->stream)) {
105 |             fclose($this->stream);
106 |         }
107 | 
108 |         $this->readBuffer     = '';
109 |         $this->writeBuffer    = '';
110 |         $this->readNeedLength = 0;
111 |     }
112 | 
113 |     public function isResource(): bool
114 |     {
115 |         return is_resource($this->stream);
116 |     }
117 | 
118 |     /**
119 |      * Read from the socket at most $len bytes.
120 |      *
121 |      * This method will not wait for all the requested data, it will return as
122 |      * soon as any data is received.
123 |      *
124 |      * @param string|int $data
125 |      */
126 |     public function read($data): void
127 |     {
128 |         $this->readBuffer .= (string) $data;
129 | 
130 |         do {
131 |             if ($this->readNeedLength === 0) { // response start
132 |                 if (strlen($this->readBuffer) < 4) {
133 |                     return;
134 |                 }
135 | 
136 |                 $dataLen              = Protocol::unpack(Protocol::BIT_B32, substr($this->readBuffer, 0, 4));
137 |                 $this->readNeedLength = $dataLen;
138 |                 $this->readBuffer     = substr($this->readBuffer, 4);
139 |             }
140 | 
141 |             if (strlen($this->readBuffer) < $this->readNeedLength) {
142 |                 return;
143 |             }
144 | 
145 |             $data = (string) substr($this->readBuffer, 0, $this->readNeedLength);
146 | 
147 |             $this->readBuffer     = substr($this->readBuffer, $this->readNeedLength);
148 |             $this->readNeedLength = 0;
149 | 
150 |             ($this->onReadable)($data, (int) $this->stream);
151 |         } while (strlen($this->readBuffer));
152 |     }
153 | 
154 |     /**
155 |      * Write to the socket.
156 |      *
157 |      * @throws Loop\InvalidWatcherError
158 |      */
159 |     public function write(?string $data = null): void
160 |     {
161 |         if ($data !== null) {
162 |             $this->writeBuffer .= $data;
163 |         }
164 | 
165 |         $bytesToWrite = strlen($this->writeBuffer);
166 |         $bytesWritten = @fwrite($this->stream, $this->writeBuffer);
167 | 
168 |         if ($bytesToWrite === $bytesWritten) {
169 |             Loop::disable($this->writeWatcherId);
170 |         } elseif ($bytesWritten >= 0) {
171 |             Loop::enable($this->writeWatcherId);
172 |         } elseif ($this->isSocketDead()) {
173 |             $this->reconnect();
174 |         }
175 | 
176 |         $this->writeBuffer = substr($this->writeBuffer, $bytesWritten);
177 |     }
178 | 
179 |     /**
180 |      * check the stream is close
181 |      */
182 |     protected function isSocketDead(): bool
183 |     {
184 |         return ! is_resource($this->stream) || @feof($this->stream);
185 |     }
186 | }
187 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | Kafka-php
  2 | ==========
  3 | 
  4 | [中文文档](README_CH.md)
  5 | 
  6 | [![QQ Group](https://img.shields.io/badge/QQ%20Group-657517955-brightgreen.svg)]()
  7 | [![Build Status](https://travis-ci.org/weiboad/kafka-php.svg?branch=master)](https://travis-ci.org/weiboad/kafka-php)
  8 | [![Packagist](https://img.shields.io/packagist/dm/nmred/kafka-php.svg?style=plastic)]()
  9 | [![Packagist](https://img.shields.io/packagist/dd/nmred/kafka-php.svg?style=plastic)]()
 10 | [![Packagist](https://img.shields.io/packagist/dt/nmred/kafka-php.svg?style=plastic)]()
 11 | [![GitHub issues](https://img.shields.io/github/issues/weiboad/kafka-php.svg?style=plastic)](https://github.com/weiboad/kafka-php/issues)
 12 | [![GitHub forks](https://img.shields.io/github/forks/weiboad/kafka-php.svg?style=plastic)](https://github.com/weiboad/kafka-php/network)
 13 | [![GitHub stars](https://img.shields.io/github/stars/weiboad/kafka-php.svg?style=plastic)](https://github.com/weiboad/kafka-php/stargazers)
 14 | [![GitHub license](https://img.shields.io/badge/license-Apache%202-blue.svg?style=plastic)](https://raw.githubusercontent.com/weiboad/kafka-php/master/LICENSE)
 15 | 
 16 | Kafka-php is a pure PHP kafka client that currently supports greater than 0.8.x version of Kafka, this project v0.2.x and v0.1.x are incompatible if using the original v0.1.x You can refer to the document 
 17 | [Kafka PHP v0.1.x Document](https://github.com/weiboad/kafka-php/blob/v0.1.6/README.md), but it is recommended to switch to v0.2.x . v0.2.x use PHP asynchronous implementation and kafka broker interaction, more stable than v0.1.x efficient, because the use of PHP language so do not compile any expansion can be used to reduce the access and maintenance costs
 18 | 
 19 | 
 20 | ## Requirements
 21 | 
 22 | * Minimum PHP version: 7.1
 23 | * Kafka version greater than 0.8
 24 | * The consumer module needs kafka broker version  greater than 0.9.0
 25 | 
 26 | ## Installation
 27 | 
 28 | Add the lib directory to the PHP include_path and use an autoloader like the one in the examples directory (the code follows the PEAR/Zend one-class-per-file convention).
 29 | 
 30 | ## Composer Install
 31 | 
 32 | Simply add a dependency `nmred/kafka-php` to your project if you use Composer to manage the dependencies of your project.
 33 |  
 34 | `$ composer require nmred/kafka-php`
 35 | 
 36 |  Here is a minimal example of a composer.json file :
 37 | ```
 38 | {
 39 | 	"require": {
 40 | 		"nmred/kafka-php": "0.2.*"
 41 | 	}
 42 | }
 43 | ```
 44 | 
 45 | ## Configuration
 46 | 
 47 | Configuration properties are documented in [Configuration](docs/Configure.md)
 48 | 
 49 | ## Producer
 50 | 
 51 | ### Asynchronous mode
 52 | 
 53 | ```php
 54 | <?php
 55 | require '../vendor/autoload.php';
 56 | date_default_timezone_set('PRC');
 57 | use Monolog\Logger;
 58 | use Monolog\Handler\StdoutHandler;
 59 | // Create the logger
 60 | $logger = new Logger('my_logger');
 61 | // Now add some handlers
 62 | $logger->pushHandler(new StdoutHandler());
 63 | 
 64 | $config = \Kafka\ProducerConfig::getInstance();
 65 | $config->setMetadataRefreshIntervalMs(10000);
 66 | $config->setMetadataBrokerList('10.13.4.159:9192');
 67 | $config->setBrokerVersion('1.0.0');
 68 | $config->setRequiredAck(1);
 69 | $config->setIsAsyn(false);
 70 | $config->setProduceInterval(500);
 71 | $producer = new \Kafka\Producer(
 72 |     function() {
 73 |         return [
 74 |             [
 75 |                 'topic' => 'test',
 76 |                 'value' => 'test....message.',
 77 |                 'key' => 'testkey',
 78 |             ],
 79 |         ];
 80 |     }
 81 | );
 82 | $producer->setLogger($logger);
 83 | $producer->success(function($result) {
 84 | 	var_dump($result);
 85 | });
 86 | $producer->error(function($errorCode) {
 87 | 		var_dump($errorCode);
 88 | });
 89 | $producer->send(true);
 90 | ```
 91 | 
 92 | ### Synchronous mode
 93 | 
 94 | ```php
 95 | <?php
 96 | require '../vendor/autoload.php';
 97 | date_default_timezone_set('PRC');
 98 | use Monolog\Logger;
 99 | use Monolog\Handler\StdoutHandler;
100 | // Create the logger
101 | $logger = new Logger('my_logger');
102 | // Now add some handlers
103 | $logger->pushHandler(new StdoutHandler());
104 | 
105 | $config = \Kafka\ProducerConfig::getInstance();
106 | $config->setMetadataRefreshIntervalMs(10000);
107 | $config->setMetadataBrokerList('127.0.0.1:9192');
108 | $config->setBrokerVersion('1.0.0');
109 | $config->setRequiredAck(1);
110 | $config->setIsAsyn(false);
111 | $config->setProduceInterval(500);
112 | $producer = new \Kafka\Producer();
113 | $producer->setLogger($logger);
114 | 
115 | for($i = 0; $i < 100; $i++) {
116 |     $producer->send([
117 |         [
118 |             'topic' => 'test1',
119 |             'value' => 'test1....message.',
120 |             'key' => '',
121 |         ],
122 |     ]);
123 | }
124 | ```
125 | 
126 | ## Consumer
127 | 
128 | ```php
129 | <?php
130 | require '../vendor/autoload.php';
131 | date_default_timezone_set('PRC');
132 | use Monolog\Logger;
133 | use Monolog\Handler\StdoutHandler;
134 | // Create the logger
135 | $logger = new Logger('my_logger');
136 | // Now add some handlers
137 | $logger->pushHandler(new StdoutHandler());
138 | 
139 | $config = \Kafka\ConsumerConfig::getInstance();
140 | $config->setMetadataRefreshIntervalMs(10000);
141 | $config->setMetadataBrokerList('10.13.4.159:9192');
142 | $config->setGroupId('test');
143 | $config->setBrokerVersion('1.0.0');
144 | $config->setTopics(['test']);
145 | //$config->setOffsetReset('earliest');
146 | $consumer = new \Kafka\Consumer();
147 | $consumer->setLogger($logger);
148 | $consumer->start(function($topic, $part, $message) {
149 | 	var_dump($message);
150 | });
151 | ```
152 | 
153 | ## Low-Level API
154 | 
155 | Refer [Example](https://github.com/weiboad/kafka-php/tree/master/example)
156 | 
157 | 
158 | ## QQ Group
159 | 
160 | Group 1: 531522091 
161 | Group 2: 657517955
162 | ![QQ Group](docs/qq_group.png)
163 | 


--------------------------------------------------------------------------------
/src/Protocol/DescribeGroups.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use function substr;
  8 | 
  9 | class DescribeGroups extends Protocol
 10 | {
 11 |     /**
 12 |      * @param mixed[] $payloads
 13 |      *
 14 |      * @throws NotSupported
 15 |      */
 16 |     public function encode(array $payloads = []): string
 17 |     {
 18 |         $header = $this->requestHeader('kafka-php', self::DESCRIBE_GROUPS_REQUEST, self::DESCRIBE_GROUPS_REQUEST);
 19 |         $data   = self::encodeArray($payloads, [$this, 'encodeString'], self::PACK_INT16);
 20 | 
 21 |         return self::encodeString($header . $data, self::PACK_INT32);
 22 |     }
 23 | 
 24 |     /**
 25 |      * @return mixed[]
 26 |      */
 27 |     public function decode(string $data): array
 28 |     {
 29 |         $offset  = 0;
 30 |         $groups  = $this->decodeArray(substr($data, $offset), [$this, 'describeGroup']);
 31 |         $offset += $groups['length'];
 32 | 
 33 |         return $groups['data'];
 34 |     }
 35 | 
 36 |     /**
 37 |      * @return mixed[]
 38 |      */
 39 |     protected function describeGroup(string $data): array
 40 |     {
 41 |         $offset       = 0;
 42 |         $errorCode    = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
 43 |         $offset      += 2;
 44 |         $groupId      = $this->decodeString(substr($data, $offset), self::BIT_B16);
 45 |         $offset      += $groupId['length'];
 46 |         $state        = $this->decodeString(substr($data, $offset), self::BIT_B16);
 47 |         $offset      += $state['length'];
 48 |         $protocolType = $this->decodeString(substr($data, $offset), self::BIT_B16);
 49 |         $offset      += $protocolType['length'];
 50 |         $protocol     = $this->decodeString(substr($data, $offset), self::BIT_B16);
 51 |         $offset      += $protocol['length'];
 52 | 
 53 |         $members = $this->decodeArray(substr($data, $offset), [$this, 'describeMember']);
 54 |         $offset += $members['length'];
 55 | 
 56 |         return [
 57 |             'length' => $offset,
 58 |             'data'   => [
 59 |                 'errorCode'    => $errorCode,
 60 |                 'groupId'      => $groupId['data'],
 61 |                 'state'        => $state['data'],
 62 |                 'protocolType' => $protocolType['data'],
 63 |                 'protocol'     => $protocol['data'],
 64 |                 'members'      => $members['data'],
 65 |             ],
 66 |         ];
 67 |     }
 68 | 
 69 |     /**
 70 |      * @return mixed[]
 71 |      */
 72 |     protected function describeMember(string $data): array
 73 |     {
 74 |         $offset     = 0;
 75 |         $memberId   = $this->decodeString(substr($data, $offset), self::BIT_B16);
 76 |         $offset    += $memberId['length'];
 77 |         $clientId   = $this->decodeString(substr($data, $offset), self::BIT_B16);
 78 |         $offset    += $clientId['length'];
 79 |         $clientHost = $this->decodeString(substr($data, $offset), self::BIT_B16);
 80 |         $offset    += $clientHost['length'];
 81 |         $metadata   = $this->decodeString(substr($data, $offset), self::BIT_B32);
 82 |         $offset    += $metadata['length'];
 83 |         $assignment = $this->decodeString(substr($data, $offset), self::BIT_B32);
 84 |         $offset    += $assignment['length'];
 85 | 
 86 |         $memberAssignment        = $assignment['data'];
 87 |         $memberAssignmentOffset  = 0;
 88 |         $version                 = self::unpack(self::BIT_B16_SIGNED, substr($memberAssignment, $memberAssignmentOffset, 2));
 89 |         $memberAssignmentOffset += 2;
 90 |         $partitionAssignments    = $this->decodeArray(substr($memberAssignment, $memberAssignmentOffset), [$this, 'describeResponsePartition']);
 91 |         $memberAssignmentOffset += $partitionAssignments['length'];
 92 |         $userData                = $this->decodeString(substr($memberAssignment, $memberAssignmentOffset), self::BIT_B32);
 93 | 
 94 |         $metaData     = $metadata['data'];
 95 |         $metaOffset   = 0;
 96 |         $version      = self::unpack(self::BIT_B16, substr($metaData, $metaOffset, 2));
 97 |         $metaOffset  += 2;
 98 |         $topics       = $this->decodeArray(substr($metaData, $metaOffset), [$this, 'decodeString'], self::BIT_B16);
 99 |         $metaOffset  += $topics['length'];
100 |         $metaUserData = $this->decodeString(substr($metaData, $metaOffset), self::BIT_B32);
101 | 
102 |         return [
103 |             'length' => $offset,
104 |             'data'   => [
105 |                 'memberId'   => $memberId['data'],
106 |                 'clientId'   => $clientId['data'],
107 |                 'clientHost' => $clientHost['data'],
108 |                 'metadata'   => [
109 |                     'version'  => $version,
110 |                     'topics'   => $topics['data'],
111 |                     'userData' => $metaUserData['data'],
112 |                 ],
113 |                 'assignment' => [
114 |                     'version'    => $version,
115 |                     'partitions' => $partitionAssignments['data'],
116 |                     'userData'   => $userData['data'],
117 |                 ],
118 |             ],
119 |         ];
120 |     }
121 | 
122 |     /**
123 |      * @return mixed[]
124 |      */
125 |     protected function describeResponsePartition(string $data): array
126 |     {
127 |         $offset     = 0;
128 |         $topicName  = $this->decodeString(substr($data, $offset), self::BIT_B16);
129 |         $offset    += $topicName['length'];
130 |         $partitions = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32);
131 |         $offset    += $partitions['length'];
132 | 
133 |         return [
134 |             'length' => $offset,
135 |             'data'   => [
136 |                 'topicName'  => $topicName['data'],
137 |                 'partitions' => $partitions['data'],
138 |             ],
139 |         ];
140 |     }
141 | }
142 | 


--------------------------------------------------------------------------------
/src/Protocol/CommitOffset.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use Kafka\Exception\Protocol as ProtocolException;
  8 | use function substr;
  9 | use function time;
 10 | 
 11 | class CommitOffset extends Protocol
 12 | {
 13 |     /**
 14 |      * @param mixed[] $payloads
 15 |      *
 16 |      * @throws ProtocolException
 17 |      * @throws NotSupported
 18 |      */
 19 |     public function encode(array $payloads = []): string
 20 |     {
 21 |         if (! isset($payloads['group_id'])) {
 22 |             throw new ProtocolException('given commit offset data invalid. `group_id` is undefined.');
 23 |         }
 24 | 
 25 |         if (! isset($payloads['data'])) {
 26 |             throw new ProtocolException('given commit data invalid. `data` is undefined.');
 27 |         }
 28 | 
 29 |         if (! isset($payloads['generation_id'])) {
 30 |             $payloads['generation_id'] = -1;
 31 |         }
 32 | 
 33 |         if (! isset($payloads['member_id'])) {
 34 |             $payloads['member_id'] = '';
 35 |         }
 36 | 
 37 |         if (! isset($payloads['retention_time'])) {
 38 |             $payloads['retention_time'] = -1;
 39 |         }
 40 | 
 41 |         $version = $this->getApiVersion(self::OFFSET_COMMIT_REQUEST);
 42 |         $header  = $this->requestHeader('kafka-php', self::OFFSET_COMMIT_REQUEST, self::OFFSET_COMMIT_REQUEST);
 43 | 
 44 |         $data = self::encodeString($payloads['group_id'], self::PACK_INT16);
 45 | 
 46 |         if ($version === self::API_VERSION1) {
 47 |             $data .= self::pack(self::BIT_B32, (string) $payloads['generation_id']);
 48 |             $data .= self::encodeString($payloads['member_id'], self::PACK_INT16);
 49 |         }
 50 | 
 51 |         if ($version === self::API_VERSION2) {
 52 |             $data .= self::pack(self::BIT_B32, (string) $payloads['generation_id']);
 53 |             $data .= self::encodeString($payloads['member_id'], self::PACK_INT16);
 54 |             $data .= self::pack(self::BIT_B64, (string) $payloads['retention_time']);
 55 |         }
 56 | 
 57 |         $data .= self::encodeArray($payloads['data'], [$this, 'encodeTopic']);
 58 |         $data  = self::encodeString($header . $data, self::PACK_INT32);
 59 | 
 60 |         return $data;
 61 |     }
 62 | 
 63 |     /**
 64 |      * @return mixed[]
 65 |      */
 66 |     public function decode(string $data): array
 67 |     {
 68 |         $offset  = 0;
 69 |         $topics  = $this->decodeArray(substr($data, $offset), [$this, 'decodeTopic']);
 70 |         $offset += $topics['length'];
 71 | 
 72 |         return $topics['data'];
 73 |     }
 74 | 
 75 |     /**
 76 |      * @param mixed[] $values
 77 |      *
 78 |      * @throws NotSupported
 79 |      * @throws ProtocolException
 80 |      */
 81 |     protected function encodeTopic(array $values): string
 82 |     {
 83 |         if (! isset($values['topic_name'])) {
 84 |             throw new ProtocolException('given commit offset data invalid. `topic_name` is undefined.');
 85 |         }
 86 |         if (! isset($values['partitions'])) {
 87 |             throw new ProtocolException('given commit offset data invalid. `partitions` is undefined.');
 88 |         }
 89 | 
 90 |         $data  = self::encodeString($values['topic_name'], self::PACK_INT16);
 91 |         $data .= self::encodeArray($values['partitions'], [$this, 'encodePartition']);
 92 | 
 93 |         return $data;
 94 |     }
 95 | 
 96 |     /**
 97 |      * @param mixed[] $values
 98 |      *
 99 |      * @throws NotSupported
100 |      * @throws ProtocolException
101 |      */
102 |     protected function encodePartition(array $values): string
103 |     {
104 |         if (! isset($values['partition'])) {
105 |             throw new ProtocolException('given commit offset data invalid. `partition` is undefined.');
106 |         }
107 | 
108 |         if (! isset($values['offset'])) {
109 |             throw new ProtocolException('given commit offset data invalid. `offset` is undefined.');
110 |         }
111 | 
112 |         if (! isset($values['metadata'])) {
113 |             $values['metadata'] = '';
114 |         }
115 | 
116 |         if (! isset($values['timestamp'])) {
117 |             $values['timestamp'] = time() * 1000;
118 |         }
119 | 
120 |         $version = $this->getApiVersion(self::OFFSET_COMMIT_REQUEST);
121 | 
122 |         $data  = self::pack(self::BIT_B32, (string) $values['partition']);
123 |         $data .= self::pack(self::BIT_B64, (string) $values['offset']);
124 | 
125 |         if ($version === self::API_VERSION1) {
126 |             $data .= self::pack(self::BIT_B64, (string) $values['timestamp']);
127 |         }
128 | 
129 |         $data .= self::encodeString($values['metadata'], self::PACK_INT16);
130 | 
131 |         return $data;
132 |     }
133 | 
134 |     /**
135 |      * @return mixed[]
136 |      */
137 |     protected function decodeTopic(string $data): array
138 |     {
139 |         $offset    = 0;
140 |         $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16);
141 |         $offset   += $topicInfo['length'];
142 | 
143 |         $partitions = $this->decodeArray(substr($data, $offset), [$this, 'decodePartition']);
144 |         $offset    += $partitions['length'];
145 | 
146 |         return [
147 |             'length' => $offset,
148 |             'data'   => [
149 |                 'topicName'  => $topicInfo['data'],
150 |                 'partitions' => $partitions['data'],
151 |             ],
152 |         ];
153 |     }
154 | 
155 |     /**
156 |      * @return mixed[]
157 |      */
158 |     protected function decodePartition(string $data): array
159 |     {
160 |         $offset = 0;
161 | 
162 |         $partitionId = self::unpack(self::BIT_B32, substr($data, $offset, 4));
163 |         $offset     += 4;
164 | 
165 |         $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
166 |         $offset   += 2;
167 | 
168 |         return [
169 |             'length' => $offset,
170 |             'data'   => [
171 |                 'partition' => $partitionId,
172 |                 'errorCode' => $errorCode,
173 |             ],
174 |         ];
175 |     }
176 | }
177 | 


--------------------------------------------------------------------------------
/src/Protocol/SyncGroup.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use Kafka\Exception\Protocol as ProtocolException;
  8 | use function substr;
  9 | 
 10 | class SyncGroup extends Protocol
 11 | {
 12 |     /**
 13 |      * @param mixed[] $payloads
 14 |      *
 15 |      * @throws NotSupported
 16 |      * @throws ProtocolException
 17 |      */
 18 |     public function encode(array $payloads = []): string
 19 |     {
 20 |         if (! isset($payloads['group_id'])) {
 21 |             throw new ProtocolException('given sync group data invalid. `group_id` is undefined.');
 22 |         }
 23 | 
 24 |         if (! isset($payloads['generation_id'])) {
 25 |             throw new ProtocolException('given sync group data invalid. `generation_id` is undefined.');
 26 |         }
 27 | 
 28 |         if (! isset($payloads['member_id'])) {
 29 |             throw new ProtocolException('given sync group data invalid. `member_id` is undefined.');
 30 |         }
 31 | 
 32 |         if (! isset($payloads['data'])) {
 33 |             throw new ProtocolException('given sync group data invalid. `data` is undefined.');
 34 |         }
 35 | 
 36 |         $header = $this->requestHeader('kafka-php', self::SYNC_GROUP_REQUEST, self::SYNC_GROUP_REQUEST);
 37 |         $data   = self::encodeString($payloads['group_id'], self::PACK_INT16);
 38 |         $data  .= self::pack(self::BIT_B32, (string) $payloads['generation_id']);
 39 |         $data  .= self::encodeString($payloads['member_id'], self::PACK_INT16);
 40 |         $data  .= self::encodeArray($payloads['data'], [$this, 'encodeGroupAssignment']);
 41 | 
 42 |         return self::encodeString($header . $data, self::PACK_INT32);
 43 |     }
 44 | 
 45 |     /**
 46 |      * @return mixed[]
 47 |      */
 48 |     public function decode(string $data): array
 49 |     {
 50 |         $offset    = 0;
 51 |         $errorCode = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
 52 |         $offset   += 2;
 53 | 
 54 |         $memberAssignments = $this->decodeString(substr($data, $offset), self::BIT_B32);
 55 |         $offset           += $memberAssignments['length'];
 56 | 
 57 |         $memberAssignment = $memberAssignments['data'];
 58 | 
 59 |         if ($memberAssignment === '') {
 60 |             return ['errorCode' => $errorCode];
 61 |         }
 62 | 
 63 |         $memberAssignmentOffset  = 0;
 64 |         $version                 = self::unpack(
 65 |             self::BIT_B16_SIGNED,
 66 |             substr($memberAssignment, $memberAssignmentOffset, 2)
 67 |         );
 68 |         $memberAssignmentOffset += 2;
 69 |         $partitionAssignments    = $this->decodeArray(
 70 |             substr($memberAssignment, $memberAssignmentOffset),
 71 |             [$this, 'syncGroupResponsePartition']
 72 |         );
 73 |         $memberAssignmentOffset += $partitionAssignments['length'];
 74 |         $userData                = $this->decodeString(
 75 |             substr($memberAssignment, $memberAssignmentOffset),
 76 |             self::BIT_B32
 77 |         );
 78 | 
 79 |         return [
 80 |             'errorCode'            => $errorCode,
 81 |             'partitionAssignments' => $partitionAssignments['data'],
 82 |             'version'              => $version,
 83 |             'userData'             => $userData['data'],
 84 |         ];
 85 |     }
 86 | 
 87 |     /**
 88 |      * @param mixed[] $values
 89 |      *
 90 |      * @throws NotSupported
 91 |      * @throws ProtocolException
 92 |      */
 93 |     protected function encodeGroupAssignment(array $values): string
 94 |     {
 95 |         if (! isset($values['version'])) {
 96 |             throw new ProtocolException('given data invalid. `version` is undefined.');
 97 |         }
 98 | 
 99 |         if (! isset($values['member_id'])) {
100 |             throw new ProtocolException('given data invalid. `member_id` is undefined.');
101 |         }
102 | 
103 |         if (! isset($values['assignments'])) {
104 |             throw new ProtocolException('given data invalid. `assignments` is undefined.');
105 |         }
106 | 
107 |         if (! isset($values['user_data'])) {
108 |             $values['user_data'] = '';
109 |         }
110 | 
111 |         $memberId = self::encodeString($values['member_id'], self::PACK_INT16);
112 | 
113 |         $data  = self::pack(self::BIT_B16, '0');
114 |         $data .= self::encodeArray($values['assignments'], [$this, 'encodeGroupAssignmentTopic']);
115 |         $data .= self::encodeString($values['user_data'], self::PACK_INT32);
116 | 
117 |         return $memberId . self::encodeString($data, self::PACK_INT32);
118 |     }
119 | 
120 |     /**
121 |      * @param mixed[] $values
122 |      *
123 |      * @throws ProtocolException
124 |      * @throws NotSupported
125 |      */
126 |     protected function encodeGroupAssignmentTopic(array $values): string
127 |     {
128 |         if (! isset($values['topic_name'])) {
129 |             throw new ProtocolException('given data invalid. `topic_name` is undefined.');
130 |         }
131 | 
132 |         if (! isset($values['partitions'])) {
133 |             throw new ProtocolException('given data invalid. `partitions` is undefined.');
134 |         }
135 | 
136 |         $topicName  = self::encodeString($values['topic_name'], self::PACK_INT16);
137 |         $partitions = self::encodeArray($values['partitions'], [$this, 'encodeGroupAssignmentTopicPartition']);
138 | 
139 |         return $topicName . $partitions;
140 |     }
141 | 
142 |     protected function encodeGroupAssignmentTopicPartition(int $values): string
143 |     {
144 |         return self::pack(self::BIT_B32, (string) $values);
145 |     }
146 | 
147 |     /**
148 |      * @return mixed[]
149 |      */
150 |     protected function syncGroupResponsePartition(string $data): array
151 |     {
152 |         $offset     = 0;
153 |         $topicName  = $this->decodeString(substr($data, $offset), self::BIT_B16);
154 |         $offset    += $topicName['length'];
155 |         $partitions = $this->decodePrimitiveArray(substr($data, $offset), self::BIT_B32);
156 |         $offset    += $partitions['length'];
157 | 
158 |         return [
159 |             'length' => $offset,
160 |             'data'   => [
161 |                 'topicName'  => $topicName['data'],
162 |                 'partitions' => $partitions['data'],
163 |             ],
164 |         ];
165 |     }
166 | }
167 | 


--------------------------------------------------------------------------------
/src/Protocol/JoinGroup.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use Kafka\Exception\Protocol as ProtocolException;
  8 | use function substr;
  9 | 
 10 | class JoinGroup extends Protocol
 11 | {
 12 |     /**
 13 |      * @param mixed[] $payloads
 14 |      *
 15 |      * @throws NotSupported
 16 |      * @throws ProtocolException
 17 |      */
 18 |     public function encode(array $payloads = []): string
 19 |     {
 20 |         if (! isset($payloads['group_id'])) {
 21 |             throw new ProtocolException('given join group data invalid. `group_id` is undefined.');
 22 |         }
 23 | 
 24 |         if (! isset($payloads['session_timeout'])) {
 25 |             throw new ProtocolException('given join group data invalid. `session_timeout` is undefined.');
 26 |         }
 27 | 
 28 |         if (! isset($payloads['member_id'])) {
 29 |             throw new ProtocolException('given join group data invalid. `member_id` is undefined.');
 30 |         }
 31 | 
 32 |         if (! isset($payloads['data'])) {
 33 |             throw new ProtocolException('given join group data invalid. `data` is undefined.');
 34 |         }
 35 | 
 36 |         if (! isset($payloads['protocol_type'])) {
 37 |             $payloads['protocol_type'] = 'consumer';
 38 |         }
 39 | 
 40 |         if (! isset($payloads['rebalance_timeout'])) {
 41 |             $payloads['rebalance_timeout'] = $payloads['session_timeout'];
 42 |         }
 43 | 
 44 |         $header = $this->requestHeader('kafka-php', self::JOIN_GROUP_REQUEST, self::JOIN_GROUP_REQUEST);
 45 |         $data   = self::encodeString($payloads['group_id'], self::PACK_INT16);
 46 |         $data  .= self::pack(self::BIT_B32, (string) $payloads['session_timeout']);
 47 | 
 48 |         if ($this->getApiVersion(self::JOIN_GROUP_REQUEST) === self::API_VERSION1) {
 49 |             $data .= self::pack(self::BIT_B32, (string) $payloads['rebalance_timeout']);
 50 |         }
 51 | 
 52 |         $data .= self::encodeString($payloads['member_id'], self::PACK_INT16);
 53 |         $data .= self::encodeString($payloads['protocol_type'], self::PACK_INT16);
 54 |         $data .= self::encodeArray($payloads['data'], [$this, 'encodeGroupProtocol']);
 55 |         $data  = self::encodeString($header . $data, self::PACK_INT32);
 56 | 
 57 |         return $data;
 58 |     }
 59 | 
 60 |     /**
 61 |      * @return mixed[]
 62 |      */
 63 |     public function decode(string $data): array
 64 |     {
 65 |         $offset        = 0;
 66 |         $errorCode     = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
 67 |         $offset       += 2;
 68 |         $generationId  = self::unpack(self::BIT_B32, substr($data, $offset, 4));
 69 |         $offset       += 4;
 70 |         $groupProtocol = $this->decodeString(substr($data, $offset), self::BIT_B16);
 71 |         $offset       += $groupProtocol['length'];
 72 |         $leaderId      = $this->decodeString(substr($data, $offset), self::BIT_B16);
 73 |         $offset       += $leaderId['length'];
 74 |         $memberId      = $this->decodeString(substr($data, $offset), self::BIT_B16);
 75 |         $offset       += $memberId['length'];
 76 | 
 77 |         $members = $this->decodeArray(substr($data, $offset), [$this, 'joinGroupMember']);
 78 |         $offset += $memberId['length'];
 79 | 
 80 |         return [
 81 |             'errorCode'     => $errorCode,
 82 |             'generationId'  => $generationId,
 83 |             'groupProtocol' => $groupProtocol['data'],
 84 |             'leaderId'      => $leaderId['data'],
 85 |             'memberId'      => $memberId['data'],
 86 |             'members'       => $members['data'],
 87 |         ];
 88 |     }
 89 | 
 90 |     /**
 91 |      * @param mixed[] $values
 92 |      *
 93 |      * @throws ProtocolException
 94 |      * @throws NotSupported
 95 |      */
 96 |     protected function encodeGroupProtocol(array $values): string
 97 |     {
 98 |         if (! isset($values['protocol_name'])) {
 99 |             throw new ProtocolException('given join group data invalid. `protocol_name` is undefined.');
100 |         }
101 | 
102 |         $protocolName = self::encodeString($values['protocol_name'], self::PACK_INT16);
103 | 
104 |         if (! isset($values['version'])) {
105 |             throw new ProtocolException('given data invalid. `version` is undefined.');
106 |         }
107 | 
108 |         if (! isset($values['subscription']) || empty($values['subscription'])) {
109 |             throw new ProtocolException('given data invalid. `subscription` is undefined.');
110 |         }
111 |         if (! isset($values['user_data'])) {
112 |             $values['user_data'] = '';
113 |         }
114 | 
115 |         $data  = self::pack(self::BIT_B16, '0');
116 |         $data .= self::encodeArray($values['subscription'], [$this, 'encodeGroupProtocolMetaTopic']);
117 |         $data .= self::encodeString($values['user_data'], self::PACK_INT32);
118 | 
119 |         return $protocolName . self::encodeString($data, self::PACK_INT32);
120 |     }
121 | 
122 |     /**
123 |      * @throws NotSupported
124 |      */
125 |     protected function encodeGroupProtocolMetaTopic(string $values): string
126 |     {
127 |         return self::encodeString($values, self::PACK_INT16);
128 |     }
129 | 
130 |     /**
131 |      * @return mixed[]
132 |      */
133 |     protected function joinGroupMember(string $data): array
134 |     {
135 |         $offset     = 0;
136 |         $memberId   = $this->decodeString(substr($data, $offset), self::BIT_B16);
137 |         $offset    += $memberId['length'];
138 |         $memberMeta = $this->decodeString(substr($data, $offset), self::BIT_B32);
139 |         $offset    += $memberMeta['length'];
140 | 
141 |         $metaData    = $memberMeta['data'];
142 |         $metaOffset  = 0;
143 |         $version     = self::unpack(self::BIT_B16, substr($metaData, $metaOffset, 2));
144 |         $metaOffset += 2;
145 |         $topics      = $this->decodeArray(substr($metaData, $metaOffset), [$this, 'decodeString'], self::BIT_B16);
146 |         $metaOffset += $topics['length'];
147 |         $userData    = $this->decodeString(substr($metaData, $metaOffset), self::BIT_B32);
148 | 
149 |         return [
150 |             'length' => $offset,
151 |             'data'   => [
152 |                 'memberId'   => $memberId['data'],
153 |                 'memberMeta' => [
154 |                     'version'  => $version,
155 |                     'topics'   => $topics['data'],
156 |                     'userData' => $userData['data'],
157 |                 ],
158 |             ],
159 |         ];
160 |     }
161 | }
162 | 


--------------------------------------------------------------------------------
/src/SwooleSocket.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use function is_resource;
  7 | use function sprintf;
  8 | use function strlen;
  9 | use function trim;
 10 | 
 11 | class SwooleSocket
 12 | {
 13 |     public const READ_MAX_LENGTH = 5242880; // read socket max length 5MB
 14 | 
 15 |     /**
 16 |      * max write socket buffer
 17 |      * fixed:send of 8192 bytes failed with errno=11 Resource temporarily
 18 |      * fixed:'fwrite(): send of ???? bytes failed with errno=35 Resource temporarily unavailable'
 19 |      * unavailable error info
 20 |      */
 21 |     public const MAX_WRITE_BUFFER = 2048;
 22 | 
 23 |     /**
 24 |      * @var int Connect timeout in seconds.
 25 |      */
 26 |     protected $connectTimeoutSec = 0;
 27 | 
 28 |     /**
 29 |      * @var int Connect timeout in microseconds.
 30 |      */
 31 |     protected $connectTimeoutUsec = 100000;
 32 | 
 33 |     /**
 34 |      * Send timeout in seconds.
 35 |      *
 36 |      * @var int
 37 |      */
 38 |     protected $sendTimeoutSec = 0;
 39 | 
 40 |     /**
 41 |      * Send timeout in microseconds.
 42 |      *
 43 |      * @var int
 44 |      */
 45 |     protected $sendTimeoutUsec = 100000;
 46 | 
 47 |     /**
 48 |      * Recv timeout in seconds
 49 |      *
 50 |      * @var int
 51 |      */
 52 |     protected $recvTimeoutSec = 10;
 53 | 
 54 |     /**
 55 |      * Recv timeout in microseconds
 56 |      *
 57 |      * @var int
 58 |      */
 59 |     protected $recvTimeoutUsec = 750000;
 60 | 
 61 |     /**
 62 |      * @var \Swoole\Coroutine\Socket
 63 |      */
 64 |     protected $stream;
 65 | 
 66 |     /**
 67 |      * @var string|null
 68 |      */
 69 |     protected $host;
 70 | 
 71 |     /**
 72 |      * @var int
 73 |      */
 74 |     protected $port = -1;
 75 | 
 76 |     /**
 77 |      * @var int
 78 |      */
 79 |     protected $maxWriteAttempts = 3;
 80 | 
 81 |     /**
 82 |      * @var Config|null
 83 |      */
 84 |     protected $config;
 85 | 
 86 |     /**
 87 |      * @var SaslMechanism|null
 88 |      */
 89 |     private $saslProvider;
 90 | 
 91 |     public function __construct(string $host, int $port, ?Config $config = null, ?SaslMechanism $saslProvider = null)
 92 |     {
 93 |         $this->host = $host;
 94 |         $this->port = $port;
 95 |         $this->config = $config;
 96 |         $this->saslProvider = $saslProvider;
 97 |     }
 98 | 
 99 |     public function setConnectTimeoutSec(int $sendTimeoutSec): void
100 |     {
101 |         $this->sendTimeoutSec = $sendTimeoutSec;
102 |     }
103 | 
104 |     public function setConnectTimeoutUsec(int $sendTimeoutUsec): void
105 |     {
106 |         $this->sendTimeoutUsec = $sendTimeoutUsec;
107 |     }
108 | 
109 |     public function setSendTimeoutSec(int $sendTimeoutSec): void
110 |     {
111 |         $this->sendTimeoutSec = $sendTimeoutSec;
112 |     }
113 | 
114 |     public function setSendTimeoutUsec(int $sendTimeoutUsec): void
115 |     {
116 |         $this->sendTimeoutUsec = $sendTimeoutUsec;
117 |     }
118 | 
119 |     public function setRecvTimeoutSec(int $recvTimeoutSec): void
120 |     {
121 |         $this->recvTimeoutSec = $recvTimeoutSec;
122 |     }
123 | 
124 |     public function setRecvTimeoutUsec(int $recvTimeoutUsec): void
125 |     {
126 |         $this->recvTimeoutUsec = $recvTimeoutUsec;
127 |     }
128 | 
129 |     public function setMaxWriteAttempts(int $number): void
130 |     {
131 |         $this->maxWriteAttempts = $number;
132 |     }
133 | 
134 |     public function getConnectTimeout()
135 |     {
136 |         return $this->connectTimeoutSec + ($this->connectTimeoutUsec / 1000000);
137 |     }
138 | 
139 |     public function getReadTimeout()
140 |     {
141 |         return $this->sendTimeoutSec + ($this->sendTimeoutUsec / 1000000);
142 |     }
143 | 
144 |     public function getWriteTimeout()
145 |     {
146 |         return $this->recvTimeoutSec + ($this->recvTimeoutUsec / 1000000);
147 |     }
148 | 
149 |     /**
150 |      * @throws Exception
151 |      */
152 |     protected function createStream(): void
153 |     {
154 |         if (trim($this->host) === '') {
155 |             throw new Exception('Cannot open null host.');
156 |         }
157 | 
158 |         if ($this->port <= 0) {
159 |             throw new Exception('Cannot open without port.');
160 |         }
161 | 
162 |         $socket = new \Swoole\Coroutine\Socket(AF_INET, SOCK_STREAM, IPPROTO_IP);
163 | 
164 |         if ($this->config !== null && $this->config->getSslEnable()) { // ssl connection
165 |             $socket->setProtocol([
166 |                 'open_ssl' => true,
167 |                 'ssl_cert_file' => $this->config->getSslLocalCert(),
168 |                 'ssl_key_file' => $this->config->getSslPassphrase()
169 |             ]);
170 |         }
171 |         if (!$socket->connect($this->host, $this->port, $this->getConnectTimeout())) {
172 |             $this->ioException();
173 |         } else {
174 |             $this->stream = $socket;
175 |         }
176 | 
177 |         // SASL auth
178 |         if ($this->saslProvider !== null) {
179 |             $this->saslProvider->authenticate($this);
180 |         }
181 |     }
182 | 
183 |     /**
184 |      * @return \Swoole\Coroutine\Socket
185 |      */
186 |     public function getSocket()
187 |     {
188 |         return $this->stream;
189 |     }
190 | 
191 |     /**
192 |      * Read from the socket at most $len bytes.
193 |      *
194 |      * This method will not wait for all the requested data, it will return as
195 |      * soon as any data is received.
196 |      *
197 |      * @throws Exception
198 |      */
199 |     public function read(int $length): string
200 |     {
201 |         if ($length > self::READ_MAX_LENGTH) {
202 |             throw Exception\Socket::invalidLength($length, self::READ_MAX_LENGTH);
203 |         }
204 | //        $data = $this->stream->recvAll($length, $this->getReadTimeout());
205 |         $data = $this->stream->recvAll($length);
206 |         if ($data === false) {
207 |             throw new Exception(
208 |                 sprintf('Read msg error %s:%d (%s [%d])', $this->host, $this->port, $this->stream->errMsg, $this->stream->errCode)
209 |             );
210 |         }
211 |         return $data;
212 |     }
213 | 
214 |     /**
215 |      * Write to the socket.
216 |      *
217 |      * @throws Exception
218 |      */
219 |     public function write(string $buffer): int
220 |     {
221 |         if ($buffer === null) {
222 |             throw new Exception('You must inform some data to be written');
223 |         }
224 | 
225 |         $bytesToWrite = strlen($buffer);
226 | 
227 |         $hasWriteLen = $this->stream->sendAll($buffer, $this->getWriteTimeout());
228 | 
229 |         if($hasWriteLen !== $bytesToWrite) {
230 |             $this->ioException();
231 |         }
232 | 
233 |         return $hasWriteLen;
234 |     }
235 | 
236 |     protected function ioException(?int $errno = null): void
237 |     {
238 |         $socket = $this->stream;
239 |         $errmsg = sprintf('Could not connect to %s:%d (%s [%d])', $this->host, $this->port, $this->stream->errMsg, $this->stream->errCode);
240 |         if ($errno !== null) {
241 |             $socket->errCode = $errno;
242 |             $socket->errMsg = $errmsg.swoole_strerror($errno);
243 |         }
244 |         $socket->close();
245 |         $this->stream = null;
246 |         throw new Exception($socket->errMsg, $socket->errCode);
247 |     }
248 | }
249 | 


--------------------------------------------------------------------------------
/src/Consumer/Assignment.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Consumer;
  5 | 
  6 | use Kafka\Broker;
  7 | use Kafka\SingletonTrait;
  8 | use function count;
  9 | 
 10 | class Assignment
 11 | {
 12 |     use SingletonTrait;
 13 | 
 14 |     /**
 15 |      * @var string
 16 |      */
 17 |     private $memberId = '';
 18 | 
 19 |     /**
 20 |      * @var int|null
 21 |      */
 22 |     private $generationId;
 23 | 
 24 |     /**
 25 |      * @var int[][][][]
 26 |      */
 27 |     private $assignments = [];
 28 | 
 29 |     /**
 30 |      * @var mixed[][]
 31 |      */
 32 |     private $topics = [];
 33 | 
 34 |     /**
 35 |      * @var int[][]
 36 |      */
 37 |     private $offsets = [];
 38 | 
 39 |     /**
 40 |      * @var int[][]
 41 |      */
 42 |     private $lastOffsets = [];
 43 | 
 44 |     /**
 45 |      * @var int[][]
 46 |      */
 47 |     private $fetchOffsets = [];
 48 | 
 49 |     /**
 50 |      * @var int[][]
 51 |      */
 52 |     private $consumerOffsets = [];
 53 | 
 54 |     /**
 55 |      * @var int[][]
 56 |      */
 57 |     private $commitOffsets = [];
 58 | 
 59 |     /**
 60 |      * @var int[][]
 61 |      */
 62 |     private $preCommitOffsets = [];
 63 | 
 64 |     public function setMemberId(string $memberId): void
 65 |     {
 66 |         $this->memberId = $memberId;
 67 |     }
 68 | 
 69 |     public function getMemberId(): string
 70 |     {
 71 |         return $this->memberId;
 72 |     }
 73 | 
 74 |     public function setGenerationId(int $generationId): void
 75 |     {
 76 |         $this->generationId = $generationId;
 77 |     }
 78 | 
 79 |     public function getGenerationId(): ?int
 80 |     {
 81 |         return $this->generationId;
 82 |     }
 83 | 
 84 |     /**
 85 |      * @return int[][][][]
 86 |      */
 87 |     public function getAssignments(): array
 88 |     {
 89 |         return $this->assignments;
 90 |     }
 91 | 
 92 |     /**
 93 |      * @param string[] $result
 94 |      */
 95 |     public function assign(array $result): void
 96 |     {
 97 |         /** @var Broker $broker */
 98 |         $broker = Broker::getInstance();
 99 |         $topics = $broker->getTopics();
100 | 
101 |         $memberCount = count($result);
102 | 
103 |         $count   = 0;
104 |         $members = [];
105 | 
106 |         foreach ($topics as $topicName => $partitionition) {
107 |             foreach ($partitionition as $partitionId => $leaderId) {
108 |                 $memberNum = $count % $memberCount;
109 | 
110 |                 if (! isset($members[$memberNum])) {
111 |                     $members[$memberNum] = [];
112 |                 }
113 | 
114 |                 if (! isset($members[$memberNum][$topicName])) {
115 |                     $members[$memberNum][$topicName] = [];
116 |                 }
117 | 
118 |                 $members[$memberNum][$topicName]['topic_name'] = $topicName;
119 | 
120 |                 if (! isset($members[$memberNum][$topicName]['partitions'])) {
121 |                     $members[$memberNum][$topicName]['partitions'] = [];
122 |                 }
123 | 
124 |                 $members[$memberNum][$topicName]['partitions'][] = $partitionId;
125 |                 ++$count;
126 |             }
127 |         }
128 | 
129 |         $data = [];
130 | 
131 |         foreach ($result as $key => $member) {
132 |             $data[] = [
133 |                 'version'     => 0,
134 |                 'member_id'   => $member['memberId'],
135 |                 'assignments' => $members[$key] ?? [],
136 |             ];
137 |         }
138 | 
139 |         $this->assignments = $data;
140 |     }
141 | 
142 |     /**
143 |      * @param mixed[][] $topics
144 |      */
145 |     public function setTopics(array $topics): void
146 |     {
147 |         $this->topics = $topics;
148 |     }
149 | 
150 |     /**
151 |      * @return mixed[][]
152 |      */
153 |     public function getTopics(): array
154 |     {
155 |         return $this->topics;
156 |     }
157 | 
158 |     /**
159 |      * @param int[][] $offsets
160 |      */
161 |     public function setOffsets(array $offsets): void
162 |     {
163 |         $this->offsets = $offsets;
164 |     }
165 | 
166 |     /**
167 |      * @return int[][]
168 |      */
169 |     public function getOffsets(): array
170 |     {
171 |         return $this->offsets;
172 |     }
173 | 
174 |     /**
175 |      * @param int[][] $offsets
176 |      */
177 |     public function setLastOffsets(array $offsets): void
178 |     {
179 |         $this->lastOffsets = $offsets;
180 |     }
181 | 
182 |     /**
183 |      * @return int[][]
184 |      */
185 |     public function getLastOffsets(): array
186 |     {
187 |         return $this->lastOffsets;
188 |     }
189 | 
190 |     /**
191 |      * @param int[][] $offsets
192 |      */
193 |     public function setFetchOffsets(array $offsets): void
194 |     {
195 |         $this->fetchOffsets = $offsets;
196 |     }
197 | 
198 |     /**
199 |      * @return int[][]
200 |      */
201 |     public function getFetchOffsets(): array
202 |     {
203 |         return $this->fetchOffsets;
204 |     }
205 | 
206 |     /**
207 |      * @param int[][] $offsets
208 |      */
209 |     public function setConsumerOffsets(array $offsets): void
210 |     {
211 |         $this->consumerOffsets = $offsets;
212 |     }
213 | 
214 |     /**
215 |      * @return int[][]
216 |      */
217 |     public function getConsumerOffsets(): array
218 |     {
219 |         return $this->consumerOffsets;
220 |     }
221 | 
222 |     public function setConsumerOffset(string $topic, int $partition, int $offset): void
223 |     {
224 |         $this->consumerOffsets[$topic][$partition] = $offset;
225 |     }
226 | 
227 |     public function getConsumerOffset(string $topic, int $partition): ?int
228 |     {
229 |         if (! isset($this->consumerOffsets[$topic][$partition])) {
230 |             return null;
231 |         }
232 | 
233 |         return $this->consumerOffsets[$topic][$partition];
234 |     }
235 | 
236 |     /**
237 |      * @param int[][] $offsets
238 |      */
239 |     public function setCommitOffsets(array $offsets): void
240 |     {
241 |         $this->commitOffsets = $offsets;
242 |     }
243 | 
244 |     /**
245 |      * @return int[][]
246 |      */
247 |     public function getCommitOffsets(): array
248 |     {
249 |         return $this->commitOffsets;
250 |     }
251 | 
252 |     public function setCommitOffset(string $topic, int $partition, int $offset): void
253 |     {
254 |         $this->commitOffsets[$topic][$partition] = $offset;
255 |     }
256 | 
257 |     /**
258 |      * @param int[][] $offsets
259 |      */
260 |     public function setPreCommitOffsets(array $offsets): void
261 |     {
262 |         $this->preCommitOffsets = $offsets;
263 |     }
264 | 
265 |     /**
266 |      * @return int[][]
267 |      */
268 |     public function getPreCommitOffsets(): array
269 |     {
270 |         return $this->preCommitOffsets;
271 |     }
272 | 
273 |     public function setPreCommitOffset(string $topic, int $partition, int $offset): void
274 |     {
275 |         $this->preCommitOffsets[$topic][$partition] = $offset;
276 |     }
277 | 
278 |     public function clearOffset(): void
279 |     {
280 |         $this->offsets          = [];
281 |         $this->lastOffsets      = [];
282 |         $this->fetchOffsets     = [];
283 |         $this->consumerOffsets  = [];
284 |         $this->commitOffsets    = [];
285 |         $this->preCommitOffsets = [];
286 |     }
287 | }
288 | 


--------------------------------------------------------------------------------
/src/Producer/CoroutineProcess.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Producer;
  5 | 
  6 | use Kafka\Broker;
  7 | use Kafka\Exception;
  8 | use Kafka\LoggerTrait;
  9 | use Kafka\ProducerConfig;
 10 | use Kafka\Protocol\Protocol;
 11 | use Monolog\Handler\StdoutHandler;
 12 | use Monolog\Logger;
 13 | use Psr\Log\LoggerAwareTrait;
 14 | use function array_keys;
 15 | use function count;
 16 | use function explode;
 17 | use function json_encode;
 18 | use function shuffle;
 19 | use function substr;
 20 | use function trim;
 21 | 
 22 | class CoroutineProcess
 23 | {
 24 |     use LoggerAwareTrait;
 25 |     use LoggerTrait;
 26 | 
 27 |     /** @var RecordValidator */
 28 |     private $recordValidator;
 29 | 
 30 |     public function __construct(?RecordValidator $recordValidator = null)
 31 |     {
 32 |         $logger = new Logger('coroutine_process');
 33 |         $logger->pushHandler(new StdoutHandler());
 34 |         $this->setLogger($logger);
 35 |         $this->recordValidator = $recordValidator ?? new RecordValidator();
 36 | 
 37 |         $config = $this->getConfig();
 38 |         \Kafka\Protocol::init($config->getBrokerVersion(), $this->logger);
 39 | 
 40 |         $broker = $this->getBroker();
 41 |         $broker->setConfig($config);
 42 | 
 43 |         $this->syncMeta();
 44 |     }
 45 | 
 46 |     /**
 47 |      * @param mixed[][] $recordSet
 48 |      *
 49 |      * @return mixed[]
 50 |      *
 51 |      * @throws \Kafka\Exception
 52 |      */
 53 |     public function send(array $recordSet): array
 54 |     {
 55 |         $broker = $this->getBroker();
 56 |         $config = $this->getConfig();
 57 | 
 58 |         $requiredAck = $config->getRequiredAck();
 59 |         $timeout     = $config->getTimeout();
 60 |         $compression = $config->getCompression();
 61 | 
 62 |         // get send message
 63 |         // data struct
 64 |         //  topic:
 65 |         //  partId:
 66 |         //  key:
 67 |         //  value:
 68 |         if (empty($recordSet)) {
 69 |             return [];
 70 |         }
 71 | 
 72 |         $sendData = $this->convertRecordSet($recordSet);
 73 |         $result   = [];
 74 |         foreach ($sendData as $brokerId => $topicList) {
 75 |             $connect = $broker->getDataConnect((string) $brokerId, true);
 76 | 
 77 |             if ($connect === null) {
 78 |                 return [];
 79 |             }
 80 | 
 81 |             $params = [
 82 |                 'required_ack' => $requiredAck,
 83 |                 'timeout'      => $timeout,
 84 |                 'data'         => $topicList,
 85 |                 'compression'  => $compression,
 86 |             ];
 87 | 
 88 |             $this->debug('Send message start, params:' . json_encode($params));
 89 |             $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::PRODUCE_REQUEST, $params);
 90 |             $connect->write($requestData);
 91 | 
 92 |             if ($requiredAck !== 0) { // If it is 0 the server will not send any response
 93 |                 $dataLen       = Protocol::unpack(Protocol::BIT_B32, $connect->read(4));
 94 |                 $recordSet     = $connect->read($dataLen);
 95 |                 $correlationId = Protocol::unpack(Protocol::BIT_B32, substr($recordSet, 0, 4));
 96 |                 $ret           = \Kafka\Protocol::decode(\Kafka\Protocol::PRODUCE_REQUEST, substr($recordSet, 4));
 97 | 
 98 |                 $result[] = $ret;
 99 |             }
100 |         }
101 | 
102 |         return $result;
103 |     }
104 | 
105 |     public function syncMeta(): void
106 |     {
107 |         $this->debug('Start sync metadata request');
108 | 
109 |         $brokerList = ProducerConfig::getInstance()->getMetadataBrokerList();
110 |         $brokerHost = [];
111 | 
112 |         foreach (explode(',', $brokerList) as $key => $val) {
113 |             if (trim($val)) {
114 |                 $brokerHost[] = $val;
115 |             }
116 |         }
117 | 
118 |         if (count($brokerHost) === 0) {
119 |             throw new Exception('No valid broker configured');
120 |         }
121 | 
122 |         shuffle($brokerHost);
123 |         $broker = $this->getBroker();
124 | 
125 |         foreach ($brokerHost as $host) {
126 |             $socket = $broker->getMetaConnect($host, true);
127 | 
128 |             if ($socket === null) {
129 |                 continue;
130 |             }
131 | 
132 |             $params = [];
133 |             $this->debug('Start sync metadata request params:' . json_encode($params));
134 |             $requestData = \Kafka\Protocol::encode(\Kafka\Protocol::METADATA_REQUEST, $params);
135 |             $socket->write($requestData);
136 |             $dataLen       = Protocol::unpack(Protocol::BIT_B32, $socket->read(4));
137 |             $data          = $socket->read($dataLen);
138 |             $correlationId = Protocol::unpack(Protocol::BIT_B32, substr($data, 0, 4));
139 |             $result        = \Kafka\Protocol::decode(\Kafka\Protocol::METADATA_REQUEST, substr($data, 4));
140 | 
141 |             if (! isset($result['brokers'], $result['topics'])) {
142 |                 throw new Exception('Get metadata is fail, brokers or topics is null.');
143 |             }
144 | 
145 |             $broker = $this->getBroker();
146 |             $broker->setData($result['topics'], $result['brokers']);
147 | 
148 |             return;
149 |         }
150 | 
151 |         throw Exception\ConnectionException::fromBrokerList($brokerList);
152 |     }
153 | 
154 |     /**
155 |      * @param string[][] $recordSet
156 |      *
157 |      * @return mixed[]
158 |      */
159 |     protected function convertRecordSet(array $recordSet): array
160 |     {
161 |         $sendData = [];
162 |         $broker   = $this->getBroker();
163 |         $topics   = $broker->getTopics();
164 | 
165 |         foreach ($recordSet as $record) {
166 |             $this->recordValidator->validate($record, $topics);
167 | 
168 |             $topicMeta = $topics[$record['topic']];
169 |             $partNums  = array_keys($topicMeta);
170 |             shuffle($partNums);
171 | 
172 |             $partId = isset($record['partId'], $topicMeta[$record['partId']]) ? $record['partId'] : $partNums[0];
173 | 
174 |             $brokerId  = $topicMeta[$partId];
175 |             $topicData = [];
176 |             if (isset($sendData[$brokerId][$record['topic']])) {
177 |                 $topicData = $sendData[$brokerId][$record['topic']];
178 |             }
179 | 
180 |             $partition = [];
181 |             if (isset($topicData['partitions'][$partId])) {
182 |                 $partition = $topicData['partitions'][$partId];
183 |             }
184 | 
185 |             $partition['partition_id'] = $partId;
186 | 
187 |             if (trim($record['key'] ?? '') !== '') {
188 |                 $partition['messages'][] = ['value' => $record['value'], 'key' => $record['key']];
189 |             } else {
190 |                 $partition['messages'][] = $record['value'];
191 |             }
192 | 
193 |             $topicData['partitions'][$partId]      = $partition;
194 |             $topicData['topic_name']               = $record['topic'];
195 |             $sendData[$brokerId][$record['topic']] = $topicData;
196 |             $this->debug("send data". json_encode($sendData));
197 |         }
198 | 
199 |         return $sendData;
200 |     }
201 | 
202 |     private function getBroker(): Broker
203 |     {
204 |         /** @var Broker $broker */
205 |         $broker = Broker::getInstance();
206 |         $broker->setLogger($this->logger);
207 |         return $broker;
208 |     }
209 | 
210 |     private function getConfig(): ProducerConfig
211 |     {
212 |         return ProducerConfig::getInstance();
213 |     }
214 | }
215 | 


--------------------------------------------------------------------------------
/src/Producer/State.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Producer;
  5 | 
  6 | use Amp\Loop;
  7 | use Kafka\ProducerConfig;
  8 | use Kafka\SingletonTrait;
  9 | use function array_keys;
 10 | use function is_array;
 11 | use function microtime;
 12 | 
 13 | class State
 14 | {
 15 |     use SingletonTrait;
 16 | 
 17 |     public const REQUEST_METADATA = 1;
 18 |     public const REQUEST_PRODUCE  = 2;
 19 | 
 20 |     public const STATUS_INIT    = 0;
 21 |     public const STATUS_STOP    = 1;
 22 |     public const STATUS_START   = 2;
 23 |     public const STATUS_LOOP    = 4;
 24 |     public const STATUS_PROCESS = 8;
 25 |     public const STATUS_FINISH  = 16;
 26 | 
 27 |     /**
 28 |      * @var mixed[]
 29 |      */
 30 |     private $callStatus = [];
 31 | 
 32 |     /**
 33 |      * @var mixed[][]
 34 |      */
 35 |     private $requests = [
 36 |         self::REQUEST_METADATA => [],
 37 |         self::REQUEST_PRODUCE => [],
 38 |     ];
 39 | 
 40 |     public function init(): void
 41 |     {
 42 |         $this->callStatus = [
 43 |             self::REQUEST_METADATA => ['status' => self::STATUS_LOOP],
 44 |             self::REQUEST_PRODUCE  => ['status' => self::STATUS_LOOP],
 45 |         ];
 46 | 
 47 |         $config = $this->getConfig();
 48 | 
 49 |         foreach (array_keys($this->requests) as $request) {
 50 |             if ($request === self::REQUEST_METADATA) {
 51 |                 $this->requests[$request]['interval'] = $config->getMetadataRefreshIntervalMs();
 52 |                 break;
 53 |             }
 54 | 
 55 |             $interval = $config->getIsAsyn() ? $config->getProduceInterval() : 1;
 56 | 
 57 |             $this->requests[$request]['interval'] = $interval;
 58 |         }
 59 |     }
 60 | 
 61 |     public function start(): void
 62 |     {
 63 |         foreach ($this->requests as $request => $option) {
 64 |             $interval = $option['interval'] ?? 200;
 65 | 
 66 |             Loop::repeat((int) $interval, function (string $watcherId) use ($request, $option): void {
 67 |                 if ($this->checkRun($request) && $option['func'] !== null) {
 68 |                     $this->processing($request, $option['func']());
 69 |                 }
 70 | 
 71 |                 $this->requests[$request]['watcher'] = $watcherId;
 72 |             });
 73 |         }
 74 | 
 75 |         // start sync metadata
 76 |         if (isset($request, $this->requests[self::REQUEST_METADATA]['func'])
 77 |             && $this->callStatus[self::REQUEST_METADATA]['status'] === self::STATUS_LOOP) {
 78 |             $context = $this->requests[self::REQUEST_METADATA]['func']();
 79 |             $this->processing($request, $context);
 80 |         }
 81 |     }
 82 | 
 83 |     /**
 84 |      * @param mixed|null $context
 85 |      */
 86 |     public function succRun(int $key, $context = null): void
 87 |     {
 88 |         $config = $this->getConfig();
 89 |         $isAsyn = $config->getIsAsyn();
 90 | 
 91 |         if (! isset($this->callStatus[$key])) {
 92 |             return;
 93 |         }
 94 | 
 95 |         switch ($key) {
 96 |             case self::REQUEST_METADATA:
 97 |                 $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH);
 98 | 
 99 |                 if ($context !== null) { // if kafka broker is change
100 |                     $this->recover();
101 |                 }
102 |                 break;
103 |             case self::REQUEST_PRODUCE:
104 |                 if ($context === null) {
105 |                     if (! $isAsyn) {
106 |                         $this->callStatus[$key]['status'] = self::STATUS_FINISH;
107 |                         Loop::stop();
108 |                     } else {
109 |                         $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH);
110 |                     }
111 |                     break;
112 |                 }
113 |                 unset($this->callStatus[$key]['context'][$context]);
114 |                 $contextStatus = $this->callStatus[$key]['context'];
115 | 
116 |                 if (empty($contextStatus)) {
117 |                     if (! $isAsyn) {
118 |                         $this->callStatus[$key]['status'] = self::STATUS_FINISH;
119 |                         Loop::stop();
120 |                     } else {
121 |                         $this->callStatus[$key]['status'] = (self::STATUS_LOOP | self::STATUS_FINISH);
122 |                     }
123 |                 }
124 |                 break;
125 |         }
126 |     }
127 | 
128 |     public function failRun(int $key): void
129 |     {
130 |         if (! isset($this->callStatus[$key])) {
131 |             return;
132 |         }
133 | 
134 |         switch ($key) {
135 |             case self::REQUEST_METADATA:
136 |                 $this->callStatus[$key]['status'] = self::STATUS_LOOP;
137 |                 break;
138 |             case self::REQUEST_PRODUCE:
139 |                 $this->recover();
140 |                 break;
141 |         }
142 |     }
143 | 
144 |     /**
145 |      * @param callable[] $callbacks
146 |      */
147 |     public function setCallback(array $callbacks): void
148 |     {
149 |         foreach ($callbacks as $request => $callback) {
150 |             $this->requests[$request]['func'] = $callback;
151 |         }
152 |     }
153 | 
154 |     public function recover(): void
155 |     {
156 |         $this->callStatus = [
157 |             self::REQUEST_METADATA => $this->callStatus[self::REQUEST_METADATA],
158 |             self::REQUEST_PRODUCE => [
159 |                 'status'=> self::STATUS_LOOP,
160 |             ],
161 |         ];
162 |     }
163 | 
164 |     protected function checkRun(int $key): bool
165 |     {
166 |         if (! isset($this->callStatus[$key])) {
167 |             return false;
168 |         }
169 | 
170 |         $status = $this->callStatus[$key]['status'];
171 | 
172 |         switch ($key) {
173 |             case self::REQUEST_METADATA:
174 |                 if (($status & self::STATUS_PROCESS) === self::STATUS_PROCESS) {
175 |                     return false;
176 |                 }
177 | 
178 |                 if (($status & self::STATUS_LOOP) === self::STATUS_LOOP) {
179 |                     return true;
180 |                 }
181 | 
182 |                 return false;
183 |             case self::REQUEST_PRODUCE:
184 |                 if (($status & self::STATUS_PROCESS) === self::STATUS_PROCESS) {
185 |                     return false;
186 |                 }
187 | 
188 |                 $syncStatus = $this->callStatus[self::REQUEST_METADATA]['status'];
189 | 
190 |                 if (($syncStatus & self::STATUS_FINISH) !== self::STATUS_FINISH) {
191 |                     return false;
192 |                 }
193 | 
194 |                 if (($status & self::STATUS_LOOP) === self::STATUS_LOOP) {
195 |                     return true;
196 |                 }
197 | 
198 |                 return false;
199 |         }
200 | 
201 |         return false;
202 |     }
203 | 
204 |     /**
205 |      * @param mixed $context
206 |      */
207 |     protected function processing(int $key, $context): void
208 |     {
209 |         if (! isset($this->callStatus[$key])) {
210 |             return;
211 |         }
212 | 
213 |         // set process start time
214 |         $this->callStatus[$key]['time'] = microtime(true);
215 |         switch ($key) {
216 |             case self::REQUEST_METADATA:
217 |                 $this->callStatus[$key]['status'] |= self::STATUS_PROCESS;
218 |                 break;
219 |             case self::REQUEST_PRODUCE:
220 |                 if (empty($context)) {
221 |                     break;
222 |                 }
223 | 
224 |                 $this->callStatus[$key]['status'] |= self::STATUS_PROCESS;
225 | 
226 |                 $contextStatus = [];
227 | 
228 |                 if (is_array($context)) {
229 |                     foreach ($context as $fd) {
230 |                         $contextStatus[$fd] = self::STATUS_PROCESS;
231 |                     }
232 | 
233 |                     $this->callStatus[$key]['context'] = $contextStatus;
234 |                 }
235 | 
236 |                 break;
237 |         }
238 |     }
239 | 
240 |     private function getConfig(): ProducerConfig
241 |     {
242 |         return ProducerConfig::getInstance();
243 |     }
244 | }
245 | 


--------------------------------------------------------------------------------
/src/Broker.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use Kafka\Sasl\Gssapi;
  7 | use Kafka\Sasl\Plain;
  8 | use Kafka\Sasl\Scram;
  9 | use function array_keys;
 10 | use function explode;
 11 | use function in_array;
 12 | use function serialize;
 13 | use function shuffle;
 14 | use function sprintf;
 15 | use function strpos;
 16 | 
 17 | class Broker
 18 | {
 19 |     use SingletonTrait;
 20 | 
 21 |     /**
 22 |      * @var int
 23 |      */
 24 |     private $groupBrokerId;
 25 | 
 26 |     /**
 27 |      * @var mixed[][]
 28 |      */
 29 |     private $topics = [];
 30 | 
 31 |     /**
 32 |      * @var string[]
 33 |      */
 34 |     private $brokers = [];
 35 | 
 36 |     /**
 37 |      * @var CommonSocket[]
 38 |      */
 39 |     private $metaSockets = [];
 40 | 
 41 |     /**
 42 |      * @var CommonSocket[]
 43 |      */
 44 |     private $dataSockets = [];
 45 | 
 46 |     /**
 47 |      * @var callable|null
 48 |      */
 49 |     private $process;
 50 | 
 51 |     /**
 52 |      * @var Config|null
 53 |      */
 54 |     private $config;
 55 | 
 56 |     public function setProcess(callable $process): void
 57 |     {
 58 |         $this->process = $process;
 59 |     }
 60 | 
 61 |     public function setConfig(Config $config): void
 62 |     {
 63 |         $this->config = $config;
 64 |     }
 65 | 
 66 |     public function setGroupBrokerId(int $brokerId): void
 67 |     {
 68 |         $this->groupBrokerId = $brokerId;
 69 |     }
 70 | 
 71 |     public function getGroupBrokerId(): int
 72 |     {
 73 |         return $this->groupBrokerId;
 74 |     }
 75 | 
 76 |     /**
 77 |      * @param mixed[][] $topics
 78 |      * @param mixed[]   $brokersResult
 79 |      */
 80 |     public function setData(array $topics, array $brokersResult): bool
 81 |     {
 82 |         $brokers = [];
 83 | 
 84 |         foreach ($brokersResult as $value) {
 85 |             $brokers[$value['nodeId']] = $value['host'] . ':' . $value['port'];
 86 |         }
 87 | 
 88 |         $changed = false;
 89 | 
 90 |         if (serialize($this->brokers) !== serialize($brokers)) {
 91 |             $this->brokers = $brokers;
 92 | 
 93 |             $changed = true;
 94 |         }
 95 | 
 96 |         $newTopics = [];
 97 |         foreach ($topics as $topic) {
 98 |             if ((int) $topic['errorCode'] !== Protocol::NO_ERROR) {
 99 |                 $this->error('Parse metadata for topic is error, error:' . Protocol::getError($topic['errorCode']));
100 |                 continue;
101 |             }
102 | 
103 |             $item = [];
104 | 
105 |             foreach ($topic['partitions'] as $part) {
106 |                 $item[$part['partitionId']] = $part['leader'];
107 |             }
108 | 
109 |             $newTopics[$topic['topicName']] = $item;
110 |         }
111 | 
112 |         if (serialize($this->topics) !== serialize($newTopics)) {
113 |             $this->topics = $newTopics;
114 | 
115 |             $changed = true;
116 |         }
117 | 
118 |         return $changed;
119 |     }
120 | 
121 |     /**
122 |      * @return mixed[][]
123 |      */
124 |     public function getTopics(): array
125 |     {
126 |         return $this->topics;
127 |     }
128 | 
129 |     /**
130 |      * @return string[]
131 |      */
132 |     public function getBrokers(): array
133 |     {
134 |         return $this->brokers;
135 |     }
136 | 
137 |     public function getMetaConnect(string $key, bool $modeSync = false)
138 |     {
139 |         return $this->getConnect($key, 'metaSockets', $modeSync);
140 |     }
141 | 
142 |     public function getRandConnect(bool $modeSync = false)
143 |     {
144 |         $nodeIds = array_keys($this->brokers);
145 |         shuffle($nodeIds);
146 | 
147 |         if (! isset($nodeIds[0])) {
148 |             return null;
149 |         }
150 | 
151 |         return $this->getMetaConnect((string) $nodeIds[0], $modeSync);
152 |     }
153 | 
154 |     public function getDataConnect(string $key, bool $modeSync = false)
155 |     {
156 |         return $this->getConnect($key, 'dataSockets', $modeSync);
157 |     }
158 | 
159 |     public function getConnect(string $key, string $type, bool $modeSync = false): SocketCoroutine
160 |     {
161 |         if (isset($this->{$type}[$key])) {
162 |             return $this->{$type}[$key];
163 |         }
164 | 
165 |         if (isset($this->brokers[$key])) {
166 |             $hostname = $this->brokers[$key];
167 |             if (isset($this->{$type}[$hostname])) {
168 |                 return $this->{$type}[$hostname];
169 |             }
170 |         }
171 | 
172 |         $host = null;
173 |         $port = null;
174 | 
175 |         if (isset($this->brokers[$key])) {
176 |             $hostname      = $this->brokers[$key];
177 | 
178 |             [$host, $port] = explode(':', $hostname);
179 |         }
180 | 
181 |         if (strpos($key, ':') !== false) {
182 |             [$host, $port] = explode(':', $key);
183 |         }
184 | 
185 |         if ($host === null || $port === null || (! $modeSync && $this->process === null)) {
186 |             return null;
187 |         }
188 | 
189 |         try {
190 |             /**
191 |              * @var SocketCoroutine $socket
192 |              */
193 |             $socket = $this->getSocket((string) $host, (int) $port, $modeSync);
194 |             $socket->connect();
195 |             $this->{$type}[$key] = $socket;
196 |             return $socket;
197 |         } catch (\Throwable $e) {
198 |             $this->debug($e->getMessage());
199 |             $this->error($e->getMessage());
200 |             return null;
201 |         }
202 |     }
203 | 
204 |     public function clear(): void
205 |     {
206 |         foreach ($this->metaSockets as $key => $socket) {
207 |             $socket->close();
208 |         }
209 |         foreach ($this->dataSockets as $key => $socket) {
210 |             $socket->close();
211 |         }
212 |         $this->brokers = [];
213 |     }
214 | 
215 |     /**
216 |      * @throws \Kafka\Exception
217 |      */
218 |     public function getSocket(string $host, int $port, bool $modeSync): SwooleSocket
219 |     {
220 |         $saslProvider = $this->judgeConnectionConfig();
221 | 
222 |         if ($modeSync) {
223 | //            return new SocketSync($host, $port, $this->config, $saslProvider);
224 |             return new SocketCoroutine($host, $port, $this->config, $saslProvider);
225 |         }
226 | 
227 |         return new Socket($host, $port, $this->config, $saslProvider);
228 |     }
229 | 
230 | 
231 |     /**
232 |      * @throws \Kafka\Exception
233 |      */
234 |     private function judgeConnectionConfig(): ?SaslMechanism
235 |     {
236 |         if ($this->config === null) {
237 |             return null;
238 |         }
239 | 
240 |         $plainConnections = [
241 |             Config::SECURITY_PROTOCOL_PLAINTEXT,
242 |             Config::SECURITY_PROTOCOL_SASL_PLAINTEXT,
243 |         ];
244 | 
245 |         $saslConnections = [
246 |             Config::SECURITY_PROTOCOL_SASL_SSL,
247 |             Config::SECURITY_PROTOCOL_SASL_PLAINTEXT,
248 |         ];
249 | 
250 |         $securityProtocol = $this->config->getSecurityProtocol();
251 | 
252 |         $this->config->setSslEnable(! in_array($securityProtocol, $plainConnections, true));
253 | 
254 |         if (in_array($securityProtocol, $saslConnections, true)) {
255 |             return $this->getSaslMechanismProvider($this->config);
256 |         }
257 | 
258 |         return null;
259 |     }
260 | 
261 |     /**
262 |      * @throws \Kafka\Exception
263 |      */
264 |     private function getSaslMechanismProvider(Config $config): SaslMechanism
265 |     {
266 |         $mechanism = $config->getSaslMechanism();
267 |         $username  = $config->getSaslUsername();
268 |         $password  = $config->getSaslPassword();
269 | 
270 |         switch ($mechanism) {
271 |             case Config::SASL_MECHANISMS_PLAIN:
272 |                 return new Plain($username, $password);
273 |             case Config::SASL_MECHANISMS_GSSAPI:
274 |                 return Gssapi::fromKeytab($config->getSaslKeytab(), $config->getSaslPrincipal());
275 |             case Config::SASL_MECHANISMS_SCRAM_SHA_256:
276 |                 return new Scram($username, $password, Scram::SCRAM_SHA_256);
277 |             case Config::SASL_MECHANISMS_SCRAM_SHA_512:
278 |                 return new Scram($username, $password, Scram::SCRAM_SHA_512);
279 |         }
280 | 
281 |         throw new Exception(sprintf('"%s" is an invalid SASL mechanism', $mechanism));
282 |     }
283 | }
284 | 


--------------------------------------------------------------------------------
/src/kafka.dio:
--------------------------------------------------------------------------------
 1 | <mxfile host="" modified="2020-06-05T10:16:25.781Z" agent="5.0 (Macintosh; Intel Mac OS X 10_14_6) AppleWebKit/537.36 (KHTML, like Gecko) Code/1.45.1 Chrome/78.0.3904.130 Electron/7.2.4 Safari/537.36" etag="iv0BXMJ8P-AfokFyi6NV" version="13.1.3">
 2 |     <diagram id="6hGFLwfOUW9BJ-s0fimq" name="Page-1">
 3 |         <mxGraphModel dx="1075" dy="2029" grid="1" gridSize="10" guides="1" tooltips="1" connect="1" arrows="1" fold="1" page="1" pageScale="1" pageWidth="827" pageHeight="1169" math="0" shadow="0">
 4 |             <root>
 5 |                 <mxCell id="0"/>
 6 |                 <mxCell id="1" parent="0"/>
 7 |                 <mxCell id="6" value="Producer" style="swimlane;fontStyle=1;align=center;verticalAlign=top;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" parent="1" vertex="1">
 8 |                     <mxGeometry x="70" y="350" width="160" height="86" as="geometry"/>
 9 |                 </mxCell>
10 |                 <mxCell id="7" value="+ process: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="6" vertex="1">
11 |                     <mxGeometry y="26" width="160" height="26" as="geometry"/>
12 |                 </mxCell>
13 |                 <mxCell id="8" value="" style="line;strokeWidth=1;fillColor=none;align=left;verticalAlign=middle;spacingTop=-1;spacingLeft=3;spacingRight=3;rotatable=0;labelPosition=right;points=[];portConstraint=eastwest;" parent="6" vertex="1">
14 |                     <mxGeometry y="52" width="160" height="8" as="geometry"/>
15 |                 </mxCell>
16 |                 <mxCell id="9" value="+ method(type): type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="6" vertex="1">
17 |                     <mxGeometry y="60" width="160" height="26" as="geometry"/>
18 |                 </mxCell>
19 |                 <mxCell id="23" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=0.006;entryY=0.846;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="2" target="14" edge="1">
20 |                     <mxGeometry relative="1" as="geometry"/>
21 |                 </mxCell>
22 |                 <mxCell id="2" value="ProducerConfig" style="swimlane;fontStyle=1;align=center;verticalAlign=top;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" parent="1" vertex="1">
23 |                     <mxGeometry x="380" y="120" width="160" height="112" as="geometry"/>
24 |                 </mxCell>
25 |                 <mxCell id="3" value="+ defaults: array" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="2" vertex="1">
26 |                     <mxGeometry y="26" width="160" height="26" as="geometry"/>
27 |                 </mxCell>
28 |                 <mxCell id="4" value="" style="line;strokeWidth=1;fillColor=none;align=left;verticalAlign=middle;spacingTop=-1;spacingLeft=3;spacingRight=3;rotatable=0;labelPosition=right;points=[];portConstraint=eastwest;" parent="2" vertex="1">
29 |                     <mxGeometry y="52" width="160" height="8" as="geometry"/>
30 |                 </mxCell>
31 |                 <mxCell id="5" value="+ set: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="2" vertex="1">
32 |                     <mxGeometry y="60" width="160" height="26" as="geometry"/>
33 |                 </mxCell>
34 |                 <mxCell id="11" value="+ get: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="2" vertex="1">
35 |                     <mxGeometry y="86" width="160" height="26" as="geometry"/>
36 |                 </mxCell>
37 |                 <mxCell id="24" style="edgeStyle=orthogonalEdgeStyle;rounded=0;orthogonalLoop=1;jettySize=auto;html=1;exitX=0.5;exitY=0;exitDx=0;exitDy=0;entryX=1;entryY=0.923;entryDx=0;entryDy=0;entryPerimeter=0;" parent="1" source="18" target="14" edge="1">
38 |                     <mxGeometry relative="1" as="geometry"/>
39 |                 </mxCell>
40 |                 <mxCell id="18" value="ConsumerConfig" style="swimlane;fontStyle=1;align=center;verticalAlign=top;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;" parent="1" vertex="1">
41 |                     <mxGeometry x="790" y="130" width="160" height="112" as="geometry"/>
42 |                 </mxCell>
43 |                 <mxCell id="19" value="+ defaults: array" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="18" vertex="1">
44 |                     <mxGeometry y="26" width="160" height="26" as="geometry"/>
45 |                 </mxCell>
46 |                 <mxCell id="20" value="" style="line;strokeWidth=1;fillColor=none;align=left;verticalAlign=middle;spacingTop=-1;spacingLeft=3;spacingRight=3;rotatable=0;labelPosition=right;points=[];portConstraint=eastwest;" parent="18" vertex="1">
47 |                     <mxGeometry y="52" width="160" height="8" as="geometry"/>
48 |                 </mxCell>
49 |                 <mxCell id="21" value="+ set: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="18" vertex="1">
50 |                     <mxGeometry y="60" width="160" height="26" as="geometry"/>
51 |                 </mxCell>
52 |                 <mxCell id="22" value="+ get: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="18" vertex="1">
53 |                     <mxGeometry y="86" width="160" height="26" as="geometry"/>
54 |                 </mxCell>
55 |                 <mxCell id="12" value="Config" style="swimlane;fontStyle=1;align=center;verticalAlign=top;childLayout=stackLayout;horizontal=1;startSize=26;horizontalStack=0;resizeParent=1;resizeParentMax=0;resizeLast=0;collapsible=1;marginBottom=0;fillColor=#008a00;strokeColor=#005700;fontColor=#ffffff;" parent="1" vertex="1">
56 |                     <mxGeometry x="590" y="-70" width="160" height="138" as="geometry"/>
57 |                 </mxCell>
58 |                 <mxCell id="13" value="+ defaults: array" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="12" vertex="1">
59 |                     <mxGeometry y="26" width="160" height="26" as="geometry"/>
60 |                 </mxCell>
61 |                 <mxCell id="14" value="+ options: array" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="12" vertex="1">
62 |                     <mxGeometry y="52" width="160" height="26" as="geometry"/>
63 |                 </mxCell>
64 |                 <mxCell id="15" value="" style="line;strokeWidth=1;fillColor=none;align=left;verticalAlign=middle;spacingTop=-1;spacingLeft=3;spacingRight=3;rotatable=0;labelPosition=right;points=[];portConstraint=eastwest;" parent="12" vertex="1">
65 |                     <mxGeometry y="78" width="160" height="8" as="geometry"/>
66 |                 </mxCell>
67 |                 <mxCell id="16" value="+ set: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="12" vertex="1">
68 |                     <mxGeometry y="86" width="160" height="26" as="geometry"/>
69 |                 </mxCell>
70 |                 <mxCell id="17" value="+ get: type" style="text;strokeColor=none;fillColor=none;align=left;verticalAlign=top;spacingLeft=4;spacingRight=4;overflow=hidden;rotatable=0;points=[[0,0.5],[1,0.5]];portConstraint=eastwest;" parent="12" vertex="1">
71 |                     <mxGeometry y="112" width="160" height="26" as="geometry"/>
72 |                 </mxCell>
73 |             </root>
74 |         </mxGraphModel>
75 |     </diagram>
76 | </mxfile>


--------------------------------------------------------------------------------
/src/Sasl/Scram.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Sasl;
  5 | 
  6 | use Kafka\CommonSocket;
  7 | use Kafka\Exception;
  8 | use Kafka\Protocol\Protocol as ProtocolTool;
  9 | use function base64_decode;
 10 | use function base64_encode;
 11 | use function chr;
 12 | use function count;
 13 | use function explode;
 14 | use function hash;
 15 | use function hash_equals;
 16 | use function hash_hmac;
 17 | use function preg_match;
 18 | use function random_int;
 19 | use function sprintf;
 20 | use function str_replace;
 21 | use function strlen;
 22 | use function substr;
 23 | use function trim;
 24 | 
 25 | class Scram extends Mechanism
 26 | {
 27 |     public const SCRAM_SHA_256 = 256;
 28 |     public const SCRAM_SHA_512 = 512;
 29 | 
 30 |     private const MECHANISM_NAME = 'SCRAM-SHA-';
 31 | 
 32 |     private const ALLOW_SHA_ALGORITHM = [
 33 |         self::SCRAM_SHA_256 => 'sha256',
 34 |         self::SCRAM_SHA_512 => 'sha512',
 35 |     ];
 36 | 
 37 |     /**
 38 |      * @var int
 39 |      */
 40 |     private $hashAlgorithm;
 41 | 
 42 |     /**
 43 |      * @var string
 44 |      */
 45 |     private $username;
 46 | 
 47 |     /**
 48 |      * @var string
 49 |      */
 50 |     private $password;
 51 | 
 52 |     /**
 53 |      * @var string
 54 |      */
 55 |     private $cnonce;
 56 | 
 57 |     /**
 58 |      * @var string
 59 |      */
 60 |     private $firstMessageBare;
 61 | 
 62 |     /**
 63 |      * @var string
 64 |      */
 65 |     private $saltedPassword;
 66 | 
 67 |     /**
 68 |      * @var string
 69 |      */
 70 |     private $authMessage;
 71 | 
 72 |     /**
 73 |      * @throws \Kafka\Exception
 74 |      */
 75 |     public function __construct(string $username, string $password, int $algorithm)
 76 |     {
 77 |         if (! isset(self::ALLOW_SHA_ALGORITHM[$algorithm])) {
 78 |             throw new Exception('Invalid hash algorithm given, it must be one of: [SCRAM_SHA_256, SCRAM_SHA_512].');
 79 |         }
 80 | 
 81 |         $this->hashAlgorithm = $algorithm;
 82 |         $this->username      = $this->formatName(trim($username));
 83 |         $this->password      = trim($password);
 84 |     }
 85 | 
 86 |     /**
 87 |      * @throws \Kafka\Exception
 88 |      * @throws \Exception
 89 |      */
 90 |     protected function performAuthentication(CommonSocket $socket): void
 91 |     {
 92 |         $firstMessage = $this->firstMessage();
 93 |         $data         = ProtocolTool::encodeString($firstMessage, ProtocolTool::PACK_INT32);
 94 |         $socket->writeBlocking($data);
 95 |         $dataLen           = ProtocolTool::unpack(ProtocolTool::BIT_B32, $socket->readBlocking(4));
 96 |         $serverFistMessage = $socket->readBlocking($dataLen);
 97 | 
 98 |         $finalMessage = $this->finalMessage($serverFistMessage);
 99 |         $data         = ProtocolTool::encodeString($finalMessage, ProtocolTool::PACK_INT32);
100 |         $socket->writeBlocking($data);
101 |         $dataLen       = ProtocolTool::unpack(ProtocolTool::BIT_B32, $socket->readBlocking(4));
102 |         $verifyMessage = $socket->readBlocking($dataLen);
103 | 
104 |         if (! $this->verifyMessage($verifyMessage)) {
105 |             throw new Exception('Verify server final response message is failure');
106 |         }
107 |     }
108 | 
109 | 
110 |     public function getName(): string
111 |     {
112 |         return self::MECHANISM_NAME . $this->hashAlgorithm;
113 |     }
114 | 
115 |     /**
116 |      * @throws \Exception
117 |      */
118 |     protected function firstMessage(): string
119 |     {
120 |         $this->cnonce = $this->generateNonce();
121 |         $message      = sprintf('n,,n=%s,r=%s', $this->username, $this->cnonce);
122 | 
123 |         $this->firstMessageBare = substr($message, 3);
124 | 
125 |         return $message;
126 |     }
127 | 
128 |     /**
129 |      * @throws \Kafka\Exception
130 |      */
131 |     protected function finalMessage(string $challenge): string
132 |     {
133 |         $challengeArray = explode(',', $challenge);
134 | 
135 |         if (count($challengeArray) < 3) {
136 |             throw new Exception('Server response challenge is invalid.');
137 |         }
138 | 
139 |         $nonce = substr($challengeArray[0], 2);
140 |         $salt  = base64_decode(substr($challengeArray[1], 2));
141 | 
142 |         if (! $salt) {
143 |             throw new Exception('Server response challenge is invalid, paser salt is failure.');
144 |         }
145 | 
146 |         $i      = (int) substr($challengeArray[2], 2);
147 |         $cnonce = substr($nonce, 0, strlen($this->cnonce));
148 | 
149 |         if ($cnonce !== $this->cnonce) {
150 |             throw new Exception('Server response challenge is invalid, cnonce is invalid.');
151 |         }
152 | 
153 |         $finalMessage = 'c=biws,r=' . $nonce; // `biws` is base64 encode of "n,,"
154 | 
155 |         /* Constructing the ClientProof attribute (p):
156 |          *
157 |          * p = Base64-encoded ClientProof
158 |          * SaltedPassword  := Hi(Normalize(password), salt, i)
159 |          * ClientKey       := HMAC(SaltedPassword, "Client Key")
160 |          * StoredKey       := H(ClientKey)
161 |          * AuthMessage     := client-first-message-bare + "," +
162 |          *                    server-first-message + "," +
163 |          *                    client-final-message-without-proof
164 |          * ClientSignature := HMAC(StoredKey, AuthMessage)
165 |          * ClientProof     := ClientKey XOR ClientSignature
166 |          * ServerKey       := HMAC(SaltedPassword, "Server Key")
167 |          * ServerSignature := HMAC(ServerKey, AuthMessage)
168 |          */
169 | 
170 |         $saltedPassword       = $this->hi($this->password, $salt, $i);
171 |         $this->saltedPassword = $saltedPassword;
172 |         $clientKey            = $this->hmac($saltedPassword, 'Client Key', true);
173 |         $storedKey            = $this->hash($clientKey);
174 |         $authMessage          = $this->firstMessageBare . ',' . $challenge . ',' . $finalMessage;
175 |         $this->authMessage    = $authMessage;
176 | 
177 |         $clientSignature = $this->hmac($storedKey, $authMessage, true);
178 |         $clientProof     = $clientKey ^ $clientSignature;
179 |         $proof           = ',p=' . base64_encode($clientProof);
180 | 
181 |         return $finalMessage . $proof;
182 |     }
183 | 
184 |     /**
185 |      * SCRAM has also a server verification step
186 |      *
187 |      * @param string $data The additional data sent along a successful outcome.
188 |      *
189 |      * @return bool Whether the server has been authenticated.
190 |      */
191 |     protected function verifyMessage(string $data): bool
192 |     {
193 |         $verifierRegexp = '#^v=((?:[A-Za-z0-9/+]{4})*(?:[A-Za-z0-9]{3}=|[A-Xa-z0-9]{2}==)?)$#';
194 | 
195 |         if ($this->saltedPassword === null || $this->authMessage === null) {
196 |             return false;
197 |         }
198 | 
199 |         if (! preg_match($verifierRegexp, $data, $matches)) {
200 |             return false;
201 |         }
202 | 
203 |         $proposedServerSignature = base64_decode($matches[1]);
204 |         $serverKey               = $this->hmac($this->saltedPassword, 'Server Key', true);
205 |         $serverSignature         = $this->hmac($serverKey, $this->authMessage, true);
206 |         return hash_equals($proposedServerSignature, $serverSignature);
207 |     }
208 | 
209 |     /**
210 |      * @throws \Exception
211 |      */
212 |     protected function generateNonce(): string
213 |     {
214 |         $str = '';
215 | 
216 |         for ($i = 0; $i < 32; $i++) {
217 |             $str .= chr(random_int(0, 255));
218 |         }
219 | 
220 |         return base64_encode($str);
221 |     }
222 | 
223 |     private function hash(string $data): string
224 |     {
225 |         return hash(self::ALLOW_SHA_ALGORITHM[$this->hashAlgorithm], $data, true);
226 |     }
227 | 
228 |     private function hmac(string $key, string $data, bool $raw): string
229 |     {
230 |         return hash_hmac(self::ALLOW_SHA_ALGORITHM[$this->hashAlgorithm], $data, $key, $raw);
231 |     }
232 | 
233 |     /**
234 |      * Prepare a name for inclusion in a SCRAM response.
235 |      * @See RFC-4013.
236 |      *
237 |      * @param string $user a name to be prepared.
238 |      * @return string the reformatted name.
239 |      */
240 |     private function formatName(string $user): string
241 |     {
242 |         return str_replace(['=', ','], ['=3D', '=2C'], $user);
243 |     }
244 | 
245 |     /**
246 |      * Hi() call, which is essentially PBKDF2 (RFC-2898) with HMAC-H() as the pseudorandom function.
247 |      */
248 |     private function hi(string $str, string $salt, int $icnt): string
249 |     {
250 |         $int1   = "\0\0\0\1";
251 |         $ui     = $this->hmac($str, $salt . $int1, true);
252 |         $result = $ui;
253 | 
254 |         for ($k = 1; $k < $icnt; $k++) {
255 |             $ui     = $this->hmac($str, $ui, true);
256 |             $result = $result ^ $ui;
257 |         }
258 | 
259 |         return $result;
260 |     }
261 | }
262 | 


--------------------------------------------------------------------------------
/src/Protocol/Produce.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception\NotSupported;
  7 | use Kafka\Exception\Protocol as ProtocolException;
  8 | use Lcobucci\Clock\Clock;
  9 | use Lcobucci\Clock\SystemClock;
 10 | use function crc32;
 11 | use function is_array;
 12 | use function substr;
 13 | 
 14 | class Produce extends Protocol
 15 | {
 16 |     /**
 17 |      * Specifies the mask for the compression code. 3 bits to hold the compression codec.
 18 |      * 0 is reserved to indicate no compression
 19 |      */
 20 |     public const COMPRESSION_CODEC_MASK = 0x07;
 21 | 
 22 |     /**
 23 |      * Specify the mask of timestamp type: 0 for CreateTime, 1 for LogAppendTime.
 24 |      */
 25 |     private const TIMESTAMP_TYPE_MASK = 0x08;
 26 | 
 27 |     private const TIMESTAMP_NONE            = -1;
 28 |     private const TIMESTAMP_CREATE_TIME     = 0;
 29 |     private const TIMESTAMP_LOG_APPEND_TIME = 1;
 30 | 
 31 |     /**
 32 |      * @var Clock
 33 |      */
 34 |     private $clock;
 35 | 
 36 |     public function __construct(string $version = self::DEFAULT_BROKER_VERION, ?Clock $clock = null)
 37 |     {
 38 |         parent::__construct($version);
 39 | 
 40 |         $this->clock = $clock ?: new SystemClock();
 41 |     }
 42 | 
 43 |     /**
 44 |      * @param mixed[] $payloads
 45 |      *
 46 |      * @throws NotSupported
 47 |      * @throws ProtocolException
 48 |      */
 49 |     public function encode(array $payloads = []): string
 50 |     {
 51 |         if (! isset($payloads['data'])) {
 52 |             throw new ProtocolException('given procude data invalid. `data` is undefined.');
 53 |         }
 54 | 
 55 |         $header = $this->requestHeader('kafka-php', 0, self::PRODUCE_REQUEST);
 56 |         $data   = self::pack(self::BIT_B16, (string) ($payloads['required_ack'] ?? 0));
 57 |         $data  .= self::pack(self::BIT_B32, (string) ($payloads['timeout'] ?? 100));
 58 |         $data  .= self::encodeArray(
 59 |             $payloads['data'],
 60 |             [$this, 'encodeProduceTopic'],
 61 |             $payloads['compression'] ?? self::COMPRESSION_NONE
 62 |         );
 63 | 
 64 |         return self::encodeString($header . $data, self::PACK_INT32);
 65 |     }
 66 | 
 67 |     /**
 68 |      * @return mixed[]
 69 |      *
 70 |      * @throws ProtocolException
 71 |      */
 72 |     public function decode(string $data): array
 73 |     {
 74 |         $offset       = 0;
 75 |         $version      = $this->getApiVersion(self::PRODUCE_REQUEST);
 76 |         $ret          = $this->decodeArray(substr($data, $offset), [$this, 'produceTopicPair'], $version);
 77 |         $offset      += $ret['length'];
 78 |         $throttleTime = 0;
 79 | 
 80 |         if ($version === self::API_VERSION2) {
 81 |             $throttleTime = self::unpack(self::BIT_B32, substr($data, $offset, 4));
 82 |         }
 83 | 
 84 |         return ['throttleTime' => $throttleTime, 'data' => $ret['data']];
 85 |     }
 86 | 
 87 |     /**
 88 |      * encode message set
 89 |      * N.B., MessageSets are not preceded by an int32 like other array elements
 90 |      * in the protocol.
 91 |      *
 92 |      * @param string[]|string[][] $messages
 93 |      *
 94 |      * @throws NotSupported
 95 |      */
 96 |     protected function encodeMessageSet(array $messages, int $compression = self::COMPRESSION_NONE): string
 97 |     {
 98 |         $data = '';
 99 |         $next = 0;
100 | 
101 |         foreach ($messages as $message) {
102 |             $encodedMessage = $this->encodeMessage($message);
103 | 
104 |             $data .= self::pack(self::BIT_B64, (string) $next)
105 |                    . self::encodeString($encodedMessage, self::PACK_INT32);
106 | 
107 |             ++$next;
108 |         }
109 | 
110 |         if ($compression === self::COMPRESSION_NONE) {
111 |             return $data;
112 |         }
113 | 
114 |         return self::pack(self::BIT_B64, '0')
115 |              . self::encodeString($this->encodeMessage($data, $compression), self::PACK_INT32);
116 |     }
117 | 
118 |     /**
119 |      * @param string[]|string $message
120 |      *
121 |      * @throws NotSupported
122 |      */
123 |     protected function encodeMessage($message, int $compression = self::COMPRESSION_NONE): string
124 |     {
125 |         $magic      = $this->computeMagicByte();
126 |         $attributes = $this->computeAttributes($magic, $compression, $this->computeTimestampType($magic));
127 | 
128 |         $data  = self::pack(self::BIT_B8, (string) $magic);
129 |         $data .= self::pack(self::BIT_B8, (string) $attributes);
130 | 
131 |         if ($magic >= self::MESSAGE_MAGIC_VERSION1) {
132 |             $data .= self::pack(self::BIT_B64, $this->clock->now()->format('Uv'));
133 |         }
134 | 
135 |         $key = '';
136 | 
137 |         if (is_array($message)) {
138 |             $key     = $message['key'];
139 |             $message = $message['value'];
140 |         }
141 | 
142 |         // message key
143 |         $data .= self::encodeString($key, self::PACK_INT32);
144 | 
145 |         // message value
146 |         $data .= self::encodeString($message, self::PACK_INT32, $compression);
147 | 
148 |         $crc = (string) crc32($data);
149 | 
150 |         // int32 -- crc code  string data
151 |         $message = self::pack(self::BIT_B32, $crc) . $data;
152 | 
153 |         return $message;
154 |     }
155 | 
156 |     private function computeMagicByte(): int
157 |     {
158 |         if ($this->getApiVersion(self::PRODUCE_REQUEST) === self::API_VERSION2) {
159 |             return self::MESSAGE_MAGIC_VERSION1;
160 |         }
161 | 
162 |         return self::MESSAGE_MAGIC_VERSION0;
163 |     }
164 | 
165 |     public function computeTimestampType(int $magic): int
166 |     {
167 |         if ($magic === self::MESSAGE_MAGIC_VERSION0) {
168 |             return self::TIMESTAMP_NONE;
169 |         }
170 | 
171 |         return self::TIMESTAMP_CREATE_TIME;
172 |     }
173 | 
174 |     private function computeAttributes(int $magic, int $compression, int $timestampType): int
175 |     {
176 |         $attributes = 0;
177 | 
178 |         if ($compression !== self::COMPRESSION_NONE) {
179 |             $attributes |= self::COMPRESSION_CODEC_MASK & $compression;
180 |         }
181 | 
182 |         if ($magic === self::MESSAGE_MAGIC_VERSION0) {
183 |             return $attributes;
184 |         }
185 | 
186 |         if ($timestampType === self::TIMESTAMP_LOG_APPEND_TIME) {
187 |             $attributes |= self::TIMESTAMP_TYPE_MASK;
188 |         }
189 | 
190 |         return $attributes;
191 |     }
192 | 
193 |     /**
194 |      * encode signal part
195 |      *
196 |      * @param mixed[] $values
197 |      *
198 |      * @throws NotSupported
199 |      * @throws ProtocolException
200 |      */
201 |     protected function encodeProducePartition(array $values, int $compression): string
202 |     {
203 |         if (! isset($values['partition_id'])) {
204 |             throw new ProtocolException('given produce data invalid. `partition_id` is undefined.');
205 |         }
206 | 
207 |         if (! isset($values['messages']) || empty($values['messages'])) {
208 |             throw new ProtocolException('given produce data invalid. `messages` is undefined.');
209 |         }
210 | 
211 |         $data  = self::pack(self::BIT_B32, (string) $values['partition_id']);
212 |         $data .= self::encodeString(
213 |             $this->encodeMessageSet((array) $values['messages'], $compression),
214 |             self::PACK_INT32
215 |         );
216 | 
217 |         return $data;
218 |     }
219 | 
220 |     /**
221 |      * encode signal topic
222 |      *
223 |      * @param mixed[] $values
224 |      *
225 |      * @throws NotSupported
226 |      * @throws ProtocolException
227 |      */
228 |     protected function encodeProduceTopic(array $values, int $compression): string
229 |     {
230 |         if (! isset($values['topic_name'])) {
231 |             throw new ProtocolException('given produce data invalid. `topic_name` is undefined.');
232 |         }
233 | 
234 |         if (! isset($values['partitions']) || empty($values['partitions'])) {
235 |             throw new ProtocolException('given produce data invalid. `partitions` is undefined.');
236 |         }
237 | 
238 |         $topic      = self::encodeString($values['topic_name'], self::PACK_INT16);
239 |         $partitions = self::encodeArray($values['partitions'], [$this, 'encodeProducePartition'], $compression);
240 | 
241 |         return $topic . $partitions;
242 |     }
243 | 
244 |     /**
245 |      * decode produce topic pair response
246 |      *
247 |      * @return mixed[]
248 |      *
249 |      * @throws ProtocolException
250 |      */
251 |     protected function produceTopicPair(string $data, int $version): array
252 |     {
253 |         $offset    = 0;
254 |         $topicInfo = $this->decodeString($data, self::BIT_B16);
255 |         $offset   += $topicInfo['length'];
256 |         $ret       = $this->decodeArray(substr($data, $offset), [$this, 'producePartitionPair'], $version);
257 |         $offset   += $ret['length'];
258 | 
259 |         return [
260 |             'length' => $offset,
261 |             'data'   => [
262 |                 'topicName'  => $topicInfo['data'],
263 |                 'partitions' => $ret['data'],
264 |             ],
265 |         ];
266 |     }
267 | 
268 |     /**
269 |      * decode produce partition pair response
270 |      *
271 |      * @return mixed[]
272 |      *
273 |      * @throws ProtocolException
274 |      */
275 |     protected function producePartitionPair(string $data, int $version): array
276 |     {
277 |         $offset          = 0;
278 |         $partitionId     = self::unpack(self::BIT_B32, substr($data, $offset, 4));
279 |         $offset         += 4;
280 |         $errorCode       = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
281 |         $offset         += 2;
282 |         $partitionOffset = self::unpack(self::BIT_B64, substr($data, $offset, 8));
283 |         $offset         += 8;
284 |         $timestamp       = 0;
285 | 
286 |         if ($version === self::API_VERSION2) {
287 |             $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8));
288 |             $offset   += 8;
289 |         }
290 | 
291 |         return [
292 |             'length' => $offset,
293 |             'data'   => [
294 |                 'partition' => $partitionId,
295 |                 'errorCode' => $errorCode,
296 |                 'offset'    => $offset,
297 |                 'timestamp' => $timestamp,
298 |             ],
299 |         ];
300 |     }
301 | }
302 | 


--------------------------------------------------------------------------------
/src/Config.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka;
  5 | 
  6 | use function array_filter;
  7 | use function array_shift;
  8 | use function count;
  9 | use function explode;
 10 | use function in_array;
 11 | use function is_file;
 12 | use function lcfirst;
 13 | use function preg_match;
 14 | use function strpos;
 15 | use function substr;
 16 | use function trim;
 17 | use function version_compare;
 18 | 
 19 | /**
 20 |  * @method string getClientId()
 21 |  * @method string getBrokerVersion()
 22 |  * @method string getMetadataBrokerList()
 23 |  * @method int getMessageMaxBytes()
 24 |  * @method int getMetadataRequestTimeoutMs()
 25 |  * @method int getMetadataRefreshIntervalMs()
 26 |  * @method int getMetadataMaxAgeMs()
 27 |  * @method string getSecurityProtocol()
 28 |  * @method bool getSslEnable()
 29 |  * @method void setSslEnable(bool $sslEnable)
 30 |  * @method string getSslLocalCert()
 31 |  * @method string getSslLocalPk()
 32 |  * @method bool getSslVerifyPeer()
 33 |  * @method void setSslVerifyPeer(bool $sslVerifyPeer)
 34 |  * @method string getSslPassphrase()
 35 |  * @method void setSslPassphrase(string $sslPassphrase)
 36 |  * @method string getSslCafile()
 37 |  * @method string getSslPeerName()
 38 |  * @method void setSslPeerName(string $sslPeerName)
 39 |  * @method string getSaslMechanism()
 40 |  * @method string getSaslUsername()
 41 |  * @method string getSaslPassword()
 42 |  * @method string getSaslKeytab()
 43 |  * @method string getSaslPrincipal()
 44 |  */
 45 | abstract class Config
 46 | {
 47 |     public const SECURITY_PROTOCOL_PLAINTEXT      = 'PLAINTEXT';
 48 |     public const SECURITY_PROTOCOL_SSL            = 'SSL';
 49 |     public const SECURITY_PROTOCOL_SASL_PLAINTEXT = 'SASL_PLAINTEXT';
 50 |     public const SECURITY_PROTOCOL_SASL_SSL       = 'SASL_SSL';
 51 | 
 52 |     public const SASL_MECHANISMS_PLAIN         = 'PLAIN';
 53 |     public const SASL_MECHANISMS_GSSAPI        = 'GSSAPI';
 54 |     public const SASL_MECHANISMS_SCRAM_SHA_256 = 'SCRAM_SHA_256';
 55 |     public const SASL_MECHANISMS_SCRAM_SHA_512 = 'SCRAM_SHA_512';
 56 | 
 57 |     private const ALLOW_SECURITY_PROTOCOLS = [
 58 |         self::SECURITY_PROTOCOL_PLAINTEXT,
 59 |         self::SECURITY_PROTOCOL_SSL,
 60 |         self::SECURITY_PROTOCOL_SASL_PLAINTEXT,
 61 |         self::SECURITY_PROTOCOL_SASL_SSL,
 62 |     ];
 63 | 
 64 |     private const ALLOW_MECHANISMS = [
 65 |         self::SASL_MECHANISMS_PLAIN,
 66 |         self::SASL_MECHANISMS_GSSAPI,
 67 |         self::SASL_MECHANISMS_SCRAM_SHA_256,
 68 |         self::SASL_MECHANISMS_SCRAM_SHA_512,
 69 |     ];
 70 | 
 71 |     /**
 72 |      * @var mixed[]
 73 |      */
 74 |     protected static $options = [];
 75 | 
 76 |     /**
 77 |      * @var mixed[]
 78 |      */
 79 |     private static $defaults = [
 80 |         'clientId'                  => 'kafka-php',
 81 |         'brokerVersion'             => '0.10.1.0',
 82 |         'metadataBrokerList'        => '',
 83 |         'messageMaxBytes'           => 1000000,
 84 |         'metadataRequestTimeoutMs'  => 60000,
 85 |         'metadataRefreshIntervalMs' => 300000,
 86 |         'metadataMaxAgeMs'          => -1,
 87 |         'securityProtocol'          => self::SECURITY_PROTOCOL_PLAINTEXT,
 88 |         'sslEnable'                 => false, // this config item will override, don't config it.
 89 |         'sslLocalCert'              => '',
 90 |         'sslLocalPk'                => '',
 91 |         'sslVerifyPeer'             => false,
 92 |         'sslPassphrase'             => '',
 93 |         'sslCafile'                 => '',
 94 |         'sslPeerName'               => '',
 95 |         'saslMechanism'             => self::SASL_MECHANISMS_PLAIN,
 96 |         'saslUsername'              => '',
 97 |         'saslPassword'              => '',
 98 |         'saslKeytab'                => '',
 99 |         'saslPrincipal'             => '',
100 |     ];
101 | 
102 |     /**
103 |      * @param mixed[] $args
104 |      *
105 |      * @return bool|mixed
106 |      */
107 |     public function __call(string $name, array $args)
108 |     {
109 |         $isGetter = strpos($name, 'get') === 0 || strpos($name, 'iet') === 0;
110 |         $isSetter = strpos($name, 'set') === 0;
111 | 
112 |         if (! $isGetter && ! $isSetter) {
113 |             return false;
114 |         }
115 | 
116 |         $option = lcfirst(substr($name, 3));
117 | 
118 |         if ($isGetter) {
119 |             if (isset(self::$options[$option])) {
120 |                 return self::$options[$option];
121 |             }
122 | 
123 |             if (isset(self::$defaults[$option])) {
124 |                 return self::$defaults[$option];
125 |             }
126 | 
127 |             if (isset(static::$defaults[$option])) {
128 |                 return static::$defaults[$option];
129 |             }
130 | 
131 |             return false;
132 |         }
133 | 
134 |         if (count($args) !== 1) {
135 |             return false;
136 |         }
137 | 
138 |         static::$options[$option] = array_shift($args);
139 | 
140 |         // check todo
141 |         return true;
142 |     }
143 | 
144 |     /**
145 |      * @throws Exception\Config
146 |      */
147 |     public function setClientId(string $val): void
148 |     {
149 |         $client = trim($val);
150 | 
151 |         if ($client === '') {
152 |             throw new Exception\Config('Set clientId value is invalid, must is not empty string.');
153 |         }
154 | 
155 |         static::$options['clientId'] = $client;
156 |     }
157 | 
158 |     /**
159 |      * @throws Exception\Config
160 |      */
161 |     public function setBrokerVersion(string $version): void
162 |     {
163 |         $version = trim($version);
164 | 
165 |         if ($version === '' || version_compare($version, '0.8.0', '<')) {
166 |             throw new Exception\Config('Set broker version value is invalid, must is not empty string and gt 0.8.0.');
167 |         }
168 | 
169 |         static::$options['brokerVersion'] = $version;
170 |     }
171 | 
172 |     /**
173 |      * @throws Exception\Config
174 |      */
175 |     public function setMetadataBrokerList(string $brokerList): void
176 |     {
177 |         $brokerList = trim($brokerList);
178 | 
179 |         $brokers = array_filter(
180 |             explode(',', $brokerList),
181 |             function (string $broker): bool {
182 |                 return preg_match('/^(.*:[\d]+)$/', $broker) === 1;
183 |             }
184 |         );
185 | 
186 |         if (empty($brokers)) {
187 |             throw new Exception\Config(
188 |                 'Broker list must be a comma-separated list of brokers (format: "host:port"), with at least one broker'
189 |             );
190 |         }
191 | 
192 |         static::$options['metadataBrokerList'] = $brokerList;
193 |     }
194 | 
195 |     public function clear(): void
196 |     {
197 |         static::$options = [];
198 |     }
199 | 
200 |     /**
201 |      * @throws Exception\Config
202 |      */
203 |     public function setMessageMaxBytes(int $messageMaxBytes): void
204 |     {
205 |         if ($messageMaxBytes < 1000 || $messageMaxBytes > 1000000000) {
206 |             throw new Exception\Config('Set message max bytes value is invalid, must set it 1000 .. 1000000000');
207 |         }
208 |         static::$options['messageMaxBytes'] = $messageMaxBytes;
209 |     }
210 | 
211 |     /**
212 |      * @throws Exception\Config
213 |      */
214 |     public function setMetadataRequestTimeoutMs(int $metadataRequestTimeoutMs): void
215 |     {
216 |         if ($metadataRequestTimeoutMs < 10 || $metadataRequestTimeoutMs > 900000) {
217 |             throw new Exception\Config('Set metadata request timeout value is invalid, must set it 10 .. 900000');
218 |         }
219 |         static::$options['metadataRequestTimeoutMs'] = $metadataRequestTimeoutMs;
220 |     }
221 | 
222 |     /**
223 |      * @throws Exception\Config
224 |      */
225 |     public function setMetadataRefreshIntervalMs(int $metadataRefreshIntervalMs): void
226 |     {
227 |         if ($metadataRefreshIntervalMs < 10 || $metadataRefreshIntervalMs > 3600000) {
228 |             throw new Exception\Config('Set metadata refresh interval value is invalid, must set it 10 .. 3600000');
229 |         }
230 |         static::$options['metadataRefreshIntervalMs'] = $metadataRefreshIntervalMs;
231 |     }
232 | 
233 |     /**
234 |      * @throws Exception\Config
235 |      */
236 |     public function setMetadataMaxAgeMs(int $metadataMaxAgeMs): void
237 |     {
238 |         if ($metadataMaxAgeMs < 1 || $metadataMaxAgeMs > 86400000) {
239 |             throw new Exception\Config('Set metadata max age value is invalid, must set it 1 .. 86400000');
240 |         }
241 |         static::$options['metadataMaxAgeMs'] = $metadataMaxAgeMs;
242 |     }
243 | 
244 |     /**
245 |      * @throws Exception\Config
246 |      */
247 |     public function setSslLocalCert(string $localCert): void
248 |     {
249 |         if (! is_file($localCert)) {
250 |             throw new Exception\Config('Set ssl local cert file is invalid');
251 |         }
252 | 
253 |         static::$options['sslLocalCert'] = $localCert;
254 |     }
255 | 
256 |     /**
257 |      * @throws Exception\Config
258 |      */
259 |     public function setSslLocalPk(string $localPk): void
260 |     {
261 |         if (! is_file($localPk)) {
262 |             throw new Exception\Config('Set ssl local private key file is invalid');
263 |         }
264 | 
265 |         static::$options['sslLocalPk'] = $localPk;
266 |     }
267 | 
268 |     /**
269 |      * @throws Exception\Config
270 |      */
271 |     public function setSslCafile(string $cafile): void
272 |     {
273 |         if (! is_file($cafile)) {
274 |             throw new Exception\Config('Set ssl ca file is invalid');
275 |         }
276 | 
277 |         static::$options['sslCafile'] = $cafile;
278 |     }
279 | 
280 |     /**
281 |      * @throws Exception\Config
282 |      */
283 |     public function setSaslKeytab(string $keytab): void
284 |     {
285 |         if (! is_file($keytab)) {
286 |             throw new Exception\Config('Set sasl gssapi keytab file is invalid');
287 |         }
288 | 
289 |         static::$options['saslKeytab'] = $keytab;
290 |     }
291 | 
292 |     /**
293 |      * @throws Exception\Config
294 |      */
295 |     public function setSecurityProtocol(string $protocol): void
296 |     {
297 |         if (! in_array($protocol, self::ALLOW_SECURITY_PROTOCOLS, true)) {
298 |             throw new Exception\Config('Invalid security protocol given.');
299 |         }
300 | 
301 |         static::$options['securityProtocol'] = $protocol;
302 |     }
303 | 
304 |     /**
305 |      * @throws Exception\Config
306 |      */
307 |     public function setSaslMechanism(string $mechanism): void
308 |     {
309 |         if (! in_array($mechanism, self::ALLOW_MECHANISMS, true)) {
310 |             throw new Exception\Config('Invalid security sasl mechanism given.');
311 |         }
312 | 
313 |         static::$options['saslMechanism'] = $mechanism;
314 |     }
315 | }
316 | 


--------------------------------------------------------------------------------
/src/Producer/Process.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Producer;
  5 | 
  6 | use Amp\Loop;
  7 | use Kafka\Broker;
  8 | use Kafka\Exception;
  9 | use Kafka\LoggerTrait;
 10 | use Kafka\ProducerConfig;
 11 | use Kafka\Protocol;
 12 | use Psr\Log\LoggerAwareTrait;
 13 | use function array_keys;
 14 | use function count;
 15 | use function explode;
 16 | use function in_array;
 17 | use function json_encode;
 18 | use function shuffle;
 19 | use function sprintf;
 20 | use function substr;
 21 | use function trim;
 22 | 
 23 | class Process
 24 | {
 25 |     use LoggerAwareTrait;
 26 |     use LoggerTrait;
 27 | 
 28 |     /**
 29 |      * @var callable|null
 30 |      */
 31 |     protected $producer;
 32 | 
 33 |     /**
 34 |      * @var bool
 35 |      */
 36 |     protected $isRunning = true;
 37 | 
 38 |     /**
 39 |      * @var callable|null
 40 |      */
 41 |     protected $success;
 42 | 
 43 |     /**
 44 |      * @var callable|null
 45 |      */
 46 |     protected $error;
 47 | 
 48 |     /**
 49 |      * @var State
 50 |      */
 51 |     private $state;
 52 | 
 53 |     /** @var RecordValidator */
 54 |     private $recordValidator;
 55 | 
 56 |     public function __construct(?callable $producer = null, ?RecordValidator $recordValidator = null)
 57 |     {
 58 |         $this->producer        = $producer;
 59 |         $this->recordValidator = $recordValidator ?? new RecordValidator();
 60 |     }
 61 | 
 62 |     public function init(): void
 63 |     {
 64 |         $config = $this->getConfig();
 65 |         Protocol::init($config->getBrokerVersion(), $this->logger);
 66 | 
 67 |         $broker = $this->getBroker();
 68 |         $broker->setConfig($config);
 69 |         $broker->setProcess(
 70 |             function (string $data, int $fd): void {
 71 |                 $this->processRequest($data, $fd);
 72 |             }
 73 |         );
 74 | 
 75 |         $this->state = State::getInstance();
 76 | 
 77 |         if ($this->logger) {
 78 |             $this->state->setLogger($this->logger);
 79 |         }
 80 | 
 81 |         $this->state->setCallback(
 82 |             [
 83 |                 State::REQUEST_METADATA => [$this, 'syncMeta'],
 84 |                 State::REQUEST_PRODUCE  => [$this, 'produce'],
 85 |             ]
 86 |         );
 87 | 
 88 |         $this->state->init();
 89 | 
 90 |         if (! empty($broker->getTopics())) {
 91 |             $this->state->succRun(State::REQUEST_METADATA);
 92 |         }
 93 |     }
 94 | 
 95 |     public function start(): void
 96 |     {
 97 |         $this->init();
 98 |         $this->state->start();
 99 | 
100 |         $config = $this->getConfig();
101 | 
102 |         if ($config->getIsAsyn()) {
103 |             return;
104 |         }
105 | 
106 |         Loop::repeat(
107 |             $config->getRequestTimeout(),
108 |             function (string $watcherId): void {
109 |                 if ($this->error !== null) {
110 |                     ($this->error)(1000);
111 |                 }
112 | 
113 |                 Loop::cancel($watcherId);
114 |                 Loop::stop();
115 |             }
116 |         );
117 |     }
118 | 
119 |     public function stop(): void
120 |     {
121 |         $this->isRunning = false;
122 |     }
123 | 
124 |     public function setSuccess(callable $success): void
125 |     {
126 |         $this->success = $success;
127 |     }
128 | 
129 |     public function setError(callable $error): void
130 |     {
131 |         $this->error = $error;
132 |     }
133 | 
134 |     /**
135 |      * @throws \Kafka\Exception
136 |      */
137 |     public function syncMeta(): void
138 |     {
139 |         $this->debug('Start sync metadata request');
140 | 
141 |         $brokerList = ProducerConfig::getInstance()->getMetadataBrokerList();
142 |         $brokerHost = [];
143 | 
144 |         foreach (explode(',', $brokerList) as $key => $val) {
145 |             if (trim($val)) {
146 |                 $brokerHost[] = $val;
147 |             }
148 |         }
149 | 
150 |         if (count($brokerHost) === 0) {
151 |             throw new Exception('No valid broker configured');
152 |         }
153 | 
154 |         shuffle($brokerHost);
155 |         $broker = $this->getBroker();
156 | 
157 |         foreach ($brokerHost as $host) {
158 |             $socket = $broker->getMetaConnect($host);
159 | 
160 |             if ($socket !== null) {
161 |                 $params = [];
162 |                 $this->debug('Start sync metadata request params:' . json_encode($params));
163 |                 $requestData = Protocol::encode(Protocol::METADATA_REQUEST, $params);
164 |                 $socket->write($requestData);
165 | 
166 |                 return;
167 |             }
168 |         }
169 | 
170 |         throw Exception\ConnectionException::fromBrokerList($brokerList);
171 |     }
172 | 
173 |     /**
174 |      * process Request
175 |      *
176 |      * @throws \Kafka\Exception
177 |      */
178 |     protected function processRequest(string $data, int $fd): void
179 |     {
180 |         $correlationId = Protocol\Protocol::unpack(Protocol\Protocol::BIT_B32, substr($data, 0, 4));
181 |         switch ($correlationId) {
182 |             case Protocol::METADATA_REQUEST:
183 |                 $result = Protocol::decode(Protocol::METADATA_REQUEST, substr($data, 4));
184 | 
185 |                 if (! isset($result['brokers'], $result['topics'])) {
186 |                     $this->error('Get metadata is fail, brokers or topics is null.');
187 |                     $this->state->failRun(State::REQUEST_METADATA);
188 |                     break;
189 |                 }
190 | 
191 |                 $broker   = $this->getBroker();
192 |                 $isChange = $broker->setData($result['topics'], $result['brokers']);
193 |                 $this->state->succRun(State::REQUEST_METADATA, $isChange);
194 | 
195 |                 break;
196 |             case Protocol::PRODUCE_REQUEST:
197 |                 $result = Protocol::decode(Protocol::PRODUCE_REQUEST, substr($data, 4));
198 |                 $this->succProduce($result, $fd);
199 |                 break;
200 |             default:
201 |                 $this->error('Error request, correlationId:' . $correlationId);
202 |         }
203 |     }
204 | 
205 |     /**
206 |      * @return int[]
207 |      */
208 |     public function produce(): array
209 |     {
210 |         $context = [];
211 |         $broker  = $this->getBroker();
212 |         $config  = $this->getConfig();
213 | 
214 |         $requiredAck = $config->getRequiredAck();
215 |         $timeout     = $config->getTimeout();
216 |         $compression = $config->getCompression();
217 | 
218 |         // get send message
219 |         // data struct
220 |         //  topic:
221 |         //  partId:
222 |         //  key:
223 |         //  value:
224 |         $data = ($this->producer)();
225 | 
226 |         if (empty($data)) {
227 |             return $context;
228 |         }
229 | 
230 |         $sendData = $this->convertRecordSet($data);
231 | 
232 |         foreach ($sendData as $brokerId => $topicList) {
233 |             $connect = $broker->getDataConnect((string) $brokerId);
234 | 
235 |             if ($connect === null) {
236 |                 return $context;
237 |             }
238 | 
239 |             $params = [
240 |                 'required_ack' => $requiredAck,
241 |                 'timeout'      => $timeout,
242 |                 'data'         => $topicList,
243 |                 'compression'  => $compression,
244 |             ];
245 | 
246 |             $this->debug('Send message start, params:' . json_encode($params));
247 |             $requestData = Protocol::encode(Protocol::PRODUCE_REQUEST, $params);
248 | 
249 |             if ($requiredAck === 0) { // If it is 0 the server will not send any response
250 |                 $this->state->succRun(State::REQUEST_PRODUCE);
251 |             } else {
252 |                 $connect->write($requestData);
253 |                 $context[] = (int) $connect->getSocket();
254 |             }
255 |         }
256 | 
257 |         return $context;
258 |     }
259 | 
260 |     /**
261 |      * @param mixed[] $result
262 |      */
263 |     protected function succProduce(array $result, int $fd): void
264 |     {
265 |         $msg = sprintf('Send message sucess, result: %s', json_encode($result));
266 |         $this->debug($msg);
267 | 
268 |         if ($this->success !== null) {
269 |             ($this->success)($result);
270 |         }
271 | 
272 |         $this->state->succRun(State::REQUEST_PRODUCE, $fd);
273 |     }
274 | 
275 |     protected function stateConvert(int $errorCode): bool
276 |     {
277 |         $this->error(Protocol::getError($errorCode));
278 | 
279 |         if ($this->error !== null) {
280 |             ($this->error)($errorCode);
281 |         }
282 | 
283 |         $recoverCodes = [
284 |             Protocol::UNKNOWN_TOPIC_OR_PARTITION,
285 |             Protocol::INVALID_REQUIRED_ACKS,
286 |             Protocol::RECORD_LIST_TOO_LARGE,
287 |             Protocol::NOT_ENOUGH_REPLICAS_AFTER_APPEND,
288 |             Protocol::NOT_ENOUGH_REPLICAS,
289 |             Protocol::NOT_LEADER_FOR_PARTITION,
290 |             Protocol::BROKER_NOT_AVAILABLE,
291 |             Protocol::GROUP_LOAD_IN_PROGRESS,
292 |             Protocol::GROUP_COORDINATOR_NOT_AVAILABLE,
293 |             Protocol::NOT_COORDINATOR_FOR_GROUP,
294 |             Protocol::INVALID_TOPIC,
295 |             Protocol::INCONSISTENT_GROUP_PROTOCOL,
296 |             Protocol::INVALID_GROUP_ID,
297 |         ];
298 | 
299 |         if (in_array($errorCode, $recoverCodes, true)) {
300 |             $this->state->recover();
301 | 
302 |             return false;
303 |         }
304 | 
305 |         return true;
306 |     }
307 | 
308 |     /**
309 |      * @param mixed[] $recordSet
310 |      *
311 |      * @return mixed[]
312 |      */
313 |     protected function convertRecordSet(array $recordSet): array
314 |     {
315 |         $sendData = [];
316 |         $broker   = $this->getBroker();
317 |         $topics   = $broker->getTopics();
318 | 
319 |         foreach ($recordSet as $record) {
320 |             $this->recordValidator->validate($record, $topics);
321 | 
322 |             $topicMeta = $topics[$record['topic']];
323 |             $partNums  = array_keys($topicMeta);
324 |             shuffle($partNums);
325 | 
326 |             $partId = ! isset($record['partId'], $topicMeta[$record['partId']]) ? $partNums[0] : $record['partId'];
327 | 
328 |             $brokerId  = $topicMeta[$partId];
329 |             $topicData = [];
330 |             if (isset($sendData[$brokerId][$record['topic']])) {
331 |                 $topicData = $sendData[$brokerId][$record['topic']];
332 |             }
333 | 
334 |             $partition = [];
335 |             if (isset($topicData['partitions'][$partId])) {
336 |                 $partition = $topicData['partitions'][$partId];
337 |             }
338 | 
339 |             $partition['partition_id'] = $partId;
340 |             if (trim($record['key'] ?? '') !== '') {
341 |                 $partition['messages'][] = ['value' => $record['value'], 'key' => $record['key']];
342 |             } else {
343 |                 $partition['messages'][] = $record['value'];
344 |             }
345 | 
346 |             $topicData['partitions'][$partId]      = $partition;
347 |             $topicData['topic_name']               = $record['topic'];
348 |             $sendData[$brokerId][$record['topic']] = $topicData;
349 |         }
350 | 
351 |         return $sendData;
352 |     }
353 | 
354 |     private function getConfig(): ProducerConfig
355 |     {
356 |         return ProducerConfig::getInstance();
357 |     }
358 | 
359 |     private function getBroker(): Broker
360 |     {
361 |         return Broker::getInstance();
362 |     }
363 | }
364 | 


--------------------------------------------------------------------------------
/src/Protocol/Fetch.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | declare(strict_types=1);
  3 | 
  4 | namespace Kafka\Protocol;
  5 | 
  6 | use Kafka\Exception;
  7 | use Kafka\Exception\NotSupported;
  8 | use Kafka\Exception\Protocol as ProtocolException;
  9 | use function array_merge;
 10 | use function strlen;
 11 | use function substr;
 12 | 
 13 | class Fetch extends Protocol
 14 | {
 15 |     /**
 16 |      * @param mixed[] $payloads
 17 |      *
 18 |      * @throws NotSupported
 19 |      * @throws ProtocolException
 20 |      */
 21 |     public function encode(array $payloads = []): string
 22 |     {
 23 |         if (! isset($payloads['data'])) {
 24 |             throw new ProtocolException('given fetch kafka data invalid. `data` is undefined.');
 25 |         }
 26 | 
 27 |         if (! isset($payloads['replica_id'])) {
 28 |             $payloads['replica_id'] = -1;
 29 |         }
 30 | 
 31 |         if (! isset($payloads['max_wait_time'])) {
 32 |             $payloads['max_wait_time'] = 100; // default timeout 100ms
 33 |         }
 34 | 
 35 |         if (! isset($payloads['min_bytes'])) {
 36 |             $payloads['min_bytes'] = 64 * 1024; // 64k
 37 |         }
 38 | 
 39 |         $header = $this->requestHeader('kafka-php', self::FETCH_REQUEST, self::FETCH_REQUEST);
 40 |         $data   = self::pack(self::BIT_B32, (string) $payloads['replica_id']);
 41 |         $data  .= self::pack(self::BIT_B32, (string) $payloads['max_wait_time']);
 42 |         $data  .= self::pack(self::BIT_B32, (string) $payloads['min_bytes']);
 43 |         $data  .= self::encodeArray($payloads['data'], [$this, 'encodeFetchTopic']);
 44 |         $data   = self::encodeString($header . $data, self::PACK_INT32);
 45 | 
 46 |         return $data;
 47 |     }
 48 | 
 49 |     /**
 50 |      * @return mixed[]
 51 |      */
 52 |     public function decode(string $data): array
 53 |     {
 54 |         $offset       = 0;
 55 |         $version      = $this->getApiVersion(self::FETCH_REQUEST);
 56 |         $throttleTime = 0;
 57 | 
 58 |         if ($version !== self::API_VERSION0) {
 59 |             $throttleTime = self::unpack(self::BIT_B32, substr($data, $offset, 4));
 60 |             $offset      += 4;
 61 |         }
 62 | 
 63 |         $topics  = $this->decodeArray(substr($data, $offset), [$this, 'fetchTopic']);
 64 |         $offset += $topics['length'];
 65 | 
 66 |         return [
 67 |             'throttleTime' => $throttleTime,
 68 |             'topics'       => $topics['data'],
 69 |         ];
 70 |     }
 71 | 
 72 |     /**
 73 |      * @return mixed[]
 74 |      */
 75 |     protected function fetchTopic(string $data): array
 76 |     {
 77 |         $offset    = 0;
 78 |         $topicInfo = $this->decodeString(substr($data, $offset), self::BIT_B16);
 79 |         $offset   += $topicInfo['length'];
 80 | 
 81 |         $partitions = $this->decodeArray(substr($data, $offset), [$this, 'fetchPartition']);
 82 |         $offset    += $partitions['length'];
 83 | 
 84 |         return [
 85 |             'length' => $offset,
 86 |             'data'   => [
 87 |                 'topicName'  => $topicInfo['data'],
 88 |                 'partitions' => $partitions['data'],
 89 |             ],
 90 |         ];
 91 |     }
 92 | 
 93 |     /**
 94 |      * @return mixed[]
 95 |      *
 96 |      * @throws ProtocolException
 97 |      */
 98 |     protected function fetchPartition(string $data): array
 99 |     {
100 |         $offset              = 0;
101 |         $partitionId         = self::unpack(self::BIT_B32, substr($data, $offset, 4));
102 |         $offset             += 4;
103 |         $errorCode           = self::unpack(self::BIT_B16_SIGNED, substr($data, $offset, 2));
104 |         $offset             += 2;
105 |         $highwaterMarkOffset = self::unpack(self::BIT_B64, substr($data, $offset, 8));
106 |         $offset             += 8;
107 | 
108 |         $messageSetSize = self::unpack(self::BIT_B32, substr($data, $offset, 4));
109 |         $offset        += 4;
110 | 
111 |         $messages = [];
112 | 
113 |         if ($messageSetSize > 0 && $offset < strlen($data)) {
114 |             $messages = $this->decodeMessageSetArray(substr($data, $offset, $messageSetSize), $messageSetSize);
115 | 
116 |             $offset += $messages['length'];
117 |         }
118 | 
119 |         return [
120 |             'length' => $offset,
121 |             'data'   => [
122 |                 'partition'           => $partitionId,
123 |                 'errorCode'           => $errorCode,
124 |                 'highwaterMarkOffset' => $highwaterMarkOffset,
125 |                 'messageSetSize'      => $messageSetSize,
126 |                 'messages'            => $messages['data'] ?? [],
127 |             ],
128 |         ];
129 |     }
130 | 
131 |     /**
132 |      * @return mixed[]
133 |      *
134 |      * @throws ProtocolException
135 |      */
136 |     protected function decodeMessageSetArray(string $data, int $messageSetSize): array
137 |     {
138 |         $offset = 0;
139 |         $result = [];
140 | 
141 |         while ($offset < strlen($data)) {
142 |             $value = substr($data, $offset);
143 |             $ret   = $this->decodeMessageSet($value);
144 | 
145 |             if ($ret === null) {
146 |                 break;
147 |             }
148 | 
149 |             if (! isset($ret['length'], $ret['data'])) {
150 |                 throw new ProtocolException('Decode array failed, given function return format is invalid');
151 |             }
152 | 
153 |             if ((int) $ret['length'] === 0) {
154 |                 continue;
155 |             }
156 | 
157 |             $offset += $ret['length'];
158 | 
159 |             if (($ret['data']['message']['attr'] & Produce::COMPRESSION_CODEC_MASK) === Produce::COMPRESSION_NONE) {
160 |                 $result[] = $ret['data'];
161 |                 continue;
162 |             }
163 | 
164 |             $innerMessages = $this->decodeMessageSetArray($ret['data']['message']['value'], $ret['length']);
165 |             $result        = array_merge($result, $innerMessages['data']);
166 |         }
167 | 
168 |         if ($offset < $messageSetSize) {
169 |             $offset = $messageSetSize;
170 |         }
171 | 
172 |         return ['length' => $offset, 'data' => $result];
173 |     }
174 | 
175 |     /**
176 |      * decode message set
177 |      * N.B., MessageSets are not preceded by an int32 like other array elements
178 |      * in the protocol.
179 |      *
180 |      * @return mixed[]|null
181 |      */
182 |     protected function decodeMessageSet(string $data): ?array
183 |     {
184 |         if (strlen($data) <= 12) {
185 |             return null;
186 |         }
187 | 
188 |         $offset      = 0;
189 |         $roffset     = self::unpack(self::BIT_B64, substr($data, $offset, 8));
190 |         $offset     += 8;
191 |         $messageSize = self::unpack(self::BIT_B32, substr($data, $offset, 4));
192 |         $offset     += 4;
193 |         $ret         = $this->decodeMessage(substr($data, $offset), $messageSize);
194 | 
195 |         if ($ret === null) {
196 |             return null;
197 |         }
198 | 
199 |         $offset += $ret['length'];
200 | 
201 |         return [
202 |             'length' => $offset,
203 |             'data'   => [
204 |                 'offset'  => $roffset,
205 |                 'size'    => $messageSize,
206 |                 'message' => $ret['data'],
207 |             ],
208 |         ];
209 |     }
210 | 
211 |     /**
212 |      * decode message
213 |      * N.B., MessageSets are not preceded by an int32 like other array elements
214 |      * in the protocol.
215 |      *
216 |      * @return mixed[]|null
217 |      */
218 |     protected function decodeMessage(string $data, int $messageSize): ?array
219 |     {
220 |         if ($messageSize === 0 || strlen($data) < $messageSize) {
221 |             return null;
222 |         }
223 | 
224 |         $offset  = 0;
225 |         $crc     = self::unpack(self::BIT_B32, substr($data, $offset, 4));
226 |         $offset += 4;
227 | 
228 |         $magic = self::unpack(self::BIT_B8, substr($data, $offset, 1));
229 |         ++$offset;
230 | 
231 |         $attr = self::unpack(self::BIT_B8, substr($data, $offset, 1));
232 |         ++$offset;
233 | 
234 |         $timestamp  = 0;
235 |         $backOffset = $offset;
236 | 
237 |         try { // try unpack message format v1, falling back to v0 if it fails
238 |             if ($magic >= self::MESSAGE_MAGIC_VERSION1) {
239 |                 $timestamp = self::unpack(self::BIT_B64, substr($data, $offset, 8));
240 |                 $offset   += 8;
241 |             }
242 | 
243 |             $keyRet  = $this->decodeString(substr($data, $offset), self::BIT_B32);
244 |             $offset += $keyRet['length'];
245 | 
246 |             $valueRet = $this->decodeString((string) substr($data, $offset), self::BIT_B32, $attr & Produce::COMPRESSION_CODEC_MASK);
247 |             $offset  += $valueRet['length'];
248 | 
249 |             if ($offset !== $messageSize) {
250 |                 throw new Exception(
251 |                     'pack message fail, message len:' . $messageSize . ' , data unpack offset :' . $offset
252 |                 );
253 |             }
254 |         } catch (Exception $e) { // try unpack message format v0
255 |             $offset    = $backOffset;
256 |             $timestamp = 0;
257 |             $keyRet    = $this->decodeString(substr($data, $offset), self::BIT_B32);
258 |             $offset   += $keyRet['length'];
259 | 
260 |             $valueRet = $this->decodeString(substr($data, $offset), self::BIT_B32);
261 |             $offset  += $valueRet['length'];
262 |         }
263 | 
264 |         return [
265 |             'length' => $offset,
266 |             'data'   => [
267 |                 'crc'       => $crc,
268 |                 'magic'     => $magic,
269 |                 'attr'      => $attr,
270 |                 'timestamp' => $timestamp,
271 |                 'key'       => $keyRet['data'],
272 |                 'value'     => $valueRet['data'],
273 |             ],
274 |         ];
275 |     }
276 | 
277 |     /**
278 |      * @param mixed[] $values
279 |      *
280 |      * @throws ProtocolException
281 |      */
282 |     protected function encodeFetchPartition(array $values): string
283 |     {
284 |         if (! isset($values['partition_id'])) {
285 |             throw new ProtocolException('given fetch data invalid. `partition_id` is undefined.');
286 |         }
287 | 
288 |         if (! isset($values['offset'])) {
289 |             $values['offset'] = 0;
290 |         }
291 | 
292 |         if (! isset($values['max_bytes'])) {
293 |             $values['max_bytes'] = 2 * 1024 * 1024;
294 |         }
295 | 
296 |         $data  = self::pack(self::BIT_B32, (string) $values['partition_id']);
297 |         $data .= self::pack(self::BIT_B64, (string) $values['offset']);
298 |         $data .= self::pack(self::BIT_B32, (string) $values['max_bytes']);
299 | 
300 |         return $data;
301 |     }
302 | 
303 |     /**
304 |      * @param mixed[] $values
305 |      *
306 |      * @throws NotSupported
307 |      * @throws ProtocolException
308 |      */
309 |     protected function encodeFetchTopic(array $values): string
310 |     {
311 |         if (! isset($values['topic_name'])) {
312 |             throw new ProtocolException('given fetch data invalid. `topic_name` is undefined.');
313 |         }
314 | 
315 |         if (! isset($values['partitions']) || empty($values['partitions'])) {
316 |             throw new ProtocolException('given fetch data invalid. `partitions` is undefined.');
317 |         }
318 | 
319 |         $topic      = self::encodeString($values['topic_name'], self::PACK_INT16);
320 |         $partitions = self::encodeArray($values['partitions'], [$this, 'encodeFetchPartition']);
321 | 
322 |         return $topic . $partitions;
323 |     }
324 | }
325 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
  1 |                                  Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 


--------------------------------------------------------------------------------