├── .gitignore ├── LICENSE ├── README.md ├── composer.json └── src ├── Kafka.php └── Kafka ├── AbstractRecord.php ├── Admin └── AdminClient.php ├── BinarySchemeInterface.php ├── Client.php ├── Common ├── Cluster.php ├── Config.php ├── Node.php ├── PartitionMetadata.php ├── RestorableTrait.php ├── TopicMetadata.php └── Utils │ └── ByteUtils.php ├── Consumer ├── Config.php ├── Internals │ └── SubscriptionState.php ├── KafkaConsumer.php ├── MemberAssignment.php ├── OffsetResetStrategy.php ├── PartitionAssignorInterface.php ├── RoundRobinAssignor.php └── Subscription.php ├── DTO ├── ApiVersionsResponseMetadata.php ├── ControlledShutdownResponsePartition.php ├── DescribeGroupResponseMember.php ├── DescribeGroupResponseMetadata.php ├── FetchRequestTopic.php ├── FetchRequestTopicPartition.php ├── FetchResponseAbortedTransaction.php ├── FetchResponsePartition.php ├── FetchResponseTopic.php ├── GroupCoordinatorResponseMetadata.php ├── Header.php ├── JoinGroupRequestProtocol.php ├── JoinGroupResponseMember.php ├── ListGroupResponseProtocol.php ├── OffsetCommitRequestPartition.php ├── OffsetCommitRequestTopic.php ├── OffsetCommitResponsePartition.php ├── OffsetCommitResponseTopic.php ├── OffsetFetchResponsePartition.php ├── OffsetFetchResponseTopic.php ├── OffsetsRequestPartition.php ├── OffsetsRequestTopic.php ├── OffsetsResponsePartition.php ├── OffsetsResponseTopic.php ├── ProduceRequestPartition.php ├── ProduceRequestTopic.php ├── ProduceResponsePartition.php ├── ProduceResponseTopic.php ├── Record.php ├── RecordBatch.php ├── SyncGroupRequestMember.php └── TopicPartitions.php ├── Enum ├── SecurityProtocol.php └── SslProtocol.php ├── Error ├── AllBrokersNotAvailable.php ├── BrokerNotAvailable.php ├── ClientExceptionInterface.php ├── ClusterAuthorizationFailed.php ├── CorruptMessage.php ├── EmptyAssignmentException.php ├── GroupAuthorizationFailed.php ├── GroupCoordinatorNotAvailable.php ├── GroupLoadInProgress.php ├── IllegalGeneration.php ├── IllegalSaslState.php ├── InconsistentGroupProtocol.php ├── InvalidCommitOffsetSize.php ├── InvalidConfiguration.php ├── InvalidFetchSize.php ├── InvalidGroupId.php ├── InvalidRequiredAcks.php ├── InvalidSessionTimeout.php ├── InvalidTimestamp.php ├── InvalidTopicException.php ├── KafkaException.php ├── LeaderNotAvailable.php ├── MessageTooLarge.php ├── NetworkException.php ├── NotCoordinatorForGroup.php ├── NotEnoughReplicas.php ├── NotEnoughReplicasAfterAppend.php ├── NotLeaderForPartition.php ├── OffsetMetadataTooLarge.php ├── OffsetOutOfRange.php ├── RebalanceInProgress.php ├── RecordListTooLarge.php ├── ReplicaNotAvailable.php ├── RequestTimedOut.php ├── RetriableException.php ├── ServerExceptionInterface.php ├── StaleControllerEpoch.php ├── TopicAuthorizationFailed.php ├── TopicPartitionRequestException.php ├── UnknownError.php ├── UnknownMemberId.php ├── UnknownTopicOrPartition.php ├── UnsupportedSaslMechanism.php └── UnsupportedVersion.php ├── Producer ├── Config.php ├── DefaultPartitioner.php ├── KafkaProducer.php └── PartitionerInterface.php ├── Record ├── AbstractRequest.php ├── AbstractResponse.php ├── ApiVersionsRequest.php ├── ApiVersionsResponse.php ├── ControlledShutdownRequest.php ├── ControlledShutdownResponse.php ├── DescribeGroupsRequest.php ├── DescribeGroupsResponse.php ├── FetchRequest.php ├── FetchResponse.php ├── GroupCoordinatorRequest.php ├── GroupCoordinatorResponse.php ├── HeartbeatRequest.php ├── HeartbeatResponse.php ├── JoinGroupRequest.php ├── JoinGroupResponse.php ├── LeaveGroupRequest.php ├── LeaveGroupResponse.php ├── ListGroupsRequest.php ├── ListGroupsResponse.php ├── MetadataRequest.php ├── MetadataResponse.php ├── OffsetCommitRequest.php ├── OffsetCommitResponse.php ├── OffsetFetchRequest.php ├── OffsetFetchResponse.php ├── OffsetsRequest.php ├── OffsetsResponse.php ├── ProduceRequest.php ├── ProduceResponse.php ├── SaslHandshakeRequest.php ├── SaslHandshakeResponse.php ├── SyncGroupRequest.php └── SyncGroupResponse.php ├── Scheme.php ├── Stream.php └── Stream ├── AbstractStream.php ├── SocketStream.php └── StringStream.php /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2018 Alpari 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is furnished 8 | to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | PHP Native Apache Kafka Client 2 | ----------------- 3 | 4 | `alpari/kafka-client` is a PHP library implementation of the Apache Kafka protocol, containing both Producer and Consumer support. It was designed to be as close to PHP as possible, keeping API and config as close to the original ones as possible. 5 | 6 | 7 | Installation 8 | ------------ 9 | 10 | `alpari/kafka-client` can be installed with composer. Installation is quite easy, just ask the Composer to download the library with its dependencies by running the command: 11 | 12 | ``` bash 13 | $ composer require alpari/kafka-client 14 | ``` 15 | 16 | This library contains several branches, each branch contains support for specfic version of Apache Kafka, see mapping below: 17 | 18 | - Branch 0.8.x is suited for Kafka 0.8.0 versions 19 | - Branch 0.9.x is suited for Kafka 0.9.0 versions 20 | - Branch 0.10.x is suited for Kafka 0.10.0 versions 21 | - Branch master is suited for Kafka 0.11.0 versions 22 | 23 | Producer API 24 | ------------ 25 | The Producer API allows applications to send streams of data to topics in the Kafka cluster. 26 | 27 | Example showing how to use the producer is given below: 28 | 29 | ```php 30 | use Alpari\Kafka\DTO\Message; 31 | use Alpari\Kafka\Producer\Config; 32 | use Alpari\Kafka\Producer\KafkaProducer; 33 | 34 | include __DIR__ . '/vendor/autoload.php'; 35 | 36 | $producer = new KafkaProducer([ 37 | Config::BOOTSTRAP_SERVERS => ['tcp://localhost'], 38 | ]); 39 | $result = $producer->send('test', Message::fromValue('foo')); 40 | ``` 41 | 42 | Only required option is `Config::BOOTSTRAP_SERVERS` which should describe list of Kafka servers used to bootstrap connections to Kafka. 43 | 44 | For additional options, please see `Alpari\Kafka\Producer\Config` constants description and [producer configuration]. 45 | 46 | 47 | Consumer API 48 | ------------ 49 | 50 | The Consumer API allows applications to read streams of data from topics in the Kafka cluster. 51 | 52 | Example showing how to use the consumer is given below. 53 | 54 | ```php 55 | use Alpari\Kafka; 56 | use Alpari\Kafka\Consumer\Config; 57 | use Alpari\Kafka\Consumer\KafkaConsumer; 58 | 59 | $consumer = new KafkaConsumer([ 60 | Config::BOOTSTRAP_SERVERS => ['tcp://localhost'], 61 | Config::GROUP_ID => 'Kafka-Daemon', 62 | Config::FETCH_MAX_WAIT_MS => 5000, 63 | Config::AUTO_OFFSET_RESET => Kafka\Consumer\OffsetResetStrategy::LATEST, 64 | Config::SESSION_TIMEOUT_MS => 30000, 65 | Config::AUTO_COMMIT_INTERVAL_MS => 10000, 66 | Config::METADATA_CACHE_FILE => '/tmp/metadata.php', 67 | ]); 68 | $consumer->subscribe(['test']); 69 | for ($i=0; $i<100; $i++) { 70 | $data = $consumer->poll(1000); 71 | echo json_encode($data), PHP_EOL; 72 | } 73 | ``` 74 | 75 | For detailed description of configuration, please visit [consumer configuration]. 76 | 77 | 78 | PHP-specific configuration 79 | -------------------------- 80 | This library introduces some specific configuration options in order to work faster with PHP: 81 | 82 | - `metadata.cache.file` File name that stores the metadata, this file will be effectively cached by the Opcode cache in production 83 | - `stream.async.connect` Should client use asynchronous connection to the broker 84 | - `stream.persistent.connection` Should client use persistent connection to the cluster or not. 85 | 86 | For publishing events from web-requests it is recommended to enable persistent connection and configuring path for metadata cache. In this case publishing of event will be as fast as possible. 87 | 88 | [producer configuration]: https://kafka.apache.org/documentation/#producerconfigs 89 | [consumer configuration]: https://kafka.apache.org/documentation/#newconsumerconfigs 90 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "alpari/kafka-client", 3 | "description": "Implementation of Kafka Client in PHP", 4 | "type": "library", 5 | "license": "MIT", 6 | "authors": [ 7 | { 8 | "name": "Alexander Lisachenko", 9 | "email": "lisachenko.it@gmail.com" 10 | } 11 | ], 12 | "autoload": { 13 | "psr-4" : { 14 | "Alpari\\" : "src/" 15 | } 16 | }, 17 | "require": { 18 | "php": "~7.1", 19 | "react/promise": "^2.7" 20 | }, 21 | "suggest": { 22 | "ext-openssl": "to able SSL socket streams usage" 23 | }, 24 | "extra": { 25 | "branch-alias": { 26 | "dev-master": "0.11.x-dev" 27 | } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /src/Kafka.php: -------------------------------------------------------------------------------- 1 | [], 25 | Config::CLIENT_ID => 'PHP/Kafka', 26 | Config::STREAM_PERSISTENT_CONNECTION => false, 27 | Config::STREAM_ASYNC_CONNECT => false, 28 | Config::METADATA_MAX_AGE_MS => 300000, 29 | Config::RECEIVE_BUFFER_BYTES => 32768, 30 | Config::SEND_BUFFER_BYTES => 131072, 31 | Config::SECURITY_PROTOCOL => SecurityProtocol::PLAINTEXT, 32 | Config::SSL_PROTOCOL => SslProtocol::TLS, 33 | Config::SSL_CLIENT_CERT_LOCATION => null, 34 | Config::SSL_KEY_PASSWORD => null, 35 | Config::SSL_KEY_LOCATION => null, 36 | 37 | Config::CONNECTIONS_MAX_IDLE_MS => 540000, 38 | Config::REQUEST_TIMEOUT_MS => 30000, 39 | Config::SASL_MECHANISM => 'GSSAPI', 40 | Config::METADATA_FETCH_TIMEOUT_MS => 60000, 41 | Config::RECONNECT_BACKOFF_MS => 50, 42 | Config::RETRY_BACKOFF_MS => 100, 43 | ]; 44 | 45 | /** 46 | * A list of host/port pairs to use for establishing the initial connection to the Kafka cluster. 47 | */ 48 | public const BOOTSTRAP_SERVERS = 'bootstrap.servers'; 49 | 50 | /** 51 | * An id string to pass to the server when making requests. 52 | * 53 | * The purpose of this is to be able to track the source of requests beyond just ip/port by allowing a logical 54 | * application name to be included in server-side request logging. 55 | */ 56 | public const CLIENT_ID = 'client.id'; 57 | 58 | /** 59 | * The configuration controls the maximum amount of time the client will wait for the response of a request. 60 | * 61 | * If the response is not received before the timeout elapses the client will resend the request if necessary or 62 | * fail the request if retries are exhausted. 63 | */ 64 | public const REQUEST_TIMEOUT_MS = 'request.timeout.ms'; 65 | 66 | /** 67 | * Should client use persistent connection to the cluster or not 68 | * 69 | * (PHP Only option) 70 | */ 71 | public const STREAM_PERSISTENT_CONNECTION = 'stream.persistent.connection'; 72 | 73 | /** 74 | * Should client use asynchronous connection to the broker 75 | * 76 | * (PHP Only option) 77 | */ 78 | public const STREAM_ASYNC_CONNECT = 'stream.async.connect'; 79 | 80 | /** 81 | * File name that stores the metadata, this file will be effectively cached by the Opcode cache in production 82 | * 83 | * Note: always use absolute paths, because cwd could change in destructors during batch flush! 84 | * 85 | * (PHP Only option) 86 | */ 87 | public const METADATA_CACHE_FILE = 'metadata.cache.file'; 88 | 89 | /** 90 | * The first time data is sent to the broker we must fetch metadata about that topic to know which servers host the 91 | * topic's partitions. This fetch to succeed before throwing an exception back to the client. 92 | */ 93 | public const METADATA_FETCH_TIMEOUT_MS = 'metadata.fetch.timeout.ms'; 94 | 95 | /** 96 | * The period of time in milliseconds after which we force a refresh of metadata even if we haven't seen any 97 | * partition leadership changes to proactively discover any new brokers or partitions. 98 | * 99 | * Applied only if the metadata.cache.file is configured 100 | */ 101 | public const METADATA_MAX_AGE_MS = 'metadata.max.age.ms'; 102 | 103 | /** 104 | * The size of the TCP send buffer (SO_SNDBUF) to use when sending data. 105 | */ 106 | public const SEND_BUFFER_BYTES = 'send.buffer.bytes'; 107 | 108 | /** 109 | * The size of the TCP receive buffer (SO_RCVBUF) to use when reading data. 110 | */ 111 | public const RECEIVE_BUFFER_BYTES = 'receive.buffer.bytes'; 112 | 113 | /** 114 | * Location of Certificate Authority file on local filesystem which should be used to authenticate 115 | * the identity of the remote peer. 116 | */ 117 | public const SSL_CA_CERT_LOCATION = 'ssl.ca.cert.location'; 118 | 119 | /** 120 | * Protocol used to communicate with brokers. Valid values are: PLAINTEXT, SSL, SASL_PLAINTEXT, SASL_SSL. 121 | * 122 | * Implemented values: PLAINTEXT, SSL 123 | */ 124 | public const SECURITY_PROTOCOL = 'security.protocol'; 125 | 126 | /** 127 | * The SSL protocol used to generate the SSLContext. Default setting is TLS, which is fine for most cases. 128 | * Allowed values are TLS, TLSv1.1 and TLSv1.2. SSL, SSLv2 and SSLv3, but their usage is discouraged due 129 | * to known security vulnerabilities. 130 | */ 131 | public const SSL_PROTOCOL = 'ssl.protocol'; 132 | 133 | /** 134 | * Path to local certificate file on filesystem. It must be a PEM encoded file which contains your 135 | * certificate and private key. It can optionally contain the certificate chain of issuers. 136 | * The private key also may be contained in a separate file specified by SSL_KEY_LOCATION. 137 | * 138 | * (PHP Only option) 139 | */ 140 | public const SSL_CLIENT_CERT_LOCATION = 'ssl.client.cert.location'; 141 | 142 | /** 143 | * The location of the private key file. This is optional for client and can be used for two-way 144 | * authentication for client. 145 | */ 146 | public const SSL_KEY_LOCATION = 'ssl.key.location'; 147 | 148 | /** 149 | * The password of the private key. This is optional for client. 150 | */ 151 | public const SSL_KEY_PASSWORD = 'ssl.key.password'; 152 | 153 | public const CONNECTIONS_MAX_IDLE_MS = 'connections.max.idle.ms'; 154 | public const SASL_MECHANISM = 'sasl.mechanism'; 155 | public const SSL_ENABLED_PROTOCOLS = 'ssl.enabled.protocols'; 156 | public const RECONNECT_BACKOFF_MS = 'reconnect.backoff.ms'; 157 | public const RETRY_BACKOFF_MS = 'retry.backoff.ms'; 158 | 159 | /** 160 | * Returns default configuration 161 | */ 162 | public static function getDefaultConfiguration(): array 163 | { 164 | return self::$generalConfiguration; 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /src/Kafka/Common/Node.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT32, 65 | 'host' => Scheme::TYPE_STRING, 66 | 'port' => Scheme::TYPE_INT32, 67 | 'rack' => Scheme::TYPE_NULLABLE_STRING, 68 | ]; 69 | } 70 | 71 | /** 72 | * Returns a connection to this node. 73 | * 74 | * @param array $configuration Client configuration 75 | * @todo Move this method outside this class 76 | * 77 | * @return Stream 78 | */ 79 | public function getConnection(array $configuration): Stream 80 | { 81 | if (!isset(self::$nodeConnections[$this->host][$this->port])) { 82 | $connection = new Stream\SocketStream("tcp://{$this->host}:{$this->port}", $configuration); 83 | 84 | self::$nodeConnections[$this->host][$this->port] = $connection; 85 | } 86 | 87 | return self::$nodeConnections[$this->host][$this->port]; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/Kafka/Common/PartitionMetadata.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT16, 65 | 'partitionId' => Scheme::TYPE_INT32, 66 | 'leader' => Scheme::TYPE_INT32, 67 | 'replicas' => [Scheme::TYPE_INT32], 68 | 'isr' => [Scheme::TYPE_INT32] 69 | ]; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/Kafka/Common/RestorableTrait.php: -------------------------------------------------------------------------------- 1 | $value) { 28 | $self->$key = $value; 29 | } 30 | 31 | return $self; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Kafka/Common/TopicMetadata.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT16, 59 | 'topic' => Scheme::TYPE_STRING, 60 | 'isInternal' => Scheme::TYPE_INT8, 61 | 'partitions' => [PartitionMetadata::class] 62 | ]; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Kafka/Common/Utils/ByteUtils.php: -------------------------------------------------------------------------------- 1 | > 63); 97 | $bytes = 1; 98 | while (($v & 0xffffffffffffff80) !== 0) { 99 | ++$bytes; 100 | $v >>= 7; 101 | } 102 | 103 | return $bytes; 104 | } 105 | 106 | /** 107 | * Return number of bytes needed to encode an integer in variable-length format. 108 | */ 109 | public static function sizeOfVarint(int $value): int 110 | { 111 | $v = ($value << 1) ^ ($value >> 31); 112 | $bytes = 1; 113 | while (($v & 0xffffff80) !== 0) { 114 | ++$bytes; 115 | $v >>= 7; 116 | } 117 | return $bytes; 118 | } 119 | 120 | /** 121 | * Performs ZigZag encoding of integer value in specific base (32 or 64) 122 | */ 123 | public static function encodeZigZag(int $value, int $base = 32): int 124 | { 125 | $value = ($value << 1) ^ ($value >> $base-1); 126 | 127 | return $value; 128 | } 129 | 130 | /** 131 | * Decodes ZigZag-encoded value 132 | */ 133 | public static function decodeZigZag(int $value): int 134 | { 135 | $value = ($value >> 1) ^ (-($value & 1)); 136 | 137 | return $value; 138 | } 139 | 140 | /** 141 | * Compute CRC-32C checksum of the data as in rfc3720 section B.4. 142 | */ 143 | public static function crc32c(string $buffer, int $initial = 0): int 144 | { 145 | $length = strlen($buffer); 146 | $crc = $initial ^ self::CRC_MASK; 147 | 148 | for ($index = 0; $index < $length; $index++) { 149 | $tableIndex = ($crc ^ ord($buffer[$index])) & 0xff; 150 | $crc = (self::CRC_TABLE[$tableIndex] ^ ($crc >> 8)) & self::CRC_MASK; 151 | } 152 | 153 | return $crc ^ self::CRC_MASK; 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/Kafka/Consumer/MemberAssignment.php: -------------------------------------------------------------------------------- 1 | Version PartitionAssignment 24 | * Version => int16 25 | * PartitionAssignment => [Topic [Partition]] 26 | * Topic => string 27 | * Partition => int32 28 | * UserData => bytes 29 | */ 30 | class MemberAssignment implements BinarySchemeInterface 31 | { 32 | /** 33 | * This is a version id. 34 | */ 35 | public $version; 36 | 37 | /** 38 | * This property holds assignments of topic partitions for member. 39 | * 40 | * @var TopicPartitions[] 41 | */ 42 | public $topicPartitions = []; 43 | 44 | /** 45 | * The UserData field can be used by custom partition assignment strategies. 46 | * 47 | * For example, in a sticky partitioning implementation, this field can contain the assignment from the previous 48 | * generation. In a resource-based assignment strategy, it could include the number of cpus on the machine hosting 49 | * each consumer instance. 50 | */ 51 | public $userData; 52 | 53 | /** 54 | * MemberAssignment constructor. 55 | * 56 | * @param array|int[][] $topicPartitions Partition assignments per topic 57 | * @param int $version Optional version 58 | * @param string $userData Additional user data 59 | */ 60 | public function __construct(array $topicPartitions = [], int $version = 0, string $userData = '') 61 | { 62 | $packedTopicAssignment = []; 63 | foreach ($topicPartitions as $topic => $partitions) { 64 | $packedTopicAssignment[$topic] = new TopicPartitions($topic, $partitions); 65 | } 66 | 67 | $this->topicPartitions = $packedTopicAssignment; 68 | $this->version = $version; 69 | $this->userData = $userData; 70 | } 71 | 72 | /** 73 | * Returns definition of binary packet for the class or object 74 | * 75 | * @return array 76 | */ 77 | public static function getScheme(): array 78 | { 79 | return [ 80 | 'version' => Scheme::TYPE_INT16, 81 | 'topicPartitions' => ['topic' => TopicPartitions::class], 82 | 'userData' => Scheme::TYPE_BYTEARRAY, 83 | ]; 84 | } 85 | } 86 | -------------------------------------------------------------------------------- /src/Kafka/Consumer/OffsetResetStrategy.php: -------------------------------------------------------------------------------- 1 | $subscriptionData) { 52 | $stringMetadata = new StringStream($subscriptionData->metadata); 53 | $subscriptionMetadata = Scheme::readObjectFromStream(Subscription::class, $stringMetadata); 54 | foreach ($subscriptionMetadata->topics as $topic) { 55 | $topicMembers[$topic][] = $memberId; 56 | } 57 | } 58 | 59 | $requestedTopics = array_keys($topicMembers); 60 | foreach ($requestedTopics as $requestedTopic) { 61 | $partitions = $metadata->partitionsForTopic($requestedTopic); 62 | $totalConsumers = count($topicMembers[$requestedTopic]); 63 | foreach ($partitions as $index=>$partition) { 64 | $memberIndex = $index % $totalConsumers; 65 | $memberId = $topicMembers[$requestedTopic][$memberIndex]; 66 | $partitionAssignments[$memberId][$requestedTopic][$partition->partitionId] = $partition->partitionId; 67 | } 68 | } 69 | 70 | $result = []; 71 | foreach ($partitionAssignments as $memberId => $partitionAssignment) { 72 | $result[$memberId] = new MemberAssignment($partitionAssignment); 73 | } 74 | $unassignedMembers = array_diff_key($subscriptions, $result); 75 | foreach (array_keys($unassignedMembers) as $unassignedMemberId) { 76 | $result[$unassignedMemberId] = new MemberAssignment(); 77 | } 78 | 79 | return $result; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/Kafka/Consumer/Subscription.php: -------------------------------------------------------------------------------- 1 | Version Subscription UserData 23 | * Version => int16 24 | * Subscription => [Topic] 25 | * Topic => string 26 | * UserData => bytes 27 | */ 28 | class Subscription implements BinarySchemeInterface 29 | { 30 | 31 | /** 32 | * This is a version id. 33 | */ 34 | public $version; 35 | 36 | /** 37 | * This property holds all the topics for the consumer. 38 | */ 39 | public $topics; 40 | 41 | /** 42 | * The UserData field can be used by custom partition assignment strategies. 43 | * 44 | * For example, in a sticky partitioning implementation, this field can contain the assignment from the previous 45 | * generation. In a resource-based assignment strategy, it could include the number of cpus on the machine hosting 46 | * each consumer instance. 47 | */ 48 | public $userData; 49 | 50 | /** 51 | * Subscription constructor. 52 | * 53 | * @param string[] $topics List of topics 54 | */ 55 | public function __construct(array $topics, int $version = 0, string $userData = '') 56 | { 57 | $this->topics = $topics; 58 | $this->version = $version; 59 | $this->userData = $userData; 60 | } 61 | 62 | /** 63 | * Returns definition of binary packet for the class or object 64 | * 65 | * @return array 66 | */ 67 | public static function getScheme(): array 68 | { 69 | return [ 70 | 'version' => Scheme::TYPE_INT16, 71 | 'topics' => [Scheme::TYPE_STRING], 72 | 'userData' => Scheme::TYPE_BYTEARRAY 73 | ]; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ApiVersionsResponseMetadata.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT16, 52 | 'minVersion' => Scheme::TYPE_INT16, 53 | 'maxVersion' => Scheme::TYPE_INT16, 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ControlledShutdownResponsePartition.php: -------------------------------------------------------------------------------- 1 | partition timestamp 23 | * topic => STRING 24 | * partition => INT32 25 | */ 26 | class ControlledShutdownResponsePartition implements BinarySchemeInterface 27 | { 28 | /** 29 | * Name of topic 30 | * 31 | * @var string 32 | */ 33 | public $topic; 34 | 35 | /** 36 | * Topic partition id 37 | * 38 | * @var integer 39 | */ 40 | public $partition; 41 | 42 | /** 43 | * @inheritdoc 44 | */ 45 | public static function getScheme(): array 46 | { 47 | return [ 48 | 'partition' => Scheme::TYPE_INT32, 49 | 'timestamp' => Scheme::TYPE_INT64 50 | ]; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Kafka/DTO/DescribeGroupResponseMember.php: -------------------------------------------------------------------------------- 1 | member_id client_id client_host member_metadata member_assignment 23 | * member_id => STRING 24 | * client_id => STRING 25 | * client_host => STRING 26 | * member_metadata => BYTES 27 | * member_assignment => BYTES 28 | */ 29 | class DescribeGroupResponseMember implements BinarySchemeInterface 30 | { 31 | /** 32 | * The memberId assigned by the coordinator 33 | * 34 | * @var string 35 | */ 36 | public $memberId; 37 | 38 | /** 39 | * The client id used in the member's latest join group request 40 | * 41 | * @var string 42 | */ 43 | public $clientId; 44 | 45 | /** 46 | * The client host used in the request session corresponding to the member's join group. 47 | * 48 | * @var string 49 | */ 50 | public $clientHost; 51 | 52 | /** 53 | * The metadata corresponding to the current group protocol in use (will only be present if the group is stable). 54 | * 55 | * @var string Binary data 56 | */ 57 | public $memberMetadata; 58 | 59 | /** 60 | * The current assignment provided by the group leader (will only be present if the group is stable). 61 | * 62 | * @var string 63 | */ 64 | public $memberAssignment; 65 | 66 | /** 67 | * @inheritdoc 68 | */ 69 | public static function getScheme(): array 70 | { 71 | return [ 72 | 'memberId' => Scheme::TYPE_STRING, 73 | 'clientId' => Scheme::TYPE_STRING, 74 | 'clientHost' => Scheme::TYPE_STRING, 75 | 'memberMetadata' => Scheme::TYPE_BYTEARRAY, 76 | 'memberAssignment' => Scheme::TYPE_BYTEARRAY, 77 | ]; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/Kafka/DTO/DescribeGroupResponseMetadata.php: -------------------------------------------------------------------------------- 1 | error_code group_id state protocol_type protocol [members] 23 | * error_code => INT16 24 | * group_id => STRING 25 | * state => STRING 26 | * protocol_type => STRING 27 | * protocol => STRING 28 | * members => member_id client_id client_host member_metadata member_assignment 29 | */ 30 | class DescribeGroupResponseMetadata implements BinarySchemeInterface 31 | { 32 | 33 | /** 34 | * Error code for the group 35 | * 36 | * @var integer 37 | */ 38 | public $errorCode; 39 | 40 | /** 41 | * Name of the group 42 | * 43 | * @var string 44 | */ 45 | public $groupId; 46 | 47 | /** 48 | * The current state of the group 49 | * (one of: Dead, Stable, AwaitingSync, or PreparingRebalance, or empty if there is no active group) 50 | * 51 | * @var string 52 | */ 53 | public $state; 54 | 55 | /** 56 | * The current group protocol type (will be empty if there is no active group) 57 | * 58 | * @var string 59 | */ 60 | public $protocolType; 61 | 62 | /** 63 | * The current group protocol (only provided if the group is Stable) 64 | * 65 | * @var string 66 | */ 67 | public $protocol; 68 | 69 | /** 70 | * Current group members (only provided if the group is not Dead) 71 | * 72 | * @var DescribeGroupResponseMember[] 73 | */ 74 | public $members = []; 75 | 76 | /** 77 | * @inheritdoc 78 | */ 79 | public static function getScheme(): array 80 | { 81 | return [ 82 | 'errorCode' => Scheme::TYPE_INT16, 83 | 'groupId' => Scheme::TYPE_STRING, 84 | 'state' => Scheme::TYPE_STRING, 85 | 'protocolType' => Scheme::TYPE_STRING, 86 | 'protocol' => Scheme::TYPE_STRING, 87 | 'members' => ['memberId' => DescribeGroupResponseMember::class] 88 | ]; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/Kafka/DTO/FetchRequestTopic.php: -------------------------------------------------------------------------------- 1 | topic [partitions] 23 | * topic => STRING 24 | * partitions => partition fetch_offset max_bytes 25 | * partition => INT32 26 | * fetch_offset => INT64 27 | * max_bytes => INT32 28 | */ 29 | class FetchRequestTopic implements BinarySchemeInterface 30 | { 31 | /** 32 | * Name of the topic for fetching 33 | */ 34 | public $topic; 35 | 36 | /** 37 | * Details about fetching for each topic's partition 38 | * 39 | * @var FetchRequestTopicPartition[] 40 | */ 41 | public $partitions; 42 | 43 | public function __construct(string $topic, array $partitions = []) 44 | { 45 | $this->topic = $topic; 46 | $this->partitions = $partitions; 47 | } 48 | 49 | /** 50 | * @inheritdoc 51 | */ 52 | public static function getScheme(): array 53 | { 54 | return [ 55 | 'topic' => Scheme::TYPE_STRING, 56 | 'partitions' => ['partition' => FetchRequestTopicPartition::class] 57 | ]; 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /src/Kafka/DTO/FetchRequestTopicPartition.php: -------------------------------------------------------------------------------- 1 | partition fetch_offset log_start_offset max_bytes 23 | * partition => INT32 24 | * fetch_offset => INT64 25 | * log_start_offset => INT64 (Since FetchRequest v5) 26 | * max_bytes => INT32 27 | */ 28 | class FetchRequestTopicPartition implements BinarySchemeInterface 29 | { 30 | /** 31 | * Topic partition id 32 | */ 33 | public $partition; 34 | 35 | /** 36 | * Message offset. 37 | */ 38 | public $fetchOffset; 39 | 40 | /** 41 | * Earliest available offset of the follower replica. 42 | * 43 | * The field is only used when request is sent by follower. 44 | * 45 | * @since 0.11.0.0 Kafka 46 | */ 47 | public $logStartOffset; 48 | 49 | /** 50 | * Maximum bytes to fetch. 51 | */ 52 | public $maxBytes; 53 | 54 | public function __construct(int $partition, int $fetchOffset, int $maxBytes, int $logStartOffset = -1) 55 | { 56 | $this->partition = $partition; 57 | $this->fetchOffset = $fetchOffset; 58 | $this->logStartOffset = $logStartOffset; 59 | $this->maxBytes = $maxBytes; 60 | } 61 | 62 | /** 63 | * @inheritdoc 64 | */ 65 | public static function getScheme(): array 66 | { 67 | return [ 68 | 'partition' => Scheme::TYPE_INT32, 69 | 'fetchOffset' => Scheme::TYPE_INT64, 70 | 'logStartOffset' => Scheme::TYPE_INT64, 71 | 'maxBytes' => Scheme::TYPE_INT32 72 | ]; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/Kafka/DTO/FetchResponseAbortedTransaction.php: -------------------------------------------------------------------------------- 1 | producer_id first_offset 23 | * producer_id => INT64 24 | * first_offset => INT64 25 | * 26 | * @since Version 4 27 | */ 28 | class FetchResponseAbortedTransaction implements BinarySchemeInterface 29 | { 30 | /** 31 | * The producer id associated with the aborted transactions 32 | * 33 | * @var integer 34 | */ 35 | public $producerId; 36 | 37 | /** 38 | * The first offset in the aborted transaction 39 | * 40 | * @var integer 41 | */ 42 | public $firstOffset; 43 | 44 | /** 45 | * @inheritdoc 46 | */ 47 | public static function getScheme(): array 48 | { 49 | return [ 50 | 'producerId' => Scheme::TYPE_INT64, 51 | 'firstOffset' => Scheme::TYPE_INT64 52 | ]; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Kafka/DTO/FetchResponsePartition.php: -------------------------------------------------------------------------------- 1 | partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] 24 | * partition => INT32 25 | * error_code => INT16 26 | * high_watermark => INT64 27 | * last_stable_offset => INT64 28 | * log_start_offset => INT64 29 | * aborted_transactions => producer_id first_offset 30 | * producer_id => INT64 31 | * first_offset => INT64 32 | */ 33 | class FetchResponsePartition implements BinarySchemeInterface 34 | { 35 | /** 36 | * The id of the partition this response is for. 37 | * 38 | * @var integer 39 | */ 40 | public $partition; 41 | 42 | /** 43 | * The error from this partition, if any. 44 | * 45 | * Errors are given on a per-partition basis because a given partition may be unavailable or maintained on a 46 | * different host, while others may have successfully accepted the produce request. 47 | * 48 | * @var integer 49 | */ 50 | public $errorCode; 51 | 52 | /** 53 | * The offset at the end of the log for this partition. This can be used by the client to determine how many 54 | * messages behind the end of the log they are. 55 | * 56 | * @var integer 57 | */ 58 | public $highWaterMarkOffset; 59 | 60 | /** 61 | * The last stable offset (or LSO) of the partition. 62 | * 63 | * This is the last offset such that the state of all transactional records prior to this offset have been decided 64 | * (ABORTED or COMMITTED) 65 | * 66 | * @since version 4 67 | * 68 | * @var integer 69 | */ 70 | public $lastStableOffset; 71 | 72 | /** 73 | * Earliest available offset. 74 | * 75 | * @since version 5 76 | * 77 | * @var integer 78 | */ 79 | public $logStartOffset; 80 | 81 | /** 82 | * List of aborted transactions 83 | * 84 | * @since version 4 85 | * 86 | * @var FetchResponseAbortedTransaction[] 87 | */ 88 | public $abortedTransactions = []; 89 | 90 | /** 91 | * @var string 92 | */ 93 | public $recordBatchBuffer; 94 | 95 | /** 96 | * @inheritdoc 97 | */ 98 | public static function getScheme(): array 99 | { 100 | return [ 101 | 'partition' => Scheme::TYPE_INT32, 102 | 'errorCode' => Scheme::TYPE_INT16, 103 | 'highWaterMarkOffset' => Scheme::TYPE_INT64, 104 | 'lastStableOffset' => Scheme::TYPE_INT64, 105 | 'logStartOffset' => Scheme::TYPE_INT64, 106 | 'abortedTransactions' => ['producerId' => FetchResponseAbortedTransaction::class], 107 | // TODO: this should be actualy dynamic array of RecordBatch::class entities 108 | 'recordBatchBuffer' => Scheme::TYPE_BYTEARRAY 109 | ]; 110 | } 111 | 112 | /** 113 | * Returns collection of RecordBatches 114 | * 115 | * TODO: is this possible somehow to do this on Scheme level? 116 | * @return RecordBatch[] 117 | */ 118 | public function getRecordBatches(): array 119 | { 120 | $recordBatches = []; 121 | // TODO: Avoid creation of temporary string buffer, this should be implemented in reader directly 122 | $buffer = new StringStream($this->recordBatchBuffer); 123 | while (!$buffer->isEmpty()) { 124 | $recordBatches[] = Scheme::readObjectFromStream(RecordBatch::class, $buffer); 125 | } 126 | 127 | return $recordBatches; 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/Kafka/DTO/FetchResponseTopic.php: -------------------------------------------------------------------------------- 1 | topic [partition_responses] 23 | * topic => STRING 24 | * partition_responses => partition_header record_set 25 | * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] 26 | * partition => INT32 27 | * error_code => INT16 28 | * high_watermark => INT64 29 | * last_stable_offset => INT64 30 | * log_start_offset => INT64 31 | * aborted_transactions => producer_id first_offset 32 | * producer_id => INT64 33 | * first_offset => INT64 34 | * record_set => RECORDS 35 | */ 36 | class FetchResponseTopic implements BinarySchemeInterface 37 | { 38 | /** 39 | * Name of the topic for fetching 40 | * 41 | * @var string 42 | */ 43 | public $topic; 44 | 45 | /** 46 | * Details about fetching for each topic's partition 47 | * 48 | * @var FetchResponsePartition[] 49 | */ 50 | public $partitions; 51 | 52 | /** 53 | * @inheritdoc 54 | */ 55 | public static function getScheme(): array 56 | { 57 | return [ 58 | 'topic' => Scheme::TYPE_STRING, 59 | 'partitions' => ['partition' => FetchResponsePartition::class] 60 | ]; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Kafka/DTO/GroupCoordinatorResponseMetadata.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT32, 52 | 'host' => Scheme::TYPE_STRING, 53 | 'port' => Scheme::TYPE_INT32 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Kafka/DTO/Header.php: -------------------------------------------------------------------------------- 1 | HeaderKey HeaderVal 23 | * HeaderKeyLen => varint 24 | * HeaderKey => string 25 | * HeaderValueLen => varint 26 | * HeaderValue => data 27 | * 28 | * @since 0.11.0 29 | * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-82+-+Add+Record+Headers 30 | */ 31 | class Header implements BinarySchemeInterface 32 | { 33 | /** 34 | * Item key name 35 | */ 36 | public $key; 37 | 38 | /** 39 | * Item arbitrary data 40 | */ 41 | public $value; 42 | 43 | /** 44 | * Header constructor. 45 | * 46 | * @param string $key Item key name 47 | * @param string $value Item arbitrary data 48 | */ 49 | public function __construct(string $key = '', string $value = '') 50 | { 51 | $this->key = $key; 52 | $this->value = $value; 53 | } 54 | 55 | /** 56 | * @inheritdoc 57 | */ 58 | public static function getScheme(): array 59 | { 60 | return [ 61 | 'key' => Scheme::TYPE_VARCHAR_ZIGZAG, 62 | 'value' => Scheme::TYPE_VARCHAR_ZIGZAG, 63 | ]; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Kafka/DTO/JoinGroupRequestProtocol.php: -------------------------------------------------------------------------------- 1 | protocol_name protocol_metadata 25 | * protocol_name => STRING 26 | * protocol_metadata => BYTES 27 | */ 28 | class JoinGroupRequestProtocol implements BinarySchemeInterface 29 | { 30 | /** 31 | * Name of the protocol 32 | * 33 | * @var string 34 | */ 35 | public $name; 36 | 37 | /** 38 | * Alpari-specific metadata 39 | * 40 | * @todo Update scheme to use Subscription instance directly 41 | * @var string 42 | */ 43 | public $metadata; 44 | 45 | public function __construct(string $name, Subscription $subscription) 46 | { 47 | // TODO: This should be on scheme-level 48 | $stringStream = new StringStream(); 49 | Scheme::writeObjectToStream($subscription, $stringStream); 50 | 51 | $this->name = $name; 52 | $this->metadata = $stringStream->getBuffer(); 53 | } 54 | 55 | /** 56 | * @inheritdoc 57 | */ 58 | public static function getScheme(): array 59 | { 60 | return [ 61 | 'name' => Scheme::TYPE_STRING, 62 | 'metadata' => Scheme::TYPE_BYTEARRAY 63 | ]; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Kafka/DTO/JoinGroupResponseMember.php: -------------------------------------------------------------------------------- 1 | member_id member_metadata 23 | * member_id => STRING 24 | * member_metadata => BYTES 25 | */ 26 | class JoinGroupResponseMember implements BinarySchemeInterface 27 | { 28 | /** 29 | * Name of the group member 30 | */ 31 | public $memberId; 32 | 33 | /** 34 | * Member-specific metadata 35 | */ 36 | public $metadata; 37 | 38 | public function __construct(string $memberId, string $metadata) 39 | { 40 | $this->memberId = $memberId; 41 | $this->metadata = $metadata; 42 | } 43 | 44 | /** 45 | * @inheritdoc 46 | */ 47 | public static function getScheme(): array 48 | { 49 | return [ 50 | 'memberId' => Scheme::TYPE_STRING, 51 | 'metadata' => Scheme::TYPE_BYTEARRAY 52 | ]; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ListGroupResponseProtocol.php: -------------------------------------------------------------------------------- 1 | group_id protocol_type 23 | * group_id => STRING 24 | * protocol_type => STRING 25 | */ 26 | class ListGroupResponseProtocol implements BinarySchemeInterface 27 | { 28 | /** 29 | * The unique group identifier 30 | * 31 | * @var string 32 | */ 33 | public $groupId; 34 | 35 | /** 36 | * Supported protocol type 37 | * 38 | * @var string 39 | */ 40 | public $protocolType; 41 | 42 | /** 43 | * @inheritdoc 44 | */ 45 | public static function getScheme(): array 46 | { 47 | return [ 48 | 'groupId' => Scheme::TYPE_STRING, 49 | 'protocolType' => Scheme::TYPE_STRING 50 | ]; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetCommitRequestPartition.php: -------------------------------------------------------------------------------- 1 | partition offset metadata 23 | * partition => INT32 24 | * offset => INT64 25 | * metadata => NULLABLE_STRING 26 | */ 27 | class OffsetCommitRequestPartition implements BinarySchemeInterface 28 | { 29 | /** 30 | * The partition this request entry corresponds to. 31 | */ 32 | public $partition; 33 | 34 | /** 35 | * The offset assigned to the first message in the message set appended to this partition. 36 | */ 37 | public $offset; 38 | 39 | /** 40 | * Any associated metadata the client wants to keep. 41 | */ 42 | public $metadata; 43 | 44 | public function __construct(int $partition, int $offset, ?string $metadata = null) 45 | { 46 | $this->partition = $partition; 47 | $this->offset = $offset; 48 | $this->metadata = $metadata; 49 | } 50 | 51 | /** 52 | * @inheritdoc 53 | */ 54 | public static function getScheme(): array 55 | { 56 | return [ 57 | 'partition' => Scheme::TYPE_INT32, 58 | 'offset' => Scheme::TYPE_INT64, 59 | 'metadata' => Scheme::TYPE_NULLABLE_STRING 60 | ]; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetCommitRequestTopic.php: -------------------------------------------------------------------------------- 1 | topic [partitions] 23 | * topic => STRING 24 | * partitions => partition offset metadata 25 | * partition => INT32 26 | * offset => INT64 27 | * metadata => NULLABLE_STRING 28 | */ 29 | class OffsetCommitRequestTopic implements BinarySchemeInterface 30 | { 31 | /** 32 | * Name of the topic 33 | */ 34 | public $topic; 35 | 36 | /** 37 | * Partitions to commit offset. 38 | * 39 | * @var OffsetCommitRequestPartition[] 40 | */ 41 | public $partitions; 42 | 43 | public function __construct(string $topic, array $partitions) 44 | { 45 | $packedPartitions = []; 46 | $this->topic = $topic; 47 | foreach ($partitions as $partition => $timestamp) { 48 | $packedPartitions[$partition] = new OffsetCommitRequestPartition($partition, $timestamp); 49 | } 50 | $this->partitions = $packedPartitions; 51 | } 52 | 53 | /** 54 | * @inheritdoc 55 | */ 56 | public static function getScheme(): array 57 | { 58 | return [ 59 | 'topic' => Scheme::TYPE_STRING, 60 | 'partitions' => ['partition' => OffsetCommitRequestPartition::class], 61 | ]; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetCommitResponsePartition.php: -------------------------------------------------------------------------------- 1 | partition error_code 23 | * partition => INT32 24 | * error_code => INT16 25 | */ 26 | class OffsetCommitResponsePartition implements BinarySchemeInterface 27 | { 28 | /** 29 | * The partition this request entry corresponds to. 30 | * 31 | * @var integer 32 | */ 33 | public $partition; 34 | 35 | /** 36 | * The error from this partition, if any. 37 | * 38 | * @var integer 39 | */ 40 | public $errorCode; 41 | 42 | /** 43 | * @inheritdoc 44 | */ 45 | public static function getScheme(): array 46 | { 47 | return [ 48 | 'partition' => Scheme::TYPE_INT32, 49 | 'errorCode' => Scheme::TYPE_INT16 50 | ]; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetCommitResponseTopic.php: -------------------------------------------------------------------------------- 1 | topic [partition_responses] 23 | * topic => STRING 24 | * partition_responses => partition error_code 25 | * partition => INT32 26 | * error_code => INT16 27 | */ 28 | class OffsetCommitResponseTopic implements BinarySchemeInterface 29 | { 30 | /** 31 | * Name of the topic 32 | * 33 | * @var string 34 | */ 35 | public $topic; 36 | 37 | /** 38 | * Result for offset committing by each topic-partition 39 | * 40 | * @var OffsetCommitResponsePartition[] 41 | */ 42 | public $partitions; 43 | 44 | /** 45 | * @inheritdoc 46 | */ 47 | public static function getScheme(): array 48 | { 49 | return [ 50 | 'topic' => Scheme::TYPE_STRING, 51 | 'partitions' => ['partition' => OffsetCommitResponsePartition::class], 52 | ]; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetFetchResponsePartition.php: -------------------------------------------------------------------------------- 1 | partition offset metadata error_code 23 | * partition => INT32 24 | * offset => INT64 25 | * metadata => NULLABLE_STRING 26 | * error_code => INT16 27 | */ 28 | class OffsetFetchResponsePartition implements BinarySchemeInterface 29 | { 30 | /** 31 | * The partition this response entry corresponds to. 32 | * 33 | * @var integer 34 | */ 35 | public $partition; 36 | 37 | /** 38 | * The offset assigned to the first message in the message set appended to this partition. 39 | * 40 | * @var integer 41 | */ 42 | public $offset; 43 | 44 | /** 45 | * Any associated metadata the client wants to keep. 46 | * 47 | * @var string 48 | */ 49 | public $metadata; 50 | 51 | /** 52 | * The error from this partition, if any. 53 | * 54 | * Errors are given on a per-partition basis because a given partition may be unavailable or maintained on a 55 | * different host, while others may have successfully accepted the produce request. 56 | * 57 | * @var integer 58 | */ 59 | public $errorCode; 60 | 61 | /** 62 | * @inheritdoc 63 | */ 64 | public static function getScheme(): array 65 | { 66 | return [ 67 | 'partition' => Scheme::TYPE_INT32, 68 | 'offset' => Scheme::TYPE_INT64, 69 | 'metadata' => Scheme::TYPE_NULLABLE_STRING, 70 | 'errorCode' => Scheme::TYPE_INT16 71 | ]; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetFetchResponseTopic.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_STRING, 45 | 'partitions' => ['partition' => OffsetFetchResponsePartition::class], 46 | ]; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetsRequestPartition.php: -------------------------------------------------------------------------------- 1 | partition timestamp 23 | * partition => INT32 24 | * timestamp => INT64 25 | */ 26 | class OffsetsRequestPartition implements BinarySchemeInterface 27 | { 28 | /** 29 | * Topic partition id 30 | */ 31 | public $partition; 32 | 33 | /** 34 | * The target timestamp for the partition. 35 | */ 36 | public $timestamp; 37 | 38 | /** 39 | * OffsetsRequestPartition constructor. 40 | * 41 | * @param int $partition Topic partition id 42 | * @param int $timestamp The target timestamp for the partition. 43 | */ 44 | public function __construct(int $partition, int $timestamp) 45 | { 46 | $this->partition = $partition; 47 | $this->timestamp = $timestamp; 48 | } 49 | 50 | /** 51 | * @inheritdoc 52 | */ 53 | public static function getScheme(): array 54 | { 55 | return [ 56 | 'partition' => Scheme::TYPE_INT32, 57 | 'timestamp' => Scheme::TYPE_INT64 58 | ]; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetsRequestTopic.php: -------------------------------------------------------------------------------- 1 | topic [partitions] 23 | * topic => STRING 24 | * partitions => partition timestamp 25 | * partition => INT32 26 | * timestamp => INT64 27 | */ 28 | class OffsetsRequestTopic implements BinarySchemeInterface 29 | { 30 | /** 31 | * Name of the topic 32 | */ 33 | public $topic; 34 | 35 | /** 36 | * Partitions to list offset. 37 | * 38 | * @var OffsetsRequestPartition[] 39 | */ 40 | public $partitions; 41 | 42 | public function __construct(string $topic, array $partitions) 43 | { 44 | $packedPartitions = []; 45 | $this->topic = $topic; 46 | foreach ($partitions as $partition => $timestamp) { 47 | $packedPartitions[$partition] = new OffsetsRequestPartition($partition, $timestamp); 48 | } 49 | $this->partitions = $packedPartitions; 50 | } 51 | 52 | /** 53 | * @inheritdoc 54 | */ 55 | public static function getScheme(): array 56 | { 57 | return [ 58 | 'topic' => Scheme::TYPE_STRING, 59 | 'partitions' => ['partition' => OffsetsRequestPartition::class], 60 | ]; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetsResponsePartition.php: -------------------------------------------------------------------------------- 1 | partition error_code timestamp offset 23 | * partition => INT32 24 | * error_code => INT16 25 | * timestamp => INT64 26 | * offset => INT64 27 | */ 28 | class OffsetsResponsePartition implements BinarySchemeInterface 29 | { 30 | /** 31 | * The partition this response entry corresponds to. 32 | * 33 | * @var integer 34 | */ 35 | public $partition; 36 | 37 | /** 38 | * The error from this partition, if any. 39 | * 40 | * Errors are given on a per-partition basis because a given partition may be unavailable or maintained on a 41 | * different host, while others may have successfully accepted the produce request. 42 | * 43 | * @var integer 44 | */ 45 | public $errorCode; 46 | 47 | /** 48 | * The timestamp associated with the returned offset 49 | * 50 | * @since 0.10.1 51 | * 52 | * @var integer 53 | */ 54 | public $timestamp; 55 | 56 | /** 57 | * Found offset 58 | * 59 | * @var integer 60 | */ 61 | public $offset; 62 | 63 | /** 64 | * @inheritdoc 65 | */ 66 | public static function getScheme(): array 67 | { 68 | return [ 69 | 'partition' => Scheme::TYPE_INT32, 70 | 'errorCode' => Scheme::TYPE_INT16, 71 | 'timestamp' => Scheme::TYPE_INT64, 72 | 'offset' => Scheme::TYPE_INT64, 73 | ]; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/Kafka/DTO/OffsetsResponseTopic.php: -------------------------------------------------------------------------------- 1 | topic [partition_responses] 23 | * topic => STRING 24 | * partition_responses => partition error_code timestamp offset 25 | * partition => INT32 26 | * error_code => INT16 27 | * timestamp => INT64 28 | * offset => INT64 29 | */ 30 | class OffsetsResponseTopic implements BinarySchemeInterface 31 | { 32 | /** 33 | * Name of the topic 34 | * 35 | * @var string 36 | */ 37 | public $topic; 38 | 39 | /** 40 | * Partition responses. 41 | * 42 | * @var OffsetsResponsePartition[] 43 | */ 44 | public $partitions; 45 | 46 | /** 47 | * @inheritdoc 48 | */ 49 | public static function getScheme(): array 50 | { 51 | return [ 52 | 'topic' => Scheme::TYPE_STRING, 53 | 'partitions' => ['partition' => OffsetsResponsePartition::class], 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ProduceRequestPartition.php: -------------------------------------------------------------------------------- 1 | partition = $partition; 46 | $recordBatch = $recordBatch ?? new RecordBatch(); 47 | 48 | $recordBatchStream = new StringStream(); 49 | Scheme::writeObjectToStream($recordBatch, $recordBatchStream); 50 | $recordBatchBuffer = $recordBatchStream->getBuffer(); 51 | 52 | // TODO: Calculation of CRC should be in the RecordBatch, but here we can work with raw buffer in one place 53 | $prefix = substr($recordBatchBuffer, 0, 17); // firstOffset..magic fields 54 | $body = substr($recordBatchBuffer, 21); 55 | $crc32c = ByteUtils::crc32c($body); 56 | 57 | $recordBatch->crc = $crc32c; 58 | $this->recordBatch = $prefix . pack('N', $crc32c) . $body; 59 | } 60 | 61 | /** 62 | * @inheritdoc 63 | */ 64 | public static function getScheme(): array 65 | { 66 | return [ 67 | 'partition' => Scheme::TYPE_INT32, 68 | 'recordBatch' => Scheme::TYPE_BYTEARRAY 69 | ]; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ProduceRequestTopic.php: -------------------------------------------------------------------------------- 1 | topic = $topic; 42 | $this->partitions = $partitionData; 43 | } 44 | 45 | /** 46 | * @inheritdoc 47 | */ 48 | public static function getScheme(): array 49 | { 50 | return [ 51 | 'topic' => Scheme::TYPE_STRING, 52 | 'partitions' => ['partition' => ProduceRequestPartition::class] 53 | ]; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ProduceResponsePartition.php: -------------------------------------------------------------------------------- 1 | partition error_code base_offset log_append_time 23 | * partition => INT32 24 | * error_code => INT16 25 | * base_offset => INT64 26 | * log_append_time => INT64 27 | */ 28 | class ProduceResponsePartition implements BinarySchemeInterface 29 | { 30 | /** 31 | * The partition this response entry corresponds to. 32 | * 33 | * @var integer 34 | */ 35 | public $partition; 36 | 37 | /** 38 | * The error from this partition, if any. 39 | * 40 | * Errors are given on a per-partition basis because a given partition may be unavailable or maintained on a 41 | * different host, while others may have successfully accepted the produce request. 42 | * 43 | * @var integer 44 | */ 45 | public $errorCode; 46 | 47 | /** 48 | * The offset assigned to the first message in the message set appended to this partition. 49 | * 50 | * @var integer 51 | */ 52 | public $baseOffset; 53 | 54 | /** 55 | * If LogAppendTime is used for the topic, this is the timestamp assigned by the broker to the message set. 56 | * All the messages in the message set have the same timestamp. 57 | * 58 | * If CreateTime is used, this field is always -1. The producer can assume the timestamp of the messages in the 59 | * produce request has been accepted by the broker if there is no error code returned. 60 | * 61 | * Unit is milliseconds since beginning of the epoch (midnight Jan 1, 1970 (UTC)). 62 | * 63 | * @var integer 64 | * @since Version 2 of protocol 65 | */ 66 | public $logAppendTime; 67 | 68 | /** 69 | * @inheritdoc 70 | */ 71 | public static function getScheme(): array 72 | { 73 | return [ 74 | 'partition' => Scheme::TYPE_INT32, 75 | 'errorCode' => Scheme::TYPE_INT16, 76 | 'baseOffset' => Scheme::TYPE_INT64, 77 | 'logAppendTime' => Scheme::TYPE_INT64 78 | ]; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/Kafka/DTO/ProduceResponseTopic.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_STRING, 45 | 'partitions' => ['partition' => ProduceResponsePartition::class] 46 | ]; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Kafka/DTO/Record.php: -------------------------------------------------------------------------------- 1 | 23 | * Length => Varint 24 | * Attributes => Int8 25 | * TimestampDelta => Varlong 26 | * OffsetDelta => Varint 27 | * Key => Bytes 28 | * Value => Bytes 29 | * Headers => [HeaderKey HeaderValue] 30 | * HeaderKey => String 31 | * HeaderValue => Bytes 32 | * 33 | * Note that in this schema, the Bytes and String types use a variable length integer to represent 34 | * the length of the field. The array type used for the headers also uses a Varint for the number of 35 | * headers. 36 | * 37 | * The current record attributes are depicted below: 38 | * 39 | * ---------------- 40 | * | Unused (0-7) | 41 | * ---------------- 42 | * 43 | * The offset and timestamp deltas compute the difference relative to the base offset and 44 | * base timestamp of the batch that this record is contained in. 45 | * 46 | * @since 0.11.0 47 | */ 48 | class Record implements BinarySchemeInterface 49 | { 50 | /** 51 | * Length of this message 52 | */ 53 | public $length = 0; 54 | 55 | /** 56 | * Record level attributes are presently unused. 57 | */ 58 | public $attributes = 0; 59 | 60 | /** 61 | * The timestamp delta of the record in the batch. 62 | * 63 | * The timestamp of each Record in the RecordBatch is its 'TimestampDelta' + 'FirstTimestamp'. 64 | * 65 | * @since Version 2 of Message structure 66 | */ 67 | public $timestampDelta = 0; 68 | 69 | /** 70 | * The offset delta of the record in the batch. 71 | * 72 | * The offset of each Record in the Batch is its 'OffsetDelta' + 'FirstOffset'. 73 | * 74 | * @since Version 2 of Message (Record) structure 75 | */ 76 | public $offsetDelta = 0; 77 | 78 | /** 79 | * The key is an optional message key that was used for partition assignment. The key can be null. 80 | */ 81 | public $key; 82 | 83 | /** 84 | * The value is the actual message contents as an opaque byte array. 85 | */ 86 | public $value; 87 | 88 | /** 89 | * Application level record level headers. 90 | * 91 | * @since Version 2 of Message (Record) structure 92 | * @see https://cwiki.apache.org/confluence/display/KAFKA/KIP-82+-+Add+Record+Headers 93 | * 94 | * @var Header[] 95 | */ 96 | public $headers = []; 97 | 98 | /** 99 | * Record constructor 100 | */ 101 | public function __construct( 102 | string $value, 103 | ?string $key = null, 104 | array $headers = [], 105 | int $attributes = 0, 106 | int $timestampDelta = 0, 107 | int $offsetDelta = 0 108 | ) { 109 | $this->value = $value; 110 | $this->key = $key; 111 | $this->headers = $headers; 112 | $this->attributes = $attributes; 113 | $this->timestampDelta = $timestampDelta; 114 | $this->offsetDelta = $offsetDelta; 115 | 116 | $this->length = Scheme::getObjectTypeSize($this) - 1; /* Varint 0 length always equal to 1 */ 117 | } 118 | 119 | /** 120 | * @inheritdoc 121 | */ 122 | public static function getScheme(): array 123 | { 124 | return [ 125 | 'length' => Scheme::TYPE_VARINT_ZIGZAG, 126 | 'attributes' => Scheme::TYPE_INT8, 127 | 'timestampDelta' => Scheme::TYPE_VARLONG_ZIGZAG, 128 | 'offsetDelta' => Scheme::TYPE_VARINT_ZIGZAG, 129 | 'key' => Scheme::TYPE_VARCHAR_ZIGZAG, 130 | 'value' => Scheme::TYPE_VARCHAR_ZIGZAG, 131 | 'headers' => ['key' => Header::class, Scheme::FLAG_VARARRAY => true] 132 | ]; 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /src/Kafka/DTO/SyncGroupRequestMember.php: -------------------------------------------------------------------------------- 1 | [MemberId MemberAssignment] 25 | * MemberId => string 26 | * MemberAssignment => MemberAssignment 27 | */ 28 | class SyncGroupRequestMember implements BinarySchemeInterface 29 | { 30 | /** 31 | * Name of the group member 32 | */ 33 | public $memberId; 34 | 35 | /** 36 | * Member-specific assignment 37 | * 38 | * @var string 39 | * @todo This field should be MemberAssignment instance in scheme 40 | */ 41 | public $assignment; 42 | 43 | /** 44 | * Default initializer 45 | * 46 | * @param string $memberId Member identifier 47 | * @param MemberAssignment $assignment Received assignment 48 | */ 49 | public function __construct(string $memberId, MemberAssignment $assignment) 50 | { 51 | $this->memberId = $memberId; 52 | // TODO: This should be done on scheme-level 53 | $stringBuffer = new StringStream(); 54 | Scheme::writeObjectToStream($assignment, $stringBuffer); 55 | $this->assignment = $stringBuffer->getBuffer(); 56 | } 57 | 58 | /** 59 | * @inheritdoc 60 | */ 61 | public static function getScheme(): array 62 | { 63 | return [ 64 | 'memberId' => Scheme::TYPE_STRING, 65 | 'assignment' => Scheme::TYPE_BYTEARRAY 66 | ]; 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /src/Kafka/DTO/TopicPartitions.php: -------------------------------------------------------------------------------- 1 | [Topic [Partition]] 23 | * Topic => string 24 | * Partition => int32 25 | */ 26 | class TopicPartitions implements BinarySchemeInterface 27 | { 28 | 29 | /** 30 | * Name of the topic to assign 31 | */ 32 | public $topic; 33 | 34 | /** 35 | * List of partitions from the topic to assign 36 | * 37 | * @var integer[] 38 | */ 39 | public $partitions = []; 40 | 41 | /** 42 | * @inheritDoc 43 | */ 44 | public function __construct(string $topic, array $partitions) 45 | { 46 | $this->topic = $topic; 47 | $this->partitions = $partitions; 48 | } 49 | 50 | /** 51 | * @inheritdoc 52 | */ 53 | public static function getScheme(): array 54 | { 55 | return [ 56 | 'topic' => Scheme::TYPE_STRING, 57 | 'partitions' => [Scheme::TYPE_INT32] 58 | ]; 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/Kafka/Enum/SecurityProtocol.php: -------------------------------------------------------------------------------- 1 | requestedTopics = $requestedTopics; 40 | } 41 | 42 | /** 43 | * Return list of requested topic subscription 44 | * 45 | * @return string[] 46 | */ 47 | public function getRequestedTopics(): array 48 | { 49 | return $this->requestedTopics; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /src/Kafka/Error/GroupAuthorizationFailed.php: -------------------------------------------------------------------------------- 1 | partition result 24 | * 25 | * @var array 26 | */ 27 | private $partialResult; 28 | 29 | /** 30 | * Exceptions during this query [topic][partition] => exception 31 | * 32 | * @var Exception[][] 33 | */ 34 | protected $exceptions; 35 | 36 | /** 37 | * TopicPartitionRequestException constructor. 38 | * 39 | * @param array $partialResult Partial result received from broker 40 | * @param Exception[][] $exceptions List of nested exceptions 41 | */ 42 | public function __construct(array $partialResult, array $exceptions) 43 | { 44 | $message = ''; 45 | foreach ($exceptions as $topic => $partitions) { 46 | foreach ($partitions as $partitionId => $exception) { 47 | $message .= sprintf("%s:%s %s\n", $topic, $partitionId, $exception->getMessage()); 48 | } 49 | } 50 | parent::__construct('Request completed with errors: ' . $message); 51 | $this->partialResult = $partialResult; 52 | $this->exceptions = $exceptions; 53 | } 54 | 55 | /** 56 | * Return partial result from request 57 | * 58 | * @return array [topic][partition] => partition result 59 | */ 60 | public function getPartialResult(): array 61 | { 62 | return $this->partialResult; 63 | } 64 | 65 | /** 66 | * Return array of occurred exceptions 67 | * 68 | * @return Exception[][] [topic][partition] => exception 69 | */ 70 | public function getExceptions(): array 71 | { 72 | return $this->exceptions; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/Kafka/Error/UnknownError.php: -------------------------------------------------------------------------------- 1 | DefaultPartitioner::class, 28 | Config::ACKS => 1, 29 | Config::TIMEOUT_MS => 2000, 30 | Config::RETRIES => 0, 31 | Config::BATCH_SIZE => 0, 32 | Config::TRANSACTIONAL_ID => null, 33 | 34 | Config::COMPRESSION_TYPE => 'none', 35 | Config::LINGER_MS => 0, 36 | Config::MAX_REQUEST_SIZE => 1048576, 37 | ]; 38 | 39 | /** 40 | * The number of acknowledgments the producer requires the leader to have received before considering a request 41 | * complete. This controls the durability of records that are sent. The following settings are common: 42 | * 43 | * acks=0 If set to zero then the producer will not wait for any acknowledgment from the server at all. The record 44 | * will be immediately added to the socket buffer and considered sent. No guarantee can be made that the server has 45 | * received the record in this case, and the retries configuration will not take effect (as the client won't 46 | * generally know of any failures). The offset given back for each record will always be set to -1. 47 | * 48 | * acks=1 This will mean the leader will write the record to its local log but will respond without awaiting full 49 | * acknowledgement from all followers. In this case should the leader fail immediately after acknowledging the 50 | * record but before the followers have replicated it then the record will be lost. 51 | * 52 | * acks=all This means the leader will wait for the full set of in-sync replicas to acknowledge the record. This 53 | * guarantees that the record will not be lost as long as at least one in-sync replica remains alive. This is the 54 | * strongest available guarantee. 55 | */ 56 | public const ACKS = 'acks'; 57 | 58 | /** 59 | * Partitioner class that implements the Partitioner interface. 60 | */ 61 | public const PARTITIONER_CLASS = 'partitioner.class'; 62 | 63 | /** 64 | * Setting a value greater than zero will cause the client to resend any record whose send fails with a potentially 65 | * transient error. 66 | * 67 | * Note that this retry is no different than if the client resent the record upon receiving the 68 | * error. Allowing retries without setting max.in.flight.requests.per.connection to 1 will potentially change the 69 | * ordering of records because if two batches are sent to a single partition, and the first fails and is retried 70 | * but the second succeeds, then the records in the second batch may appear first. 71 | */ 72 | public const RETRIES = 'retries'; 73 | 74 | /** 75 | * The producer will attempt to batch records together into fewer requests whenever multiple records are being sent 76 | * to the same partition. This helps performance on both the client and the server. This configuration controls the 77 | * default batch size in bytes. 78 | * 79 | * No attempt will be made to batch records larger than this size. 80 | * 81 | * Requests sent to brokers will contain multiple batches, one for each partition with data available to be sent. 82 | * 83 | * A small batch size will make batching less common and may reduce throughput (a batch size of zero will disable 84 | * batching entirely). A very large batch size may use memory a bit more wastefully as we will always allocate a 85 | * buffer of the specified batch size in anticipation of additional records. 86 | */ 87 | public const BATCH_SIZE = 'batch.size'; 88 | 89 | /** 90 | * The configuration controls the maximum amount of time the server will wait for acknowledgments from followers to 91 | * meet the acknowledgment requirements the producer has specified with the acks configuration. If the requested 92 | * number of acknowledgments are not met when the timeout elapses an error will be returned. This timeout is 93 | * measured on the server side and does not include the network latency of the request. 94 | */ 95 | public const TIMEOUT_MS = 'timeout.ms'; 96 | 97 | /** 98 | * The TransactionalId to use for transactional delivery. 99 | * 100 | * This enables reliability semantics which span multiple producer sessions since it allows the client to guarantee 101 | * that transactions using the same TransactionalId have been completed prior to starting any new transactions. If 102 | * no TransactionalId is provided, then the producer is limited to idempotent delivery. Note that 103 | * enable.idempotence must be enabled if a TransactionalId is configured. The default is empty, which means 104 | * transactions cannot be used. 105 | */ 106 | public const TRANSACTIONAL_ID = 'transactional.id'; 107 | 108 | public const COMPRESSION_TYPE = 'compression.type'; 109 | public const LINGER_MS = 'linger.ms'; 110 | public const MAX_REQUEST_SIZE = 'max.request.size'; 111 | 112 | /** 113 | * Returns default configuration for producer 114 | */ 115 | public static function getDefaultConfiguration(): array 116 | { 117 | return self::$producerConfiguration + parent::$generalConfiguration; 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /src/Kafka/Producer/DefaultPartitioner.php: -------------------------------------------------------------------------------- 1 | partitionsForTopic($topic); 45 | $totalPartitions = count($partitions); 46 | 47 | if (isset($key)) { 48 | $partitionIndex = crc32($key) % $totalPartitions; 49 | } else { 50 | if (!isset(self::$counter)) { 51 | self::$counter = (integer) (microtime(true) * 1e6); 52 | } 53 | $partitionIndex = (self::$counter++) % $totalPartitions; 54 | } 55 | 56 | return $partitionIndex; 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /src/Kafka/Producer/PartitionerInterface.php: -------------------------------------------------------------------------------- 1 | api_key api_version correlation_id client_id 23 | * api_key => INT16 24 | * api_version => INT16 25 | * correlation_id => INT32 26 | * client_id => NULLABLE_STRING 27 | */ 28 | abstract class AbstractRequest extends AbstractRecord 29 | { 30 | /** 31 | * Version of API request, could be overridden in children classes 32 | */ 33 | protected const VERSION = 0; 34 | 35 | /** 36 | * The id of the request type. (INT16) 37 | */ 38 | protected $apiKey; 39 | 40 | /** 41 | * The version of the API. (INT16) 42 | */ 43 | protected $apiVersion; 44 | 45 | /** 46 | * A user specified identifier for the client making the request. 47 | */ 48 | protected $clientId; 49 | 50 | /** 51 | * Global request counter, ideally this should be stored somewhere in the shared config to survive between requests 52 | */ 53 | private static $counter = 0; 54 | 55 | public function __construct(int $apiKey, string $clientId = '', int $correlationId = 0) 56 | { 57 | $this->apiKey = $apiKey; 58 | $this->clientId = $clientId; 59 | $this->correlationId = $correlationId ?: self::$counter++; 60 | $this->apiVersion = static::VERSION; 61 | $this->messageSize = Scheme::getObjectTypeSize($this) - 4 /* INT32 MessageSize */; 62 | } 63 | 64 | /** 65 | * @inheritdoc 66 | */ 67 | public static function getScheme(): array 68 | { 69 | return [ 70 | 'messageSize' => Scheme::TYPE_INT32, 71 | 'apiKey' => Scheme::TYPE_INT16, 72 | 'apiVersion' => Scheme::TYPE_INT16, 73 | 'correlationId' => Scheme::TYPE_INT32, 74 | 'clientId' => Scheme::TYPE_NULLABLE_STRING 75 | ]; 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /src/Kafka/Record/AbstractResponse.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT32, 31 | 'correlationId' => Scheme::TYPE_INT32, 32 | ]; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /src/Kafka/Record/ApiVersionsRequest.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT16, 46 | 'apiVersions' => ['apiKey' => ApiVersionsResponseMetadata::class], 47 | ]; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Kafka/Record/ControlledShutdownRequest.php: -------------------------------------------------------------------------------- 1 | broker_id 23 | * broker_id => INT32 24 | */ 25 | class ControlledShutdownRequest extends AbstractRequest 26 | { 27 | /** 28 | * @inheritDoc 29 | */ 30 | protected const VERSION = 1; 31 | 32 | /** 33 | * Broker identifier to shutdown 34 | */ 35 | private $brokerId; 36 | 37 | public function __construct(int $brokerId, string $clientId = '', int $correlationId = 0) 38 | { 39 | $this->brokerId = $brokerId; 40 | 41 | parent::__construct(Kafka::CONTROLLED_SHUTDOWN, $clientId, $correlationId); 42 | } 43 | 44 | /** 45 | * @inheritdoc 46 | */ 47 | public static function getScheme(): array 48 | { 49 | $header = parent::getScheme(); 50 | 51 | return $header + [ 52 | 'brokerId' => Scheme::TYPE_INT32 53 | ]; 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /src/Kafka/Record/ControlledShutdownResponse.php: -------------------------------------------------------------------------------- 1 | error_code [partitions_remaining] 23 | * error_code => INT16 24 | * partitions_remaining => topic partition 25 | * topic => STRING 26 | * partition => INT32 27 | */ 28 | class ControlledShutdownResponse extends AbstractResponse 29 | { 30 | /** 31 | * Error code. 32 | * 33 | * @var integer 34 | */ 35 | public $errorCode; 36 | 37 | /** 38 | * The topic partitions that the broker still leads. 39 | * 40 | * @var ControlledShutdownResponsePartition[] 41 | */ 42 | public $remainingTopicPartitions = []; 43 | 44 | /** 45 | * @inheritdoc 46 | */ 47 | public static function getScheme(): array 48 | { 49 | $header = parent::getScheme(); 50 | 51 | return $header + [ 52 | 'errorCode' => Scheme::TYPE_INT16, 53 | 'remainingTopicPartitions' => [ControlledShutdownResponsePartition::class] 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Kafka/Record/DescribeGroupsRequest.php: -------------------------------------------------------------------------------- 1 | [group_ids] 26 | * group_ids => STRING 27 | */ 28 | class DescribeGroupsRequest extends AbstractRequest 29 | { 30 | /** 31 | * List of groups to describe 32 | */ 33 | private $groups; 34 | 35 | public function __construct(array $groups, string $clientId = '', int $correlationId = 0) 36 | { 37 | $this->groups = $groups; 38 | parent::__construct(Kafka::DESCRIBE_GROUPS, $clientId, $correlationId); 39 | } 40 | 41 | /** 42 | * @inheritdoc 43 | */ 44 | public static function getScheme(): array 45 | { 46 | $header = parent::getScheme(); 47 | 48 | return $header + [ 49 | 'groups' => [Scheme::TYPE_STRING] 50 | ]; 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /src/Kafka/Record/DescribeGroupsResponse.php: -------------------------------------------------------------------------------- 1 | ['groupId' => DescribeGroupResponseMetadata::class] 37 | ]; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Kafka/Record/FetchResponse.php: -------------------------------------------------------------------------------- 1 | throttle_time_ms [responses] 23 | * throttle_time_ms => INT32 24 | * responses => topic [partition_responses] 25 | * topic => STRING 26 | * partition_responses => partition_header record_set 27 | * partition_header => partition error_code high_watermark last_stable_offset log_start_offset [aborted_transactions] 28 | * partition => INT32 29 | * error_code => INT16 30 | * high_watermark => INT64 31 | * last_stable_offset => INT64 32 | * log_start_offset => INT64 33 | * aborted_transactions => producer_id first_offset 34 | * producer_id => INT64 35 | * first_offset => INT64 36 | * record_set => RECORDS 37 | */ 38 | class FetchResponse extends AbstractResponse 39 | { 40 | 41 | /** 42 | * Duration in milliseconds for which the request was throttled due to quota violation. 43 | * (Zero if the request did not violate any quota.) 44 | * 45 | * @var integer 46 | * @since Version 1 of protocol 47 | */ 48 | public $throttleTimeMs; 49 | 50 | /** 51 | * List of fetch responses 52 | * 53 | * @var FetchResponseTopic[] 54 | */ 55 | public $topics = []; 56 | 57 | /** 58 | * @inheritdoc 59 | */ 60 | public static function getScheme(): array 61 | { 62 | return parent::getScheme() + [ 63 | 'throttleTimeMs' => Scheme::TYPE_INT32, 64 | 'topics' => ['topic' => FetchResponseTopic::class], 65 | ]; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/Kafka/Record/GroupCoordinatorRequest.php: -------------------------------------------------------------------------------- 1 | consumerGroup = $consumerGroup; 35 | 36 | parent::__construct(Kafka::GROUP_COORDINATOR, $clientId, $correlationId); 37 | } 38 | 39 | /** 40 | * @inheritdoc 41 | */ 42 | public static function getScheme(): array 43 | { 44 | $header = parent::getScheme(); 45 | 46 | return $header + [ 47 | 'consumerGroup' => Scheme::TYPE_STRING 48 | ]; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Kafka/Record/GroupCoordinatorResponse.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT16, 47 | 'coordinator' => GroupCoordinatorResponseMetadata::class 48 | ]; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Kafka/Record/HeartbeatRequest.php: -------------------------------------------------------------------------------- 1 | group_id generation_id member_id 27 | * group_id => STRING 28 | * generation_id => INT32 29 | * member_id => STRING 30 | */ 31 | class HeartbeatRequest extends AbstractRequest 32 | { 33 | /** 34 | * The consumer group id. 35 | */ 36 | private $consumerGroup; 37 | 38 | /** 39 | * The generation of the group. 40 | */ 41 | private $generationId; 42 | 43 | /** 44 | * The member id assigned by the group coordinator. 45 | */ 46 | private $memberId; 47 | 48 | public function __construct( 49 | string $consumerGroup, 50 | int $generationId, 51 | string $memberId, 52 | string $clientId = '', 53 | int $correlationId = 0 54 | ) { 55 | $this->consumerGroup = $consumerGroup; 56 | $this->generationId = $generationId; 57 | $this->memberId = $memberId; 58 | 59 | parent::__construct(Kafka::HEARTBEAT, $clientId, $correlationId); 60 | } 61 | 62 | /** 63 | * @inheritdoc 64 | */ 65 | public static function getScheme(): array 66 | { 67 | $header = parent::getScheme(); 68 | 69 | return $header + [ 70 | 'consumerGroup' => Scheme::TYPE_STRING, 71 | 'generationId' => Scheme::TYPE_INT32, 72 | 'memberId' => Scheme::TYPE_STRING 73 | ]; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/Kafka/Record/HeartbeatResponse.php: -------------------------------------------------------------------------------- 1 | error_code 22 | * error_code => INT16 23 | */ 24 | class HeartbeatResponse extends AbstractResponse 25 | { 26 | /** 27 | * Error code. 28 | * 29 | * @var integer 30 | */ 31 | public $errorCode; 32 | 33 | /** 34 | * @inheritdoc 35 | */ 36 | public static function getScheme(): array 37 | { 38 | $header = parent::getScheme(); 39 | 40 | return $header + [ 41 | 'errorCode' => Scheme::TYPE_INT16 42 | ]; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Kafka/Record/JoinGroupRequest.php: -------------------------------------------------------------------------------- 1 | value pairs, where value is metadata 67 | * 68 | * @var array 69 | */ 70 | private $groupProtocols; 71 | 72 | public function __construct( 73 | string $consumerGroup, 74 | int $sessionTimeout, 75 | int $rebalanceTimeout, 76 | string $memberId, 77 | string $protocolType, 78 | array $groupProtocols, 79 | string $clientId = '', 80 | int $correlationId = 0 81 | ) { 82 | $this->consumerGroup = $consumerGroup; 83 | $this->sessionTimeout = $sessionTimeout; 84 | $this->rebalanceTimeout = $rebalanceTimeout; 85 | $this->memberId = $memberId; 86 | $this->protocolType = $protocolType; 87 | $packedProtocols = []; 88 | foreach ($groupProtocols as $protocolName => $protocolMetadata) { 89 | $packedProtocols[$protocolName] = new JoinGroupRequestProtocol($protocolName, $protocolMetadata); 90 | } 91 | 92 | $this->groupProtocols = $packedProtocols; 93 | 94 | parent::__construct(Kafka::JOIN_GROUP, $clientId, $correlationId); 95 | } 96 | 97 | /** 98 | * @inheritdoc 99 | */ 100 | public static function getScheme(): array 101 | { 102 | $header = parent::getScheme(); 103 | 104 | return $header + [ 105 | 'consumerGroup' => Scheme::TYPE_STRING, 106 | 'sessionTimeout' => Scheme::TYPE_INT32, 107 | 'rebalanceTimeout' => Scheme::TYPE_INT32, 108 | 'memberId' => Scheme::TYPE_STRING, 109 | 'protocolType' => Scheme::TYPE_STRING, 110 | 'groupProtocols' => ['protocolName' => JoinGroupRequestProtocol::class] 111 | ]; 112 | } 113 | } 114 | -------------------------------------------------------------------------------- /src/Kafka/Record/JoinGroupResponse.php: -------------------------------------------------------------------------------- 1 | error_code generation_id group_protocol leader_id member_id [members] 23 | * error_code => INT16 24 | * generation_id => INT32 25 | * group_protocol => STRING 26 | * leader_id => STRING 27 | * member_id => STRING 28 | * members => member_id member_metadata 29 | * member_id => STRING 30 | * member_metadata => BYTES 31 | */ 32 | class JoinGroupResponse extends AbstractResponse 33 | { 34 | /** 35 | * Error code. 36 | * 37 | * @var integer 38 | */ 39 | public $errorCode; 40 | 41 | /** 42 | * The generation of the consumer group. 43 | * 44 | * @var integer 45 | */ 46 | public $generationId; 47 | 48 | /** 49 | * The group protocol selected by the coordinator 50 | * 51 | * @var string 52 | */ 53 | public $groupProtocol; 54 | 55 | /** 56 | * The leader of the group 57 | * 58 | * @var string 59 | */ 60 | public $leaderId; 61 | 62 | /** 63 | * The consumer id assigned by the group coordinator. 64 | * 65 | * @var string 66 | */ 67 | public $memberId; 68 | 69 | /** 70 | * List of members of the group with metadata as value 71 | * 72 | * @var array 73 | */ 74 | public $members = []; 75 | 76 | /** 77 | * @inheritdoc 78 | */ 79 | public static function getScheme(): array 80 | { 81 | $header = parent::getScheme(); 82 | 83 | return $header + [ 84 | 'errorCode' => Scheme::TYPE_INT16, 85 | 'generationId' => Scheme::TYPE_INT32, 86 | 'groupProtocol' => Scheme::TYPE_STRING, 87 | 'leaderId' => Scheme::TYPE_STRING, 88 | 'memberId' => Scheme::TYPE_STRING, 89 | 'members' => ['memberId' => JoinGroupResponseMember::class] 90 | ]; 91 | } 92 | } 93 | -------------------------------------------------------------------------------- /src/Kafka/Record/LeaveGroupRequest.php: -------------------------------------------------------------------------------- 1 | group_id member_id 27 | * group_id => STRING 28 | * member_id => STRING 29 | */ 30 | class LeaveGroupRequest extends AbstractRequest 31 | { 32 | /** 33 | * The consumer group id. 34 | */ 35 | private $consumerGroup; 36 | 37 | /** 38 | * The member id assigned by the group coordinator. 39 | */ 40 | private $memberId; 41 | 42 | public function __construct(string $consumerGroup, string $memberId, string $clientId = '', int $correlationId = 0) 43 | { 44 | $this->consumerGroup = $consumerGroup; 45 | $this->memberId = $memberId; 46 | 47 | parent::__construct(Kafka::LEAVE_GROUP, $clientId, $correlationId); 48 | } 49 | 50 | /** 51 | * @inheritdoc 52 | */ 53 | public static function getScheme(): array 54 | { 55 | $header = parent::getScheme(); 56 | 57 | return $header + [ 58 | 'consumerGroup' => Scheme::TYPE_STRING, 59 | 'memberId' => Scheme::TYPE_STRING 60 | ]; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Kafka/Record/LeaveGroupResponse.php: -------------------------------------------------------------------------------- 1 | error_code 22 | * error_code => INT16 23 | */ 24 | class LeaveGroupResponse extends AbstractResponse 25 | { 26 | /** 27 | * Error code. 28 | * 29 | * @var integer 30 | */ 31 | public $errorCode; 32 | 33 | /** 34 | * @inheritdoc 35 | */ 36 | public static function getScheme(): array 37 | { 38 | $header = parent::getScheme(); 39 | 40 | return $header + [ 41 | 'errorCode' => Scheme::TYPE_INT16 42 | ]; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Kafka/Record/ListGroupsRequest.php: -------------------------------------------------------------------------------- 1 | error_code [groups] 23 | * error_code => INT16 24 | * groups => group_id protocol_type 25 | * group_id => STRING 26 | * protocol_type => STRING 27 | */ 28 | class ListGroupsResponse extends AbstractResponse 29 | { 30 | /** 31 | * Error code. 32 | * 33 | * @var integer 34 | */ 35 | public $errorCode; 36 | 37 | /** 38 | * List of groups as keys and current protocols as values 39 | * 40 | * @var ListGroupResponseProtocol[] 41 | */ 42 | public $groups = []; 43 | 44 | /** 45 | * @inheritdoc 46 | */ 47 | public static function getScheme(): array 48 | { 49 | $header = parent::getScheme(); 50 | 51 | return $header + [ 52 | 'errorCode' => Scheme::TYPE_INT16, 53 | 'groups' => ['groupId' => ListGroupResponseProtocol::class] 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Kafka/Record/MetadataRequest.php: -------------------------------------------------------------------------------- 1 | topics = $topics; 55 | 56 | parent::__construct(Kafka::METADATA, $clientId, $correlationId); 57 | } 58 | 59 | /** 60 | * @inheritdoc 61 | */ 62 | public static function getScheme(): array 63 | { 64 | $header = parent::getScheme(); 65 | 66 | return $header + [ 67 | 'topics' => [Scheme::TYPE_STRING, Scheme::FLAG_NULLABLE => true] 68 | ]; 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/Kafka/Record/MetadataResponse.php: -------------------------------------------------------------------------------- 1 | [Node::class], 68 | 'clusterId' => Scheme::TYPE_NULLABLE_STRING, 69 | 'controllerId' => Scheme::TYPE_INT32, 70 | 'topics' => ['topic' => TopicMetadata::class], 71 | ]; 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /src/Kafka/Record/OffsetCommitRequest.php: -------------------------------------------------------------------------------- 1 | group_id generation_id member_id retention_time [topics] 28 | * group_id => STRING 29 | * generation_id => INT32 30 | * member_id => STRING 31 | * retention_time => INT64 32 | * topics => topic [partitions] 33 | * topic => STRING 34 | * partitions => partition offset metadata 35 | * partition => INT32 36 | * offset => INT64 37 | * metadata => NULLABLE_STRING 38 | */ 39 | class OffsetCommitRequest extends AbstractRequest 40 | { 41 | /** 42 | * Generation id for unsubscribed consumer 43 | */ 44 | public const DEFAULT_GENERATION_ID = -1; 45 | 46 | /** 47 | * @inheritDoc 48 | */ 49 | protected const VERSION = 2; 50 | 51 | /** 52 | * The consumer group id. 53 | */ 54 | private $consumerGroup; 55 | 56 | /** 57 | * The generation of the group. 58 | * 59 | * @since Version 1 of protocol 60 | */ 61 | private $generationId; 62 | 63 | /** 64 | * The member id assigned by the group coordinator. 65 | * 66 | * @since Version 1 of protocol 67 | */ 68 | private $memberName; 69 | 70 | /** 71 | * Time period in ms to retain the offset. 72 | * 73 | * @since Version 2 of protocol 74 | */ 75 | private $retentionTime; 76 | 77 | /** 78 | * @var OffsetCommitRequestTopic[] 79 | */ 80 | private $topicPartitions; 81 | 82 | public function __construct( 83 | string $consumerGroup, 84 | int $generationId, 85 | string $memberName, 86 | int $retentionTime, 87 | array $topicPartitions, 88 | string $clientId = '', 89 | int $correlationId = 0 90 | ) { 91 | 92 | $this->consumerGroup = $consumerGroup; 93 | $this->generationId = $generationId; 94 | $this->memberName = $memberName; 95 | $this->retentionTime = $retentionTime; 96 | $packedTopicPartitions = []; 97 | foreach ($topicPartitions as $topic => $partitions) { 98 | $packedTopicPartitions[$topic] = new OffsetCommitRequestTopic($topic, $partitions); 99 | } 100 | $this->topicPartitions = $packedTopicPartitions; 101 | 102 | parent::__construct(Kafka::OFFSET_COMMIT, $clientId, $correlationId); 103 | } 104 | 105 | /** 106 | * @inheritdoc 107 | */ 108 | public static function getScheme(): array 109 | { 110 | $header = parent::getScheme(); 111 | 112 | return $header + [ 113 | 'consumerGroup' => Scheme::TYPE_STRING, 114 | 'generationId' => Scheme::TYPE_INT32, 115 | 'memberName' => Scheme::TYPE_STRING, 116 | 'retentionTime' => Scheme::TYPE_INT64, 117 | 'topicPartitions' => ['topic' => OffsetCommitRequestTopic::class] 118 | ]; 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /src/Kafka/Record/OffsetCommitResponse.php: -------------------------------------------------------------------------------- 1 | [responses] 22 | * responses => topic [partition_responses] 23 | * topic => STRING 24 | * partition_responses => partition error_code 25 | * partition => INT32 26 | * error_code => INT16 27 | */ 28 | class OffsetCommitResponse extends AbstractResponse 29 | { 30 | /** 31 | * List of topics with partition result 32 | * 33 | * @var OffsetCommitResponseTopic[] 34 | */ 35 | public $topics = []; 36 | 37 | /** 38 | * @inheritdoc 39 | */ 40 | public static function getScheme(): array 41 | { 42 | $header = parent::getScheme(); 43 | 44 | return $header + [ 45 | 'topics' => ['topic' => OffsetCommitResponseTopic::class] 46 | ]; 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /src/Kafka/Record/OffsetFetchRequest.php: -------------------------------------------------------------------------------- 1 | group_id [topics] 33 | * group_id => STRING 34 | * topics => topic [partitions] 35 | * topic => STRING 36 | * partitions => partition 37 | * partition => INT32 38 | */ 39 | class OffsetFetchRequest extends AbstractRequest 40 | { 41 | /** 42 | * @inheritDoc 43 | */ 44 | protected const VERSION = 2; 45 | 46 | /** 47 | * The consumer group id. 48 | */ 49 | protected $consumerGroup; 50 | 51 | /** 52 | * @var TopicPartitions[]|null 53 | */ 54 | protected $topicPartitions; 55 | 56 | /** 57 | * OffsetFetchRequest constructor. 58 | * 59 | * @param string $consumerGroup Name of the consumer group 60 | * @param TopicPartitions[]|null $topicPartitions List of topic => partitions to fetch or null for all topics 61 | * @param string $clientId Unique client identifier 62 | * @param int $correlationId Correlated request ID 63 | */ 64 | public function __construct( 65 | string $consumerGroup, 66 | array $topicPartitions = null, 67 | string $clientId = '', 68 | int $correlationId = 0 69 | ) { 70 | $this->consumerGroup = $consumerGroup; 71 | $this->topicPartitions = $topicPartitions; 72 | 73 | parent::__construct(Kafka::OFFSET_FETCH, $clientId, $correlationId); 74 | } 75 | 76 | /** 77 | * @inheritdoc 78 | */ 79 | public static function getScheme(): array 80 | { 81 | $header = parent::getScheme(); 82 | 83 | return $header + [ 84 | 'consumerGroup' => Scheme::TYPE_STRING, 85 | 'topicPartitions' => ['topic' => TopicPartitions::class, Scheme::FLAG_NULLABLE => true] 86 | ]; 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /src/Kafka/Record/OffsetFetchResponse.php: -------------------------------------------------------------------------------- 1 | [responses] error_code 23 | * responses => topic [partition_responses] 24 | * topic => STRING 25 | * partition_responses => partition offset metadata error_code 26 | * partition => INT32 27 | * offset => INT64 28 | * metadata => NULLABLE_STRING 29 | * error_code => INT16 30 | * error_code => INT16 31 | */ 32 | class OffsetFetchResponse extends AbstractResponse 33 | { 34 | /** 35 | * List of topic responses 36 | * 37 | * @var OffsetFetchResponseTopic[] 38 | */ 39 | public $topics = []; 40 | 41 | /** 42 | * Error code returned by the coordinator 43 | * 44 | * @since Version 2 of protocol 45 | * 46 | * @var integer 47 | */ 48 | public $errorCode; 49 | 50 | /** 51 | * @inheritdoc 52 | */ 53 | public static function getScheme(): array 54 | { 55 | $header = parent::getScheme(); 56 | 57 | return $header + [ 58 | 'topics' => ['topic' => OffsetFetchResponseTopic::class], 59 | 'errorCode' => Scheme::TYPE_INT16, 60 | ]; 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Kafka/Record/OffsetsRequest.php: -------------------------------------------------------------------------------- 1 | replica_id [topics] 31 | * replica_id => INT32 32 | * topics => topic [partitions] 33 | * topic => STRING 34 | * partitions => partition timestamp 35 | * partition => INT32 36 | * timestamp => INT64 37 | */ 38 | class OffsetsRequest extends AbstractRequest 39 | { 40 | /** 41 | * Special value for the offset of the next coming message 42 | */ 43 | public const LATEST = -1; 44 | 45 | /** 46 | * Special value for receiving the earliest available offset 47 | */ 48 | public const EARLIEST = -2; 49 | 50 | /** 51 | * @inheritDoc 52 | */ 53 | protected const VERSION = 1; 54 | 55 | /** 56 | * @var array 57 | */ 58 | private $topicPartitions; 59 | 60 | /** 61 | * The replica id indicates the node id of the replica initiating this request. Normal client consumers should 62 | * always specify this as -1 as they have no node id. Other brokers set this to be their own node id. The value -2 63 | * is accepted to allow a non-broker to issue fetch requests as if it were a replica broker for debugging purposes. 64 | */ 65 | private $replicaId; 66 | 67 | public function __construct( 68 | array $topicPartitions, 69 | int $replicaId = -1, 70 | string $clientId = '', 71 | int $correlationId = 0 72 | ) { 73 | $packedTopicPartitions = []; 74 | foreach ($topicPartitions as $topic => $partitions) { 75 | $packedTopicPartitions[$topic] = new OffsetsRequestTopic($topic, $partitions); 76 | } 77 | $this->topicPartitions = $packedTopicPartitions; 78 | $this->replicaId = $replicaId; 79 | 80 | parent::__construct(Kafka::OFFSETS, $clientId, $correlationId); 81 | } 82 | 83 | /** 84 | * @inheritdoc 85 | */ 86 | public static function getScheme(): array 87 | { 88 | $header = parent::getScheme(); 89 | 90 | return $header + [ 91 | 'replicaId' => Scheme::TYPE_INT32, 92 | 'topicPartitions' => ['topic' => OffsetsRequestTopic::class] 93 | ]; 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /src/Kafka/Record/OffsetsResponse.php: -------------------------------------------------------------------------------- 1 | [responses] 22 | * responses => topic [partition_responses] 23 | * topic => STRING 24 | * partition_responses => partition error_code timestamp offset 25 | * partition => INT32 26 | * error_code => INT16 27 | * timestamp => INT64 28 | * offset => INT64 29 | */ 30 | class OffsetsResponse extends AbstractResponse 31 | { 32 | /** 33 | * List of broker metadata info 34 | * 35 | * @var OffsetsResponseTopic[] 36 | */ 37 | public $topics = []; 38 | 39 | /** 40 | * @inheritdoc 41 | */ 42 | public static function getScheme(): array 43 | { 44 | $header = parent::getScheme(); 45 | 46 | return $header + [ 47 | 'topics' => ['topic' => OffsetsResponseTopic::class] 48 | ]; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Kafka/Record/ProduceRequest.php: -------------------------------------------------------------------------------- 1 | transactional_id acks timeout [topic_data] 32 | * transactional_id => NULLABLE_STRING 33 | * acks => INT16 34 | * timeout => INT32 35 | * topic_data => topic [data] 36 | * topic => STRING 37 | * data => partition record_set 38 | * partition => INT32 39 | * record_set => RECORDS 40 | */ 41 | class ProduceRequest extends AbstractRequest 42 | { 43 | /** 44 | * @inheritDoc 45 | */ 46 | protected const VERSION = 3; 47 | 48 | /** 49 | * The transactional ID of the producer. 50 | * 51 | * This is used to authorize transaction produce requests. 52 | * This can be null for non-transactional producers. 53 | * 54 | * @var string|null 55 | */ 56 | private $transactionalId; 57 | 58 | /** 59 | * The number of acknowledgments the producer requires the leader to have received before considering a request 60 | * complete. Allowed values: 0 for no acknowledgments, 1 for only the leader and -1 for the full ISR. 61 | */ 62 | private $requiredAcks; 63 | 64 | /** 65 | * The time to await a response in ms. 66 | */ 67 | private $timeout; 68 | 69 | /** 70 | * @var ProduceRequestTopic[] 71 | */ 72 | public $topicMessages; 73 | 74 | /** 75 | * ProduceRequest constructor. 76 | * 77 | * @param array $topicMessages List of messages in format: topic => [partition => [messages]] 78 | * @param int $requiredAcks This field indicates how many acknowledgements the servers should receive before 79 | * responding to the request. 80 | * If it is 0 the server will not send any response 81 | * (this is the only case where the server will not reply to a request). 82 | * If it is 1, the server will wait the data is written to the local log before 83 | * sending a response. If it is -1 the server will block until the message is 84 | * committed by all in sync replicas before sending a response. 85 | * @param string $transactionalId The transactional ID of the producer. This is used to authorize transaction 86 | * produce requests. This can be null for non-transactional producers. 87 | * @param int $timeout This provides a maximum time in milliseconds the server can await the receipt of 88 | * the number of acknowledgements in RequiredAcks. 89 | * @param string $clientId Kafka client identifier 90 | * @param int $correlationId Correlation request ID (will be returned in the response) 91 | */ 92 | public function __construct( 93 | array $topicMessages = [], 94 | int $requiredAcks = 1, 95 | ?string $transactionalId = null, 96 | int $timeout = 0, 97 | string $clientId = '', 98 | int $correlationId = 0 99 | ) { 100 | 101 | foreach ($topicMessages as $topic => $partitionMessages) { 102 | $partitions = []; 103 | foreach ($partitionMessages as $partition => $records) { 104 | $recordBatch = new RecordBatch($records); 105 | $partitions[$partition] = new ProduceRequestPartition($partition, $recordBatch); 106 | } 107 | 108 | $this->topicMessages[$topic] = new ProduceRequestTopic($topic, $partitions); 109 | 110 | } 111 | 112 | $this->requiredAcks = $requiredAcks; 113 | $this->transactionalId = $transactionalId; 114 | $this->timeout = $timeout; 115 | 116 | parent::__construct(Kafka::PRODUCE, $clientId, $correlationId); 117 | } 118 | 119 | /** 120 | * @inheritdoc 121 | */ 122 | public static function getScheme(): array 123 | { 124 | $header = parent::getScheme(); 125 | 126 | return $header + [ 127 | 'transactionalId' => Scheme::TYPE_NULLABLE_STRING, 128 | 'requiredAcks' => Scheme::TYPE_INT16, 129 | 'timeout' => Scheme::TYPE_INT32, 130 | 'topicMessages' => ['topic' => ProduceRequestTopic::class] 131 | ]; 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /src/Kafka/Record/ProduceResponse.php: -------------------------------------------------------------------------------- 1 | [responses] throttle_time_ms 23 | * responses => topic [partition_responses] 24 | * topic => STRING 25 | * partition_responses => partition error_code base_offset log_append_time 26 | * partition => INT32 27 | * error_code => INT16 28 | * base_offset => INT64 29 | * log_append_time => INT64 30 | * throttle_time_ms => INT32 31 | */ 32 | class ProduceResponse extends AbstractResponse 33 | { 34 | /** 35 | * List of broker metadata info 36 | * 37 | * @var ProduceResponseTopic[] 38 | */ 39 | public $topics; 40 | 41 | /** 42 | * Duration in milliseconds for which the request was throttled due to quota violation. (Zero if the request did not violate any quota). 43 | * 44 | * @var integer 45 | * @since Version 1 of protocol 46 | */ 47 | public $throttleTime; 48 | 49 | /** 50 | * @inheritdoc 51 | */ 52 | public static function getScheme(): array 53 | { 54 | $header = parent::getScheme(); 55 | 56 | return $header + [ 57 | 'topics' => ['topic' => ProduceResponseTopic::class], 58 | 'throttleTime' => Scheme::TYPE_INT32 59 | ]; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/Kafka/Record/SaslHandshakeRequest.php: -------------------------------------------------------------------------------- 1 | mechanism = $mechanism; 32 | 33 | parent::__construct(Kafka::SASL_HANDSHAKE, $clientId, $correlationId); 34 | } 35 | 36 | /** 37 | * @inheritdoc 38 | */ 39 | public static function getScheme(): array 40 | { 41 | $header = parent::getScheme(); 42 | 43 | return $header + [ 44 | 'mechanism' => Scheme::TYPE_STRING 45 | ]; 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /src/Kafka/Record/SaslHandshakeResponse.php: -------------------------------------------------------------------------------- 1 | Scheme::TYPE_INT16, 47 | 'enabledMechanisms' => [Scheme::TYPE_STRING] 48 | ]; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Kafka/Record/SyncGroupRequest.php: -------------------------------------------------------------------------------- 1 | GroupId GenerationId MemberId GroupAssignment 29 | * GroupId => string 30 | * GenerationId => int32 31 | * MemberId => string 32 | * GroupAssignment => [MemberId MemberAssignment] 33 | * MemberId => string 34 | * MemberAssignment => bytes 35 | */ 36 | class SyncGroupRequest extends AbstractRequest 37 | { 38 | /** 39 | * @inheritDoc 40 | */ 41 | protected const VERSION = 1; 42 | 43 | /** 44 | * The consumer group id. 45 | */ 46 | private $consumerGroup; 47 | 48 | /** 49 | * The generation of the group. 50 | */ 51 | private $generationId; 52 | 53 | /** 54 | * The member id assigned by the group coordinator. 55 | */ 56 | private $memberId; 57 | 58 | /** 59 | * List of group member assignments 60 | * 61 | * @var SyncGroupRequestMember[] 62 | */ 63 | private $groupAssignments; 64 | 65 | /** 66 | * SyncGroupRequest constructor. 67 | * 68 | * @param string $consumerGroup The consumer group id 69 | * @param int $generationId The generation of the group 70 | * @param string|null $memberId The member id assigned by the group coordinator 71 | * @param MemberAssignment[] $groupAssignments List of group member assignments 72 | * @param string $clientId Client identifier 73 | * @param int $correlationId Correlated request ID 74 | */ 75 | public function __construct( 76 | string $consumerGroup, 77 | int $generationId, 78 | ?string $memberId = null, 79 | array $groupAssignments = [], 80 | string $clientId = '', 81 | int $correlationId = 0 82 | ) { 83 | $this->consumerGroup = $consumerGroup; 84 | $this->generationId = $generationId; 85 | $this->memberId = $memberId; 86 | $packedGroupAssignments = []; 87 | foreach ($groupAssignments as $groupMemberId => $memberAssignment) { 88 | $packedGroupAssignments[$groupMemberId] = new SyncGroupRequestMember($groupMemberId, $memberAssignment); 89 | } 90 | $this->groupAssignments = $packedGroupAssignments; 91 | 92 | parent::__construct(Kafka::SYNC_GROUP, $clientId, $correlationId); 93 | } 94 | 95 | /** 96 | * @inheritdoc 97 | */ 98 | public static function getScheme(): array 99 | { 100 | $header = parent::getScheme(); 101 | 102 | return $header + [ 103 | 'consumerGroup' => Scheme::TYPE_STRING, 104 | 'generationId' => Scheme::TYPE_INT32, 105 | 'memberId' => Scheme::TYPE_NULLABLE_STRING, 106 | 'groupAssignments' => ['memberId' => SyncGroupRequestMember::class], 107 | ]; 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /src/Kafka/Record/SyncGroupResponse.php: -------------------------------------------------------------------------------- 1 | throttle_time_ms error_code member_assignment 22 | * throttle_time_ms => INT32 23 | * error_code => INT16 24 | * member_assignment => BYTES 25 | */ 26 | class SyncGroupResponse extends AbstractResponse 27 | { 28 | /** 29 | * Duration in milliseconds for which the request was throttled due to quota violation 30 | * 31 | * (Zero if the request did not violate any quota) 32 | * 33 | * @var integer 34 | */ 35 | public $throttleTimeMs; 36 | 37 | /** 38 | * Error code. 39 | * 40 | * @var integer 41 | */ 42 | public $errorCode; 43 | 44 | /** 45 | * Assigned data to the member 46 | * 47 | * @todo This should be implemented on scheme-level as MemberAssignment 48 | * @var string 49 | */ 50 | public $memberAssignment; 51 | 52 | /** 53 | * @inheritdoc 54 | */ 55 | public static function getScheme(): array 56 | { 57 | $header = parent::getScheme(); 58 | 59 | return $header + [ 60 | 'throttleTimeMs' => Scheme::TYPE_INT32, 61 | 'errorCode' => Scheme::TYPE_INT16, 62 | 'memberAssignment' => Scheme::TYPE_BYTEARRAY 63 | ]; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Kafka/Stream.php: -------------------------------------------------------------------------------- 1 | read('nlength')['length']; 27 | if ($stringLength === 0xFFFF) { 28 | throw new \UnexpectedValueException('Received -1 length for not nullable string'); 29 | } 30 | 31 | return $this->read("a{$stringLength}string")['string']; 32 | } 33 | 34 | /** 35 | * Writes the string to the stream 36 | */ 37 | public function writeString(string $string): void 38 | { 39 | $stringLength = strlen($string); 40 | $this->write("na{$stringLength}", $stringLength, $string); 41 | } 42 | 43 | /** 44 | * Reads a byte array from the stream 45 | * 46 | * @return string|null 47 | */ 48 | public function readByteArray(): ?string 49 | { 50 | $dataLength = $this->read('Nlength')['length']; 51 | if ($dataLength === 0xFFFFFFFF) { 52 | throw new \UnexpectedValueException('Received -1 length for not nullable byte array'); 53 | } 54 | 55 | return $this->read("a{$dataLength}data")['data']; 56 | } 57 | 58 | /** 59 | * Writes the string to the stream 60 | * 61 | * @param string|null $data 62 | */ 63 | public function writeByteArray(?string $data): void 64 | { 65 | $dataLength = strlen($data); 66 | $this->write("Na{$dataLength}", $dataLength, $data); 67 | } 68 | 69 | /** 70 | * Reads varint from the stream 71 | */ 72 | public function readVarint(): int 73 | { 74 | $value = 0; 75 | $offset = 0; 76 | do { 77 | $byte = $this->read('Cbyte')['byte']; 78 | $value += ($byte & 0x7f) << $offset; 79 | $offset += 7; 80 | } while (($byte & 0x80) !== 0); 81 | 82 | return $value; 83 | } 84 | 85 | /** 86 | * Writes a varint value to the stream 87 | */ 88 | public function writeVarint(int $value): void 89 | { 90 | do { 91 | $byte = $value & 0x7f; 92 | $value >>= 7; 93 | $byte = $value > 0 ? ($byte | 0x80) : $byte; 94 | $this->write('C', $byte); 95 | } while ($value > 0); 96 | } 97 | 98 | /** 99 | * Writes the raw buffer into the stream as-is 100 | * 101 | * @param string|null $buffer 102 | */ 103 | public function writeBuffer(?string $buffer): void 104 | { 105 | $bufferLength = $buffer ? strlen($buffer) : 0; 106 | $this->write("a{$bufferLength}", $buffer); 107 | } 108 | 109 | /** 110 | * Calculates the format size for unpack() operation 111 | */ 112 | protected static function packetSize(string $format): int 113 | { 114 | static $tableSize = [ 115 | 'a' => 1, 116 | 'c' => 1, 117 | 'C' => 1, 118 | 's' => 2, 119 | 'S' => 2, 120 | 'n' => 2, 121 | 'v' => 2, 122 | 'i' => PHP_INT_SIZE, 123 | 'I' => PHP_INT_SIZE, 124 | 'l' => 4, 125 | 'L' => 4, 126 | 'N' => 4, 127 | 'V' => 4, 128 | 'q' => 8, 129 | 'Q' => 8, 130 | 'J' => 8, 131 | 'P' => 8, 132 | ]; 133 | static $cache = []; 134 | if (isset($cache[$format])) { 135 | return $cache[$format]; 136 | } 137 | 138 | $numMatches = preg_match_all('/(?:\/|^)(\w)(\d*)/', $format, $matches); 139 | if(empty($numMatches)) { 140 | throw new \InvalidArgumentException("Unknown format specified: {$format}"); 141 | } 142 | $size = 0; 143 | for ($matchIndex = 0; $matchIndex < $numMatches; $matchIndex ++) { 144 | [$modifier, $repitition] = [$matches[1][$matchIndex], $matches[2][$matchIndex]]; 145 | if (!isset($tableSize[$modifier])) { 146 | throw new \InvalidArgumentException("Unknown modifier specified: $modifier"); 147 | } 148 | $size += $tableSize[$modifier] * ($repitition !== '' ? $repitition : 1); 149 | } 150 | 151 | $cache[$format] = $size; 152 | 153 | return $size; 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /src/Kafka/Stream/StringStream.php: -------------------------------------------------------------------------------- 1 | buffer = $stringBuffer ?? ''; 32 | } 33 | 34 | /** 35 | * Writes arguments to the stream 36 | * 37 | * @param string $format Format for packing arguments 38 | * @param array ...$arguments List of arguments for packing 39 | * 40 | * @return void 41 | * @see pack() manual for format 42 | * 43 | */ 44 | public function write(string $format, ...$arguments): void 45 | { 46 | $this->buffer .= pack($format, ...$arguments); 47 | } 48 | 49 | /** 50 | * Reads information from the stream, advanced internal pointer 51 | * 52 | * @param string $format Format for unpacking arguments 53 | * 54 | * @return array List of unpacked arguments 55 | * @see unpack() manual for format 56 | * 57 | */ 58 | public function read(string $format): array 59 | { 60 | $arguments = unpack($format, $this->buffer); 61 | $this->buffer = substr($this->buffer, self::packetSize($format)); 62 | 63 | return $arguments; 64 | } 65 | 66 | /** 67 | * {@inheritdoc} 68 | */ 69 | public function isConnected(): bool 70 | { 71 | return true; 72 | } 73 | 74 | /** 75 | * Returns the current buffer, useful for write operations 76 | */ 77 | public function getBuffer(): string 78 | { 79 | return $this->buffer; 80 | } 81 | 82 | /** 83 | * Checks if stream is empty 84 | */ 85 | public function isEmpty(): bool 86 | { 87 | return $this->buffer === ''; 88 | } 89 | } 90 | --------------------------------------------------------------------------------