├── .dockerignore ├── .gitignore ├── Dockerfile ├── README.md ├── composer.json ├── composer.lock ├── docker-compose.yaml ├── examples ├── AsyncCallback.php ├── AsyncHandledCallback.php ├── SyncCallback.php ├── consume_async_commit.php ├── consumeautocommit.php ├── consumebatch.php ├── consumesynccommit.php ├── produce_async.php ├── produceorder.php ├── producesync.php └── producewithheaders.php └── src ├── Common ├── CallbacksCollection.php ├── ConfigurationCallbacksKeys.php └── DefaultCallbacks.php ├── Consume └── HighLevel │ ├── ConsumerProperties.php │ ├── ConsumerWrapper.php │ ├── Contracts │ └── Callback.php │ ├── Exceptions │ ├── ConsumerShouldBeInstantiatedException.php │ ├── KafkaConsumeException.php │ ├── KafkaRebalanceCbException.php │ └── KafkaTopicNameException.php │ └── VendorExtends │ └── Output.php ├── Exceptions ├── KafkaBrokerException.php └── KafkaConfigErrorCallbackException.php └── Produce ├── Exceptions ├── KafkaProduceFlushNotImplementedException.php └── KafkaProduceFlushTimeoutException.php ├── ProducerData.php ├── ProducerProperties.php └── ProducerWrapper.php /.dockerignore: -------------------------------------------------------------------------------- 1 | data 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | vendor 3 | 4 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM 74-fpm-alpine-lib-1.5.0-ext-4.0.4 2 | 3 | COPY ./ /app 4 | 5 | WORKDIR /app 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # phpkafkacore 2 | 3 | This library is a *pure PHP* wrapper for https://arnaud.le-blanc.net/php-rdkafka-doc/phpdoc/book.rdkafka.html. It's been tested against https://kafka.apache.org/. 4 | 5 | The library was used for the PHP projects to simplify work and handle common use cases. Inspired by https://github.com/php-amqplib/php-amqplib 6 | 7 | ## Project Maintainers 8 | 9 | https://github.com/Spartaques 10 | 11 | ## Supported Kafka Versions 12 | 13 | Kafka version >= [0.9.0.0 ](https://github.com/apache/kafka/releases/tag/0.9.0.0) 14 | 15 | ## Setup 16 | 17 | Ensure you have [composer](http://getcomposer.org/) installed, then run the following command: 18 | 19 | ```php 20 | composer require spartaques/phpcorekafka 21 | ``` 22 | 23 | 24 | 25 | # Topics 26 | 27 | There are some things, that you should know before creating a topic for your application. 28 | 29 | 1. **Message ordering** 30 | 2. **Replication factor** 31 | 3. **Count of partitions** 32 | 33 | About ordering: **any events that need to stay in a fixed order must go in the same topic** (and they must also use the same partitioning key). So, as a rule of thumb, we could say that all events about the same entity need to go in the same topic. Partition key described below. 34 | 35 | Replication factor should be used for achieving fault tolerance. 36 | 37 | Partitions is scaling unit in kafka, so we must know our data and calculate proper number. 38 | 39 | # Producing 40 | 41 | The most important things to know about producing is: 42 | 43 | 1. **mode (sync, async, fire & forget)** 44 | 2. **configurations** 45 | 3. **messages order** 46 | 4. **payload schema** 47 | 48 | 49 | 1 Because kafka works in async mode by default, we can loose some messages if smth went wrong with broker. To avoid this, we can simply wait for response from broker. 50 | 51 | examples/producesync.php describes sync producing. We simply use timeout for produce message, that means we wait for response from server. 52 | 53 | examples/produce_async.php describes async producing. We just use 0 as timeout value for poll. 54 | 55 | Dont forget to call flush() to be sure that all events are published when using async mode. It's related to php because php dies after each request, so some events might lost. 56 | 57 | 2 We should understand what we need from producer and broker. All things can be configured for best result. 58 | 59 | The most important configuration parameters for producer: 60 | 61 | ***acks** - The acks parameter controls how many partition replicas must receive the record before the producer can consider the write successful. This option has a significant impact on how likely messages are to be lost.* 62 | 63 | ***buffer.memory** - This sets the amount of memory the producer will use to buffer messages waiting to be sent to brokers.* 64 | 65 | ***compression.type** - By default, messages are sent uncompressed. This parameter can be set to snappy, gzip, or lz4, in which case the corresponding compression algorithms will be used to compress the data before sending it to the brokers.* 66 | 67 | ***retries** - When the producer receives an error message from the server, the error could be transient (e.g., a lack of leader for a partition). In this case, the value of the retries parameter will control how many times the producer will retry sending the message before giving up and notifying the client of an issue.* 68 | 69 | *etc...* 70 | 71 | 3 Kafka use ordering strategy (partitioner) to deliver messages to partitions depends on use case. https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md describes partitioner parameter that handle this. 72 | 73 | example: examples/produceorder.php 74 | 75 | *TIP: Always use RD_KAFKA_PARTITION_UA as partition number. Kafka will care about everything.* 76 | 77 | In most of the situations, ordering is not important. But if we, for example, use kafka for storing client, fact that client address info came before general info is not appropriate. 78 | 79 | So, in such situations, we have 2 choise: 80 | 81 | 1. Use only 1 partition and 1 consumer, so messages will be handled in one order. 82 | 2. Use many partitions and consumers, but messages related to some entity should always go to one partition. Kafka consistent hashing and partitioner handle for us this case by default. (Using Rabbitmq for example, you should write this logic yourself). 83 | 84 | 4 Message payload schema should be clean and simple, and contain only data that are used. 85 | 86 | For more advanced schema, use https://avro.apache.org/ serializer. 87 | 88 | 89 | 90 | # Consuming 91 | 92 | 1. **Committing** 93 | 94 | 2. **rebalancing** 95 | 96 | 3. **configurations** 97 | 98 | 99 | 1 Depend on data that should be processed, we can choose what behaviour is appropriate for us. 100 | 101 | If duplication or loosing is not a problem, using automatic commit (that works by default) will be good decision. 102 | 103 | When we want to avoid such behaviour, we should use manual commit. 104 | 105 | Manual commit works only when **enable.auto.commit** is set to false, and have 2 mode: 106 | 107 | 108 | 1) Synchronous - commit last offset after processing message. example: examples/consumesynccommit.php 109 | 110 | 2) Async - non-blocking commit last offset after processing message. example: examples/consume_async_commit.php 111 | 112 | 3) Autocommit (by default) - examples/consumeautocommit.php 113 | 114 | 115 | 2 Rebalancing is a process of reassigning partitions to available consumers. It starts by consumers leader when it not receive heartbeats by one of the consumers after some period. When we use manual commit mode, we should commit offset before rebalancing starts. 116 | 117 | Example: src/Common/DefaultCallbacks.php , syncRebalance(). 118 | 119 | 120 | 3 configurations 121 | 122 | most important configuration parameters: 123 | 124 | ***enable.auto.commit** - This parameter controls whether the consumer will commit offsets automatically, and defaults to true.* 125 | 126 | ***fetch.min.bytes** - This property allows a consumer to specify the minimum amount of data that it wants to receive from the broker when fetching records.* 127 | 128 | ***fetch.max.wait.ms** - By setting fetch.min.bytes, you tell Kafka to wait until it has enough data to send before responding to the consumer.* 129 | 130 | ***max.partition.fetch.bytes** - This property controls the maximum number of bytes the server will return per parti‐ tion. The default is 1 MB.* 131 | 132 | ***session.timeout.ms** - The amount of time a consumer can be out of contact with the brokers while still considered alive defaults to 3 seconds.* 133 | 134 | ***auto.offset.reset** - This property controls the behavior of the consumer when it starts reading a partition for which it doesn’t have a committed offset or if the committed offset it has is invalid (usually because the consumer was down for so long that the record with that offset was already aged out of the broker).* 135 | 136 | ***max.poll.records** - This controls the maximum number of records that a single call to poll() will return.* 137 | 138 | ***receive.buffer.bytes and send.buffer.bytes** - These are the sizes of the TCP send and receive buffers used by the sockets when writing and reading data.* 139 | 140 | *For more info: https://github.com/edenhill/librdkafka/blob/master/CONFIGURATION.md* 141 | 142 | 143 | 144 | # Using in Production 145 | 146 | There are 2 problems that should be handled: 147 | 148 | 1. kafka 149 | 2. server where code works. 150 | 151 | In 1 situation, we must handle all errors, log and analyse. For this purpose we can register callbacks 152 | 153 | And push some notifications using custom callback. Example: 154 | 155 | src/Common/DefaultCallbacks.php error() method. 156 | 157 | In 2 sutiation, we should use some process manager aka **supervisor** for monitoring our processes, and be confidence that our consumers handle signals and exit (close connections) gracefully. 158 | 159 | # Using with frameworks 160 | 161 | This library is framework agnostic. 162 | 163 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "spartaques/phpcorekafka", 3 | "description": "Wrapper for php rdkafka", 4 | "type": "library", 5 | "config": { 6 | "preferred-install": { 7 | "*": "dist" 8 | }, 9 | "sort-packages": true 10 | }, 11 | "authors": [ 12 | { 13 | "name": "Andrew Bashuk ", 14 | "email": "96andlgrac@gmail.com", 15 | "role": "Developer" 16 | } 17 | ], 18 | "autoload": { 19 | "psr-4": { 20 | "Spartaques\\CoreKafka\\": "src", 21 | "Spartaques\\CoreKafka\\Examples\\": "Examples" 22 | } 23 | }, 24 | 25 | "require": { 26 | "php": "*", 27 | "ext-pcntl": "*", 28 | "ext-rdkafka": "*", 29 | "symfony/console": "~6.0|~5.0|~4.0|~3.0|^2.4.2|~2.3.10" 30 | }, 31 | "minimum-stability": "stable", 32 | "prefer-stable": true 33 | } 34 | -------------------------------------------------------------------------------- /composer.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_readme": [ 3 | "This file locks the dependencies of your project to a known state", 4 | "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", 5 | "This file is @generated automatically" 6 | ], 7 | "content-hash": "d48db8879e1faedf92190cd771adfb4b", 8 | "packages": [ 9 | { 10 | "name": "psr/container", 11 | "version": "2.0.2", 12 | "source": { 13 | "type": "git", 14 | "url": "https://github.com/php-fig/container.git", 15 | "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963" 16 | }, 17 | "dist": { 18 | "type": "zip", 19 | "url": "https://api.github.com/repos/php-fig/container/zipball/c71ecc56dfe541dbd90c5360474fbc405f8d5963", 20 | "reference": "c71ecc56dfe541dbd90c5360474fbc405f8d5963", 21 | "shasum": "" 22 | }, 23 | "require": { 24 | "php": ">=7.4.0" 25 | }, 26 | "type": "library", 27 | "extra": { 28 | "branch-alias": { 29 | "dev-master": "2.0.x-dev" 30 | } 31 | }, 32 | "autoload": { 33 | "psr-4": { 34 | "Psr\\Container\\": "src/" 35 | } 36 | }, 37 | "notification-url": "https://packagist.org/downloads/", 38 | "license": [ 39 | "MIT" 40 | ], 41 | "authors": [ 42 | { 43 | "name": "PHP-FIG", 44 | "homepage": "https://www.php-fig.org/" 45 | } 46 | ], 47 | "description": "Common Container Interface (PHP FIG PSR-11)", 48 | "homepage": "https://github.com/php-fig/container", 49 | "keywords": [ 50 | "PSR-11", 51 | "container", 52 | "container-interface", 53 | "container-interop", 54 | "psr" 55 | ], 56 | "support": { 57 | "issues": "https://github.com/php-fig/container/issues", 58 | "source": "https://github.com/php-fig/container/tree/2.0.2" 59 | }, 60 | "time": "2021-11-05T16:47:00+00:00" 61 | }, 62 | { 63 | "name": "symfony/console", 64 | "version": "v6.0.1", 65 | "source": { 66 | "type": "git", 67 | "url": "https://github.com/symfony/console.git", 68 | "reference": "fafd9802d386bf1c267e0249ddb7ceb14dcfdad4" 69 | }, 70 | "dist": { 71 | "type": "zip", 72 | "url": "https://api.github.com/repos/symfony/console/zipball/fafd9802d386bf1c267e0249ddb7ceb14dcfdad4", 73 | "reference": "fafd9802d386bf1c267e0249ddb7ceb14dcfdad4", 74 | "shasum": "" 75 | }, 76 | "require": { 77 | "php": ">=8.0.2", 78 | "symfony/polyfill-mbstring": "~1.0", 79 | "symfony/service-contracts": "^1.1|^2|^3", 80 | "symfony/string": "^5.4|^6.0" 81 | }, 82 | "conflict": { 83 | "symfony/dependency-injection": "<5.4", 84 | "symfony/dotenv": "<5.4", 85 | "symfony/event-dispatcher": "<5.4", 86 | "symfony/lock": "<5.4", 87 | "symfony/process": "<5.4" 88 | }, 89 | "provide": { 90 | "psr/log-implementation": "1.0|2.0|3.0" 91 | }, 92 | "require-dev": { 93 | "psr/log": "^1|^2|^3", 94 | "symfony/config": "^5.4|^6.0", 95 | "symfony/dependency-injection": "^5.4|^6.0", 96 | "symfony/event-dispatcher": "^5.4|^6.0", 97 | "symfony/lock": "^5.4|^6.0", 98 | "symfony/process": "^5.4|^6.0", 99 | "symfony/var-dumper": "^5.4|^6.0" 100 | }, 101 | "suggest": { 102 | "psr/log": "For using the console logger", 103 | "symfony/event-dispatcher": "", 104 | "symfony/lock": "", 105 | "symfony/process": "" 106 | }, 107 | "type": "library", 108 | "autoload": { 109 | "psr-4": { 110 | "Symfony\\Component\\Console\\": "" 111 | }, 112 | "exclude-from-classmap": [ 113 | "/Tests/" 114 | ] 115 | }, 116 | "notification-url": "https://packagist.org/downloads/", 117 | "license": [ 118 | "MIT" 119 | ], 120 | "authors": [ 121 | { 122 | "name": "Fabien Potencier", 123 | "email": "fabien@symfony.com" 124 | }, 125 | { 126 | "name": "Symfony Community", 127 | "homepage": "https://symfony.com/contributors" 128 | } 129 | ], 130 | "description": "Eases the creation of beautiful and testable command line interfaces", 131 | "homepage": "https://symfony.com", 132 | "keywords": [ 133 | "cli", 134 | "command line", 135 | "console", 136 | "terminal" 137 | ], 138 | "support": { 139 | "source": "https://github.com/symfony/console/tree/v6.0.1" 140 | }, 141 | "funding": [ 142 | { 143 | "url": "https://symfony.com/sponsor", 144 | "type": "custom" 145 | }, 146 | { 147 | "url": "https://github.com/fabpot", 148 | "type": "github" 149 | }, 150 | { 151 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 152 | "type": "tidelift" 153 | } 154 | ], 155 | "time": "2021-12-09T12:47:37+00:00" 156 | }, 157 | { 158 | "name": "symfony/polyfill-ctype", 159 | "version": "v1.23.0", 160 | "source": { 161 | "type": "git", 162 | "url": "https://github.com/symfony/polyfill-ctype.git", 163 | "reference": "46cd95797e9df938fdd2b03693b5fca5e64b01ce" 164 | }, 165 | "dist": { 166 | "type": "zip", 167 | "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/46cd95797e9df938fdd2b03693b5fca5e64b01ce", 168 | "reference": "46cd95797e9df938fdd2b03693b5fca5e64b01ce", 169 | "shasum": "" 170 | }, 171 | "require": { 172 | "php": ">=7.1" 173 | }, 174 | "suggest": { 175 | "ext-ctype": "For best performance" 176 | }, 177 | "type": "library", 178 | "extra": { 179 | "branch-alias": { 180 | "dev-main": "1.23-dev" 181 | }, 182 | "thanks": { 183 | "name": "symfony/polyfill", 184 | "url": "https://github.com/symfony/polyfill" 185 | } 186 | }, 187 | "autoload": { 188 | "psr-4": { 189 | "Symfony\\Polyfill\\Ctype\\": "" 190 | }, 191 | "files": [ 192 | "bootstrap.php" 193 | ] 194 | }, 195 | "notification-url": "https://packagist.org/downloads/", 196 | "license": [ 197 | "MIT" 198 | ], 199 | "authors": [ 200 | { 201 | "name": "Gert de Pagter", 202 | "email": "BackEndTea@gmail.com" 203 | }, 204 | { 205 | "name": "Symfony Community", 206 | "homepage": "https://symfony.com/contributors" 207 | } 208 | ], 209 | "description": "Symfony polyfill for ctype functions", 210 | "homepage": "https://symfony.com", 211 | "keywords": [ 212 | "compatibility", 213 | "ctype", 214 | "polyfill", 215 | "portable" 216 | ], 217 | "support": { 218 | "source": "https://github.com/symfony/polyfill-ctype/tree/v1.23.0" 219 | }, 220 | "funding": [ 221 | { 222 | "url": "https://symfony.com/sponsor", 223 | "type": "custom" 224 | }, 225 | { 226 | "url": "https://github.com/fabpot", 227 | "type": "github" 228 | }, 229 | { 230 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 231 | "type": "tidelift" 232 | } 233 | ], 234 | "time": "2021-02-19T12:13:01+00:00" 235 | }, 236 | { 237 | "name": "symfony/polyfill-intl-grapheme", 238 | "version": "v1.23.1", 239 | "source": { 240 | "type": "git", 241 | "url": "https://github.com/symfony/polyfill-intl-grapheme.git", 242 | "reference": "16880ba9c5ebe3642d1995ab866db29270b36535" 243 | }, 244 | "dist": { 245 | "type": "zip", 246 | "url": "https://api.github.com/repos/symfony/polyfill-intl-grapheme/zipball/16880ba9c5ebe3642d1995ab866db29270b36535", 247 | "reference": "16880ba9c5ebe3642d1995ab866db29270b36535", 248 | "shasum": "" 249 | }, 250 | "require": { 251 | "php": ">=7.1" 252 | }, 253 | "suggest": { 254 | "ext-intl": "For best performance" 255 | }, 256 | "type": "library", 257 | "extra": { 258 | "branch-alias": { 259 | "dev-main": "1.23-dev" 260 | }, 261 | "thanks": { 262 | "name": "symfony/polyfill", 263 | "url": "https://github.com/symfony/polyfill" 264 | } 265 | }, 266 | "autoload": { 267 | "psr-4": { 268 | "Symfony\\Polyfill\\Intl\\Grapheme\\": "" 269 | }, 270 | "files": [ 271 | "bootstrap.php" 272 | ] 273 | }, 274 | "notification-url": "https://packagist.org/downloads/", 275 | "license": [ 276 | "MIT" 277 | ], 278 | "authors": [ 279 | { 280 | "name": "Nicolas Grekas", 281 | "email": "p@tchwork.com" 282 | }, 283 | { 284 | "name": "Symfony Community", 285 | "homepage": "https://symfony.com/contributors" 286 | } 287 | ], 288 | "description": "Symfony polyfill for intl's grapheme_* functions", 289 | "homepage": "https://symfony.com", 290 | "keywords": [ 291 | "compatibility", 292 | "grapheme", 293 | "intl", 294 | "polyfill", 295 | "portable", 296 | "shim" 297 | ], 298 | "support": { 299 | "source": "https://github.com/symfony/polyfill-intl-grapheme/tree/v1.23.1" 300 | }, 301 | "funding": [ 302 | { 303 | "url": "https://symfony.com/sponsor", 304 | "type": "custom" 305 | }, 306 | { 307 | "url": "https://github.com/fabpot", 308 | "type": "github" 309 | }, 310 | { 311 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 312 | "type": "tidelift" 313 | } 314 | ], 315 | "time": "2021-05-27T12:26:48+00:00" 316 | }, 317 | { 318 | "name": "symfony/polyfill-intl-normalizer", 319 | "version": "v1.23.0", 320 | "source": { 321 | "type": "git", 322 | "url": "https://github.com/symfony/polyfill-intl-normalizer.git", 323 | "reference": "8590a5f561694770bdcd3f9b5c69dde6945028e8" 324 | }, 325 | "dist": { 326 | "type": "zip", 327 | "url": "https://api.github.com/repos/symfony/polyfill-intl-normalizer/zipball/8590a5f561694770bdcd3f9b5c69dde6945028e8", 328 | "reference": "8590a5f561694770bdcd3f9b5c69dde6945028e8", 329 | "shasum": "" 330 | }, 331 | "require": { 332 | "php": ">=7.1" 333 | }, 334 | "suggest": { 335 | "ext-intl": "For best performance" 336 | }, 337 | "type": "library", 338 | "extra": { 339 | "branch-alias": { 340 | "dev-main": "1.23-dev" 341 | }, 342 | "thanks": { 343 | "name": "symfony/polyfill", 344 | "url": "https://github.com/symfony/polyfill" 345 | } 346 | }, 347 | "autoload": { 348 | "psr-4": { 349 | "Symfony\\Polyfill\\Intl\\Normalizer\\": "" 350 | }, 351 | "files": [ 352 | "bootstrap.php" 353 | ], 354 | "classmap": [ 355 | "Resources/stubs" 356 | ] 357 | }, 358 | "notification-url": "https://packagist.org/downloads/", 359 | "license": [ 360 | "MIT" 361 | ], 362 | "authors": [ 363 | { 364 | "name": "Nicolas Grekas", 365 | "email": "p@tchwork.com" 366 | }, 367 | { 368 | "name": "Symfony Community", 369 | "homepage": "https://symfony.com/contributors" 370 | } 371 | ], 372 | "description": "Symfony polyfill for intl's Normalizer class and related functions", 373 | "homepage": "https://symfony.com", 374 | "keywords": [ 375 | "compatibility", 376 | "intl", 377 | "normalizer", 378 | "polyfill", 379 | "portable", 380 | "shim" 381 | ], 382 | "support": { 383 | "source": "https://github.com/symfony/polyfill-intl-normalizer/tree/v1.23.0" 384 | }, 385 | "funding": [ 386 | { 387 | "url": "https://symfony.com/sponsor", 388 | "type": "custom" 389 | }, 390 | { 391 | "url": "https://github.com/fabpot", 392 | "type": "github" 393 | }, 394 | { 395 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 396 | "type": "tidelift" 397 | } 398 | ], 399 | "time": "2021-02-19T12:13:01+00:00" 400 | }, 401 | { 402 | "name": "symfony/polyfill-mbstring", 403 | "version": "v1.23.1", 404 | "source": { 405 | "type": "git", 406 | "url": "https://github.com/symfony/polyfill-mbstring.git", 407 | "reference": "9174a3d80210dca8daa7f31fec659150bbeabfc6" 408 | }, 409 | "dist": { 410 | "type": "zip", 411 | "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/9174a3d80210dca8daa7f31fec659150bbeabfc6", 412 | "reference": "9174a3d80210dca8daa7f31fec659150bbeabfc6", 413 | "shasum": "" 414 | }, 415 | "require": { 416 | "php": ">=7.1" 417 | }, 418 | "suggest": { 419 | "ext-mbstring": "For best performance" 420 | }, 421 | "type": "library", 422 | "extra": { 423 | "branch-alias": { 424 | "dev-main": "1.23-dev" 425 | }, 426 | "thanks": { 427 | "name": "symfony/polyfill", 428 | "url": "https://github.com/symfony/polyfill" 429 | } 430 | }, 431 | "autoload": { 432 | "psr-4": { 433 | "Symfony\\Polyfill\\Mbstring\\": "" 434 | }, 435 | "files": [ 436 | "bootstrap.php" 437 | ] 438 | }, 439 | "notification-url": "https://packagist.org/downloads/", 440 | "license": [ 441 | "MIT" 442 | ], 443 | "authors": [ 444 | { 445 | "name": "Nicolas Grekas", 446 | "email": "p@tchwork.com" 447 | }, 448 | { 449 | "name": "Symfony Community", 450 | "homepage": "https://symfony.com/contributors" 451 | } 452 | ], 453 | "description": "Symfony polyfill for the Mbstring extension", 454 | "homepage": "https://symfony.com", 455 | "keywords": [ 456 | "compatibility", 457 | "mbstring", 458 | "polyfill", 459 | "portable", 460 | "shim" 461 | ], 462 | "support": { 463 | "source": "https://github.com/symfony/polyfill-mbstring/tree/v1.23.1" 464 | }, 465 | "funding": [ 466 | { 467 | "url": "https://symfony.com/sponsor", 468 | "type": "custom" 469 | }, 470 | { 471 | "url": "https://github.com/fabpot", 472 | "type": "github" 473 | }, 474 | { 475 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 476 | "type": "tidelift" 477 | } 478 | ], 479 | "time": "2021-05-27T12:26:48+00:00" 480 | }, 481 | { 482 | "name": "symfony/service-contracts", 483 | "version": "v3.0.0", 484 | "source": { 485 | "type": "git", 486 | "url": "https://github.com/symfony/service-contracts.git", 487 | "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603" 488 | }, 489 | "dist": { 490 | "type": "zip", 491 | "url": "https://api.github.com/repos/symfony/service-contracts/zipball/36715ebf9fb9db73db0cb24263c79077c6fe8603", 492 | "reference": "36715ebf9fb9db73db0cb24263c79077c6fe8603", 493 | "shasum": "" 494 | }, 495 | "require": { 496 | "php": ">=8.0.2", 497 | "psr/container": "^2.0" 498 | }, 499 | "conflict": { 500 | "ext-psr": "<1.1|>=2" 501 | }, 502 | "suggest": { 503 | "symfony/service-implementation": "" 504 | }, 505 | "type": "library", 506 | "extra": { 507 | "branch-alias": { 508 | "dev-main": "3.0-dev" 509 | }, 510 | "thanks": { 511 | "name": "symfony/contracts", 512 | "url": "https://github.com/symfony/contracts" 513 | } 514 | }, 515 | "autoload": { 516 | "psr-4": { 517 | "Symfony\\Contracts\\Service\\": "" 518 | } 519 | }, 520 | "notification-url": "https://packagist.org/downloads/", 521 | "license": [ 522 | "MIT" 523 | ], 524 | "authors": [ 525 | { 526 | "name": "Nicolas Grekas", 527 | "email": "p@tchwork.com" 528 | }, 529 | { 530 | "name": "Symfony Community", 531 | "homepage": "https://symfony.com/contributors" 532 | } 533 | ], 534 | "description": "Generic abstractions related to writing services", 535 | "homepage": "https://symfony.com", 536 | "keywords": [ 537 | "abstractions", 538 | "contracts", 539 | "decoupling", 540 | "interfaces", 541 | "interoperability", 542 | "standards" 543 | ], 544 | "support": { 545 | "source": "https://github.com/symfony/service-contracts/tree/v3.0.0" 546 | }, 547 | "funding": [ 548 | { 549 | "url": "https://symfony.com/sponsor", 550 | "type": "custom" 551 | }, 552 | { 553 | "url": "https://github.com/fabpot", 554 | "type": "github" 555 | }, 556 | { 557 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 558 | "type": "tidelift" 559 | } 560 | ], 561 | "time": "2021-11-04T17:53:12+00:00" 562 | }, 563 | { 564 | "name": "symfony/string", 565 | "version": "v6.0.1", 566 | "source": { 567 | "type": "git", 568 | "url": "https://github.com/symfony/string.git", 569 | "reference": "0cfed595758ec6e0a25591bdc8ca733c1896af32" 570 | }, 571 | "dist": { 572 | "type": "zip", 573 | "url": "https://api.github.com/repos/symfony/string/zipball/0cfed595758ec6e0a25591bdc8ca733c1896af32", 574 | "reference": "0cfed595758ec6e0a25591bdc8ca733c1896af32", 575 | "shasum": "" 576 | }, 577 | "require": { 578 | "php": ">=8.0.2", 579 | "symfony/polyfill-ctype": "~1.8", 580 | "symfony/polyfill-intl-grapheme": "~1.0", 581 | "symfony/polyfill-intl-normalizer": "~1.0", 582 | "symfony/polyfill-mbstring": "~1.0" 583 | }, 584 | "conflict": { 585 | "symfony/translation-contracts": "<2.0" 586 | }, 587 | "require-dev": { 588 | "symfony/error-handler": "^5.4|^6.0", 589 | "symfony/http-client": "^5.4|^6.0", 590 | "symfony/translation-contracts": "^2.0|^3.0", 591 | "symfony/var-exporter": "^5.4|^6.0" 592 | }, 593 | "type": "library", 594 | "autoload": { 595 | "psr-4": { 596 | "Symfony\\Component\\String\\": "" 597 | }, 598 | "files": [ 599 | "Resources/functions.php" 600 | ], 601 | "exclude-from-classmap": [ 602 | "/Tests/" 603 | ] 604 | }, 605 | "notification-url": "https://packagist.org/downloads/", 606 | "license": [ 607 | "MIT" 608 | ], 609 | "authors": [ 610 | { 611 | "name": "Nicolas Grekas", 612 | "email": "p@tchwork.com" 613 | }, 614 | { 615 | "name": "Symfony Community", 616 | "homepage": "https://symfony.com/contributors" 617 | } 618 | ], 619 | "description": "Provides an object-oriented API to strings and deals with bytes, UTF-8 code points and grapheme clusters in a unified way", 620 | "homepage": "https://symfony.com", 621 | "keywords": [ 622 | "grapheme", 623 | "i18n", 624 | "string", 625 | "unicode", 626 | "utf-8", 627 | "utf8" 628 | ], 629 | "support": { 630 | "source": "https://github.com/symfony/string/tree/v6.0.1" 631 | }, 632 | "funding": [ 633 | { 634 | "url": "https://symfony.com/sponsor", 635 | "type": "custom" 636 | }, 637 | { 638 | "url": "https://github.com/fabpot", 639 | "type": "github" 640 | }, 641 | { 642 | "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", 643 | "type": "tidelift" 644 | } 645 | ], 646 | "time": "2021-12-08T15:13:44+00:00" 647 | } 648 | ], 649 | "packages-dev": [], 650 | "aliases": [], 651 | "minimum-stability": "stable", 652 | "stability-flags": [], 653 | "prefer-stable": true, 654 | "prefer-lowest": false, 655 | "platform": { 656 | "php": "*", 657 | "ext-pcntl": "*", 658 | "ext-rdkafka": "5.*" 659 | }, 660 | "platform-dev": [], 661 | "plugin-api-version": "2.1.0" 662 | } 663 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | #you can include needed services to your network using docker-compose service merging : docker-compose -f docker-compose.yml -f docker-composer.elk.yml up 4 | 5 | services: 6 | app: 7 | build: 8 | context: ./ 9 | dockerfile: Dockerfile 10 | ports: 11 | - "8085:80" 12 | volumes: 13 | - ./:/app 14 | depends_on: 15 | # - redis 16 | - kafka 17 | 18 | zookeeper: 19 | image: wurstmeister/zookeeper 20 | ports: 21 | - "2181:2181" 22 | kafka: 23 | image: wurstmeister/kafka 24 | ports: 25 | - "9092:9092" 26 | environment: 27 | KAFKA_CREATE_TOPICS: "hell2:10:1,oauthdata:5:1,test:1:1" 28 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 29 | KAFKA_LISTENERS: PLAINTEXT://kafka:9092 30 | -------------------------------------------------------------------------------- /examples/AsyncCallback.php: -------------------------------------------------------------------------------- 1 | offset); 15 | $consumerWrapper->commitAsync(); 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /examples/AsyncHandledCallback.php: -------------------------------------------------------------------------------- 1 | offset); 21 | $consumerWrapper->commitAsync(); 22 | } finally { 23 | try { 24 | $consumerWrapper->commitSync(); 25 | } finally { 26 | $consumerWrapper->close(); 27 | } 28 | } 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /examples/SyncCallback.php: -------------------------------------------------------------------------------- 1 | commitSync(); 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/consume_async_commit.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->consume(), 20 | ConfigurationCallbacksKeys::DELIVERY_REPORT => $callbacksInstance->delivery(), 21 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 22 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 23 | ConfigurationCallbacksKeys::OFFSET_COMMIT => $callbacksInstance->commit(), 24 | ConfigurationCallbacksKeys::REBALANCE => $callbacksInstance->rebalance(), 25 | ConfigurationCallbacksKeys::STATISTICS => $callbacksInstance->statistics(), 26 | ] 27 | ); 28 | 29 | $consumer = new ConsumerWrapper(); 30 | 31 | $consumeDataObject = new ConsumerProperties( 32 | [ 33 | 'group.id' => 'test2', 34 | 'client.id' => 'test', 35 | 'metadata.broker.list' => 'kafka:9092', 36 | 'auto.offset.reset' => 'smallest', 37 | 'enable.auto.commit' => "false", 38 | // 'auto.commit.interval.ms' => 0 39 | ], 40 | $collection 41 | ); 42 | 43 | 44 | // ітак, питання в тому як передати обєкт ConsumerWrapper у всі коллбеки? 45 | $consumer->init($consumeDataObject)->consume(['test123'], new AsyncCallback()); 46 | -------------------------------------------------------------------------------- /examples/consumeautocommit.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->consume(), 19 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 20 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 21 | ConfigurationCallbacksKeys::REBALANCE => $callbacksInstance->rebalance(), 22 | ] 23 | ); 24 | 25 | $consumer = new ConsumerWrapper(); 26 | 27 | $consumeDataObject = new ConsumerProperties( 28 | [ 29 | 'group.id' => 'test2', 30 | 'client.id' => 'test', 31 | 'metadata.broker.list' => 'kafka:9092', 32 | 'auto.offset.reset' => 'smallest', 33 | // 'enable.auto.commit' => "false", 34 | // 'auto.commit.interval.ms' => 0 35 | 'log_level' => 6 36 | ], 37 | $collection 38 | ); 39 | 40 | 41 | // ітак, питання в тому як передати обєкт ConsumerWrapper у всі коллбеки? 42 | $consumer->init($consumeDataObject)->consume(['test123'], function (\RdKafka\Message $message, ConsumerWrapper $consumer) { 43 | var_dump($message); 44 | }); 45 | -------------------------------------------------------------------------------- /examples/consumebatch.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->consume(), 19 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 20 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 21 | ConfigurationCallbacksKeys::REBALANCE => $callbacksInstance->rebalance(), 22 | ] 23 | ); 24 | 25 | $consumer = new ConsumerWrapper(); 26 | 27 | $consumeDataObject = new ConsumerProperties( 28 | [ 29 | 'group.id' => 'test2', 30 | 'client.id' => 'test', 31 | 'metadata.broker.list' => 'kafka:9092', 32 | 'auto.offset.reset' => 'latest', 33 | 'auto.commit.interval.ms' => 1000, 34 | 'message.max.bytes' => 15729152, 35 | // 'enable.auto.commit' => "false", 36 | // 'auto.commit.interval.ms' => 0 37 | 'log_level' => 6 38 | ], 39 | $collection 40 | ); 41 | 42 | 43 | // ітак, питання в тому як передати обєкт ConsumerWrapper у всі коллбеки? 44 | $consumer->initOld('kafka:9092', $consumeDataObject)->consumeBatch('test123', 0, 5000, 10000, function (array $messages, ConsumerWrapper $consumer) { 45 | var_dump(microtime(true). ' | '.count($messages)); 46 | }); 47 | -------------------------------------------------------------------------------- /examples/consumesynccommit.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->consume(), 19 | ConfigurationCallbacksKeys::DELIVERY_REPORT => $callbacksInstance->delivery(), 20 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 21 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 22 | ConfigurationCallbacksKeys::OFFSET_COMMIT => $callbacksInstance->commit(), 23 | ConfigurationCallbacksKeys::REBALANCE => $callbacksInstance->rebalance(), 24 | ConfigurationCallbacksKeys::STATISTICS => $callbacksInstance->statistics(), 25 | ] 26 | ); 27 | 28 | $consumer = new ConsumerWrapper(); 29 | 30 | $consumeDataObject = new ConsumerProperties( 31 | [ 32 | 'group.id' => 'test1', 33 | 'client.id' => 'test', 34 | 'metadata.broker.list' => 'kafka:9092', 35 | 'auto.offset.reset' => 'smallest', 36 | 'enable.auto.commit' => "false", 37 | // 'auto.commit.interval.ms' => 0 38 | ], 39 | $collection 40 | ); 41 | 42 | 43 | // ітак, питання в тому як передати обєкт ConsumerWrapper у всі коллбеки? 44 | $consumer->init($consumeDataObject)->consume(['test123'], new SyncCallback()); 45 | -------------------------------------------------------------------------------- /examples/produce_async.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->delivery(), 20 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 21 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 22 | ]); 23 | 24 | // producer initialization object 25 | $produceData = new ProducerProperties( 26 | 'test123', 27 | [ 28 | 'metadata.broker.list' => 'kafka:9092', 29 | 'client.id' => 'clientid', 30 | // 'debug' => 'all' 31 | ], 32 | [], 33 | $collection 34 | ); 35 | 36 | for ($i = 0; $i < 100; $i++) { 37 | // produce message using ProducerDataObject 38 | $producer->init($produceData)->produce(new ProducerData("Message $i", RD_KAFKA_PARTITION_UA, 0, $i)); 39 | var_dump($i); 40 | // sleep(2); 41 | } 42 | 43 | 44 | $producer->flush(); 45 | 46 | -------------------------------------------------------------------------------- /examples/produceorder.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->delivery(), 20 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 21 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 22 | ]); 23 | 24 | // producer initialization object 25 | $produceData = new ProducerProperties( 26 | 'test123', 27 | [ 28 | 'metadata.broker.list' => 'kafka:9092', 29 | 'client.id' => 'clientid', 30 | ], 31 | [ 32 | ], 33 | $collection 34 | ); 35 | 36 | $json = json_encode([ 37 | 'sfqsf' => 'fwqdfwfqwfqwf', 38 | 1 => 'fwqdfwfqwfqwf' , 39 | 2 => 'fwqdfwfqwfqwf' , 40 | 3 => 'fwqdfwfqwfqwf' , 41 | 4 => 'fwqdfwfqwfqwf' , 42 | 5 => 'fwqdfwfqwfqwf' , 43 | 6 => 'fwqdfwfqwfqwf' , 44 | 7 => 'fwqdfwfqwfqwf' , 45 | 8 => 'fwqdfwfqwfqwf' , 46 | 9 => 'fwqdfwfqwfqwf' 47 | ]); 48 | 49 | for ($i = 0; $i < 100; $i++) { 50 | var_dump($i); 51 | 52 | // produce message using ProducerDataObject 53 | $producer->init($produceData)->produce(new ProducerData($json, 0)); 54 | } 55 | 56 | $producer->flush(); 57 | 58 | -------------------------------------------------------------------------------- /examples/producesync.php: -------------------------------------------------------------------------------- 1 | $callbacksInstance->delivery(), 20 | ConfigurationCallbacksKeys::ERROR => $callbacksInstance->error(), 21 | ConfigurationCallbacksKeys::LOG => $callbacksInstance->log(), 22 | ]); 23 | 24 | // producer initialization object 25 | $produceData = new ProducerProperties( 26 | 'test123', 27 | [ 28 | 'metadata.broker.list' => 'kafka:9092', 29 | 'client.id' => 'clientid', 30 | ], 31 | [], 32 | $collection 33 | ); 34 | 35 | for ($i = 0; $i < 10; $i++) { 36 | // produce message using ProducerDataObject 37 | $producer->init($produceData)->produce(new ProducerData("Message $i", RD_KAFKA_PARTITION_UA, 0, $i), 100); 38 | } 39 | 40 | $producer->flush(); 41 | 42 | -------------------------------------------------------------------------------- /examples/producewithheaders.php: -------------------------------------------------------------------------------- 1 | 2 | 'kafka:9092', 18 | 'client.id' => 'clientid' 19 | ], 20 | [] 21 | ); 22 | 23 | $headers = [ 24 | 'SomeKey' => 'SomeValue', 25 | 'AnotherKey' => 'AnotherValue', 26 | ]; 27 | 28 | for ($i = 0; $i < 1; $i++) { 29 | // produce message using ProducerDataObject 30 | $producer->init($produceData)->produceWithHeaders(new ProducerData("Message $i", RD_KAFKA_PARTITION_UA, 0, null, $headers)); 31 | } 32 | 33 | $producer->flush(); 34 | 35 | -------------------------------------------------------------------------------- /src/Common/CallbacksCollection.php: -------------------------------------------------------------------------------- 1 | $item) { 16 | if (!in_array($key, ConfigurationCallbacksKeys::CALLBACKS_MAP, true)) { 17 | throw new \RuntimeException('wrong key for callback'); 18 | } 19 | $this->set($key, $item); 20 | } 21 | } 22 | 23 | public function set( string $key, \Closure $callback) 24 | { 25 | $this->items[$key] = $callback; 26 | } 27 | 28 | public function get($key): \Closure 29 | { 30 | return $this->items[$key]; 31 | } 32 | 33 | /** 34 | * @inheritDoc 35 | */ 36 | public function getIterator() 37 | { 38 | return new \ArrayIterator($this->items); 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/Common/ConfigurationCallbacksKeys.php: -------------------------------------------------------------------------------- 1 | getOutput()->info('Assign: '); 27 | var_dump($partitions); 28 | $this->assign($partitions); 29 | break; 30 | 31 | case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: 32 | 33 | $this->getOutput()->writeln('Revoke: '); 34 | 35 | var_dump($partitions); 36 | 37 | $this->commitSync(); 38 | 39 | $this->getOutput()->error('offset commited:'); 40 | $this->assign(NULL); 41 | break; 42 | 43 | default: 44 | throw new KafkaRebalanceCbException($err); 45 | } 46 | }; 47 | } 48 | 49 | /** 50 | * @return \Closure 51 | */ 52 | public function rebalance() : \Closure 53 | { 54 | return function (KafkaConsumer $kafka, $err, array $partitions = null) { 55 | switch ($err) { 56 | case RD_KAFKA_RESP_ERR__ASSIGN_PARTITIONS: 57 | $this->getOutput()->info('Assign: '); 58 | var_dump($partitions); 59 | $this->assign($partitions); 60 | break; 61 | 62 | case RD_KAFKA_RESP_ERR__REVOKE_PARTITIONS: 63 | 64 | $this->getOutput()->writeln('Revoke: '); 65 | 66 | var_dump($partitions); 67 | 68 | $this->assign(NULL); 69 | break; 70 | 71 | default: 72 | throw new KafkaRebalanceCbException($err); 73 | } 74 | }; 75 | } 76 | 77 | /** 78 | * @return \Closure 79 | */ 80 | public function consume(): \Closure 81 | { 82 | return function ($message) { 83 | $this->getOutput()->info('consume callback'); 84 | var_dump($message); 85 | }; 86 | } 87 | 88 | /** 89 | * @return \Closure 90 | */ 91 | public function delivery(): \Closure 92 | { 93 | return function (Kafka $kafka, Message $message) { 94 | if ($message->err) { 95 | $this->getOutput()->warn('message permanently failed to be delivered'); 96 | } else { 97 | $this->getOutput()->info('message successfully delivered'); 98 | // message successfully delivered 99 | } 100 | }; 101 | } 102 | 103 | /** 104 | * @return \Closure 105 | */ 106 | public function error() : \Closure 107 | { 108 | return function ($kafka, $err, $reason) { 109 | $this->getOutput()->warn(sprintf("Kafka error: %s (reason: %s)\n", rd_kafka_err2str($err), $reason)); 110 | }; 111 | } 112 | 113 | /** 114 | * @return \Closure 115 | */ 116 | public function log(): \Closure 117 | { 118 | return function ($kafka, $level, $facility, $message) { 119 | $this->getOutput()->warn(sprintf("Kafka %s: %s (level: %d)\n", $facility, $message, $level)); 120 | }; 121 | } 122 | 123 | /** 124 | * @return \Closure 125 | */ 126 | public function commit(): \Closure 127 | { 128 | return function (KafkaConsumer $kafka, $err, array $partitions) { 129 | 130 | if($err === RD_KAFKA_RESP_ERR__NO_OFFSET) { 131 | return; 132 | } 133 | 134 | $text = 'commit callback. '; 135 | 136 | foreach ($partitions as $partition) { 137 | $text .= "partition # {$partition->getPartition()} . offset # {$partition->getOffset()} | "; 138 | } 139 | 140 | $this->getOutput()->info($text); 141 | }; 142 | } 143 | 144 | /** 145 | * @return \Closure 146 | */ 147 | public function statistics(): \Closure 148 | { 149 | return function ($kafka, $json, $json_len) { 150 | echo 'statistics'; 151 | }; 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /src/Consume/HighLevel/ConsumerProperties.php: -------------------------------------------------------------------------------- 1 | kafkaConf = $kafkaConf; 31 | $this->callbacksCollection = $callbacksCollection; 32 | } 33 | 34 | /** 35 | * @return array 36 | */ 37 | public function getKafkaConf(): array 38 | { 39 | return $this->kafkaConf; 40 | } 41 | 42 | /** 43 | * @return CallbacksCollection 44 | */ 45 | public function getCallbacksCollection(): CallbacksCollection 46 | { 47 | return $this->callbacksCollection; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Consume/HighLevel/ConsumerWrapper.php: -------------------------------------------------------------------------------- 1 | instantiated) { 46 | return $this; 47 | } 48 | 49 | $this->output = new Output(); 50 | 51 | $this->output->comment('Consumer initialization...'); 52 | 53 | $this->defineSignalsHandling(); 54 | 55 | $this->consumer = $this->initConsumerConnection($consumerProperties, $consumerProperties->getCallbacksCollection()); 56 | 57 | $this->instantiated = true; 58 | 59 | $this->output->comment('Consumer initialized'); 60 | 61 | return $this; 62 | } 63 | 64 | // An application should make sure to call consume() at regular intervals, even if no messages are expected, to serve any queued callbacks waiting to be called. 65 | // This is especially important when a rebalnce_cb has been registered as it needs to be called and handled properly to synchronize internal consumer state. 66 | 67 | /** 68 | * @param array $topics 69 | * @param $callback 70 | * @param int $timeout 71 | * @throws KafkaConsumeException 72 | */ 73 | public function consume(array $topics, $callback, int $timeout = 10000):void 74 | { 75 | $this->consumer->subscribe($topics); 76 | 77 | $this->output->info('Waiting for partition assignment... (make take some time when'); 78 | $this->output->info('quickly re-joining the group after leaving it'); 79 | 80 | while (true) { 81 | $message = $this->consumer->consume($timeout); 82 | switch ($message->err) { 83 | case RD_KAFKA_RESP_ERR_NO_ERROR: 84 | $this->callback($callback, $message); 85 | break; 86 | case RD_KAFKA_RESP_ERR__PARTITION_EOF: 87 | $this->output->info('No more messages; will wait for more'); 88 | break; 89 | case RD_KAFKA_RESP_ERR__TIMED_OUT: 90 | $this->output->info('Timed out'); 91 | break; 92 | default: 93 | $this->output->error($message->err); 94 | throw new KafkaConsumeException($message->errstr(), $message->err); 95 | break; 96 | } 97 | } 98 | } 99 | 100 | public function initOld(string $brokerList, ConsumerProperties $consumerProperties) 101 | { 102 | if($this->instantiated) { 103 | return $this; 104 | } 105 | 106 | $this->output = new Output(); 107 | 108 | $this->output->comment('Old Consumer initialization...'); 109 | 110 | $this->defineSignalsHandling(); 111 | 112 | $this->oldConsumer = $this->initOldConsumerCollection($consumerProperties, $consumerProperties->getCallbacksCollection()); 113 | 114 | $this->oldConsumer->addBrokers($brokerList); 115 | 116 | $this->instantiated = true; 117 | 118 | $this->output->comment('Old Consumer initialized'); 119 | 120 | return $this; 121 | } 122 | 123 | public function consumeBatch(string $topic, int $partition , int $timeout_ms , int $batch_size, $callback, $topicConf = null ): array 124 | { 125 | $consumeTopic = $this->oldConsumer->newTopic($topic, $topicConf); 126 | 127 | $this->output->info('batch consuming...'); 128 | 129 | $consumeTopic->consumeStart($partition, RD_KAFKA_OFFSET_STORED); 130 | 131 | while (true) { 132 | $messages = $consumeTopic->consumeBatch($partition, $timeout_ms, $batch_size); 133 | 134 | if($messages === null) { 135 | $this->output->info('Timed out'); 136 | } else { 137 | $this->callbackBatch($callback, $messages); 138 | } 139 | } 140 | } 141 | 142 | /** 143 | * @param $callback 144 | * @param int $timeout 145 | * @throws KafkaConsumeException 146 | */ 147 | public function consumeWithManualAssign($callback, int $timeout = 10000): void 148 | { 149 | $this->output->info('Waiting for partition assignment... (make take some time when'); 150 | $this->output->info('quickly re-joining the group after leaving it'); 151 | 152 | while (true) { 153 | $message = $this->consumer->consume($timeout); 154 | switch ($message->err) { 155 | case RD_KAFKA_RESP_ERR_NO_ERROR: 156 | $this->callback($callback, $message); 157 | break; 158 | case RD_KAFKA_RESP_ERR__PARTITION_EOF: 159 | $this->output->info('No more messages; will wait for more'); 160 | break; 161 | case RD_KAFKA_RESP_ERR__TIMED_OUT: 162 | $this->output->info('Timed out'); 163 | break; 164 | default: 165 | $this->output->info($message->err); 166 | throw new KafkaConsumeException($message->errstr(), $message->err); 167 | break; 168 | } 169 | } 170 | } 171 | 172 | /** 173 | * @param mixed|null $message_or_offsets 174 | * @throws ConsumerShouldBeInstantiatedException 175 | */ 176 | public function commitSync($message_or_offsets = NULL):void 177 | { 178 | if(!$this->instantiated) { 179 | throw new ConsumerShouldBeInstantiatedException(); 180 | } 181 | 182 | $this->consumer->commit($message_or_offsets); 183 | } 184 | 185 | /** 186 | * @param null $message_or_offsets 187 | * @throws ConsumerShouldBeInstantiatedException 188 | */ 189 | public function commitAsync($message_or_offsets = NULL):void 190 | { 191 | if(!$this->instantiated) { 192 | throw new ConsumerShouldBeInstantiatedException(); 193 | } 194 | 195 | $this->consumer->commitAsync($message_or_offsets); 196 | } 197 | 198 | /** 199 | * @return array 200 | * @throws ConsumerShouldBeInstantiatedException 201 | */ 202 | public function getAssignment():array 203 | { 204 | if(!$this->instantiated) { 205 | throw new ConsumerShouldBeInstantiatedException(); 206 | } 207 | 208 | return $this->consumer->getAssignment(); 209 | } 210 | 211 | /** 212 | * @param array $topics 213 | * @param int $timeout_ms 214 | * @return array 215 | * @throws ConsumerShouldBeInstantiatedException 216 | */ 217 | public function getCommittedOffsets(array $topics , int $timeout_ms = 10000): array 218 | { 219 | if(!$this->instantiated) { 220 | throw new ConsumerShouldBeInstantiatedException(); 221 | } 222 | 223 | return $this->consumer->getCommittedOffsets($topics, $timeout_ms); 224 | } 225 | 226 | /** 227 | * @return array 228 | * @throws ConsumerShouldBeInstantiatedException 229 | */ 230 | public function getSubscription(): array 231 | { 232 | if(!$this->instantiated) { 233 | throw new ConsumerShouldBeInstantiatedException(); 234 | } 235 | 236 | return $this->consumer->getSubscription(); 237 | } 238 | 239 | /** 240 | * @param string $topic 241 | * @param int $partition 242 | * @param int $low 243 | * @param int $high 244 | * @param int $timeout_ms 245 | * @throws ConsumerShouldBeInstantiatedException 246 | */ 247 | public function queryWatermarkOffsets(string $topic , int $partition , int &$low , int &$high , int $timeout_ms):void 248 | { 249 | if(!$this->instantiated) { 250 | throw new ConsumerShouldBeInstantiatedException(); 251 | } 252 | 253 | $this->consumer->queryWatermarkOffsets($topic, $partition, $low, $high, $timeout_ms); 254 | } 255 | 256 | /** 257 | * @param array $topics 258 | * @throws ConsumerShouldBeInstantiatedException 259 | */ 260 | public function subscribe(array $topics) :void 261 | { 262 | if(!$this->instantiated) { 263 | throw new ConsumerShouldBeInstantiatedException(); 264 | } 265 | 266 | $this->consumer->subscribe($topics); 267 | } 268 | 269 | 270 | /** 271 | * 272 | */ 273 | public function unsubscribe():void 274 | { 275 | $this->consumer->unsubscribe(); 276 | } 277 | 278 | /** 279 | * @param ConsumerProperties $consumerProperties 280 | * @param CallbacksCollection $callbacksCollection 281 | * @return KafkaConsumer 282 | */ 283 | public function initConsumerConnection(ConsumerProperties $consumerProperties, CallbacksCollection $callbacksCollection): KafkaConsumer 284 | { 285 | return new KafkaConsumer($this->getKafkaConf($consumerProperties, $callbacksCollection)); 286 | } 287 | 288 | public function initOldConsumerCollection(ConsumerProperties $consumerProperties, CallbacksCollection $callbacksCollection): Consumer 289 | { 290 | return new Consumer($this->getKafkaConf($consumerProperties, $callbacksCollection)); 291 | } 292 | 293 | private function getKafkaConf(ConsumerProperties $consumerProperties, CallbacksCollection $callbacksCollection) 294 | { 295 | $kafkaConf = new Conf(); 296 | 297 | foreach ($consumerProperties->getKafkaConf() as $key => $value) { 298 | $kafkaConf->set($key, $value); 299 | } 300 | 301 | /** 302 | * @var \Closure $callback 303 | */ 304 | foreach ($callbacksCollection as $key => $callback) { 305 | switch ($key) { 306 | case ConfigurationCallbacksKeys::CONSUME: {$kafkaConf->setConsumeCb($callback->bindTo($this));} break; 307 | case ConfigurationCallbacksKeys::ERROR: {$kafkaConf->setErrorCb($callback->bindTo($this)); break;} 308 | case ConfigurationCallbacksKeys::LOG: {$kafkaConf->setLogCb($callback->bindTo($this)); break;} 309 | case ConfigurationCallbacksKeys::OFFSET_COMMIT: {$kafkaConf->setOffsetCommitCb($callback->bindTo($this)); break;} 310 | case ConfigurationCallbacksKeys::REBALANCE: {$kafkaConf->setRebalanceCb($callback->bindTo($this)); break;} 311 | case ConfigurationCallbacksKeys::STATISTICS: {$kafkaConf->setStatsCb($callback->bindTo($this)); break;} 312 | } 313 | } 314 | 315 | $this->output->comment('callbacks registered'); 316 | 317 | return $kafkaConf; 318 | } 319 | 320 | /** 321 | * @return bool 322 | */ 323 | public function isInstantiated(): bool 324 | { 325 | return $this->instantiated; 326 | } 327 | 328 | /** 329 | * @param array|null $topic_partitions 330 | * @return $this 331 | * @throws ConsumerShouldBeInstantiatedException 332 | */ 333 | public function assign( array $topic_partitions = null) 334 | { 335 | if(!$this->instantiated) { 336 | throw new ConsumerShouldBeInstantiatedException(); 337 | } 338 | 339 | $this->consumer->assign($topic_partitions); 340 | 341 | return $this; 342 | } 343 | 344 | /** 345 | * @return Output 346 | */ 347 | public function getOutput(): Output 348 | { 349 | return $this->output; 350 | } 351 | 352 | /** 353 | * 354 | */ 355 | public function close() 356 | { 357 | if($this->consumer !== null) { 358 | $this->output->info('Stopping consumer by closing connection'); 359 | $this->consumer->close(); 360 | } else { 361 | $this->output->info('flushing old consumer...'); 362 | $this->oldConsumer->flush(-1); 363 | } 364 | } 365 | 366 | 367 | /** 368 | * @param array $topicPartitions 369 | * @return mixed 370 | */ 371 | public function getOffsetPositions(array $topicPartitions) 372 | { 373 | return $this->consumer->getOffsetPositions($topicPartitions); 374 | } 375 | 376 | /** 377 | * @param int $signalNumber 378 | */ 379 | public function signalHandler(int $signalNumber): void 380 | { 381 | $this->output->error('Handling signal: #' . $signalNumber); 382 | 383 | switch ($signalNumber) { 384 | case SIGTERM: // 15 : supervisor default stop 385 | case SIGQUIT: // 3 : kill -s QUIT 386 | echo 'process closed with SIGQUIT'. PHP_EOL; 387 | $this->close(); 388 | exit(1); 389 | break; 390 | case SIGINT: // 2 : ctrl+c 391 | $this->output->warn('process closed with SIGINT'); 392 | $this->close(); 393 | exit(1); 394 | break; 395 | case SIGHUP: // 1 : kill -s HUP 396 | // $this->consumer->restart(); 397 | break; 398 | case SIGUSR1: // 10 : kill -s USR1 399 | // send an alarm in 1 second 400 | pcntl_alarm(1); 401 | break; 402 | case SIGUSR2: // 12 : kill -s USR2 403 | // send an alarm in 10 seconds 404 | pcntl_alarm(10); 405 | break; 406 | default: 407 | break; 408 | } 409 | } 410 | 411 | /** 412 | * Alarm handler 413 | * 414 | * @param int $signalNumber 415 | * @return void 416 | */ 417 | public function alarmHandler($signalNumber) 418 | { 419 | $this->output->warn("Handling alarm: # . $signalNumber. memory usage: ". memory_get_usage(true)); 420 | } 421 | 422 | /** 423 | * 424 | */ 425 | public function defineSignalsHandling():void 426 | { 427 | if (extension_loaded('pcntl')) { 428 | pcntl_signal(SIGTERM, [$this, 'signalHandler']); 429 | pcntl_signal(SIGHUP, [$this, 'signalHandler']); 430 | pcntl_signal(SIGINT, [$this, 'signalHandler']); 431 | pcntl_signal(SIGQUIT, [$this, 'signalHandler']); 432 | pcntl_signal(SIGUSR1, [$this, 'signalHandler']); 433 | pcntl_signal(SIGUSR2, [$this, 'signalHandler']); 434 | pcntl_signal(SIGALRM, [$this, 'alarmHandler']); 435 | } else { 436 | $this->output->error('Unable to process signal.'); 437 | exit(1); 438 | } 439 | } 440 | 441 | /** 442 | * @param $callback 443 | * @param \RdKafka\Message $message 444 | */ 445 | public function callback($callback, \RdKafka\Message $message): void 446 | { 447 | if($callback instanceof \Closure) { 448 | $callback($message, $this); 449 | return; 450 | } 451 | 452 | if($callback instanceof Callback) { 453 | /** @var Callback $instance */ 454 | $callback->callback($message, $this); 455 | return; 456 | } 457 | 458 | throw new \RuntimeException('wrong instance'); 459 | } 460 | 461 | /** 462 | * @param $callback 463 | * @param array $messages 464 | */ 465 | public function callbackBatch($callback, array $messages): void 466 | { 467 | if($callback instanceof \Closure) { 468 | $callback($messages, $this); 469 | return; 470 | } 471 | 472 | if($callback instanceof Callback) { 473 | /** @var Callback $instance */ 474 | $callback->callback($messages, $this); 475 | return; 476 | } 477 | 478 | throw new \RuntimeException('wrong instance'); 479 | } 480 | } 481 | -------------------------------------------------------------------------------- /src/Consume/HighLevel/Contracts/Callback.php: -------------------------------------------------------------------------------- 1 | setHeaders((array) $headers)->setRows($rows)->setStyle($tableStyle); 29 | 30 | foreach ($columnStyles as $columnIndex => $columnStyle) { 31 | $table->setColumnStyle($columnIndex, $columnStyle); 32 | } 33 | 34 | $table->render(); 35 | } 36 | 37 | 38 | /** 39 | * @param $string 40 | * @param null $verbosity 41 | */ 42 | public function info($string, $verbosity = null) 43 | { 44 | $this->line($string, 'info', $verbosity); 45 | } 46 | 47 | 48 | /** 49 | * @param $string 50 | * @param null $style 51 | * @param null $verbosity 52 | */ 53 | public function line($string, $style = null, $verbosity = null) 54 | { 55 | $styled = $style ? "<$style>$string" : $string; 56 | 57 | $this->writeln($styled); 58 | } 59 | 60 | 61 | /** 62 | * @param $string 63 | * @param null $verbosity 64 | */ 65 | public function comment($string, $verbosity = null) 66 | { 67 | $this->line($string, 'comment', $verbosity); 68 | } 69 | 70 | 71 | /** 72 | * @param $string 73 | * @param null $verbosity 74 | */ 75 | public function question($string, $verbosity = null) 76 | { 77 | $this->line($string, 'question', $verbosity); 78 | } 79 | 80 | 81 | /** 82 | * @param $string 83 | * @param null $verbosity 84 | */ 85 | public function error($string, $verbosity = null) 86 | { 87 | $this->line($string, 'error', $verbosity); 88 | } 89 | 90 | 91 | /** 92 | * @param $string 93 | * @param null $verbosity 94 | */ 95 | public function warn($string, $verbosity = null) 96 | { 97 | if (! $this->getFormatter()->hasStyle('warning')) { 98 | $style = new OutputFormatterStyle('yellow'); 99 | 100 | $this->getFormatter()->setStyle('warning', $style); 101 | } 102 | 103 | $this->line($string, 'warning', $verbosity); 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /src/Exceptions/KafkaBrokerException.php: -------------------------------------------------------------------------------- 1 | payload = $payload; 48 | $this->partition = $partition; 49 | $this->msgFlags = $msgFlags; 50 | $this->messageKey = $messageKey; 51 | $this->headers = $headers; 52 | } 53 | 54 | /** 55 | * @return string 56 | */ 57 | public function getPayload(): string 58 | { 59 | return $this->payload; 60 | } 61 | 62 | /** 63 | * @return int 64 | */ 65 | public function getPartition(): int 66 | { 67 | return $this->partition; 68 | } 69 | 70 | /** 71 | * @return int 72 | */ 73 | public function getMsgFlags(): int 74 | { 75 | return $this->msgFlags; 76 | } 77 | 78 | /** 79 | * @return string|null 80 | */ 81 | public function getMessageKey(): ?string 82 | { 83 | return $this->messageKey; 84 | } 85 | 86 | /** 87 | * @return array 88 | */ 89 | public function getHeaders(): array 90 | { 91 | return $this->headers; 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /src/Produce/ProducerProperties.php: -------------------------------------------------------------------------------- 1 | topicName = $topicName; 40 | $this->kafkaConf = $kafkaConf; 41 | $this->topicConf = $topicConf; 42 | $this->callbacksCollection = $callbacksCollection; 43 | } 44 | 45 | /** 46 | * @return string 47 | */ 48 | public function getTopicName(): string 49 | { 50 | return $this->topicName; 51 | } 52 | 53 | /** 54 | * @return array 55 | */ 56 | public function getKafkaConf(): array 57 | { 58 | return $this->kafkaConf; 59 | } 60 | 61 | /** 62 | * @return array 63 | */ 64 | public function getTopicConf(): array 65 | { 66 | return $this->topicConf; 67 | } 68 | 69 | /** 70 | * @return CallbacksCollection 71 | */ 72 | public function getCallbacksCollection(): ?CallbacksCollection 73 | { 74 | return $this->callbacksCollection; 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /src/Produce/ProducerWrapper.php: -------------------------------------------------------------------------------- 1 | instantiated) { 58 | return $this; 59 | } 60 | 61 | $this->producer = $this->initProducer($producerProperties); 62 | 63 | $this->topic = $this->instantiateTopic($producerProperties); 64 | 65 | if($producerProperties->getCallbacksCollection() !== null) { 66 | $this->registerConfigurationCallbacks($this->kafkaConf, $producerProperties->getCallbacksCollection()); 67 | } 68 | $this->instantiated = true; 69 | 70 | return $this; 71 | } 72 | 73 | /** 74 | * @param ProducerData $dataObject 75 | * @param int $timeout 76 | * @return $this 77 | */ 78 | public function produce(ProducerData $dataObject, $timeout = 0): self 79 | { 80 | $this->topic->produce( 81 | $dataObject->getPartition(), 82 | $dataObject->getMsgFlags(), 83 | $dataObject->getPayload(), 84 | $dataObject->getMessageKey() 85 | ); 86 | 87 | $this->producer->poll($timeout); 88 | 89 | return $this; 90 | } 91 | 92 | /** 93 | * 94 | */ 95 | public function produceWithHeaders(ProducerData $dataObject, $timeout = 0) 96 | { 97 | $this->topic->producev( 98 | $dataObject->getPartition(), 99 | $dataObject->getMsgFlags(), 100 | $dataObject->getPayload(), 101 | $dataObject->getMessageKey(), 102 | $dataObject->getHeaders() 103 | ); 104 | 105 | $this->producer->poll($timeout); 106 | 107 | return $this; 108 | } 109 | 110 | /** 111 | * @param int $ms 112 | * @throws KafkaProduceFlushNotImplementedException 113 | * @throws KafkaProduceFlushTimeoutException 114 | */ 115 | public function flush(int $ms = 10000): void 116 | { 117 | for ($flushRetries = 0; $flushRetries < 10; $flushRetries++) { 118 | $result = $this->producer->flush($ms); 119 | if (RD_KAFKA_RESP_ERR_NO_ERROR === $result) { 120 | break; 121 | } 122 | } 123 | 124 | if (RD_KAFKA_RESP_ERR__TIMED_OUT === $result) { 125 | throw new KafkaProduceFlushTimeoutException('Flush timeout exception!!'); 126 | } 127 | 128 | if (RD_KAFKA_RESP_ERR__NOT_IMPLEMENTED === $result) { 129 | throw new KafkaProduceFlushNotImplementedException('Was unable to flush, messages might be lost!'); 130 | } 131 | } 132 | 133 | /** 134 | * @param ProducerProperties $producerProperties 135 | * @return array 136 | */ 137 | private function initProducer(ProducerProperties $producerProperties): Producer 138 | { 139 | $this->kafkaConf = new Conf(); 140 | 141 | foreach ($producerProperties->getKafkaConf() as $key => $value) { 142 | $this->kafkaConf->set($key, $value); 143 | } 144 | 145 | return new Producer($this->kafkaConf); 146 | } 147 | 148 | /** 149 | * @param ProducerProperties $producerProperties 150 | * @return Topic 151 | * @throws KafkaTopicNameException 152 | */ 153 | private function instantiateTopic(ProducerProperties $producerProperties): Topic 154 | { 155 | $this->topicConf = new TopicConf(); 156 | 157 | foreach ($producerProperties->getTopicConf() as $key => $value) { 158 | $this->topicConf->set($key, $value); 159 | } 160 | 161 | if (empty($producerProperties->getTopicName())) { 162 | throw new KafkaTopicNameException(); 163 | } 164 | 165 | return $this->producer->newTopic($producerProperties->getTopicName(), $this->topicConf); 166 | } 167 | 168 | /** 169 | * @return bool 170 | */ 171 | public function isInstantiated(): bool 172 | { 173 | return $this->instantiated; 174 | } 175 | 176 | private function registerConfigurationCallbacks(Conf $conf, CallbacksCollection $callbacksCollection) 177 | { 178 | /** 179 | * @var \Closure $callback 180 | */ 181 | foreach ($callbacksCollection as $key => $callback) { 182 | switch ($key) { 183 | case ConfigurationCallbacksKeys::DELIVERY_REPORT: { $conf->setDrMsgCb($callback->bindTo($this)); break;} 184 | case ConfigurationCallbacksKeys::ERROR: {$conf->setErrorCb($callback->bindTo($this)); break;} 185 | case ConfigurationCallbacksKeys::LOG: {$conf->setLogCb($callback->bindTo($this)); break;} 186 | } 187 | } 188 | } 189 | } 190 | --------------------------------------------------------------------------------