├── .github └── workflows │ └── php.yml ├── Dockerfile ├── README.md ├── composer.json ├── config └── streamer.php ├── docker-compose.yml ├── phpunit.xml.dist ├── pint.json ├── rector.php └── src ├── Archiver ├── NullStorage.php ├── StorageManager.php └── StreamArchiver.php ├── Commands ├── Archive │ ├── ArchiveCommand.php │ ├── ArchiveRestoreCommand.php │ ├── ProcessMessagesCommand.php │ └── PurgeCommand.php ├── FailedMessages │ ├── FlushFailedCommand.php │ ├── ListFailedCommand.php │ └── RetryFailedCommand.php ├── ListCommand.php └── ListenCommand.php ├── Concerns ├── ConnectsWithRedis.php ├── EmitsStreamerEvents.php └── HashableMessage.php ├── Contracts ├── ArchiveStorage.php ├── Archiver.php ├── Emitter.php ├── Errors │ ├── MessagesFailer.php │ ├── Repository.php │ ├── Specification.php │ └── StreamableMessage.php ├── Event.php ├── History.php ├── Listener.php ├── MessageReceiver.php └── Replayable.php ├── Eloquent └── EloquentModelEvent.php ├── Errors ├── FailedMessage.php ├── FailedMessagesHandler.php ├── MessagesRepository.php └── Specifications │ ├── IdentifierSpecification.php │ ├── MatchAllSpecification.php │ ├── ReceiverSpecification.php │ └── StreamSpecification.php ├── EventDispatcher ├── Message.php ├── ReceivedMessage.php ├── StreamMessage.php └── Streamer.php ├── Exceptions ├── AcknowledgingFailedException.php ├── ArchivizationFailedException.php ├── InvalidListeningArgumentsException.php ├── MessageRetryFailedException.php └── RestoringFailedException.php ├── Facades └── Streamer.php ├── History ├── EventHistory.php └── Snapshot.php ├── ListenersStack.php ├── Stream.php ├── Stream ├── Consumer.php ├── MultiStream.php └── Range.php ├── StreamNotFoundException.php ├── StreamerProvider.php └── Streams.php /.github/workflows/php.yml: -------------------------------------------------------------------------------- 1 | name: Build 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | pull_request: 8 | branches: 9 | - master 10 | 11 | jobs: 12 | run: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | php-version: ['8.1', '8.2'] 17 | redis-version: [6, 7] 18 | name: PHP ${{ matrix.php-version }}, Redis ${{ matrix.redis-version }} 19 | steps: 20 | - name: Checkout 21 | uses: actions/checkout@v4 22 | 23 | - name: Setup PHP 24 | uses: shivammathur/setup-php@v2 25 | with: 26 | php-version: ${{ matrix.php-version }} 27 | coverage: pcov 28 | tools: phpunit 29 | 30 | - name: Redis Server 31 | uses: supercharge/redis-github-action@1.2.0 32 | with: 33 | redis-version: ${{ matrix.redis-version }} 34 | 35 | - name: Validate composer.json 36 | run: composer validate 37 | 38 | - name: Install dependencies 39 | run: composer install --prefer-dist --no-progress --no-suggest 40 | 41 | - name: Run test suite 42 | run: vendor/bin/phpunit --coverage-text 43 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM php:8.2-fpm-alpine3.20 2 | 3 | RUN apk add --update --no-cache autoconf g++ make \ 4 | && apk add linux-headers \ 5 | && pecl install redis \ 6 | && pecl install xdebug \ 7 | && docker-php-ext-enable redis \ 8 | && docker-php-ext-enable xdebug \ 9 | && curl -sS https://getcomposer.org/installer | php -- --install-dir=/usr/bin --filename=composer 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel Streamer 2 | 3 | Streamer is a Laravel package for events functionality between different applications, powered by Redis Streams. This 4 | package utilizes all main commands of Redis 5.0 Streams providing a simple usage of Streams as Events. 5 | 6 | Main concept of this package is to provide easy way of emitting new events from your application and to allow listening to them in your other applications that are using same Redis server. 7 | 8 | # Installation 9 | 1. Install package via composer command `composer require prwnr/laravel-streamer` or by adding it to your composer.json file with version. 10 | 2. Discover the package 11 | 3. Publish configuration with `vendor:publish` command. 12 | 4. Make sure that you have running Redis 5.0 instance and that Laravel is configured to use it 13 | 5. Make sure that you have [PHPRedis extension](https://github.com/phpredis/phpredis) installed. 14 | # Usage 15 | There are two main ends of this package usage - emiting new event and listening to events. Whereas emiting requires a bit more work to get it used, such as creating own Event classes, then listening to events is available with artisan command and is working without much work needed. 16 | 17 | ## Version Compatibility 18 | 19 | | PHP | Laravel | Streamer | Redis driver | Redis | 20 | |:-------------------------|:--------------------------------|:---------|:-------------|--------------| 21 | | 7.2^|8.0^ | 5.6.x | 1.6.x | Predis | | 22 | | 7.2^|8.0^ | 5.7.x | 1.6.x | Predis | | 23 | | 7.2^|8.0^ | 5.8.x | 1.6.x | Predis | | 24 | | 7.2^|8.0^ | 6.x | 2.x | PhpRedis | | 25 | | 7.2^|8.0^ | 6.x|7.x | ^2.1 | PhpRedis | | 26 | | 7.2^|8.0^ | 6.x|7.x|8.x | ^2.3 | PhpRedis | | 27 | | 7.4^|8.0^|8.1^ | 6.x|7.x|8.x | ^3.0 | PhpRedis | | 28 | | 7.4^|8.0^|8.1^ | 6.x|7.x|8.x|9.x | ^3.3 | PhpRedis | | 29 | | 7.4^|8.0^|8.1^ | 7.x|8.x|9.x|10.x | ^3.5 | PhpRedis | | 30 | | ^8.1 | 10.x|11.x | ^4.0 | PhpRedis | 6.0|7.0 | 31 | | ^8.1|^8.2 | 10.x|11.x|12.0 | ^4.4 | PhpRedis | 6.0|7.0 | 32 | 33 | ### Emiting new events 34 | 35 | In order to emit new event few things needs to be done. 36 | First of all, you will need to have a valid class that implements `Prwnr\Streamer\Contracts\Event` like this: 37 | ```php 38 | class ExampleStreamerEvent implements Prwnr\Streamer\Contracts\Event { 39 | /** 40 | * Require name method, must return a string. 41 | * Event name can be anything, but remember that it will be used for listening 42 | */ 43 | public function name(): string 44 | { 45 | return 'example.streamer.event'; 46 | } 47 | /** 48 | * Required type method, must return a string. 49 | * Type can be any string or one of predefined types from Event 50 | */ 51 | public function type(): string 52 | { 53 | return Event::TYPE_EVENT; 54 | } 55 | /** 56 | * Required payload method, must return array 57 | * This array will be your message data content 58 | */ 59 | public function payload(): array 60 | { 61 | return ['message' => 'content']; 62 | } 63 | } 64 | ``` 65 | Then, at any point in your application all you need to do is to emit that event by using either `Streamer` instance or `Streamer` facade. 66 | ```php 67 | $event = new ExampleStreamerEvent(); 68 | $id = \Prwnr\Streamer\Facades\Streamer::emit($event); 69 | ``` 70 | This will create a message on a stream named (if such does not exist): `example.streamer.event`. Emit method will return 71 | an ID if emitting your event ended up with success. 72 | 73 | ### Listening for new messages on events 74 | 75 | In order to listen to event you will have to properly configure `config/streamer.php` (or use `ListenerStack::add` method) and run `php artisan streamer:listen` command. 76 | At config file you will find *Application listeners* configuration with default values for it, that should be changed if you want to start listening with streamer listen command. 77 | Other way to add listeners for events is to use `ListenersStack` static class. This class is being booted with listeners from configuration file and is then used by 78 | command to get all of them. So, the addition of this class is that it allows adding listeners not only via configuration file, but also programmatically. 79 | 80 | Remember that local listener should implement `MessageReceiver` contract to ensure that it has `handle` method 81 | which accepts `ReceivedMessage` as an argument. 82 | ```php 83 | /* 84 | |-------------------------------------------------------------------------- 85 | | Application listeners 86 | |-------------------------------------------------------------------------- 87 | | 88 | | Listeners classes that should be invoked with Streamer listen command 89 | | based on streamer.event.name => [local_handlers] pairs 90 | | 91 | | Local listeners should implement MessageReceiver contract 92 | | 93 | */ 94 | 'listen_and_fire' => [ 95 | 'example.streamer.event' => [ 96 | //List of local listeners that should be invoked 97 | //\App\Listeners\ExampleListener::class 98 | ], 99 | ], 100 | ``` 101 | 102 | Above configuration is an array of Streamer events with an array of local listeners related to Streamer event. When 103 | listening to `example.streamer.event` all local listeners from its config definition will be created and their `handle` 104 | method fired with message received from Stream. For listener instance creation this package uses Laravel Container, 105 | therefore you can type hint anything into your listener constructor to use Laravel dependency injection. 106 | 107 | To start listening for an event, use [listen](#listen) command. 108 | 109 | ### Commands 110 | 111 | #### Listen 112 | 113 | ```bash 114 | streamer:listen example.streamer.event 115 | ``` 116 | 117 | This command will start listening on a given stream (or streams separated by comma) starting from "now". 118 | It will be listening in a blocking way, meaning that it will run until Redis will time out or crash. 119 | All listener related errors are being caught and logged into 120 | console as well as stored in Failed Messages list for later debugging and/or retrying. 121 | 122 | That's a basic usage of this command, where event name is a required argument (unless `--all` option is provided). 123 | So in this case it simply starts listening for only new events. 124 | This command however has few options that are extending its usage, those are: 125 | 126 | ```text 127 | --all= : Will trigger listener mode to start listening on all events that are registered with local listeners classes (from the ListenersStack). Event name argument is no longer required in this case. 128 | --group= : Name of your streaming group. Only when group is provided listener will listen on group as consumer 129 | --consumer= : Name of your group consumer. If not provided a name will be created as groupname-timestamp 130 | --reclaim= : Milliseconds of pending messages idle time, that should be reclaimed for current consumer in this group. Can be only used with group listening 131 | --last_id= : ID from which listener should start reading messages (using 0-0 will process all old messages) 132 | --keep-alive : Will keep listener alive when any unexpected non-listener related error will occur by simply restarting listening 133 | --max-attempts= : Number of maximum attempts to restart a listener on an unexpected non-listener related error (requires --keep-alive to be used) 134 | --purge : Will remove message from the stream if it will be processed successfully by all local listeners in the current stack. 135 | --archive : Will remove message from the stream and store it in database if it will be processed successfully by all local listeners in the current stack. 136 | ``` 137 | 138 | When `consumer` and `group` options are being in use, every message on a stream will be marked as acknowledged for the 139 | given consumer, thus it will not be processed by consequent 140 | `streamer:listen` command call with the same options. Note that listening from a specific ID without consumer and group 141 | being set will ignore acknowledgments. 142 | 143 | The `purge` and `archive` options (available since v2.6) are designed to be used to release memory or storage of the 144 | Redis instance 145 | (in a cases when there are tons of streamed messages or the payloads are big and Redis runs out of memory/storage). When 146 | using those options, keep in mind, that they are not going to take into account listeners running in other instances or 147 | other servers - meaning, that when first listener hooked to specific event will process its messages, the `purge` and 148 | `archive` options will delete the message not waiting for other listeners to finish. To fully use `archive` option 149 | see [Stream Archive][#stream-archive] for more details and instructions. 150 | 151 | Using multiple events in argument or the `--all` option with any other option (like group, consumer, last_id) 152 | will apply those options to every stream event that is being in use. 153 | 154 | #### Failed List 155 | 156 | ```bash 157 | streamer:failed:list 158 | ``` 159 | 160 | This command will show list of stream messages that failed to be handled by their listeners. It will yield all the 161 | important information about them like: ID, stream name, listener class, error message that cause it to fail, and a date 162 | when that happened. 163 | 164 | Table example: 165 | 166 | ```text 167 | +-----+-----------+---------------------------+-------+---------------------+ 168 | | ID | Stream | Receiver | Error | Date | 169 | +-----+-----------+---------------------------+-------+---------------------+ 170 | | 123 | foo.bar | Tests\Stubs\LocalListener | error | 2021-12-12 12:12:12 | 171 | | 321 | other.bar | Tests\Stubs\LocalListener | error | 2021-12-12 12:15:12 | 172 | +-----+-----------+---------------------------+-------+---------------------+ 173 | ``` 174 | 175 | There's one addition option for this command, called `--compact` which will limit the table output to only ID, Stream 176 | and Error columns. 177 | 178 | #### Failed Retry 179 | 180 | ```bash 181 | streamer:failed:retry 182 | ``` 183 | 184 | This command is meant to try again failed listening. It simply reads the message from a stream and attempts to handle it 185 | again by the listener that it was originally processed. 186 | 187 | When the listener fails to process the message again, the message failed information will be re-stored (with a newer 188 | date and updated error message) and will be available to be retried again. There's no limit to how many times message 189 | can be processed. It will remain available after each fail unless flush command will be used. 190 | 191 | This command has few options that are available: 192 | 193 | ```text 194 | --all : retries all existing failed messages 195 | --id= : retries only those messages that are matching given ID 196 | --stream= : retries only those messages that are matching given stream name 197 | --receiver= : retries only those messages that are matching given listener full class name (may require to be in quotation) 198 | ``` 199 | 200 | At least one of those options is required to be used with the command to process failed messages. The `all` option can 201 | be only used solely, while the other three options can be used together or not. This means, that any combination of `id` 202 | , `stream` and `receiver` 203 | can be used to match any number of failed messages and retry them. So, for example a `stream` 204 | can be used together with `id` or in other case `id` can be used with `receiver`, or only one of them can be used, or 205 | all three at once, its all up to the use case. 206 | 207 | #### Failed Flush 208 | 209 | ```bash 210 | streamer:failed:flush 211 | ``` 212 | 213 | This command will remove all existing failed messages from the messages' repository. Can be used to prune entries that 214 | cannot be processed at all by listeners. 215 | 216 | This command **WILL NOT** remove the message from the Stream itself - the message will remain there untouched, but 217 | acknowledged by its original consumer (if used). 218 | 219 | #### List 220 | 221 | ```bash 222 | streamer:list 223 | ``` 224 | 225 | This command will list all registered events, and their associated listeners. The option `--compact` will yield only a 226 | list of the events, skipping listeners column. 227 | 228 | This command may be useful to see what events are being actually handled by a listener, what can help to find out what's 229 | missing. This list can be also used to start listening to available events by 3rd party app. 230 | 231 | Table example: 232 | 233 | ```textmate 234 | +------------------------+------------------------------------+ 235 | | Event | Listeners | 236 | +------------------------+------------------------------------+ 237 | | example.streamer.event | none | 238 | | foo.bar | Tests\Stubs\LocalListener | 239 | | other.foo.bar | Tests\Stubs\LocalListener | 240 | | | Tests\Stubs\AnotherLocalListener | 241 | +------------------------+------------------------------------+ 242 | ``` 243 | 244 | #### Archive 245 | 246 | ```bash 247 | streamer:archive 248 | ``` 249 | 250 | This command will archive messages from a selected streams older than days/weeks or so. It will process all stream 251 | messages, verifying their `created` timestamp and will attempt to archive (deleting them from redis and attempting to 252 | store them in associated archive [storage](#stream-archive)) 253 | each one of them. 254 | 255 | This command has two required options: 256 | 257 | ```text 258 | --streams : list of streams separated by comma to archive messages from 259 | --older_than= : information how old messages should be to archive them. The suggested format is: 60 min, 1 day, 1 week, 5 days, 2 weeks etc. 260 | ``` 261 | 262 | Be aware of using this command, as it will not take into account whether listeners processed messages it tries to 263 | archive or not. This should be used with caution and only for older messages, so that it will be more certain, that all 264 | listeners processed their messages. 265 | 266 | #### Purge 267 | 268 | ```bash 269 | streamer:purge 270 | ``` 271 | 272 | This command will purge messages from a selected streams older than days/weeks or so. It will process all stream 273 | messages, verifying their `created` timestamp and will attempt to purge them (deleting them from the redis entirely). 274 | 275 | This command has two required options: 276 | 277 | ```text 278 | --streams : list of streams separated by comma to purge messages from 279 | --older_than= : information how old messages should be to purge them. The suggested format is: 60 min, 1 day, 1 week, 5 days, 2 weeks etc. 280 | ``` 281 | 282 | Be aware of using this command, as it will not take into account whether listeners processed messages it tries to purge 283 | or not. This should be used with caution and only for older messages, so that it will be more certain, that all 284 | listeners processed their messages. 285 | 286 | #### Archive Restore 287 | 288 | ```bash 289 | streamer:archive:restore 290 | ``` 291 | 292 | This command will restore archived stream messages from the associated archive storage. It will essentially fetch 293 | messages (all or selection) and will try to put them back onto the stream, while also deleting them from the archive. 294 | This action will trigger listeners that are hooked to the restored streams! 295 | 296 | This command has few options that are available: 297 | 298 | ```text 299 | --all : restores all archived messages back to the stream. 300 | --id= : restores archived message back to the stream by ID. Requires --stream option to be used as well. 301 | --stream= : restores all archived messages from a selected stream. 302 | ``` 303 | 304 | At least one of those options is required to attempt restoration of the messages. If any error occurs while restoring a 305 | message, it will be reported for that particular attempt not preventing other message from being processed. 306 | 307 | Restoring message puts it back onto a stream with NEW ID - this is Redis requirement and limitation, that any message 308 | added to stream, needs to have ID higher than the last generated one. The original ID of the message that is being 309 | restored will be stored in meta information in `original_id` field. 310 | 311 | ### Stream Archive 312 | 313 | Stream Archive allows storing processed message in any kind of storage, to free up Redis memory and/or space since 2.6 314 | version. Archive allows restoring those messages, releasing them back onto the stream. 315 | 316 | To fully use archive storage, a new storage driver needs to be written and added to manager. To do so, few quick steps 317 | needs to be finished: 318 | 319 | 1) define your storage driver class and make it implement `\Prwnr\Streamer\Contracts\ArchiveStorage` contract. This 320 | interface is mandatory. 321 | 2) extend the manager with your driver like this: 322 | 323 | ```php 324 | $manager = $this->app->make(StorageManager::class); 325 | $manager->extend('your_driver_name', static function () { 326 | return new YourDriver(); 327 | }); 328 | ``` 329 | 330 | 3) define your driver as default in streamer config file 331 | 332 | ```php 333 | 'archive' => [ 334 | 'storage_driver' => 'your_driver_name' 335 | ] 336 | ``` 337 | 338 | ### Replaying Events 339 | 340 | Since 2.2 version, Stream events can be "replayed". This means, that the specific message (with a unique identifier) 341 | can be "reconstructed" until "now" (or until a selected date). 342 | 343 | What "replaying" messages really means? It means, that all the messages that are in the stream, will be read from the 344 | very beginning, and payload of each single entry will be "combined" into a final version of the message - each filed 345 | will be replaced with its "newer" value, if such exists in the history. 346 | 347 | This is going to be useful with events that don't hold all the information about the resource they may represent, 348 | but have only data about fields that changed. 349 | 350 | So, for example having a resource with fields `name` and `surname`, 351 | we will emit 3 different events: 352 | - first for its creation populating both fields with values (`name: foo; surname: bar`) 353 | - second event that will change only `name` into `foo bar` 354 | - third event that changes name again to `bar foo`. 355 | 356 | While replaying this set of messages (remember that each one has the same unique identifier) 357 | our final replayed resource will be: `name: bar foo; surname: bar`. If we replay the event until the time before third 358 | change, we would have `name: foo bar; surname: bar` 359 | 360 | #### Usage 361 | 362 | To make Event replayable, it needs to implement the `Prwnr\Streamer\Contracts\Replayable` contract. 363 | This will enforce adding a `getIdentifier` method, that should return unique identifier for the resource 364 | (like UUID of the resource that this event represents). With this contract being fulfilled, all events that will go 365 | through `Streamer` emit method, will be also "marked" as available to be replayed. 366 | 367 | To actually replay messages, `Hsitory` interface implementation needs to be used. 368 | 369 | Method that should be used is: `replay(string $event, string $identifier, Carbon $until = null): array`. 370 | This method will return the "current state" of the event, rebuilding it from its history. As seen in method definition, 371 | it asks for event string name and resource identifier (that was applied by `Replayable` contract). Third parameter is 372 | optional and if used, it will stop replaying messages when first message with matching date will be encountered. 373 | 374 | ### Eloquent Model Events 375 | 376 | With use of a `EmitsStreamerEvents` trait you can easily make your Eloquent Models emit basic events. This trait will 377 | integrate your model with Streamer and will emit events on actions like: `save`, `create` and `delete`. It will emit an 378 | event of your model name with suffix of the action and a payload of what happened. In case of a `create` 379 | and `save` actions the payload will have a list of changed fields and a before/after for each of those fields (with 380 | create action fields before will basically have all values set to null), in case of a `delete` action, payload will 381 | simply state that the model has been deleted. Each payload includes a `[key_name => key_value]` pair of your model ID. 382 | 383 | By default, events will take names from their models with a suffix of the action, but the name can be changed by 384 | assigning it to a `baseEventName` attribute. This name will replace the model name but will keep suffix of what action 385 | has been taken. 386 | 387 | Check example's directory in this package to see how can you exactly use each command with package Stream and Consumer 388 | instances. 389 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "prwnr/laravel-streamer", 3 | "description": "Events streaming package for Laravel that uses Redis 5 streams", 4 | "type": "library", 5 | "license": "MIT", 6 | "authors": [ 7 | { 8 | "name": "Rafal Purwin", 9 | "email": "purwinr@gmail.com" 10 | } 11 | ], 12 | "require": { 13 | "ext-json": "*", 14 | "ext-redis": "*", 15 | "php": "^8.1|^8.2", 16 | "illuminate/console": "^10.0|^11.0|^12.0", 17 | "illuminate/support": "^10.0|^11.0|^12.0", 18 | "illuminate/redis": "^10.0|^11.0|^12.0", 19 | "illuminate/container": "^10.0|^11.0|^12.0" 20 | }, 21 | "require-dev": { 22 | "orchestra/testbench": "^8.0|^9.0|^10.0", 23 | "phpunit/phpunit": "^10.0|^11.0", 24 | "predis/predis": "^1.1", 25 | "rector/rector": "^1.0", 26 | "laravel/pint": "^1.2" 27 | }, 28 | "autoload": { 29 | "psr-4": { 30 | "Prwnr\\Streamer\\": "src/" 31 | } 32 | }, 33 | "autoload-dev": { 34 | "psr-4": { 35 | "Tests\\": "tests/" 36 | } 37 | }, 38 | "minimum-stability": "dev", 39 | "prefer-stable": true, 40 | "extra": { 41 | "laravel": { 42 | "aliases": { 43 | "Streamer": "Prwnr\\Streamer\\Facades\\Streamer" 44 | }, 45 | "providers": [ 46 | "Prwnr\\Streamer\\StreamerProvider" 47 | ] 48 | } 49 | }, 50 | "config": { 51 | "allow-plugins": { 52 | "infection/extension-installer": true 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /config/streamer.php: -------------------------------------------------------------------------------- 1 | 0, 19 | 20 | /* 21 | |-------------------------------------------------------------------------- 22 | | Streamer read timeout 23 | |-------------------------------------------------------------------------- 24 | | 25 | | Seconds after which Streamer listen block should timeout 26 | | Setting 0 never timeouts. 27 | | 28 | | Time in seconds 29 | | 30 | */ 31 | 'stream_read_timeout' => 0, 32 | 33 | /* 34 | |-------------------------------------------------------------------------- 35 | | Streamer reading sleep 36 | |-------------------------------------------------------------------------- 37 | | 38 | | Seconds of a sleep time that happens between reading messages from Stream 39 | | 40 | | Time in seconds 41 | | 42 | */ 43 | 'read_sleep' => 1, 44 | 45 | /* 46 | |-------------------------------------------------------------------------- 47 | | Streamer Redis connection 48 | |-------------------------------------------------------------------------- 49 | | 50 | | Connection name which Streamer should use for all Redis commands 51 | | 52 | */ 53 | 'redis_connection' => 'default', 54 | 55 | /* 56 | |-------------------------------------------------------------------------- 57 | | Streamer event domain 58 | |-------------------------------------------------------------------------- 59 | | 60 | | Domain name which streamer should use when 61 | | building message with JSON schema 62 | | 63 | */ 64 | 'domain' => env('APP_NAME', ''), 65 | 66 | /* 67 | |-------------------------------------------------------------------------- 68 | | Application handlers 69 | |-------------------------------------------------------------------------- 70 | | 71 | | Handlers classes that should be invoked with Streamer listen command 72 | | based on streamer.event.name => [local_handlers] pairs 73 | | 74 | | Local handlers should implement MessageReceiver contract 75 | | 76 | */ 77 | 'listen_and_fire' => [ 78 | 'example.streamer.event' => [ 79 | //List of local listeners that should be invoked 80 | //\App\Listeners\ExampleListener::class 81 | ], 82 | ], 83 | 84 | /* 85 | |-------------------------------------------------------------------------- 86 | | Archive Storage Driver 87 | |-------------------------------------------------------------------------- 88 | | 89 | | Name of the driver that should be used by StreamArchiver while performing 90 | | archivisation action. 91 | | Null driver being the default driver will not store stream message, 92 | | that will make it only removed. 93 | | 94 | | To fully use archiver functionality, the driver should be added to 95 | | \Prwnr\Streamer\Archiver\StorageManager and save the received message 96 | | in some kind of database. 97 | | 98 | | Driver should implement \Prwnr\Streamer\Contracts\ArchiveStorage contract. 99 | */ 100 | 'archive' => [ 101 | 'storage_driver' => 'null', 102 | ], 103 | ]; 104 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | services: 3 | 4 | app: 5 | build: . 6 | volumes: 7 | - ./:/var/www/html 8 | 9 | redis: 10 | image: redis:latest 11 | ports: 12 | - 6379:6379 13 | command: redis-server --appendonly yes 14 | -------------------------------------------------------------------------------- /phpunit.xml.dist: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ./tests 6 | 7 | 8 | 9 | 10 | ./src 11 | 12 | 13 | 14 | -------------------------------------------------------------------------------- /pint.json: -------------------------------------------------------------------------------- 1 | { 2 | "preset": "psr12", 3 | "rules": { 4 | "is_null": true, 5 | "no_blank_lines_after_class_opening": true, 6 | "method_chaining_indentation": true, 7 | "no_superfluous_phpdoc_tags": true, 8 | "no_empty_phpdoc": true, 9 | "ordered_class_elements": true, 10 | "ordered_imports": true, 11 | "phpdoc_line_span": true, 12 | "phpdoc_annotation_without_dot": true, 13 | "phpdoc_no_useless_inheritdoc": true, 14 | "phpdoc_summary": true, 15 | "protected_to_private": true, 16 | "standardize_increment": true, 17 | "strict_comparison": true, 18 | "trailing_comma_in_multiline": true, 19 | "use_arrow_functions": true, 20 | "ternary_to_null_coalescing": true, 21 | "statement_indentation": true, 22 | "array_indentation": true, 23 | "assign_null_coalescing_to_coalesce_equal": true, 24 | "align_multiline_comment": true, 25 | "cast_spaces": true, 26 | "combine_consecutive_issets": true, 27 | "curly_braces_position": true, 28 | "no_empty_comment": true, 29 | "operator_linebreak": true, 30 | "class_attributes_separation": true, 31 | "no_extra_blank_lines": { 32 | "tokens": ["extra", "throw", "use", "use_trait", "curly_brace_block", "continue"] 33 | }, 34 | "single_space_after_construct": true, 35 | "binary_operator_spaces": true, 36 | "declare_strict_types": true 37 | } 38 | } -------------------------------------------------------------------------------- /rector.php: -------------------------------------------------------------------------------- 1 | withPaths([ 13 | __DIR__ . '/src', 14 | __DIR__ . '/tests', 15 | ]) 16 | ->withPhpSets(php81: true) 17 | ->withPreparedSets( 18 | deadCode: true, 19 | codeQuality: true, 20 | typeDeclarations: true, 21 | ) 22 | ->withRules([ 23 | MixedTypeRector::class, 24 | ReturnUnionTypeRector::class, 25 | RemoveUselessReturnTagRector::class, 26 | RemoveUselessParamTagRector::class, 27 | ]); 28 | -------------------------------------------------------------------------------- /src/Archiver/NullStorage.php: -------------------------------------------------------------------------------- 1 | container->make(NullStorage::class); 34 | } 35 | 36 | /** 37 | * @inheritDoc 38 | */ 39 | public function getDefaultDriver(): string 40 | { 41 | return 'null'; 42 | } 43 | 44 | /** 45 | * @inheritDoc 46 | */ 47 | protected function callCustomCreator($driver) 48 | { 49 | $custom = $this->customCreators[$driver]($this->container); 50 | if (!$custom instanceof ArchiveStorage) { 51 | $message = sprintf('Custom driver needs to implement [%s] interface.', ArchiveStorage::class); 52 | throw new RuntimeException($message); 53 | } 54 | 55 | return $custom; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/Archiver/StreamArchiver.php: -------------------------------------------------------------------------------- 1 | storage = $manager->driver(config('streamer.archive.storage_driver')); 26 | } 27 | 28 | /** 29 | * @inheritDoc 30 | * @throws ArchivizationFailedException 31 | * @throws JsonException 32 | */ 33 | public function archive(ReceivedMessage $message): void 34 | { 35 | $content = $message->getContent(); 36 | $this->storage->create(new Message([ 37 | '_id' => $message->getId(), 38 | 'original_id' => $content['original_id'] ?? null, 39 | 'name' => $content['name'], 40 | 'domain' => $content['domain'], 41 | 'created' => $content['created'] ?? null, 42 | ], $content['data'])); 43 | 44 | $stream = new Stream($message->getEventName()); 45 | $result = $stream->delete($message->getId()); 46 | 47 | if ($result === 0) { 48 | $this->storage->delete($message->getEventName(), $message->getId()); 49 | 50 | throw new ArchivizationFailedException( 51 | 'Stream message could not be deleted, message will not be archived.' 52 | ); 53 | } 54 | } 55 | 56 | /** 57 | * @inheritDoc 58 | * @throws RestoringFailedException 59 | * @throws JsonException 60 | */ 61 | public function restore(Message $message): string 62 | { 63 | $result = $this->storage->delete($message->getEventName(), $message->getId()); 64 | if ($result === 0) { 65 | throw new RestoringFailedException( 66 | 'Message was not deleted from the archive storage, message will not be restored.' 67 | ); 68 | } 69 | 70 | $content = $message->getContent(); 71 | $message = new Message([ 72 | 'original_id' => $message->getId(), 73 | 'type' => $content['type'], 74 | 'name' => $content['name'], 75 | 'domain' => $content['domain'], 76 | 'created' => $content['created'], 77 | ], $message->getData()); 78 | 79 | $stream = new Stream($message->getEventName()); 80 | $id = $stream->add($message); 81 | if (!$id) { 82 | $this->storage->create($message); 83 | 84 | throw new RestoringFailedException( 85 | 'Message was not deleted from the archive storage, message will not be restored.' 86 | ); 87 | } 88 | 89 | return $id; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Commands/Archive/ArchiveCommand.php: -------------------------------------------------------------------------------- 1 | archiver->archive($received); 38 | $this->info("Message [$id] has been archived from the '$stream' stream."); 39 | } catch (Exception $e) { 40 | report($e); 41 | $this->error("Message [$id] from the '$stream' stream could not be archived. Error: " . $e->getMessage()); 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Commands/Archive/ArchiveRestoreCommand.php: -------------------------------------------------------------------------------- 1 | storage = $manager->driver(config('streamer.archive.storage_driver')); 40 | } 41 | 42 | public function handle(): int 43 | { 44 | $confirm = $this->confirm( 45 | 'Restoring a message will add it back onto the stream and will trigger listeners hooked to its event. Do you want to continue?', 46 | true 47 | ); 48 | 49 | if (!$confirm) { 50 | return 0; 51 | } 52 | 53 | if ($this->option('all')) { 54 | foreach ($this->storage->all() as $message) { 55 | $this->restore($message); 56 | } 57 | 58 | return 0; 59 | } 60 | 61 | if ($this->option('id')) { 62 | if (!$this->option('stream')) { 63 | $this->warn('To restore by ID, a stream name needs to be provided as well.'); 64 | return 1; 65 | } 66 | 67 | $message = $this->storage->find($this->option('stream'), $this->option('id')); 68 | if (!$message instanceof Message) { 69 | $this->error('The message could not be found in archive storage.'); 70 | return 1; 71 | } 72 | 73 | $this->restore($message); 74 | return 0; 75 | } 76 | 77 | if ($this->option('stream')) { 78 | foreach ($this->storage->findMany($this->option('stream')) as $message) { 79 | $this->restore($message); 80 | } 81 | 82 | return 0; 83 | } 84 | 85 | $this->error('At least one option must be used to restore the message.'); 86 | 87 | return 1; 88 | } 89 | 90 | /** 91 | * @inheritDoc 92 | */ 93 | protected function getOptions(): array 94 | { 95 | return [ 96 | [ 97 | 'all', 98 | null, 99 | InputOption::VALUE_NONE, 100 | 'Restores all archived messages back to the stream.', 101 | ], 102 | [ 103 | 'id', 104 | null, 105 | InputOption::VALUE_REQUIRED, 106 | 'Restores archived message back to the stream by ID. Requires --stream option to be used as well.', 107 | ], 108 | [ 109 | 'stream', 110 | null, 111 | InputOption::VALUE_REQUIRED, 112 | 'Restores all archived messages from a selected stream.', 113 | ], 114 | ]; 115 | } 116 | 117 | private function restore(Message $message): void 118 | { 119 | try { 120 | $id = $this->archiver->restore($message); 121 | $this->info("Successfully restored [{$message->getEventName()}][{$message->getId()}] message. New ID: $id"); 122 | } catch (Exception $e) { 123 | report($e); 124 | $this->info( 125 | "Failed to restore [{$message->getEventName()}][{$message->getId()}] message. Error: {$e->getMessage()}" 126 | ); 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /src/Commands/Archive/ProcessMessagesCommand.php: -------------------------------------------------------------------------------- 1 | option('streams')) { 18 | $this->error('Streams option is required with at least one stream name provided.'); 19 | return 1; 20 | } 21 | 22 | $olderThan = new Carbon('-' . $this->option('older_than')); 23 | $streams = explode(',', $this->option('streams')); 24 | 25 | $messageCount = 0; 26 | foreach ($streams as $name) { 27 | $stream = new Stream($name); 28 | $messages = $stream->read(); 29 | foreach ($messages[$name] ?? [] as $id => $message) { 30 | if ($olderThan->lt(Carbon::createFromTimestamp($message['created']))) { 31 | continue; 32 | } 33 | 34 | $this->process($name, $id, $message); 35 | $messageCount++; 36 | } 37 | } 38 | 39 | $this->info( 40 | sprintf( 41 | "Total of %d %s processed.", 42 | $messageCount, 43 | Str::plural('message', $messageCount) 44 | ) 45 | ); 46 | 47 | return 0; 48 | } 49 | 50 | abstract protected function process(string $stream, string $id, array $message): void; 51 | 52 | /** 53 | * @inheritDoc 54 | */ 55 | protected function getOptions(): array 56 | { 57 | return [ 58 | [ 59 | 'streams', 60 | null, 61 | InputOption::VALUE_REQUIRED, 62 | 'List of streams to process separated by comma.', 63 | ], 64 | [ 65 | 'older_than', 66 | null, 67 | InputOption::VALUE_REQUIRED, 68 | 'How old messages should be to get process. The format to use this option looks like: 1 day, 1 week, 5 days, 4 weeks etc. It will take the current time and subtract the option value.', 69 | ], 70 | ]; 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/Commands/Archive/PurgeCommand.php: -------------------------------------------------------------------------------- 1 | delete($id); 30 | if ($result === 0) { 31 | $this->warn("Message [$id] from the '$stream' stream could not be purged or is already deleted."); 32 | return; 33 | } 34 | 35 | $this->info("Message [$id] has been purged from the '$stream' stream."); 36 | } catch (Exception $e) { 37 | report($e); 38 | $this->warn("Message [$id] from the '$stream' stream could not be purged. Error: " . $e->getMessage()); 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Commands/FailedMessages/FlushFailedCommand.php: -------------------------------------------------------------------------------- 1 | repository->count(); 31 | if ($count === 0) { 32 | $this->info('No messages to remove.'); 33 | return 0; 34 | } 35 | 36 | $this->repository->flush(); 37 | $this->info(sprintf('Flushed %d %s.', $count, Str::plural('message', $count))); 38 | 39 | return 0; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/Commands/FailedMessages/ListFailedCommand.php: -------------------------------------------------------------------------------- 1 | repository->count() === 0) { 46 | $this->info('No failed messages'); 47 | return 0; 48 | } 49 | 50 | $this->table( 51 | $this->option('compact') ? $this->compactHeaders : $this->headers, 52 | $this->getMessages() 53 | ); 54 | 55 | return 0; 56 | } 57 | 58 | protected function getMessages(): array 59 | { 60 | $isCompact = $this->option('compact'); 61 | 62 | return $this->repository->all()->map(static function (FailedMessage $message) use ($isCompact): array { 63 | $serialized = $message->jsonSerialize(); 64 | if ($isCompact) { 65 | unset($serialized['receiver'], $serialized['date']); 66 | } 67 | 68 | return $serialized; 69 | })->toArray(); 70 | } 71 | 72 | /** 73 | * @inheritDoc 74 | */ 75 | protected function getOptions(): array 76 | { 77 | return [ 78 | [ 79 | 'compact', 80 | null, 81 | InputOption::VALUE_NONE, 82 | 'Returns only IDs, Stream names and Errors of failed messages.', 83 | ], 84 | ]; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/Commands/FailedMessages/RetryFailedCommand.php: -------------------------------------------------------------------------------- 1 | IdentifierSpecification::class, 38 | 'receiver' => ReceiverSpecification::class, 39 | 'stream' => StreamSpecification::class, 40 | ]; 41 | 42 | public function __construct( 43 | private readonly Repository $repository, 44 | private readonly MessagesFailer $failer, 45 | private readonly Archiver $archiver 46 | ) { 47 | parent::__construct(); 48 | } 49 | 50 | public function handle(): int 51 | { 52 | if ($this->repository->count() === 0) { 53 | $this->info('There are no failed messages to retry.'); 54 | return 0; 55 | } 56 | 57 | if ($this->option('all')) { 58 | $this->retry($this->repository->all()); 59 | 60 | return 0; 61 | } 62 | 63 | $options = $this->options(); 64 | if ($options['id'] || $options['stream'] || $options['receiver']) { 65 | return $this->retryBy(Arr::only($options, ['id', 'stream', 'receiver'])); 66 | } 67 | 68 | $this->warn('No retry option has been selected'); 69 | $this->info("Use '--all' flag or at least one of '--all', '--stream', '--receiver'"); 70 | 71 | return 0; 72 | } 73 | 74 | protected function retryBy(array $filters): int 75 | { 76 | $messages = $this->getMessages($filters); 77 | if ($messages->isEmpty()) { 78 | $this->info('There are no failed messages matching your criteria.'); 79 | return 0; 80 | } 81 | 82 | $this->retry($messages); 83 | 84 | return 0; 85 | } 86 | 87 | /** 88 | * @inheritDoc 89 | */ 90 | protected function getOptions(): array 91 | { 92 | return [ 93 | [ 94 | 'all', 95 | null, 96 | InputOption::VALUE_NONE, 97 | 'Retries all failed messages.', 98 | ], 99 | [ 100 | 'id', 101 | null, 102 | InputOption::VALUE_REQUIRED, 103 | 'Retries messages with given ID (messages from different streams may have same IDs and some messages may fail for multiple listeners).', 104 | ], 105 | [ 106 | 'stream', 107 | null, 108 | InputOption::VALUE_REQUIRED, 109 | 'Retries messages from given Stream name.', 110 | ], 111 | [ 112 | 'receiver', 113 | null, 114 | InputOption::VALUE_REQUIRED, 115 | 'Retries messages with given receiver associated with them.', 116 | ], 117 | [ 118 | 'purge', 119 | null, 120 | InputOption::VALUE_NONE, 121 | 'Will remove message from the stream if it will be retried successfully and there will be no other failures saved.', 122 | ], 123 | [ 124 | 'archive', 125 | null, 126 | InputOption::VALUE_NONE, 127 | 'Will remove message from the stream and store it in database if it will be retried successfully and there will be no other failures saved.', 128 | ], 129 | ]; 130 | } 131 | 132 | /** 133 | * Retries set of messages. 134 | * 135 | * @param Collection&FailedMessage[] $messages 136 | */ 137 | private function retry(Collection $messages): void 138 | { 139 | foreach ($messages as $message) { 140 | try { 141 | $this->failer->retry($message); 142 | $this->printSuccess($message); 143 | 144 | if ($this->option('archive')) { 145 | $this->archive($message); 146 | continue; 147 | } 148 | 149 | if ($this->option('purge')) { 150 | $this->purge($message); 151 | } 152 | } catch (MessageRetryFailedException $e) { 153 | report($e); 154 | 155 | $this->error($e->getMessage()); 156 | } 157 | } 158 | } 159 | 160 | private function printSuccess(FailedMessage $message): void 161 | { 162 | $this->info( 163 | sprintf( 164 | 'Successfully retried [%s] on %s stream by [%s] listener', 165 | $message->getId(), 166 | $message->getStream()->getName(), 167 | $message->getReceiver() 168 | ) 169 | ); 170 | } 171 | 172 | private function archive(FailedMessage $message): void 173 | { 174 | if ($this->hasOtherFailures($message)) { 175 | return; 176 | } 177 | 178 | try { 179 | $receivedMessage = new ReceivedMessage($message->getId(), $message->getStreamMessage()); 180 | $this->archiver->archive($receivedMessage); 181 | $this->info( 182 | "Message [{$message->getId()}] has been archived from the '{$message->getStream()->getName()}' stream." 183 | ); 184 | } catch (Exception $e) { 185 | report($e); 186 | $this->warn( 187 | "Message [{$message->getId()}] from the '{$message->getStream()->getName()}' stream could not be archived. Error: " . $e->getMessage( 188 | ) 189 | ); 190 | } 191 | } 192 | 193 | private function hasOtherFailures(FailedMessage $message): bool 194 | { 195 | return $this->getMessages([ 196 | 'id' => $message->getId(), 197 | 'stream' => $message->getStream()->getName(), 198 | ])->isNotEmpty(); 199 | } 200 | 201 | /** 202 | * @return Collection 203 | */ 204 | private function getMessages(array $filters): Collection 205 | { 206 | $specification = $this->prepareSpecification(array_filter($filters)); 207 | 208 | return $this->repository->all() 209 | ->filter(static fn (FailedMessage $message): bool => $specification->isSatisfiedBy($message)); 210 | } 211 | 212 | private function prepareSpecification(array $filters): Specification 213 | { 214 | $specifications = []; 215 | foreach ($filters as $filter => $value) { 216 | if (!isset($this->specifications[$filter])) { 217 | continue; 218 | } 219 | 220 | $specifications[] = new $this->specifications[$filter]($value); 221 | } 222 | 223 | return new MatchAllSpecification(...$specifications); 224 | } 225 | 226 | private function purge(FailedMessage $message): void 227 | { 228 | if ($this->hasOtherFailures($message)) { 229 | return; 230 | } 231 | 232 | $result = $message->getStream()->delete($message->getId()); 233 | if ($result !== 0) { 234 | $this->info( 235 | "Message [{$message->getId()}] has been purged from the '{$message->getStream()->getName()}' stream." 236 | ); 237 | } 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /src/Commands/ListCommand.php: -------------------------------------------------------------------------------- 1 | option('compact'); 27 | 28 | if (!$isCompact) { 29 | $headers[] = 'Listeners'; 30 | } 31 | 32 | $this->table( 33 | $headers, 34 | $this->makeRows(), 35 | $isCompact ? 'compact' : 'default' 36 | ); 37 | 38 | return 0; 39 | } 40 | 41 | protected function makeRows(): array 42 | { 43 | $listeners = ListenersStack::all(); 44 | $rows = []; 45 | 46 | $isCompact = $this->option('compact'); 47 | 48 | foreach ($listeners as $event => $listener) { 49 | if ($isCompact) { 50 | $rows[] = [$event]; 51 | 52 | continue; 53 | } 54 | 55 | $eventListeners = implode(PHP_EOL, $listener); 56 | 57 | $rows[] = [ 58 | $event, 59 | $eventListeners ?: 'none', 60 | ]; 61 | } 62 | 63 | return $rows; 64 | } 65 | 66 | /** 67 | * @inheritDoc 68 | */ 69 | protected function getOptions(): array 70 | { 71 | return [ 72 | [ 73 | 'compact', 74 | null, 75 | InputOption::VALUE_NONE, 76 | 'Returns only names of events that are registered in streamer.', 77 | ], 78 | ]; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /src/Commands/ListenCommand.php: -------------------------------------------------------------------------------- 1 | getEventsToListen(); 61 | $missingListeners = 0; 62 | foreach ($events as $event) { 63 | if (ListenersStack::hasListener($event)) { 64 | continue; 65 | } 66 | 67 | $this->warn("There are no local listeners associated with '$event' event in configuration."); 68 | $missingListeners++; 69 | } 70 | 71 | if (count($events) === $missingListeners) { 72 | return 1; 73 | } 74 | 75 | if ($this->option('last_id') !== null) { 76 | if (count($events) > 1) { 77 | $this->info('The last_id value will be used for all listened events.'); 78 | } 79 | $this->streamer->startFrom($this->option('last_id')); 80 | } 81 | 82 | if ($this->option('group')) { 83 | $this->setupGroupListening(); 84 | } 85 | 86 | $maxAttempts = $this->option('max-attempts'); 87 | if ($maxAttempts !== null) { 88 | $this->maxAttempts = (int) $maxAttempts; 89 | } 90 | 91 | $this->listen($events, function (ReceivedMessage $message): void { 92 | $failed = false; 93 | foreach (ListenersStack::getListenersFor($message->getEventName()) as $listener) { 94 | $receiver = app()->make($listener); 95 | if (!$receiver instanceof MessageReceiver) { 96 | $this->error("Listener class [$listener] needs to implement MessageReceiver"); 97 | continue; 98 | } 99 | 100 | try { 101 | $receiver->handle($message); 102 | } catch (Throwable $e) { 103 | $failed = true; 104 | report($e); 105 | 106 | $this->printError($message, $listener, $e); 107 | $this->failer->store($message, $receiver, new \Exception($e->getMessage(), $e->getCode(), $e)); 108 | 109 | continue; 110 | } 111 | 112 | $this->printInfo($message, $listener); 113 | } 114 | 115 | if ($failed) { 116 | return; 117 | } 118 | 119 | if ($this->option('archive')) { 120 | $this->archive($message); 121 | } 122 | 123 | if (!$this->option('archive') && $this->option('purge')) { 124 | $this->purge($message); 125 | } 126 | }); 127 | 128 | return 0; 129 | } 130 | 131 | /** 132 | * @inheritDoc 133 | */ 134 | protected function getArguments(): array 135 | { 136 | return [ 137 | [ 138 | 'events', 139 | InputArgument::OPTIONAL, 140 | 'Name (or names separated by comma) of an event(s) that should be listened to', 141 | ], 142 | ]; 143 | } 144 | 145 | /** 146 | * @inheritDoc 147 | */ 148 | protected function getOptions(): array 149 | { 150 | return [ 151 | [ 152 | 'all', 153 | null, 154 | InputOption::VALUE_NONE, 155 | 'Will start listening to all events that are registered in Listeners Stack. Will override usage of "events" argument', 156 | ], 157 | [ 158 | 'group', 159 | null, 160 | InputOption::VALUE_REQUIRED, 161 | 'Name of your streaming group. Only when group is provided listener will listen on group as consumer', 162 | ], 163 | [ 164 | 'consumer', 165 | null, 166 | InputOption::VALUE_REQUIRED, 167 | 'Name of your group consumer. If not provided a name will be created as groupname-timestamp', 168 | ], 169 | [ 170 | 'reclaim', 171 | null, 172 | InputOption::VALUE_REQUIRED, 173 | 'Milliseconds of pending messages idle time, that should be reclaimed for current consumer in this group. Can be only used with group listening', 174 | ], 175 | [ 176 | 'last_id', 177 | null, 178 | InputOption::VALUE_REQUIRED, 179 | 'ID from which listener should start reading messages', 180 | ], 181 | [ 182 | 'keep-alive', 183 | null, 184 | InputOption::VALUE_NONE, 185 | 'Will keep listener alive when any unexpected non-listener related error will occur by simply restarting listening.', 186 | ], 187 | [ 188 | 'max-attempts', 189 | null, 190 | InputOption::VALUE_REQUIRED, 191 | 'Number of maximum attempts to restart a listener on an unexpected non-listener related error', 192 | ], 193 | [ 194 | 'purge', 195 | null, 196 | InputOption::VALUE_NONE, 197 | 'Will remove message from the stream if it will be processed successfully by all listeners in the current stack.', 198 | ], 199 | [ 200 | 'archive', 201 | null, 202 | InputOption::VALUE_NONE, 203 | 'Will remove message from the stream and store it in database if it will be processed successfully by all listeners in the current stack.', 204 | ], 205 | ]; 206 | } 207 | 208 | private function getEventsToListen(): array 209 | { 210 | if ($this->option('all')) { 211 | return array_keys(ListenersStack::all()); 212 | } 213 | 214 | $events = $this->argument('events'); 215 | if (!$events) { 216 | $this->error( 217 | 'Either "events" argument with list of events or "--all" option is required to start listening.' 218 | ); 219 | } 220 | 221 | return explode(',', $events); 222 | } 223 | 224 | private function setupGroupListening(): void 225 | { 226 | $multiStream = new Stream\MultiStream($this->getEventsToListen(), $this->option('group')); 227 | $consumer = $this->option('consumer'); 228 | if (!$consumer) { 229 | $consumer = $this->option('group') . '-' . Str::random(); 230 | } 231 | 232 | if ($this->option('reclaim')) { 233 | if ($multiStream->streams()->count() > 1) { 234 | $this->info('Reclaiming will reclaim pending messages on all listened events.'); 235 | } 236 | $this->reclaimMessages($multiStream, $consumer); 237 | } 238 | 239 | $this->streamer->asConsumer($consumer, $this->option('group')); 240 | } 241 | 242 | private function reclaimMessages(Stream\MultiStream $multiStream, string $consumerName): void 243 | { 244 | foreach ($multiStream->streams() as $stream) { 245 | $pendingMessages = $stream->pending($this->option('group')); 246 | $messages = array_map(static fn ($message) => $message[0], $pendingMessages); 247 | if ($messages === []) { 248 | continue; 249 | } 250 | 251 | $consumer = new Stream\Consumer($consumerName, $stream, $this->option('group')); 252 | $consumer->claim($messages, (int) $this->option('reclaim')); 253 | } 254 | } 255 | 256 | /** 257 | * @throws Throwable 258 | */ 259 | private function listen(array $events, callable $handler): void 260 | { 261 | try { 262 | $this->streamer->listen($events, $handler); 263 | } catch (Throwable $e) { 264 | if (!$this->option('keep-alive')) { 265 | throw $e; 266 | } 267 | 268 | $this->error($e->getMessage()); 269 | report($e); 270 | 271 | if ($this->maxAttempts === 0) { 272 | return; 273 | } 274 | 275 | $this->warn('Starting listener again due to unexpected error.'); 276 | if ($this->maxAttempts !== null) { 277 | $this->warn("Attempts left: $this->maxAttempts"); 278 | $this->maxAttempts--; 279 | } 280 | 281 | $this->listen($events, $handler); 282 | } 283 | } 284 | 285 | private function printError(ReceivedMessage $message, string $listener, Throwable $e): void 286 | { 287 | $this->error( 288 | sprintf( 289 | "Listener error. Failed processing message with ID %s on '%s' stream by %s. Error: %s", 290 | $message->getId(), 291 | $message->getEventName(), 292 | $listener, 293 | $e->getMessage() 294 | ) 295 | ); 296 | } 297 | 298 | private function printInfo(ReceivedMessage $message, string $listener): void 299 | { 300 | $this->info( 301 | sprintf( 302 | "Processed message [%s] on '%s' stream by [%s] listener.", 303 | $message->getId(), 304 | $message->getEventName(), 305 | $listener 306 | ) 307 | ); 308 | } 309 | 310 | /** 311 | * Removes message from the stream and stores message in DB. 312 | * No verification for other consumers is made in this archiving option. 313 | */ 314 | private function archive(ReceivedMessage $message): void 315 | { 316 | try { 317 | $this->archiver->archive($message); 318 | $this->info( 319 | "Message [{$message->getId()}] has been archived from the '{$message->getEventName()}' stream." 320 | ); 321 | } catch (Exception $e) { 322 | $this->warn( 323 | "Message [{$message->getId()}] from the '{$message->getEventName()}' stream could not be archived. Error: " . $e->getMessage( 324 | ) 325 | ); 326 | } 327 | } 328 | 329 | /** 330 | * Removes message from the stream. 331 | * No verification for other consumers is made in this purging option. 332 | */ 333 | private function purge(ReceivedMessage $message): void 334 | { 335 | $stream = new Stream($message->getEventName()); 336 | $result = $stream->delete($message->getId()); 337 | if ($result !== 0) { 338 | $this->info("Message [{$message->getId()}] has been purged from the '{$message->getEventName()}' stream."); 339 | } 340 | } 341 | } 342 | -------------------------------------------------------------------------------- /src/Concerns/ConnectsWithRedis.php: -------------------------------------------------------------------------------- 1 | setOption(\Redis::OPT_PREFIX, ''); 21 | 22 | return $connection; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Concerns/EmitsStreamerEvents.php: -------------------------------------------------------------------------------- 1 | postSave(); 22 | }); 23 | 24 | static::created(static function (Model $model): void { 25 | $model->postCreate(); 26 | }); 27 | 28 | static::deleted(static function (Model $model): void { 29 | $model->postDelete(); 30 | }); 31 | } 32 | 33 | /** 34 | * Called after record is successfully updated. 35 | */ 36 | public function postSave(): void 37 | { 38 | if (!$this->wasChanged() || !$this->canStream()) { 39 | return; 40 | } 41 | 42 | $payload = $this->makeBasePayload(); 43 | foreach ($this->getChanges() as $field => $change) { 44 | $payload['fields'][] = $field; 45 | $payload['before'][$field] = $this->getOriginal($field); 46 | $payload['after'][$field] = $change; 47 | } 48 | 49 | Streamer::emit(new EloquentModelEvent($this->getEventName('updated'), $payload)); 50 | } 51 | 52 | /** 53 | * Called after record is successfully created. 54 | */ 55 | public function postCreate(): void 56 | { 57 | if (!$this->canStream()) { 58 | return; 59 | } 60 | $payload = $this->makeBasePayload(); 61 | foreach ($this->getAttributes() as $field => $change) { 62 | $payload['fields'][] = $field; 63 | $payload['before'][$field] = null; 64 | $payload['after'][$field] = $change; 65 | } 66 | 67 | Streamer::emit(new EloquentModelEvent($this->getEventName('created'), $payload)); 68 | } 69 | 70 | /** 71 | * Called after record is successfully deleted. 72 | */ 73 | public function postDelete(): void 74 | { 75 | if (!$this->canStream()) { 76 | return; 77 | } 78 | 79 | $payload = $this->makeBasePayload(); 80 | $payload['deleted'] = true; 81 | Streamer::emit(new EloquentModelEvent($this->getEventName('deleted'), $payload)); 82 | } 83 | 84 | /** 85 | * Method that can be overridden to add custom logic which will determine 86 | * whether the given model should have events emitted or not. 87 | * Returns true by default, emitting events for any case. 88 | */ 89 | protected function canStream(): bool 90 | { 91 | return true; 92 | } 93 | 94 | /** 95 | * Method that can be overridden to add additional data to each event payload. 96 | * It will be added as 'top' level array. If method returns empty array, 97 | * then the 'additional' data won't be added to payload. 98 | */ 99 | protected function getAdditionalPayloadData(): array 100 | { 101 | return []; 102 | } 103 | 104 | private function makeBasePayload(): array 105 | { 106 | return array_filter([ 107 | $this->getKeyName() => $this->getKey(), 108 | 'additional' => $this->getAdditionalPayloadData(), 109 | ]); 110 | } 111 | 112 | private function getEventName(string $action): string 113 | { 114 | $suffix = '.' . $action; 115 | $name = class_basename($this) . $suffix; 116 | if ($this->baseEventName) { 117 | $name = $this->baseEventName . $suffix; 118 | } 119 | 120 | return strtolower($name); 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /src/Concerns/HashableMessage.php: -------------------------------------------------------------------------------- 1 | content) { 21 | return; 22 | } 23 | 24 | $data = $this->content['data']; 25 | if (is_array($this->content['data']) || is_object($this->content['data'])) { 26 | $data = json_encode($this->content['data'], JSON_THROW_ON_ERROR); 27 | } 28 | 29 | $key = $this->content['type'] . $this->content['name'] . $this->content['domain'] . $data; 30 | $hash = hash('SHA256', $key); 31 | $this->content['hash'] = $hash; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/Contracts/ArchiveStorage.php: -------------------------------------------------------------------------------- 1 | handler] - one handler for one stream. 14 | */ 15 | public function listen(string|array $events, array|callable $handlers): void; 16 | } 17 | -------------------------------------------------------------------------------- /src/Contracts/MessageReceiver.php: -------------------------------------------------------------------------------- 1 | name; 22 | } 23 | 24 | /** 25 | * Event type. Can be one of the predefined types from this contract. 26 | */ 27 | public function type(): string 28 | { 29 | return Event::TYPE_EVENT; 30 | } 31 | 32 | /** 33 | * Event payload that will be sent as message to Stream. 34 | */ 35 | public function payload(): array 36 | { 37 | return $this->payload; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Errors/FailedMessage.php: -------------------------------------------------------------------------------- 1 | date = $date ?? Carbon::now()->toDateTimeString(); 25 | } 26 | 27 | public function getReceiver(): string 28 | { 29 | return $this->receiver; 30 | } 31 | 32 | public function getError(): string 33 | { 34 | return $this->error; 35 | } 36 | 37 | public function getDate(): string 38 | { 39 | return $this->date; 40 | } 41 | 42 | /** 43 | * Returns stream message for the given failed message. 44 | * 45 | * @throws MessageRetryFailedException 46 | */ 47 | public function getStreamMessage(): array 48 | { 49 | $range = new Range($this->getId(), $this->getId()); 50 | $stream = $this->getStream(); 51 | $messages = $stream->readRange($range, 1); 52 | 53 | if (!$messages || count($messages) !== 1) { 54 | throw new MessageRetryFailedException( 55 | $this, 56 | "No matching messages found on a '{$stream->getName()}' stream for ID #{$this->getId()}." 57 | ); 58 | } 59 | 60 | return array_pop($messages); 61 | } 62 | 63 | public function getId(): string 64 | { 65 | return $this->id; 66 | } 67 | 68 | public function getStream(): Stream 69 | { 70 | return new Stream($this->stream); 71 | } 72 | 73 | /** 74 | * @inheritDoc 75 | */ 76 | public function jsonSerialize(): array 77 | { 78 | return [ 79 | 'id' => $this->id, 80 | 'stream' => $this->stream, 81 | 'receiver' => $this->receiver, 82 | 'error' => $this->error, 83 | 'date' => $this->date, 84 | ]; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /src/Errors/FailedMessagesHandler.php: -------------------------------------------------------------------------------- 1 | makeReceiver($message); 29 | 30 | $receivedMessage = null; 31 | try { 32 | $receivedMessage = new ReceivedMessage($message->getId(), $message->getStreamMessage()); 33 | $listener->handle($receivedMessage); 34 | } catch (Throwable $e) { 35 | if ($receivedMessage === null) { 36 | throw $e; 37 | } 38 | 39 | $this->store($receivedMessage, $listener, $e); 40 | 41 | throw new MessageRetryFailedException($message, $e->getMessage()); 42 | } finally { 43 | $this->repository->remove($message); 44 | } 45 | } 46 | 47 | /** 48 | * @inheritDoc 49 | */ 50 | public function store(ReceivedMessage $message, MessageReceiver $receiver, Exception $e): void 51 | { 52 | $this->repository->add(new FailedMessage(...[ 53 | $message->getId(), 54 | $message->getContent()['name'] ?? '', 55 | $receiver::class, 56 | $e->getMessage(), 57 | ])); 58 | } 59 | 60 | /** 61 | * @throws MessageRetryFailedException 62 | */ 63 | private function makeReceiver(FailedMessage $message): MessageReceiver 64 | { 65 | if (!class_exists($message->getReceiver())) { 66 | throw new MessageRetryFailedException($message, 'Receiver class does not exists'); 67 | } 68 | 69 | $listener = app($message->getReceiver()); 70 | if (!$listener instanceof MessageReceiver) { 71 | throw new MessageRetryFailedException( 72 | $message, 73 | 'Receiver class is not an instance of MessageReceiver contract' 74 | ); 75 | } 76 | 77 | return $listener; 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/Errors/MessagesRepository.php: -------------------------------------------------------------------------------- 1 | redis()->sMembers(self::ERRORS_SET); 23 | if (!$elements) { 24 | return collect(); 25 | } 26 | 27 | $decode = static fn ($item): array => array_values(json_decode((string) $item, true, 512, JSON_THROW_ON_ERROR)); 28 | 29 | return collect($elements) 30 | ->map(static fn ($item): FailedMessage => new FailedMessage(...$decode($item))) 31 | ->sortBy(static fn (FailedMessage $message): string => $message->getDate()); 32 | } 33 | 34 | /** 35 | * @inheritDoc 36 | */ 37 | public function add(FailedMessage $message): void 38 | { 39 | $this->redis()->sAdd(self::ERRORS_SET, json_encode($message, JSON_THROW_ON_ERROR)); 40 | } 41 | 42 | /** 43 | * @inheritDoc 44 | */ 45 | public function remove(FailedMessage $message): void 46 | { 47 | $this->redis()->sRem(self::ERRORS_SET, json_encode($message, JSON_THROW_ON_ERROR)); 48 | } 49 | 50 | /** 51 | * @inheritDoc 52 | */ 53 | public function flush(): void 54 | { 55 | $this->redis()->spop(self::ERRORS_SET, $this->count()); 56 | } 57 | 58 | /** 59 | * @inheritDoc 60 | */ 61 | public function count(): int 62 | { 63 | return $this->redis()->sCard(self::ERRORS_SET); 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /src/Errors/Specifications/IdentifierSpecification.php: -------------------------------------------------------------------------------- 1 | getId() === $this->id; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Errors/Specifications/MatchAllSpecification.php: -------------------------------------------------------------------------------- 1 | specifications = $specifications; 20 | } 21 | 22 | public function isSatisfiedBy(FailedMessage $message): bool 23 | { 24 | foreach ($this->specifications as $specification) { 25 | if (!$specification->isSatisfiedBy($message)) { 26 | return false; 27 | } 28 | } 29 | 30 | return true; 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Errors/Specifications/ReceiverSpecification.php: -------------------------------------------------------------------------------- 1 | getReceiver() === $this->receiver; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Errors/Specifications/StreamSpecification.php: -------------------------------------------------------------------------------- 1 | getStream()->getName() === $this->stream; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/EventDispatcher/Message.php: -------------------------------------------------------------------------------- 1 | $meta['_id'] ?? '*', 24 | 'original_id' => $meta['original_id'] ?? null, 25 | 'type' => $meta['type'] ?? Event::TYPE_EVENT, 26 | 'version' => '1.3', 27 | 'name' => $meta['name'], 28 | 'domain' => $meta['domain'] ?? '', 29 | 'created' => $meta['created'] ?? time(), 30 | 'data' => json_encode($data, JSON_THROW_ON_ERROR), 31 | ], static fn ($v): bool => $v !== null); 32 | 33 | $this->content = $payload; 34 | $this->hashIt(); 35 | } 36 | 37 | /** 38 | * @throws JsonException 39 | */ 40 | public function getData(): array 41 | { 42 | return json_decode((string) $this->content['data'], true, 512, JSON_THROW_ON_ERROR); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/EventDispatcher/ReceivedMessage.php: -------------------------------------------------------------------------------- 1 | content = $content; 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/EventDispatcher/StreamMessage.php: -------------------------------------------------------------------------------- 1 | content['_id'] ?? ''; 17 | } 18 | 19 | public function getEventName(): string 20 | { 21 | return $this->content['name'] ?? ''; 22 | } 23 | 24 | public function getContent(): array 25 | { 26 | return $this->content; 27 | } 28 | 29 | /** 30 | * Retrieves values directly from the content data. 31 | * 32 | * @param null|string $key dot.notation string 33 | * @param null $default 34 | */ 35 | public function get(?string $key = null, $default = null) 36 | { 37 | return Arr::get($this->getData(), $key, $default); 38 | } 39 | 40 | public function getData(): array 41 | { 42 | return $this->content['data'] ?? []; 43 | } 44 | 45 | /** 46 | * Retrieves values directly from the content data. 47 | */ 48 | public function only(array|string $keys): array 49 | { 50 | return Arr::only($this->getData(), $keys); 51 | } 52 | 53 | /** 54 | * Retrieves values directly from the content data. 55 | */ 56 | public function except(array|string $keys): array 57 | { 58 | return Arr::except($this->getData(), $keys); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/EventDispatcher/Streamer.php: -------------------------------------------------------------------------------- 1 | readTimeout = (float) config('streamer.stream_read_timeout', 1.0); 45 | $this->listenTimeout = (float) config('streamer.listen_timeout', 1.0); 46 | $this->readSleep = (float) config('streamer.read_sleep', 1.0); 47 | $this->readTimeout *= 1000.0; 48 | $this->listenTimeout *= 1000.0; 49 | } 50 | 51 | public function startFrom(string $startFrom): self 52 | { 53 | $this->startFrom = $startFrom; 54 | 55 | return $this; 56 | } 57 | 58 | public function asConsumer(string $consumer, string $group): self 59 | { 60 | $this->consumer = $consumer; 61 | $this->group = $group; 62 | 63 | return $this; 64 | } 65 | 66 | /** 67 | * @throws JsonException 68 | */ 69 | public function emit(Event $event, string $id = '*'): string 70 | { 71 | $meta = [ 72 | 'type' => $event->type(), 73 | 'domain' => config('streamer.domain'), 74 | 'name' => $event->name(), 75 | 'created' => time(), 76 | ]; 77 | 78 | $message = new Message($meta, $event->payload()); 79 | $stream = new Stream($event->name()); 80 | 81 | $id = $stream->add($message, $id); 82 | 83 | if ($event instanceof Replayable) { 84 | $this->history->record(new Snapshot($id, $event)); 85 | } 86 | 87 | return $id; 88 | } 89 | 90 | /** 91 | * Handler is invoked with \Prwnr\Streamer\EventDispatcher\ReceivedMessage instance as first argument 92 | * and with \Prwnr\Streamer\EventDispatcher\Streamer as second argument. 93 | * 94 | * @inheritdoc 95 | * 96 | * @throws Throwable 97 | */ 98 | public function listen(string|array $events, array|callable $handlers): void 99 | { 100 | if ($this->inLoop) { 101 | return; 102 | } 103 | 104 | [$events, $handlers] = $this->parseArgs($events, $handlers); 105 | 106 | if ($this->consumer && $this->group) { 107 | $this->adjustGroupReadTimeout(); 108 | } 109 | 110 | try { 111 | $multiStream = new Stream\MultiStream($events, $this->group, $this->consumer); 112 | $this->listenOn($multiStream, $handlers); 113 | } finally { 114 | $this->inLoop = false; 115 | } 116 | } 117 | 118 | /** 119 | * Cancels current listener loop. 120 | */ 121 | public function cancel(): void 122 | { 123 | $this->canceled = true; 124 | } 125 | 126 | /** 127 | * @throws Exception 128 | */ 129 | private function parseArgs($events, $handlers): array 130 | { 131 | $eventsList = Arr::wrap($events); 132 | if (is_array($handlers) && count($eventsList) === 1 && count($handlers) > 1) { 133 | throw new InvalidListeningArgumentsException(); 134 | } 135 | 136 | if (is_callable($handlers)) { 137 | return [$eventsList, [$handlers]]; 138 | } 139 | 140 | foreach ($eventsList as $event) { 141 | if (!isset($handlers[$event])) { 142 | throw new InvalidListeningArgumentsException(); 143 | } 144 | } 145 | 146 | return [$eventsList, $handlers]; 147 | } 148 | 149 | /** 150 | * When listening on group, timeout should not be equal to 0, because it is required to know 151 | * when reading history of message is finished and when listener should start 152 | * reading only new messages via '>' key. 153 | */ 154 | private function adjustGroupReadTimeout(): void 155 | { 156 | if ($this->readTimeout === 0.0) { 157 | $this->readTimeout = 2000.0; 158 | } 159 | } 160 | 161 | private function listenOn(Stream\MultiStream $streams, array $handlers): void 162 | { 163 | $start = microtime(true) * 1000; 164 | $lastSeenId = $this->startFrom ?? $streams->getNewEntriesKey(); 165 | while (!$this->canceled) { 166 | $this->inLoop = true; 167 | $payload = $streams->await($lastSeenId, $this->readTimeout); 168 | if (!$payload) { 169 | $lastSeenId = $streams->getNewEntriesKey(); 170 | sleep((int) $this->readSleep); 171 | if ($this->shouldStop($start)) { 172 | break; 173 | } 174 | continue; 175 | } 176 | 177 | $this->processPayload($payload, $handlers, $streams); 178 | 179 | $lastSeenId = $streams->getNewEntriesKey(); 180 | 181 | $start = microtime(true) * 1000; 182 | } 183 | } 184 | 185 | private function shouldStop(float $start): bool 186 | { 187 | if ($this->listenTimeout === 0.0) { 188 | return false; 189 | } 190 | return microtime(true) * 1000 - $start > $this->listenTimeout; 191 | } 192 | 193 | private function processPayload(array $payload, array $handlers, Stream\MultiStream $streams): void 194 | { 195 | foreach ($payload as $message) { 196 | try { 197 | $this->forward($message, $this->getHandler($message['stream'], $handlers)); 198 | $streams->acknowledge([$message['stream'] => $message['id']]); 199 | if ($this->canceled) { 200 | break; 201 | } 202 | } catch (Throwable $ex) { 203 | $this->report($message['id'], $streams->streams()->get($message['stream']), $ex); 204 | continue; 205 | } 206 | } 207 | } 208 | 209 | /** 210 | * @throws JsonException 211 | */ 212 | private function forward(array $message, callable $handler): void 213 | { 214 | $handler(new ReceivedMessage($message['id'], $message['message']), $this); 215 | } 216 | 217 | private function getHandler(string $stream, array $handlers): callable 218 | { 219 | if (count($handlers) === 1) { 220 | return Arr::first($handlers); 221 | } 222 | 223 | return $handlers[$stream]; 224 | } 225 | 226 | private function report(string $id, Stream $on, Throwable $ex): void 227 | { 228 | $error = "Listener error. Failed processing message with ID $id on '{$on->getName()}' stream. Error: {$ex->getMessage()}"; 229 | Log::error($error); 230 | } 231 | } 232 | -------------------------------------------------------------------------------- /src/Exceptions/AcknowledgingFailedException.php: -------------------------------------------------------------------------------- 1 | getId(), 17 | $message->getStream()->getName(), 18 | $message->getReceiver(), 19 | $additional 20 | ); 21 | 22 | parent::__construct($errorMessage); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/Exceptions/RestoringFailedException.php: -------------------------------------------------------------------------------- 1 | redis()->lPush($snapshot->getKey(), json_encode($snapshot->toArray(), JSON_THROW_ON_ERROR)); 24 | } 25 | 26 | /** 27 | * @inheritDoc 28 | * @throws JsonException 29 | */ 30 | public function replay(string $event, string $identifier, Carbon $until = null): array 31 | { 32 | $key = $event . Snapshot::KEY_SEPARATOR . $identifier; 33 | $snapshots = $this->redis()->lRange($key, 0, $this->redis()->lLen($key)); 34 | $snapshotsCount = count($snapshots) - 1; 35 | $last = json_decode((string) $snapshots[0], true, 512, JSON_THROW_ON_ERROR)['id']; 36 | $first = json_decode((string) $snapshots[$snapshotsCount], true, 512, JSON_THROW_ON_ERROR)['id']; 37 | 38 | $stream = new Stream($event); 39 | $range = $stream->readRange(new Stream\Range($first, $last)); 40 | 41 | $result = []; 42 | for ($i = $snapshotsCount; $i >= 0; $i--) { 43 | $snapshot = json_decode((string) $snapshots[$i], true, 512, JSON_THROW_ON_ERROR); 44 | if ($until && $until <= Carbon::createFromFormat('Y-m-d H:i:s', $snapshot['date'])) { 45 | return $result; 46 | } 47 | 48 | $message = $range[$snapshot['id']] ?? null; 49 | if (!$message) { 50 | continue; 51 | } 52 | 53 | $record = json_decode((string) $message['data'], true, 512, JSON_THROW_ON_ERROR); 54 | foreach ($record as $field => $value) { 55 | $result[$field] = $value; 56 | } 57 | } 58 | 59 | return $result; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /src/History/Snapshot.php: -------------------------------------------------------------------------------- 1 | name = $event->name(); 33 | $this->identifier = $event->getIdentifier(); 34 | $this->date = Carbon::now(); 35 | } 36 | 37 | public function getId(): string 38 | { 39 | return $this->id; 40 | } 41 | 42 | public function getDate(): Carbon 43 | { 44 | return $this->date; 45 | } 46 | 47 | /** 48 | * Returns combination of event name and identifier. 49 | */ 50 | public function getKey(): string 51 | { 52 | return $this->name . self::KEY_SEPARATOR . $this->identifier; 53 | } 54 | 55 | /** 56 | * @inheritDoc 57 | */ 58 | public function toArray(): array 59 | { 60 | return [ 61 | 'id' => $this->id, 62 | 'name' => $this->name, 63 | 'identifier' => $this->identifier, 64 | 'date' => $this->date->format('Y-m-d H:i:s'), 65 | ]; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/ListenersStack.php: -------------------------------------------------------------------------------- 1 | [listeners]] 21 | */ 22 | public static function addMany(array $listenersStack): void 23 | { 24 | foreach ($listenersStack as $event => $listeners) { 25 | if (is_string($listeners)) { 26 | $listeners = [$listeners]; 27 | } 28 | 29 | foreach ($listeners as $listener) { 30 | self::add($event, $listener); 31 | } 32 | } 33 | } 34 | 35 | /** 36 | * Add event listener to stack. 37 | */ 38 | public static function add(string $event, string $listener): void 39 | { 40 | if (!isset(self::$events[$event])) { 41 | self::$events[$event] = []; 42 | } 43 | 44 | if (!in_array($listener, self::$events[$event], true)) { 45 | self::$events[$event][] = $listener; 46 | } 47 | } 48 | 49 | public static function getListenersFor(string $event): array 50 | { 51 | if (self::hasListener($event)) { 52 | return self::$events[$event]; 53 | } 54 | 55 | return []; 56 | } 57 | 58 | public static function hasListener(string $event): bool 59 | { 60 | return isset(self::$events[$event]); 61 | } 62 | 63 | public static function all(): array 64 | { 65 | return self::$events; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/Stream.php: -------------------------------------------------------------------------------- 1 | name; 35 | } 36 | 37 | public function getNewEntriesKey(): string 38 | { 39 | return self::NEW_ENTRIES; 40 | } 41 | 42 | public function add(StreamableMessage $message, string $id = '*'): string 43 | { 44 | return (string) $this->redis()->xAdd($this->name, $id, $message->getContent()); 45 | } 46 | 47 | public function delete(string $id): int 48 | { 49 | return (int) $this->redis()->xDel($this->name, [$id]); 50 | } 51 | 52 | public function read(string $from = self::FROM_START, int $limit = 0): array 53 | { 54 | $result = []; 55 | if ($limit !== 0) { 56 | $result = $this->redis()->xRead([$this->name => $from], $limit); 57 | } 58 | 59 | if ($limit === 0) { 60 | $result = $this->redis()->xRead([$this->name => $from]); 61 | } 62 | 63 | if (!is_array($result)) { 64 | return []; 65 | } 66 | 67 | return $result; 68 | } 69 | 70 | public function await(string $lastSeenId = self::FROM_START, float $timeout = 0.0): array 71 | { 72 | $result = $this->redis()->xRead([$this->name => $lastSeenId], 0, $timeout); 73 | if (!is_array($result)) { 74 | return []; 75 | } 76 | 77 | return $result; 78 | } 79 | 80 | public function acknowledge(string $id): void 81 | { 82 | // When listening on Stream without a group we are not acknowledging any messages 83 | } 84 | 85 | public function readRange(Range $range, ?int $limit = null): array 86 | { 87 | $method = 'xRANGE'; 88 | $start = $range->getStart(); 89 | $stop = $range->getStop(); 90 | if ($range->getDirection() === Range::BACKWARD) { 91 | $method = 'xREVRANGE'; 92 | $start = $range->getStop(); 93 | $stop = $range->getStart(); 94 | } 95 | 96 | $result = []; 97 | if ($limit) { 98 | $result = $this->redis()->$method($this->name, $start, $stop, $limit); 99 | } 100 | 101 | if (!$limit) { 102 | $result = $this->redis()->$method($this->name, $start, $stop); 103 | } 104 | 105 | if (!is_array($result)) { 106 | return []; 107 | } 108 | 109 | return $result; 110 | } 111 | 112 | public function createGroup( 113 | string $name, 114 | string $from = self::FROM_START, 115 | bool $createStreamIfNotExists = true 116 | ): bool { 117 | if ($createStreamIfNotExists) { 118 | return $this->redis()->xGroup(self::CREATE, $this->name, $name, $from, 'MKSTREAM'); 119 | } 120 | 121 | return $this->redis()->xGroup(self::CREATE, $this->name, $name, $from); 122 | } 123 | 124 | /** 125 | * Return all pending messages from given group. 126 | * Optionally it can return pending message for single consumer. 127 | * 128 | * 129 | */ 130 | public function pending(string $group, ?string $consumer = null): array 131 | { 132 | $pending = $this->redis()->xPending($this->name, $group); 133 | if (!$pending) { 134 | return []; 135 | } 136 | 137 | $pendingCount = array_shift($pending); 138 | 139 | if ($consumer) { 140 | return $this->redis()->xPending($this->name, $group, Range::FIRST, Range::LAST, $pendingCount, $consumer); 141 | } 142 | 143 | return $this->redis()->xPending($this->name, $group, Range::FIRST, Range::LAST, $pendingCount); 144 | } 145 | 146 | public function len(): int 147 | { 148 | return $this->redis()->xLen($this->name); 149 | } 150 | 151 | /** 152 | * Returns XINFO for stream with FULL flag. 153 | * Available since Redis v6.0.0. 154 | * 155 | * @throws StreamNotFoundException 156 | */ 157 | public function fullInfo(): array 158 | { 159 | $info = $this->redis()->info(); 160 | if (!version_compare($info['redis_version'], '6.0.0', '>=')) { 161 | throw new BadMethodCallException('fullInfo only available for Redis 6.0 or above.'); 162 | } 163 | 164 | $result = $this->redis()->xInfo(self::STREAM, $this->name, 'FULL'); 165 | if (!$result) { 166 | throw new StreamNotFoundException("No results for stream $this->name"); 167 | } 168 | 169 | return [ 170 | 'length' => null, 171 | 'radix-tree-keys' => null, 172 | 'radix-tree-nodes' => null, 173 | 'last-generated-id' => null, 174 | 'max-deleted-entry-id' => null, 175 | 'entries-added' => null, 176 | 'recorded-first-entry-id' => null, 177 | 'entries' => null, 178 | 'groups' => null, 179 | ...$result, 180 | ]; 181 | } 182 | 183 | /** 184 | * @throws StreamNotFoundException 185 | */ 186 | public function info(): array 187 | { 188 | $result = $this->redis()->xInfo(self::STREAM, $this->name); 189 | if (!$result) { 190 | throw new StreamNotFoundException("No results for stream $this->name"); 191 | } 192 | 193 | return [ 194 | 'length' => null, 195 | 'radix-tree-keys' => null, 196 | 'radix-tree-nodes' => null, 197 | 'last-generated-id' => null, 198 | 'max-deleted-entry-id' => null, 199 | 'entries-added' => null, 200 | 'recorded-first-entry-id' => null, 201 | 'groups' => null, 202 | 'first-entry' => null, 203 | 'last-entry' => null, 204 | ...$result, 205 | ]; 206 | } 207 | 208 | /** 209 | * 210 | * @throws StreamNotFoundException 211 | * 212 | */ 213 | public function consumers(string $group): array 214 | { 215 | $result = $this->redis()->xInfo(self::CONSUMERS, $this->name, $group); 216 | if (!$result) { 217 | throw new StreamNotFoundException("No results for stream $this->name"); 218 | } 219 | 220 | return $result; 221 | } 222 | 223 | public function groupExists(string $name): bool 224 | { 225 | try { 226 | $groups = $this->groups(); 227 | } catch (StreamNotFoundException) { 228 | return false; 229 | } 230 | 231 | foreach ($groups as $group) { 232 | if ($group['name'] === $name) { 233 | return true; 234 | } 235 | } 236 | 237 | return false; 238 | } 239 | 240 | /** 241 | * @throws StreamNotFoundException 242 | * 243 | */ 244 | public function groups(): array 245 | { 246 | $result = $this->redis()->xInfo(self::GROUPS, $this->name); 247 | if (!$result) { 248 | throw new StreamNotFoundException("No results for stream $this->name"); 249 | } 250 | 251 | return $result; 252 | } 253 | } 254 | -------------------------------------------------------------------------------- /src/Stream/Consumer.php: -------------------------------------------------------------------------------- 1 | '; 19 | 20 | public function __construct( 21 | private readonly string $consumer, 22 | private readonly Stream $stream, 23 | private readonly string $group 24 | ) { 25 | } 26 | 27 | public function getNewEntriesKey(): string 28 | { 29 | return self::NEW_ENTRIES; 30 | } 31 | 32 | public function await(string $lastSeenId = self::NEW_ENTRIES, float $timeout = 0.0): array 33 | { 34 | $result = $this->redis()->xReadGroup( 35 | $this->group, 36 | $this->consumer, 37 | [$this->stream->getName() => $lastSeenId], 38 | 0, 39 | $timeout 40 | ); 41 | 42 | if (!is_array($result)) { 43 | return []; 44 | } 45 | 46 | return $result; 47 | } 48 | 49 | public function getName(): string 50 | { 51 | return $this->stream->getName(); 52 | } 53 | 54 | /** 55 | * @throws AcknowledgingFailedException 56 | */ 57 | public function acknowledge(string $id): void 58 | { 59 | $result = $this->redis()->xAck($this->stream->getName(), $this->group, [$id]); 60 | if ($result === 0) { 61 | throw new AcknowledgingFailedException("Could not acknowledge message with ID $id"); 62 | } 63 | } 64 | 65 | /** 66 | * Return pending message only for this particular consumer. 67 | */ 68 | public function pending(): array 69 | { 70 | return $this->stream->pending($this->group, $this->consumer); 71 | } 72 | 73 | /** 74 | * Claim all given messages that have minimum idle time of $idleTime milliseconds. 75 | */ 76 | public function claim(array $ids, int $idleTime, bool $justId = true): array 77 | { 78 | if ($justId) { 79 | return $this->redis()->xClaim( 80 | $this->stream->getName(), 81 | $this->group, 82 | $this->consumer, 83 | $idleTime, 84 | $ids, 85 | ['JUSTID'] 86 | ); 87 | } 88 | 89 | return $this->redis()->xClaim($this->stream->getName(), $this->group, $this->consumer, $idleTime, $ids); 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /src/Stream/MultiStream.php: -------------------------------------------------------------------------------- 1 | streams = new Collection(); 31 | 32 | foreach ($streams as $name) { 33 | if (!$name || !is_string($name)) { 34 | continue; 35 | } 36 | 37 | $stream = new Stream($name); 38 | $this->streams->put($name, $stream); 39 | 40 | if ($consumer && $group) { 41 | $stream->createGroup($group); 42 | } 43 | } 44 | 45 | $this->consumer = $consumer; 46 | $this->group = $group; 47 | } 48 | 49 | /** 50 | * @return Collection&Stream[] 51 | */ 52 | public function streams(): Collection 53 | { 54 | return $this->streams; 55 | } 56 | 57 | /** 58 | * Adds new message to Streams (if such is in MultiStream collection). 59 | * 60 | * @param array $streams [stream => id] format. if ID is not a string, assuming "*" 61 | */ 62 | public function add(array $streams, StreamableMessage $message): array 63 | { 64 | $added = []; 65 | foreach ($streams as $stream => $id) { 66 | if (!$this->streams->has($stream)) { 67 | continue; 68 | } 69 | 70 | if (!is_string($id)) { 71 | $id = '*'; 72 | } 73 | 74 | $added[$stream] = $this->streams->get($stream)->add($message, $id); 75 | } 76 | 77 | return $added; 78 | } 79 | 80 | /** 81 | * Deletes message from a Stream (if such is in MultiStream collection). 82 | * 83 | * @param array $streams [stream => [ids]] format 84 | */ 85 | public function delete(array $streams): int 86 | { 87 | $deleted = 0; 88 | foreach ($streams as $stream => $ids) { 89 | if (!$this->streams->has($stream)) { 90 | continue; 91 | } 92 | 93 | $deleted += $this->redis()->xDel($stream, Arr::wrap($ids)); 94 | } 95 | 96 | return $deleted; 97 | } 98 | 99 | public function await(string $lastSeenId = '', float $timeout = 0.0): ?array 100 | { 101 | if ($lastSeenId === '' || $lastSeenId === '0') { 102 | $lastSeenId = $this->getNewEntriesKey(); 103 | } 104 | 105 | if ($this->streams->count() === 1) { 106 | return $this->parseResult($this->awaitSingle($this->streams->first(), $lastSeenId, $timeout)); 107 | } 108 | 109 | $result = $this->awaitMultiple($lastSeenId, $timeout); 110 | if ($result === []) { 111 | return []; 112 | } 113 | 114 | return $this->sortByTimestamps($this->parseResult($result)); 115 | } 116 | 117 | public function getNewEntriesKey(): string 118 | { 119 | if ($this->consumer && $this->group) { 120 | return Consumer::NEW_ENTRIES; 121 | } 122 | 123 | return Stream::NEW_ENTRIES; 124 | } 125 | 126 | /** 127 | * Acknowledges multiple messages if MultiStream has a group. 128 | * 129 | * @param array $streams [stream => [ids]] format 130 | * @throws Exception 131 | */ 132 | public function acknowledge(array $streams): void 133 | { 134 | if ($this->group === '' || $this->group === '0') { 135 | return; 136 | } 137 | 138 | $notAcknowledged = []; 139 | foreach ($streams as $stream => $ids) { 140 | if (!$this->streams->has($stream)) { 141 | continue; 142 | } 143 | 144 | $result = $this->redis()->xAck($stream, $this->group, Arr::wrap($ids)); 145 | if ($result === 0) { 146 | $notAcknowledged[] = $stream; 147 | } 148 | } 149 | 150 | if ($notAcknowledged !== []) { 151 | throw new AcknowledgingFailedException( 152 | "Not all messages were acknowledged. Streams affected: " . implode(', ', $notAcknowledged) 153 | ); 154 | } 155 | } 156 | 157 | private function parseResult($result): array 158 | { 159 | $list = []; 160 | foreach ($result as $stream => $messages) { 161 | foreach ($messages as $id => $message) { 162 | $list[] = [ 163 | 'stream' => $stream, 164 | 'id' => $id, 165 | 'message' => $message, 166 | ]; 167 | } 168 | } 169 | return $list; 170 | } 171 | 172 | private function awaitSingle(Stream $stream, string $lastSeenId, float $timeout): array 173 | { 174 | if (!$this->consumer && !$this->group) { 175 | return $stream->await($lastSeenId, $timeout); 176 | } 177 | 178 | $consumer = new Consumer($this->consumer, $stream, $this->group); 179 | if ($lastSeenId === '' || $lastSeenId === '0') { 180 | $lastSeenId = $consumer->getNewEntriesKey(); 181 | } 182 | 183 | return $consumer->await($lastSeenId, $timeout); 184 | } 185 | 186 | private function awaitMultiple(string $lastSeenId, float $timeout): array 187 | { 188 | $streams = $this->streams->map(static fn (Stream $s): string => $lastSeenId)->toArray(); 189 | 190 | if (!$this->consumer || !$this->group) { 191 | $result = $this->redis()->xRead($streams, 0, $timeout); 192 | if (!is_array($result)) { 193 | return []; 194 | } 195 | 196 | return $result; 197 | } 198 | 199 | $result = $this->redis()->xReadGroup($this->group, $this->consumer, $streams, 0, $timeout); 200 | if (!is_array($result)) { 201 | return []; 202 | } 203 | 204 | return $result; 205 | } 206 | 207 | private function sortByTimestamps(array $list): array 208 | { 209 | usort($list, static function ($a, $b): int { 210 | $aID = $a['id'] ?? null; 211 | $bID = $b['id'] ?? null; 212 | if ($aID === $bID) { 213 | return 0; 214 | } 215 | 216 | [$aTimestamp, $aSequence] = explode("-", $aID); 217 | [$bTimestamp, $bSequence] = explode("-", $bID); 218 | 219 | if ($aTimestamp === $bTimestamp) { 220 | if ($aSequence > $bSequence) { 221 | return 1; 222 | } 223 | 224 | if ($aSequence < $bSequence) { 225 | return -1; 226 | } 227 | 228 | return 0; 229 | } 230 | 231 | if ($aTimestamp > $bTimestamp) { 232 | return 1; 233 | } 234 | 235 | if ($aTimestamp < $bTimestamp) { 236 | return -1; 237 | } 238 | 239 | return 0; 240 | }); 241 | 242 | return $list; 243 | } 244 | } 245 | -------------------------------------------------------------------------------- /src/Stream/Range.php: -------------------------------------------------------------------------------- 1 | start; 27 | } 28 | 29 | public function getStop(): string 30 | { 31 | return $this->stop; 32 | } 33 | 34 | public function getDirection(): int 35 | { 36 | return $this->direction; 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /src/StreamNotFoundException.php: -------------------------------------------------------------------------------- 1 | app->bind(History::class, EventHistory::class); 35 | $this->app->bind(MessagesFailer::class, FailedMessagesHandler::class); 36 | $this->app->bind(Repository::class, MessagesRepository::class); 37 | $this->app->bind(Archiver::class, StreamArchiver::class); 38 | 39 | $this->app->when(StorageManager::class) 40 | ->needs('$container') 41 | ->give($this->app); 42 | $this->app->singleton(StorageManager::class); 43 | 44 | $this->app->bind('Streamer', fn () => $this->app->make(Streamer::class)); 45 | 46 | $this->offerPublishing(); 47 | $this->configure(); 48 | $this->registerCommands(); 49 | 50 | ListenersStack::boot(config('streamer.listen_and_fire', [])); 51 | } 52 | 53 | /** 54 | * Setup the resource publishing groups. 55 | */ 56 | private function offerPublishing(): void 57 | { 58 | if ($this->app->runningInConsole()) { 59 | $this->publishes([ 60 | __DIR__ . '/../config/streamer.php' => app()->basePath('config/streamer.php'), 61 | ], 'config'); 62 | } 63 | } 64 | 65 | /** 66 | * Setup the configuration. 67 | */ 68 | private function configure(): void 69 | { 70 | $this->mergeConfigFrom( 71 | __DIR__ . '/../config/streamer.php', 72 | 'streamer' 73 | ); 74 | } 75 | 76 | /** 77 | * Register the Artisan commands. 78 | */ 79 | private function registerCommands(): void 80 | { 81 | if ($this->app->runningInConsole()) { 82 | $this->commands([ 83 | ListenCommand::class, 84 | ListCommand::class, 85 | ListFailedCommand::class, 86 | RetryFailedCommand::class, 87 | FlushFailedCommand::class, 88 | ArchiveRestoreCommand::class, 89 | ArchiveCommand::class, 90 | PurgeCommand::class, 91 | ]); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /src/Streams.php: -------------------------------------------------------------------------------- 1 | streams as $stream) { 22 | $ids[] = $this->redis()->XADD($stream, $id, $message->getContent()); 23 | } 24 | 25 | return $ids; 26 | } 27 | 28 | public function read(array $from = [], ?int $limit = null): array 29 | { 30 | $read = []; 31 | foreach ($this->streams as $key => $stream) { 32 | $read[$stream] = $from[$key] ?? Stream::FROM_START; 33 | } 34 | 35 | $result = []; 36 | if ($limit) { 37 | $result = $this->redis()->xRead($read, $limit); 38 | } 39 | 40 | if (!$limit) { 41 | $result = $this->redis()->xRead($read); 42 | } 43 | 44 | if (!is_array($result)) { 45 | return []; 46 | } 47 | 48 | return $result; 49 | } 50 | } 51 | --------------------------------------------------------------------------------