├── .editorconfig ├── .github └── workflows │ └── tests.yml ├── .gitignore ├── .styleci.yml ├── CHANGELOG.md ├── LICENSE ├── README.md ├── composer.json ├── config └── stream.php ├── docker-compose.yml ├── phpunit.xml ├── src ├── ClientProxy.php ├── Common │ ├── Deserializer │ │ └── IdentityDeserializer.php │ └── Serializer │ │ └── IdentitySerializer.php ├── Console │ ├── ConsumeCommand.php │ ├── DeclareGroupCommand.php │ └── DestroyGroupCommand.php ├── Data │ ├── Commands.php │ ├── Options.php │ └── XGROUPOptions.php ├── LaravelRedisStreamServiceProvider.php ├── RedisStream.php └── TransporterServer.php └── tests ├── Functional └── 01ConsoleCommandsTest.php ├── Helper.php └── TestCase.php /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | insert_final_newline = true 7 | indent_style = space 8 | indent_size = 4 9 | trim_trailing_whitespace = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | 14 | [*.{yml,yaml}] 15 | indent_size = 2 -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | push: 5 | pull_request: 6 | schedule: 7 | - cron: "0 0 * * *" 8 | 9 | jobs: 10 | tests: 11 | runs-on: ubuntu-latest 12 | 13 | strategy: 14 | fail-fast: true 15 | matrix: 16 | php: ["7.4", "8.0"] 17 | stability: [prefer-lowest, prefer-stable] 18 | exclude: 19 | - php: "8.0" 20 | stability: "prefer-lowest" 21 | 22 | name: PHP ${{ matrix.php }} - ${{ matrix.stability }} 23 | 24 | steps: 25 | - name: Checkout code 26 | uses: actions/checkout@v2 27 | 28 | - name: Setup PHP 29 | uses: shivammathur/setup-php@v2 30 | with: 31 | php-version: ${{ matrix.php }} 32 | extensions: dom, curl, libxml, mbstring, zip 33 | coverage: none 34 | 35 | - name: Set up Docker 36 | run: | 37 | sudo rm /usr/local/bin/docker-compose 38 | curl -L https://github.com/docker/compose/releases/download/1.24.1/docker-compose-`uname -s`-`uname -m` > docker-compose 39 | chmod +x docker-compose 40 | sudo mv docker-compose /usr/local/bin 41 | 42 | - name: Start Docker container 43 | run: docker-compose up -d redis 44 | 45 | - name: Install dependencies 46 | run: composer update --${{ matrix.stability }} --prefer-dist --no-interaction --no-progress 47 | 48 | - name: Execute tests 49 | run: sleep 10 && vendor/bin/phpunit --verbose 50 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/ 2 | composer.lock 3 | .phpunit.result.cache 4 | -------------------------------------------------------------------------------- /.styleci.yml: -------------------------------------------------------------------------------- 1 | preset: laravel 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | ## ![1.1.4 (2022-01-13)](https://github.com/afikrim/laravel-redis-stream/compare/1.1.3...1.1.4) 6 | 7 | - 8 | 9 | ## ![1.1.3 (2022-01-12)](https://github.com/afikrim/laravel-redis-stream/compare/1.1.2...1.1.3) 10 | 11 | - Decode json to array before deserializing 12 | 13 | ## ![1.1.2 (2022-01-12)](https://github.com/afikrim/laravel-redis-stream/compare/1.1.1...1.1.2) 14 | 15 | - Remove `is_dispossed` from deserializer 16 | - Modify handle default value for options 17 | 18 | ## ![1.1.1 (2022-01-12)](https://github.com/afikrim/laravel-redis-stream/compare/1.1.0...1.1.1) 19 | 20 | - Add predis to require 21 | - Fix undefined function 'now' 22 | 23 | ## ![1.1.0 (2022-01-11)](https://github.com/afikrim/laravel-redis-stream/compare/1.0.4...1.1.0) 24 | 25 | - Remove support for array or object 26 | - Add transporter strategy 27 | - Add client proxy 28 | - Add serializer and deserializer 29 | - Add stream config 30 | 31 | ## ![1.0.4 (2021-12-28)](https://github.com/afikrim/laravel-redis-stream/compare/1.0.3...1.0.4) 32 | 33 | - Add support for array 34 | - Add support for object 35 | 36 | ## ![1.0.3 (2021-12-27)](https://github.com/afikrim/laravel-redis-stream/compare/1.0.2...1.0.3) 37 | 38 | - Fix group and consumer option in command 39 | 40 | ## ![1.0.2 (2021-12-18)](https://github.com/afikrim/laravel-redis-stream/compare/1.0.1...1.0.2) 41 | 42 | - Fix typos 43 | 44 | ## ![1.0.1 (2021-12-18)](https://github.com/afikrim/laravel-redis-stream/compare/1.0.0...1.0.1) 45 | 46 | - Modify functions in `RedisStream` to static 47 | - Update documentation in `README` 48 | - Add test cases for console commands 49 | - Fix Redis Execute result handler in `RedisStream` 50 | 51 | ## ![1.0.0 (2021-12-18)](https://github.com/afikrim/laravel-redis-stream/tree/1.0.0) 52 | 53 | - Add `stream:declare-group` artisan command 54 | - Add `stream:destroy-group` artisan command 55 | - Add `stream:consume` artisan command 56 | - Add `LaravelRedisStreamServiceProvider` service provider 57 | - Add `RedisStream` helper class 58 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2021 Aziz Fikri 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel Redis Stream 2 | 3 | Laravel Redis Stream is a package to help you handle event streaming between different applications powered by Redis. 4 | 5 | Main concept of this package is to provide an easy way of storing new events from your application and consume it in your other applications. 6 | 7 | ## Installation 8 | 9 | You can install this package via composer using this command: 10 | 11 | ```sh 12 | composer require afikrim/laravel-redis-stream 13 | ``` 14 | 15 | ### Installation on Lumen 16 | 17 | After you install the package via composer, register a new service provider in `bootstrap/app.php` 18 | 19 | ```php 20 | $app->register(\Afikrim\LaravelRedisStream\LaravelRedisStreamServiceProvider::class); 21 | ``` 22 | 23 | > Note: don't forget to uncomment facades and register redis (please use predis because it can't work in phpredis) 24 | 25 | ## Add configuration 26 | 27 | Add new redis connection for the `stream` : 28 | 29 | ```php 30 | [ 33 | ... 34 | 35 | 'stream' => [ 36 | 'url' => env('REDIS_URL'), 37 | 'host' => env('REDIS_HOST', '127.0.0.1'), 38 | 'password' => env('REDIS_PASSWORD', null), 39 | 'port' => env('REDIS_PORT', '6379'), 40 | 'database' => env('REDIS_STREAM_DB', '3'), 41 | 'prefix' => env('REDIS_STREAM_PREFIX', Str::slug(strtolower(config('app.env')) . '.' . strtolower(config('app.name')), '.')) . ':', 42 | ] 43 | ], 44 | ... 45 | ``` 46 | 47 | ## Available commands 48 | 49 | There are three commands includes in this packages, 50 | 51 | 1. `stream:declare-group` command to declare a group for a stream 52 | 2. `stream:destroy-group` command to destroy a group for a stream 53 | 3. `stream:consume` command to consume incoming event 54 | 55 | ## Add custom consume handler 56 | 57 | You can add your custom consumer by extending our `ConsumeCommand` in `app/Console/Commands` 58 | 59 | ```php 60 | hasOption('group')) { 75 | $options['group'] = $this->option('group'); 76 | } 77 | if ($this->hasOption('consumer')) { 78 | $options['consumer'] = $this->option('consumer'); 79 | } 80 | if ($this->hasOption('count')) { 81 | $options['count'] = $this->option('count'); 82 | } 83 | if ($this->hasOption('block')) { 84 | $options['block'] = $this->option('block'); 85 | } 86 | if ($this->laravel->config->get('redis.stream.prefix')) { 87 | $options['prefix'] = $this->laravel->config->get('redis.stream.prefix'); 88 | } 89 | 90 | $server = new TransporterServer($options); 91 | $server->addHandler('mystream', function ($result) {return $result;}); 92 | $server->addHandler('mystream2', function ($result) {return $result;}); 93 | // And another awesome handlers... 94 | $server->listen(); 95 | } 96 | } 97 | ``` 98 | 99 | Then add your custom `ConsumeCommand` to `App\Console\Kernel` class 100 | 101 | ```php 102 | protected $commands = [ 103 | // you other command, 104 | \App\Console\Commands\ConsumeCommad::class 105 | ] 106 | ``` 107 | 108 | ## Send data to the stream 109 | 110 | To send your data to the stream, you can use `ClientProxy` class. 111 | 112 | ### Send data and listen for the response 113 | 114 | ```php 115 | ... 116 | use Afikrim\LaravelRedisStream\ClientProxy; 117 | 118 | ... 119 | $results = ClientProxy::init($options) 120 | ->publish('mystream2', [ 121 | 'name' => 'Aziz', 122 | 'email' => "afikrim10@gmail.com", 123 | ]) 124 | ->subscribe('mystream2', 60); 125 | ... 126 | ``` 127 | 128 | ### Send data without waiting any response 129 | 130 | ```php 131 | ... 132 | use Afikrim\LaravelRedisStream\ClientProxy; 133 | 134 | ... 135 | ClientProxy::init($options) 136 | ->dispatch('mystream2', [ 137 | 'name' => 'Aziz', 138 | 'email' => "afikrim10@gmail.com", 139 | ]); 140 | ... 141 | ``` 142 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "afikrim/laravel-redis-stream", 3 | "description": "A composer library to use redis stream as a message broker in laravel", 4 | "type": "library", 5 | "authors": [ 6 | { 7 | "name": "Aziz Fikri M", 8 | "email": "afikrim10@gmail.com" 9 | } 10 | ], 11 | "minimum-stability": "dev", 12 | "require": { 13 | "php": "^7.4|^8.0", 14 | "predis/predis": "v1.1.9", 15 | "illuminate/redis": "*" 16 | }, 17 | "require-dev": { 18 | "phpunit/phpunit": "^9.3", 19 | "orchestra/testbench": "^6.0" 20 | }, 21 | "autoload": { 22 | "psr-4": { 23 | "Afikrim\\LaravelRedisStream\\": "src/" 24 | } 25 | }, 26 | "autoload-dev": { 27 | "psr-4": { 28 | "Afikrim\\LaravelRedisStream\\Tests\\": "tests/" 29 | } 30 | }, 31 | "suggest": { 32 | "ext-redis": "Required to use redis above 5.0" 33 | }, 34 | "extra": { 35 | "branch-alias": { 36 | "dev-master": "1.0.x-dev" 37 | }, 38 | "laravel": { 39 | "providers": [ 40 | "Afikrim\\LaravelRedisStream\\LaravelRedisStreamServiceProvider" 41 | ] 42 | } 43 | }, 44 | "scripts": { 45 | "test:unit": "@php vendor/bin/phpunit" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /config/stream.php: -------------------------------------------------------------------------------- 1 | env('REDIS_URL'), 5 | 'host' => env('REDIS_HOST', '127.0.0.1'), 6 | 'password' => env('REDIS_PASSWORD', null), 7 | 'port' => env('REDIS_PORT', '6379'), 8 | 'database' => env('REDIS_STREAM_DB', '3'), 9 | 'prefix' => env('REDIS_STREAM_PREFIX', \Illuminate\Support\Str::slug(strtolower(config('app.env')) . '.' . strtolower(config('app.name')), '.')) . ':', 10 | ]; 11 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.7' 2 | 3 | services: 4 | redis: 5 | image: redis 6 | ports: 7 | - "6379:6379" 8 | -------------------------------------------------------------------------------- /phpunit.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | ./tests 6 | 7 | 8 | 9 | 10 | ./app 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /src/ClientProxy.php: -------------------------------------------------------------------------------- 1 | options = $options; 21 | } 22 | 23 | public static function init($options = []) 24 | { 25 | return (new static($options)); 26 | } 27 | 28 | /** 29 | * Function to send a request that need a reply 30 | * 31 | * @param string $pattern 32 | * @param array $data 33 | * @return this 34 | */ 35 | public function publish(string $pattern, array $data) 36 | { 37 | Log::info('Generating data to publish at pattern: ' . $pattern); 38 | $partial_packet = [ 39 | 'data' => json_encode($data), 40 | 'pattern' => $pattern, 41 | ]; 42 | $request = (array) (new IdentitySerializer($partial_packet, false, true)); 43 | 44 | Log::info('Sending data to pattern: ' . $pattern); 45 | $this->id = RedisStream::xadd($this->getPattern($pattern), '*', $request); 46 | 47 | return $this; 48 | } 49 | 50 | /** 51 | * Function to send a forgetable request 52 | * 53 | * @param string $pattern 54 | * @param array $data 55 | * @return void 56 | */ 57 | public function dispatch(string $pattern, array $data) 58 | { 59 | Log::info('Generating data to publish at pattern: ' . $pattern); 60 | $partial_packet = [ 61 | 'data' => json_encode($data), 62 | 'pattern' => $pattern, 63 | ]; 64 | $request = (array) (new IdentitySerializer($partial_packet)); 65 | 66 | Log::info('Sending data to pattern: ' . $pattern); 67 | RedisStream::xadd($this->getPattern($pattern), '*', $request); 68 | } 69 | 70 | public function subscribe(string $pattern, int $max_tries = 5, int $attempts = 0) 71 | { 72 | try { 73 | // create consumer group 74 | RedisStream::xgroup( 75 | XGROUPOptions::OPTION_CREATE, 76 | $this->getReplyPattern($pattern), 77 | $this->getOption('group'), 78 | true, 79 | [ 80 | '$', 81 | ] 82 | ); 83 | } catch (\Exception$e) { 84 | // do nothing 85 | } 86 | 87 | $results = RedisStream::xreadgroup( 88 | $this->getOption('group'), 89 | $this->getOption('consumer'), 90 | [$this->getReplyPattern($pattern)], 91 | ['>'], 92 | [ 93 | Options::OPTION_COUNT, 94 | $this->getOption('count'), 95 | Options::OPTION_BLOCK, 96 | $this->getOption('block'), 97 | ], 98 | ); 99 | 100 | if ($attempts > $max_tries) { 101 | return []; 102 | } 103 | 104 | return $this->handleReply($pattern, $results, $max_tries, $attempts); 105 | } 106 | 107 | private function handleReply(string $pattern, array $results, int $max_tries, int $attempts) 108 | { 109 | if (count($results) === 0) { 110 | RedisStream::xdel($pattern, [$this->id]); 111 | return []; 112 | } 113 | 114 | [ 115 | 'data' => $raw_messages, 116 | ] = $results[0]; 117 | 118 | $raw_message_index = array_search($this->id, array_column($raw_messages, 'id')); 119 | if (!is_integer($raw_message_index)) { 120 | if ($attempts === $max_tries) { 121 | RedisStream::xdel($pattern, [$this->id]); 122 | } 123 | 124 | return $this->subscribe($pattern, $max_tries, $attempts + 1); 125 | } 126 | 127 | [ 128 | 'id' => $_id, 129 | 'data' => $packet, 130 | ] = $raw_messages[$raw_message_index]; 131 | 132 | $packet = (array) (new IdentityDeserializer($packet, true)); 133 | $error = $packet['error']; 134 | $response = $packet['response'] ? json_decode($packet['response'], true) : null; 135 | 136 | RedisStream::xack($this->getReplyPattern($packet['pattern']), $this->getOption('group'), [$_id]); 137 | 138 | return [ 139 | 'error' => $error, 140 | 'response' => $response, 141 | ]; 142 | } 143 | 144 | private function getPattern(string $pattern) 145 | { 146 | return $pattern; 147 | } 148 | 149 | private function getReplyPattern(string $pattern) 150 | { 151 | return $pattern . '.reply'; 152 | } 153 | 154 | private function getOption(string $key) 155 | { 156 | $value = array_key_exists($key, $this->options) ? $this->options[$key] : null; 157 | if (!$value) { 158 | if ($key === 'count') { 159 | return 1; 160 | } else if ($key === 'block') { 161 | return 2000; 162 | } else if ($key === 'group') { 163 | return Str::slug( 164 | config('app.env') 165 | . '.' 166 | . config('app.name') 167 | . '.group' 168 | , '.'); 169 | } else if ($key === 'consumer') { 170 | return Str::slug( 171 | config('app.env') 172 | . '.' 173 | . config('app.name') 174 | . '.consumer' 175 | , '.'); 176 | } 177 | 178 | return null; 179 | } 180 | 181 | if ($key === 'count' || $key === 'block') { 182 | return (int) $value; 183 | } 184 | 185 | return $value; 186 | } 187 | } 188 | -------------------------------------------------------------------------------- /src/Common/Deserializer/IdentityDeserializer.php: -------------------------------------------------------------------------------- 1 | >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); 12 | Log::info(json_encode($packet)); 13 | Log::info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); 14 | 15 | $this->id = $packet['id']; 16 | $this->pattern = $packet['pattern']; 17 | if ($reply) { 18 | $this->response = $packet['response']; 19 | $this->error = $packet['error'] === '' ? null : $packet['error']; 20 | } else { 21 | $this->data = $packet['data']; 22 | $this->need_reply = $packet['need_reply'] ?? false; 23 | } 24 | 25 | $this->time = time(); 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /src/Common/Serializer/IdentitySerializer.php: -------------------------------------------------------------------------------- 1 | >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); 13 | Log::info(json_encode($packet)); 14 | Log::info(">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>"); 15 | 16 | $this->id = $packet['id'] ?? Uuid::uuid4(); 17 | $this->pattern = $packet['pattern']; 18 | if ($reply) { 19 | $this->response = $packet['response']; 20 | $this->error = $packet['error'] ?? null; 21 | } else { 22 | $this->data = $packet['data']; 23 | $this->need_reply = $need_reply; 24 | } 25 | 26 | $this->time = time(); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Console/ConsumeCommand.php: -------------------------------------------------------------------------------- 1 | listen(); 24 | 25 | if (config('app.env') === 'testing') { 26 | break; 27 | } 28 | 29 | $this->rest(); 30 | } 31 | } 32 | 33 | protected function listen() 34 | { 35 | $options = []; 36 | if ($this->hasOption('group')) { 37 | $options['group'] = $this->option('group'); 38 | } 39 | if ($this->hasOption('consumer')) { 40 | $options['consumer'] = $this->option('consumer'); 41 | } 42 | if ($this->hasOption('count')) { 43 | $options['count'] = $this->option('count'); 44 | } 45 | if ($this->hasOption('block')) { 46 | $options['block'] = $this->option('block'); 47 | } 48 | if ($this->laravel->config->get('redis.stream.prefix')) { 49 | $options['prefix'] = $this->laravel->config->get('redis.stream.prefix'); 50 | } 51 | 52 | $server = new TransporterServer($options); 53 | $server->addHandler('mystream', function ($result) {return $result;}); 54 | $server->addHandler('mystream2', function ($result) {return $result;}); 55 | $server->listen(); 56 | } 57 | 58 | private function rest() 59 | { 60 | sleep($this->option('rest')); 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /src/Console/DeclareGroupCommand.php: -------------------------------------------------------------------------------- 1 | hasArgument('key')) { 23 | return 1; 24 | } 25 | 26 | RedisStream::xgroup( 27 | XGROUPOptions::OPTION_CREATE, 28 | $this->argument('key'), 29 | $this->getGroup(), 30 | $this->option('mkstream'), 31 | [ 32 | '$', 33 | ] 34 | ); 35 | 36 | return 0; 37 | } 38 | 39 | protected function getGroup() 40 | { 41 | return $this->hasArgument('group') 42 | ? $this->argument('group') 43 | : Str::slug( 44 | $this->laravel->config->get('app.env') 45 | . '.' 46 | . $this->laravel->config->get('app.name') 47 | . '.group' 48 | , '.'); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /src/Console/DestroyGroupCommand.php: -------------------------------------------------------------------------------- 1 | hasArgument('key')) { 21 | return 1; 22 | } 23 | 24 | RedisStream::xgroup( 25 | XGROUPOptions::OPTION_DESTROY, 26 | $this->argument('key'), 27 | $this->getGroup(), 28 | ); 29 | 30 | return 0; 31 | } 32 | 33 | protected function getGroup() 34 | { 35 | return $this->hasArgument('group') 36 | ? $this->argument('group') 37 | : Str::slug( 38 | $this->laravel->config->get('app.env') 39 | . '.' 40 | . $this->laravel->config->get('app.name') 41 | . '.group' 42 | , '.'); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /src/Data/Commands.php: -------------------------------------------------------------------------------- 1 | mergeConfigFrom( 17 | __DIR__ . '/../config/stream.php', 18 | 'database.redis.stream' 19 | ); 20 | 21 | if ($this->app->runningInConsole()) { 22 | $this->commands([ 23 | Console\ConsumeCommand::class, 24 | ]); 25 | } 26 | 27 | $this->commands([ 28 | Console\DeclareGroupCommand::class, 29 | Console\DestroyGroupCommand::class, 30 | ]); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/RedisStream.php: -------------------------------------------------------------------------------- 1 | newXadd($key, $id, $data); 34 | } 35 | 36 | /** 37 | * Function to create static function 'xadd' 38 | * 39 | * @param string $key The stream key 40 | * @param string $id New object ID 41 | * @param array $data The data that will be input 42 | * @return string 43 | */ 44 | public function newXadd(string $key, string $id = '*', array $data): string 45 | { 46 | $fieldsAndValues = $this->parseData($data); 47 | 48 | $result = Redis::connection('stream') 49 | ->executeRaw([ 50 | Commands::XADD, 51 | config('database.redis.stream.prefix', '') . $key, 52 | $id, 53 | ...$fieldsAndValues, 54 | ]); 55 | 56 | return $result; 57 | } 58 | 59 | /** 60 | * Function to read from the stream 61 | * 62 | * @param array $streams Streams that will be read 63 | * @param array $ids IDs from the stream that will retrieve 64 | * @param array $options 65 | * @return \Illuminate\Support\Collection 66 | */ 67 | public static function xread(array $streams, array $ids, array $options): Collection 68 | { 69 | return (new static )->xread($streams, $ids, $options); 70 | } 71 | 72 | /** 73 | * Function to create static function 'xread' 74 | * 75 | * @param array $streams Streams that will be read 76 | * @param array $ids IDs from the stream that will retrieve 77 | * @param array $options 78 | * @return array 79 | */ 80 | public function newXread(array $streams, array $ids, array $options): array 81 | { 82 | $redisArguments = [ 83 | Commands::XREAD, 84 | ...$options, 85 | Options::OPTION_STREAMS, 86 | ]; 87 | foreach ($streams as $stream) { 88 | $redisArguments[] = config('database.redis.stream.prefix', '') . $stream; 89 | } 90 | foreach ($ids as $id) { 91 | $redisArguments[] = !$id ? '0' : $id; 92 | } 93 | 94 | $results = Redis::connection('stream') 95 | ->executeRaw($redisArguments); 96 | 97 | if (is_string($results)) { 98 | throw new \Exception($results); 99 | } 100 | 101 | return $this->parseResults($results ?? []); 102 | } 103 | 104 | /** 105 | * Function to delete object(s) from stream 106 | * 107 | * @param string $key Stream that will be read 108 | * @param array $ids A List of id that will be delete 109 | * @return void 110 | */ 111 | public static function xdel(string $key, array $ids): void 112 | { 113 | $result = Redis::connection('stream') 114 | ->executeRaw([ 115 | Commands::XDEL, 116 | config('database.redis.stream.prefix', '') . $key, 117 | ...$ids, 118 | ]); 119 | 120 | if (!preg_match('/^\d+$/', $result)) { 121 | throw new \Exception($result); 122 | } 123 | } 124 | 125 | /** 126 | * Function to acknowledge event(s) 127 | * 128 | * @param string $key The stream that will be read 129 | * @param string $group The consumer group 130 | * @param array $ids ID of objects 131 | * @return void 132 | */ 133 | public static function xack(string $key, string $group, array $ids): void 134 | { 135 | $result = Redis::connection('stream') 136 | ->executeRaw([ 137 | Commands::XACK, 138 | config('database.redis.stream.prefix', '') . $key, 139 | config('database.redis.stream.prefix', '') . $group, 140 | ...$ids, 141 | ]); 142 | 143 | if (!preg_match('/^\d+$/', $result)) { 144 | throw new \Exception($result); 145 | } 146 | } 147 | 148 | /** 149 | * Function to create stream group 150 | * 151 | * @param string $option XGROUP Options as listed on \Afikrim\LaravelRedisStream\Data\XGROUPOptions 152 | * @param string $key The stream that will be read 153 | * @param string $groupname The group name 154 | * @param boolean $mkstream To make a stream when creating group 155 | * @param array $arguments Extra arguments, like ID or consumername. eg, ['$'] || ['consumername'] 156 | * @return void 157 | */ 158 | public static function xgroup(string $option, string $key, string $groupname, bool $mkstream = false, array $arguments = []): void 159 | { 160 | $redisArguments = [ 161 | Commands::XGROUP, 162 | $option, 163 | config('database.redis.stream.prefix', '') . $key, 164 | config('database.redis.stream.prefix', '') . $groupname, 165 | ...$arguments, 166 | ]; 167 | if ($option === XGROUPOptions::OPTION_CREATE 168 | && $mkstream) { 169 | $redisArguments[] = XGROUPOptions::OPTION_CREATE_OPTION_MKSTREAM; 170 | } 171 | 172 | $result = Redis::connection('stream') 173 | ->executeRaw($redisArguments); 174 | 175 | if ($result !== 'OK' && !preg_match('/^\d+$/', $result)) { 176 | throw new \Exception($result); 177 | } 178 | } 179 | 180 | /** 181 | * Function to read from a certain group 182 | * 183 | * @param string $group The group 184 | * @param string $consumer Consumer name 185 | * @param array $streams Streams that will be read 186 | * @param array $ids ID that will be retrieve 187 | * @param array $options Extra options, as define in \Afikrim\LaravelRedisStream\Data\Options. eg, [Options::OPTION_BLOCK, 2000] 188 | * @return array 189 | */ 190 | public static function xreadgroup(string $group, string $consumer, array $streams, array $ids, array $options): array 191 | { 192 | return (new static )->newXreadgroup($group, $consumer, $streams, $ids, $options); 193 | } 194 | 195 | /** 196 | * Function to read from a certain group 197 | * 198 | * @param string $group The group 199 | * @param string $consumer Consumer name 200 | * @param array $streams Streams that will be read 201 | * @param array $ids ID that will be retrieve 202 | * @param array $options Extra options, as define in \Afikrim\LaravelRedisStream\Data\Options. eg, [Options::OPTION_BLOCK, 2000] 203 | * @return array 204 | */ 205 | public function newXreadgroup(string $group, string $consumer, array $streams, array $ids, array $options): array 206 | { 207 | $redisArguments = [ 208 | Commands::XREADGROUP, 209 | Options::OPTION_GROUP, 210 | config('database.redis.stream.prefix', '') . $group, 211 | config('database.redis.stream.prefix', '') . $consumer, 212 | ...$options, 213 | Options::OPTION_STREAMS, 214 | ]; 215 | foreach ($streams as $stream) { 216 | $redisArguments[] = config('database.redis.stream.prefix', '') . $stream; 217 | } 218 | foreach ($ids as $id) { 219 | $redisArguments[] = !$id ? '>' : $id; 220 | } 221 | 222 | $results = Redis::connection('stream') 223 | ->executeRaw($redisArguments); 224 | 225 | if (is_string($results)) { 226 | throw new \Exception($results); 227 | } 228 | 229 | return $this->parseResults($results ?? []); 230 | } 231 | 232 | /** 233 | * Function to parse inserted data to redis arguments 234 | * 235 | * @param array $raw 236 | * @return array 237 | */ 238 | protected function parseData(array $raw): array 239 | { 240 | $data = []; 241 | foreach ($raw as $field => $value) { 242 | $data[] = $field; 243 | $data[] = $value; 244 | } 245 | 246 | return $data; 247 | } 248 | 249 | /** 250 | * Function to parse read result to associative array 251 | * 252 | * @param array $result 253 | * @return array 254 | */ 255 | protected function parseResult(array $result): array 256 | { 257 | [$key, $rawData] = $result; 258 | 259 | $data = collect($rawData) 260 | ->map(function ($rawData) { 261 | [$id, $rawData] = $rawData; 262 | 263 | $data = []; 264 | for ($i = 0; $i < count($rawData); $i += 2) { 265 | $value = $rawData[$i + 1]; 266 | $data["{$rawData[$i]}"] = $value; 267 | } 268 | 269 | return ['id' => $id, 'data' => $data]; 270 | }); 271 | 272 | return ['key' => $key, 'data' => $data->toArray()]; 273 | } 274 | 275 | /** 276 | * Function to parse read result to associative array 277 | * 278 | * @param array $results 279 | * @return array 280 | */ 281 | protected function parseResults(array $results): array 282 | { 283 | $data = collect($results) 284 | ->reduce(function ($prev, $current) { 285 | $result = $this->parseResult($current); 286 | 287 | return [$result, ...$prev]; 288 | }, []); 289 | 290 | return (array) $data; 291 | } 292 | } 293 | -------------------------------------------------------------------------------- /src/TransporterServer.php: -------------------------------------------------------------------------------- 1 | handlers = collect([]); 22 | $this->options = $options; 23 | } 24 | 25 | public function addHandler(string $pattern, Closure $handler) 26 | { 27 | $existingHandler = $this->handlers->where('pattern', $pattern)->first(); 28 | 29 | if ($existingHandler) { 30 | $this->handlers->where('pattern', $pattern)->replace([ 31 | 'pattern' => $pattern, 32 | 'handler' => $handler, 33 | ]); 34 | return; 35 | } 36 | $this->handlers->add([ 37 | 'pattern' => $pattern, 38 | 'handler' => $handler, 39 | ]); 40 | } 41 | 42 | public function removeHandler(string $pattern) 43 | { 44 | $this->handlers = $this->handlers 45 | ->filter(function ($handler) use ($pattern) { 46 | return $handler['pattern'] !== $pattern; 47 | }); 48 | } 49 | 50 | public function listen() 51 | { 52 | Log::info('Collecting patterns to listen'); 53 | $streams = $this->handlers 54 | ->reduce(function ($prev, $handler) { 55 | $pattern = $handler['pattern']; 56 | $id = '>'; 57 | 58 | return [ 59 | 'patterns' => [...$prev['patterns'], $pattern], 60 | 'ids' => [...$prev['ids'], $id], 61 | ]; 62 | }, ['patterns' => [], 'ids' => []]); 63 | 64 | collect($streams['patterns'])->each(function ($pattern) { 65 | try { 66 | // create consumer group 67 | RedisStream::xgroup( 68 | XGROUPOptions::OPTION_CREATE, 69 | $pattern, 70 | $this->getOption('group'), 71 | true, 72 | [ 73 | '$', 74 | ] 75 | ); 76 | } catch (\Exception$e) { 77 | // do nothing 78 | } 79 | }); 80 | 81 | Log::info('Listening to the stream'); 82 | $results = RedisStream::xreadgroup( 83 | $this->getOption('group'), 84 | $this->getOption('consumer'), 85 | $streams['patterns'], 86 | $streams['ids'], 87 | [ 88 | Options::OPTION_COUNT, 89 | $this->getOption('count'), 90 | Options::OPTION_BLOCK, 91 | $this->getOption('block'), 92 | ], 93 | ); 94 | 95 | $this->handle($results); 96 | } 97 | 98 | private function handle(array $results) 99 | { 100 | foreach ($results as $result) { 101 | [ 102 | 'key' => $key, 103 | 'data' => $raw_messages, 104 | ] = $result; 105 | 106 | foreach ($raw_messages as $raw_message) { 107 | [ 108 | 'id' => $_id, 109 | 'data' => $packet, 110 | ] = $raw_message; 111 | 112 | [ 113 | 'id' => $id, 114 | 'data' => $message, 115 | 'pattern' => $pattern, 116 | 'need_reply' => $need_reply, 117 | ] = (array) (new IdentityDeserializer($packet)); 118 | $message = json_decode($message, true); 119 | if (!$need_reply) { 120 | Log::info('Request with pattern: ' . $pattern . ' doesn\'t need any reply. Processing next request...'); 121 | RedisStream::xack($this->getPattern($pattern), $this->getOption('group'), [$_id]); 122 | continue; 123 | } 124 | 125 | $handler = $this->handlers 126 | ->where('pattern', $pattern) 127 | ->first(); 128 | if (!$handler) { 129 | Log::critical("There is no handler for pattern: {$pattern}"); 130 | 131 | $response_packet = [ 132 | 'id' => $id, 133 | 'response' => null, 134 | 'error' => 'There is no handler for this pattern', 135 | 'pattern' => $pattern, 136 | ]; 137 | $response = (array) (new IdentitySerializer($response_packet, true)); 138 | 139 | Log::info('Sending reply to pattern: ' . $pattern); 140 | RedisStream::xadd($this->getReplyPattern($pattern), '*', $response); 141 | RedisStream::xack($this->getPattern($pattern), $this->getOption('group'), [$_id]); 142 | continue; 143 | } 144 | 145 | Log::info('Handling request from pattern: ' . $pattern); 146 | [$error, $response_data] = $handler['handler']($message); 147 | $response_packet = [ 148 | 'id' => $id, 149 | 'response' => $response_data ? json_encode($response_data) : null, 150 | 'pattern' => $pattern, 151 | 'error' => $error, 152 | ]; 153 | $response = (array) (new IdentitySerializer($response_packet, true)); 154 | 155 | Log::info('Sending reply to pattern: ' . $pattern); 156 | RedisStream::xadd($this->getReplyPattern($pattern), '*', $response); 157 | RedisStream::xack($this->getPattern($pattern), $this->getOption('group'), [$_id]); 158 | } 159 | } 160 | } 161 | 162 | private function getPattern(string $pattern) 163 | { 164 | return $pattern; 165 | } 166 | 167 | private function getReplyPattern(string $pattern) 168 | { 169 | return $pattern . '.reply'; 170 | } 171 | 172 | private function getOption(string $key) 173 | { 174 | $value = array_key_exists($key, $this->options) ? $this->options[$key] : null; 175 | if (!$value) { 176 | if ($key === 'count') { 177 | return 1; 178 | } else if ($key === 'block') { 179 | return 2000; 180 | } else if ($key === 'group') { 181 | return Str::slug( 182 | config('app.env') 183 | . '.' 184 | . config('app.name') 185 | . '.group' 186 | , '.'); 187 | } else if ($key === 'consumer') { 188 | return Str::slug( 189 | config('app.env') 190 | . '.' 191 | . config('app.name') 192 | . '.consumer' 193 | , '.'); 194 | } 195 | 196 | return null; 197 | } 198 | 199 | if ($key === 'count' || $key === 'block') { 200 | return (int) $value; 201 | } 202 | 203 | return $value; 204 | } 205 | } 206 | -------------------------------------------------------------------------------- /tests/Functional/01ConsoleCommandsTest.php: -------------------------------------------------------------------------------- 1 | 'mystream', 27 | 'group' => 'mygroup', 28 | '--mkstream' => true, 29 | ]); 30 | 31 | $infos = Helper::xinfo([ 32 | 'XINFO', 33 | 'GROUPS', 34 | config('database.redis.stream.prefix') . 'mystream', 35 | ]); 36 | $info = $infos->filter(function ($info) {return $info['name'] === config('database.redis.stream.prefix') . 'mygroup';})->first(); 37 | 38 | $this->assertNotNull($info, "Group not created"); 39 | } 40 | 41 | public function testDestroyGroup() 42 | { 43 | // delete and create a new group first 44 | try { 45 | RedisStream::xgroup( 46 | XGROUPOptions::OPTION_DESTROY, 47 | 'mystream', 48 | 'mygroup', 49 | ); 50 | RedisStream::xgroup( 51 | XGROUPOptions::OPTION_CREATE, 52 | 'mystream', 53 | 'mygroup', 54 | true, 55 | ['$'] 56 | ); 57 | } catch (\Exception$e) {} 58 | 59 | Artisan::call('stream:destroy-group', [ 60 | 'key' => 'mystream', 61 | 'group' => 'mygroup', 62 | ]); 63 | 64 | $infos = Helper::xinfo([ 65 | 'XINFO', 66 | 'GROUPS', 67 | config('database.redis.stream.prefix') . 'mystream', 68 | ]); 69 | $info = $infos->filter(function ($info) {return $info['name'] === config('database.redis.stream.prefix') . 'mygroup';})->first(); 70 | 71 | $this->assertEquals(null, $info, "Group destroyed"); 72 | } 73 | 74 | public function testConsume() 75 | { 76 | // delete and create a new group first 77 | try { 78 | RedisStream::xgroup( 79 | XGROUPOptions::OPTION_DESTROY, 80 | 'mystream', 81 | 'mygroup', 82 | ); 83 | RedisStream::xgroup( 84 | XGROUPOptions::OPTION_CREATE, 85 | 'mystream', 86 | 'mygroup', 87 | true, 88 | ['$'] 89 | ); 90 | 91 | // populate stream 92 | $this->populateStream(); 93 | } catch (\Exception$e) {} 94 | 95 | Artisan::call('stream:consume', [ 96 | '--group' => 'mygroup', 97 | '--count' => 10, 98 | '--rest' => 0, 99 | ]); 100 | 101 | $infos = Helper::xinfo([ 102 | 'XINFO', 103 | 'GROUPS', 104 | config('database.redis.stream.prefix') . 'mystream', 105 | ]); 106 | $info = $infos->filter(function ($info) {return $info['name'] === config('database.redis.stream.prefix') . 'mygroup';})->first(); 107 | 108 | $this->assertEquals(0, $info['pending'], "There is still pending event"); 109 | } 110 | 111 | /** Not part of console commands */ 112 | // public function testClient() 113 | // { 114 | // exec("/usr/bin/php " . __DIR__ . "/../../artisan stream:consume --group=mygroup --count=1 --rest=0 --block=0"); 115 | // $results = ClientProxy::init([ 116 | // 'group' => 'mygroup', 117 | // ]) 118 | // ->publish('mystream2', [ 119 | // 'name' => 'Aziz', 120 | // 'email' => "afikrim10@gmail.com", 121 | // ]) 122 | // ->subscribe('mystream2'); 123 | 124 | // // $this->assertArrayHasKey('response', $results, "Array doesn't have 'response' key"); 125 | // } 126 | 127 | private function populateStream() 128 | { 129 | for ($i = 0; $i < 10; $i += 1) { 130 | ClientProxy::init([ 131 | 'group' => 'mygroup', 132 | ]) 133 | ->dispatch('mystream', [ 134 | 'name' => 'Aziz', 135 | 'email' => "afikrim1{$i}@gmail.com", 136 | ]); 137 | } 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /tests/Helper.php: -------------------------------------------------------------------------------- 1 | executeRaw($arguments); 13 | $infos = collect($infosRaw) 14 | ->map(function ($infoRaw) { 15 | $info = []; 16 | for ($i = 0; $i < count($infoRaw); $i += 2) { 17 | $info[$infoRaw[$i]] = $infoRaw[$i + 1]; 18 | } 19 | 20 | return $info; 21 | }); 22 | 23 | return $infos; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /tests/TestCase.php: -------------------------------------------------------------------------------- 1 |