├── README.md ├── composer.json ├── examples └── test.php └── src ├── Client.php └── UnretryableException.php /README.md: -------------------------------------------------------------------------------- 1 | # redis-queue 2 | Message queue system written in PHP based on [workerman](https://github.com/walkor/workerman) and backed by Redis. 3 | 4 | # Install 5 | ``` 6 | composer require workerman/redis-queue 7 | ``` 8 | 9 | # Usage 10 | test.php 11 | ```php 12 | onWorkerStart = function () { 21 | $client = new Client('redis://127.0.0.1:6379'); 22 | 23 | $client->subscribe('user-1', function($data) { 24 | echo "user-1\n"; 25 | var_export($data); 26 | }); 27 | 28 | $client->subscribe('user-2', function($data) { 29 | echo "user-2\n"; 30 | var_export($data); 31 | }); 32 | 33 | $client->onConsumeFailure(function (\Throwable $exception, $package) { 34 | echo "consume failure\n"; 35 | echo $exception->getMessage(), "\n"; 36 | var_export($package); 37 | }); 38 | 39 | Timer::add(1, function() use ($client) { 40 | $client->send('user-1', ['some', 'data']); 41 | }); 42 | }; 43 | 44 | Worker::runAll(); 45 | ``` 46 | 47 | Run with command `php test.php start` or `php test.php start -d`. 48 | 49 | # API 50 | 51 | * Client::__construct() 52 | * Client::send() 53 | * Client::subscribe() 54 | * Client::unsubscribe() 55 | * Client::onConsumeFailure() 56 | 57 | ------------------------------------------------------- 58 | 59 | 60 | ### __construct (string $address, [array $options]) 61 | 62 | Create an instance by $address and $options. 63 | 64 | * `$address` for example `redis://ip:6379`. 65 | 66 | * `$options` is the client connection options. Defaults: 67 | * `auth`: default '' 68 | * `db`: default 0 69 | * `retry_seconds`: Retry interval after consumption failure 70 | * `max_attempts`: Maximum number of retries after consumption failure 71 | 72 | ------------------------------------------------------- 73 | 74 | 75 | ### send(String $queue, Mixed $data, [int $dely=0]) 76 | 77 | Send a message to a queue 78 | 79 | * `$queue` is the queue to publish to, `String` 80 | * `$data` is the message to publish, `Mixed` 81 | * `$dely` is delay seconds for delayed consumption, `Int` 82 | 83 | ------------------------------------------------------- 84 | 85 | 86 | ### subscribe(mixed $queue, callable $callback) 87 | 88 | Subscribe to a queue or queues 89 | 90 | * `$queue` is a `String` queue or an `Array` which has as keys the queue name to subscribe. 91 | * `$callback` - `function (Mixed $data)`, `$data` is the data sent by `send($queue, $data)`. 92 | 93 | ------------------------------------------------------- 94 | 95 | 96 | ### unsubscribe(mixed $queue) 97 | 98 | Unsubscribe from a queue or queues 99 | 100 | ------------------------------------------------------- 101 | 102 | 103 | ### onConsumeFailure(callable $callback) 104 | 105 | When consumption fails onConsumeFailure is triggered. 106 | 107 | * `$callback` - `function (\Throwable $exception, array $package)`, `$package` contains information such as data queue attempts 108 | 109 | ------------------------------------------------------- 110 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name" : "workerman/redis-queue", 3 | "type" : "library", 4 | "homepage": "http://www.workerman.net", 5 | "license" : "MIT", 6 | "description": "Message queue system written in PHP based on workerman and backed by Redis.", 7 | "require": { 8 | "php": ">=7.0", 9 | "workerman/redis" : "^1.0||^2.0", 10 | "workerman/workerman" : ">=4.0.20" 11 | }, 12 | "autoload": { 13 | "psr-4": {"Workerman\\RedisQueue\\": "./src"} 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /examples/test.php: -------------------------------------------------------------------------------- 1 | onWorkerStart = function () { 11 | $client = new Client('redis://127.0.0.1:6379'); 12 | $client->subscribe('user-1', function ($data) { 13 | echo "user-1\n"; 14 | var_export($data); 15 | }, function ($data) { 16 | echo "user-1 failed\n"; 17 | var_export($data); 18 | }); 19 | $client->subscribe('user-2', function ($data) { 20 | echo "user-2\n"; 21 | var_export($data); 22 | }, function ($data) { 23 | echo "user-2 failed\n"; 24 | var_export($data); 25 | }); 26 | $client->onConsumeFailure(function (\Throwable $exception, $package) { 27 | echo "consume failure\n"; 28 | echo $exception->getMessage(), "\n"; 29 | var_export($package); 30 | }); 31 | Timer::add(1, function () use ($client) { 32 | $client->send('user-1', [666, 777]); 33 | }); 34 | }; 35 | 36 | Worker::runAll(); 37 | -------------------------------------------------------------------------------- /src/Client.php: -------------------------------------------------------------------------------- 1 | 11 | * @copyright walkor 12 | * @link http://www.workerman.net/ 13 | * @license http://www.opensource.org/licenses/mit-license.php MIT License 14 | */ 15 | 16 | namespace Workerman\RedisQueue; 17 | 18 | use RuntimeException; 19 | use Workerman\Timer; 20 | use Workerman\Redis\Client as Redis; 21 | use Psr\Log\LoggerInterface; 22 | 23 | /** 24 | * Class Client 25 | * @package Workerman\RedisQueue 26 | */ 27 | #[\AllowDynamicProperties] 28 | class Client 29 | { 30 | /** 31 | * Queue waiting for consumption 32 | */ 33 | const QUEUE_WAITING = '{redis-queue}-waiting'; 34 | 35 | /** 36 | * Queue with delayed consumption 37 | */ 38 | const QUEUE_DELAYED = '{redis-queue}-delayed'; 39 | 40 | /** 41 | * Queue with consumption failure 42 | */ 43 | const QUEUE_FAILED = '{redis-queue}-failed'; 44 | 45 | /** 46 | * @var Redis 47 | */ 48 | protected $_redisSubscribe; 49 | 50 | /** 51 | * @var Redis 52 | */ 53 | protected $_redisSend; 54 | 55 | /** 56 | * @var array 57 | */ 58 | protected $_subscribeQueues = []; 59 | 60 | /** 61 | * @var LoggerInterface 62 | */ 63 | protected $_logger = null; 64 | 65 | /** 66 | * consume failure callback 67 | * @var callable 68 | */ 69 | protected $_consumeFailure = null; 70 | /** 71 | * @var array 72 | */ 73 | protected $_options = [ 74 | 'retry_seconds' => 5, 75 | 'max_attempts' => 5, 76 | 'auth' => '', 77 | 'db' => 0, 78 | 'prefix' => '', 79 | ]; 80 | 81 | /** 82 | * Client constructor. 83 | * @param $address 84 | * @param array $options 85 | */ 86 | public function __construct($address, $options = []) 87 | { 88 | $this->_redisSubscribe = new Redis($address, $options); 89 | $this->_redisSubscribe->brPoping = 0; 90 | $this->_redisSend = new Redis($address, $options); 91 | if (isset($options['auth']) && $options['auth'] !== '') { 92 | $this->_redisSubscribe->auth($options['auth']); 93 | $this->_redisSend->auth($options['auth']); 94 | } 95 | if (isset($options['db'])) { 96 | $this->_redisSubscribe->select($options['db']); 97 | $this->_redisSend->select($options['db']); 98 | } 99 | $this->_options = array_merge($this->_options, $options); 100 | } 101 | 102 | /** 103 | * Send. 104 | * 105 | * @param $queue 106 | * @param $data 107 | * @param int $delay 108 | * @param callable $cb 109 | */ 110 | public function send($queue, $data, $delay = 0, $cb = null) 111 | { 112 | static $_id = 0; 113 | $id = \microtime(true) . '.' . (++$_id); 114 | $now = time(); 115 | $package_str = \json_encode([ 116 | 'id' => $id, 117 | 'time' => $now, 118 | 'delay' => $delay, 119 | 'attempts' => 0, 120 | 'queue' => $queue, 121 | 'data' => $data 122 | ]); 123 | if (\is_callable($delay)) { 124 | $cb = $delay; 125 | $delay = 0; 126 | } 127 | if ($cb) { 128 | $cb = function ($ret) use ($cb) { 129 | $cb((bool)$ret); 130 | }; 131 | if ($delay == 0) { 132 | $this->_redisSend->lPush($this->_options['prefix'] . static::QUEUE_WAITING . $queue, $package_str, $cb); 133 | } else { 134 | $this->_redisSend->zAdd($this->_options['prefix'] . static::QUEUE_DELAYED, $now + $delay, $package_str, $cb); 135 | } 136 | return; 137 | } 138 | if ($delay == 0) { 139 | $this->_redisSend->lPush($this->_options['prefix'] . static::QUEUE_WAITING . $queue, $package_str); 140 | } else { 141 | $this->_redisSend->zAdd($this->_options['prefix'] . static::QUEUE_DELAYED, $now + $delay, $package_str); 142 | } 143 | } 144 | 145 | /** 146 | * Set the consume failure callback. 147 | * 148 | * @param callable $callback 149 | */ 150 | public function onConsumeFailure(callable $callback) 151 | { 152 | $this->_consumeFailure = $callback; 153 | } 154 | 155 | /** 156 | * Subscribe. 157 | * 158 | * @param string|array $queue 159 | * @param callable $callback 160 | */ 161 | public function subscribe($queue, callable $callback) 162 | { 163 | $queue = (array)$queue; 164 | foreach ($queue as $q) { 165 | $redis_key = $this->_options['prefix'] . static::QUEUE_WAITING . $q; 166 | $this->_subscribeQueues[$redis_key] = $callback; 167 | } 168 | $this->pull(); 169 | } 170 | 171 | /** 172 | * Unsubscribe. 173 | * 174 | * @param string|array $queue 175 | * @return void 176 | */ 177 | public function unsubscribe($queue) 178 | { 179 | $queue = (array)$queue; 180 | foreach ($queue as $q) { 181 | $redis_key = $this->_options['prefix'] . static::QUEUE_WAITING . $q; 182 | unset($this->_subscribeQueues[$redis_key]); 183 | } 184 | } 185 | 186 | /** 187 | * tryToPullDelayQueue. 188 | */ 189 | protected function tryToPullDelayQueue() 190 | { 191 | static $retry_timer = 0; 192 | if ($retry_timer) { 193 | return; 194 | } 195 | $retry_timer = Timer::add(1, function () { 196 | $now = time(); 197 | $options = ['LIMIT', 0, 128]; 198 | $this->_redisSend->zrevrangebyscore($this->_options['prefix'] . static::QUEUE_DELAYED, $now, '-inf', $options, function ($items) { 199 | if ($items === false) { 200 | throw new RuntimeException($this->_redisSend->error()); 201 | } 202 | foreach ($items as $package_str) { 203 | $this->_redisSend->zRem($this->_options['prefix'] . static::QUEUE_DELAYED, $package_str, function ($result) use ($package_str) { 204 | if ($result !== 1) { 205 | return; 206 | } 207 | $package = \json_decode($package_str, true); 208 | if (!$package) { 209 | $this->_redisSend->lPush($this->_options['prefix'] . static::QUEUE_FAILED, $package_str); 210 | return; 211 | } 212 | $this->_redisSend->lPush($this->_options['prefix'] . static::QUEUE_WAITING . $package['queue'], $package_str); 213 | }); 214 | } 215 | }); 216 | }); 217 | } 218 | 219 | /** 220 | * pull. 221 | */ 222 | public function pull() 223 | { 224 | $this->tryToPullDelayQueue(); 225 | if (!$this->_subscribeQueues || $this->_redisSubscribe->brPoping) { 226 | return; 227 | } 228 | $cb = function ($data) use (&$cb) { 229 | if ($data) { 230 | $this->_redisSubscribe->brPoping = 0; 231 | $redis_key = $data[0]; 232 | $package_str = $data[1]; 233 | $package = json_decode($package_str, true); 234 | if (!$package) { 235 | $this->_redisSend->lPush($this->_options['prefix'] . static::QUEUE_FAILED, $package_str); 236 | } else { 237 | if (!isset($this->_subscribeQueues[$redis_key])) { 238 | // 取消订阅，放回队列 239 | $this->_redisSend->rPush($redis_key, $package_str); 240 | } else { 241 | $callback = $this->_subscribeQueues[$redis_key]; 242 | try { 243 | \call_user_func($callback, $package['data']); 244 | } catch (UnretryableException $e) { 245 | $this->log((string)$e); 246 | $package['max_attempts'] = $this->_options['max_attempts']; 247 | $package['error'] = $e->getMessage(); 248 | $this->fail($package); 249 | } catch (\Throwable $e) { 250 | $this->log((string)$e); 251 | $package['max_attempts'] = $this->_options['max_attempts']; 252 | $package['error'] = $e->getMessage(); 253 | $package_modified = null; 254 | if ($this->_consumeFailure) { 255 | try { 256 | $package_modified = \call_user_func($this->_consumeFailure, $e, $package); 257 | } catch (\Throwable $ta) { 258 | $this->log((string)$ta); 259 | } 260 | } 261 | if (is_array($package_modified)) { 262 | $package['data'] = $package_modified['data'] ?? $package['data']; 263 | $package['attempts'] = $package_modified['attempts'] ?? $package['attempts']; 264 | $package['max_attempts'] = $package_modified['max_attempts'] ?? $package['max_attempts']; 265 | $package['error'] = $package_modified['error'] ?? $package['error']; 266 | } 267 | if (++$package['attempts'] > $package['max_attempts']) { 268 | $this->fail($package); 269 | } else { 270 | $this->retry($package); 271 | } 272 | } 273 | } 274 | } 275 | } 276 | if ($this->_subscribeQueues) { 277 | $this->_redisSubscribe->brPoping = 1; 278 | Timer::add(0.000001, [$this->_redisSubscribe, 'brPop'], [\array_keys($this->_subscribeQueues), 1, $cb], false); 279 | } 280 | }; 281 | $this->_redisSubscribe->brPoping = 1; 282 | $this->_redisSubscribe->brPop(\array_keys($this->_subscribeQueues), 1, $cb); 283 | } 284 | 285 | /** 286 | * @param $package 287 | */ 288 | protected function retry($package) 289 | { 290 | $delay = time() + $this->_options['retry_seconds'] * ($package['attempts']); 291 | $this->_redisSend->zAdd($this->_options['prefix'] . static::QUEUE_DELAYED, $delay, \json_encode($package, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); 292 | } 293 | 294 | /** 295 | * @param $package 296 | */ 297 | protected function fail($package) 298 | { 299 | $this->_redisSend->lPush($this->_options['prefix'] . static::QUEUE_FAILED, \json_encode($package, JSON_UNESCAPED_UNICODE | JSON_PRETTY_PRINT)); 300 | } 301 | 302 | /** 303 | * @param $message 304 | * @return void 305 | */ 306 | protected function log($message) 307 | { 308 | if ($this->_logger) { 309 | $this->_logger->info($message); 310 | return; 311 | } 312 | echo $message . PHP_EOL; 313 | } 314 | 315 | /** 316 | * @param $logger 317 | * @return mixed|LoggerInterface 318 | */ 319 | public function logger($logger = null) 320 | { 321 | if ($logger) { 322 | $this->_logger = $logger; 323 | } 324 | return $this->_logger; 325 | } 326 | } 327 | -------------------------------------------------------------------------------- /src/UnretryableException.php: -------------------------------------------------------------------------------- 1 |