├── .gitignore ├── README.md ├── composer.json ├── src ├── AutoLoader.php ├── Bucket.php ├── DelayQueue.php ├── Exception │ ├── ClassNotFoundException.php │ ├── InvalidJobException.php │ └── ServiceNotFoundException.php ├── Job.php ├── JobHandler.php ├── JobPool.php ├── Packer │ ├── JsonPacker.php │ └── PackerInterface.php ├── Process │ └── DelayQueueProcess.php └── ReadyQueue.php └── test ├── .env └── bootstrap.php /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | .buildpath 3 | .settings/ 4 | .project 5 | *.patch 6 | /.idea 7 | .git/ 8 | vendor/ 9 | *.lock 10 | .phpintel/ 11 | .DS_Store 12 | composer.lock 13 | Thumbs.db 14 | /.vscode -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 基于swoft2.0的redis延时队列 3 | 详细内容参考[有赞的延迟队列 ](https://tech.youzan.com/queuing_delay/) 4 | 5 | ### 欢迎使用queue 2.0 6 | queue V2.0版本的，配置更简单了，使用更简洁。 7 | 去除msg扩展，直接用json代替 8 | 9 | 10 | ### 安装 composer require poolbang/queue 11 | 12 | 添加config/queue.php文件，然后内容为： 13 | ``` 14 | reutrn [ 15 | 'contrast' => 10, //每次对比的元素数量， 默认: 10 16 | 'interval' => 2, //空数据时等待时长， 默认: 1 17 | 'log' => false, //是否写入日志， 默认: true 18 | ]; 19 | ``` 20 | 21 | 开启DelayQueue进程，在app/bean.php中的process添加下面内容： 22 | ``` 23 | 'process' => [ 24 | 'queue' => bean(\Queue\Process\DelayQueueProcess::class), 25 | ], 26 | ``` 27 | 28 | ##### job任务的类完成时执行的逻辑 29 | ``` 30 | namespace App\Models\Logic; 31 | 32 | 33 | use Queue\JobHandler; 34 | use Swoft\Bean\Annotation\Mapping\Bean; 35 | 36 | /** 37 | * @Bean(scope=Bean::PROTOTYPE) 38 | * Class QueueLogic 39 | * @package App\Models\Logic 40 | */ 41 | class QueueLogic extends JobHandler 42 | { 43 | 44 | protected function perform() 45 | { 46 | echo 'JobId: ' . $this->id . PHP_EOL; 47 | var_dump($this->args); 48 | } 49 | } 50 | ``` 51 | #### 用法 52 | 53 | ``` 54 | use Queue\DelayQueue; 55 | 56 | /** 57 | * 添加 58 | * 59 | * @param string $topic 一组相同类型Job的集合（队列）。 60 | * @param string $jobName job任务的类名，是延迟队列里的基本单元。与具体的Topic关联在一起。 61 | * @param integer $delay job任务延迟时间 传入相对于当前时间的延迟时间即可 例如延迟10分钟执行 传入 10*60 62 | * @param integer $ttr job任务超时时间,保证job至少被消费一次,如果时间内未删除Job方法,则会再次投入ready队列中 63 | * @param array $args 执行Job任务时传递的可选参数。 64 | * @param string $jobId 任务id可传入或默认生成 65 | */ 66 | DelayQueue::enqueue('test',QueueLogic::class,5,10,['order_id'=>uniqid('queue_')]); 67 | //获取 68 | DelayQueue::get($jobId); 69 | //删除 70 | DelayQueue::remove($jobId); 71 | ``` 72 | 73 | #### [future] 74 | 用redis的发布订阅来处理消息 75 | 76 | 由于现在swoft的发布订阅出现问题，暂时不能用发布订阅来处理消息 77 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "poolbang/queue", 3 | "keywords": [ 4 | "php", 5 | "swoole", 6 | "swoft", 7 | "debug", 8 | "queue" 9 | ], 10 | "description": "queue for swoft", 11 | "license": "Apache-2.0", 12 | "minimum-stability": "beta", 13 | "authors": [ 14 | { 15 | "name": "su", 16 | "email": "1346233126@qq.com" 17 | } 18 | ], 19 | "require": { 20 | "php": ">7.1", 21 | "ext-redis": "*", 22 | "ext-json": "*", 23 | "swoft/framework": "~2.0.0", 24 | "swoft/swoole-ide-helper": "dev-master", 25 | "swoft/redis": "~2.0.0", 26 | "swoft/process": "~2.0.0" 27 | }, 28 | "autoload": { 29 | "psr-4": { 30 | "Queue\\": "src/" 31 | } 32 | }, 33 | "require-dev": { 34 | "phpunit/phpunit": "^5.7" 35 | }, 36 | "scripts": { 37 | "test": "./vendor/bin/phpunit" 38 | } 39 | } -------------------------------------------------------------------------------- /src/AutoLoader.php: -------------------------------------------------------------------------------- 1 | 0; 35 | } 36 | 37 | /** 38 | * Get namespace and dir 39 | * 40 | * @return array 41 | * [ 42 | * namespace => dir path 43 | * ] 44 | */ 45 | public function getPrefixDirs(): array 46 | { 47 | return [ 48 | __NAMESPACE__ => __DIR__, 49 | ]; 50 | } 51 | 52 | /** 53 | * Metadata information for the component. 54 | * 55 | * @return array 56 | * @see ComponentInterface::getMetadata() 57 | */ 58 | public function metadata(): array 59 | { 60 | $jsonFile = dirname(__DIR__) . '/composer.json'; 61 | 62 | return ComposerJSON::open($jsonFile)->getMetadata(); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /src/Bucket.php: -------------------------------------------------------------------------------- 1 | generateBucketName(); 38 | return $this->redis->zAdd($bucketName, [$jobId => intval($delay)]); 39 | 40 | } 41 | 42 | /** 43 | * 从bucket中获取延迟时间最小的一批Job任务 44 | * 45 | * @param integer $index 索引位置 46 | * @return array 47 | */ 48 | public function getJobsMinDelayTime($index) 49 | { 50 | $bucketName = $this->generateBucketName(); 51 | return $this->redis->zrange($bucketName, 0, $index - 1, true); 52 | } 53 | 54 | /** 55 | * 从bucket中删除JobId 56 | * 57 | * @param string $jobId 任务id 58 | * @return boolean 59 | */ 60 | public function removeBucket($jobId) 61 | { 62 | $bucketName = $this->generateBucketName(); 63 | return $this->redis->zRem($bucketName, $jobId); 64 | } 65 | 66 | /** 67 | * 获取bucket 68 | * @return string 69 | */ 70 | public function generateBucketName() 71 | { 72 | return 'bucket'; 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /src/DelayQueue.php: -------------------------------------------------------------------------------- 1 | 20847]); 50 | * 51 | * @param string $topic 一组相同类型Job的集合（队列）。供消费者来订阅。 52 | * @param string $jobName job任务的类名，是延迟队列里的基本单元。与具体的Topic关联在一起。 53 | * @param integer $delay job任务延迟时间 传入相对于当前时间的延迟时间即可 例如延迟10分钟执行 传入 10*60 54 | * @param integer $ttr job任务超时时间,保证job至少被消费一次,如果时间内未删除Job方法,则会再次投入ready队列中 55 | * @param array $args 执行Job任务时传递的可选参数。 56 | * @param string $jobId 任务id可传入或默认生成 57 | * @return string|boolean 58 | * 59 | */ 60 | public static function enqueue($topic, $jobName, $delay, $ttr, $args = null, $jobId = null) 61 | { 62 | if (empty($topic) || empty($jobName)) { 63 | return false; 64 | } 65 | $job = new Job(); 66 | $job['id'] = is_null($jobId) ? md5(uniqid(microtime(true), true)) : $jobId; 67 | $job['class'] = $jobName; 68 | $job['topic'] = $topic; 69 | $job['args'] = $args; 70 | $job['delay'] = time() + intval($delay); 71 | $job['ttr'] = intval($ttr); 72 | if (!static::push($job)) { 73 | return false; 74 | } 75 | return $job['id']; 76 | } 77 | 78 | /** 79 | * 80 | * @param Job $job 81 | * @return bool|mixed 82 | * @throws InvalidJobException 83 | */ 84 | private static function push(Job $job) 85 | { 86 | if ( 87 | empty($job['id']) 88 | || empty($job['topic']) 89 | || empty($job['class']) 90 | || $job['delay'] < 0 91 | || $job['ttr'] < 0 92 | ) { 93 | throw new InvalidJobException("job attribute cannot be empty."); 94 | } 95 | $jobPool = Swoft::getBean(JobPool::class); 96 | $result = $jobPool->putJob($job); 97 | if (!$result) { 98 | return false; 99 | } 100 | $result = Swoft::getBean(Bucket::class)->pushBucket($job['id'], $job['delay']); 101 | //Bucket添加失败 删除元数据 102 | if (!$result) { 103 | $jobPool->removeJob($job['id']); 104 | return false; 105 | } 106 | return $job['id']; 107 | } 108 | 109 | /** 110 | * 删除job任务,元数据和bucket等信息都会删除 111 | * 在job任务处理结束后调用,不删除在达到超时时间后，会再次投递到可消费队列中,等待再次消费 112 | * 113 | * @param string $jobId 任务id 114 | * @return boolean 115 | */ 116 | public static function remove($jobId) 117 | { 118 | Swoft::getBean(Bucket::class)->removeBucket($jobId); 119 | return Swoft::getBean(JobPool::class)->removeJob($jobId); 120 | } 121 | 122 | /** 123 | * 获取job任务信息 124 | * 125 | * @param string $jobId 任务id 126 | * @return array 127 | */ 128 | public static function get($jobId) 129 | { 130 | return Swoft::getBean(JobPool::class)->getJob($jobId); 131 | } 132 | 133 | /** 134 | * 立即弹出 135 | * 136 | * @param array $topics 一组相同类型Job的集合（队列）。 137 | * @return array 138 | */ 139 | public function pop(array $topics) 140 | { 141 | $readyJob = $this->readyQueue->popReadyQueue($topics); 142 | if (empty($readyJob)) { 143 | return []; 144 | } 145 | $jobInfo = static::get($readyJob); 146 | if (empty($jobInfo)) { 147 | return []; 148 | } 149 | $this->bucket->pushBucket($jobInfo['id'], time() + $jobInfo['ttr']); 150 | return $jobInfo; 151 | } 152 | 153 | /** 154 | * 阻塞等待弹出 155 | * 156 | * @param array $topics 一组相同类型Job的集合（队列）。 157 | * @param integer $timeout 阻塞等待超时时间 158 | * @return array 159 | */ 160 | public function bpop(array $topics, $timeout) 161 | { 162 | $readyJob = $this->readyQueue->bPopReadyQueue($topics, $timeout); 163 | if (empty($readyJob) || count($readyJob) != 2) { 164 | return []; 165 | } 166 | $jobInfo = static::get($readyJob[1]); 167 | if (empty($jobInfo)) { 168 | return []; 169 | } 170 | $this->bucket->pushBucket($jobInfo['id'], time() + $jobInfo['ttr']); 171 | return $jobInfo; 172 | } 173 | 174 | /** 175 | * Timer触发器 扫描bucket, 将符合执行时间的任务放到readyqueue中 176 | * @param int $index 177 | * @return bool 178 | */ 179 | public function touchTimer(int $index) 180 | { 181 | while (true) { 182 | $bucketJobs = $this->bucket->getJobsMinDelayTime($index); 183 | // 集合为空 184 | if (empty($bucketJobs)) { 185 | return false; 186 | } 187 | $isBreak = false; 188 | foreach ($bucketJobs as $jobId => $time) { 189 | if ($time > time()) { 190 | $isBreak = true; 191 | break; 192 | } 193 | $jobInfo =(array) $this->jobPool->getJob($jobId); 194 | // job元信息不存在, 从bucket中删除 195 | if (empty($jobInfo)) { 196 | $this->bucket->removeBucket($jobId); 197 | continue; 198 | } 199 | // 元信息中delay是否小于等于当前时间 200 | if ($jobInfo['delay'] > time()) { 201 | $this->bucket->removeBucket($jobInfo['id']); 202 | $this->bucket->pushBucket($jobInfo['id'], $jobInfo['delay']); 203 | continue; 204 | } 205 | if (config('queue.log', true)) { 206 | CLog::info('Found jobId:%s on Bucket. ensure time: %u' , $jobId, $time); 207 | CLog::info('Push jobId: %s to topic: %s ' , $jobId, $jobInfo['topic']); 208 | } 209 | $this->bucket->removeBucket($jobId); 210 | if (Swoft::hasBean($jobInfo['class'])) { 211 | sgo(function () use ($jobInfo){ 212 | $queueClass = Swoft::getBean($jobInfo['class']); 213 | if ($queueClass instanceof JobHandler) { 214 | $queueClass->setId($jobInfo['id']); 215 | $queueClass->setArgs($jobInfo['args']); 216 | $queueClass->run(); 217 | } 218 | }); 219 | } 220 | } 221 | if ($isBreak) { 222 | return false; 223 | } 224 | } 225 | return false; 226 | } 227 | } 228 | -------------------------------------------------------------------------------- /src/Exception/ClassNotFoundException.php: -------------------------------------------------------------------------------- 1 | '', 20 | 'topic' => '', 21 | 'class' => '', 22 | 'args' => '', 23 | 'delay' => 0, 24 | 'ttr' => 0, 25 | ]; 26 | 27 | /** 28 | * 获取属性字段 29 | * @return array 30 | */ 31 | public function getAttribute(){ 32 | return $this->attribute; 33 | } 34 | 35 | /** 36 | * 调用 isset($job[$offset]) 时自动触发 37 | * @param string $offset 38 | * @return string 39 | */ 40 | public function offsetExists($offset) 41 | { 42 | return isset($this->attribute[$offset]); 43 | } 44 | 45 | /** 46 | * 调用 $job[$offset] 时自动触发 47 | * @param string $offset 48 | * @return mixed 49 | */ 50 | public function offsetGet($offset) 51 | { 52 | return $this->attribute[$offset]; 53 | } 54 | 55 | /** 56 | * 调用 $job[$offset]=$value 时自动触发 57 | * @param string $offset 58 | * @param mixed $value 59 | */ 60 | public function offsetSet($offset, $value) 61 | { 62 | if(array_key_exists($offset,$this->attribute)){ 63 | $this->attribute[$offset] = $value; 64 | } 65 | } 66 | 67 | /** 68 | * 调用 unset($job[$offset]) 时自动触发 69 | * @param string $offset 70 | * @return string 71 | */ 72 | public function offsetUnset($offset) 73 | { 74 | unset($this->attribute[$offset]); 75 | } 76 | 77 | 78 | } -------------------------------------------------------------------------------- /src/JobHandler.php: -------------------------------------------------------------------------------- 1 | id = $id; 39 | } 40 | 41 | /** 42 | * @param mixed $args 43 | */ 44 | public function setArgs($args) 45 | { 46 | $this->args = $args; 47 | } 48 | 49 | public function run() 50 | { 51 | $this->setUp(); 52 | $delayQueue = Swoft::getBean(DelayQueue::class); 53 | try { 54 | $this->perform(); 55 | $delayQueue->remove($this->id); 56 | } catch (Exception $exception) { 57 | CLog::info('Job execution failed %s', $exception->getMessage()); 58 | //失败时删除job任务避免重复的投递到bucket中,一直触发执行报错的job任务,如果需要执行重载次方法删除下面一行代码即可 59 | $delayQueue->remove($this->id); 60 | } 61 | 62 | $this->tearDown(); 63 | } 64 | 65 | protected function setUp() 66 | { 67 | } 68 | 69 | protected function tearDown() 70 | { 71 | } 72 | 73 | abstract protected function perform(); 74 | } 75 | -------------------------------------------------------------------------------- /src/JobPool.php: -------------------------------------------------------------------------------- 1 | redis->get($jobId); 43 | if(empty($data)){ 44 | return []; 45 | } 46 | return $this->packer->unpack($data); 47 | } 48 | 49 | 50 | /** 51 | * 放入job元数据 52 | * 53 | * @param \Queue\Job $job 54 | * @return boolean 55 | */ 56 | public function putJob(Job $job) 57 | { 58 | $data = $this->packer->pack($job->getAttribute()); 59 | return $this->redis->set($job['id'],$data); 60 | 61 | } 62 | 63 | /** 64 | * 删除job元数据 65 | * 66 | * @param string $jobId job id 67 | * @return mixed 68 | */ 69 | public function removeJob($jobId) 70 | { 71 | return $this->redis->del($jobId); 72 | } 73 | 74 | 75 | } 76 | -------------------------------------------------------------------------------- /src/Packer/JsonPacker.php: -------------------------------------------------------------------------------- 1 | touchTimer($contrast); 32 | Coroutine::sleep($interval); 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /src/ReadyQueue.php: -------------------------------------------------------------------------------- 1 | redis->lRange($topic, $start, $end); 32 | } 33 | 34 | /** 35 | * 添加JobId到队列中 36 | * 37 | * @param string $queueName 队列名称即主题topic名称 38 | * @param string $jobId 任务id 39 | * @return boolean 40 | */ 41 | public function pushReadyQueue($queueName, $jobId) 42 | { 43 | return $this->redis->rPush($queueName, $jobId); 44 | } 45 | 46 | /** 47 | * 从队列中获取JobId 即时性要求不高的 48 | * 49 | * @param array $queueNames 多个队列名称即多个主题topic名称 50 | * @return mixed 51 | */ 52 | public function popReadyQueue(array $queueNames) 53 | { 54 | foreach ($queueNames as $queueName) { 55 | $job = $this->redis->lPop($queueName); 56 | if (!empty($job)) { 57 | return $job; 58 | } 59 | } 60 | return []; 61 | } 62 | 63 | /** 64 | * 从队列中阻塞获取JobId 即时性要求高的时候使用 65 | * 66 | * @param array $queueNames 多个队列名称即多个主题topic名称 67 | * @param integer $timeout 超时时间 68 | * @return array 69 | */ 70 | public function bPopReadyQueue(array $queueNames, $timeout) 71 | { 72 | return $this->redis->blPop($queueNames, $timeout); 73 | } 74 | } 75 | -------------------------------------------------------------------------------- /test/.env: -------------------------------------------------------------------------------- 1 | # test config 2 | TEST_NAME=test 3 | TEST_URI=127.0.0.1:6378 4 | TEST_MAX_IDEL=2 5 | TEST_MAX_ACTIVE=2 6 | TEST_MAX_WAIT=2 7 | TEST_TIMEOUT=2 8 | TEST_BALANCER=r1 9 | TEST_USE_PROVIDER=true 10 | TEST_PROVIDER=c1 11 | 12 | # the pool of master nodes pool 13 | DB_NAME=master2 14 | DB_URI=127.0.0.1:3302,127.0.0.1:3302 15 | DB_MAX_IDEL=20 16 | DB_MAX_ACTIVE=20 17 | DB_MAX_WAIT=20 18 | DB_TIMEOUT=20 19 | DB_USE_PROVIDER=true 20 | DB_BALANCER=random2 21 | DB_PROVIDER=consul2 22 | 23 | # the pool of slave nodes pool 24 | DB_SLAVE_NAME=slave2 25 | DB_SLAVE_URI=127.0.0.1:3306/test?user=root&password=&charset=utf8,127.0.0.1:3306/test?user=root&password=&charset=utf8 26 | DB_SLAVE_MAX_IDEL=2 27 | DB_SLAVE_MAX_ACTIVE=2 28 | DB_SLAVE_MAX_WAIT=2 29 | DB_SLAVE_TIMEOUT=2 30 | DB_SLAVE_USE_PROVIDER=false 31 | DB_SLAVE_BALANCER=random 32 | DB_SLAVE_PROVIDER=consul2 33 | 34 | # the pool of redis 35 | REDIS_NAME=redis2 36 | REDIS_URI=127.0.0.1:2222,127.0.0.1:2222 37 | REDIS_MAX_IDEL=2 38 | REDIS_MAX_ACTIVE=2 39 | REDIS_MAX_WAIT=2 40 | REDIS_TIMEOUT=2 41 | REDIS_USE_PROVIDER=true 42 | REDIS_BALANCER=random2 43 | REDIS_PROVIDER=consul2 44 | 45 | 46 | # consul provider 47 | PROVIDER_CONSUL_ADDRESS=http://127.0.0.1:82 48 | PROVIDER_CONSUL_TAGS=1,2 49 | PROVIDER_CONSUL_TIMEOUT=2 50 | PROVIDER_CONSUL_INTERVAL=2 -------------------------------------------------------------------------------- /test/bootstrap.php: -------------------------------------------------------------------------------- 1 | bootstrap(); 10 | \Swoft\Bean\BeanFactory::reload([ 11 | 'application' => [ 12 | 'class' => \Swoft\Testing\Application::class, 13 | 'inTest' => true 14 | ], 15 | ]); 16 | $initApplicationContext = new \Swoft\Core\InitApplicationContext(); 17 | $initApplicationContext->init(); --------------------------------------------------------------------------------