├── .gitignore ├── .atoum.php ├── .php_cs ├── Makefile ├── .travis.yml ├── src ├── LoggedPredisClient │ ├── PredisClientInterface.php │ └── LoggedPredisClient.php ├── MessageHandler │ ├── Producer.php │ ├── AbstractMessageHandler.php │ ├── LostMessagesConsumer.php │ └── Consumer.php ├── Event │ ├── PredisEvent.php │ └── ConsumerEvent.php ├── Queue │ ├── AbstractQueueTool.php │ ├── Cleanup.php │ ├── Definition.php │ └── Inspector.php └── MessageEnvelope.php ├── composer.json ├── tests ├── MessageHandler │ ├── Producer.php │ ├── LostMessagesConsumer.php │ └── Consumer.php ├── Queue │ ├── Definition.php │ └── Cleanup.php ├── MessageEnvelope.php └── LoggedPredisClient │ └── LoggedPredisClient.php └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /vendor/* 2 | composer.lock 3 | bin/ 4 | .php_cs.cache -------------------------------------------------------------------------------- /.atoum.php: -------------------------------------------------------------------------------- 1 | addTestsFromDirectory(__DIR__.'/tests'); 4 | -------------------------------------------------------------------------------- /.php_cs: -------------------------------------------------------------------------------- 1 | getFinder() 6 | ->in([ 7 | __DIR__.'/src' 8 | ])->exclude([ 9 | 'tests' 10 | ]); 11 | 12 | return $config; -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Coding Style 2 | 3 | cs: 4 | ./bin/php-cs-fixer fix --dry-run --stop-on-violation --diff 5 | 6 | cs-fix: 7 | ./bin/php-cs-fixer fix 8 | 9 | cs-ci: 10 | ./bin/php-cs-fixer fix --dry-run --using-cache=no --verbose -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: php 2 | 3 | services: 4 | - redis-server 5 | 6 | php: 7 | - 7.1 8 | 9 | before_script: 10 | - phpenv config-rm xdebug.ini 11 | - wget http://getcomposer.org/composer.phar 12 | - php composer.phar install --dev 13 | 14 | script: 15 | - make cs-ci 16 | - bin/atoum -d tests 17 | -------------------------------------------------------------------------------- /src/LoggedPredisClient/PredisClientInterface.php: -------------------------------------------------------------------------------- 1 | =7.1", 19 | "predis/predis": "^1.1.1" 20 | }, 21 | "require-dev": { 22 | "atoum/atoum": "~3.0.0", 23 | "m6web/php-cs-fixer-config": "^1.0", 24 | "m6web/redis-mock": "^3.3.1" 25 | }, 26 | "autoload": { 27 | "psr-4": { "M6Web\\Component\\RedisMessageBroker\\": "src/" } 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /tests/MessageHandler/Producer.php: -------------------------------------------------------------------------------- 1 | if( 16 | $queue = new Definition('queue1'), 17 | $redisClient = new \mock\Predis\Client(), 18 | $producer = $this->newTestedInstance($queue, $redisClient), 19 | $message = new MessageEnvelope(uniqid(), 'message in the bottle') 20 | ) 21 | ->then 22 | ->integer($producer->publishMessage($message)) 23 | ->mock($redisClient) 24 | ->call('lpush') 25 | ->once() 26 | ; 27 | } 28 | } -------------------------------------------------------------------------------- /src/MessageHandler/Producer.php: -------------------------------------------------------------------------------- 1 | redisClient->lpush($this->queue->getARandomListName(), $message->getStorableValue($this->doMessageCompression)); 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/Event/PredisEvent.php: -------------------------------------------------------------------------------- 1 | command = $command; 22 | $this->executionTime = $executionTime; 23 | } 24 | 25 | public function getCommand(): string 26 | { 27 | return $this->command; 28 | } 29 | 30 | public function getExecutionTime(): float 31 | { 32 | return $this->executionTime; 33 | } 34 | 35 | /** 36 | * Get execution time in milliseconds (used by statsd). 37 | * 38 | * @return float 39 | */ 40 | public function getTiming(): float 41 | { 42 | return $this->executionTime * 1000; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /tests/Queue/Definition.php: -------------------------------------------------------------------------------- 1 | if($t = $this->newTestedInstance('raoul', 4)) 13 | ->then 14 | ->array($l = $t->getListNames()) 15 | ->isIdentiCalTo( 16 | ['raoul_list__1', 'raoul_list__2', 'raoul_list__3', 'raoul_list__4' ] 17 | ) 18 | ; 19 | } 20 | 21 | public function testConstructor() 22 | { 23 | $this 24 | ->exception( 25 | function () { 26 | $this->newTestedInstance('raoul', 0); 27 | } 28 | )->isInstanceOf('\InvalidArgumentException'); 29 | 30 | $this 31 | ->exception( 32 | function () { 33 | $this->newTestedInstance('', 2); 34 | } 35 | )->isInstanceOf('\InvalidArgumentException'); 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /src/Queue/AbstractQueueTool.php: -------------------------------------------------------------------------------- 1 | queue = $queue; 34 | $this->redisClient = new LoggedPredisClient($redisClient); 35 | } 36 | 37 | /** 38 | * @param \Closure $closure 39 | * 40 | * @return $this 41 | */ 42 | public function setEventCallback(\Closure $closure) 43 | { 44 | $this->eventCallback = $closure; 45 | $this->redisClient->setEventCallback($closure); 46 | 47 | return $this; 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/Event/ConsumerEvent.php: -------------------------------------------------------------------------------- 1 | eventName = $eventName; 34 | $this->nbMessage = $nbMessage; 35 | $this->listName = $listName; 36 | } 37 | 38 | public function getEventName(): string 39 | { 40 | return $this->eventName; 41 | } 42 | 43 | public function getNbMessage(): int 44 | { 45 | return $this->nbMessage; 46 | } 47 | 48 | public function getListName(): string 49 | { 50 | return $this->listName; 51 | } 52 | 53 | public function getValue(): int 54 | { 55 | return $this->nbMessage; 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /src/MessageHandler/AbstractMessageHandler.php: -------------------------------------------------------------------------------- 1 | queue = $queue; 38 | $this->doMessageCompression = $doMessageCompression; 39 | $this->redisClient = new LoggedPredisClient($redisClient); 40 | } 41 | 42 | /** 43 | * @param \Closure $closure 44 | * 45 | * @return $this 46 | */ 47 | public function setEventCallback(\Closure $closure): self 48 | { 49 | $this->eventCallback = $closure; 50 | $this->redisClient->setEventCallback($closure); 51 | 52 | return $this; 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /tests/MessageEnvelope.php: -------------------------------------------------------------------------------- 1 | if($t = $this->newTestedInstance(uniqid(), 'raoul the message', \DateTime::createFromFormat('Y-m-d H:i:s', '2017-07-10 14:05:00'))) 15 | ->then 16 | ->object($t->getCreatedAt()) 17 | ->isInstanceOf('DateTime') 18 | ->isEqualTo(\DateTime::createFromFormat('Y-m-d H:i:s', '2017-07-10 14:05:00')) 19 | ->and 20 | ->string($t->getMessage()) 21 | ->isEqualTo('raoul the message') 22 | ; 23 | } 24 | 25 | public function testStore(bool $doCompression, $message) 26 | { 27 | $this 28 | ->if($messageEnvelope = $this->newTestedInstance(uniqid(), $message)) 29 | ->then 30 | ->string($storedMessage = $messageEnvelope->getStorableValue($doCompression)) 31 | ->and 32 | ->object($transportedMessage = Base::unstoreMessage($storedMessage, $doCompression)) 33 | ->isEqualTo($messageEnvelope) 34 | ; 35 | } 36 | 37 | public function testUnstoreBadMessage() 38 | { 39 | $this 40 | ->if($message = serialize(new \DateTime())) 41 | ->then 42 | ->variable($transportedMessage = Base::unstoreMessage($message)) 43 | ->isNull() 44 | ; 45 | } 46 | 47 | protected function testStoreDataProvider() 48 | { 49 | return [ 50 | [true, 'raoul the message'], 51 | [false, 'raoul the message'], 52 | [true, new \SplStack()], 53 | [false, new \SplStack()], 54 | ]; 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Queue/Cleanup.php: -------------------------------------------------------------------------------- 1 | getCreatedAt()->format('U')) > $maxAge; // if message was created before $maxAge 29 | }; 30 | $r += $this->cleanMessage($this->queue->getWorkingLists($this->redisClient), $hasToDelete); 31 | 32 | if ($eraseReadyMessages) { 33 | $r += $this->cleanMessage($this->queue->getListNames(), $hasToDelete); 34 | } 35 | 36 | return $r; 37 | } 38 | 39 | /** 40 | * this method delete deadletter lists 41 | * 42 | * @return int number of deleted messages in dead letter lists 43 | */ 44 | public function cleanDeadLetterLists(): int 45 | { 46 | $r = 0; 47 | 48 | foreach ($this->queue->getDeadLetterLists($this->redisClient) as $deadLetterList) { 49 | $this->redisClient->multi(); 50 | $this->redisClient->llen($deadLetterList); 51 | $this->redisClient->del($deadLetterList); 52 | $result = $this->redisClient->exec(); 53 | 54 | $r += $result[0] ?? 0; 55 | } 56 | 57 | return $r; 58 | } 59 | 60 | private function cleanMessage(iterable $lists, callable $hasToDelete): int 61 | { 62 | $r = 0; 63 | foreach ($lists as $list) { 64 | $listLen = $this->redisClient->llen($list); 65 | for ($i = 1; $i <= $listLen; $i++) { 66 | $message = $this->redisClient->rpoplpush($list, $list); 67 | if ($hasToDelete(MessageEnvelope::unstoreMessage($message))) { 68 | $r += $this->redisClient->lrem($list, 0, $message); 69 | } 70 | } 71 | } 72 | 73 | return $r; 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/Queue/Definition.php: -------------------------------------------------------------------------------- 1 | name = $name; 40 | $this->listCount = abs($listCount); 41 | } 42 | 43 | /** 44 | * get physical redis list names 45 | */ 46 | public function getListNames(): array 47 | { 48 | $t = range(1, $this->listCount); 49 | array_walk( 50 | $t, function (&$v) { 51 | $v = $this->getListPrefixName().'_'.$v; 52 | } 53 | ); 54 | 55 | return $t; 56 | } 57 | 58 | public function getARandomListName(): string 59 | { 60 | return $this->getListNames()[array_rand($this->getListNames())]; 61 | } 62 | 63 | public function getName(): string 64 | { 65 | return $this->name; 66 | } 67 | 68 | public function getListPrefixName(): string 69 | { 70 | return $this->name.'_list_'; 71 | } 72 | 73 | public function getWorkingListPrefixName(): string 74 | { 75 | return $this->name.'_working_list_'; 76 | } 77 | 78 | public function getDeadLetterListName(): string 79 | { 80 | return $this->name.'_dead_letter_list_'; 81 | } 82 | 83 | /** 84 | * scan the database to get the list of working list for the queue 85 | * 86 | * @param ClientInterface $client 87 | * 88 | * @return \Iterator 89 | */ 90 | public function getWorkingLists(ClientInterface $client): \Iterator 91 | { 92 | return new Iterator\Keyspace($client, $this->getWorkingListPrefixName().'*'); // SCAN the database 93 | } 94 | 95 | public function getQueueLists(ClientInterface $client): \Iterator 96 | { 97 | return new Iterator\Keyspace($client, $this->getListPrefixName().'*'); // SCAN the database 98 | } 99 | 100 | public function getDeadLetterLists(ClientInterface $client): \Iterator 101 | { 102 | return new Iterator\Keyspace($client, $this->getDeadLetterListName().'*'); // SCAN the database 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/MessageEnvelope.php: -------------------------------------------------------------------------------- 1 | id = $id; 44 | $this->message = $message; 45 | $this->createdAt = $createdAt; 46 | if (null === $this->createdAt) { 47 | $this->createdAt = new \DateTime(); 48 | } 49 | $this->updatedAt = $this->createdAt; 50 | $this->retry = 0; 51 | } 52 | 53 | public function __clone() 54 | { 55 | if (is_object($this->message)) { 56 | $this->message = clone $this->message; 57 | } 58 | } 59 | 60 | public function getId(): string 61 | { 62 | return $this->id; 63 | } 64 | 65 | public function setUpdatedAt(\DateTime $dateTime): self 66 | { 67 | $this->createdAt = $dateTime; 68 | 69 | return $this; 70 | } 71 | 72 | public function getCreatedAt(): \DateTime 73 | { 74 | return $this->createdAt; 75 | } 76 | 77 | public function getUpdatedAt(): \DateTime 78 | { 79 | return $this->updatedAt; 80 | } 81 | 82 | public function incrementRetry(): self 83 | { 84 | $this->setUpdatedAt(new \DateTime()); 85 | 86 | $this->retry++; 87 | 88 | return $this; 89 | } 90 | 91 | public function getRetry(): int 92 | { 93 | return $this->retry; 94 | } 95 | 96 | public function getMessage() 97 | { 98 | return $this->message; 99 | } 100 | 101 | public function getStorableValue(bool $doCompression = false): string 102 | { 103 | $serializedValue = \serialize($this); 104 | 105 | if ($doCompression) { 106 | return \gzdeflate($serializedValue, 9); 107 | } 108 | 109 | return $serializedValue; 110 | } 111 | 112 | public static function unstoreMessage(string $storedMessage, bool $isCompressed = false): ?self 113 | { 114 | // If the message is compressed, uncompress it. 115 | if ($isCompressed && ($messageUncompressed = @gzinflate($storedMessage)) !== false) { 116 | $storedMessage = $messageUncompressed; 117 | } 118 | 119 | $unserializeMessage = \unserialize($storedMessage); 120 | 121 | return ($unserializeMessage instanceof self) ? $unserializeMessage : null; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /src/Queue/Inspector.php: -------------------------------------------------------------------------------- 1 | queue->getListNames() as $listName) { 20 | $i += $this->redisClient->llen($listName); 21 | } 22 | 23 | return $i; 24 | } 25 | 26 | public function countListQueues(): int 27 | { 28 | return iterator_count($this->queue->getQueueLists($this->redisClient)); 29 | } 30 | 31 | public function countWorkingListQueues(): int 32 | { 33 | return iterator_count($this->queue->getWorkingLists($this->redisClient)); 34 | } 35 | 36 | public function countInProgressMessages(): int 37 | { 38 | $i = 0; 39 | foreach ($this->queue->getWorkingLists($this->redisClient) as $list) { 40 | $i += $this->redisClient->llen($list); 41 | } 42 | 43 | return $i; 44 | } 45 | 46 | public function countInErrorMessages(): int 47 | { 48 | $i = 0; 49 | foreach ($this->queue->getDeadLetterLists($this->redisClient) as $list) { 50 | $i += $this->redisClient->llen($list); 51 | } 52 | 53 | return $i; 54 | } 55 | 56 | /** 57 | * Get the REDIS memory usage in bytes 58 | * 59 | * @see https://redis.io/commands/INFO 60 | */ 61 | public function getRedisMemoryUsage(): ?int 62 | { 63 | $redisInfoMemory = $this->redisClient->info(self::REDIS_INFO_MEMORY); 64 | 65 | if (!empty($redisInfoMemory) && isset($redisInfoMemory[self::REDIS_INFO_MEMORY]['used_memory'])) { 66 | return $redisInfoMemory[self::REDIS_INFO_MEMORY]['used_memory']; 67 | } 68 | 69 | return null; 70 | } 71 | 72 | /** 73 | * Get the REDIS total memory in bytes 74 | * 75 | * @see https://redis.io/commands/INFO 76 | */ 77 | public function getRedisMemoryTotal(): ?int 78 | { 79 | $redisInfoMemory = $this->redisClient->info(self::REDIS_INFO_MEMORY); 80 | 81 | if (!empty($redisInfoMemory) && isset($redisInfoMemory[self::REDIS_INFO_MEMORY]['total_system_memory'])) { 82 | return $redisInfoMemory[self::REDIS_INFO_MEMORY]['total_system_memory']; 83 | } 84 | 85 | return null; 86 | } 87 | 88 | /** 89 | * Calculate the REDIS free memory in bytes 90 | * 91 | * @see https://redis.io/commands/INFO 92 | */ 93 | public function getRedisMemoryFree(): ?int 94 | { 95 | $redisInfoMemory = $this->redisClient->info(self::REDIS_INFO_MEMORY); 96 | 97 | if (!empty($redisInfoMemory) && isset($redisInfoMemory[self::REDIS_INFO_MEMORY]['total_system_memory']) && $redisInfoMemory[self::REDIS_INFO_MEMORY]['used_memory']) { 98 | return $redisInfoMemory[self::REDIS_INFO_MEMORY]['total_system_memory'] - $redisInfoMemory[self::REDIS_INFO_MEMORY]['used_memory']; 99 | } 100 | 101 | return null; 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /tests/LoggedPredisClient/LoggedPredisClient.php: -------------------------------------------------------------------------------- 1 | if( 13 | $predisClient = $this->getPredisClientMock(), 14 | $loggedPredisClient = $this->newTestedInstance($predisClient) 15 | ) 16 | ->when($response = $loggedPredisClient->del(['some', 'keys'])) 17 | ->then 18 | ->mock($predisClient)->call('del')->once(); 19 | } 20 | 21 | public function testExec() 22 | { 23 | $this 24 | ->if( 25 | $predisClient = $this->getPredisClientMock(), 26 | $loggedPredisClient = $this->newTestedInstance($predisClient) 27 | ) 28 | ->when($response = $loggedPredisClient->exec()) 29 | ->then 30 | ->mock($predisClient)->call('exec')->once(); 31 | } 32 | 33 | public function testLlen() 34 | { 35 | $this 36 | ->if( 37 | $predisClient = $this->getPredisClientMock(), 38 | $loggedPredisClient = $this->newTestedInstance($predisClient) 39 | ) 40 | ->when($response = $loggedPredisClient->llen('key')) 41 | ->then 42 | ->mock($predisClient)->call('llen')->once(); 43 | } 44 | 45 | public function testLpush() 46 | { 47 | $this 48 | ->if( 49 | $predisClient = $this->getPredisClientMock(), 50 | $loggedPredisClient = $this->newTestedInstance($predisClient) 51 | ) 52 | ->when($response = $loggedPredisClient->lpush('key', ['some', 'values'])) 53 | ->then 54 | ->mock($predisClient)->call('lpush')->once(); 55 | } 56 | 57 | public function testLrem() 58 | { 59 | $this 60 | ->if( 61 | $predisClient = $this->getPredisClientMock(), 62 | $loggedPredisClient = $this->newTestedInstance($predisClient) 63 | ) 64 | ->when($response = $loggedPredisClient->lrem('key', 100, 'value')) 65 | ->then 66 | ->mock($predisClient)->call('lrem')->once(); 67 | } 68 | 69 | public function testMulti() 70 | { 71 | $this 72 | ->if( 73 | $predisClient = $this->getPredisClientMock(), 74 | $loggedPredisClient = $this->newTestedInstance($predisClient) 75 | ) 76 | ->when($response = $loggedPredisClient->multi()) 77 | ->then 78 | ->mock($predisClient)->call('multi')->once(); 79 | } 80 | 81 | public function testRpop() 82 | { 83 | $this 84 | ->if( 85 | $predisClient = $this->getPredisClientMock(), 86 | $loggedPredisClient = $this->newTestedInstance($predisClient) 87 | ) 88 | ->when($response = $loggedPredisClient->rpop('key')) 89 | ->then 90 | ->mock($predisClient)->call('rpop')->once(); 91 | } 92 | 93 | public function testRpoplpush() 94 | { 95 | $this 96 | ->if( 97 | $predisClient = $this->getPredisClientMock(), 98 | $loggedPredisClient = $this->newTestedInstance($predisClient) 99 | ) 100 | ->when($response = $loggedPredisClient->rpoplpush('source', 'destination')) 101 | ->then 102 | ->mock($predisClient)->call('rpoplpush')->once(); 103 | } 104 | 105 | protected function getPredisClientMock() 106 | { 107 | $this->mockGenerator->orphanize('__construct'); 108 | $predisClientMock = new \mock\Predis\Client(); 109 | $predisClientMock->getMockController()->del = 1; 110 | $predisClientMock->getMockController()->exec = []; 111 | $predisClientMock->getMockController()->llen = 1; 112 | $predisClientMock->getMockController()->lpush = 1; 113 | $predisClientMock->getMockController()->lrem = 1; 114 | $predisClientMock->getMockController()->multi = []; 115 | $predisClientMock->getMockController()->rpop = 'something'; 116 | $predisClientMock->getMockController()->rpoplpush = 'something'; 117 | 118 | return $predisClientMock; 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /tests/Queue/Cleanup.php: -------------------------------------------------------------------------------- 1 | if( 20 | $queue = new Definition('queue1'.uniqid('test_redis', false)), 21 | $redisClient = new \Predis\Client(), 22 | $producer = new Producer($queue, $redisClient), 23 | $message = new MessageEnvelope(uniqid(), 'message in the bottle 1'), 24 | $message2 = new MessageEnvelope(uniqid(), 'message in the bottle 2'), 25 | $producer->publishMessage($message), 26 | $producer->publishMessage($message2), 27 | $inspector = new Inspector($queue, $redisClient), 28 | $cleanup = $this->newTestedInstance($queue, $redisClient) 29 | ) 30 | ->and( 31 | sleep(4) // build two 4s old message 32 | ) 33 | ->then 34 | ->integer($inspector->countReadyMessages()) 35 | ->isEqualTo(2) 36 | 37 | ->and 38 | ->integer($cleanup->cleanWorkingListsOldMessages(2, true)) 39 | ->isEqualTo(2) // two messages cleaned 40 | ->and 41 | ->integer($inspector->countReadyMessages()) 42 | ->isEqualTo(0) 43 | ; 44 | 45 | $this 46 | ->if( 47 | $queue = new Definition('queue2'.uniqid('test_redis', false)), 48 | $redisClient = new \Predis\Client(), 49 | $producer = new Producer($queue, $redisClient), 50 | $message = new MessageEnvelope(uniqid(), 'message in the bottle 1'), 51 | $message2 = new MessageEnvelope(uniqid(), 'message in the bottle 2'), 52 | $producer->publishMessage($message), 53 | $producer->publishMessage($message2), 54 | $inspector = new Inspector($queue, $redisClient), 55 | $cleanup = $this->newTestedInstance($queue, $redisClient), 56 | $consumer = new Consumer($queue, $redisClient, uniqid('test_consumer', false)), 57 | $consumer->setNoAutoAck() 58 | ) 59 | ->and( 60 | $consumer->getMessageEnvelope(), // get one message 61 | sleep(4) // build two 4s old message 62 | ) 63 | ->and 64 | ->integer($cleanup->cleanWorkingListsOldMessages(2)) // clean in progress messages 65 | ->isEqualTo(1) // one message cleaned 66 | ->and 67 | ->integer($inspector->countReadyMessages()) 68 | ->isEqualTo(1) // ready message is still here 69 | ; 70 | } 71 | 72 | public function testCleanDeadLetterLists() 73 | { 74 | $this 75 | ->if( 76 | $queue = new Definition('queue1'.uniqid('test_redis', false)), 77 | $redisClient = new \Predis\Client(), 78 | $producer = new Producer($queue, $redisClient), 79 | 80 | $message = new MessageEnvelope(uniqid(), 'message in the bottle 1'), 81 | $message->incrementRetry(), 82 | $producer->publishMessage($message), 83 | 84 | $inspector = new Inspector($queue, $redisClient), 85 | 86 | $consumer = new Consumer($queue, $redisClient, 'testConsumer2'.uniqid('test_redis', false)), 87 | $consumer->setNoAutoAck(), 88 | $consumer->getMessageEnvelope(), 89 | 90 | 91 | sleep(2), 92 | $lostMessagesConsumer = new LostMessagesConsumer($queue, $redisClient, 1, 1), 93 | $lostMessagesConsumer->requeueOldMessages(), 94 | 95 | $cleanup = $this->newTestedInstance($queue, $redisClient) 96 | ) 97 | ->then 98 | ->integer($inspector->countInErrorMessages()) 99 | ->isEqualTo(1) 100 | 101 | ->and 102 | ->integer($cleanup->cleanDeadLetterLists()) 103 | ->isEqualTo(1) // one message cleaned 104 | ->and 105 | ->integer($inspector->countReadyMessages()) 106 | ->isEqualTo(0) 107 | ->integer($inspector->countInProgressMessages()) 108 | ->isEqualTo(0) 109 | ; 110 | } 111 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RedisMessageBroker 2 | 3 | This component will help you to build a messages brocker system over a [redis](redis.io) backend. It will take advantage of the redis cluster capabilities with the possibility to shard messages into several redis lists while producing messages. Consumer, in no auto-ack mode, implements a working list to be sure not to loose any messages when processing fail. 4 | 5 | By design, producing is super fast (one Redis command) and consuming can be slow if you want to ack messages manually. 6 | 7 | You should use it with Redis >= 2.8. 8 | 9 | ## usage 10 | 11 | ### producer 12 | 13 | ```php 14 | publishMessage($message); 24 | 25 | ``` 26 | 27 | ### consumer 28 | 29 | Consumer should be wrapped in a worker. A unique Id should be pass to the consumer constructor. If you work with a worker, uniqId has to be constant per worker. 30 | 31 | ```php 32 | getMessageEnvelope(); 41 | 42 | ``` 43 | 44 | ### inspector 45 | 46 | Inspector methods allow you to count the messages in ready or processing in a queue. 47 | 48 | ```php 49 | countInProgressMessages(); 58 | $countReady = $inspector->countReadyMessages(); 59 | ``` 60 | 61 | ### cleanup 62 | 63 | Cleanup methods let you perform a cleanup in the message queue. Cleanup is very slow as all the message in the queue will be scanned. 64 | 65 | ```php 66 | cleanOldMessages( 75 | 3600, // erase messages older than 3600 seconds 76 | true // clean message in the ready queue too. Mandatory use if you are in no-autoack mode 77 | ); 78 | ``` 79 | 80 | ## queue option 81 | 82 | To avoid hotpsots you can shard a queue on several lists : 83 | 84 | ```php 85 | $queue = new RedisMessageBroker\Queue\Definition('raoul', 10); // shard on 10 lists 86 | ``` 87 | 88 | In this mode, messages will be written and read among the 10 lists. FIFO is no more guaranteed. 89 | 90 | ## consumer options 91 | 92 | ### manual message acknowledgment 93 | 94 | with `setNoAutoAck()` 95 | 96 | 97 | ```php 98 | setNoAutoAck(); 106 | 107 | $message = $consumer->getMessageEnvelope(); 108 | if ($message) { 109 | // do something with the message 110 | $consumer->ack($message); // erase the message from the working list 111 | } 112 | ``` 113 | 114 | ### look for old messages not acked by consumers 115 | 116 | Each consumer got an unique Id defined during the construction of the object. This Id allow the consumer to define a unique working list where a message is stored between the `getMessage` and the `ack`. 117 | Is it possible to use LostMessageConsumer class to look on consumer working lists and move message more than x second old (`messageTtl` parameter) from those lists to a queue list. 118 | The `maxRetry` parameter will put a message in a dead letter list when the retry number is reached. 119 | 120 | ```php 121 | requeueOldMessages(); 129 | ``` 130 | 131 | `messageTtl` parameter need to be superior of your max time to process a message. (Otherwise it will consider a message old when its processing) 132 | -------------------------------------------------------------------------------- /src/MessageHandler/LostMessagesConsumer.php: -------------------------------------------------------------------------------- 1 | messageTtl = $messageTtl; 35 | $this->maxRetry = $maxRetry; 36 | parent::__construct($queue, $redisClient, $compressMessages); 37 | } 38 | 39 | public function setMessageTtl(int $messageTtl): void 40 | { 41 | $this->messageTtl = $messageTtl; 42 | } 43 | 44 | /** 45 | * Requeue message with date superior to message ttl 46 | * Then max retry reached, message is put in dead letter list 47 | * 48 | * @param int|null $maxExecuteTime 49 | */ 50 | public function requeueOldMessages(?int $maxExecuteTime = null): void 51 | { 52 | $workingLists = iterator_to_array($this->queue->getWorkingLists($this->redisClient)); 53 | 54 | $startTime = time(); 55 | 56 | shuffle($workingLists); 57 | 58 | $restTime = null; 59 | 60 | foreach ($workingLists as $workingList) { 61 | if (null !== $maxExecuteTime) { 62 | $restTime = $maxExecuteTime - (time() - $startTime); 63 | 64 | if ($restTime <= 0) { 65 | break; 66 | } 67 | } 68 | 69 | $this->requeueOldMessageFromList($workingList, $restTime); 70 | } 71 | } 72 | 73 | /** 74 | * Requeue old messages from $list, and remove them if 75 | * $messageTTL or time is exceed. 76 | * 77 | * @param string $list 78 | * @param int|null $maxExecutionTime 79 | */ 80 | protected function requeueOldMessageFromList(string $list, ?int $maxExecutionTime): void 81 | { 82 | $startTime = time(); 83 | 84 | $maxMessages = $this->redisClient->llen($list); 85 | 86 | while (($maxMessages-- > 0) && ($storredMessage = $this->redisClient->rpoplpush($list, $list))) { 87 | $message = empty($storredMessage) ? null : MessageEnvelope::unstoreMessage($storredMessage, $this->doMessageCompression); 88 | 89 | if (!empty($message)) { 90 | // Message is old enough 91 | if ((time() - $message->getUpdatedAt()->format('U')) > $this->messageTtl) { 92 | // Message is on the list 93 | if ($this->redisClient->lrem($list, 0, $storredMessage)) { 94 | // Message is not retry too many times 95 | if ($message->getRetry() < $this->maxRetry) { 96 | // Add it on the Queue list (again) 97 | $this->addMessageInQueueList($message); 98 | } else { 99 | // Message is retried too many times; add it to the deadLetterList 100 | $this->addMessageInDeadLetterList($message); 101 | } 102 | } 103 | } // Else : Message isn't too old : Do nothing. 104 | } else { 105 | // Message is empty, Remove it. 106 | $this->redisClient->lrem($list, 0, $storredMessage); 107 | } 108 | 109 | if (null !== $maxExecutionTime && $maxExecutionTime - (time() - $startTime) <= 0) { 110 | break; 111 | } 112 | } 113 | 114 | $this->redisClient->rpoplpush($list, $list); 115 | } 116 | 117 | protected function addMessageInDeadLetterList(MessageEnvelope $message): void 118 | { 119 | $list = $this->queue->getDeadLetterListName(); 120 | $this->redisClient->lpush($list, $message->getStorableValue($this->doMessageCompression)); 121 | 122 | if ($this->eventCallback) { 123 | ($this->eventCallback)(new ConsumerEvent(ConsumerEvent::MAX_RETRY_REACHED, 1, $list)); 124 | } 125 | } 126 | 127 | protected function addMessageInQueueList(MessageEnvelope $message): void 128 | { 129 | $message->incrementRetry(); 130 | 131 | $queueName = $this->queue->getARandomListName(); 132 | $this->redisClient->lpush($queueName, $message->getStorableValue($this->doMessageCompression)); 133 | 134 | if ($this->eventCallback) { 135 | ($this->eventCallback)(new ConsumerEvent(ConsumerEvent::REQUEUE_OLD_MESSAGE, 1, $queueName)); 136 | } 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /src/MessageHandler/Consumer.php: -------------------------------------------------------------------------------- 1 | uniqueId = $uniqueId ?? uniqid(); 37 | 38 | parent::__construct($queue, $redisClient, $compressMessages); 39 | } 40 | 41 | public function setNoAutoAck() 42 | { 43 | $this->autoAck = false; 44 | } 45 | 46 | /** 47 | * Get last MessageEnvelope 48 | */ 49 | public function getMessageEnvelope(): ?MessageEnvelope 50 | { 51 | $lists = iterator_to_array($this->queue->getQueueLists($this->redisClient)); 52 | shuffle($lists); 53 | 54 | // #1 autoack 55 | if ($this->autoAck) { 56 | foreach ($lists as $list) { 57 | if ($message = $this->redisClient->rpop($list)) { 58 | return MessageEnvelope::unstoreMessage($message, $this->doMessageCompression); 59 | } 60 | } 61 | } 62 | 63 | // #2 no-autoack - messages have to be acked manually via ->ack(message) 64 | // grab something in the queue and put it in the workinglist while returning the message 65 | foreach ($lists as $list) { 66 | if ($message = $this->redisClient->rpoplpush($list, $this->getWorkingList())) { 67 | return MessageEnvelope::unstoreMessage($message, $this->doMessageCompression); 68 | } 69 | } 70 | 71 | return null; 72 | } 73 | 74 | /** 75 | * the working list has to be unique per consumer worker 76 | */ 77 | protected function getWorkingList(): string 78 | { 79 | return $this->queue->getWorkingListPrefixName().'-'.$this->uniqueId; 80 | } 81 | 82 | /** 83 | * @param MessageEnvelope $message this message will be acked in the working lists 84 | * @param int $count number of removed first elements from the list stored 85 | * 86 | * @return int the number of messages acked 87 | */ 88 | public function ack(MessageEnvelope $message, int $count = 0): int 89 | { 90 | $nbMessageAck = $this->removeMessageInWorkingList($message, $count); 91 | 92 | if ($this->eventCallback) { 93 | ($this->eventCallback)(new ConsumerEvent(ConsumerEvent::ACK_EVENT, $nbMessageAck, $this->getWorkingList())); 94 | } 95 | 96 | return $nbMessageAck; 97 | } 98 | 99 | /** 100 | * @param MessageEnvelope $message 101 | * @param int $count number of removed first elements from the list stored 102 | * 103 | * @see https://redis.io/commands/lrem for more informations on the $count parameter 104 | * 105 | * @return int the number of messages deleted 106 | */ 107 | protected function removeMessageInWorkingList(MessageEnvelope $message, int $count = 0): int 108 | { 109 | $storedMessage = $message->getStorableValue($this->doMessageCompression); 110 | 111 | return $this->redisClient->lrem($this->getWorkingList(), $count, $storedMessage); 112 | } 113 | 114 | /** 115 | * @return int the number of messages acked 116 | */ 117 | public function ackAll(): int 118 | { 119 | $nbMessageAck = $this->removeList($this->getWorkingList()); 120 | 121 | if ($this->eventCallback) { 122 | ($this->eventCallback)(new ConsumerEvent(ConsumerEvent::ACK_EVENT, $nbMessageAck, $this->getWorkingList())); 123 | } 124 | 125 | return $nbMessageAck; 126 | } 127 | 128 | /** 129 | * @param string $list 130 | * 131 | * @return int nb of elements in deleted list 132 | */ 133 | protected function removeList(string $list) 134 | { 135 | $this->redisClient->multi(); 136 | $this->redisClient->llen($list); 137 | $this->redisClient->del([$list]); 138 | $result = $this->redisClient->exec(); 139 | 140 | return $result[0] ?? 0; 141 | } 142 | 143 | /** 144 | * @param MessageEnvelope $message 145 | * @param int $count number of removed first elements from the list stored 146 | * 147 | * @return int the number of messages unack 148 | */ 149 | public function unack(MessageEnvelope $message, int $count = 0): int 150 | { 151 | $queueName = $this->queue->getARandomListName(); 152 | 153 | if ($nbMessageUnack = $this->removeMessageInWorkingList($message, $count)) { 154 | $message->incrementRetry(); 155 | 156 | $this->redisClient->lpush($queueName, [$message->getStorableValue($this->doMessageCompression)]); 157 | } 158 | 159 | if ($this->eventCallback) { 160 | ($this->eventCallback)(new ConsumerEvent(ConsumerEvent::UNACK_EVENT, $nbMessageUnack, $queueName)); 161 | } 162 | 163 | return $nbMessageUnack; 164 | } 165 | 166 | /** 167 | * @return int the number of messages unack 168 | */ 169 | public function unackAll(): int 170 | { 171 | $queueName = $this->queue->getARandomListName(); 172 | $nbMessageUnack = 0; 173 | 174 | do { 175 | $message = $this->redisClient->rpop($this->getWorkingList()); 176 | 177 | if (!empty($message)) { 178 | $messageEnvelope = MessageEnvelope::unstoreMessage($message, $this->doMessageCompression); 179 | 180 | $this->redisClient->lpush($queueName, [$messageEnvelope->getStorableValue($this->doMessageCompression)]); 181 | 182 | $nbMessageUnack++; 183 | } 184 | } while (!empty($message)); 185 | 186 | if ($this->eventCallback) { 187 | ($this->eventCallback)(new ConsumerEvent(ConsumerEvent::UNACK_EVENT, $nbMessageUnack, $queueName)); 188 | } 189 | 190 | return $nbMessageUnack; 191 | } 192 | 193 | public function getQueueListsLength(): array 194 | { 195 | $queueListsLength = []; 196 | foreach ($this->queue->getQueueLists($this->redisClient) as $queueName) { 197 | $queueListsLength[$queueName] = $this->redisClient->llen($queueName); 198 | } 199 | 200 | return $queueListsLength; 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /tests/MessageHandler/LostMessagesConsumer.php: -------------------------------------------------------------------------------- 1 | if( 18 | $queue = new Definition('queue4'.uniqid('test_redis', false)), // unique queue with a lot of list 19 | $pRedisClient = $this->getPredisMock(), 20 | $producer = new Producer($queue, $pRedisClient), 21 | $message = new MessageEnvelope(uniqid(), 'message in the bottle'), 22 | $producer->publishMessage($message), // publish a message 23 | $inspector = new Inspector($queue, $pRedisClient), 24 | 25 | $consumer = new Consumer($queue, $pRedisClient, 'testConsumer2'.uniqid('test_redis', false)), 26 | $consumer->setNoAutoAck(), 27 | 28 | $consumer->getMessageEnvelope() 29 | ) 30 | ->then( 31 | sleep(10), // build an 10s old message 32 | $lostMessagesConsumer = new \M6Web\Component\RedisMessageBroker\MessageHandler\LostMessagesConsumer($queue, $pRedisClient, 3600), 33 | $lostMessagesConsumer->requeueOldMessages() 34 | ) 35 | ->then 36 | ->integer($inspector->countReadyMessages()) 37 | ->isEqualTo(0) 38 | ->integer($inspector->countInProgressMessages()) // still in the working list 39 | ->isEqualTo(1) 40 | ->if( 41 | $lostMessagesConsumer = new \M6Web\Component\RedisMessageBroker\MessageHandler\LostMessagesConsumer($queue, $pRedisClient, 1), 42 | $lostMessagesConsumer->requeueOldMessages() 43 | ) // consider now just 5s to get an old message 44 | ->then 45 | ->integer($inspector->countReadyMessages()) 46 | ->isEqualTo(1) 47 | ->integer($inspector->countInProgressMessages()) // still in the working list 48 | ->isEqualTo(0) 49 | ; 50 | } 51 | 52 | public function testRetryMessageList() 53 | { 54 | $this 55 | ->if( 56 | $queue = new Definition('queue-testRetryMessageList'.uniqid('test_redis', false)), // unique queue with a lot of list 57 | $pRedisClient = $this->getPredisMock(), 58 | $producer = new Producer($queue, $pRedisClient), 59 | 60 | $message = new MessageEnvelope(uniqid(), 'message in the bottle'), 61 | 62 | $producer->publishMessage($message), // publish a message 63 | $inspector = new Inspector($queue, $pRedisClient), 64 | 65 | $consumer = new Consumer($queue, $pRedisClient, 'testConsumer2'.uniqid('test_redis', false)), 66 | $consumer->setNoAutoAck(), 67 | $consumer->getMessageEnvelope() 68 | ) 69 | ->then( 70 | sleep(2), // build an 2s old message, retry set to 1 71 | $lostMessagesConsumer = new \M6Web\Component\RedisMessageBroker\MessageHandler\LostMessagesConsumer($queue, $pRedisClient, 3600, 1), 72 | $lostMessagesConsumer->requeueOldMessages() 73 | ) 74 | ->then 75 | ->integer($inspector->countReadyMessages()) // one message is ready. 76 | ->isEqualTo(0) 77 | ->integer($inspector->countInProgressMessages()) // still in the working list 78 | ->isEqualTo(1) 79 | ->integer($inspector->countInErrorMessages()) // No message in deadLetterlist 80 | ->isEqualTo(0) 81 | ; 82 | } 83 | 84 | // Check more messages. 85 | public function testAllOldMessagesAreRequeue() 86 | { 87 | $this 88 | ->if( 89 | $queue = new Definition('queue-testAllOldMessagesAreRequeue'.uniqid('test_redis', false)), // unique queue with a lot of list 90 | $pRedisClient = $this->getPredisMock(), 91 | $producer = new Producer($queue, $pRedisClient), 92 | 93 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle')), // publish a first message 94 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottlea')), // publish a second message 95 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottleb')), // publish a third message 96 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottlec')), // publish a fourth message 97 | 98 | $inspector = new Inspector($queue, $pRedisClient), 99 | 100 | $consumer = new Consumer($queue, $pRedisClient, 'testConsumer2'.uniqid('test_redis', false)), 101 | $consumer->setNoAutoAck(), 102 | 103 | $consumer->getMessageEnvelope(), 104 | $consumer->getMessageEnvelope(), 105 | $consumer->getMessageEnvelope(), 106 | $consumer->getMessageEnvelope() 107 | ) 108 | ->then( 109 | sleep(5), // build an 5s old message 110 | $lostMessagesConsumer = new \M6Web\Component\RedisMessageBroker\MessageHandler\LostMessagesConsumer($queue, $pRedisClient, 3600, 1), 111 | $lostMessagesConsumer->requeueOldMessages() 112 | ) 113 | ->then 114 | ->integer($inspector->countReadyMessages()) 115 | ->isEqualTo(0) 116 | ->integer($inspector->countInProgressMessages()) // still in the working list 117 | ->isEqualTo(4) 118 | ; 119 | } 120 | 121 | public function testDeadLetterList() 122 | { 123 | $this 124 | ->if( 125 | $queue = new Definition('queue-testDeadLetterList'.uniqid('test_redis', false)), // unique queue with a lot of list 126 | $pRedisClient = $this->getPredisMock(), 127 | $producer = new Producer($queue, $pRedisClient), 128 | 129 | $message = new MessageEnvelope(uniqid(), 'message in the bottle'), 130 | $message->incrementRetry(), 131 | 132 | $producer->publishMessage($message), // publish a message 133 | $inspector = new Inspector($queue, $pRedisClient), 134 | 135 | $consumer = new Consumer($queue, $pRedisClient, 'testConsumer2'.uniqid('test_redis', false)), 136 | $consumer->setNoAutoAck(), 137 | $consumer->getMessageEnvelope() 138 | ) 139 | ->then( 140 | sleep(2), // build an 2s old message, retry set to 1 141 | $lostMessagesConsumer = new \M6Web\Component\RedisMessageBroker\MessageHandler\LostMessagesConsumer($queue, $pRedisClient, 1, 1), 142 | $lostMessagesConsumer->requeueOldMessages() 143 | ) 144 | ->then 145 | ->integer($inspector->countReadyMessages()) 146 | ->isEqualTo(0) 147 | ->integer($inspector->countInProgressMessages()) // still in the working list 148 | ->isEqualTo(0) 149 | ->integer($inspector->countInErrorMessages()) // max retry reached, stay in dead letter list 150 | ->isEqualTo(1) 151 | ; 152 | 153 | } 154 | 155 | /** 156 | * Get mock of the redis Client and define a default profile 157 | * 158 | * @return \M6Web\Component\RedisMock\RedisMock 159 | */ 160 | protected function getPredisMock() 161 | { 162 | $factory = new \M6Web\Component\RedisMock\RedisMockFactory(); 163 | $myRedisMock = $factory->getAdapter(\Predis\Client::class, false, false, '', [null, ['profile' => 'default']]); 164 | $myRedisMock->reset(); 165 | 166 | return $myRedisMock; 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /src/LoggedPredisClient/LoggedPredisClient.php: -------------------------------------------------------------------------------- 1 | decorated = $redisClient; 169 | } 170 | 171 | /** 172 | * Decorate all redis calls in order to have execution time for all predis methods. 173 | * 174 | * @param string $method 175 | * @param array $arguments 176 | * 177 | * @return mixed 178 | */ 179 | public function __call($method, $arguments) 180 | { 181 | $microtimeStart = microtime(true); 182 | 183 | $result = call_user_func_array([$this->decorated, $method], $arguments); 184 | 185 | $executionTime = microtime(true) - $microtimeStart; 186 | 187 | if ($this->eventCallback) { 188 | ($this->eventCallback)(new PredisEvent($method, $executionTime)); 189 | } 190 | 191 | return $result; 192 | } 193 | 194 | public function getProfile() 195 | { 196 | return $this->decorated->getProfile(); 197 | } 198 | 199 | public function getOptions() 200 | { 201 | return $this->decorated->getOptions(); 202 | } 203 | 204 | public function connect() 205 | { 206 | return $this->decorated->connect(); 207 | } 208 | 209 | public function disconnect() 210 | { 211 | return $this->decorated->disconnect(); 212 | } 213 | 214 | public function getConnection() 215 | { 216 | return $this->decorated->getConnection(); 217 | } 218 | 219 | public function createCommand($method, $arguments = []) 220 | { 221 | return $this->decorated->createCommand($method, $arguments); 222 | } 223 | 224 | public function executeCommand(CommandInterface $command) 225 | { 226 | return $this->decorated->executeCommand($command); 227 | } 228 | 229 | public function getClientFor($connectionID) 230 | { 231 | return $this->decorated->getClientFor($connectionID); 232 | } 233 | 234 | public function quit() 235 | { 236 | return $this->decorated->quit(); 237 | } 238 | 239 | public function isConnected() 240 | { 241 | return $this->decorated->isConnected(); 242 | } 243 | 244 | public function getConnectionById($connectionID) 245 | { 246 | return $this->decorated->getConnectionById($connectionID); 247 | } 248 | 249 | public function executeRaw(array $arguments, &$error = null) 250 | { 251 | return $this->decorated->executeRaw($arguments, $error); 252 | } 253 | 254 | public function pipeline(/* arguments */) 255 | { 256 | return $this->decorated->pipeline(); 257 | } 258 | 259 | public function transaction(/* arguments */) 260 | { 261 | return $this->decorated->transaction(); 262 | } 263 | 264 | public function pubSubLoop(/* arguments */) 265 | { 266 | return $this->decorated->pubSubLoop(); 267 | } 268 | 269 | public function monitor() 270 | { 271 | return $this->decorated->monitor(); 272 | } 273 | 274 | public function getIterator() 275 | { 276 | return $this->decorated->getIterator(); 277 | } 278 | 279 | /** 280 | * @param \Closure $closure 281 | * 282 | * @return $this 283 | */ 284 | public function setEventCallback(\Closure $closure) 285 | { 286 | $this->eventCallback = $closure; 287 | 288 | return $this; 289 | } 290 | } 291 | -------------------------------------------------------------------------------- /tests/MessageHandler/Consumer.php: -------------------------------------------------------------------------------- 1 | if( 18 | $queue = new Definition('queue1'.uniqid('test_redis', false)), // unique queue 19 | $redisClient = new \Predis\Client(), 20 | $producer = new Producer($queue, $redisClient), 21 | $message = new MessageEnvelope(uniqid(), 'message in the bottle'), 22 | $producer->publishMessage($message) 23 | ) 24 | ->and( 25 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer'.uniqid('test_redis', false)) 26 | ) 27 | ->then 28 | // get the message produced and see if its the same 29 | ->object($newMessage = $consumer->getMessageEnvelope()) 30 | ->and 31 | ->string($newMessage->getMessage()) 32 | ->isIdenticalTo($message->getMessage()) 33 | ->and 34 | ->object($newMessage->getCreatedAt()) 35 | ->isInstanceOf('Datetime') 36 | ->isEqualTo($message->getCreatedAt()) 37 | ; 38 | } 39 | 40 | public function testAck() 41 | { 42 | $this 43 | ->if( 44 | $queue = new Definition('queue2'.uniqid('test_redis', false)), // unique queue 45 | $redisClient = new \mock\Predis\Client(), 46 | $producer = new Producer($queue, $redisClient), 47 | $message = new MessageEnvelope(uniqid(), 'message in the bottle'), 48 | $producer->publishMessage($message) 49 | ) 50 | ->and( 51 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer1'.uniqid('test_redis', false)), 52 | $consumer->setNoAutoAck() 53 | ) 54 | ->and( 55 | $inspector = new Inspector($queue, $redisClient) 56 | ) 57 | ->then 58 | ->object($newMessage = $consumer->getMessageEnvelope()) 59 | ->and 60 | // message should not be in the queue but in the working list 61 | ->integer($inspector->countReadyMessages()) 62 | ->isEqualTo(0) 63 | ->integer($inspector->countInProgressMessages()) 64 | ->isEqualTo(1) 65 | ->then 66 | // ack message 67 | ->integer($consumer->ack($newMessage)) 68 | // message is gone 69 | ->integer($inspector->countReadyMessages()) 70 | ->isEqualTo(0) 71 | ->integer($inspector->countInProgressMessages()) 72 | ->isEqualTo(0) 73 | ; 74 | 75 | $this 76 | ->if($producer->publishMessage($message)) 77 | ->and($newMessage = $consumer->getMessageEnvelope()) 78 | ->then( 79 | // a new consumer should NOT be able to get the message even if is not acked 80 | $consumer2 = $this->newTestedInstance($queue, $redisClient, 'testConsumer2'.uniqid('test_redis', false)), 81 | $consumer2->ack($message) 82 | ) 83 | ->and 84 | ->integer($inspector->countReadyMessages()) 85 | ->isEqualTo(0) 86 | ->integer($inspector->countInProgressMessages()) 87 | ->isEqualTo(1) 88 | ; 89 | } 90 | 91 | public function testAckAll() 92 | { 93 | $this 94 | ->if( 95 | $queue = new Definition('queue2'.uniqid('test_redis', false)), // unique queue 96 | $redisClient = new \mock\Predis\Client(), 97 | 98 | // publish two messages 99 | $producer = new Producer($queue, $redisClient), 100 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle')), 101 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle2')), 102 | 103 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer1'.uniqid('test_redis', false)), 104 | $consumer->setNoAutoAck(), 105 | 106 | $inspector = new Inspector($queue, $redisClient) 107 | ) 108 | ->then 109 | ->object($consumer->getMessageEnvelope()) 110 | ->object($consumer->getMessageEnvelope()) 111 | 112 | // messages should not be in the queue but in the working list 113 | ->integer($inspector->countReadyMessages()) 114 | ->isEqualTo(0) 115 | ->integer($inspector->countInProgressMessages()) 116 | ->isEqualTo(2) 117 | ->and 118 | // ack messages 119 | ->integer($consumer->ackAll()) 120 | // messages is gone 121 | ->integer($inspector->countReadyMessages()) 122 | ->isEqualTo(0) 123 | ->integer($inspector->countInProgressMessages()) 124 | ->isEqualTo(0) 125 | ; 126 | } 127 | 128 | public function testUnack() 129 | { 130 | $this 131 | ->if( 132 | $queue = new Definition('queue2'.uniqid('test_redis', false)), // unique queue 133 | $redisClient = new \mock\Predis\Client(), 134 | 135 | $producer = new Producer($queue, $redisClient), 136 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle')), 137 | 138 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer1'.uniqid('test_redis', false)), 139 | $consumer->setNoAutoAck(), 140 | 141 | $inspector = new Inspector($queue, $redisClient) 142 | ) 143 | ->then 144 | ->object($message = $consumer->getMessageEnvelope()) 145 | 146 | // messages should not be in the queue but in the working list 147 | ->integer($inspector->countReadyMessages()) 148 | ->isEqualTo(0) 149 | ->integer($inspector->countInProgressMessages()) 150 | ->isEqualTo(1) 151 | ->and 152 | // ack messages 153 | ->integer($consumer->unack($message)) 154 | // messages is gone 155 | ->integer($inspector->countReadyMessages()) 156 | ->isEqualTo(1) 157 | ->integer($inspector->countInProgressMessages()) 158 | ->isEqualTo(0) 159 | ; 160 | } 161 | 162 | public function testUnackAll() 163 | { 164 | $this 165 | ->if( 166 | $queue = new Definition('queue2'.uniqid('test_redis', false)), // unique queue 167 | $redisClient = new \mock\Predis\Client(), 168 | 169 | // publish two messages 170 | $producer = new Producer($queue, $redisClient), 171 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle')), 172 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle2')), 173 | 174 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer1'.uniqid('test_redis', false)), 175 | $consumer->setNoAutoAck(), 176 | 177 | $inspector = new Inspector($queue, $redisClient) 178 | ) 179 | ->then 180 | ->object($consumer->getMessageEnvelope()) 181 | ->object($consumer->getMessageEnvelope()) 182 | 183 | // messages should not be in the queue but in the working list 184 | ->integer($inspector->countReadyMessages()) 185 | ->isEqualTo(0) 186 | ->integer($inspector->countInProgressMessages()) 187 | ->isEqualTo(2) 188 | ->and 189 | // unack messages 190 | ->integer($consumer->unackAll()) 191 | // messages is gone 192 | ->integer($inspector->countReadyMessages()) 193 | ->isEqualTo(2) 194 | ->integer($inspector->countInProgressMessages()) 195 | ->isEqualTo(0) 196 | ; 197 | } 198 | 199 | public function testConsumeOnMultipleList() 200 | { 201 | $this 202 | ->if( 203 | $queue = new Definition('queue3'.uniqid('test_redis', false), 10000), // unique queue with a lot of list 204 | $redisClient = new \Predis\Client(), 205 | $producer = new Producer($queue, $redisClient), 206 | $message = new MessageEnvelope(uniqid(), 'message in the bottle'), 207 | $producer->publishMessage($message), 208 | $inspector = new Inspector($queue, $redisClient) 209 | ) 210 | ->then 211 | ->integer($inspector->countListQueues()) 212 | ->isEqualTo(1) // one list is created 213 | ->integer($inspector->countWorkingListQueues()) 214 | ->isEqualTo(0) 215 | ->if( 216 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer3'.uniqid('test_redis', false)), 217 | $consumer->setNoAutoAck(), 218 | $message = $consumer->getMessageEnvelope() 219 | ) 220 | ->then 221 | ->integer($inspector->countListQueues()) 222 | ->isEqualTo(0) 223 | ->integer($inspector->countWorkingListQueues()) 224 | ->isEqualTo(1) // one working list is created 225 | ->if( 226 | $consumer->ack($message) 227 | ) 228 | ->then 229 | ->integer($inspector->countListQueues()) 230 | ->isEqualTo(0) 231 | ->integer($inspector->countWorkingListQueues()) 232 | ->isEqualTo(0) // working list is empty so has been deleted 233 | ; 234 | } 235 | 236 | public function testGetQueueListsLength() 237 | { 238 | $this 239 | ->if( 240 | $queueName = 'queue2'.uniqid('test_redis', false), 241 | $queue = new Definition($queueName), 242 | $redisClient = new \mock\Predis\Client(), 243 | 244 | // publish two messages 245 | $producer = new Producer($queue, $redisClient), 246 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle')), 247 | $producer->publishMessage(new MessageEnvelope(uniqid(), 'message in the bottle2')), 248 | 249 | $consumer = $this->newTestedInstance($queue, $redisClient, 'testConsumer1'.uniqid('test_redis', false)) 250 | ) 251 | ->then 252 | ->array($queueListLength = $consumer->getQueueListsLength()) 253 | ->hasSize(1) 254 | ->hasKey($queueName.'_list__1') 255 | ->integer($queueListLength[$queueName.'_list__1']) 256 | ->isEqualTo(2) 257 | ; 258 | } 259 | } --------------------------------------------------------------------------------