├── composer.json ├── config └── sqs-queue-reader.php └── src ├── Jobs └── DispatcherJob.php ├── Sqs ├── Connector.php └── Queue.php └── SqsQueueReaderServiceProvider.php /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "palpalani/laravel-sqs-queue-json-reader", 3 | "description": "Custom SQS queue reader for Laravel", 4 | "keywords": [ 5 | "palpalani", 6 | "sqs-reader", 7 | "sqs-json", 8 | "sqs-text-reader", 9 | "sqs-plain-message", 10 | "sqs-bulk-message", 11 | "laravel-sqs", 12 | "laravel-sqs-queue-json-reader" 13 | ], 14 | "homepage": "https://github.com/palpalani/laravel-sqs-queue-json-reader", 15 | "license": "MIT", 16 | "authors": [ 17 | { 18 | "name": "palPalani", 19 | "email": "palani.p@gmail.com", 20 | "homepage": "https://github.com/palpalani", 21 | "role": "Developer" 22 | } 23 | ], 24 | "require": { 25 | "php": "^8.2", 26 | "illuminate/contracts": "^9.0|^10.0|^11.0|^12.0", 27 | "illuminate/support": "^9.0|^10.0|^11.0|^12.0", 28 | "illuminate/queue": "^9.0|^10.0|^11.0|^12.0", 29 | "illuminate/bus": "^9.0|^10.0|^11.0|^12.0", 30 | "aws/aws-sdk-php": "^3.250" 31 | }, 32 | "require-dev": { 33 | "larastan/larastan": "^2.0", 34 | "laravel/pint": "^1.2", 35 | "nunomaduro/collision": "^6.3|^7.0|^8.1", 36 | "orchestra/testbench": "^7.15|^8.0|^9.0|^10.0", 37 | "phpstan/extension-installer": "^1.2", 38 | "phpstan/phpstan-deprecation-rules": "^1.0", 39 | "phpunit/phpunit": "^9.5|^10.0|^11.0" 40 | }, 41 | "autoload": { 42 | "psr-4": { 43 | "palPalani\\SqsQueueReader\\": "src" 44 | } 45 | }, 46 | "autoload-dev": { 47 | "psr-4": { 48 | "palPalani\\SqsQueueReader\\Tests\\": "tests" 49 | } 50 | }, 51 | "scripts": { 52 | "analyse": "vendor/bin/phpstan analyse", 53 | "test": "./vendor/bin/testbench package:test --no-coverage", 54 | "test-coverage": "vendor/bin/phpunit --coverage-html coverage", 55 | "format": "vendor/bin/pint" 56 | }, 57 | "config": { 58 | "sort-packages": true, 59 | "allow-plugins": { 60 | "phpstan/extension-installer": true 61 | } 62 | }, 63 | "extra": { 64 | "laravel": { 65 | "providers": [ 66 | "palPalani\\SqsQueueReader\\SqsQueueReaderServiceProvider" 67 | ] 68 | } 69 | }, 70 | "minimum-stability": "dev", 71 | "prefer-stable": true, 72 | "funding": [ 73 | { 74 | "type": "github", 75 | "url": "https://github.com/sponsors/palpalani" 76 | } 77 | ] 78 | } 79 | -------------------------------------------------------------------------------- /config/sqs-queue-reader.php: -------------------------------------------------------------------------------- 1 | [ 12 | 'stripe-webhooks' => [ 13 | 'class' => App\Jobs\StripeHandler::class, 14 | 'count' => 10, 15 | ], 16 | 'mailgun-webhooks' => [ 17 | 'class' => App\Jobs\MailgunHandler::class, 18 | 'count' => 100, 19 | ], 20 | ], 21 | 22 | // If no handlers specified then default handler will be executed. 23 | 'default-handler' => [ 24 | 25 | // Name of the handler class 26 | 'class' => App\Jobs\SqsHandler::class, 27 | 28 | // Number of messages need to read from SQS. 29 | 'count' => 1, 30 | ], 31 | ]; 32 | -------------------------------------------------------------------------------- /src/Jobs/DispatcherJob.php: -------------------------------------------------------------------------------- 1 | isPlain()) { 28 | return [ 29 | 'job' => app('config')->get('sqs-queue-reader.default-handler'), 30 | 'data' => $this->data, 31 | ]; 32 | } 33 | 34 | return $this->data; 35 | } 36 | 37 | /** 38 | * @return $this 39 | */ 40 | public function setPlain(bool $plain = true): self 41 | { 42 | $this->plain = $plain; 43 | 44 | return $this; 45 | } 46 | 47 | public function isPlain(): bool 48 | { 49 | return $this->plain; 50 | } 51 | 52 | public function __invoke() 53 | { 54 | $this->getPayload(); 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /src/Sqs/Connector.php: -------------------------------------------------------------------------------- 1 | getDefaultConfiguration($config); 21 | 22 | if (isset($config['key'], $config['secret'])) { 23 | $config['credentials'] = Arr::only($config, ['key', 'secret']); 24 | } 25 | 26 | return new Queue( 27 | new SqsClient($config), 28 | $config['queue'], 29 | Arr::get($config, 'prefix', '') 30 | ); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/Sqs/Queue.php: -------------------------------------------------------------------------------- 1 | getClass($queue) . '@handle'; 36 | 37 | return $job->isPlain() ? \json_encode($job->getPayload(), JSON_THROW_ON_ERROR) : \json_encode([ 38 | 'job' => $handlerJob, 39 | 'data' => $job->getPayload(), 40 | ], JSON_THROW_ON_ERROR); 41 | } 42 | 43 | private function getClass($queue = null): string 44 | { 45 | if (! $queue) { 46 | return Config::get('sqs-queue-reader.default-handler')['class']; 47 | } 48 | 49 | $queueId = explode('/', $queue); 50 | $queueId = array_pop($queueId); 51 | 52 | return (\array_key_exists($queueId, Config::get('sqs-queue-reader.handlers'))) 53 | ? Config::get('sqs-queue-reader.handlers')[$queueId]['class'] 54 | : Config::get('sqs-queue-reader.default-handler')['class']; 55 | } 56 | 57 | /** 58 | * Pop the next job off of the queue. 59 | * 60 | * @param string $queue 61 | * @return \Illuminate\Contracts\Queue\Job|null 62 | * 63 | * @throws JsonException 64 | */ 65 | public function pop($queue = null) 66 | { 67 | $queue = $this->getQueue($queue); 68 | 69 | $queueId = explode('/', $queue); 70 | $queueId = array_pop($queueId); 71 | 72 | $count = (\array_key_exists($queueId, Config::get('sqs-queue-reader.handlers'))) 73 | ? Config::get('sqs-queue-reader.handlers')[$queueId]['count'] 74 | : Config::get('sqs-queue-reader.default-handler')['count']; 75 | 76 | try { 77 | $response = $this->sqs->receiveMessage([ 78 | 'QueueUrl' => $queue, 79 | 'AttributeNames' => ['ApproximateReceiveCount'], 80 | 'MaxNumberOfMessages' => $count, 81 | 'MessageAttributeNames' => ['All'], 82 | ]); 83 | 84 | if (isset($response['Messages']) && count($response['Messages']) > 0) { 85 | $class = (\array_key_exists($queueId, $this->container['config']->get('sqs-queue-reader.handlers'))) 86 | ? $this->container['config']->get('sqs-queue-reader.handlers')[$queueId]['class'] 87 | : $this->container['config']->get('sqs-queue-reader.default-handler')['class']; 88 | 89 | if ($count === 1) { 90 | $response = $this->modifySinglePayload($response['Messages'][0], $class); 91 | } else { 92 | $response = $this->modifyMultiplePayload($response['Messages'], $class); 93 | } 94 | 95 | return new SqsJob($this->container, $this->sqs, $response, $this->connectionName, $queue); 96 | } 97 | } catch (AwsException $e) { 98 | $msg = 'Line: ' . $e->getLine() . ', ' . $e->getFile() . ', ' . $e->getMessage(); 99 | 100 | throw new \RuntimeException('Aws SQS error: ' . $msg); 101 | } 102 | } 103 | 104 | /** 105 | * @throws JsonException 106 | */ 107 | private function modifySinglePayload(array|string $payload, string $class): array|string 108 | { 109 | if (! is_array($payload)) { 110 | $payload = \json_decode($payload, true, 512, JSON_THROW_ON_ERROR); 111 | } 112 | 113 | $body = \json_decode($payload['Body'], true, 512, JSON_THROW_ON_ERROR); 114 | 115 | $payload['Body'] = \json_encode([ 116 | 'uuid' => (string) Str::uuid(), 117 | 'job' => $class . '@handle', 118 | 'data' => $body['data'] ?? $body, 119 | ], JSON_THROW_ON_ERROR); 120 | 121 | return $payload; 122 | } 123 | 124 | /** 125 | * @throws JsonException 126 | */ 127 | private function modifyMultiplePayload(array|string $payload, string $class): array 128 | { 129 | if (! is_array($payload)) { 130 | $payload = \json_decode($payload, true, 512, JSON_THROW_ON_ERROR); 131 | } 132 | 133 | $body = []; 134 | $attributes = []; 135 | $messageId = null; 136 | $receiptHandle = null; 137 | 138 | foreach ($payload as $k => $item) { 139 | try { 140 | $message = \json_decode($item['Body'], true, 512, JSON_THROW_ON_ERROR); 141 | } catch (JsonException $e) { 142 | $message = []; 143 | } 144 | 145 | $body[$k] = [ 146 | 'messages' => $message, 147 | 'attributes' => $item['Attributes'], 148 | 'batchIds' => [ 149 | 'Id' => $item['MessageId'], 150 | 'ReceiptHandle' => $item['ReceiptHandle'], 151 | ], 152 | ]; 153 | $attributes = $item['Attributes']; 154 | $messageId = $item['MessageId']; 155 | $receiptHandle = $item['ReceiptHandle']; 156 | } 157 | 158 | return [ 159 | 'MessageId' => $messageId, 160 | 'ReceiptHandle' => $receiptHandle, 161 | 'Body' => \json_encode([ 162 | 'uuid' => (string) Str::uuid(), 163 | 'job' => $class . '@handle', 164 | 'data' => $body, 165 | ], JSON_THROW_ON_ERROR), 166 | 'Attributes' => $attributes, 167 | ]; 168 | } 169 | 170 | /** 171 | * @param string $payload 172 | * @param string|null $queue 173 | * 174 | * @throws JsonException 175 | */ 176 | public function pushRaw($payload, $queue = null, array $options = []): mixed 177 | { 178 | $payload = \json_decode($payload, true, 512, JSON_THROW_ON_ERROR); 179 | 180 | if (isset($payload['data'], $payload['job'])) { 181 | $payload = $payload['data']; 182 | } 183 | 184 | return parent::pushRaw(\json_encode($payload, JSON_THROW_ON_ERROR), $queue, $options); 185 | } 186 | } 187 | -------------------------------------------------------------------------------- /src/SqsQueueReaderServiceProvider.php: -------------------------------------------------------------------------------- 1 | app->runningInConsole()) { 22 | $this->publishes([ 23 | __DIR__ . '/../config/sqs-queue-reader.php' => config_path('sqs-queue-reader.php'), 24 | ], 'config'); 25 | 26 | Queue::after(function (JobProcessed $event) { 27 | $connections = Config::get('queue.connections'); 28 | if (\in_array($event->connectionName, array_keys($connections), true)) { 29 | $queue = $event->job->getQueue(); 30 | 31 | $queueId = explode('/', $queue); 32 | $queueId = array_pop($queueId); 33 | 34 | $count = (\array_key_exists($queueId, Config::get('sqs-queue-reader.handlers'))) 35 | ? Config::get('sqs-queue-reader.handlers')[$queueId]['count'] 36 | : Config::get('sqs-queue-reader.default-handler')['count']; 37 | 38 | if ($count === 1) { 39 | $event->job->delete(); 40 | } else { 41 | $this->removeMessages($event->job->payload(), $queue, $event->connectionName); 42 | } 43 | } 44 | }); 45 | } 46 | } 47 | 48 | public function register(): void 49 | { 50 | $this->mergeConfigFrom(__DIR__ . '/../config/sqs-queue-reader.php', 'sqs-queue-reader'); 51 | 52 | $this->app->booted(function () { 53 | $this->app['queue']->extend('sqs-json', static function () { 54 | return new Connector; 55 | }); 56 | }); 57 | } 58 | 59 | private function removeMessages(array $data, $queue, string $connection): void 60 | { 61 | $batchIds = array_column($data['data'], 'batchIds'); 62 | $batchIds = array_chunk($batchIds, 10); 63 | 64 | $config = Config::get('queue.connections.' . $connection); 65 | 66 | $sqsClientConfig = [ 67 | //'profile' => 'default', 68 | 'region' => Config::get('queue.connections.' . $connection . '.region'), 69 | 'version' => '2012-11-05', 70 | 'http' => [ 71 | 'timeout' => 30, 72 | 'connect_timeout' => 30, 73 | ], 74 | ]; 75 | 76 | if (isset($config['key'], $config['secret'])) { 77 | $sqsClientConfig['credentials'] = Arr::only($config, ['key', 'secret']); 78 | } 79 | 80 | $client = new SqsClient($sqsClientConfig); 81 | 82 | foreach ($batchIds as $batch) { 83 | //Deletes up to ten messages from the specified queue. 84 | try { 85 | $result = $client->deleteMessageBatch([ 86 | 'Entries' => $batch, 87 | 'QueueUrl' => $queue, 88 | ]); 89 | 90 | if (isset($result['Failed'])) { 91 | $msg = ''; 92 | foreach ($result['Failed'] as $failed) { 93 | $msg .= sprintf('Deleting message failed, code = %s, id = %s, msg = %s, senderfault = %s', $failed['Code'], $failed['Id'], $failed['Message'], $failed['SenderFault']); 94 | } 95 | Log::error('Cannot delete some SQS messages: ', [$msg]); 96 | 97 | throw new \RuntimeException('Cannot delete some messages, consult log for more info!'); 98 | } 99 | //Log::info('Message remove report:', [$result]); 100 | } catch (AwsException $e) { 101 | Log::error('AWS SQS client error:', [$e->getMessage()]); 102 | } 103 | } 104 | } 105 | } 106 | --------------------------------------------------------------------------------