├── .php_cs.dist.php ├── CHANGELOG.md ├── LICENSE.md ├── README.md ├── composer.json └── src ├── AsyncSqsQueueServiceProvider.php ├── Connector └── SqsBulkConnector.php ├── Exception └── BulkSqsDispatchFailed.php └── SqsBulkQueue.php /.php_cs.dist.php: -------------------------------------------------------------------------------- 1 | in([ 5 | __DIR__ . '/src', 6 | __DIR__ . '/tests', 7 | ]) 8 | ->name('*.php') 9 | ->notName('*.blade.php') 10 | ->ignoreDotFiles(true) 11 | ->ignoreVCS(true); 12 | 13 | return (new PhpCsFixer\Config()) 14 | ->setRules([ 15 | '@PSR2' => true, 16 | 'array_syntax' => ['syntax' => 'short'], 17 | 'ordered_imports' => ['sort_algorithm' => 'alpha'], 18 | 'no_unused_imports' => true, 19 | 'not_operator_with_successor_space' => true, 20 | 'trailing_comma_in_multiline' => true, 21 | 'phpdoc_scalar' => true, 22 | 'unary_operator_spaces' => true, 23 | 'binary_operator_spaces' => true, 24 | 'blank_line_before_statement' => [ 25 | 'statements' => ['break', 'continue', 'declare', 'return', 'throw', 'try'], 26 | ], 27 | 'phpdoc_single_line_var_spacing' => true, 28 | 'phpdoc_var_without_name' => true, 29 | 'class_attributes_separation' => [ 30 | 'elements' => [ 31 | 'method' => 'one', 32 | ], 33 | ], 34 | 'method_argument_space' => [ 35 | 'on_multiline' => 'ensure_fully_multiline', 36 | 'keep_multiple_spaces_after_comma' => true, 37 | ], 38 | 'single_trait_insert_per_statement' => true, 39 | ]) 40 | ->setFinder($finder); 41 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to `laravel-bulk-sqs-queue` will be documented in this file. 4 | 5 | ## 1.0.0 - 202X-XX-XX 6 | 7 | - initial release 8 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) atymic 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Laravel SQS Bulk Queue 2 | 3 | [![Latest Version on Packagist](https://img.shields.io/packagist/v/atymic/laravel-bulk-sqs-queue.svg?style=flat-square)](https://packagist.org/packages/atymic/laravel-bulk-sqs-queue) 4 | [![GitHub Tests Action Status](https://img.shields.io/github/actions/workflow/status/atymic/laravel-bulk-sqs-queue/run-tests.yml?label=tests&branch=master)](https://github.com/atymic/laravel-bulk-sqs-queue/actions?query=workflow%3Arun-tests+branch%3Amain) 5 | [![GitHub Code Style Action Status](https://img.shields.io/github/actions/workflow/status/atymic/laravel-bulk-sqs-queue/php-cs-fixer.yml?label=code%20style&branch=master)](https://github.com/atymic/laravel-bulk-sqs-queue/actions?query=workflow%3A"Check+%26+fix+styling"+branch%3Amain) 6 | [![Total Downloads](https://img.shields.io/packagist/dt/atymic/laravel-bulk-sqs-queue.svg?style=flat-square)](https://packagist.org/packages/atymic/laravel-bulk-sqs-queue) 7 | 8 | ## Installation 9 | 10 | You can install the package via composer: 11 | 12 | ```bash 13 | composer require atymic/laravel-bulk-sqs-queue 14 | ``` 15 | 16 | ## How it works 17 | 18 | By default, Laravel allows you to easily execute a batch of jobs with `Queue::bulk()` method or with built-in [job batching](https://laravel.com/docs/master/queues#job-batching). Both methods accept an array of jobs for dispatch and loop over every job, making one HTTP request for each job. 19 | 20 | This isn't an issue when you are dispatching a few jobs, but there's two major issues with bigger batches: 21 | 22 | - Waiting for 1000 http requests (even SQS, with 20-50ms latency) is two to five seconds, that's slow! 23 | - SQS is billed per request, and they support batching of up to 10 messages for the same cost as a single `sendMessage` call. That's a 10x cost saving! 24 | 25 | But AWS SQS has a [batch action](https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-batch-api-actions.html) that let you group up to 10 messages with a single request, in order to reduce costs. Under the hood, this package override the bulk method to: 26 | 27 | - Batch jobs into 10 per request, or 200kb chunks (SQS has a maximum of 256kb, including request overhead/etc) 28 | - Dispatch those batches asynchronously, up to `$concurrency` at a time (default 5) 29 | 30 | That's about it. With this package, the laravel queue system should work the exact same as normal. You should have the same result in your application and your AWS SQS dashboard but with a smaller AWS bill :) 31 | 32 | ## Usage 33 | 34 | This package provides a queue connector called `sqs-bulk`. Inside your `queue.php` config file, add it to `connections`: 35 | 36 | ```php 37 | 'connections' => [ 38 | 'sqs-bulk' => [ 39 | 'driver' => 'sqs-bulk', 40 | 'key' => env('AWS_ACCESS_KEY_ID'), 41 | 'secret' => env('AWS_SECRET_ACCESS_KEY'), 42 | 'prefix' => env('SQS_PREFIX', 'https://sqs.us-east-1.amazonaws.com/your-account-id'), 43 | 'queue' => env('SQS_QUEUE', 'default'), 44 | 'suffix' => env('SQS_SUFFIX'), 45 | 'region' => env('AWS_DEFAULT_REGION', 'us-east-1'), 46 | 'after_commit' => false, 47 | 'concurrency' => 5, // Set the request concurrency, defaults to 5 48 | ], 49 | ] 50 | ``` 51 | 52 | Then you can start a queue worker for the new "connection" and the given queue ('default' queue can be override with `SQS_QUEUE`): 53 | 54 | ```bash 55 | php artisan queue:work sqs-bulk --queue=default 56 | ``` 57 | 58 | It will process new jobs as they are pushed onto the queue. You can group jobs with queue's bulk method: 59 | 60 | ```php 61 | Illuminate\Support\Facades\Queue::bulk([ 62 | new \App\Jobs\Foo, 63 | new \App\Jobs\Bar, 64 | new \App\Jobs\Baz, 65 | ], '', 'default'); 66 | ``` 67 | 68 | or with laravel's built-in [job batching](https://laravel.com/docs/master/queues#job-batching) feature: 69 | 70 | ```php 71 | Illuminate\Support\Facades\Bus::batch([ 72 | new \App\Jobs\Foo, 73 | new \App\Jobs\Bar, 74 | new \App\Jobs\Baz, 75 | ])->name('My sqs batch')->onQueue('default')->dispatch(); 76 | ``` 77 | 78 | It should have processed the 3 jobs by creating 3 "messages" on AWS SQS but with only 1 request. 79 | 80 | ### Failing jobs 81 | 82 | This package only affect the way jobs are transmitted to AWS SQS. Once received by AWS SQS, they are handled as separate messages. From AWS SQS perspective, there is no relation between messages. The batch request can result in a combination of successful and unsuccessful actions that doesn't affect each others. 83 | 84 | ## Testing 85 | 86 | ```bash 87 | composer test 88 | ``` 89 | 90 | ## Changelog 91 | 92 | Please see [CHANGELOG](CHANGELOG.md) for more information on what has changed recently. 93 | 94 | ## Contributing 95 | 96 | Please see [CONTRIBUTING](.github/CONTRIBUTING.md) for details. 97 | 98 | ## Security Vulnerabilities 99 | 100 | Please review [our security policy](../../security/policy) on how to report security vulnerabilities. 101 | 102 | ## Credits 103 | 104 | - [atymic](https://github.com/atymic) 105 | - [Laravel-BatchSQS](https://github.com/CoInvestor/Laravel-BatchSQS) 106 | - [All Contributors](../../contributors) 107 | 108 | ## License 109 | 110 | The MIT License (MIT). Please see [License File](LICENSE.md) for more information. 111 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "atymic/laravel-bulk-sqs-queue", 3 | "description": "Laravel SQS Bulk Queue", 4 | "keywords": [ 5 | "atymic", 6 | "laravel", 7 | "laravel-bulk-sqs-queue", 8 | "queue", 9 | "sqs" 10 | ], 11 | "homepage": "https://github.com/atymic/laravel-bulk-sqs-queue", 12 | "license": "MIT", 13 | "authors": [ 14 | { 15 | "name": "atymic", 16 | "email": "atymicq@gmail.com", 17 | "role": "Developer" 18 | } 19 | ], 20 | "require": { 21 | "php": "^8.1", 22 | "illuminate/contracts": "^9.0 || ^10.0 || ^11.0", 23 | "aws/aws-sdk-php": "^3" 24 | }, 25 | "require-dev": { 26 | "brianium/paratest": "^6.2", 27 | "nunomaduro/collision": "^7.0", 28 | "orchestra/testbench": "^7.0 || ^8.0 || ^9.0", 29 | "phpunit/phpunit": "^9.3" 30 | }, 31 | "autoload": { 32 | "psr-4": { 33 | "Atymic\\AsyncSqsQueue\\": "src" 34 | } 35 | }, 36 | "autoload-dev": { 37 | "psr-4": { 38 | "Atymic\\AsyncSqsQueue\\Tests\\": "tests" 39 | } 40 | }, 41 | "scripts": { 42 | "psalm": "vendor/bin/psalm", 43 | "test": "./vendor/bin/testbench package:test --parallel --no-coverage", 44 | "test-coverage": "vendor/bin/phpunit --coverage-html coverage" 45 | }, 46 | "config": { 47 | "sort-packages": true 48 | }, 49 | "extra": { 50 | "laravel": { 51 | "providers": [ 52 | "Atymic\\AsyncSqsQueue\\AsyncSqsQueueServiceProvider" 53 | ], 54 | "aliases": { 55 | "AsyncSqsQueue": "Atymic\\AsyncSqsQueue\\AsyncSqsQueueFacade" 56 | } 57 | } 58 | }, 59 | "minimum-stability": "dev", 60 | "prefer-stable": true 61 | } 62 | -------------------------------------------------------------------------------- /src/AsyncSqsQueueServiceProvider.php: -------------------------------------------------------------------------------- 1 | app['queue']; 15 | 16 | $queueManager->addConnector('sqs-bulk', function () { 17 | return new SqsBulkConnector(); 18 | }); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /src/Connector/SqsBulkConnector.php: -------------------------------------------------------------------------------- 1 | getDefaultConfiguration($config); 20 | 21 | if ($config['key'] && $config['secret']) { 22 | $config['credentials'] = Arr::only($config, ['key', 'secret', 'token']); 23 | } 24 | 25 | $queue = new SqsBulkQueue( 26 | new SqsClient($config), 27 | $config['queue'], 28 | $config['prefix'] ?? '', 29 | $config['suffix'] ?? '', 30 | $config['after_commit'] ?? null 31 | ); 32 | 33 | if ($concurrency !== null) { 34 | $queue->setConcurrency($concurrency); 35 | } 36 | 37 | return $queue; 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /src/Exception/BulkSqsDispatchFailed.php: -------------------------------------------------------------------------------- 1 | map(fn (Result $fail) => sprintf('%s failed with %s, message `%s`', $fail['Id'], $fail['Code'], $fail['Message'])) 16 | ->prepend('Bulk dispatch failed, errors:') 17 | ->join("\n"); 18 | 19 | return new self($message); 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /src/SqsBulkQueue.php: -------------------------------------------------------------------------------- 1 | batchGenerator($jobs, $data, $queue), 29 | $this->concurrency, 30 | fn (Result $res) => $responses->push($res) 31 | ); 32 | 33 | $promise->wait(); 34 | 35 | $failed = $responses 36 | ->filter(fn (Result $res) => count($res['Failed'] ?? [])) 37 | ->flatten(1); 38 | 39 | if ($failed->isNotEmpty()) { 40 | throw BulkSqsDispatchFailed::withFailureResponses($failed); 41 | } 42 | } 43 | 44 | protected function batchGenerator($jobs, $data = '', $queue = null): Generator 45 | { 46 | $queue = $queue ?: $this->default; 47 | 48 | $batchPayloads = []; 49 | $batchBytes = 0; 50 | 51 | foreach ($jobs as $job) { 52 | $payload = $this->createPayload($job, $queue ?: $this->default, $data); 53 | 54 | $batchPayloads[] = $payload; 55 | $batchBytes += strlen($payload); 56 | 57 | if ($batchBytes >= self::BATCH_SIZE_LIMIT || count($batchPayloads) >= self::BATCH_LIMIT) { 58 | yield $this->dispatchBatchAsync($queue, $batchPayloads); 59 | $batchPayloads = []; 60 | $batchBytes = 0; 61 | } 62 | } 63 | 64 | if (count($batchPayloads)) { 65 | yield $this->dispatchBatchAsync($queue, $batchPayloads); 66 | } 67 | } 68 | 69 | protected function dispatchBatchAsync(string $queue, array $payloads): Promise 70 | { 71 | return $this->sqs->sendMessageBatchAsync([ 72 | 'QueueUrl' => $this->getQueue($queue), 73 | 'Entries' => array_map( 74 | fn (string $payload) => [ 75 | 'Id' => (string) Str::uuid(), 76 | 'MessageBody' => $payload, 77 | ], 78 | $payloads 79 | ), 80 | ]); 81 | } 82 | 83 | public function setConcurrency(int $concurrency): void 84 | { 85 | $this->concurrency = $concurrency; 86 | } 87 | } 88 | --------------------------------------------------------------------------------