├── .php-cs-fixer.dist.php ├── CODE_OF_CONDUCT.md ├── composer.json └── src ├── BulkChunkDispatcher.php ├── Chunk.php ├── ChunkRange.php ├── ChunkableJob.php ├── Exceptions └── BulkChunkDispatcherException.php └── UnknownSizeChunk.php /.php-cs-fixer.dist.php: -------------------------------------------------------------------------------- 1 | in([ 5 | __DIR__, 6 | ]) 7 | ->name('*.php') 8 | ->notName('*.blade.php') 9 | ->ignoreDotFiles(true) 10 | ->ignoreVCS(true) 11 | ->exclude(['vendor', 'node_modules']); 12 | 13 | $config = new PhpCsFixer\Config(); 14 | 15 | // Rules from: https://cs.symfony.com/doc/rules/index.html 16 | 17 | return $config->setRules([ 18 | '@PSR2' => true, 19 | 'array_syntax' => ['syntax' => 'short'], 20 | 'ordered_imports' => ['sort_algorithm' => 'length'], 21 | 'no_unused_imports' => true, 22 | 'not_operator_with_successor_space' => true, 23 | 'trailing_comma_in_multiline' => true, 24 | 'single_quote' => ['strings_containing_single_quote_chars' => true], 25 | 'phpdoc_scalar' => true, 26 | 'unary_operator_spaces' => true, 27 | 'binary_operator_spaces' => true, 28 | 'blank_line_before_statement' => [ 29 | 'statements' => ['declare', 'return', 'throw', 'try'], 30 | ], 31 | 'phpdoc_single_line_var_spacing' => true, 32 | 'phpdoc_var_without_name' => true, 33 | 'method_argument_space' => [ 34 | 'on_multiline' => 'ensure_fully_multiline', 35 | 'keep_multiple_spaces_after_comma' => true, 36 | ], 37 | 'return_type_declaration' => [ 38 | 'space_before' => 'none' 39 | ], 40 | 'declare_strict_types' => true 41 | ])->setFinder($finder); 42 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported to the community leaders responsible for enforcement at 63 | github@yeehaw.dev. 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All community leaders are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. 129 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "sammyjo20/laravel-chunkable-jobs", 3 | "description": "Split a job into multiple separate job chunks", 4 | "type": "library", 5 | "license": "MIT", 6 | "autoload": { 7 | "psr-4": { 8 | "Sammyjo20\\ChunkableJobs\\": "src/", 9 | "Sammyjo20\\ChunkableJobs\\Tests\\": "tests/" 10 | } 11 | }, 12 | "authors": [ 13 | { 14 | "name": "Sammyjo20", 15 | "email": "29132017+Sammyjo20@users.noreply.github.com" 16 | } 17 | ], 18 | "minimum-stability": "stable", 19 | "require": { 20 | "php": "^8.1" 21 | }, 22 | "require-dev": { 23 | "friendsofphp/php-cs-fixer": "^3.5", 24 | "pestphp/pest": "^2.34", 25 | "spatie/ray": "^1.33", 26 | "orchestra/testbench": "^8.0 || ^9.0" 27 | }, 28 | "scripts": { 29 | "test": [ 30 | "./vendor/bin/pest" 31 | ], 32 | "fix-code": [ 33 | "./vendor/bin/php-cs-fixer fix --allow-risky=yes" 34 | ] 35 | }, 36 | "config": { 37 | "allow-plugins": { 38 | "pestphp/pest-plugin": true 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /src/BulkChunkDispatcher.php: -------------------------------------------------------------------------------- 1 | defineChunk(); 19 | 20 | if (is_null($chunk)) { 21 | return; 22 | } 23 | 24 | if ($chunk instanceof UnknownSizeChunk) { 25 | throw new BulkChunkDispatcherException('You cannot iterate through an UnknownSizeChunk.'); 26 | } 27 | 28 | $chunkRange = ChunkRange::create($chunk->totalItems, $chunk->originalSize); 29 | 30 | // Now we'll loop through the chunk range and dispatch every job. We'll 31 | // also make sure to disable the next functionality of every chunk 32 | // so they don't chain, that would be bad! 33 | 34 | foreach ($chunkRange as $chunk) { 35 | $chunk->disableNext(); 36 | 37 | $dispatchJob = (clone $job)->setChunk($chunk); 38 | 39 | dispatch($dispatchJob); 40 | } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /src/Chunk.php: -------------------------------------------------------------------------------- 1 | originalSize = $chunkSize; 109 | $this->totalItems = $totalItems; 110 | $this->totalChunks = $totalChunks; 111 | $this->remainingItems = $totalItems; 112 | $this->remainingChunks = $totalChunks > 0 ? $totalChunks - 1 : 0; 113 | $this->limit = min($totalItems, $chunkSize); 114 | $this->size = $this->limit; 115 | $this->offset = 0; 116 | $this->position = 1; 117 | $this->metadata = $metadata; 118 | 119 | if ($startingPosition !== 1) { 120 | $this->move($startingPosition, true); 121 | } 122 | } 123 | 124 | /** 125 | * Return the next chunk. 126 | * 127 | * @return Chunk 128 | */ 129 | public function next(): Chunk 130 | { 131 | if ($this->isLast() || $this->isNextDisabled()) { 132 | return $this; 133 | } 134 | 135 | return $this->move($this->position + 1); 136 | } 137 | 138 | /** 139 | * Move the chunk to a specific position 140 | * 141 | * @param int $position 142 | * @param bool $mutable 143 | * @return $this 144 | */ 145 | public function move(int $position, bool $mutable = false): Chunk 146 | { 147 | if ($position === $this->position) { 148 | return $this; 149 | } 150 | 151 | if ($position <= 0 || $position > $this->totalChunks) { 152 | throw new InvalidArgumentException(sprintf('The position must be between 1 and %s.', $this->totalChunks)); 153 | } 154 | 155 | // We'll calculate the remaining items with some maths 156 | 157 | $remaining = ((($position * $this->originalSize) - $this->totalItems) * -1) + $this->originalSize; 158 | 159 | // Now we'll create a new chunk to process it with. 160 | 161 | $newChunk = clone $this; 162 | 163 | $newChunk->position = $position; 164 | $newChunk->remainingItems = $remaining; 165 | $newChunk->remainingChunks = $this->totalChunks - $position; 166 | $newChunk->offset = ($position - 1) * $this->originalSize; 167 | $newChunk->limit = min($remaining, $this->originalSize); 168 | $newChunk->size = $newChunk->limit; 169 | 170 | return $mutable === true ? $this->replace($newChunk) : $newChunk; 171 | } 172 | 173 | /** 174 | * Replace the object with another chunk. 175 | * 176 | * @param Chunk $chunk 177 | * @return $this 178 | */ 179 | public function replace(Chunk $chunk): static 180 | { 181 | $this->totalItems = $chunk->totalItems; 182 | $this->totalChunks = $chunk->totalChunks; 183 | $this->remainingItems = $chunk->remainingItems; 184 | $this->remainingChunks = $chunk->remainingChunks; 185 | $this->originalSize = $chunk->originalSize; 186 | $this->size = $chunk->size; 187 | $this->limit = $chunk->limit; 188 | $this->offset = $chunk->offset; 189 | $this->position = $chunk->position; 190 | $this->metadata = $chunk->metadata; 191 | 192 | return $this; 193 | } 194 | 195 | /** 196 | * Determine if the chunk is the first chunk. 197 | * 198 | * @return bool 199 | */ 200 | public function isFirst(): bool 201 | { 202 | return $this->offset === 0; 203 | } 204 | 205 | /** 206 | * Determine if the chunk is not the first. 207 | * 208 | * @return bool 209 | */ 210 | public function isNotFirst(): bool 211 | { 212 | return ! $this->isFirst(); 213 | } 214 | 215 | /** 216 | * Determine if the chunk is the last chunk. 217 | * 218 | * @return bool 219 | */ 220 | public function isLast(): bool 221 | { 222 | return $this->remainingChunks === 0; 223 | } 224 | 225 | /** 226 | * Determine if the chunk is not the last. 227 | * 228 | * @return bool 229 | */ 230 | public function isNotLast(): bool 231 | { 232 | return ! $this->isLast(); 233 | } 234 | 235 | /** 236 | * Determines if the chunk is empty. 237 | * 238 | * @return bool 239 | */ 240 | public function isEmpty(): bool 241 | { 242 | return $this->totalItems === 0; 243 | } 244 | 245 | /** 246 | * Determines if the chunk is not empty. 247 | * 248 | * @return bool 249 | */ 250 | public function isNotEmpty(): bool 251 | { 252 | return ! $this->isEmpty(); 253 | } 254 | 255 | /** 256 | * Disable the next chunk functionality 257 | * 258 | * @return $this 259 | */ 260 | public function disableNext(): Chunk 261 | { 262 | $this->disableNext = true; 263 | 264 | return $this; 265 | } 266 | 267 | /** 268 | * Enable the next chunk functionality 269 | * 270 | * @return $this 271 | */ 272 | public function enableNext(): Chunk 273 | { 274 | $this->disableNext = false; 275 | 276 | return $this; 277 | } 278 | 279 | /** 280 | * Check if the next is disabled. 281 | * 282 | * @return bool 283 | */ 284 | public function isNextDisabled(): bool 285 | { 286 | return $this->disableNext === true; 287 | } 288 | } 289 | -------------------------------------------------------------------------------- /src/ChunkRange.php: -------------------------------------------------------------------------------- 1 | totalChunks; $i++) { 25 | yield $chunk->move($i + 1); 26 | } 27 | }; 28 | 29 | return $generator(); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /src/ChunkableJob.php: -------------------------------------------------------------------------------- 1 | chunk)) { 54 | $this->setChunk($this->defineChunk()); 55 | } 56 | 57 | $chunk = $this->chunk; 58 | 59 | // If we have a chunk, and it isn't empty, we will start 60 | // processing it. 61 | 62 | if (! $chunk instanceof Chunk || $chunk->isEmpty()) { 63 | return; 64 | } 65 | 66 | // If it's the first chunk we will run "setUp". After 67 | // that we will run the "handleChunk" method. 68 | 69 | if ($chunk->isFirst()) { 70 | $this->setUp(); 71 | } 72 | 73 | $this->handleChunk($chunk); 74 | 75 | // Next we'll calculate the next job to dispatch. 76 | 77 | $this->prependNextJob(); 78 | 79 | // If the job has stopped chunking early, or we have hit 80 | // the last chunk, we will run the tear down method. 81 | 82 | if ($this->processNextChunk === false || $chunk->isLast()) { 83 | $this->tearDown(); 84 | } 85 | } 86 | 87 | /** 88 | * Extend this method to write logic before the chunkable job is processed. 89 | * 90 | * @return void 91 | */ 92 | protected function setUp(): void 93 | { 94 | // 95 | } 96 | 97 | /** 98 | * Extend this method to write logic after the chunkable job is processed. 99 | * 100 | * @return void 101 | */ 102 | protected function tearDown(): void 103 | { 104 | // 105 | } 106 | 107 | /** 108 | * Prepend the next job in the chunkable job chain. 109 | * 110 | * @return void 111 | */ 112 | protected function prependNextJob(): void 113 | { 114 | $chunk = $this->chunk; 115 | 116 | // We don't want to process the next chunk if it's the last chunk, we've stopped chunking 117 | // or if the job has been released/deleted. 118 | 119 | if ($this->processNextChunk === false || $chunk->isLast() || $chunk->isNextDisabled() || $this?->job->isDeletedOrReleased()) { 120 | return; 121 | } 122 | 123 | // We will need to unset the following properties because they can cause issues 124 | // when we need to serialize the cloned job in the database. The main culprit 125 | // is the "job" instance on the class, but we also want to ignore chunk 126 | // and nextChunk. 127 | 128 | $ignoredProperties = array_merge([ 129 | 'job', 'middleware', 'chunk', 'nextChunk', 130 | ], $this->ignoredProperties); 131 | 132 | $clone = clone $this; 133 | 134 | foreach ($ignoredProperties as $property) { 135 | unset($clone->$property); 136 | } 137 | 138 | // We'll also trigger a method that can be implemented to unset 139 | 140 | $clone = $this->modifyClone($clone); 141 | 142 | // Next, we'll set the chunk of the clone to the next chunk. 143 | 144 | $clone->setChunk($this->nextChunk ?? $chunk->next()); 145 | 146 | // Finally, we'll dispatch the next chunk 147 | 148 | $this->dispatchNextChunk($clone); 149 | } 150 | 151 | /** 152 | * Dispatch the next chunk 153 | * 154 | * @param object $job 155 | * @return void 156 | */ 157 | protected function dispatchNextChunk(object $job): void 158 | { 159 | if (is_null($job->delay)) { 160 | $job->delay($this->chunkInterval); 161 | } 162 | 163 | // We'll now dispatch the job to the queue, and it will 164 | // be processed with the chunk and delay. 165 | 166 | dispatch($job); 167 | } 168 | 169 | /** 170 | * Set the chunk on the chunkable job 171 | * 172 | * @param Chunk|null $chunk 173 | * @return $this 174 | */ 175 | public function setChunk(?Chunk $chunk): static 176 | { 177 | $this->chunk = $chunk; 178 | 179 | return $this; 180 | } 181 | 182 | /** 183 | * Set the next chunk to be processed. 184 | * 185 | * @param Chunk|null $nextChunk 186 | * @return ChunkableJob 187 | */ 188 | public function setNextChunk(?Chunk $nextChunk): ChunkableJob 189 | { 190 | $this->nextChunk = $nextChunk; 191 | 192 | return $this; 193 | } 194 | 195 | /** 196 | * Stop chunking 197 | * 198 | * @return $this 199 | */ 200 | public function stopChunking(): static 201 | { 202 | $this->processNextChunk = false; 203 | 204 | return $this; 205 | } 206 | 207 | /** 208 | * Dispatch all chunks at once 209 | * 210 | * @param ...$arguments 211 | * @return void 212 | * @throws Exceptions\BulkChunkDispatcherException 213 | */ 214 | public static function dispatchAllChunks(...$arguments): void 215 | { 216 | BulkChunkDispatcher::dispatch(new static(...$arguments)); 217 | } 218 | 219 | /** 220 | * Modify the clone before it is sent. 221 | * 222 | * @param ChunkableJob $job 223 | * @return $this 224 | */ 225 | protected function modifyClone(ChunkableJob $job): static 226 | { 227 | return $job; 228 | } 229 | 230 | /** 231 | * Define the chunk. If it's null or a chunk with zero items the chunkable job will stop early. 232 | * 233 | * @return Chunk|null 234 | */ 235 | abstract public function defineChunk(): ?Chunk; 236 | 237 | /** 238 | * Handle the chunk. 239 | * 240 | * @param Chunk $chunk 241 | * @return void 242 | */ 243 | abstract protected function handleChunk(Chunk $chunk): void; 244 | } 245 | -------------------------------------------------------------------------------- /src/Exceptions/BulkChunkDispatcherException.php: -------------------------------------------------------------------------------- 1 |