├── .gitignore ├── Console └── Command │ └── ThreadProcessorCommand.php ├── LICENSE ├── Model ├── Dimension │ ├── ParallelStoreProcessor.php │ └── ParallelWebsiteProcessor.php ├── ForkedArrayProcessor.php ├── ForkedCollectionProcessor.php ├── ForkedSearchResultProcessor.php ├── ItemProvider │ ├── ArrayWrapper.php │ ├── CollectionWrapper.php │ ├── ItemProviderInterface.php │ └── SearchResultWrapper.php └── Processor │ ├── ForkedProcessor.php │ └── ForkedProcessorRunner.php ├── README.md ├── composer.json ├── etc ├── di.xml └── module.xml └── registration.php /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | composer.lock 3 | /vendor 4 | -------------------------------------------------------------------------------- /Console/Command/ThreadProcessorCommand.php: -------------------------------------------------------------------------------- 1 | setDescription('Wrapper command to run a command line indefinitely in a dedicated thread'); 28 | $this->setName( 29 | 'thread:processor' 30 | )->addArgument( 31 | 'command_name', 32 | InputArgument::REQUIRED, 33 | 'The name of the command to be started.' 34 | )->addOption( 35 | 'timeout', 36 | '', 37 | InputOption::VALUE_OPTIONAL, 38 | 'Define the process timeout in seconds' 39 | )->addOption( 40 | 'iterations', 41 | '', 42 | InputOption::VALUE_OPTIONAL, 43 | 'Define the number of iteration' 44 | )->addOption( 45 | 'environment', 46 | 'env', 47 | InputOption::VALUE_OPTIONAL, 48 | 'Set environment variables separate by comma' 49 | )->addOption( 50 | 'progress', 51 | 'p', 52 | InputOption::VALUE_NONE, 53 | 'Show progress bar while executing command' 54 | ); 55 | 56 | parent::configure(); 57 | } 58 | 59 | /** 60 | * @param InputInterface $input 61 | * @param OutputInterface $output 62 | * @return int 63 | * @throws Exception 64 | */ 65 | protected function execute(InputInterface $input, OutputInterface $output): int 66 | { 67 | // Argument and option 68 | $commandName = $input->getArgument('command_name'); 69 | //$this->getApplication()->get($commandName); 70 | 71 | $environment = $input->getOption('environment'); 72 | $envExploded = !empty($environment) ? explode(',', $environment) : null; 73 | $timeout = $input->getOption('timeout') ?: 300; 74 | $iterations = $input->getOption('iterations') ?: 0; 75 | 76 | // Build extra env values 77 | $arrayEnv = null; 78 | if (is_array($envExploded)) { 79 | foreach ($envExploded as $variableEnv) { 80 | $env = explode('=', $variableEnv); 81 | if (isset($env[0], $env[1])) { 82 | $arrayEnv[$env[0]] = $env[1]; 83 | } 84 | } 85 | } 86 | 87 | $showProgress = $input->getOption('progress'); 88 | if ($showProgress) { 89 | $progressBar = new ProgressBar($output); 90 | $maxIteration = $iterations ?: null; 91 | $progressBar->start((int)$maxIteration); 92 | } 93 | 94 | // Build command 95 | $command = [ 96 | PHP_BINARY, 97 | self::BINARY_MAGENTO, 98 | $commandName 99 | ]; 100 | 101 | // Add options 102 | if ($output->isVerbose()) { 103 | $command[] = "-v"; 104 | } 105 | if ($output->isVeryVerbose()) { 106 | $command[] = "-vv"; 107 | } 108 | 109 | $i = 0; 110 | while (true) { 111 | // Limit the number of iterations 112 | if ($iterations !== 0 && $i >= $iterations) { 113 | break; 114 | } 115 | 116 | // Run single thread process 117 | $process = new Process($command, BP, $arrayEnv); 118 | $process->setTimeout($timeout); 119 | 120 | // Handle interrupt signal 121 | pcntl_signal(SIGINT, function () use ($process) { 122 | $process->stop(); 123 | }); 124 | 125 | // Run the process and output 126 | $process->mustRun(); 127 | $output->write($process->getErrorOutput()); 128 | $output->write($process->getOutput()); 129 | 130 | if ($showProgress) { 131 | $progressBar->advance(); 132 | } 133 | $i++; 134 | } 135 | 136 | if ($showProgress) { 137 | $progressBar->finish(); 138 | } 139 | 140 | return Cli::RETURN_SUCCESS; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2023 Username - Benjamin Calef 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Model/Dimension/ParallelStoreProcessor.php: -------------------------------------------------------------------------------- 1 | forkedProcessorRunner = $forkedProcessorRunner; 33 | $this->arrayWrapperFactory = $arrayWrapperFactory; 34 | $this->storeRepository = $storeRepository; 35 | } 36 | 37 | /** 38 | * @param callable $callback 39 | * @param int|null $maxChildrenProcess 40 | * @param bool $onlyActiveStores 41 | * @param bool $withDefaultStore 42 | * @return void 43 | */ 44 | public function process( 45 | callable $callback, 46 | int $maxChildrenProcess = null, 47 | bool $onlyActiveStores = true, 48 | bool $withDefaultStore = false 49 | ): void { 50 | $stores = array_filter($this->storeRepository->getList(), 51 | function (StoreInterface $store) use ($onlyActiveStores, $withDefaultStore) { 52 | if (!$withDefaultStore && (int)$store->getId() === 0) { 53 | return false; 54 | } 55 | if ($onlyActiveStores && !$store->getIsActive()) { 56 | return false; 57 | } 58 | return true; 59 | }); 60 | 61 | /** @var ArrayWrapper $itemProvider */ 62 | $itemProvider = $this->arrayWrapperFactory->create([ 63 | 'items' => $stores, 64 | 'pageSize' => 1 65 | ]); 66 | $storeCount = $itemProvider->getSize(); 67 | $maxChildrenProcess = ($maxChildrenProcess >= $storeCount || $maxChildrenProcess === null) 68 | ? $storeCount : $maxChildrenProcess; 69 | 70 | $this->forkedProcessorRunner->run($itemProvider, $callback, $maxChildrenProcess); 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Model/Dimension/ParallelWebsiteProcessor.php: -------------------------------------------------------------------------------- 1 | forkedProcessorRunner = $forkedProcessorRunner; 33 | $this->arrayWrapperFactory = $arrayWrapperFactory; 34 | $this->websiteRepository = $websiteRepository; 35 | } 36 | 37 | /** 38 | * @param callable $callback 39 | * @param int|null $maxChildrenProcess 40 | * @param bool $withDefaultWebsite 41 | * @return void 42 | */ 43 | public function process( 44 | callable $callback, 45 | int $maxChildrenProcess = null, 46 | bool $withDefaultWebsite = false 47 | ): void { 48 | $websites = array_filter($this->websiteRepository->getList(), 49 | function (WebsiteInterface $website) use ($withDefaultWebsite) { 50 | if (!$withDefaultWebsite && (int)$website->getId() === 0) { 51 | return false; 52 | } 53 | return true; 54 | }); 55 | 56 | /** @var ArrayWrapper $itemProvider */ 57 | $itemProvider = $this->arrayWrapperFactory->create([ 58 | 'items' => $websites, 59 | 'pageSize' => 1 60 | ]); 61 | $websiteCount = $itemProvider->getSize(); 62 | $maxChildrenProcess = ($maxChildrenProcess >= $websiteCount || $maxChildrenProcess === null) 63 | ? $websiteCount : $maxChildrenProcess; 64 | 65 | $this->forkedProcessorRunner->run($itemProvider, $callback, $maxChildrenProcess); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /Model/ForkedArrayProcessor.php: -------------------------------------------------------------------------------- 1 | forkedProcessorRunner = $forkedProcessorRunner; 31 | $this->arrayWrapperFactory = $arrayWrapperFactory; 32 | } 33 | 34 | /** 35 | * @param array $array 36 | * @param callable $callback 37 | * @param int $pageSize 38 | * @param int $maxChildrenProcess 39 | * @return void 40 | */ 41 | public function process( 42 | array $array, 43 | callable $callback, 44 | int $pageSize = 1000, 45 | int $maxChildrenProcess = 10 46 | ): void { 47 | /** @var ArrayWrapper $itemProvider */ 48 | $itemProvider = $this->arrayWrapperFactory->create([ 49 | 'items' => $array, 50 | 'pageSize' => $pageSize 51 | ]); 52 | 53 | $this->forkedProcessorRunner->run($itemProvider, $callback, $maxChildrenProcess); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Model/ForkedCollectionProcessor.php: -------------------------------------------------------------------------------- 1 | forkedProcessorRunner = $forkedProcessorRunner; 32 | $this->collectionWrapperFactory = $collectionWrapperFactory; 33 | } 34 | 35 | /** 36 | * @param Collection $collection 37 | * @param callable $callback 38 | * @param int $pageSize 39 | * @param int $maxChildrenProcess 40 | * @param bool $isIdempotent 41 | * @return void 42 | */ 43 | public function process( 44 | Collection $collection, 45 | callable $callback, 46 | int $pageSize = 1000, 47 | int $maxChildrenProcess = 10, 48 | bool $isIdempotent = true 49 | ): void { 50 | /** @var CollectionWrapper $itemProvider */ 51 | $itemProvider = $this->collectionWrapperFactory->create([ 52 | 'collection' => $collection, 53 | 'pageSize' => $pageSize, 54 | 'maxChildrenProcess' => $maxChildrenProcess, 55 | 'isIdempotent' => $isIdempotent 56 | ]); 57 | 58 | $this->forkedProcessorRunner->run($itemProvider, $callback, $maxChildrenProcess); 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Model/ForkedSearchResultProcessor.php: -------------------------------------------------------------------------------- 1 | forkedProcessorRunner = $forkedProcessorRunner; 29 | $this->searchResultWrapperFactory = $searchResultWrapperFactory; 30 | } 31 | 32 | /** 33 | * @param SearchCriteria $searchCriteria 34 | * @param $repository 35 | * @param callable $callback 36 | * @param int $pageSize 37 | * @param int $maxChildrenProcess 38 | * @param bool $isIdempotent 39 | * @return void 40 | */ 41 | public function process( 42 | SearchCriteria $searchCriteria, 43 | $repository, 44 | callable $callback, 45 | int $pageSize = 1000, 46 | int $maxChildrenProcess = 10, 47 | bool $isIdempotent = true 48 | ): void { 49 | if (!method_exists($repository, 'getList')) { 50 | throw new InvalidArgumentException('The repository class must have a method called "getList"'); 51 | } 52 | 53 | /** @var SearchResultWrapper $itemProvider */ 54 | $itemProvider = $this->searchResultWrapperFactory->create([ 55 | 'searchCriteria' => $searchCriteria, 56 | 'repository' => $repository, 57 | 'pageSize' => $pageSize, 58 | 'maxChildrenProcess' => $maxChildrenProcess, 59 | 'isIdempotent' => $isIdempotent 60 | ]); 61 | 62 | $this->forkedProcessorRunner->run($itemProvider, $callback, $maxChildrenProcess); 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Model/ItemProvider/ArrayWrapper.php: -------------------------------------------------------------------------------- 1 | items = $items; 28 | $this->pageSize = $pageSize; 29 | } 30 | 31 | /** 32 | * @inheritDoc 33 | */ 34 | public function setCurrentPage(int $currentPage): void 35 | { 36 | $this->currentPage = $currentPage; 37 | } 38 | 39 | /** 40 | * @inheirtDoc 41 | */ 42 | public function getSize(): int 43 | { 44 | return count($this->items); 45 | } 46 | 47 | /** 48 | * @inheritDoc 49 | */ 50 | public function getPageSize(): int 51 | { 52 | return $this->pageSize; 53 | } 54 | 55 | /** 56 | * @inheritDoc 57 | */ 58 | public function getTotalPages(): int 59 | { 60 | return (int)ceil($this->getSize() / $this->pageSize); 61 | } 62 | 63 | /** 64 | * @inheritDoc 65 | */ 66 | public function getItems(): array 67 | { 68 | $offset = ($this->currentPage - 1) * $this->pageSize; 69 | 70 | return array_slice($this->items, $offset, $this->pageSize); 71 | } 72 | 73 | /** 74 | * @inheirtDoc 75 | */ 76 | public function isIdempotent(): bool 77 | { 78 | return true; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /Model/ItemProvider/CollectionWrapper.php: -------------------------------------------------------------------------------- 1 | collection = $collection; 39 | $this->pageSize = $pageSize; 40 | $this->maxChildrenProcess = $maxChildrenProcess; 41 | $this->isIdempotent = $maxChildrenProcess > 1 ? $isIdempotent : true; 42 | } 43 | 44 | /** 45 | * @inheirtDoc 46 | */ 47 | public function setCurrentPage(int $currentPage): void 48 | { 49 | $this->collection->setPageSize($this->getPageSize()); 50 | if (!$this->isIdempotent()) { 51 | $moduloPage = $currentPage % $this->maxChildrenProcess; 52 | $currentPage = $moduloPage === 0 ? $this->maxChildrenProcess : $moduloPage; 53 | } 54 | $this->collection->setCurPage($currentPage); 55 | } 56 | 57 | /** 58 | * @inheirtDoc 59 | */ 60 | public function getSize(): int 61 | { 62 | $this->collection->clear(); 63 | $this->collection->setPageSize(null); 64 | $this->collection->setCurPage(null); 65 | 66 | return $this->collection->getSize(); 67 | } 68 | 69 | /** 70 | * @inheirtDoc 71 | */ 72 | public function getPageSize(): int 73 | { 74 | return (int)$this->pageSize; 75 | } 76 | 77 | /** 78 | * @inheirtDoc 79 | */ 80 | public function getTotalPages(): int 81 | { 82 | return (int)ceil($this->getSize() / $this->getPageSize()); 83 | } 84 | 85 | /** 86 | * @inheirtDoc 87 | */ 88 | public function getItems(): array 89 | { 90 | $this->collection->load(false, true); 91 | 92 | return $this->collection->getItems(); 93 | } 94 | 95 | /** 96 | * @inheirtDoc 97 | */ 98 | public function isIdempotent(): bool 99 | { 100 | return $this->isIdempotent; 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Model/ItemProvider/ItemProviderInterface.php: -------------------------------------------------------------------------------- 1 | searchCriteria = $searchCriteria; 45 | $this->repository = $repository; 46 | $this->pageSize = $pageSize; 47 | $this->maxChildrenProcess = $maxChildrenProcess; 48 | $this->isIdempotent = $maxChildrenProcess > 1 ? $isIdempotent : true; 49 | } 50 | 51 | /** 52 | * @inheirtDoc 53 | */ 54 | public function setCurrentPage(int $currentPage): void 55 | { 56 | $this->searchCriteria->setPageSize($this->getPageSize()); 57 | if (!$this->isIdempotent()) { 58 | $moduloPage = $currentPage % $this->maxChildrenProcess; 59 | $currentPage = $moduloPage === 0 ? $this->maxChildrenProcess : $moduloPage; 60 | } 61 | $this->searchCriteria->setCurrentPage($currentPage); 62 | } 63 | 64 | /** 65 | * @inheirtDoc 66 | */ 67 | public function getSize(): int 68 | { 69 | $this->searchCriteria->setPageSize(null); 70 | $this->searchCriteria->setCurrentPage(null); 71 | 72 | return $this->getSearchResults()->getTotalCount(); 73 | } 74 | 75 | /** 76 | * @inheirtDoc 77 | */ 78 | public function getPageSize(): int 79 | { 80 | return (int)$this->pageSize; 81 | } 82 | 83 | /** 84 | * @inheirtDoc 85 | */ 86 | public function getTotalPages(): int 87 | { 88 | return (int)ceil($this->getSize() / $this->getPageSize()); 89 | } 90 | 91 | /** 92 | * @inheirtDoc 93 | */ 94 | public function getItems(): array 95 | { 96 | return $this->getSearchResults()->getItems(); 97 | } 98 | 99 | /** 100 | * @inheirtDoc 101 | */ 102 | public function isIdempotent(): bool 103 | { 104 | return $this->isIdempotent; 105 | } 106 | 107 | /** 108 | * @return SearchResultsInterface 109 | */ 110 | public function getSearchResults(): SearchResultsInterface 111 | { 112 | return $this->repository->getList($this->searchCriteria); 113 | } 114 | } 115 | -------------------------------------------------------------------------------- /Model/Processor/ForkedProcessor.php: -------------------------------------------------------------------------------- 1 | logger = $logger; 44 | $this->itemProvider = $itemProvider; 45 | $this->callback = $callback; 46 | $this->maxChildrenProcess = $maxChildrenProcess; 47 | pcntl_signal(SIGINT, [$this, 'handleSig']); 48 | pcntl_signal(SIGTERM, [$this, 'handleSig']); 49 | } 50 | 51 | public function process(): void 52 | { 53 | while ($this->running) { 54 | if ($this->maxChildrenProcess > 1) { 55 | $this->handleMultipleChildProcesses(); 56 | } else { 57 | $this->handleSingleChildProcesses(); 58 | } 59 | } 60 | } 61 | 62 | public function handleSig(): void 63 | { 64 | $this->running = false; 65 | } 66 | 67 | private function handleSingleChildProcesses(): void 68 | { 69 | $currentPage = 1; 70 | $childProcessCounter = 0; 71 | $totalPages = $this->itemProvider->getTotalPages(); 72 | if ($totalPages <= 0) { 73 | $this->logger->info('There is nothing to process'); 74 | $this->running = false; 75 | return; 76 | } 77 | 78 | while ($currentPage <= $totalPages) { 79 | // create fork 80 | $pid = pcntl_fork(); 81 | if ($pid == -1) { 82 | $this->logger->error('Could not fork the process'); 83 | } elseif ($pid) { 84 | // parent process 85 | $childProcessCounter++; 86 | } else { 87 | // child process 88 | $this->processChild($currentPage, $totalPages, $childProcessCounter); 89 | register_shutdown_function(function() { 90 | posix_kill(getmypid(), SIGKILL); 91 | }); 92 | exit; 93 | } 94 | 95 | $pid = pcntl_waitpid($pid, $status); 96 | if (pcntl_wtermsig($status) != 9) { 97 | $this->logger->error('Error with child process', [ 98 | 'pid' => $pid, 99 | 'exit_code' => $status 100 | ]); 101 | } 102 | 103 | $childProcessCounter--; 104 | $currentPage++; 105 | if ($currentPage > $totalPages) { 106 | $this->running = false; 107 | break; 108 | } 109 | } 110 | } 111 | 112 | private function handleMultipleChildProcesses(): void 113 | { 114 | $currentPage = 1; 115 | $childProcessCounter = 0; 116 | $childPids = []; 117 | $totalPages = $this->itemProvider->getTotalPages(); 118 | if ($totalPages <= 0) { 119 | $this->logger->info('There is nothing to process'); 120 | $this->running = false; 121 | return; 122 | } 123 | 124 | while ($currentPage <= $totalPages) { 125 | // manage children 126 | while ($childProcessCounter >= $this->maxChildrenProcess) { 127 | $pid = pcntl_wait($status); 128 | if (pcntl_wtermsig($status) != 9) { 129 | $this->logger->error('Error with child process', [ 130 | 'pid' => $pid, 131 | 'exit_code' => $status 132 | ]); 133 | unset($childPids[$pid]); 134 | } 135 | $childProcessCounter--; 136 | } 137 | 138 | // create fork 139 | $pid = pcntl_fork(); 140 | if ($pid == -1) { 141 | $this->logger->error('Could not fork the process'); 142 | } elseif ($pid) { 143 | // parent process 144 | $childProcessCounter++; 145 | $childPids[$pid] = $currentPage; 146 | } else { 147 | // child process 148 | $this->processChild($currentPage, $totalPages, $childProcessCounter); 149 | register_shutdown_function(function() { 150 | posix_kill(getmypid(), SIGKILL); 151 | }); 152 | exit; 153 | } 154 | 155 | $currentPage++; 156 | if ($currentPage > $totalPages) { 157 | $this->running = false; 158 | break; 159 | } 160 | } 161 | 162 | // wait children process before releasing parent 163 | while ($childProcessCounter > 0) { 164 | $pid = pcntl_wait($status); 165 | $childProcessCounter--; 166 | if (pcntl_wtermsig($status) != 9) { 167 | $this->logger->error('Error with child process', [ 168 | 'pid' => $pid, 169 | 'exit_code' => $status 170 | ]); 171 | unset($childPids[$pid]); 172 | } 173 | $this->logger->info('Finished last child process', [ 174 | 'pid' => $pid, 175 | 'child_process_counter' => $childProcessCounter + 1, 176 | 'memory_usage' => $this->getMemoryUsage() 177 | ]); 178 | } 179 | 180 | // Fallback based on missing pages 181 | $missingPages = array_unique(array_diff(range(1, $totalPages), $childPids)); 182 | if (!empty($missingPages)) { 183 | $this->logger->info('Fallback on missing pages', ['missing_pages' => array_values($missingPages)]); 184 | } 185 | foreach ($missingPages as $page) { 186 | $this->processChild($page, $totalPages, 0); 187 | } 188 | 189 | // Fallback based on database query 190 | if (!$this->itemProvider->isIdemPotent()) { 191 | $size = $this->itemProvider->getSize(); 192 | $this->logger->info('Missing items from original query collection', ['total_items' => $size]); 193 | if ($size !== 0) { 194 | $this->handleMultipleChildProcesses(); 195 | } 196 | } 197 | } 198 | 199 | private function processChild(int $currentPage, int $totalPages, int $childProcessCounter): void 200 | { 201 | $itemProceed = 0; 202 | try { 203 | $this->itemProvider->setCurrentPage($currentPage); 204 | $items = $this->itemProvider->getItems(); 205 | } catch (Throwable $e) { 206 | $this->logger->error('Error while loading collection', [ 207 | 'pid' => getmypid(), 208 | 'current_page' => $currentPage, 209 | 'exception' => $e 210 | ]); 211 | register_shutdown_function(function() { 212 | posix_kill(getmypid(), SIGABRT); 213 | }); 214 | exit; 215 | } 216 | 217 | $this->logger->info('Running child process', [ 218 | 'pid' => getmypid(), 219 | 'child_process_counter' => $childProcessCounter + 1, 220 | 'memory_usage' => $this->getMemoryUsage(), 221 | 'item_counter' => count($items), 222 | 'current_page' => $currentPage, 223 | 'remaining_pages' => $totalPages - $currentPage, 224 | 'total_pages' => $totalPages 225 | ]); 226 | 227 | foreach ($items as $item) { 228 | try { 229 | call_user_func($this->callback, $item); 230 | $itemProceed++; 231 | } catch (Throwable $e) { 232 | $this->logger->error('Error on callback function will processing item', [ 233 | 'pid' => getmypid(), 234 | 'current_page' => $currentPage, 235 | 'exception' => $e, 236 | ]); 237 | } 238 | } 239 | 240 | if ($itemProceed === 0) { 241 | register_shutdown_function(function() { 242 | posix_kill(getmypid(), SIGABRT); 243 | }); 244 | exit; 245 | } 246 | 247 | $this->logger->info('Finished child process', [ 248 | 'pid' => getmypid(), 249 | 'child_process_counter' => $childProcessCounter + 1, 250 | 'memory_usage' => $this->getMemoryUsage(), 251 | 'current_page' => $currentPage, 252 | 'remaining_pages' => $totalPages - $currentPage, 253 | 'total_pages' => $totalPages, 254 | 'item_proceed' => $itemProceed 255 | ]); 256 | } 257 | 258 | private function getMemoryUsage(): string 259 | { 260 | return round(memory_get_usage(true) / (1024 * 1024), 2) . 'MB'; 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /Model/Processor/ForkedProcessorRunner.php: -------------------------------------------------------------------------------- 1 | forkedProcessorFactory = $forkedProcessorFactory; 22 | } 23 | 24 | /** 25 | * @param ItemProviderInterface $itemProvider 26 | * @param callable $callback 27 | * @param int $maxChildrenProcess 28 | * @return void 29 | */ 30 | public function run( 31 | ItemProviderInterface $itemProvider, 32 | callable $callback, 33 | int $maxChildrenProcess 34 | ): void { 35 | /** @var $forkedProcessor ForkedProcessor */ 36 | $forkedProcessor = $this->forkedProcessorFactory->create([ 37 | 'itemProvider' => $itemProvider, 38 | 'callback' => $callback, 39 | 'maxChildrenProcess' => $maxChildrenProcess, 40 | ]); 41 | 42 | $forkedProcessor->process(); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Multi-Threading for Magento 2 2 | 3 | This module is a powerful tool for developers who want to process large data sets in 4 | a short amount of time. It allows you to process large collections of data in parallel 5 | using multiple child processes, improving performance and reducing processing time. 6 | 7 | ## Installation 8 | ```php 9 | composer require zepgram/module-multi-threading 10 | bin/magento module:enable Zepgram_MultiThreading 11 | bin/magento setup:upgrade 12 | ``` 13 | 14 | ## Usage 15 | 16 | These classes allows you to process a search criteria, a collection or an array using multi-threading. 17 | 18 | ### ForkedSearchResultProcessor 19 | 20 | ```php 21 | use Zepgram\MultiThreading\Model\ForkedSearchResultProcessor; 22 | use Magento\Catalog\Api\ProductRepositoryInterface; 23 | use Magento\Framework\Api\SearchCriteriaBuilder; 24 | 25 | class MyAwesomeClass 26 | { 27 | /** @var ForkedSearchResultProcessor */ 28 | private $forkedSearchResultProcessor; 29 | 30 | /** @var ProductRepositoryInterface */ 31 | private $productRepository; 32 | 33 | public function __construct( 34 | ForkedSearchResultProcessor $forkedSearchResultProcessor, 35 | ProductRepositoryInterface $productRepository, 36 | SearchCriteriaBuilder $searchCriteriaBuilder 37 | ) { 38 | $this->forkedSearchResultProcessor = $forkedSearchResultProcessor; 39 | $this->productRepository = $productRepository; 40 | $this->searchCriteriaBuilder = $searchCriteriaBuilder; 41 | } 42 | 43 | $searchCriteria = $this->searchCriteriaBuilder->create(); 44 | $productRepository = $this->productRepository; 45 | $callback = function ($item) { 46 | $item->getData(); 47 | // do your business logic here 48 | }; 49 | 50 | $this->forkedSearchResultProcessor->process( 51 | $searchCriteria, 52 | $productRepository, 53 | $callback, 54 | $pageSize = 1000, 55 | $maxChildrenProcess = 10, 56 | $isIdempotent = true 57 | ); 58 | } 59 | ``` 60 | 61 | ### ForkedCollectionProcessor 62 | ```php 63 | use Zepgram\MultiThreading\Model\ForkedCollectionProcessor; 64 | use Magento\Catalog\Model\ResourceModel\Product\CollectionFactory; 65 | 66 | class MyAwesomeClass 67 | { 68 | /** @var ForkedCollectionProcessor */ 69 | private $forkedCollectionProcessor; 70 | 71 | public function __construct( 72 | ForkedCollectionProcessor $forkedCollectionProcessor, 73 | CollectionFactory $collectionFactory 74 | ) { 75 | $this->forkedCollectionProcessor = $forkedCollectionProcessor; 76 | $this->collectionFactory = $collectionFactory; 77 | } 78 | 79 | $collection = $this->collectionFactory->create(); 80 | $callback = function ($item) { 81 | $item->getData(); 82 | // do your business logic here 83 | }; 84 | 85 | $this->forkedCollectionProcessor->process( 86 | $searchCriteria, 87 | $productRepository, 88 | $callback, 89 | $pageSize = 1000, 90 | $maxChildrenProcess = 10, 91 | $isIdempotent = true 92 | ); 93 | } 94 | ``` 95 | 96 | ### ForkedArrayProcessor 97 | This class allows you to process an array of data using multi-threading. 98 | 99 | ```php 100 | use Zepgram\MultiThreading\Model\ForkedArrayProcessor; 101 | 102 | class MyAwesomeClass 103 | { 104 | /** @var ForkedArrayProcessor */ 105 | private $forkedArrayProcessor; 106 | 107 | public function __construct(ForkedArrayProcessor $forkedArrayProcessor) 108 | { 109 | $this->forkedArrayProcessor = $forkedArrayProcessor; 110 | } 111 | 112 | $array = [1,2,3,4,5,...]; 113 | $callback = function ($item) { 114 | echo $item; 115 | // do your business logic here 116 | }; 117 | 118 | $this->forkedArrayProcessor->process( 119 | $array, 120 | $callback, 121 | $pageSize = 2, 122 | $maxChildrenProcess = 2 123 | ); 124 | } 125 | ``` 126 | 127 | ### ParallelStoreProcessor or ParallelWebsiteProcessor 128 | 129 | ```php 130 | use Zepgram\MultiThreading\Model\Dimension\ParallelStoreProcessor; 131 | use Magento\Catalog\Model\ResourceModel\Product\Collection; 132 | use Magento\Catalog\Model\ResourceModel\Product\CollectionFactory; 133 | 134 | class MyAwesomeClass 135 | { 136 | /** @var ParallelStoreProcessor */ 137 | private $parallelStoreProcessor; 138 | 139 | /** @var CollectionFactory */ 140 | private $collectionFactory; 141 | 142 | public function __construct( 143 | ParallelStoreProcessor $parallelStoreProcessor, 144 | CollectionFactory $collectionFactory 145 | ) { 146 | $this->parallelStoreProcessor = $parallelStoreProcessor; 147 | $this->collectionFactory = $collectionFactory; 148 | } 149 | 150 | $array = [1,2,3,4,5,...]; 151 | $callback = function (StoreInterface $store) { 152 | // retrieve data from database foreach stores (do not load the collection !) 153 | $collection = $this->collectionFactory->create(); 154 | $collection->addFieldToFilter('type_id', 'simple') 155 | ->addFieldToSelect(['sku', 'description', 'created_at']) 156 | ->setStoreId($store->getId()) 157 | ->addStoreFilter($store->getId()) 158 | ->distinct(true); 159 | 160 | // handle pagination system to avoid memory leak 161 | $currentPage = 1; 162 | $pageSize = 1000; 163 | $collection->setPageSize($pageSize); 164 | $totalPages = $collection->getLastPageNumber(); 165 | while ($currentPage <= $totalPages) { 166 | $collection->clear(); 167 | $collection->setCurPage($currentPage); 168 | foreach ($collection->getItems() as $product) { 169 | // do your business logic here 170 | } 171 | $currentPage++; 172 | } 173 | }; 174 | 175 | // your collection will be processed foreach store by a dedicated child process 176 | $this->parallelStoreProcessor->process( 177 | $callback, 178 | $maxChildrenProcess = null, 179 | $onlyActiveStores = true, 180 | $withDefaultStore = false 181 | ); 182 | } 183 | ``` 184 | 185 | ### bin/magento thread:processor command 186 | 187 | This command allows running a command indefinitely in a dedicated thread using 188 | the Process Symfony Component. 189 | ```php 190 | bin/magento thread:processor [--timeout=] [--iterations=] [--environment=] [--progress] 191 | ``` 192 | 193 | #### Options 194 | 195 | - `timeout`: Define the process timeout in seconds (default: 300) 196 | - `iterations`: Define the number of iteration (default: 0) 197 | - `environment`: Set environment variables separate by comma 198 | - `progress`: Show progress bar while executing command 199 | 200 | ### How it works 201 | The `thread:processor` command creates a dedicated child process to execute existing command line. 202 | The child process runs the command specified by the user, while the parent process 203 | monitors the child process and can act accordingly. You can define iterations and execute the same command 204 | multiple times with a dedicated child foreach execution. 205 | 206 | The `ForkedSearchResultProcessor`,`ForkedCollectionProcessor` and `ForkedArrayProcessor` classes 207 | use a similar approach to process a search criteria or a collection. The process is divided 208 | into several pages, and for each page, a child process is created to run the callback 209 | function specified by the user on each item of that page. 210 | 211 | The `ParallelStoreProcessor` and `ParallelWebsiteProcessor` classes are designed to make it easier 212 | to process a list of stores or websites in parallel. To use either of these classes, you'll need to 213 | provide a callback function that will be called for each store or website in the list. The callback 214 | function should take one parameter, which will be a single store or website object.
215 | Each store or website will be passed to the callback function in a separate process, 216 | allowing faster processing times. 217 | The number of children process cannot exceed the number of stores or websites: for example, 218 | if you have 10 stores, the maximum number of child processes that can be created in parallel is 10. 219 | 220 | #### Here is a breakdown of the parameters: 221 | - `$collection`/`$searchCriteria`/`$array`: The first parameter is the data source, 222 | either a `Magento\Framework\Api\SearchCriteriaInterface` for ForkedSearchResultProcessor or 223 | a `Magento\Framework\Data\Collection` for ForkedCollectionProcessor, and an `array` 224 | for ForkedArrayProcessor 225 | 226 | - `$callback`: This parameter is a callable that will be executed on each item of the collection. 227 | It is a callback function that is passed the current item from the collection to be processed. 228 | This function should contain the business logic that should be executed on each item. 229 | 230 | - `$pageSize`: This parameter is used to set the number of items per page. 231 | It is used to paginate the collection so that it can be processed in smaller chunks. 232 | 233 | - `$maxChildrenProcess`: This parameter is used to set the maximum number of child 234 | processes that can be run simultaneously. This is used to control the number of threads 235 | that will be used by the multi-threading process. If set to 1, by definition you will have no parallelization, 236 | the parent process will wait the child process to finish before creating another one. 237 | 238 | - `$isIdempotent`: This parameter is a flag set to `true` by default and can be used for `ForkedSearchResultProcessor` 239 | or `ForkedCollectionProcessor` when your `$maxChildrenProcess` is greater than one. 240 | While fetching data from database with `ForkedSearchResult` and `ForkedCollectionProcessor` you may change values 241 | queried: by modifying items on columns queried you will change the nature of the initial collection query and at the end, 242 | the OFFSET limit in the query will be invalid because the native pagination system expect the pagination to be 243 | processed by only one process. To avoid that, set `$isIdempotent` to `false`.
244 | E.G.: In your collection query, you request all products `disabled`, in your callback method you `enable` and save 245 | them in database, then in this particular case you are modifying the column that you request in your collection, 246 | your query is not idempotent. 247 | 248 | ### Memory Limit 249 | This module allows to bypass the limitation of the memory limit, because the memory 250 | limit is reset on each child process creation. This means that even if the memory limit 251 | is set to a low value, this module can still process large amounts of data without 252 | running out of memory. However, it is important to keep in mind that this also means 253 | that the overall resource usage will be higher, so it is important to monitor the 254 | system and adjust the parameters accordingly. 255 | 256 | ### Limitations 257 | This module uses `pcntl_fork()` function which is not available on Windows. 258 | 259 | ### Conclusion 260 | This module provides a useful tool for running commands or processing collections 261 | and search criteria in a multi-threaded way, making it a great solution for improving 262 | performance and reducing execution time. 263 | The module is easy to install and use, and provides options for controlling the number 264 | of child processes, timeout, and environment variables. 265 | 266 | ### Disclaimer 267 | The Multi-Threading for Magento 2 module is provided as is, without any guarantees or warranties. 268 | While this module has been tested and is believed to be functional, it is important to note 269 | that the use of multi-threading in PHP can be complex and may have unintended consequences. 270 | As such, it is the responsibility of the user of this module to thoroughly test it in a 271 | development environment before deploying it to a production environment. 272 | I decline all responsibility for any issues or damages that may occur as a result of using 273 | this module. With great power comes great responsibility, use it wisely. 274 | 275 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "zepgram/module-multi-threading", 3 | "description": "This module is a powerful tool for developers who want to process large data sets in a short amount of time", 4 | "type": "magento2-module", 5 | "version": "0.1.6", 6 | "authors": [ 7 | { 8 | "name": "Benjamin Calef", 9 | "email": "zepgram@gmail.com" 10 | } 11 | ], 12 | "require": { 13 | "magento/framework": "^101.0.0|^102.0.0|^103.0.0", 14 | "magento/module-store": "^101", 15 | "ext-pcntl": "*", 16 | "ext-posix": "*" 17 | }, 18 | "autoload": { 19 | "files": [ 20 | "registration.php" 21 | ], 22 | "psr-4": { 23 | "Zepgram\\MultiThreading\\": "" 24 | } 25 | }, 26 | "license": "MIT", 27 | "repositories": { 28 | "repo.magento.com": { 29 | "type": "composer", 30 | "url": "https://repo.magento.com/" 31 | } 32 | }, 33 | "config": { 34 | "allow-plugins": { 35 | "magento/composer-dependency-version-audit-plugin": true 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /etc/di.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 9 | 10 | 11 | 12 | 13 | Zepgram\MultiThreading\Console\Command\ThreadProcessorCommand 14 | 15 | 16 | 17 | 18 | 19 | 20 | Magento\Framework\Filesystem\Driver\File 21 | var/log/zepgram/multi_threading.log 22 | 23 | 24 | 25 | 26 | 27 | Zepgram\MultiThreading\Logger\Handler 28 | 29 | 30 | 31 | 32 | 33 | Zepgram\MultiThreading\Logger\Logger 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /etc/module.xml: -------------------------------------------------------------------------------- 1 | 2 | 7 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /registration.php: -------------------------------------------------------------------------------- 1 |