├── Api ├── IndexGenerationObserver.php ├── IndexGenerator.php ├── IndexRecord.php ├── IndexRecordLoader.php ├── IndexRecordMutable.php ├── IndexScope.php ├── IndexScopeProvider.php ├── IndexStorageMap.php ├── IndexStorageWriter.php ├── IndexStorageWriterFactory.php └── IndexWriter.php ├── LICENSE ├── Model ├── ArrayIndexRecord.php ├── ArrayIndexRecordFactory.php ├── CompositeIndexRecordLoader.php ├── IndexRangeGenerator.php ├── Indexer.php ├── MinMaxIndexRangeGenerator.php └── ResourceModel │ ├── BatchStatementRegistry.php │ ├── IndexTableStructure.php │ ├── InsertOnDuplicateSqlGenerator.php │ ├── TableBatchIndexStorageWriter.php │ ├── TableBatchIndexStorageWriterFactory.php │ └── TableIndexRangeGenerator.php ├── README.md ├── Test ├── FakeBatchStatementRegistry.php ├── FakeIndexStorageWriter.php ├── FakeIndexTableStructure.php └── Unit │ └── Model │ ├── ArrayIndexRecordTest.php │ ├── CompositeIndexRecordLoaderTest.php │ ├── IndexerTest.php │ ├── MinMaxIndexRangeGeneratorTest.php │ └── ResourceModel │ ├── InsertOnDuplicateSqlGeneratorTest.php │ └── TableBatchIndexStorageWriterTest.php ├── composer.json ├── etc ├── di.xml └── module.xml └── registration.php /Api/IndexGenerationObserver.php: -------------------------------------------------------------------------------- 1 | data = []; 27 | $this->scopeData = []; 28 | } 29 | 30 | public function set(int $entityId, array $data): void 31 | { 32 | $this->data[$entityId] = $data; 33 | } 34 | 35 | public function add(int $entityId, string $field, mixed $value): void 36 | { 37 | $this->data[$entityId][$field] = $value; 38 | } 39 | 40 | public function append(int $entityId, string $field, string $key, mixed $value): void 41 | { 42 | $this->data[$entityId][$field][$key] = $value; 43 | } 44 | 45 | public function addInScope(int $entityId, int $storeId, string $field, mixed $value): void 46 | { 47 | if (!isset($this->data[$entityId])) { 48 | return; 49 | } 50 | 51 | $this->scopeData[$entityId][$storeId][$field] = $value; 52 | } 53 | 54 | public function getInScope(int $entityId, int $scopeId, string $field): mixed 55 | { 56 | return $this->scopeData[$entityId][$scopeId][$field] 57 | ?? $this->scopeData[$entityId][0][$field] 58 | ?? null; 59 | } 60 | 61 | public function get(int $entityId, string $field): mixed 62 | { 63 | return $this->data[$entityId][$field] ?? null; 64 | } 65 | 66 | public function listEntityIds(): iterable 67 | { 68 | foreach ($this->data as $entityId => $item) { 69 | yield $entityId; 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Model/ArrayIndexRecordFactory.php: -------------------------------------------------------------------------------- 1 | loaders as $loader) { 28 | $loader->loadByRange($indexScope, $data, $minEntityId, $maxEntityId); 29 | } 30 | } 31 | 32 | public function loadByIds(IndexScope $indexScope, IndexRecordMutable $data, array $entityIds): void 33 | { 34 | foreach ($this->loaders as $loader) { 35 | $loader->loadByIds($indexScope, $data, $entityIds); 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Model/IndexRangeGenerator.php: -------------------------------------------------------------------------------- 1 | arrayIndexRecordFactory->create(); 43 | 44 | foreach ($this->indexScopeProvider->getScopes() as $scope) { 45 | $this->indexGenerationObserver->beforeGeneration($scope); 46 | $writer = $this->indexStorageWriterFactory->createFullReindex($scope); 47 | foreach ($this->indexRangeGenerator->ranges($this->fullIndexBatchSize) as $minEntityId => $maxEntityId) { 48 | $this->indexRecordLoader->loadByRange($scope, $data, $minEntityId, $maxEntityId); 49 | $this->indexGenerator->process($scope, $data, $writer); 50 | $data->reset(); 51 | } 52 | $writer->finish(); 53 | $this->indexGenerationObserver->afterGeneration($scope); 54 | } 55 | } 56 | 57 | public function executeList(array $ids) 58 | { 59 | $this->reindexByIds($ids); 60 | } 61 | 62 | public function executeRow($id) 63 | { 64 | $this->reindexByIds([$id]); 65 | } 66 | 67 | public function execute($ids) 68 | { 69 | $this->reindexByIds($ids); 70 | } 71 | 72 | private function reindexByIds(array $ids): void 73 | { 74 | $idChunks = array_chunk( 75 | array_values(array_unique(array_map('intval', $ids), SORT_REGULAR)), 76 | $this->liveIndexBatchSize 77 | ); 78 | 79 | $data = $this->arrayIndexRecordFactory->create(); 80 | 81 | foreach ($this->indexScopeProvider->getScopes() as $scope) { 82 | $this->indexGenerationObserver->beforeGeneration($scope); 83 | $writer = $this->indexStorageWriterFactory->createPartialReindex($scope); 84 | foreach ($idChunks as $ids) { 85 | $writer->clear($ids); 86 | $this->indexRecordLoader->loadByIds($scope, $data, $ids); 87 | $this->indexGenerator->process($scope, $data, $writer); 88 | $data->reset(); 89 | } 90 | $writer->finish(); 91 | $this->indexGenerationObserver->afterGeneration($scope); 92 | } 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /Model/MinMaxIndexRangeGenerator.php: -------------------------------------------------------------------------------- 1 | start; $i <= $this->end; $i += $batchSize) { 21 | yield $i => min($i + $batchSize - 1, $this->end); 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /Model/ResourceModel/BatchStatementRegistry.php: -------------------------------------------------------------------------------- 1 | "`$column` = VALUES(`$column`)", $onUpdate) 28 | ) 29 | ); 30 | } 31 | 32 | $rowLine = rtrim(str_repeat('?,', count($columns)), ','); 33 | $rowLines = str_repeat( 34 | "($rowLine),", 35 | $rowCount 36 | ); 37 | 38 | $sql = sprintf( 39 | 'INSERT INTO `%s` (%s) VALUES %s%s', 40 | $tableName, 41 | implode(',', array_map(fn($column) => "`$column`", $columns)), 42 | rtrim( 43 | $rowLines, 44 | ',' 45 | ), 46 | $sqlOnUpdate 47 | ); 48 | 49 | return $sql; 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Model/ResourceModel/TableBatchIndexStorageWriter.php: -------------------------------------------------------------------------------- 1 | indexStorageMap->getStorageByRow($row); 32 | $this->currentBatchByTable[$tableName] = ($this->currentBatchByTable[$tableName] ?? 0) + 1; 33 | if (!isset($this->parametersByTable[$tableName])) { 34 | $this->parametersByTable[$tableName] = []; 35 | } 36 | 37 | $this->indexTableStructure->prepareRow($this->parametersByTable[$tableName], $row); 38 | 39 | if ($this->currentBatchByTable[$tableName] === $this->batchSize) { 40 | $batchSize = $this->currentBatchByTable[$tableName]; 41 | $this->executeBatch($tableName, $batchSize); 42 | } 43 | } 44 | 45 | public function clear(array $entityIds): void 46 | { 47 | // TODO: Implement clear() method. 48 | } 49 | 50 | public function finish(): void 51 | { 52 | foreach ($this->currentBatchByTable as $tableName => $batchSize) { 53 | if ($batchSize === 0) { 54 | continue; 55 | } 56 | 57 | $this->executeBatch($tableName, $batchSize); 58 | } 59 | } 60 | 61 | private function executeBatch(string $tableName, mixed $batchSize): void 62 | { 63 | if (!$this->batchInsertStatementRegistry->hasInsertStatement($tableName, $batchSize)) { 64 | $this->batchInsertStatementRegistry->createInsertStatement( 65 | $tableName, 66 | $batchSize, 67 | $this->indexTableStructure->generateInsertOnDuplicate($tableName, $batchSize) 68 | ); 69 | } 70 | 71 | $this->batchInsertStatementRegistry->executeInsertStatement( 72 | $tableName, 73 | $batchSize, 74 | $this->parametersByTable[$tableName] 75 | ); 76 | 77 | $this->currentBatchByTable[$tableName] = 0; 78 | $this->parametersByTable[$tableName] = []; 79 | } 80 | } 81 | -------------------------------------------------------------------------------- /Model/ResourceModel/TableBatchIndexStorageWriterFactory.php: -------------------------------------------------------------------------------- 1 | objectManager->create( 27 | TableBatchIndexStorageWriter::class, 28 | [ 29 | 'indexStorageMap' => $indexStorageMap, 30 | 'indexTableStructure' => $indexTableStructure, 31 | 'batchSize' => $batchSize 32 | ] 33 | ); 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Model/ResourceModel/TableIndexRangeGenerator.php: -------------------------------------------------------------------------------- 1 | resourceConnection->getConnection($this->connectionName); 30 | $primaryKey = $connection->quoteIdentifier($this->primaryKey); 31 | 32 | $select = $connection->select() 33 | ->from( 34 | $this->tableName, 35 | [ 36 | 'min' => sprintf('MIN(%s)', $primaryKey), 37 | 'max' => sprintf('MAX(%s)', $primaryKey) 38 | ] 39 | ) 40 | ->group( 41 | new Expression(sprintf( 42 | 'CEIL(%s / %d)', $primaryKey, $batchSize 43 | )) 44 | ); 45 | 46 | foreach ($select->query() as $row) { 47 | yield (int)$row['min'] => (int)$row['max']; 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Magento Indexation Framework 2 | -------------------------------------------------------------------------------- /Test/FakeBatchStatementRegistry.php: -------------------------------------------------------------------------------- 1 | insertStatements[$tableName][$batchSize]); 24 | } 25 | 26 | public function createInsertStatement(string $tableName, int $batchSize, string $sql): void 27 | { 28 | $this->insertStatements[$tableName][$batchSize] = $sql; 29 | } 30 | 31 | public function executeInsertStatement(string $tableName, int $batchSize, array $params): void 32 | { 33 | $this->executedInsertStatements[$tableName][$batchSize] = $params; 34 | } 35 | 36 | 37 | public function hasDeleteStatement(string $tableName, int $batchSize): bool 38 | { 39 | return isset($this->deleteStatements[$tableName][$batchSize]); 40 | } 41 | 42 | public function createDeleteStatement(string $tableName, int $batchSize, string $sql): void 43 | { 44 | $this->deleteStatements[$tableName][$batchSize] = $sql; 45 | } 46 | 47 | public function executeDeleteStatement(string $tableName, int $batchSize, array $params): void 48 | { 49 | $this->executedDeleteStatements[$tableName][$batchSize] = $params; 50 | } 51 | 52 | public function withInsertStatement(string $tableName, int $batchSize, string $sql): self 53 | { 54 | $registry = clone $this; 55 | $registry->createInsertStatement($tableName, $batchSize, $sql); 56 | 57 | return $registry; 58 | } 59 | 60 | public function withInsertExecuted(string $tableName, int $batchSize, array $parameters): self 61 | { 62 | $registry = clone $this; 63 | $registry->executeInsertStatement($tableName, $batchSize, $parameters); 64 | 65 | return $registry; 66 | } 67 | 68 | public function withDeleteStatement(string $tableName, int $batchSize, string $sql): self 69 | { 70 | $registry = clone $this; 71 | $registry->createDeleteStatement($tableName, $batchSize, $sql); 72 | 73 | return $registry; 74 | } 75 | 76 | public function withDeleteExecuted(string $tableName, int $batchSize, array $parameters): self 77 | { 78 | $registry = clone $this; 79 | $registry->executeDeleteStatement($tableName, $batchSize, $parameters); 80 | 81 | return $registry; 82 | } 83 | 84 | public static function create(): self 85 | { 86 | return new self(); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Test/FakeIndexStorageWriter.php: -------------------------------------------------------------------------------- 1 | pendingRows[] = $row; 21 | } 22 | 23 | public function clear(array $entityIds): void 24 | { 25 | // TODO: Implement clear() method. 26 | } 27 | 28 | public function finish(): void 29 | { 30 | $this->finishedRows[] = $this->pendingRows; 31 | $this->pendingRows = []; 32 | } 33 | 34 | public static function create(): self 35 | { 36 | return new self(); 37 | } 38 | 39 | public function withPendingRows(array ...$row): self 40 | { 41 | $storage = clone $this; 42 | $storage->pendingRows = array_merge($storage->pendingRows, $row); 43 | return $storage; 44 | } 45 | 46 | public function withFinishedRows(array ...$row): self 47 | { 48 | $storage = clone $this; 49 | $storage->finishedRows[] = $row; 50 | 51 | return $storage; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Test/FakeIndexTableStructure.php: -------------------------------------------------------------------------------- 1 | indexRecord = new ArrayIndexRecord(); 21 | } 22 | 23 | /** @test */ 24 | public function returnsEmptyContainer() 25 | { 26 | $this->assertEquals( 27 | [], 28 | iterator_to_array($this->indexRecord->listEntityIds()) 29 | ); 30 | } 31 | 32 | /** @test */ 33 | public function createsMainEntityEntry() 34 | { 35 | $this->indexRecord->set(1, ['sku' => 123]); 36 | $this->indexRecord->set(2, ['sku' => 124]); 37 | 38 | $this->assertEquals( 39 | [1, 2], 40 | iterator_to_array($this->indexRecord->listEntityIds()) 41 | ); 42 | } 43 | 44 | /** @test */ 45 | public function extendsMainEntityEntry() 46 | { 47 | $this->indexRecord->set(1, ['sku' => 123]); 48 | $this->indexRecord->set(2, ['sku' => 124]); 49 | $this->indexRecord->add(1, 'name', 'Name 1'); 50 | 51 | $this->assertEquals('Name 1', $this->indexRecord->get(1, 'name')); 52 | } 53 | 54 | /** @test */ 55 | public function extendsArrayValueOfEntity() 56 | { 57 | $this->indexRecord->set(1, ['tier_price' => []]); 58 | $this->indexRecord->append(1, 'tier_price', 'all_websites', 1); 59 | $this->indexRecord->append(1, 'tier_price', 'all_groups', 2); 60 | $this->indexRecord->append(1, 'tier_price', 'website_1', 3); 61 | 62 | 63 | $this->assertEquals( 64 | [ 65 | 'all_websites' => 1, 66 | 'all_groups' => 2, 67 | 'website_1' => 3 68 | ], 69 | $this->indexRecord->get(1, 'tier_price') 70 | ); 71 | } 72 | 73 | 74 | /** @test */ 75 | public function dataIsResetWhenRequested() 76 | { 77 | $this->indexRecord->set(1, ['item' => 1]); 78 | $this->indexRecord->set(2, ['item' => 2]); 79 | $this->indexRecord->reset(); 80 | 81 | $this->assertEquals( 82 | [], 83 | iterator_to_array($this->indexRecord->listEntityIds()) 84 | ); 85 | } 86 | 87 | /** @test */ 88 | public function takesValueFromStoreOneWhenItIsProvided() 89 | { 90 | $this->indexRecord->set(1, ['sku' => 123]); 91 | $this->indexRecord->addInScope(1, 0, 'name', 'Name in Store Default'); 92 | $this->indexRecord->addInScope(1, 1, 'name', 'Name in Store 1'); 93 | 94 | $this->assertEquals( 95 | 'Name in Store 1', 96 | $this->indexRecord->getInScope(1, 1, 'name') 97 | ); 98 | } 99 | 100 | /** @test */ 101 | public function defaultsToDefaultStoreViewIfStoreSpecificValueIsNotFound() 102 | { 103 | $this->indexRecord->set(3, []); 104 | $this->indexRecord->addInScope(3, 0, 'status', 1); 105 | $this->indexRecord->addInScope(3, 1, 'status', 2); 106 | 107 | $this->assertEquals( 108 | 1, 109 | $this->indexRecord->getInScope(3, 2, 'status') 110 | ); 111 | } 112 | 113 | /** @test */ 114 | public function defaultToNullWhenNoStoreValuesProvided() 115 | { 116 | $this->assertEquals( 117 | null, 118 | $this->indexRecord->getInScope(1, 2, 'name') 119 | ); 120 | } 121 | 122 | /** @test */ 123 | public function doesNotPopulateStoreValueWhenMainEntityIsNotSelected() 124 | { 125 | $this->indexRecord->addInScope(1, 2, 'name', 'Name in Store 2'); 126 | 127 | $this->assertEquals( 128 | null, 129 | $this->indexRecord->getInScope(1, 2, 'name') 130 | ); 131 | } 132 | 133 | /** @test */ 134 | public function returnsDataForMainField() 135 | { 136 | $this->indexRecord->set(1, ['level' => 2, 'path' => '1/2/3']); 137 | $this->assertEquals( 138 | 2, 139 | $this->indexRecord->get(1, 'level') 140 | ); 141 | } 142 | 143 | /** @test */ 144 | public function fallsBackToNullWhenEntityFieldIsNotPresent() 145 | { 146 | $this->indexRecord->set(1, ['level' => 3]); 147 | 148 | $this->assertEquals( 149 | null, 150 | $this->indexRecord->get(1, 'path') 151 | ); 152 | } 153 | 154 | /** @test */ 155 | public function storeDataIsReset() 156 | { 157 | $this->indexRecord->set(1, []); 158 | $this->indexRecord->addInScope(1, 0, 'status', 1); 159 | $this->indexRecord->reset(); 160 | 161 | $this->assertEquals( 162 | null, 163 | $this->indexRecord->getInScope(1, 1, 'status') 164 | ); 165 | } 166 | 167 | } 168 | -------------------------------------------------------------------------------- /Test/Unit/Model/CompositeIndexRecordLoaderTest.php: -------------------------------------------------------------------------------- 1 | recordLoader = new CompositeIndexRecordLoader([ 28 | $this, 29 | $this, 30 | $this 31 | ]); 32 | } 33 | 34 | /** @test */ 35 | public function invokesLoadByRangeOnAllChildLoaders() 36 | { 37 | $arrayRecordData = new ArrayIndexRecord(); 38 | $this->recordLoader->loadByRange( 39 | IndexScope::create([]), 40 | $arrayRecordData, 41 | 1, 42 | 1000 43 | ); 44 | 45 | $this->assertEquals( 46 | new ArrayIndexRecord( 47 | [ 48 | 1 => ['loadByRange' => true], 49 | 2 => ['loadByRange' => true], 50 | 3 => ['loadByRange' => true] 51 | ] 52 | ), 53 | $arrayRecordData, 54 | ); 55 | } 56 | 57 | /** @test */ 58 | public function invokesLoadByIdsOnAllChildLoaders() 59 | { 60 | $arrayRecordData = new ArrayIndexRecord(); 61 | $this->recordLoader->loadByIds( 62 | IndexScope::create([]), 63 | $arrayRecordData, 64 | [1, 2, 3] 65 | ); 66 | 67 | $this->assertEquals( 68 | new ArrayIndexRecord( 69 | [ 70 | 1 => ['loadByIds' => true], 71 | 2 => ['loadByIds' => true], 72 | 3 => ['loadByIds' => true] 73 | ] 74 | ), 75 | $arrayRecordData, 76 | ); 77 | } 78 | 79 | public function loadByRange(IndexScope $indexScope, IndexRecordMutable $data, int $minEntityId, int $maxEntityId): void 80 | { 81 | $data->set(++$this->timesCalled, ['loadByRange' => true]); 82 | } 83 | 84 | public function loadByIds(IndexScope $indexScope, IndexRecordMutable $data, array $entityIds): void 85 | { 86 | $data->set(++$this->timesCalled, ['loadByIds' => true]); 87 | } 88 | } 89 | -------------------------------------------------------------------------------- /Test/Unit/Model/IndexerTest.php: -------------------------------------------------------------------------------- 1 | indexer = new Indexer( 46 | $this, 47 | $this, 48 | $this, 49 | $this, 50 | new MinMaxIndexRangeGenerator(1, 7), 51 | $this, 52 | new ArrayIndexRecordFactory(), 53 | 3, 54 | 3 55 | ); 56 | 57 | $this->scopes['one'] = IndexScope::create([1]); 58 | $this->scopes['two'] = IndexScope::create([2]); 59 | } 60 | 61 | /** @test */ 62 | public function executesActionOnFullReindexForEveryScope() 63 | { 64 | $this->indexer->executeFull(); 65 | 66 | $this->assertSame( 67 | [ 68 | ['full', $this->scopes['one'], 1], 69 | ['full', $this->scopes['one'], 2], 70 | ['full', $this->scopes['one'], 3], 71 | ['full', $this->scopes['one'], 4], 72 | ['full', $this->scopes['one'], 5], 73 | ['full', $this->scopes['one'], 6], 74 | ['full', $this->scopes['one'], 7], 75 | ['finish_full'], 76 | ['full', $this->scopes['two'], 1], 77 | ['full', $this->scopes['two'], 2], 78 | ['full', $this->scopes['two'], 3], 79 | ['full', $this->scopes['two'], 4], 80 | ['full', $this->scopes['two'], 5], 81 | ['full', $this->scopes['two'], 6], 82 | ['full', $this->scopes['two'], 7], 83 | ['finish_full'], 84 | ], 85 | $this->actions 86 | ); 87 | } 88 | 89 | /** @test */ 90 | public function executesIndexObserverAfterEachFullIndexationOnScope() 91 | { 92 | $this->indexer->executeFull(); 93 | 94 | $this->assertSame( 95 | [ 96 | ['before', $this->scopes['one']], 97 | ['after', $this->scopes['one']], 98 | ['before', $this->scopes['two']], 99 | ['after', $this->scopes['two']], 100 | ], 101 | $this->events 102 | ); 103 | } 104 | 105 | /** @test */ 106 | public function executesPartialActionOnSingleIdUpdate() 107 | { 108 | $this->indexer->executeRow(1); 109 | 110 | $this->assertEquals( 111 | [ 112 | ['clear', [1]], 113 | ['partial', $this->scopes['one'], 1], 114 | ['finish_partial'], 115 | ['clear', [1]], 116 | ['partial', $this->scopes['two'], 1], 117 | ['finish_partial'], 118 | ], 119 | $this->actions 120 | ); 121 | } 122 | 123 | /** @test */ 124 | public function executesGenerationObserverOnSingleActionUpdate() 125 | { 126 | $this->indexer->executeRow(1); 127 | 128 | $this->assertSame( 129 | [ 130 | ['before', $this->scopes['one']], 131 | ['after', $this->scopes['one']], 132 | ['before', $this->scopes['two']], 133 | ['after', $this->scopes['two']] 134 | ], 135 | $this->events 136 | ); 137 | } 138 | 139 | /** @test */ 140 | public function executesPartialActionOnMultipleIdUpdate() 141 | { 142 | $this->indexer->execute([1, 2]); 143 | 144 | $this->assertSame( 145 | [ 146 | ['clear', [1, 2]], 147 | ['partial', $this->scopes['one'], 1], 148 | ['partial', $this->scopes['one'], 2], 149 | ['finish_partial'], 150 | ['clear', [1, 2]], 151 | ['partial', $this->scopes['two'], 1], 152 | ['partial', $this->scopes['two'], 2], 153 | ['finish_partial'], 154 | ], 155 | $this->actions 156 | ); 157 | } 158 | 159 | /** @test */ 160 | public function executesGenerationObserverOnMultipleActionUpdate() 161 | { 162 | $this->indexer->execute([3, 4]); 163 | 164 | $this->assertSame( 165 | [ 166 | ['before', $this->scopes['one']], 167 | ['after', $this->scopes['one']], 168 | ['before', $this->scopes['two']], 169 | ['after', $this->scopes['two']] 170 | ], 171 | $this->events 172 | ); 173 | } 174 | 175 | /** @test */ 176 | public function executesPartialActionOnExecuteListUpdate() 177 | { 178 | $this->indexer->executeList([3, 4]); 179 | 180 | $this->assertSame( 181 | [ 182 | ['clear', [3, 4]], 183 | ['partial', $this->scopes['one'], 3], 184 | ['partial', $this->scopes['one'], 4], 185 | ['finish_partial'], 186 | ['clear', [3, 4]], 187 | ['partial', $this->scopes['two'], 3], 188 | ['partial', $this->scopes['two'], 4], 189 | ['finish_partial'], 190 | ], 191 | $this->actions 192 | ); 193 | } 194 | 195 | /** @test */ 196 | public function executesGenerationObserverOnExecuteListUpdate() 197 | { 198 | $this->indexer->executeList([1, 2]); 199 | 200 | $this->assertSame( 201 | [ 202 | ['before', $this->scopes['one']], 203 | ['after', $this->scopes['one']], 204 | ['before', $this->scopes['two']], 205 | ['after', $this->scopes['two']] 206 | ], 207 | $this->events 208 | ); 209 | } 210 | 211 | /** @test */ 212 | public function fixesInputDataOnPartialUpdate() 213 | { 214 | $this->indexer->execute(['3', '4']); 215 | $this->indexer->executeList(['1', '2']); 216 | $this->indexer->executeRow('1'); 217 | 218 | $this->assertSame( 219 | [ 220 | ['clear', [3, 4]], 221 | ['partial', $this->scopes['one'], 3], 222 | ['partial', $this->scopes['one'], 4], 223 | ['finish_partial'], 224 | ['clear', [3, 4]], 225 | ['partial', $this->scopes['two'], 3], 226 | ['partial', $this->scopes['two'], 4], 227 | ['finish_partial'], 228 | ['clear', [1, 2]], 229 | ['partial', $this->scopes['one'], 1], 230 | ['partial', $this->scopes['one'], 2], 231 | ['finish_partial'], 232 | ['clear', [1, 2]], 233 | ['partial', $this->scopes['two'], 1], 234 | ['partial', $this->scopes['two'], 2], 235 | ['finish_partial'], 236 | ['clear', [1]], 237 | ['partial', $this->scopes['one'], 1], 238 | ['finish_partial'], 239 | ['clear', [1]], 240 | ['partial', $this->scopes['two'], 1], 241 | ['finish_partial'], 242 | ], 243 | $this->actions 244 | ); 245 | } 246 | 247 | /** @test */ 248 | public function removesDuplicateIdsFromList() 249 | { 250 | $this->indexer->execute(['3', '3', '4']); 251 | 252 | $this->assertSame( 253 | [ 254 | ['clear', [3, 4]], 255 | ['partial', $this->scopes['one'], 3], 256 | ['partial', $this->scopes['one'], 4], 257 | ['finish_partial'], 258 | ['clear', [3, 4]], 259 | ['partial', $this->scopes['two'], 3], 260 | ['partial', $this->scopes['two'], 4], 261 | ['finish_partial'], 262 | ], 263 | $this->actions 264 | ); 265 | } 266 | 267 | /** @test */ 268 | public function splitsIndexationByIdsByMaximumBatchSize() 269 | { 270 | $this->indexer->execute([1, 2, 3, 4, 5, 6, 7]); 271 | 272 | $this->assertSame( 273 | [ 274 | ['clear', [1, 2, 3]], 275 | ['partial', $this->scopes['one'], 1], 276 | ['partial', $this->scopes['one'], 2], 277 | ['partial', $this->scopes['one'], 3], 278 | ['clear', [4, 5, 6]], 279 | ['partial', $this->scopes['one'], 4], 280 | ['partial', $this->scopes['one'], 5], 281 | ['partial', $this->scopes['one'], 6], 282 | ['clear', [7]], 283 | ['partial', $this->scopes['one'], 7], 284 | ['finish_partial'], 285 | ['clear', [1, 2, 3]], 286 | ['partial', $this->scopes['two'], 1], 287 | ['partial', $this->scopes['two'], 2], 288 | ['partial', $this->scopes['two'], 3], 289 | ['clear', [4, 5, 6]], 290 | ['partial', $this->scopes['two'], 4], 291 | ['partial', $this->scopes['two'], 5], 292 | ['partial', $this->scopes['two'], 6], 293 | ['clear', [7]], 294 | ['partial', $this->scopes['two'], 7], 295 | ['finish_partial'], 296 | ], 297 | $this->actions 298 | ); 299 | } 300 | 301 | public function getScopes(): iterable 302 | { 303 | return [ 304 | $this->scopes['one'], 305 | $this->scopes['two'], 306 | ]; 307 | } 308 | 309 | public function beforeGeneration(IndexScope $scope) 310 | { 311 | $this->events[] = ['before', $scope]; 312 | } 313 | 314 | public function afterGeneration(IndexScope $scope) 315 | { 316 | $this->events[] = ['after', $scope]; 317 | } 318 | 319 | public function createPartialReindex(IndexScope $indexScope): IndexStorageWriter 320 | { 321 | return new class($this->actions) implements IndexStorageWriter { 322 | 323 | public function __construct(private array &$actions) 324 | { 325 | 326 | } 327 | 328 | public function add($row): void 329 | { 330 | $this->actions[] = ['partial', $row['scope'], $row['entity_id']]; 331 | } 332 | 333 | public function clear(array $entityIds): void 334 | { 335 | $this->actions[] = ['clear', $entityIds]; 336 | } 337 | 338 | public function finish(): void 339 | { 340 | $this->actions[] = ['finish_partial']; 341 | } 342 | }; 343 | } 344 | 345 | public function createFullReindex(IndexScope $indexScope): IndexStorageWriter 346 | { 347 | return new class($this->actions) implements IndexStorageWriter { 348 | public function __construct(private array &$actions) 349 | { 350 | 351 | } 352 | 353 | public function add($row): void 354 | { 355 | $this->actions[] = ['full', $row['scope'], $row['entity_id']]; 356 | } 357 | 358 | public function clear(array $entityIds): void 359 | { 360 | $this->actions[] = ['clear', $entityIds]; 361 | } 362 | 363 | public function finish(): void 364 | { 365 | $this->actions[] = ['finish_full']; 366 | } 367 | }; 368 | } 369 | 370 | public function loadByRange(IndexScope $indexScope, IndexRecordMutable $data, int $minEntityId, int $maxEntityId): void 371 | { 372 | foreach (range($minEntityId, $maxEntityId) as $entityId) { 373 | $data->set($entityId, ['type' => 'range']); 374 | } 375 | } 376 | 377 | public function loadByIds(IndexScope $indexScope, IndexRecordMutable $data, array $entityIds): void 378 | { 379 | foreach ($entityIds as $entityId) { 380 | $data->set($entityId, ['type' => 'partial']); 381 | } 382 | } 383 | 384 | public function process(IndexScope $scope, IndexRecord $record, IndexWriter $writer) 385 | { 386 | foreach ($record->listEntityIds() as $entityId) { 387 | $writer->add(['entity_id' => $entityId, 'scope' => $scope]); 388 | } 389 | } 390 | } 391 | -------------------------------------------------------------------------------- /Test/Unit/Model/MinMaxIndexRangeGeneratorTest.php: -------------------------------------------------------------------------------- 1 | rangeGenerator = new MinMaxIndexRangeGenerator(1, 10); 21 | } 22 | 23 | /** @test */ 24 | public function generatesOneBatchWhenStartAndEndAreBiggerThenBatchSize() 25 | { 26 | $this->assertEquals( 27 | [1 => 10], 28 | iterator_to_array( 29 | $this->rangeGenerator->ranges(20) 30 | ) 31 | ); 32 | } 33 | 34 | /** @test */ 35 | public function splitsRangesWhenBatchSizeIsSmallerThenMaxAndMin() 36 | { 37 | $this->assertEquals( 38 | [1 => 3, 4 => 6, 7 => 9, 10 => 10], 39 | iterator_to_array( 40 | $this->rangeGenerator->ranges(3) 41 | ) 42 | ); 43 | } 44 | 45 | /** @test */ 46 | public function whenBatchSizeIsOneItWillStillWork() 47 | { 48 | $this->assertEquals( 49 | [ 50 | 1 => 1, 51 | 2 => 2, 52 | 3 => 3, 53 | 4 => 4, 54 | 5 => 5, 55 | 6 => 6, 56 | 7 => 7, 57 | 8 => 8, 58 | 9 => 9, 59 | 10 => 10 60 | ], 61 | iterator_to_array( 62 | $this->rangeGenerator->ranges(1) 63 | ) 64 | ); 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /Test/Unit/Model/ResourceModel/InsertOnDuplicateSqlGeneratorTest.php: -------------------------------------------------------------------------------- 1 | insertOnDuplicateSqlGenerator = new InsertOnDuplicateSqlGenerator(); 21 | } 22 | 23 | 24 | /** @test */ 25 | public function generatesSingleRow() 26 | { 27 | $this->assertEquals( 28 | 'INSERT INTO `table1` (`column_one`,`column_two`) VALUES (?,?)', 29 | $this->insertOnDuplicateSqlGenerator 30 | ->generate('table1', ['column_one', 'column_two'], 1) 31 | ); 32 | } 33 | 34 | /** @test */ 35 | public function generatesMultipleRows() 36 | { 37 | $this->assertEquals( 38 | 'INSERT INTO `table1` (`column_one`,`column_two`) VALUES (?,?),(?,?),(?,?)', 39 | $this->insertOnDuplicateSqlGenerator 40 | ->generate('table1', ['column_one', 'column_two'], 3) 41 | ); 42 | } 43 | 44 | /** @test */ 45 | public function generatesSingleRowWithOnDuplicate() 46 | { 47 | $this->assertEquals( 48 | 'INSERT INTO `table1` (`column_one`,`column_two`) VALUES (?,?) ON DUPLICATE KEY UPDATE `column_two` = VALUES(`column_two`)', 49 | $this->insertOnDuplicateSqlGenerator 50 | ->generate( 51 | 'table1', 52 | ['column_one', 'column_two'], 53 | 1, 54 | ['column_two'] 55 | ) 56 | ); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Test/Unit/Model/ResourceModel/TableBatchIndexStorageWriterTest.php: -------------------------------------------------------------------------------- 1 | statementRegistry = FakeBatchStatementRegistry::create(); 27 | $this->batchIndexStorageWriter = new TableBatchIndexStorageWriter( 28 | $this, 29 | $this->statementRegistry, 30 | new FakeIndexTableStructure(), 31 | 5, 32 | ); 33 | } 34 | 35 | /** @test */ 36 | public function statementsAreEmptyWhenNothingIsWrittenToStorage() 37 | { 38 | $this->batchIndexStorageWriter->finish(); 39 | 40 | $this->assertEquals( 41 | FakeBatchStatementRegistry::create(), 42 | $this->statementRegistry 43 | ); 44 | } 45 | 46 | /** @test */ 47 | public function executesStatementsAsSoonAsBatchSizeIsReached() 48 | { 49 | $this->batchIndexStorageWriter->add(['entity_id' => 1, 'store_id' => 2, 'visibility' => 3]); 50 | $this->batchIndexStorageWriter->add(['entity_id' => 2, 'store_id' => 2, 'visibility' => 4]); 51 | $this->batchIndexStorageWriter->add(['entity_id' => 3, 'store_id' => 2, 'visibility' => 4]); 52 | $this->batchIndexStorageWriter->add(['entity_id' => 4, 'store_id' => 2, 'visibility' => 2]); 53 | $this->batchIndexStorageWriter->add(['entity_id' => 5, 'store_id' => 2, 'visibility' => 1]); 54 | $this->batchIndexStorageWriter->add(['entity_id' => 6, 'store_id' => 2, 'visibility' => 3]); 55 | $this->batchIndexStorageWriter->add(['entity_id' => 7, 'store_id' => 2, 'visibility' => 4]); 56 | $this->batchIndexStorageWriter->add(['entity_id' => 8, 'store_id' => 2, 'visibility' => 4]); 57 | $this->batchIndexStorageWriter->add(['entity_id' => 9, 'store_id' => 2, 'visibility' => 2]); 58 | $this->batchIndexStorageWriter->add(['entity_id' => 10, 'store_id' => 2, 'visibility' => 1]); 59 | 60 | $this->assertEquals( 61 | FakeBatchStatementRegistry::create() 62 | ->withInsertStatement('storage_2', 5, 'BATCH INSERT SQL storage_2 5') 63 | ->withInsertExecuted('storage_2', 5, 64 | [ 65 | 1, 2, 3, 66 | 2, 2, 4, 67 | 3, 2, 4, 68 | 4, 2, 2, 69 | 5, 2, 1 70 | ] 71 | ) 72 | ->withInsertExecuted('storage_2', 5, 73 | [ 74 | 6, 2, 3, 75 | 7, 2, 4, 76 | 8, 2, 4, 77 | 9, 2, 2, 78 | 10, 2, 1 79 | ] 80 | ) 81 | , 82 | $this->statementRegistry 83 | ); 84 | } 85 | 86 | /** @test */ 87 | public function whenBatchSizeIsNotReachedStatementIsNotExecuted() 88 | { 89 | $this->batchIndexStorageWriter->add(['entity_id' => 2, 'store_id' => 2, 'visibility' => 3]); 90 | $this->batchIndexStorageWriter->add(['entity_id' => 3, 'store_id' => 2, 'visibility' => 4]); 91 | $this->batchIndexStorageWriter->add(['entity_id' => 4, 'store_id' => 2, 'visibility' => 4]); 92 | $this->batchIndexStorageWriter->add(['entity_id' => 5, 'store_id' => 2, 'visibility' => 2]); 93 | 94 | $this->assertEquals( 95 | FakeBatchStatementRegistry::create(), 96 | $this->statementRegistry 97 | ); 98 | } 99 | 100 | 101 | /** @test */ 102 | public function executesBatchesForEachScopeSeparately() 103 | { 104 | $this->batchIndexStorageWriter->add(['entity_id' => 1, 'store_id' => 1, 'visibility' => 3]); 105 | $this->batchIndexStorageWriter->add(['entity_id' => 1, 'store_id' => 2, 'visibility' => 3]); 106 | $this->batchIndexStorageWriter->add(['entity_id' => 2, 'store_id' => 1, 'visibility' => 4]); 107 | $this->batchIndexStorageWriter->add(['entity_id' => 2, 'store_id' => 2, 'visibility' => 4]); 108 | $this->batchIndexStorageWriter->add(['entity_id' => 3, 'store_id' => 1, 'visibility' => 4]); 109 | $this->batchIndexStorageWriter->add(['entity_id' => 3, 'store_id' => 2, 'visibility' => 4]); 110 | $this->batchIndexStorageWriter->add(['entity_id' => 4, 'store_id' => 1, 'visibility' => 2]); 111 | $this->batchIndexStorageWriter->add(['entity_id' => 4, 'store_id' => 2, 'visibility' => 2]); 112 | $this->batchIndexStorageWriter->add(['entity_id' => 5, 'store_id' => 1, 'visibility' => 1]); 113 | $this->batchIndexStorageWriter->add(['entity_id' => 5, 'store_id' => 2, 'visibility' => 1]); 114 | 115 | $this->assertEquals( 116 | FakeBatchStatementRegistry::create() 117 | ->withInsertStatement('storage_1', 5, 'BATCH INSERT SQL storage_1 5') 118 | ->withInsertStatement('storage_2', 5, 'BATCH INSERT SQL storage_2 5') 119 | ->withInsertExecuted('storage_1', 5, 120 | [ 121 | 1, 1, 3, 122 | 2, 1, 4, 123 | 3, 1, 4, 124 | 4, 1, 2, 125 | 5, 1, 1 126 | ] 127 | ) 128 | ->withInsertExecuted('storage_2', 5, 129 | [ 130 | 1, 2, 3, 131 | 2, 2, 4, 132 | 3, 2, 4, 133 | 4, 2, 2, 134 | 5, 2, 1 135 | ] 136 | ) 137 | , 138 | $this->statementRegistry 139 | ); 140 | } 141 | 142 | 143 | /** @test */ 144 | public function createsAndExecutesStatementsForLeftOverRowsOnFinish() 145 | { 146 | $this->batchIndexStorageWriter->add(['entity_id' => 2, 'store_id' => 1, 'visibility' => 3]); 147 | $this->batchIndexStorageWriter->add(['entity_id' => 3, 'store_id' => 1, 'visibility' => 4]); 148 | $this->batchIndexStorageWriter->add(['entity_id' => 4, 'store_id' => 2, 'visibility' => 4]); 149 | $this->batchIndexStorageWriter->add(['entity_id' => 5, 'store_id' => 2, 'visibility' => 4]); 150 | $this->batchIndexStorageWriter->add(['entity_id' => 5, 'store_id' => 3, 'visibility' => 2]); 151 | $this->batchIndexStorageWriter->finish(); 152 | 153 | $this->assertEquals( 154 | FakeBatchStatementRegistry::create() 155 | ->withInsertStatement('storage_1', 2, 'BATCH INSERT SQL storage_1 2') 156 | ->withInsertStatement('storage_2', 2, 'BATCH INSERT SQL storage_2 2') 157 | ->withInsertStatement('storage_3', 1, 'BATCH INSERT SQL storage_3 1') 158 | ->withInsertExecuted('storage_1', 2, [2, 1, 3, 3, 1, 4]) 159 | ->withInsertExecuted('storage_2', 2, [4, 2, 4, 5, 2, 4]) 160 | ->withInsertExecuted('storage_3', 1, [5, 3, 2]), 161 | $this->statementRegistry 162 | ); 163 | } 164 | 165 | public function getStorageByRow(array $row): string 166 | { 167 | return 'storage_' . $row['store_id']; 168 | } 169 | 170 | public function getStorageListByScope(IndexScope $scope): iterable 171 | { 172 | // TODO: Implement getStorageListByScope() method. 173 | } 174 | } 175 | -------------------------------------------------------------------------------- /composer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ecomdev/magento-indexation-framework", 3 | "description": "Api implementation", 4 | "type": "magento2-module", 5 | "require": { 6 | "magento/framework": "*", 7 | "magento/module-indexer": "*" 8 | }, 9 | "license": [ 10 | "MIT" 11 | ], 12 | "autoload": { 13 | "files": [ 14 | "registration.php" 15 | ], 16 | "psr-4": { 17 | "MageOS\\Indexer\\": "" 18 | } 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /etc/di.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /etc/module.xml: -------------------------------------------------------------------------------- 1 | 2 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | -------------------------------------------------------------------------------- /registration.php: -------------------------------------------------------------------------------- 1 |