├── README.md ├── app ├── code │ └── community │ │ └── Dngdev │ │ └── Indexer │ │ ├── Model │ │ └── Resource │ │ │ ├── Indexer │ │ │ └── Stock.php │ │ │ └── Product │ │ │ └── Indexer │ │ │ ├── Eav │ │ │ └── Source.php │ │ │ └── Price.php │ │ └── etc │ │ └── config.xml └── etc │ └── modules │ └── Dngdev_Indexer.xml └── modman /README.md: -------------------------------------------------------------------------------- 1 | # Dngdev_Indexfix 2 | This module increase the performance during full reindexing. 3 | 4 | The repo has moved to https://github.com/smart-devs/smartdevs-indexer/ 5 | -------------------------------------------------------------------------------- /app/code/community/Dngdev/Indexer/Model/Resource/Indexer/Stock.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | class Dngdev_Indexer_Model_Resource_Indexer_Stock extends Mage_CatalogInventory_Model_Resource_Indexer_Stock 11 | { 12 | 13 | /** 14 | * Clean up temporary index table 15 | * 16 | * magento runs per default delete from table which blows up 17 | * the mysql transaction log 18 | * 19 | */ 20 | public function clearTemporaryIndexTable() 21 | { 22 | $this->_getWriteAdapter()->truncateTable($this->getIdxTable()); 23 | } 24 | 25 | /** 26 | * Synchronize data between index storage and original storage 27 | * 28 | * we use here a table rotation instead of deleting the whole table inside an transaction 29 | * which blows up the mysql transaction log and creates not iops inside the database 30 | * 31 | * @return Mage_Index_Model_Resource_Abstract 32 | */ 33 | public function syncData() 34 | { 35 | //create table names for rotation 36 | $newTableName = $this->getMainTable() . '_new'; 37 | $oldTableName = $this->getMainTable() . '_old'; 38 | //clean up last rotation if there was an error 39 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $newTableName)); 40 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $oldTableName)); 41 | 42 | try { 43 | //create new table with same schema like the original one 44 | $this->_getIndexAdapter()->query(sprintf('CREATE TABLE %s LIKE %s', $newTableName, $this->getMainTable())); 45 | 46 | //CREATE TABLE %s LIKE %s doesn't copy foreign keys so we have to add them 47 | //foreign keys are unique and have a maximun lenght of 64 characters. 48 | //so wecreate them with an custom suffix 49 | //@todo seems dirty needs some refactoring 50 | $config = $this->_getIndexAdapter()->getConfig(); 51 | $foreignKeys = $this->_getIndexAdapter()->query( 52 | sprintf('SELECT 53 | CONSTRAINT_NAME, 54 | COLUMN_NAME, 55 | REFERENCED_TABLE_NAME , 56 | REFERENCED_COLUMN_NAME 57 | FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE 58 | WHERE TABLE_NAME LIKE \'%s\' AND TABLE_SCHEMA = \'%s\' AND REFERENCED_TABLE_NAME IS NOT NULL;', 59 | $this->getMainTable(), 60 | $config['dbname'])); 61 | //add foreign keys to new table 62 | foreach ($foreignKeys->fetchAll() as $fk) { 63 | $fkName = sprintf(substr($fk['CONSTRAINT_NAME'], 0, strrpos($fk['CONSTRAINT_NAME'], '_'))); 64 | $fkName = strlen($fkName) > 50 ? substr($fkName, 0, 50) : $fkName; 65 | $fkName = $fkName . '_' . uniqid(); 66 | 67 | $this->_getIndexAdapter()->addForeignKey( 68 | $fkName, $newTableName, $fk['COLUMN_NAME'], $fk['REFERENCED_TABLE_NAME'], $fk['REFERENCED_COLUMN_NAME'] 69 | ); 70 | 71 | } 72 | //get columns mapping and insert data to new table 73 | $sourceColumns = array_keys($this->_getWriteAdapter()->describeTable($this->getIdxTable())); 74 | $targetColumns = array_keys($this->_getWriteAdapter()->describeTable($newTableName)); 75 | $select = $this->_getIndexAdapter()->select()->from($this->getIdxTable(), $sourceColumns); 76 | $this->insertFromSelect($select, $newTableName, $targetColumns, false); 77 | //rotate the tables 78 | $this->_getIndexAdapter()->query(sprintf('RENAME TABLE %s TO %s, %s TO %s', 79 | $this->getMainTable(), 80 | $oldTableName, 81 | $newTableName, 82 | $this->getMainTable() 83 | )); 84 | //drop table to reclaim table space 85 | $this->_getIndexAdapter()->dropTable($oldTableName); 86 | } catch (Exception $e) { 87 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $newTableName)); 88 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $oldTableName)); 89 | throw $e; 90 | } 91 | return $this; 92 | } 93 | } -------------------------------------------------------------------------------- /app/code/community/Dngdev/Indexer/Model/Resource/Product/Indexer/Eav/Source.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | class Dngdev_Indexer_Model_Resource_Product_Indexer_Eav_Source extends Mage_Catalog_Model_Resource_Product_Indexer_Eav_Source 11 | { 12 | 13 | /** 14 | * Rebuild all index data 15 | * 16 | * we simply remove here _removeNotVisibleEntityFromIndex because we can handle this with a join 17 | * 18 | * @return Mage_Catalog_Model_Resource_Product_Indexer_Eav_Abstract 19 | */ 20 | public function reindexAll() 21 | { 22 | $this->useIdxTable(true); 23 | $this->beginTransaction(); 24 | try { 25 | $this->clearTemporaryIndexTable(); 26 | $this->_prepareIndex(); 27 | $this->_prepareRelationIndex(); 28 | //change start 29 | #$this->_removeNotVisibleEntityFromIndex(); 30 | //change end 31 | 32 | $this->syncData(); 33 | $this->commit(); 34 | } catch (Exception $e) { 35 | $this->rollBack(); 36 | throw $e; 37 | } 38 | 39 | return $this; 40 | } 41 | 42 | /** 43 | * Clean up temporary index table 44 | * 45 | * magento runs per default delete from table which blows up 46 | * the mysql transaction log 47 | * 48 | */ 49 | public function clearTemporaryIndexTable() 50 | { 51 | $this->_getWriteAdapter()->truncateTable($this->getIdxTable()); 52 | } 53 | 54 | /** 55 | * Prepare data index for indexable select attributes 56 | * 57 | * added missing where condition for attributes which creates a horrible big temp table on disk 58 | * 59 | * @param array $entityIds the entity ids limitation 60 | * @param int $attributeId the attribute id limitation 61 | * @return Mage_Catalog_Model_Resource_Product_Indexer_Eav_Source 62 | */ 63 | protected function _prepareSelectIndex($entityIds = null, $attributeId = null) 64 | { 65 | $adapter = $this->_getWriteAdapter(); 66 | $idxTable = $this->getIdxTable(); 67 | // prepare select attributes 68 | if (is_null($attributeId)) { 69 | $attrIds = $this->_getIndexableAttributes(false); 70 | } else { 71 | $attrIds = array($attributeId); 72 | } 73 | 74 | if (!$attrIds) { 75 | return $this; 76 | } 77 | 78 | /**@var $subSelect Varien_Db_Select */ 79 | $subSelect = $adapter->select() 80 | ->from( 81 | array('s' => $this->getTable('core/store')), 82 | array('store_id', 'website_id') 83 | ) 84 | ->joinLeft( 85 | array('d' => $this->getValueTable('catalog/product', 'int')), 86 | '1 = 1 AND d.store_id = 0', 87 | array('entity_id', 'attribute_id', 'value') 88 | ) 89 | //added missing attribute filter 90 | ->where('s.store_id != 0 and d.attribute_id IN (?)', array_map('intval', $attrIds)); 91 | 92 | if (!is_null($entityIds)) { 93 | $subSelect->where('d.entity_id IN(?)', array_map('intval', $entityIds)); 94 | } 95 | 96 | /**@var $select Varien_Db_Select */ 97 | $select = $adapter->select() 98 | ->from( 99 | array('pid' => new Zend_Db_Expr(sprintf('(%s)', $subSelect->assemble()))), 100 | array() 101 | ) 102 | ->joinLeft( 103 | array('pis' => $this->getValueTable('catalog/product', 'int')), 104 | 'pis.entity_id = pid.entity_id AND pis.attribute_id = pid.attribute_id AND pis.store_id = pid.store_id', 105 | array() 106 | ) 107 | ->columns( 108 | array( 109 | 'pid.entity_id', 110 | 'pid.attribute_id', 111 | 'pid.store_id', 112 | 'value' => $adapter->getIfNullSql('pis.value', 'pid.value') 113 | ) 114 | ) 115 | ->where('pid.attribute_id IN(?)', $attrIds); 116 | 117 | $select->where(Mage::getResourceHelper('catalog')->getIsNullNotNullCondition('pis.value', 'pid.value')); 118 | 119 | /** 120 | * Add additional external limitation 121 | */ 122 | Mage::dispatchEvent('prepare_catalog_product_index_select', array( 123 | 'select' => $select, 124 | 'entity_field' => new Zend_Db_Expr('pid.entity_id'), 125 | 'website_field' => new Zend_Db_Expr('pid.website_id'), 126 | 'store_field' => new Zend_Db_Expr('pid.store_id') 127 | )); 128 | $query = $select->insertFromSelect($idxTable); 129 | $adapter->query($query); 130 | return $this; 131 | } 132 | 133 | /** 134 | * Prepare data index for indexable multiply select attributes 135 | * @todo this code is solvable via direct sql 136 | * 137 | * 138 | * @param array $entityIds the entity ids limitation 139 | * @param int $attributeId the attribute id limitation 140 | * @return Mage_Catalog_Model_Resource_Product_Indexer_Eav_Source 141 | */ 142 | protected function _prepareMultiselectIndex($entityIds = null, $attributeId = null) 143 | { 144 | $adapter = $this->_getWriteAdapter(); 145 | 146 | // prepare multiselect attributes 147 | if (is_null($attributeId)) { 148 | $attrIds = $this->_getIndexableAttributes(true); 149 | } else { 150 | $attrIds = array($attributeId); 151 | } 152 | 153 | if (!$attrIds) { 154 | return $this; 155 | } 156 | 157 | // load attribute options 158 | $options = array(); 159 | $select = $adapter->select() 160 | ->from($this->getTable('eav/attribute_option'), array('attribute_id', 'option_id')) 161 | ->where('attribute_id IN(?)', $attrIds); 162 | $query = $select->query(); 163 | while ($row = $query->fetch()) { 164 | $options[$row['attribute_id']][$row['option_id']] = true; 165 | } 166 | 167 | // prepare get multiselect values query 168 | $productValueExpression = $adapter->getCheckSql('pvs.value_id > 0', 'pvs.value', 'pvd.value'); 169 | $select = $adapter->select() 170 | ->from( 171 | array('pvd' => $this->getValueTable('catalog/product', 'varchar')), 172 | array('entity_id', 'attribute_id')) 173 | ->join( 174 | array('cs' => $this->getTable('core/store')), 175 | '', 176 | array('store_id')) 177 | ->joinLeft( 178 | array('pvs' => $this->getValueTable('catalog/product', 'varchar')), 179 | 'pvs.entity_id = pvd.entity_id AND pvs.attribute_id = pvd.attribute_id' 180 | . ' AND pvs.store_id=cs.store_id', 181 | array('value' => $productValueExpression)) 182 | ->where('pvd.store_id=?', 183 | $adapter->getIfNullSql('pvs.store_id', Mage_Catalog_Model_Abstract::DEFAULT_STORE_ID)) 184 | ->where('cs.store_id!=?', Mage_Catalog_Model_Abstract::DEFAULT_STORE_ID) 185 | ->where('pvd.attribute_id IN(?)', $attrIds); 186 | 187 | $statusCond = $adapter->quoteInto('=?', Mage_Catalog_Model_Product_Status::STATUS_ENABLED); 188 | $this->_addAttributeToSelect($select, 'status', 'pvd.entity_id', 'cs.store_id', $statusCond); 189 | 190 | if (!is_null($entityIds)) { 191 | $select->where('pvd.entity_id IN(?)', $entityIds); 192 | } 193 | 194 | /** 195 | * Add additional external limitation 196 | */ 197 | Mage::dispatchEvent('prepare_catalog_product_index_select', array( 198 | 'select' => $select, 199 | 'entity_field' => new Zend_Db_Expr('pvd.entity_id'), 200 | 'website_field' => new Zend_Db_Expr('cs.website_id'), 201 | 'store_field' => new Zend_Db_Expr('cs.store_id') 202 | )); 203 | $i = 0; 204 | $data = array(); 205 | $query = $select->query(); 206 | while ($row = $query->fetch()) { 207 | $values = explode(',', $row['value']); 208 | foreach ($values as $valueId) { 209 | if (isset($options[$row['attribute_id']][$valueId])) { 210 | $data[] = array( 211 | $row['entity_id'], 212 | $row['attribute_id'], 213 | $row['store_id'], 214 | $valueId 215 | ); 216 | $i++; 217 | if ($i % 10000 == 0) { 218 | $this->_saveIndexData($data); 219 | $data = array(); 220 | } 221 | } 222 | } 223 | } 224 | 225 | $this->_saveIndexData($data); 226 | unset($options); 227 | unset($data); 228 | return $this; 229 | } 230 | 231 | /** 232 | * Synchronize data between index storage and original storage 233 | * 234 | * we use here a table rotation instead of deleting the whole table inside an transaction 235 | * which blows up the mysql transaction log and creates not iops inside the database 236 | * 237 | * @return Mage_Index_Model_Resource_Abstract 238 | */ 239 | public function syncData() 240 | { 241 | //create table names for rotation 242 | $newTableName = $this->getMainTable() . '_new'; 243 | $oldTableName = $this->getMainTable() . '_old'; 244 | //clean up last rotation if there was an error 245 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $newTableName)); 246 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $oldTableName)); 247 | 248 | try { 249 | //create new table with same schema like the original one 250 | $this->_getIndexAdapter()->query(sprintf('CREATE TABLE %s LIKE %s', $newTableName, $this->getMainTable())); 251 | 252 | //CREATE TABLE %s LIKE %s doesn't copy foreign keys so we have to add them 253 | //foreign keys are unique and have a maximun lenght of 64 characters. 254 | //so wecreate them with an custom suffix 255 | //@todo seems dirty needs some refactoring 256 | $config = $this->_getIndexAdapter()->getConfig(); 257 | $foreignKeys = $this->_getIndexAdapter()->query( 258 | sprintf('SELECT 259 | CONSTRAINT_NAME, 260 | COLUMN_NAME, 261 | REFERENCED_TABLE_NAME , 262 | REFERENCED_COLUMN_NAME 263 | FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE 264 | WHERE TABLE_NAME LIKE \'%s\' AND TABLE_SCHEMA = \'%s\' AND REFERENCED_TABLE_NAME IS NOT NULL;', 265 | $this->getMainTable(), 266 | $config['dbname'])); 267 | //add foreign keys to new table 268 | foreach ($foreignKeys->fetchAll() as $fk) { 269 | $fkName = sprintf(substr($fk['CONSTRAINT_NAME'], 0, strrpos($fk['CONSTRAINT_NAME'], '_'))); 270 | $fkName = strlen($fkName) > 50 ? substr($fkName, 0, 50) : $fkName; 271 | $fkName = $fkName . '_' . uniqid(); 272 | 273 | $this->_getIndexAdapter()->addForeignKey( 274 | $fkName, $newTableName, $fk['COLUMN_NAME'], $fk['REFERENCED_TABLE_NAME'], $fk['REFERENCED_COLUMN_NAME'] 275 | ); 276 | 277 | } 278 | //get columns mapping and insert data to new table 279 | $sourceColumns = array_keys($this->_getWriteAdapter()->describeTable($this->getIdxTable())); 280 | $targetColumns = array_keys($this->_getWriteAdapter()->describeTable($newTableName)); 281 | $select = $this->_getIndexAdapter()->select()->from($this->getIdxTable(), $sourceColumns); 282 | //add join for bypassing disabled products and avoid heavy delete query 283 | $condition = $this->_getIndexAdapter()->quoteInto('=?',Mage_Catalog_Model_Product_Visibility::VISIBILITY_NOT_VISIBLE); 284 | $this->_addAttributeToSelect( 285 | $select, 286 | 'visibility', 287 | $this->getIdxTable() . '.entity_id', 288 | $this->getIdxTable() . '.store_id', 289 | $condition 290 | ); 291 | $this->insertFromSelect($select, $newTableName, $targetColumns, false); 292 | //rotate the tables 293 | $this->_getIndexAdapter()->query(sprintf('RENAME TABLE %s TO %s, %s TO %s', 294 | $this->getMainTable(), 295 | $oldTableName, 296 | $newTableName, 297 | $this->getMainTable() 298 | )); 299 | //drop table to reclaim table space 300 | $this->_getIndexAdapter()->dropTable($oldTableName); 301 | } catch (Exception $e) { 302 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $newTableName)); 303 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $oldTableName)); 304 | throw $e; 305 | } 306 | return $this; 307 | } 308 | } -------------------------------------------------------------------------------- /app/code/community/Dngdev/Indexer/Model/Resource/Product/Indexer/Price.php: -------------------------------------------------------------------------------- 1 | 9 | */ 10 | class Dngdev_Indexer_Model_Resource_Product_Indexer_Price extends Mage_Catalog_Model_Resource_Product_Indexer_Price 11 | { 12 | /** 13 | * Clean up temporary index table 14 | * 15 | * magento runs per default delete from table which blows up 16 | * the mysql transaction log 17 | * 18 | */ 19 | public function clearTemporaryIndexTable() 20 | { 21 | $this->_getWriteAdapter()->truncateTable($this->getIdxTable()); 22 | } 23 | 24 | /** 25 | * Synchronize data between index storage and original storage 26 | * 27 | * we use here a table rotation instead of deleting the whole table inside an transaction 28 | * which blows up the mysql transaction log and creates not iops inside the database 29 | * 30 | * @return Mage_Index_Model_Resource_Abstract 31 | */ 32 | public function syncData() 33 | { 34 | //create table names for rotation 35 | $newTableName = $this->getMainTable() . '_new'; 36 | $oldTableName = $this->getMainTable() . '_old'; 37 | //clean up last rotation if there was an error 38 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $newTableName)); 39 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $oldTableName)); 40 | 41 | try { 42 | //create new table with same schema like the original one 43 | $this->_getIndexAdapter()->query(sprintf('CREATE TABLE %s LIKE %s', $newTableName, $this->getMainTable())); 44 | 45 | //CREATE TABLE %s LIKE %s doesn't copy foreign keys so we have to add them 46 | //foreign keys are unique and have a maximun lenght of 64 characters. 47 | //so wecreate them with an custom suffix 48 | //@todo seems dirty needs some refactoring 49 | $config = $this->_getIndexAdapter()->getConfig(); 50 | $foreignKeys = $this->_getIndexAdapter()->query( 51 | sprintf('SELECT 52 | CONSTRAINT_NAME, 53 | COLUMN_NAME, 54 | REFERENCED_TABLE_NAME , 55 | REFERENCED_COLUMN_NAME 56 | FROM INFORMATION_SCHEMA.KEY_COLUMN_USAGE 57 | WHERE TABLE_NAME LIKE \'%s\' AND TABLE_SCHEMA = \'%s\' AND REFERENCED_TABLE_NAME IS NOT NULL;', 58 | $this->getMainTable(), 59 | $config['dbname'])); 60 | //add foreign keys to new table 61 | foreach ($foreignKeys->fetchAll() as $fk) { 62 | $fkName = sprintf(substr($fk['CONSTRAINT_NAME'], 0, strrpos($fk['CONSTRAINT_NAME'], '_'))); 63 | $fkName = strlen($fkName) > 50 ? substr($fkName, 0, 50) : $fkName; 64 | $fkName = $fkName . '_' . uniqid(); 65 | 66 | $this->_getIndexAdapter()->addForeignKey( 67 | $fkName, $newTableName, $fk['COLUMN_NAME'], $fk['REFERENCED_TABLE_NAME'], $fk['REFERENCED_COLUMN_NAME'] 68 | ); 69 | 70 | } 71 | //get columns mapping and insert data to new table 72 | $sourceColumns = array_keys($this->_getWriteAdapter()->describeTable($this->getIdxTable())); 73 | $targetColumns = array_keys($this->_getWriteAdapter()->describeTable($newTableName)); 74 | $select = $this->_getIndexAdapter()->select()->from($this->getIdxTable(), $sourceColumns); 75 | $this->insertFromSelect($select, $newTableName, $targetColumns, false); 76 | //rotate the tables 77 | $this->_getIndexAdapter()->query(sprintf('RENAME TABLE %s TO %s, %s TO %s', 78 | $this->getMainTable(), 79 | $oldTableName, 80 | $newTableName, 81 | $this->getMainTable() 82 | )); 83 | //drop table to reclaim table space 84 | $this->_getIndexAdapter()->dropTable($oldTableName); 85 | } catch (Exception $e) { 86 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $newTableName)); 87 | $this->_getIndexAdapter()->query(sprintf('DROP TABLE IF EXISTS %s', $oldTableName)); 88 | throw $e; 89 | } 90 | return $this; 91 | } 92 | } -------------------------------------------------------------------------------- /app/code/community/Dngdev/Indexer/etc/config.xml: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 11 | 12 | 1.0.0 13 | 14 | 15 | 16 | 17 | 18 | Dngdev_Indexer_Model 19 | 20 | 21 | 22 | Dngdev_Indexer_Model_Resource_Indexer_Stock 23 | 24 | 25 | 26 | 27 | Dngdev_Indexer_Model_Resource_Product_Indexer_Eav_Source 28 | 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /app/etc/modules/Dngdev_Indexer.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | true 6 | community 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /modman: -------------------------------------------------------------------------------- 1 | app/etc/modules/Dngdev_Indexer.xml app/etc/modules/Dngdev_Indexer.xml 2 | app/code/community/Dngdev/Indexer app/code/community/Dngdev/Indexer 3 | --------------------------------------------------------------------------------