├── README.md └── upgrade_replay.php /README.md: -------------------------------------------------------------------------------- 1 | Magento-Upgrade-Replay 2 | ====================== 3 | 4 | This is prototype for multi-process Optimized Upgrade Replay from Magento 1.5 (1.10 EE) and lower to 1.6 (1.11 EE) and higher. 5 | In order to run optimized replay you need to run original upgrade in database with same structure. 6 | Replay script uses pdo_mysql.log to read queries and estimate execution time. 7 | 8 | Disk may become bottleneck during upgrade replay. 9 | 10 | #### Warning 11 | * Was not tested with table prefixes 12 | * **THIS IS PROTOTTYPE**. Please make sure it works before running in production. 13 | 14 | #### Real life results 15 | * 1.9 EE -> 1.11 EE ==> Original upgrade ~34 hours vs Optimized Replay 4 hours 16 | * 1.4 CE -> 1.12 EE ==> Original upgrade ~15 hours vs Optimized Replay 2 hours 17 | 18 | #### Requrements 19 | * PHP 5.2+ 20 | * POSIX extension 21 | * Magento 1.6+ (1.11 EE) to upgrade to 22 | 23 | Prepare **Varien_Db_Adapter_Pdo_Mysql** in Magento 1.6+ 24 | Enable debug mode: 25 | ```php 26 | //protected $_debug = false; // Original 27 | protected $_debug = true; 28 | ``` 29 | Log all queries: 30 | ```php 31 | // protected $_logAllQueries = false; // Original 32 | protected $_logAllQueries = true; 33 | ``` 34 | 35 | #### Steps to upgrade 36 | 1. Trigger upgrade and wait until it ends. 37 | 2. Copy var/debug/pdo_mysql.log from upgraded Magento to shell direcory of new copy. 38 | 3. Copy upgrade_replay.php and shell directory of new copy. 39 | 4. cd [new copy]/shell && php -f upgrade_replay.php 40 | 41 | #### Script internals 42 | 43 | Create posix FIFO for inter-process communications. 44 | Start multiple php processes using proc_open(). 45 | Prepare queries for execution: 46 | * Strip out duplicate queries. 47 | * Merge similar ALTER TABLE queries. 48 | * Optimize one data-specific query 49 | * Split in chunks for parallel execution. 50 | Child processes are waiting for data in STDIN and puting their IDs in fifo once process is finished. 51 | Parent process reads fifo in blocking mode. 52 | Once Parent gets ID of child, it sends next query or waits till othe queries from chunk will be executed. 53 | Data upgrades are triggered the same way as in original. 54 | 55 | #### Some code 56 | Debug function to make shure that parsing FSM works fine. 57 | ```php 58 | $track->echoQueries(); 59 | ``` 60 | Triggering upgrade in multiple threads ('true' will really trigger upgrade). 61 | ```php 62 | // public function run($threads, $real = false) 63 | //$track->run(4); 64 | $track->run(12, true); 65 | ``` 66 | 67 | 68 | 69 | -------------------------------------------------------------------------------- /upgrade_replay.php: -------------------------------------------------------------------------------- 1 | _addQuery($sql, $bind, $matches[1]); 65 | 66 | } elseif ($state == self::READ_SQL && preg_match('/^SQL: (.*)/',$line, $matches)) { 67 | $match = $matches[1]; 68 | if (!preg_match('/^(DESCRIBE |SHOW CREATE TABLE |SHOW TABLE STATUS LIKE |SELECT |SHOW INDEX FROM |SHOW TABLES)/', $match) 69 | ) { 70 | //echo "Reading Line:\r\n$match"; 71 | $state = self::READ_SQL_LINE; 72 | $buffer .= $match; 73 | } else { 74 | $state = self::SKIP_LINE; 75 | } 76 | } elseif (in_array($state, array(self::READ_SQL_LINE, self::READ_BIND_LINE))) { 77 | $buffer .= ' '.$line; 78 | } else { 79 | $time_match = array(); 80 | if (preg_match('/^TIME: ([0-9]+\.[0-9]+)/',$line,$time_match)) { 81 | $totalTime+=floatval($time_match[1]); 82 | } 83 | $state = self::SKIP_LINE; 84 | } 85 | 86 | if (!in_array($state, array(self::SKIP_LINE, self::CLEAN_BUFFER))) { 87 | //echo "$state > $line"; 88 | //usleep(5000); 89 | } 90 | } 91 | echo "/*totalTime: ". $this->_getTimeString($totalTime) ."\r\n"; 92 | $cleanupAffect = $this->_clean(); 93 | var_dump($cleanupAffect); 94 | 95 | //$mergeIC = $this->_mergeIndexAndConstraint(); 96 | //var_dump($mergeIC); 97 | 98 | $merged = $this->_mergeQueries2(); 99 | var_dump($merged); 100 | 101 | echo "*/"; 102 | 103 | for ($i =1; $i <= 24;$i++) { 104 | $this->_estimate($i); 105 | } 106 | //$this->_estimate(4); 107 | } 108 | 109 | protected function _addQuery($sql, $bind, $time) { 110 | // 3 hour query fix 111 | if (preg_match('/INSERT INTO `salesrule_product_attribute`/', $sql)) { 112 | $sql = " 113 | drop temporary TABLE IF EXISTS tt1; 114 | create temporary table tt1 115 | as 116 | SELECT sr.rule_id, 117 | -- cw.website_id, 118 | -- cg.customer_group_id, 119 | ea.attribute_id, 120 | CONCAT(sr.conditions_serialized, sr.actions_serialized) LIKE 121 | CONCAT( '%s:32:\"salesrule/rule_condition_product\";s:9:\"attribute\";s:', 122 | LENGTH(ea.attribute_code), ':\"', ea.attribute_code, '\"%') as res 123 | 124 | from `salesrule` AS sr 125 | inner join `eav_attribute` AS ea ON ea.entity_type_id = 10 ; 126 | 127 | INSERT INTO `salesrule_product_attribute` 128 | select sr.rule_id, 129 | cw.website_id, 130 | cg.customer_group_id, 131 | tmp.attribute_id 132 | from `salesrule` AS sr 133 | inner join (select * from tt1 where res = 1) as tmp ON tmp.rule_id = sr.rule_id 134 | INNER JOIN `core_website` AS cw 135 | ON FIND_IN_SET(cw.website_id, sr.website_ids) INNER JOIN `customer_group` AS cg 136 | ON FIND_IN_SET(cg.customer_group_id , sr.customer_group_ids); 137 | drop temporary TABLE IF EXISTS tt1; 138 | "; 139 | $bind =''; 140 | } 141 | 142 | $bind_data =null; 143 | if($bind !='') { 144 | eval('$bind_data = '.$bind.";"); 145 | //$bind_data=serialize($bind_data); 146 | } 147 | $query = array('sql'=>$sql, 'bind' => $bind_data, 'time'=> floatval($time)); 148 | 149 | $this->_queries[] = $query; 150 | } 151 | 152 | /** 153 | * Remove duplicate queries and KEEP FIRST query mode 154 | */ 155 | protected function _clean() 156 | { 157 | $keys = array_keys($this->_queries); 158 | sort($keys); 159 | $affected = array('index' => 0,'fk' => 0); 160 | foreach($keys as $queryKey) { 161 | $matches = array(); 162 | if (!isset($this->_queries[$queryKey])) { 163 | continue; 164 | } 165 | 166 | if (preg_match('/ALTER TABLE `([a-zA-Z_0-9]+)` ADD (UNIQUE|INDEX) `([a-zA-Z_0-9]+)`/',$this->_queries[$queryKey]['sql'],$matches)) { 167 | //echo "match $matches[2]\r\n"; 168 | foreach($keys as $queryKey_f) { 169 | if (!isset($this->_queries[$queryKey_f]) || $queryKey >= $queryKey_f) { 170 | continue; 171 | } 172 | if (preg_match('/ALTER TABLE `'.$matches[1].'` DROP (INDEX|KEY) `'.$matches[3].'`/',$this->_queries[$queryKey_f]['sql'])) { 173 | unset($this->_queries[$queryKey]); 174 | unset($this->_queries[$queryKey_f]); 175 | $affected['index'] +=2; 176 | break; 177 | } 178 | } 179 | continue; 180 | } 181 | 182 | if (preg_match('/ALTER TABLE `([a-zA-Z_0-9]+)` ADD CONSTRAINT `([a-zA-Z_0-9]+)`/',$this->_queries[$queryKey]['sql'],$matches)) { 183 | 184 | foreach($keys as $queryKey_f) { 185 | if (!isset($this->_queries[$queryKey_f]) || $queryKey >= $queryKey_f) { 186 | continue; 187 | } 188 | 189 | if (preg_match('/ALTER TABLE `'.$matches[1].'` DROP FOREIGN KEY `'.$matches[2].'`/',$this->_queries[$queryKey_f]['sql'])) { 190 | //echo "match ADD FK $matches[1] $matches[2]\r\n"; 191 | unset($this->_queries[$queryKey]); 192 | unset($this->_queries[$queryKey_f]); 193 | $affected['fk'] +=2; 194 | break; 195 | } 196 | } 197 | continue; 198 | } 199 | 200 | 201 | } 202 | return $affected; 203 | } 204 | 205 | 206 | protected function _mergeIndexAndConstraint() 207 | { 208 | $keys = array_keys($this->_queries); 209 | rsort($keys); 210 | $affected = 0; 211 | foreach($keys as $queryKey) { 212 | $matches = array(); 213 | if (!isset($this->_queries[$queryKey])) { 214 | continue; 215 | } 216 | 217 | if (preg_match('/ALTER TABLE `([a-zA-Z_0-9]+)` ADD CONSTRAINT `([a-zA-Z_0-9]+)` FOREIGN KEY \(`([a-zA-Z_0-9]+)`\)/',$this->_queries[$queryKey]['sql'],$matches)) { 218 | //echo "match $matches[2]\r\n"; 219 | foreach($keys as $queryKey_f) { 220 | if (!isset($this->_queries[$queryKey_f]) || $queryKey <= $queryKey_f) { 221 | continue; 222 | } 223 | if (preg_match('/ALTER TABLE `'.$matches[1].'` ADD (INDEX|KEY|UNIQUE) `([a-zA-Z_0-9]+)` \(`'.$matches[3].'`\)/',$this->_queries[$queryKey_f]['sql'], $addMatch)) { 224 | $this->_queries[$queryKey]['sql'] = str_replace('FOREIGN KEY', 'FOREIGN KEY `'.$addMatch[2].'`', $this->_queries[$queryKey]['sql']); 225 | unset($this->_queries[$queryKey_f]); 226 | $affected +=1; 227 | break; 228 | } 229 | } 230 | continue; 231 | } 232 | } 233 | return $affected; 234 | } 235 | 236 | protected function _getDDLBlocks() 237 | { 238 | //ksort($similarStatements); 239 | $ddlBlocks = array(array()); 240 | 241 | foreach ($this->_queries as $qId => $query) { 242 | if(preg_match('/ALTER TABLE/', $query['sql'])) { 243 | $ddlBlocks[max(array_keys($ddlBlocks))][] = $qId; 244 | } else { 245 | $ddlBlocks[] = array(); 246 | } 247 | } 248 | 249 | foreach ($ddlBlocks as $blockId =>&$block) { 250 | if (empty($block)) { 251 | unset($ddlBlocks[$blockId]); 252 | } else { 253 | usort($block,array($this,'_sortQueries')); 254 | } 255 | } 256 | 257 | return $ddlBlocks; 258 | } 259 | 260 | 261 | /** 262 | * Sort queries in ddl block 263 | * 264 | * @param int $qId1 QueryId 265 | * @param int $qId2 QueryId 266 | */ 267 | protected function _sortQueries($qId1, $qId2) 268 | { 269 | preg_match('/ALTER TABLE `([^`]+)` (.*)/',$this->_queries[$qId1]['sql'] ,$q1Matches); 270 | preg_match('/ALTER TABLE `([^`]+)` (.*)/',$this->_queries[$qId2]['sql'] ,$q2Matches); 271 | 272 | 273 | $ranks = array( 274 | '/\s*(DROP FOREIGN KEY)(.*)/', 275 | '/\s*(DROP INDEX|DROP KEY)(.*)/', 276 | '/\s*(DROP PRIMARY KEY)(.*)/', 277 | '/\s*(ADD COLUMN|MODIFY COLUMN)\s*`([^`]+)`(.*)/', 278 | '/\s*(CHANGE COLUMN|CHANGE)(.*)/', 279 | '/\s*(COMMENT|ENGINE|DEFAULT CHARACTER SET)(.*)/', 280 | '/\s*(ADD UNIQUE|ADD INDEX|ADD FULLTEXT|ADD PRIMARY KEY)(.*)/', 281 | '/\s*(ADD CONSTRAINT|DROP COLUMN)(.*)/', 282 | '/\s*(DISABLE KEYS|ENABLE KEYS)(.*)/', 283 | ); 284 | $q1Rank = $q2Rank = -1; 285 | foreach ($ranks as $rank => $regex) { 286 | if (preg_match($regex, $q1Matches[2]) && $q1Rank = -1) {$q1Rank = $rank;} 287 | if (preg_match($regex, $q2Matches[2]) && $q2Rank = -1) {$q2Rank = $rank;} 288 | } 289 | if ($q1Rank < 0 || $q2Rank <0) { 290 | echo "UNKNOWN ALTER"; 291 | var_dump($q1Matches, $q2Matches); 292 | exit; 293 | } 294 | 295 | $res = 0; 296 | if ($q1Rank < $q2Rank) { 297 | $res = -1; 298 | } else if($q1Rank > $q2Rank) { 299 | $res = 1; 300 | } else if ($q1Matches[1] < $q2Matches[1]) { 301 | $res = -1; 302 | } else if ($q1Matches[1] > $q2Matches[1]) { 303 | $res = 1; 304 | } else if ($q1Matches[2] < $q2Matches[2]) { 305 | $res = -1; 306 | } else if ($q1Matches[2] < $q2Matches[2]) { 307 | $res = 1; 308 | } 309 | 310 | return $res; 311 | } 312 | 313 | /** 314 | * Merge LaterStatement for same tables 315 | */ 316 | protected function _mergeQueries2() 317 | { 318 | $affected = 0; 319 | // MERGING Backward and forward 320 | $similarStatements = array( 321 | '/\s*(DROP FOREIGN KEY)(.*)/', 322 | '/\s*(DROP INDEX|DROP KEY)(.*)/', 323 | '/\s*(DROP PRIMARY KEY)(.*)/', 324 | '/\s*(ADD COLUMN|MODIFY COLUMN|COMMENT|ENGINE|DEFAULT CHARACTER SET)(.*)/', 325 | '/\s*(CHANGE COLUMN|CHANGE|COMMENT|ENGINE|DEFAULT CHARACTER SET)(.*)/', 326 | '/\s*(ADD UNIQUE|ADD INDEX|ADD FULLTEXT|ADD PRIMARY KEY)(.*)/', 327 | '/\s*(ADD CONSTRAINT|DROP COLUMN)(.*)/', 328 | ); 329 | $ddlBlocks = $this->_getDDLBlocks(); 330 | 331 | foreach ($ddlBlocks as $block) { 332 | foreach ($block as $q1 =>$qId1) { 333 | if(!isset($this->_queries[$qId1])) continue; 334 | foreach ($block as $q2 =>$qId2) { 335 | //Move forward 336 | if(!isset($this->_queries[$qId2]) || $q1>=$q2 337 | || !empty($this->_queries[$qId1]['bind']) 338 | || !empty($this->_queries[$qId2]['bind'])) continue; 339 | $q1Matches = $q2Matches = array(); 340 | preg_match('/ALTER TABLE `([^`]+)`([^`]+)`([^`]+)`(.*)/',$this->_queries[$qId1]['sql'] ,$q1Matches); 341 | preg_match('/ALTER TABLE `([^`]+)`([^`]+)`([^`]+)`(.*)/',$this->_queries[$qId2]['sql'] ,$q2Matches); 342 | if (count($q1Matches) && $q1Matches[1] == $q2Matches[1] && $q1Matches[2] == $q2Matches[2] && $q1Matches[3] == $q2Matches[3]){ 343 | //echo "Similar $qId1,$qId2 \r\n"; 344 | //var_dump($q1Matches, $q2Matches); 345 | unset($this->_queries[min($qId1,$qId2)]); 346 | break; 347 | } 348 | } 349 | } 350 | } 351 | 352 | foreach ($ddlBlocks as $block) { 353 | foreach ($block as $q1 =>$qId1) { 354 | if(!isset($this->_queries[$qId1])) continue; 355 | foreach ($block as $q2 =>$qId2) { 356 | //Move forward 357 | if(!isset($this->_queries[$qId2]) || $q1>=$q2 358 | || !empty($this->_queries[$qId1]['bind']) 359 | || !empty($this->_queries[$qId2]['bind'])) continue; 360 | $q1Matches = $q2Matches = array(); 361 | preg_match('/ALTER TABLE `([^`]+)`(.*)/',$this->_queries[$qId1]['sql'] ,$q1Matches); 362 | preg_match('/ALTER TABLE `([^`]+)`(.*)/',$this->_queries[$qId2]['sql'] ,$q2Matches); 363 | // Stop merging if table is different 364 | if ($q1Matches[1] != $q2Matches[1]) break; 365 | // Check is statement can be merged; 366 | $areSimilar = false; 367 | foreach ($similarStatements as $regex) { 368 | // Merge only if both are matching description 369 | if (preg_match($regex,$q1Matches[2]) && preg_match($regex,$q2Matches[2])) { 370 | $areSimilar = true; 371 | break; 372 | } 373 | } 374 | // if they are similar, we can merge them 375 | if ($areSimilar) { 376 | //$this->_queries[max($qId1,$qId2)]['sql'] .= $this->_queries[min($qId1,$qId2)]['sql'] . ', '. $q2Matches[2]; 377 | //unset($this->_queries[min($qId1,$qId2)]); 378 | 379 | $this->_queries[$qId1]['sql'] .= ', '. $q2Matches[2]; 380 | unset($this->_queries[$qId2]); 381 | $affected++; 382 | } else { 383 | //var_dump($this->_queries[$qId1]['sql'], $this->_queries[$qId2]['sql']); 384 | } 385 | 386 | } 387 | 388 | } 389 | //exit; 390 | } 391 | return $affected; 392 | } 393 | 394 | public function echoQueries() 395 | { 396 | //return; 397 | $estimatedTime = 0; 398 | foreach ($this->_queries as $qid => $query) 399 | { 400 | 401 | echo "/* QID:$qid */ ". $query['sql'] .";\r\n"; 402 | if ($query['bind']) { 403 | echo '/* '. serialize($query['bind']) ." */\r\n"; 404 | } 405 | $estimatedTime += $query['time']; 406 | } 407 | //echo '-- Estimated Time: '. $this->_getTimeString($estimatedTime); 408 | } 409 | 410 | protected function _estimate($threadNum) 411 | { 412 | $estimatedTime = 0; 413 | $empty_threads = array_pad(array(),$threadNum,0); 414 | 415 | /** 416 | * There will 2 Sync points: 417 | * and 2 point of going parallel. 418 | * Going parallel: 419 | * 1. ALTER TABLE WITH ADD CONSTRAINT 420 | * 2. ALTER TABLE WITH CONSTRAINT 421 | * Sync points: 422 | * 1. FIRST ALTER TABLE WITH ADD CONSTRAINT 423 | * 2. ALTER TABLE WITH CONSTRAINT 424 | */ 425 | $acMode = false; 426 | 427 | $threads = $empty_threads; 428 | echo "\r\n/* "; 429 | foreach ($this->_queries as $query) 430 | { 431 | if ($query) { 432 | 433 | } 434 | 435 | $match = array(); 436 | if (preg_match('/ALTER TABLE `([^`]+)`(.*)/', $query['sql'], $match)) { 437 | if (preg_match('/ADD CONSTRAINT/',$match[2]) && $acMode!=1) { 438 | $acMode=1; 439 | $estimatedTime+=max($threads); 440 | $threads = $empty_threads; 441 | } elseif (preg_match('/(DROP INDEX|DROP KEY)/',$match[2]) && $acMode!=2) { 442 | $acMode=2; 443 | $estimatedTime+=max($threads); 444 | $threads = $empty_threads; 445 | } elseif($acMode !=3) { 446 | $acMode=3; 447 | $estimatedTime+=max($threads); 448 | $threads = $empty_threads; 449 | } 450 | $threads[0] += $query['time']; 451 | sort($threads); 452 | } else { 453 | $acMode = false; 454 | $oldEst = $estimatedTime; 455 | $estimatedTime +=max($threads) + $query['time']; 456 | //echo "\r\nDML sync: " . max($threads). ' AND '.$query['time']; 457 | $threads = $empty_threads; 458 | } 459 | } 460 | echo "\r\n ESTIMATED TIME AT $threadNum threads:". $this->_getTimeString($estimatedTime) ." */"; 461 | } 462 | 463 | protected function _getNextQid() 464 | { 465 | foreach($this->_queries as $qId=>&$query) { 466 | if (isset($query['status']) && $query['status'] == 'finished') continue; 467 | $canStart = true; 468 | foreach($query['dep'] as $depId) { 469 | if(!isset($this->_queries[$depId]['status']) || $this->_queries[$depId]['status'] != 'finished') { 470 | $canStart = false; 471 | break; 472 | } 473 | } 474 | if ($canStart) { 475 | return $qId; 476 | } 477 | } 478 | return null; 479 | } 480 | 481 | protected function _getParallelQueries() 482 | { 483 | echo "-- Preparing queries"; 484 | $queryFlow = array(array()); 485 | 486 | $estimatedTime = 0; 487 | //$empty_threads = array_pad(array(),$threadNum,0); 488 | 489 | /** 490 | * There will 2 Sync points: 491 | * and 2 point of going parallel. 492 | * Going parallel: 493 | * 1. ALTER TABLE WITH ADD CONSTRAINT 494 | * 2. ALTER TABLE WITH CONSTRAINT 495 | * Sync points: 496 | * 1. FIRST ALTER TABLE WITH ADD CONSTRAINT 497 | * 2. ALTER TABLE WITH CONSTRAINT 498 | */ 499 | $acMode = 0; 500 | 501 | //$threads = $empty_threads; 502 | echo "\r\n/* "; 503 | foreach ($this->_queries as $qId=>$query) 504 | { 505 | $match = array(); 506 | if (preg_match('/ALTER TABLE `([^`]+)`(.*)/', $query['sql'], $match)) { 507 | if (preg_match('/ADD CONSTRAINT/',$match[2]) && $acMode!=1) { 508 | $acMode=1; 509 | $queryFlow[] = array(); 510 | } elseif (preg_match('/(DROP INDEX|DROP KEY)/',$match[2]) && $acMode!=2) { 511 | $acMode=2; 512 | $queryFlow[] = array(); 513 | } elseif($acMode !=3) { 514 | $acMode=3; 515 | $queryFlow[] = array(); 516 | } 517 | $queryFlow[max(array_keys($queryFlow))][] = $qId; 518 | } else { 519 | $queryFlow[] = array($qId); 520 | $acMode = false; 521 | } 522 | } 523 | echo "\r\n*/"; 524 | return $queryFlow; 525 | } 526 | 527 | public function run($threads, $real = false) 528 | { 529 | $parallelQueries = $this->_getParallelQueries(); 530 | $app = Mage::app(); 531 | $app->cleanCache(); 532 | echo "\nCleaned Cache\r\n"; 533 | $queryIds = array_keys($this->_queries); 534 | sort($queryIds); 535 | /** @var $connection Zend_Db_Adapter_Abstract */ 536 | $connection = Mage::getSingleton('core/resource')->getConnection('default_setup'); 537 | $connection->closeConnection(); 538 | // init callback pipe 539 | $waitPipe = dirname(__FILE__).'/upgrade_'.getmypid().'.pipe'; 540 | $waitPipeMode = 0600; 541 | posix_mkfifo($waitPipe, $waitPipeMode); 542 | //touch($waitPipe); 543 | echo "Creating Pipe $waitPipe\r\n"; 544 | 545 | 546 | $descriptorspec = array( 547 | 0 => array("pipe", "r"), // stdin 548 | 1 => array("pipe", "w"), // stdout 549 | 2 => array("pipe", "w") // stderr ?? instead of a file 550 | ); 551 | $procs = $pipes = array(); 552 | 553 | echo "\r\nopening workers"; 554 | for ($i = 0; $i<$threads;$i++) { 555 | 556 | /* ACTUAL CODE OF WORKER */ 557 | // NIGHTMARE Worker Program To run queries and send response in pipes 558 | $workerCode = "include_once '".dirname(dirname(__FILE__))."/app/Mage.php'; Mage::app(); ". 559 | "\$responsePipe=fopen('".$waitPipe."','ab');". 560 | "\$myWorkerId = ".$i."; ". 561 | "\$connection = Mage::getSingleton('core/resource')->getConnection('default_setup');". 562 | 'function runQuery($sQuery) {global $connection, $responsePipe, $myWorkerId; '. 563 | '$query = unserialize($sQuery);'. 564 | ($real?'try{ $connection->query("SET SQL_MODE=\'\',@OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0,'. 565 | ' @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE=\'NO_AUTO_VALUE_ON_ZERO\';".$query["sql"].";SET SQL_MODE=IFNULL(@OLD_SQL_MODE,\'\'), FOREIGN_KEY_CHECKS=IF(@OLD_FOREIGN_KEY_CHECKS=0, 0, 1); ",'. 566 | '(empty($query["bind"])?array():$query["bind"])); $res = "OK";}catch(Exception $e) {echo $e; $res = "error";}' 567 | :'$res = $connection->query("show processlist;")->fetchAll(); $res="OK"; '). 568 | ' $connection->closeConnection(); return $res; }'. 569 | "ini_set('track_errors',true); do{ ob_start(); \$_R_=array('','','');". 570 | "@\$_C_=fread(STDIN,array_pop(unpack('N',fread(STDIN,4)))); @trigger_error(''); ". 571 | "if (eval('return true;if(0){'.\$_C_.'}')) { try { \$_R_[0]=eval(\$_C_); }". 572 | " catch (Exception \$e) { \$php_errormsg = \$e; } } else { \$_R_[0]='E;'; } ". 573 | "\$_R_[1]=\$php_errormsg; \$_R_[2]=ob_get_clean(); \$_RS_ = serialize(\$_R_); ". 574 | "fwrite(STDOUT,pack('N',strlen(\$_RS_)).\$_RS_); fwrite(\$responsePipe, \$myWorkerId.\"\\r\\n\");} while(!empty(\$_C_)); exit;"; 575 | //echo "\r\n$workerCode\r\n"; 576 | $workerProc = "php -d memory_limit=2000M -r ". escapeshellarg($workerCode); 577 | //echo $workerProc."\r\n"; 578 | $procs[$i]= proc_open($workerProc,$descriptorspec, $pipes[$i]); 579 | stream_set_blocking($pipes[$i][0],false); 580 | stream_set_blocking($pipes[$i][1],false); 581 | stream_set_blocking($pipes[$i][2],false); 582 | } 583 | 584 | $queryIds = array_keys($this->_queries); 585 | sort($queryIds); 586 | $queryInThread = array(); 587 | $justCounter = 0; 588 | 589 | $finish = false; 590 | $errors = array(); 591 | $error = ''; 592 | $busyArray = array_pad(array(),$threads,0); 593 | // Sending Queries to each pipe 594 | foreach ($parallelQueries as $queryIds) 595 | { 596 | // Skipping Empty 597 | if ( empty($queryIds)) { 598 | continue; 599 | } 600 | //send: 601 | foreach ($queryIds as $nextQueryId) { 602 | 603 | if ($this->_queries[$nextQueryId]['sql'] == "UPDATE `core_resource` SET `code` = ?, `data_version` = ? WHERE (code = 'core_setup')") { 604 | echo " -- $justCounter/".(count($this->_queries)-1)."\r\n"; 605 | echo "Running Data Upgrade\r\n"; 606 | var_dump($connection->query("show processlist")->fetch()); 607 | if ($real) Mage_Core_Model_Resource_Setup::applyAllDataUpdates(); 608 | $finish =true; 609 | 610 | } 611 | 612 | if (min($busyArray)>0) { 613 | // Open pipe if needed; 614 | if (!isset($waitPipeResource)) { 615 | echo "\r\nWaiting for replies from\r\n"; 616 | $waitPipeResource = fopen($waitPipe,"rb"); 617 | } 618 | $workerId = (int)fgets($waitPipeResource, 1000); 619 | //$workerId = (int)$workerId; 620 | $output = unserialize(fread($pipes[$workerId][1],array_pop(unpack('N',fread($pipes[$workerId][1],4))))); 621 | $error = fread($pipes[$workerId][2],100000); 622 | //var_dump($output, $error); 623 | $nextWorker = $workerId; 624 | 625 | if (!empty($error)) { 626 | $errors[]= array( $nextWorker, $output,$error); 627 | break; 628 | } 629 | if ($output[0] != 'OK') { 630 | var_dump($this->_queries[$queryInThread[$nextWorker]]); 631 | echo $output[2]."\r\n"; 632 | } 633 | $this->_queries[$queryInThread[$nextWorker]]['status']= 'finished'; 634 | } elseif(count($queryIds) > 1) { 635 | asort($busyArray); 636 | reset($busyArray); 637 | $nextWorker = key($busyArray); 638 | } else { 639 | $nextWorker =0; 640 | } 641 | //echo "-- Sending $nextQueryId to $nextWorker\r\n"; 642 | $this->_sendToThread($pipes[$nextWorker][0], $nextWorker, $nextQueryId); 643 | $queryInThread[$nextWorker] = $nextQueryId; 644 | 645 | $busyArray[$nextWorker] = 1; 646 | if($justCounter% 10==1) echo " -- $justCounter/".(count($this->_queries)-1).""; 647 | $justCounter++; 648 | } 649 | 650 | //sync; 651 | while (max($busyArray) > 0) { 652 | // Open pipe if needed; 653 | if (!isset($waitPipeResource)) { 654 | echo "\r\nWaiting for replies, did not use all threads\r\n"; 655 | $waitPipeResource = fopen($waitPipe,"rb"); 656 | } 657 | $workerId = (int)fgets($waitPipeResource, 1000); 658 | $output = unserialize(fread($pipes[$workerId][1],array_pop(unpack('N',fread($pipes[$workerId][1],4))))); 659 | $error = fread($pipes[$workerId][2],100000); 660 | if (!empty($error)) { 661 | $errors[]= array($workerId, $output,$error); 662 | echo "Emergency Shutdown!\r\n"; 663 | } 664 | //var_dump($output, $error); 665 | //echo "loop\r\n"; 666 | //var_dump($output, $error); 667 | $busyArray[$workerId] = 0; 668 | $this->_queries[$queryInThread[$workerId]]['status']= 'finished'; 669 | } 670 | if (!empty($errors)) { 671 | echo "Emergency Shutdown!\r\n"; 672 | echo "Dumping current query and state\r\n"; 673 | var_dump($errors); 674 | foreach($queryInThread as $thread=> $qId) { 675 | echo "Dumping thread $thread\r\n"; 676 | var_dump($thread, $this->_queries[$qId]); 677 | } 678 | break; 679 | } 680 | 681 | if ($output[0] != 'OK') { 682 | var_dump($this->_queries[$queryInThread[$nextWorker]]); 683 | echo $output[2]."\r\n"; 684 | } 685 | 686 | if ($finish) { 687 | break; 688 | } 689 | } 690 | 691 | echo "closed Pipes\r\n"; 692 | foreach($procs as $proc) { 693 | proc_close($proc); 694 | } 695 | echo "Closed Pipes\r\n"; 696 | 697 | fclose($waitPipeResource); 698 | 699 | 700 | //getting reply in pipe, and send next query. 701 | 702 | } 703 | 704 | public function runSingle($real = false, $skipQuery = '') 705 | { 706 | $skipFlag = !empty($skipQuery); 707 | echo "/* Starting\r\n"; 708 | include_once dirname(dirname(__FILE__)).'/app/Mage.php'; 709 | $app = Mage::app(); 710 | $app->cleanCache(); 711 | echo "\nCleaned Cache\r\n"; 712 | $queryIds = array_keys($this->_queries); 713 | sort($queryIds); 714 | $connection = Mage::getSingleton('core/resource')->getConnection('default_setup'); 715 | foreach ($queryIds as $counter=>$qid) { 716 | 717 | if ($skipFlag && preg_match($skipQuery,$this->_queries[$qid]['sql'])) { 718 | $skipFlag = false; 719 | continue; 720 | } 721 | if ($skipFlag) continue; 722 | if ($this->_queries[$qid]['sql'] == "UPDATE `core_resource` SET `code` = ?, `data_version` = ? WHERE (code = 'core_setup')") { 723 | echo "$counter/".(count($queryIds)-1)."\r\n"; 724 | echo "Running Data Upgrade\r\n"; 725 | if ($real) Mage_Core_Model_Resource_Setup::applyAllDataUpdates(); 726 | break; 727 | } 728 | if ($real){ 729 | try { 730 | if (!empty($this->_queries[$qid]['bind'])) { 731 | $connection->query($this->_queries[$qid]['sql'],$this->_queries[$qid]['bind']); 732 | } else { 733 | $connection->query($this->_queries[$qid]['sql']); 734 | } 735 | }catch (Exception $e) { 736 | echo $e; 737 | var_dump($this->_queries[$qid]['sql'], $this->_queries[$qid]['bind']); 738 | } 739 | } 740 | if($counter%10 == 7) echo "$counter/".(count($queryIds)-1)."\r\n"; 741 | } 742 | 743 | echo "done */\r\n"; 744 | 745 | } 746 | 747 | protected function _sendToThread($pipe, $thread, $qId) 748 | { 749 | $sQuery = " return runQuery('".str_replace("'","\\'",serialize($this->_queries[$qId]))."');"; 750 | 751 | $this->_queries[$qId]['thread'] = $thread; 752 | $this->_queries[$qId]['status'] = 'started'; 753 | fwrite($pipe,pack('N',strlen($sQuery)).$sQuery); 754 | } 755 | 756 | protected function _getTimeString($floatTime) 757 | { 758 | $hours = (int)($floatTime/3600); 759 | $minutes = (($floatTime/3600)-$hours)*60; 760 | 761 | return "$hours hours $minutes minutes"; 762 | } 763 | 764 | } 765 | //var_dump('test:'. preg_match('/^## [0-9]+ ## QUERY/','## 9803 ## QUERY')); 766 | 767 | $track = new Upgrade_Replay(); 768 | //$track->prepare("/ALTER TABLE `customer_entity` MODIFY COLUMN `website_id` smallint UNSIGNED NULL COMMENT ''/"); 769 | $track->prepare(); 770 | 771 | $track->echoQueries(); 772 | //$noOfThreads = 4; 773 | echo date(DATE_RFC822) ." ---- Started\r\n"; 774 | //$track->runSingle(false); 775 | //$track->runSingle(true); 776 | //$track->run(4); 777 | $track->run(12, true); 778 | 779 | echo date(DATE_RFC822) ." ---- Finished\r\n"; 780 | //$track->runSingle(true,'/UNQ_CORE_CONFIG_DATA_SCOPE_SCOPE_ID_PATH/'); 781 | //$track->runSingle(true ,'/ALTER TABLE `core_config_data` ADD UNIQUE `UNQ_CORE_CONFIG_DATA_SCOPE_SCOPE_ID_PATH` (`scope`,`scope_id`,`path`)/'); 782 | //$track->runSingle(true,'ALTER TABLE `core_config_data` DROP KEY `config_scope`'); 783 | 784 | 785 | 786 | 787 | 788 | 789 | 790 | 791 | --------------------------------------------------------------------------------