├── screenshots └── Command.png ├── SiteMigration.php ├── Migrator ├── SiteMigrator.php ├── MigratorSettings.php ├── SiteGoalMigrator.php ├── SiteUrlMigrator.php ├── Archive │ └── ArchiveLister.php ├── VisitMigrator.php ├── ConversionItemMigrator.php ├── ConversionMigrator.php ├── LinkVisitActionMigrator.php ├── TableMigrator.php ├── ActionMigrator.php ├── ArchiveMigrator.php └── Migrator.php ├── plugin.json ├── Helper ├── GCHelper.php └── DBHelper.php ├── README.md ├── Test ├── BatchProviderTest.php ├── ConversionItemMigratorTest.php ├── VisitMigratorTest.php ├── ConversionMigratorTest.php ├── LinkVisitActionMigratorTest.php ├── Archive │ └── ArchiveListerTest.php ├── DBHelperTest.php └── ActionMigratorTest.php ├── DataProvider └── BatchProvider.php ├── .travis.yml └── Commands └── MigrateSite.php /screenshots/Command.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/PiwikPRO/plugin-SiteMigration/HEAD/screenshots/Command.png -------------------------------------------------------------------------------- /SiteMigration.php: -------------------------------------------------------------------------------- 1 | =5.3.3", 12 | "piwik": ">=2.11.0" 13 | }, 14 | "authors": [ 15 | { 16 | "name": "Piwik PRO", 17 | "email": "contact@piwik.pro", 18 | "homepage": "http://piwik.pro" 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /Helper/GCHelper.php: -------------------------------------------------------------------------------- 1 | enableGC(); 18 | 19 | parent::__construct(); 20 | } 21 | 22 | public function enableGC() 23 | { 24 | if (!gc_enabled()) { 25 | gc_enable(); 26 | } 27 | } 28 | 29 | public function cleanup() 30 | { 31 | gc_collect_cycles(); 32 | } 33 | 34 | public function cleanVariable(&$variable) 35 | { 36 | unset($variable); 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Migrator/MigratorSettings.php: -------------------------------------------------------------------------------- 1 | siteMigrator = $siteMigrator; 24 | 25 | parent::__construct($targetDb, $gcHelper); 26 | } 27 | 28 | protected function translateRow(&$row) 29 | { 30 | $row['idsite'] = $this->siteMigrator->getNewId($row['idsite']); 31 | } 32 | 33 | /** 34 | * @return string Name of the table migrated by this migration 35 | */ 36 | protected function getTableName() 37 | { 38 | return 'goal'; 39 | } 40 | 41 | /** 42 | * @param array $row 43 | * 44 | * @return int The current id stored in the given row 45 | */ 46 | protected function getIdFromRow(&$row) 47 | { 48 | return null; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Migrator/SiteUrlMigrator.php: -------------------------------------------------------------------------------- 1 | siteMigrator = $siteMigrator; 24 | 25 | parent::__construct($targetDb, $gcHelper); 26 | } 27 | 28 | protected function translateRow(&$row) 29 | { 30 | $row['idsite'] = $this->siteMigrator->getNewId($row['idsite']); 31 | } 32 | 33 | /** 34 | * @return string Name of the table migrated by this migration 35 | */ 36 | protected function getTableName() 37 | { 38 | return 'site_url'; 39 | } 40 | 41 | /** 42 | * @param array $row 43 | * 44 | * @return int The current id stored in the given row 45 | */ 46 | protected function getIdFromRow(&$row) 47 | { 48 | return null; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Migrator/Archive/ArchiveLister.php: -------------------------------------------------------------------------------- 1 | database = $database; 25 | } 26 | 27 | /** 28 | * Returns the archive list as strings looking like this: '2014-01' 29 | * 30 | * @param \DateTime $from 31 | * @param \DateTime $to 32 | * 33 | * @return string[] 34 | */ 35 | public function getArchiveList(\DateTime $from = null, \DateTime $to = null) 36 | { 37 | $tablePrefix = $this->database->prefixTable('archive_numeric_'); 38 | 39 | // We can't use Piwik\DataAccess\ArchiveTableCreator::getTablesArchivesInstalled() 40 | // because of the global DB object: it will use the "sourceDb" instead of the "targetDb"... 41 | // TODO Fix later when we have dependency injection 42 | $archives = $this->database->getAdapter()->fetchCol("SHOW TABLES LIKE '" . $tablePrefix . "%'"); 43 | 44 | $archives = array_map(function ($value) use ($tablePrefix) { 45 | return str_replace($tablePrefix, '', $value); 46 | }, $archives); 47 | 48 | $archives = array_filter($archives, function ($archive) use ($from, $to) { 49 | $date = new \DateTime(str_replace('_', '-', $archive) . '-01'); 50 | 51 | $excluded = ($from && $from > $date) || ($to && $to < $date); 52 | 53 | return !$excluded; 54 | }); 55 | 56 | return array_values($archives); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Migrator/VisitMigrator.php: -------------------------------------------------------------------------------- 1 | actionMigrator = $actionMigrator; 29 | $this->siteMigrator = $siteMigrator; 30 | parent::__construct($targetDb, $gcHelper); 31 | } 32 | 33 | protected function getTableName() 34 | { 35 | return 'log_visit'; 36 | } 37 | 38 | protected function translateRow(&$row) 39 | { 40 | unset($row['idvisit']); 41 | 42 | $row['idsite'] = $this->siteMigrator->getNewId($row['idsite']); 43 | $row['visit_exit_idaction_url'] = $this->actionMigrator->getNewId($row['visit_exit_idaction_url']); 44 | $row['visit_exit_idaction_name'] = $this->actionMigrator->getNewId($row['visit_exit_idaction_name']); 45 | $row['visit_entry_idaction_url'] = $this->actionMigrator->getNewId($row['visit_entry_idaction_url']); 46 | $row['visit_entry_idaction_name'] = $this->actionMigrator->getNewId($row['visit_entry_idaction_name']); 47 | } 48 | 49 | protected function getIdFromRow(&$row) 50 | { 51 | return $row['idvisit']; 52 | } 53 | 54 | public function getIdRanges($chunkSize = 1000) 55 | { 56 | return array_chunk(array_keys($this->getIdMap()), $chunkSize, true); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /Migrator/ConversionItemMigrator.php: -------------------------------------------------------------------------------- 1 | siteMigrator = $siteMigrator; 49 | $this->visitMigrator = $visitMigrator; 50 | $this->actionMigrator = $actionMigrator; 51 | 52 | parent::__construct($targetDb, $gcHelper); 53 | } 54 | 55 | protected function translateRow(&$row) 56 | { 57 | $row['idsite'] = $this->siteMigrator->getNewId($row['idsite']); 58 | $row['idvisit'] = $this->visitMigrator->getNewId($row['idvisit']); 59 | 60 | foreach ($this->actionsToTranslate as $translationKey) { 61 | if ($row[$translationKey] == 0) { 62 | continue; 63 | } 64 | 65 | $row[$translationKey] = $this->actionMigrator->getNewId($row[$translationKey]); 66 | } 67 | } 68 | 69 | /** 70 | * @return string Name of the table migrated by this migration 71 | */ 72 | protected function getTableName() 73 | { 74 | return 'log_conversion_item'; 75 | } 76 | 77 | /** 78 | * @param array $row 79 | * 80 | * @return int The current id stored in the given row 81 | */ 82 | protected function getIdFromRow(&$row) 83 | { 84 | return null; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Migrator/ConversionMigrator.php: -------------------------------------------------------------------------------- 1 | siteMigrator = $siteMigrator; 45 | $this->visitMigrator = $visitMigrator; 46 | $this->actionMigrator = $actionMigrator; 47 | $this->linkVisitActionMigrator = $linkVisitActionMigrator; 48 | 49 | parent::__construct($targetDb, $gcHelper); 50 | } 51 | 52 | protected function translateRow(&$row) 53 | { 54 | $row['idsite'] = $this->siteMigrator->getNewId($row['idsite']); 55 | $row['idvisit'] = $this->visitMigrator->getNewId($row['idvisit']); 56 | 57 | if ($row['idlink_va']) { 58 | $row['idlink_va'] = $this->linkVisitActionMigrator->getNewId($row['idlink_va']); 59 | } 60 | 61 | if ($row['idaction_url']) { 62 | $row['idaction_url'] = $this->actionMigrator->getNewId( 63 | $row['idaction_url'] 64 | ); 65 | } else { 66 | $row['idaction_url'] = 0; 67 | } 68 | } 69 | 70 | /** 71 | * @return string Name of the table migrated by this migration 72 | */ 73 | protected function getTableName() 74 | { 75 | return 'log_conversion'; 76 | } 77 | 78 | /** 79 | * @param array $row 80 | * 81 | * @return int The current id stored in the given row 82 | */ 83 | protected function getIdFromRow(&$row) 84 | { 85 | return null; 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Migrator/LinkVisitActionMigrator.php: -------------------------------------------------------------------------------- 1 | siteMigrator = $siteMigrator; 39 | $this->visitMigrator = $visitMigrator; 40 | $this->actionMigrator = $actionMigrator; 41 | 42 | parent::__construct($targetDb, $gcHelper); 43 | } 44 | 45 | 46 | protected function translateRow(&$row) 47 | { 48 | unset($row['idlink_va']); 49 | 50 | $row['idsite'] = $this->siteMigrator->getNewId($row['idsite']); 51 | $row['idvisit'] = $this->visitMigrator->getNewId($row['idvisit']); 52 | 53 | $row['idaction_url'] = $this->actionMigrator->getNewId($row['idaction_url']); 54 | $row['idaction_url_ref'] = $this->actionMigrator->getNewId($row['idaction_url_ref']); 55 | $row['idaction_name'] = $this->actionMigrator->getNewId($row['idaction_name']); 56 | $row['idaction_name_ref'] = $this->actionMigrator->getNewId($row['idaction_name_ref']); 57 | $row['idaction_event_category'] = $this->actionMigrator->getNewId($row['idaction_event_category']); 58 | $row['idaction_event_action'] = $this->actionMigrator->getNewId($row['idaction_event_action']); 59 | } 60 | 61 | /** 62 | * @return string Name of the table migrated by this migration 63 | */ 64 | protected function getTableName() 65 | { 66 | return 'log_link_visit_action'; 67 | } 68 | 69 | /** 70 | * @param array $row 71 | * 72 | * @return int The current id stored in the given row 73 | */ 74 | protected function getIdFromRow(&$row) 75 | { 76 | return $row['idlink_va']; 77 | } 78 | } 79 | -------------------------------------------------------------------------------- /Migrator/TableMigrator.php: -------------------------------------------------------------------------------- 1 | targetDb = $targetDb; 40 | $this->gcHelper = $gcHelper; 41 | } 42 | 43 | public function migrate(\Traversable $dataProvider) 44 | { 45 | foreach ($dataProvider as $row) { 46 | $this->processRow($row); 47 | } 48 | } 49 | 50 | protected function processRow(&$row) 51 | { 52 | $id = $this->getIdFromRow($row); 53 | $this->translateRow($row); 54 | 55 | $this->targetDb->executeInsert($this->getTableName(), $row); 56 | 57 | if ($id != null) { 58 | $this->addNewId($id, $this->targetDb->lastInsertId()); 59 | } 60 | } 61 | 62 | /** 63 | * @param int $oldId 64 | * @return int 65 | */ 66 | public function getNewId($oldId) 67 | { 68 | if (!array_key_exists($oldId, $this->idMap)) { 69 | throw new \InvalidArgumentException('Id ' . $oldId . ' not found in ' . __CLASS__); 70 | } 71 | 72 | return $this->idMap[$oldId]; 73 | } 74 | 75 | /** 76 | * @param int $oldId 77 | * @param int $newId 78 | */ 79 | public function addNewId($oldId, $newId) 80 | { 81 | $this->idMap[$oldId] = $newId; 82 | } 83 | 84 | /** 85 | * @return int[] 86 | */ 87 | public function getIdMap() 88 | { 89 | return $this->idMap; 90 | } 91 | 92 | public function checkColumns() 93 | { 94 | } 95 | 96 | protected abstract function translateRow(&$row); 97 | 98 | /** 99 | * @return string Name of the table migrated by this migration 100 | */ 101 | protected abstract function getTableName(); 102 | 103 | /** 104 | * @param array $row 105 | * 106 | * @return int The current id stored in the given row 107 | */ 108 | protected abstract function getIdFromRow(&$row); 109 | } 110 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Piwik SiteMigration Plugin 2 | 3 | [![Build Status](https://travis-ci.org/PiwikPRO/plugin-SiteMigration.svg?branch=master)](https://travis-ci.org/PiwikPRO/plugin-SiteMigration) 4 | 5 | ## Description 6 | 7 | Migrate websites, and all the tracking data between two Piwik installations. 8 | 9 | This tool is useful in case you want to merge two Piwik installations, or if you want to move one or several websites to another Piwik server. 10 | 11 | ### Requirements 12 | 13 | To migrate data from one Piwik server to another server, you must: 14 | 15 | * First make sure that both Piwik servers are using the latest Piwik version. 16 | * You must be able to connect to the Mysql server of the Target Piwik Server. 17 | * You must run the console command on the Piwik Server that data will be copied from. 18 | 19 | ### Migrating the data 20 | 21 | Start the migration by calling from the command line CLI the following command: 22 | 23 | ./console migration:site idSite --db-prefix piwik_ 24 | 25 | The command will ask for the credentials to the target database. 26 | 27 | It will then migrate the data from the current Piwik to the target Piwik. 28 | 29 | ### Options 30 | 31 | Run `./console migration:site --help` to get a full list of options. 32 | 33 | ## FAQ 34 | 35 | **How do I migrate site data between two dates only?** 36 | 37 | You can use command options: `--date-from` and `--date-to`. 38 | 39 | **How do I migrate tracking log data only, and skip migrating archived data?** 40 | 41 | Just add the `--skip-archive-data` option. 42 | 43 | **How do I migrate the archived data and skip the tracking data?** 44 | 45 | Just add the `--skip-log-data` option. 46 | 47 | **Can I run the command on the Target Piwik server (where data will be imported)?** 48 | 49 | No, you must run the command from the source Piwik server (the server which contains the data you want to migrate). 50 | 51 | **Can I run SiteMigration in big load environment?** 52 | 53 | That is not advised. Process of moving large amounts of data may interfere with regular processing. 54 | 55 | **Will SiteMigration take full data into the account?** 56 | 57 | Data related to some of the plugins may be lost. If you are using such, we don't advise using automated migration process provided by this plugin. 58 | 59 | We strongly advise to test whole process against your use case before using it in production environment. 60 | 61 | ## Changelog 62 | 63 | **v1.0.7** 64 | 65 | - Updated the plugin for compatibility with Piwik 2.10. 66 | 67 | **v1.0.6** 68 | 69 | - [#6](https://github.com/PiwikPRO/plugin-SiteMigration/issues/6): fixed a PHP 5.3 incompatibility 70 | 71 | **v1.0.5** 72 | 73 | - [#5](https://github.com/PiwikPRO/plugin-SiteMigration/issues/5): fixed `Integrity constraint violation: 1048 Column 'idaction_url' cannot be null` 74 | 75 | **v1.0.4** 76 | 77 | - [#3](https://github.com/PiwikPRO/plugin-SiteMigration/issues/3): fixed `--db-prefix` option 78 | 79 | **v1.0.3** 80 | 81 | - Documentation update 82 | 83 | **v1.0.2** 84 | 85 | - Documentation update & fixed bug when archive_blob tables are not found 86 | 87 | **v1.0.1** 88 | 89 | - Documentation update 90 | 91 | **v1.0.0** 92 | 93 | - First stable release 94 | - Bugfixes 95 | 96 | **v0.1.1** 97 | 98 | - Changed license to free plugin 99 | - Changed name to SiteMigration 100 | 101 | **v0.1.0** 102 | 103 | - Initial release 104 | 105 | ## Credits 106 | 107 | Created by [Piwik PRO](http://piwik.pro/) 108 | -------------------------------------------------------------------------------- /Migrator/ActionMigrator.php: -------------------------------------------------------------------------------- 1 | sourceDb = $sourceDb; 26 | $this->targetDb = $targetDb; 27 | } 28 | 29 | protected function processAction($action) 30 | { 31 | if (array_key_exists($action['type'], $this->existingActions) 32 | && array_key_exists($action['hash'], $this->existingActions[$action['type']]) 33 | ) { 34 | $this->idMap[$action['idaction']] = $this->existingActions[$action['type']][$action['hash']]; 35 | 36 | return; 37 | } 38 | 39 | $idAction = $action['idaction']; 40 | unset($action['idaction']); 41 | $this->targetDb->executeInsert('log_action', $action); 42 | $this->idMap[$idAction] = $this->targetDb->lastInsertId(); 43 | unset($action); 44 | } 45 | 46 | public function loadExistingActions() 47 | { 48 | $query = $this->targetDb->getAdapter()->prepare( 49 | 'SELECT idaction, hash, type FROM ' . $this->targetDb->prefixTable('log_action') 50 | ); 51 | $query->execute(); 52 | 53 | $this->existingActions = array(); 54 | 55 | while ($action = $query->fetch()) { 56 | $this->addExistingAction($action); 57 | } 58 | } 59 | 60 | public function addExistingAction($action) 61 | { 62 | if (!array_key_exists($action['type'], $this->existingActions)) { 63 | $this->existingActions[$action['type']] = array(); 64 | } 65 | 66 | $this->existingActions[$action['type']][$action['hash']] = $action['idaction']; 67 | } 68 | 69 | public function ensureActionIsMigrated($idAction) 70 | { 71 | if (array_key_exists($idAction, $this->idMap)) { 72 | return true; 73 | } else { 74 | $action = $this->sourceDb->getAdapter()->fetchRow( 75 | 'SELECT * FROM ' . $this->sourceDb->prefixTable('log_action') . ' WHERE idaction = ?', 76 | array($idAction) 77 | ); 78 | 79 | if ($action) { 80 | $this->processAction($action); 81 | 82 | return true; 83 | } 84 | } 85 | 86 | return false; 87 | } 88 | 89 | public function getNewId($idAction) 90 | { 91 | if ($this->ensureActionIsMigrated($idAction)) { 92 | return $this->idMap[$idAction]; 93 | } else { 94 | return 0; 95 | } 96 | 97 | } 98 | 99 | /** 100 | * @param array $existingActions 101 | */ 102 | public function setExistingActions($existingActions) 103 | { 104 | $this->existingActions = $existingActions; 105 | } 106 | 107 | /** 108 | * @return array 109 | */ 110 | public function getExistingActions() 111 | { 112 | return $this->existingActions; 113 | } 114 | 115 | /** 116 | * @param int $oldId 117 | * @param int $newId 118 | */ 119 | public function addNewId($oldId, $newId) 120 | { 121 | $this->idMap[$oldId] = $newId; 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /Test/BatchProviderTest.php: -------------------------------------------------------------------------------- 1 | fromDbHelper = $this->getMock( 48 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 49 | array('getAdapter'), 50 | array(), 51 | '', 52 | false 53 | ); 54 | 55 | $this->adapter = $this->getMock( 56 | 'Zend_Db_Adapter_Pdo_Mysql', 57 | array('fetchRow', 'fetchAll', 'prepare'), 58 | array(), 59 | '', 60 | false 61 | ); 62 | 63 | $this->statement = $this->getMock( 64 | 'Zend_Db_Statement_Pdo', 65 | array('execute', 'fetch', 'closeCursor', 'rowCount'), 66 | array(), 67 | '', 68 | false 69 | ); 70 | 71 | $this->fromDbHelper->expects($this->any())->method('getAdapter')->willReturn($this->adapter); 72 | 73 | $this->gcHelper = $this->getMock('Piwik\Plugins\SiteMigration\Helper\GCHelper', array(), array(), '', false); 74 | } 75 | 76 | public function test_it_splitsQueryCorrectly() 77 | { 78 | $this->setupBatchProvider('SELECT * FROM table'); 79 | 80 | $this->adapter->expects($this->exactly(2))->method('prepare')->withConsecutive(array('SELECT * FROM table LIMIT 0, 2'), array('SELECT * FROM table LIMIT 2, 2'))->willReturn($this->statement); 81 | $this->statement->expects($this->exactly(4))->method('fetch')->willReturn(array('foo' => 'bar')); 82 | 83 | $this->batchProvider->rewind(); 84 | $this->batchProvider->next(); 85 | $this->batchProvider->next(); 86 | $this->batchProvider->next(); 87 | } 88 | 89 | public function test_it_supportsMultipleQueries() 90 | { 91 | $this->setupBatchProvider(array('SELECT * FROM table1', 'SELECT * FROM table2', 'SELECT * FROM table3')); 92 | 93 | $this->adapter->expects($this->exactly(3))->method('prepare')->withConsecutive( 94 | array('SELECT * FROM table1 LIMIT 0, 2'), 95 | array('SELECT * FROM table2 LIMIT 0, 2'), 96 | array('SELECT * FROM table3 LIMIT 0, 2') 97 | )->willReturn($this->statement); 98 | 99 | $this->statement->expects($this->exactly(3))->method('fetch')->willReturn(null); 100 | 101 | $this->batchProvider->rewind(); 102 | $this->batchProvider->next(); 103 | $this->batchProvider->next(); 104 | 105 | } 106 | 107 | protected function setupBatchProvider($queries) 108 | { 109 | $this->batchProvider = new BatchProvider($queries, $this->fromDbHelper, $this->gcHelper, 2); 110 | } 111 | } 112 | -------------------------------------------------------------------------------- /Helper/DBHelper.php: -------------------------------------------------------------------------------- 1 | adapter = $adapter; 24 | $this->config = $config; 25 | } 26 | 27 | public function getInsertSQL($table, $values) 28 | { 29 | $vals = array(); 30 | foreach ($values as $key => $val) { 31 | $vals['`' . $key . '`'] = $this->adapter->quote($val); 32 | } 33 | 34 | return "INSERT INTO " . $this->prefixTable($table) . '(' . implode( 35 | ', ', 36 | array_keys($vals) 37 | ) . ') VALUES (' . implode(', ', $vals) . ')'; 38 | } 39 | 40 | public function executeInsert($table, $values) 41 | { 42 | $this->adapter->insert($this->prefixTable($table), $values); 43 | } 44 | 45 | public function insert($table, $values) 46 | { 47 | if (!array_key_exists($table, $this->inserts)) { 48 | $this->inserts[$table] = array(); 49 | } 50 | 51 | $this->inserts[$table][] = $values; 52 | } 53 | 54 | public function flushInserts() 55 | { 56 | foreach ($this->inserts as $table => $inserts) { 57 | $query = 'INSERT INTO ' . $this->prefixTable($table) . ' (`' . implode('`, `', array_keys($inserts[0])) . '`) VALUES '; 58 | 59 | for ($i = 0; $i < count($inserts); $i++) { 60 | if ($i > 0) { 61 | $query .= ', '; 62 | } 63 | 64 | /** 65 | * Workaround for php 5.3 66 | */ 67 | $dbHelper = &$this; 68 | 69 | $values = array_map(function (&$item) use ($dbHelper){ 70 | return $dbHelper->getAdapter()->quote($item); 71 | }, 72 | $inserts[$i]); 73 | 74 | $query .= '(' . implode(', ', $values) . ')'; 75 | } 76 | 77 | $this->adapter->query($query); 78 | } 79 | /** 80 | * Clear inserts 81 | */ 82 | $this->inserts = array(); 83 | } 84 | 85 | public function getDBName() 86 | { 87 | return $this->config['dbname']; 88 | } 89 | 90 | public function acquireLock($lockName, $maxRetries = 30) 91 | { 92 | $sql = 'SELECT GET_LOCK(?, 1)'; 93 | 94 | while ($maxRetries > 0) { 95 | if ($this->adapter->fetchOne($sql, array($lockName)) == '1') { 96 | return true; 97 | } 98 | $maxRetries--; 99 | } 100 | return false; 101 | } 102 | 103 | public function releaseLock($lockName) 104 | { 105 | $sql = 'SELECT RELEASE_LOCK(?)'; 106 | 107 | return $this->adapter->fetchOne($sql, array($lockName)) == '1'; 108 | } 109 | 110 | public function prefixTable($table) 111 | { 112 | return $this->config['tables_prefix'] . $table; 113 | } 114 | 115 | /** 116 | * @return \Zend_Db_Adapter_Abstract 117 | */ 118 | public function getAdapter() 119 | { 120 | return $this->adapter; 121 | } 122 | 123 | /** 124 | * @return mixed 125 | */ 126 | public function getConfig() 127 | { 128 | return $this->config; 129 | } 130 | 131 | public function lastInsertId($tableName = null, $primaryKey = null) 132 | { 133 | return $this->adapter->lastInsertId($tableName, $primaryKey); 134 | } 135 | 136 | public function startTransaction() 137 | { 138 | $this->adapter->beginTransaction(); 139 | } 140 | 141 | public function commitTransaction() 142 | { 143 | $this->adapter->commit(); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /DataProvider/BatchProvider.php: -------------------------------------------------------------------------------- 1 | queries = $queries; 49 | $this->queriesCanonical = $queries; 50 | $this->fromDbHelper = $fromDbHelper; 51 | $this->loadAtOnce = $loadAtOnce; 52 | $this->gcHelper = $gcHelper; 53 | $this->query = array_shift($this->queries); 54 | } 55 | 56 | public function current() 57 | { 58 | return $this->row; 59 | } 60 | 61 | public function next() 62 | { 63 | $this->loadNextRow(); 64 | } 65 | 66 | public function key() 67 | { 68 | return $this->start + $this->currentPosition; 69 | } 70 | 71 | public function valid() 72 | { 73 | return ($this->row != null); 74 | } 75 | 76 | public function rewind() 77 | { 78 | $this->start = 0; 79 | $this->row = null; 80 | $this->rows = null; 81 | $this->queries = $this->queriesCanonical; 82 | $this->query = array_shift($this->queries); 83 | 84 | $this->initQuery(); 85 | $this->loadNextRow(); 86 | } 87 | 88 | protected function initQuery() 89 | { 90 | if ($this->rows instanceof \PDOStatement) { 91 | $this->cleanupResults(); 92 | } 93 | 94 | $this->rows = $this->fromDbHelper->getAdapter()->prepare( 95 | $this->query . ' LIMIT ' . $this->start . ', ' . $this->loadAtOnce 96 | ); 97 | 98 | $this->rows->execute(); 99 | 100 | $this->currentPosition = 0; 101 | } 102 | 103 | protected function loadNextRow() 104 | { 105 | if (!$this->rows) { 106 | return; 107 | } 108 | 109 | if ($this->currentPosition == $this->loadAtOnce) { 110 | $this->start += $this->loadAtOnce; 111 | $this->initQuery(); 112 | } 113 | 114 | $this->row = null; 115 | 116 | while (!$this->row && $this->query) { 117 | $this->row = $this->rows->fetch(); 118 | $this->ensureNextRowIsAvailable(); 119 | } 120 | } 121 | 122 | protected function ensureNextRowIsAvailable() 123 | { 124 | if ($this->row) { 125 | $this->currentPosition++; 126 | } else { 127 | $this->query = array_shift($this->queries); 128 | 129 | if ($this->query) { 130 | $this->start = 0; 131 | $this->initQuery(); 132 | } else { 133 | /** 134 | * No more results, exit 135 | */ 136 | $this->cleanupResults(); 137 | } 138 | } 139 | } 140 | 141 | protected function cleanupResults() 142 | { 143 | $this->rows->closeCursor(); 144 | /** 145 | * Cleanup variable to free memory 146 | */ 147 | $this->gcHelper->cleanVariable($this->rows); 148 | $this->rows = null; 149 | $this->gcHelper->cleanup(); 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /Test/ConversionItemMigratorTest.php: -------------------------------------------------------------------------------- 1 | reset(); 59 | } 60 | 61 | protected function reset() 62 | { 63 | $this->toDbHelper = $this->getMock( 64 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 65 | array('executeInsert', 'lastInsertId', 'getAdapter', 'prefixTable', 'acquireLock', 'releaseLock'), 66 | array(), 67 | '', 68 | false 69 | ); 70 | 71 | 72 | $this->actionMigrator = $this->getMock( 73 | 'Piwik\Plugins\SiteMigration\Migrator\ActionMigrator', 74 | array(), 75 | array(), 76 | '', 77 | false 78 | ); 79 | 80 | $this->siteMigrator = $this->getMock( 81 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 82 | array(), 83 | array(), 84 | '', 85 | false 86 | ); 87 | 88 | $this->visitMigrator = $this->getMock( 89 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 90 | array(), 91 | array(), 92 | '', 93 | false 94 | ); 95 | 96 | $this->linkVisitActionMigrator = $this->getMock( 97 | 'Piwik\Plugins\SiteMigration\Migrator\LinkVisitActionMigrator', 98 | array(), 99 | array(), 100 | '', 101 | false 102 | ); 103 | 104 | $this->gcHelper = $this->getMock('Piwik\Plugins\SiteMigration\Helper\GCHelper', array(), array(), '', false); 105 | 106 | $this->conversionItemMigrator = new ConversionItemMigrator($this->toDbHelper, $this->gcHelper, $this->siteMigrator, $this->visitMigrator, $this->actionMigrator); 107 | } 108 | 109 | public function test_migrateConversionItems() 110 | { 111 | $conversionItem = array( 112 | 'idsite' => 1, 113 | 'idvisit' => 3, 114 | 'idlink_va' => 5, 115 | 'idaction_sku' => 7, 116 | 'idaction_name' => 0, 117 | 'idaction_category' => 11, 118 | 'idaction_category2' => 13, 119 | 'idaction_category3' => 15, 120 | 'idaction_category4' => 17, 121 | 'idaction_category5' => 19 122 | ); 123 | 124 | $this->toDbHelper->expects($this->once())->method('executeInsert')->with( 125 | 'log_conversion_item', 126 | $this->anything() 127 | ); 128 | 129 | $this->siteMigrator->expects($this->once())->method('getNewId')->with(1)->willReturn(2); 130 | $this->visitMigrator->expects($this->once())->method('getNewId')->with(3)->willReturn(4); 131 | $this->actionMigrator->expects($this->exactly(6))->method('getNewId')->will( 132 | $this->onConsecutiveCalls(2, 4, 6, 8, 12, 14, 16, 18, 20) 133 | ); 134 | 135 | $this->conversionItemMigrator->migrate(new \ArrayIterator(array($conversionItem))); 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /Test/VisitMigratorTest.php: -------------------------------------------------------------------------------- 1 | reset(); 55 | } 56 | 57 | protected function reset() 58 | { 59 | $this->adapter = $this->getMock( 60 | 'Zend_Db_Adapter_Pdo_Mysql', 61 | array('fetchRow', 'fetchAll', 'fetchCol', 'prepare', 'query'), 62 | array(), 63 | '', 64 | false 65 | ); 66 | 67 | $this->toDbHelper = $this->getMock( 68 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 69 | array('executeInsert', 'lastInsertId', 'getAdapter', 'prefixTable', 'acquireLock', 'releaseLock'), 70 | array(), 71 | '', 72 | false 73 | ); 74 | 75 | $this->actionMigrator = $this->getMock( 76 | 'Piwik\Plugins\SiteMigration\Migrator\ActionMigrator', 77 | array(), 78 | array(), 79 | '', 80 | false 81 | ); 82 | 83 | 84 | $this->siteMigrator = $this->getMock( 85 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 86 | array(), 87 | array(), 88 | '', 89 | false 90 | ); 91 | 92 | $this->statement = $this->getMock( 93 | 'Zend_Db_Statement_Pdo', 94 | array('execute', 'fetchColumn', 'closeCursor', 'fetch'), 95 | array(), 96 | '', 97 | false 98 | ); 99 | 100 | 101 | $this->gcHelper = $this->getMock('Piwik\Plugins\SiteMigration\Helper\GCHelper', array(), array(), '', false); 102 | 103 | $this->visitMigrator = new VisitMigrator($this->toDbHelper, $this->gcHelper, $this->siteMigrator, $this->actionMigrator); 104 | } 105 | 106 | public function test_migrateVisits() 107 | { 108 | $visit = array( 109 | 'idvisit' => 123, 110 | 'idsite' => 2, 111 | 'visit_exit_idaction_url' => 4, 112 | 'visit_exit_idaction_name' => 6, 113 | 'visit_entry_idaction_url' => 8, 114 | 'visit_entry_idaction_name' => 10, 115 | ); 116 | 117 | $batchProvider = new \ArrayIterator(array($visit)); 118 | 119 | $this->toDbHelper->expects($this->once())->method('executeInsert')->with('log_visit', $this->anything()); 120 | $this->toDbHelper->expects($this->once())->method('lastInsertId')->will($this->returnValue(321)); 121 | 122 | $this->siteMigrator->expects($this->once())->method('getNewId')->with(2)->will($this->returnValue(3)); 123 | $this->actionMigrator->expects($this->exactly(4))->method('getNewId')->will($this->onConsecutiveCalls( 124 | 5, 7, 9, 11 125 | )); 126 | 127 | $this->visitMigrator->migrate($batchProvider); 128 | 129 | $this->assertEquals(321, $this->visitMigrator->getNewId(123)); 130 | } 131 | 132 | protected function setupDbHelperGetAdapter(\PHPUnit_Framework_MockObject_MockObject $adapter) 133 | { 134 | $adapter->expects($this->atLeastOnce())->method('getAdapter')->with()->will( 135 | $this->returnValue($this->adapter) 136 | ); 137 | } 138 | } 139 | -------------------------------------------------------------------------------- /Test/ConversionMigratorTest.php: -------------------------------------------------------------------------------- 1 | reset(); 59 | } 60 | 61 | protected function reset() 62 | { 63 | $this->toDbHelper = $this->getMock( 64 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 65 | array('executeInsert', 'lastInsertId', 'getAdapter', 'prefixTable', 'acquireLock', 'releaseLock'), 66 | array(), 67 | '', 68 | false 69 | ); 70 | 71 | 72 | $this->actionMigrator = $this->getMock( 73 | 'Piwik\Plugins\SiteMigration\Migrator\ActionMigrator', 74 | array(), 75 | array(), 76 | '', 77 | false 78 | ); 79 | 80 | $this->siteMigrator = $this->getMock( 81 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 82 | array(), 83 | array(), 84 | '', 85 | false 86 | ); 87 | 88 | $this->visitMigrator = $this->getMock( 89 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 90 | array(), 91 | array(), 92 | '', 93 | false 94 | ); 95 | 96 | $this->linkVisitActionMigrator = $this->getMock( 97 | 'Piwik\Plugins\SiteMigration\Migrator\LinkVisitActionMigrator', 98 | array(), 99 | array(), 100 | '', 101 | false 102 | ); 103 | 104 | $this->gcHelper = $this->getMock('Piwik\Plugins\SiteMigration\Helper\GCHelper', array(), array(), '', false); 105 | 106 | $this->conversionMigrator = new ConversionMigrator($this->toDbHelper, $this->gcHelper, $this->siteMigrator, $this->visitMigrator, $this->actionMigrator, $this->linkVisitActionMigrator); 107 | 108 | } 109 | 110 | public function test_migrateConversions() 111 | { 112 | $conversion = array( 113 | 'idsite' => 1, 114 | 'idvisit' => 3, 115 | 'idlink_va' => 5, 116 | 'idaction_url' => 7, 117 | ); 118 | 119 | $this->toDbHelper->expects($this->once())->method('executeInsert') 120 | ->with( 121 | 'log_conversion', 122 | $this->equalTo( 123 | array( 124 | 'idsite' => 2, 125 | 'idvisit' => 4, 126 | 'idlink_va' => 6, 127 | 'idaction_url' => 8, 128 | ) 129 | ) 130 | ); 131 | 132 | $this->siteMigrator->expects($this->once())->method('getNewId')->with(1)->willReturn(2); 133 | $this->visitMigrator->expects($this->once())->method('getNewId')->with(3)->willReturn(4); 134 | $this->linkVisitActionMigrator->expects($this->once())->method('getNewId')->with(5)->willReturn(6); 135 | $this->actionMigrator->expects($this->once())->method('getNewId')->with(7)->willReturn(8); 136 | 137 | 138 | $this->conversionMigrator->migrate(new \ArrayIterator(array($conversion))); 139 | } 140 | } 141 | -------------------------------------------------------------------------------- /Test/LinkVisitActionMigratorTest.php: -------------------------------------------------------------------------------- 1 | reset(); 55 | } 56 | 57 | protected function reset() 58 | { 59 | $this->toDbHelper = $this->getMock( 60 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 61 | array('executeInsert', 'lastInsertId', 'getAdapter', 'prefixTable', 'acquireLock', 'releaseLock'), 62 | array(), 63 | '', 64 | false 65 | ); 66 | 67 | $this->actionMigrator = $this->getMock( 68 | 'Piwik\Plugins\SiteMigration\Migrator\ActionMigrator', 69 | array(), 70 | array(), 71 | '', 72 | false 73 | ); 74 | 75 | $this->siteMigrator = $this->getMock( 76 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 77 | array(), 78 | array(), 79 | '', 80 | false 81 | ); 82 | 83 | $this->visitMigrator = $this->getMock( 84 | 'Piwik\Plugins\SiteMigration\Migrator\SiteMigrator', 85 | array(), 86 | array(), 87 | '', 88 | false 89 | ); 90 | 91 | $this->gcHelper = $this->getMock('Piwik\Plugins\SiteMigration\Helper\GCHelper', array(), array(), '', false); 92 | 93 | $this->linkVisitActionMigrator = new LinkVisitActionMigrator($this->toDbHelper, $this->gcHelper, $this->siteMigrator, $this->visitMigrator, $this->actionMigrator); 94 | } 95 | 96 | 97 | public function test_migrateVisitActions() 98 | { 99 | $linkVisitAction = array( 100 | 'idsite' => 1, 101 | 'idvisit' => 3, 102 | 'idlink_va' => 5, 103 | 'idaction_url' => 7, 104 | 'idaction_url_ref' => 9, 105 | 'idaction_name' => 11, 106 | 'idaction_name_ref' => 13, 107 | 'idaction_event_category' => 15, 108 | 'idaction_event_action' => 17, 109 | ); 110 | 111 | $this->toDbHelper->expects($this->once())->method('executeInsert')->with( 112 | 'log_link_visit_action', 113 | $this->equalTo( 114 | array( 115 | 'idsite' => 2, 116 | 'idvisit' => 4, 117 | 'idaction_url' => 8, 118 | 'idaction_url_ref' => 10, 119 | 'idaction_name' => 12, 120 | 'idaction_name_ref' => 14, 121 | 'idaction_event_category' => 16, 122 | 'idaction_event_action' => 18, 123 | ) 124 | ) 125 | ); 126 | $this->toDbHelper->expects($this->once())->method('lastInsertId')->will($this->returnValue(6)); 127 | 128 | $this->siteMigrator->expects($this->once())->method('getNewId')->with(1)->willReturn(2); 129 | $this->visitMigrator->expects($this->once())->method('getNewId')->with(3)->willReturn(4); 130 | 131 | $this->actionMigrator->expects($this->exactly(6))->method('getNewId')->will($this->onConsecutiveCalls( 132 | 8, 10, 12, 14, 16, 18 133 | )); 134 | 135 | $this->linkVisitActionMigrator->migrate(new \ArrayIterator(array($linkVisitAction))); 136 | 137 | $this->assertEquals(6, $this->linkVisitActionMigrator->getNewId(5)); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /Migrator/ArchiveMigrator.php: -------------------------------------------------------------------------------- 1 | sourceDb = $sourceDb; 51 | $this->targetDb = $targetDb; 52 | $this->siteMigrator = $siteMigrator; 53 | $this->archiveLister = $archiveLister; 54 | } 55 | 56 | public function migrate($siteId, \DateTime $from = null, \DateTime $to = null) 57 | { 58 | $archives = $this->archiveLister->getArchiveList($from, $to); 59 | 60 | foreach ($archives as $archiveDate) { 61 | Log::debug('Migrating archive ' . $archiveDate); 62 | 63 | $this->migrateArchive($archiveDate, 'archive_numeric_' . $archiveDate, $siteId); 64 | 65 | try { 66 | $this->migrateArchive($archiveDate, 'archive_blob_' . $archiveDate, $siteId); 67 | } catch(\Exception $e) { 68 | // blob tables can be missing 69 | } 70 | } 71 | } 72 | 73 | private function migrateArchive($archiveDate, $archiveTable, $siteId) 74 | { 75 | $this->ensureTargetTableExists($archiveTable); 76 | 77 | $records = $this->getArchiveRecordsQuery($archiveTable, $siteId); 78 | 79 | while ($record = $records->fetch()) { 80 | $this->processArchive($archiveDate, $archiveTable, $record); 81 | } 82 | } 83 | 84 | private function processArchive($archiveDate, $archiveTable, $record) 85 | { 86 | $record['idarchive'] = $this->getArchiveId($archiveDate, $record['idarchive']); 87 | $record['idsite'] = $this->siteMigrator->getNewId($record['idsite']); 88 | 89 | $this->targetDb->executeInsert($archiveTable, $record); 90 | } 91 | 92 | private function ensureTargetTableExists($archiveTable) 93 | { 94 | $data = $this->targetDb->getAdapter()->fetchCol( 95 | "SHOW TABLES LIKE '" . $this->targetDb->prefixTable($archiveTable) . "'" 96 | ); 97 | 98 | if (count($data) == 0) { 99 | $tableType = (strpos($archiveTable, 'blob')) ? 'archive_blob' : 'archive_numeric'; 100 | $sql = PiwikDbHelper::getTableCreateSql($tableType); 101 | $sql = str_replace($tableType, $archiveTable, $sql); 102 | $sql = str_replace($this->sourceDb->prefixTable($tableType), $this->targetDb->prefixTable($tableType), $sql); 103 | 104 | $this->targetDb->getAdapter()->query($sql); 105 | } 106 | } 107 | 108 | private function getArchiveRecordsQuery($archiveTable, $idSite) 109 | { 110 | $query = $this->sourceDb->getAdapter()->prepare( 111 | 'SELECT * FROM ' . $this->sourceDb->prefixTable($archiveTable) . ' WHERE idsite = ?' 112 | ); 113 | $query->execute(array($idSite)); 114 | 115 | return $query; 116 | } 117 | 118 | private function getArchiveId($archiveDate, $archiveId) 119 | { 120 | if (! isset($this->archiveIdMap[$archiveDate][$archiveId])) { 121 | 122 | $sequence = new Sequence( 123 | $this->targetDb->prefixTable('archive_numeric_' . $archiveDate), 124 | $this->targetDb->getAdapter(), 125 | $this->targetDb->prefixTable('') 126 | ); 127 | 128 | if (! $sequence->exists()) { 129 | $sequence->create(); 130 | } 131 | 132 | $this->archiveIdMap[$archiveDate][$archiveId] = $sequence->getNextId(); 133 | } 134 | 135 | return $this->archiveIdMap[$archiveDate][$archiveId]; 136 | } 137 | } 138 | -------------------------------------------------------------------------------- /Test/Archive/ArchiveListerTest.php: -------------------------------------------------------------------------------- 1 | createDbAdapter(array( 18 | 'archive_numeric_2014_10', 19 | 'archive_numeric_2014_11', 20 | )); 21 | $db = $this->createDbHelper($dbAdapter, ''); 22 | 23 | $lister = new ArchiveLister($db); 24 | 25 | $expected = array( 26 | '2014_10', 27 | '2014_11', 28 | ); 29 | $this->assertEquals($expected, $lister->getArchiveList()); 30 | } 31 | 32 | public function testWithPrefix() 33 | { 34 | $dbAdapter = $this->createDbAdapter(array( 35 | 'piwik_archive_numeric_2014_10', 36 | 'piwik_archive_numeric_2014_11', 37 | )); 38 | $db = $this->createDbHelper($dbAdapter, 'piwik_'); 39 | 40 | $lister = new ArchiveLister($db); 41 | 42 | $expected = array( 43 | '2014_10', 44 | '2014_11', 45 | ); 46 | $this->assertEquals($expected, $lister->getArchiveList()); 47 | } 48 | 49 | public function testWithDates() 50 | { 51 | $dbAdapter = $this->createDbAdapter(array( 52 | 'archive_numeric_2013_11', 53 | 'archive_numeric_2014_09', 54 | 'archive_numeric_2014_10', 55 | 'archive_numeric_2014_11', 56 | 'archive_numeric_2014_12', 57 | 'archive_numeric_2015_11', 58 | )); 59 | $db = $this->createDbHelper($dbAdapter, ''); 60 | 61 | $lister = new ArchiveLister($db); 62 | 63 | $expected = array( 64 | '2014_10', 65 | '2014_11', 66 | ); 67 | $this->assertEquals($expected, $lister->getArchiveList(new \DateTime('2014-10-01'), new \DateTime('2014-11-01'))); 68 | } 69 | 70 | public function testWithFromDate() 71 | { 72 | $dbAdapter = $this->createDbAdapter(array( 73 | 'archive_numeric_2013_11', 74 | 'archive_numeric_2014_09', 75 | 'archive_numeric_2014_10', 76 | 'archive_numeric_2014_11', 77 | 'archive_numeric_2014_12', 78 | 'archive_numeric_2015_11', 79 | )); 80 | $db = $this->createDbHelper($dbAdapter, ''); 81 | 82 | $lister = new ArchiveLister($db); 83 | 84 | $expected = array( 85 | '2014_10', 86 | '2014_11', 87 | '2014_12', 88 | '2015_11', 89 | ); 90 | $this->assertEquals($expected, $lister->getArchiveList(new \DateTime('2014-10-01'))); 91 | } 92 | 93 | public function testWithToDate() 94 | { 95 | $dbAdapter = $this->createDbAdapter(array( 96 | 'archive_numeric_2013_11', 97 | 'archive_numeric_2014_09', 98 | 'archive_numeric_2014_10', 99 | 'archive_numeric_2014_11', 100 | 'archive_numeric_2014_12', 101 | 'archive_numeric_2015_11', 102 | )); 103 | $db = $this->createDbHelper($dbAdapter, ''); 104 | 105 | $lister = new ArchiveLister($db); 106 | 107 | $expected = array( 108 | '2013_11', 109 | '2014_09', 110 | '2014_10', 111 | '2014_11', 112 | ); 113 | $this->assertEquals($expected, $lister->getArchiveList(null, new \DateTime('2014-11-01'))); 114 | } 115 | 116 | private function createDbAdapter(array $archiveTables) 117 | { 118 | $dbAdapter = $this->getMockForAbstractClass('Zend_Db_Adapter_Abstract', array(), '', false, false, true, array('fetchCol')); 119 | 120 | $dbAdapter->expects($this->any()) 121 | ->method('fetchCol') 122 | ->willReturn($archiveTables); 123 | 124 | return $dbAdapter; 125 | } 126 | 127 | private function createDbHelper($dbAdapter, $tablePrefix) 128 | { 129 | $db = $this->getMock('Piwik\Plugins\SiteMigration\Helper\DBHelper', array(), array(), '', false); 130 | $db->expects($this->any()) 131 | ->method('getAdapter') 132 | ->willReturn($dbAdapter); 133 | 134 | $db->expects($this->any()) 135 | ->method('prefixTable') 136 | ->willReturnCallback(function ($table) use ($tablePrefix) { 137 | return $tablePrefix . $table; 138 | }); 139 | 140 | return $db; 141 | } 142 | } 143 | -------------------------------------------------------------------------------- /Test/DBHelperTest.php: -------------------------------------------------------------------------------- 1 | reset(); 35 | } 36 | 37 | protected function reset($translations = array()) 38 | { 39 | $this->adapter = $this->getMock( 40 | 'Zend_Db_Adapter_Pdo_Mysql', 41 | array(), 42 | array(), 43 | '', 44 | false 45 | ); 46 | $this->config = array( 47 | 'dbname' => 'piwik__test', 48 | 'host' => 'example.com', 49 | 'username' => 'example', 50 | 'password' => 'foobar', 51 | 'tables_prefix' => 'piwik_', 52 | ); 53 | 54 | $this->dbHelper = new DBHelper($this->adapter, $this->config); 55 | } 56 | 57 | public function test_getAdapter() 58 | { 59 | $this->assertEquals($this->adapter, $this->dbHelper->getAdapter()); 60 | } 61 | 62 | public function test_getConfig() 63 | { 64 | $this->assertEquals($this->config, $this->dbHelper->getConfig()); 65 | } 66 | 67 | public function test_getInsertSQL() 68 | { 69 | $values = array('foo' => 'bar', 'dummy' => 'dummy'); 70 | $sql = "INSERT INTO piwik_table(`foo`, `dummy`) VALUES ('bar', 'dummy')"; 71 | 72 | $this->adapter->expects($this->exactly(2))->method('quote')->will( 73 | $this->returnCallback( 74 | function ($text) { 75 | return "'" . $text . "'"; 76 | } 77 | ) 78 | ); 79 | 80 | $this->assertEquals($sql, $this->dbHelper->getInsertSQL('table', $values)); 81 | } 82 | 83 | public function test_executeInsert() 84 | { 85 | $this->adapter->expects($this->once())->method('insert')->with($this->dbHelper->prefixTable('table'), $this->anything()); 86 | $this->dbHelper->executeInsert('table', array()); 87 | } 88 | 89 | public function test_getDBName() 90 | { 91 | $this->assertEquals('piwik__test', $this->dbHelper->getDBName()); 92 | } 93 | 94 | public function test_acquireLock_positive() 95 | { 96 | $this->adapter->expects($this->once())->method('fetchOne')->will($this->returnValue('1')); 97 | 98 | $this->assertEquals(true, $this->dbHelper->acquireLock('dummy')); 99 | } 100 | 101 | public function test_acquireLock_negative() 102 | { 103 | $this->adapter->expects($this->exactly(10))->method('fetchOne')->will($this->returnValue('0')); 104 | 105 | $this->assertEquals(false, $this->dbHelper->acquireLock('dummy', 10)); 106 | } 107 | 108 | public function test_releaseLock() 109 | { 110 | $this->adapter->expects($this->once())->method('fetchOne')->will($this->returnValue('1')); 111 | 112 | $this->assertEquals(true, $this->dbHelper->releaseLock('dummy')); 113 | } 114 | 115 | public function test_lastInsertId() 116 | { 117 | $this->adapter->expects($this->once())->method('lastInsertId')->with('table', null)->will($this->returnValue(2)); 118 | 119 | $this->assertEquals(2, $this->dbHelper->lastInsertId('table')); 120 | } 121 | 122 | /** 123 | * @test 124 | */ 125 | public function test_it_flushes_correctly() 126 | { 127 | $this->dbHelper->insert('test1', array('k1' => 'v1', 'k2' => 2)); 128 | $this->dbHelper->insert('test1', array('k1' => 'v3', 'k2' => 4)); 129 | $this->dbHelper->insert('test2', array('k3' => 'v5', 'k4' => 6)); 130 | 131 | $this->adapter->expects($this->exactly(6))->method('quote')->willReturnCallback(function ($arg) { 132 | return "'" . $arg . "'"; 133 | }); 134 | $this->adapter->expects($this->exactly(2))->method('query')->withConsecutive( 135 | array( 136 | "INSERT INTO piwik_test1 (`k1`, `k2`) VALUES ('v1', '2'), ('v3', '4')" 137 | ), 138 | array( 139 | "INSERT INTO piwik_test2 (`k3`, `k4`) VALUES ('v5', '6')" 140 | ) 141 | ); 142 | 143 | $this->dbHelper->flushInserts(); 144 | } 145 | } 146 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | # do not edit this file manually, instead run the generate:travis-yml console command 2 | 3 | language: php 4 | 5 | php: 6 | - 5.6 7 | - 5.3 8 | # - hhvm 9 | 10 | services: 11 | - redis-server 12 | 13 | addons: 14 | apt: 15 | sources: 16 | - deadsnakes 17 | 18 | packages: 19 | - python2.6 20 | - python2.6-dev 21 | - nginx 22 | - realpath 23 | - lftp 24 | 25 | # Separate different test suites 26 | env: 27 | global: 28 | - PLUGIN_NAME=SiteMigration 29 | - PIWIK_ROOT_DIR=$TRAVIS_BUILD_DIR/piwik 30 | - PIWIK_LATEST_STABLE_TEST_TARGET=2.15.0-rc4 31 | - secure: "MYN09GOHLHwhPl8+MXK4iDstpv4tVcE2mnAzoxOPCgwtkVGtoSKRwhjwsIWv74FxbwI82s4iSwy9/u8HDUYtPFtAqSiOTC/O26uIoUxn/I3Zgiw5D2IlryL8NJ1xzIjh2nsoSMzQ2ipoGq/DjGsO2nq/S9m7JosaPHoMfujYkWg=" 32 | - secure: "dLzccYWjSBxe7SNph/mPIEDljLqCJgdnAEv7vpqLObLYGdYlnJbYbS/xmCQCwhjMeGDXduhwZfWqhEGwhZnoSDy6codXBevwI7MZXMQFWmkMbkdj5ukBMNP3YmbLAV3y5VlY0Y7L8Qu5JpWsI/IcALSLIHDvBLgztufx2DepNs4=" 33 | matrix: 34 | - TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET 35 | - TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=minimum_required_piwik 36 | 37 | matrix: 38 | exclude: 39 | # execute latest stable tests only w/ PHP 5.6 40 | - php: 5.3.3 41 | env: TEST_SUITE=PluginTests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_CORE=minimum_required_piwik 42 | # execute UI tests only w/ PHP 5.6 43 | - php: 5.3.3 44 | env: TEST_SUITE=UITests MYSQL_ADAPTER=PDO_MYSQL TEST_AGAINST_PIWIK_BRANCH=$PIWIK_LATEST_STABLE_TEST_TARGET 45 | 46 | sudo: required 47 | 48 | script: $PIWIK_ROOT_DIR/tests/travis/travis.sh 49 | 50 | before_install: 51 | # do not use the Zend allocator on PHP 5.3 since it will randomly segfault after program execution 52 | - '[[ "$TRAVIS_PHP_VERSION" == 5.3* ]] && export USE_ZEND_ALLOC=0 || true' 53 | 54 | install: 55 | # move all contents of current repo (which contains the plugin) to a new directory 56 | - mkdir $PLUGIN_NAME 57 | - cp -R !($PLUGIN_NAME) $PLUGIN_NAME 58 | - cp -R .git/ $PLUGIN_NAME/ 59 | - cp .travis.yml $PLUGIN_NAME 60 | # checkout piwik in the current directory 61 | - git clone -q https://github.com/piwik/piwik.git piwik 62 | - cd piwik 63 | - git fetch -q --all 64 | - git submodule update 65 | 66 | # make sure travis-scripts repo is latest for initial travis setup 67 | - '[ -d ./tests/travis/.git ] || sh -c "rm -rf ./tests/travis && git clone https://github.com/piwik/travis-scripts.git ./tests/travis"' 68 | - cd ./tests/travis ; git checkout master ; cd ../.. 69 | 70 | - export GENERATE_TRAVIS_YML_COMMAND="php ./tests/travis/generator/main.php generate:travis-yml --plugin=\"SiteMigration\" --verbose" 71 | 72 | - ./tests/travis/checkout_test_against_branch.sh 73 | 74 | - '[ "$PLUGIN_NAME" == "" ] || [ ! -f ./tests/travis/check_plugin_compatible_with_piwik.php ] || php ./tests/travis/check_plugin_compatible_with_piwik.php "$PLUGIN_NAME"' 75 | 76 | - ./tests/travis/configure_git.sh 77 | 78 | # disable tls for php 5.3 as openssl isn't available 79 | - '[[ "$TRAVIS_PHP_VERSION" == 5.3* ]] && composer config -g -- disable-tls true || true' 80 | 81 | # travis now complains about this failing 9 times out of 10, so removing it 82 | #- travis_retry composer self-update 83 | 84 | - '[ "$SKIP_COMPOSER_INSTALL" == "1" ] || travis_retry composer install' 85 | 86 | # move plugin contents to folder in the plugins subdirectory 87 | - rm -rf plugins/$PLUGIN_NAME 88 | - mv ../$PLUGIN_NAME plugins 89 | 90 | # clone dependent repos 91 | - ./tests/travis/checkout_dependent_plugins.sh 92 | 93 | before_script: 94 | - if [[ "$TRAVIS_PHP_VERSION" != 7* ]]; then phpenv config-rm xdebug.ini; fi 95 | 96 | # add always_populate_raw_post_data=-1 to php.ini 97 | - echo "always_populate_raw_post_data=-1" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini 98 | 99 | # disable opcache to avoid random failures on travis 100 | - echo "opcache.enable=0" >> ~/.phpenv/versions/$(phpenv version-name)/etc/php.ini 101 | 102 | # make tmpfs and run MySQL on it for reasonnable performance 103 | - sudo mkdir /mnt/ramdisk 104 | - sudo mount -t tmpfs -o size=1024m tmpfs /mnt/ramdisk 105 | - sudo stop mysql 106 | - sudo mv /var/lib/mysql /mnt/ramdisk 107 | - sudo ln -s /mnt/ramdisk/mysql /var/lib/mysql 108 | - sudo start mysql 109 | 110 | # print out mysql information 111 | - mysql --version 112 | - mysql -e "SELECT VERSION();" 113 | 114 | # configure mysql 115 | - mysql -e "SET GLOBAL sql_mode = 'NO_ENGINE_SUBSTITUTION,STRICT_TRANS_TABLES'" # Travis default 116 | # try to avoid 'mysql has gone away' errors 117 | - mysql -e "SET GLOBAL wait_timeout = 36000;" 118 | - mysql -e "SET GLOBAL max_allowed_packet = 134209536;" 119 | - mysql -e "SHOW VARIABLES LIKE 'max_allowed_packet';" 120 | - mysql -e "SHOW VARIABLES LIKE 'wait_timeout';" 121 | 122 | - mysql -e "SELECT @@sql_mode;" 123 | # - mysql -e "SHOW GLOBAL VARIABLES;" 124 | 125 | # print out more debugging info 126 | - uname -a 127 | - date 128 | - php -r "var_dump(gd_info());" 129 | - mysql -e 'create database piwik_tests;' 130 | 131 | - ./tests/travis/prepare.sh 132 | - ./tests/travis/setup_webserver.sh 133 | 134 | - cd tests/PHPUnit 135 | 136 | after_script: 137 | # change directory back to root travis dir 138 | - cd $PIWIK_ROOT_DIR 139 | 140 | # output contents of files w/ debugging info to screen 141 | - cat $PIWIK_ROOT_DIR/tests/travis/error.log 142 | - cat $PIWIK_ROOT_DIR/tmp/php-fpm.log 143 | - cat $PIWIK_ROOT_DIR/tmp/logs/piwik.log 144 | - cat $PIWIK_ROOT_DIR/config/config.ini.php 145 | 146 | # upload test artifacts (for debugging travis failures) 147 | - ./tests/travis/upload_artifacts.sh 148 | 149 | after_success: 150 | - cd $PIWIK_ROOT_DIR 151 | -------------------------------------------------------------------------------- /Test/ActionMigratorTest.php: -------------------------------------------------------------------------------- 1 | 123, 47 | 'name' => 'Dummy action', 48 | 'hash' => '1239394545', 49 | 'type' => '4', 50 | 'url_prefix' => null, 51 | ); 52 | 53 | protected $dummyExistingActions = array('4' => array('1239394545' => 321)); 54 | 55 | public function setUp() 56 | { 57 | parent::setUp(); 58 | 59 | $this->resetActionConfig(); 60 | } 61 | 62 | protected function resetActionConfig() 63 | { 64 | $this->adapter = $this->getMock( 65 | 'Zend_Db_Adapter_Pdo_Mysql', 66 | array('fetchRow', 'fetchAll', 'prepare'), 67 | array(), 68 | '', 69 | false 70 | ); 71 | $this->fromDbHelper = $this->getMock( 72 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 73 | array('executeInsert', 'lastInsertId', 'getAdapter', 'prefixTable'), 74 | array(), 75 | '', 76 | false 77 | ); 78 | 79 | $this->toDbHelper = $this->getMock( 80 | 'Piwik\Plugins\SiteMigration\Helper\DBHelper', 81 | array('executeInsert', 'lastInsertId', 'getAdapter', 'prefixTable'), 82 | array(), 83 | '', 84 | false 85 | ); 86 | 87 | $this->statement = $this->getMock( 88 | 'Zend_Db_Statement_Pdo', 89 | array('execute', 'fetch'), 90 | array(), 91 | '', 92 | false 93 | ); 94 | 95 | $this->actionMigrator = new ActionMigrator($this->fromDbHelper, $this->toDbHelper); 96 | 97 | } 98 | 99 | 100 | public function test_ensureActionIsMigrated_actionIsMigrated() 101 | { 102 | $this->actionMigrator->addNewId(123, 321); 103 | 104 | $this->actionMigrator->ensureActionIsMigrated(123); 105 | } 106 | 107 | public function test_ensureActionIsMigrated_actionIsNotMigrated() 108 | { 109 | $action = $this->dummyAction; 110 | 111 | $actionTranslated = $action; 112 | unset($actionTranslated['idaction']); 113 | 114 | $this->setupEnsureActionIsMigratedMigrationTest($action); 115 | 116 | $this->toDbHelper->expects($this->once())->method('executeInsert')->with( 117 | $this->equalTo('log_action'), 118 | $this->equalTo($actionTranslated) 119 | ); 120 | 121 | $this->toDbHelper->expects($this->once())->method('lastInsertId')->will($this->returnValue(321)); 122 | $this->actionMigrator->ensureActionIsMigrated(123); 123 | 124 | $this->assertEquals($this->actionMigrator->getNewId(123), 321); 125 | } 126 | 127 | public function test_ensureActionIsMigrated_actionDoesNotExist() 128 | { 129 | $action = null; 130 | 131 | $this->setupDbHelperGetAdapter($this->fromDbHelper); 132 | $this->assertFalse($this->actionMigrator->ensureActionIsMigrated(123)); 133 | } 134 | 135 | public function test_ensureActionIsMigrated_actionAlreadyMigrated() 136 | { 137 | $existingActions = $this->dummyExistingActions; 138 | 139 | $this->actionMigrator->setExistingActions($existingActions); 140 | $this->toDbHelper->expects($this->never())->method('executeInsert'); 141 | $this->adapter->expects($this->once())->method('fetchRow')->will($this->returnValue($this->dummyAction)); 142 | $this->setupDbHelperGetAdapter($this->fromDbHelper); 143 | 144 | $this->assertTrue($this->actionMigrator->ensureActionIsMigrated(123)); 145 | } 146 | 147 | public function test_addExistingAction_addAction() 148 | { 149 | $action = $this->dummyAction; 150 | $action['idaction'] = 321; 151 | $this->actionMigrator->addExistingAction($action); 152 | $this->assertEquals($this->dummyExistingActions, $this->actionMigrator->getExistingActions()); 153 | } 154 | 155 | public function test_loadExistingActions() 156 | { 157 | $action = $this->dummyAction; 158 | $action['idaction'] = 321; 159 | $this->setupDbHelperGetAdapter($this->toDbHelper); 160 | 161 | $this->adapter->expects($this->once())->method('prepare')->will($this->returnValue($this->statement)); 162 | $this->toDbHelper->expects($this->once())->method('prefixTable')->will($this->returnValue('piwik_')); 163 | $this->statement->expects($this->once())->method('execute'); 164 | $this->statement->expects($this->exactly(2))->method('fetch')->will($this->onConsecutiveCalls($action, null)); 165 | 166 | $this->actionMigrator->loadExistingActions(); 167 | 168 | $this->assertEquals($this->dummyExistingActions, $this->actionMigrator->getExistingActions()); 169 | } 170 | 171 | protected function setupEnsureActionIsMigratedMigrationTest($action) 172 | { 173 | $this->setupDbHelperGetAdapter($this->fromDbHelper); 174 | $this->adapter->expects($this->once())->method('fetchRow')->will($this->returnValue($action)); 175 | } 176 | 177 | protected function setupDbHelperGetAdapter(\PHPUnit_Framework_MockObject_MockObject $helper) 178 | { 179 | $helper->expects($this->once())->method('getAdapter')->with()->will( 180 | $this->returnValue($this->adapter) 181 | ); 182 | } 183 | 184 | } 185 | -------------------------------------------------------------------------------- /Migrator/Migrator.php: -------------------------------------------------------------------------------- 1 | sourceDbHelper = $sourceDbHelper; 81 | $this->targetDbHelper = $targetDbHelper; 82 | $this->gcHelper = $gcHelper; 83 | $this->settings = $migratorSettings; 84 | $this->archiveLister = $archiveLister; 85 | 86 | $this->setupMigrators(); 87 | } 88 | 89 | private function setupMigrators() 90 | { 91 | $this->siteMigrator = new SiteMigrator($this->targetDbHelper, $this->gcHelper); 92 | $this->siteGoalMigrator = new SiteGoalMigrator($this->targetDbHelper, $this->gcHelper, $this->siteMigrator); 93 | $this->siteUrlMigrator = new SiteUrlMigrator($this->targetDbHelper, $this->gcHelper, $this->siteMigrator); 94 | $this->actionMigrator = new ActionMigrator($this->sourceDbHelper, $this->targetDbHelper); 95 | $this->visitMigrator = new VisitMigrator($this->targetDbHelper, $this->gcHelper, $this->siteMigrator, $this->actionMigrator); 96 | $this->visitActionMigrator = new LinkVisitActionMigrator($this->targetDbHelper, $this->gcHelper, $this->siteMigrator, $this->visitMigrator, $this->actionMigrator); 97 | $this->conversionMigrator = new ConversionMigrator($this->targetDbHelper, $this->gcHelper, $this->siteMigrator, $this->visitMigrator, $this->actionMigrator, $this->visitActionMigrator); 98 | $this->conversionItemMigrator = new ConversionItemMigrator($this->targetDbHelper, $this->gcHelper, $this->siteMigrator, $this->visitMigrator, $this->actionMigrator); 99 | $this->archiveMigrator = new ArchiveMigrator($this->sourceDbHelper, $this->targetDbHelper, $this->siteMigrator, $this->archiveLister); 100 | } 101 | 102 | public function migrate() 103 | { 104 | $this->startTransaction(); 105 | 106 | $this->migrateSiteConfig(); 107 | 108 | $this->loadActions(); 109 | 110 | if (!$this->settings->skipLogData) { 111 | $this->migrateLogVisits(); 112 | $this->migrateLogVisitActions(); 113 | $this->migrateLogVisitConversions(); 114 | } 115 | 116 | if (!$this->settings->skipArchiveData) { 117 | $this->migrateArchives(); 118 | } 119 | 120 | $this->commitTransaction(); 121 | } 122 | 123 | private function startTransaction() 124 | { 125 | Log::info('Start transaction'); 126 | $this->targetDbHelper->startTransaction(); 127 | } 128 | 129 | private function commitTransaction() 130 | { 131 | Log::info('Commit transaction'); 132 | $this->targetDbHelper->commitTransaction(); 133 | } 134 | 135 | private function migrateSiteConfig() 136 | { 137 | Log::info('Migrating site config'); 138 | 139 | $this->siteMigrator->migrate( 140 | $this->getBatchProvider( 141 | 'SELECT * FROM ' . $this->sourceDbHelper->prefixTable('site') . ' WHERE idsite = ' . $this->settings->idSite 142 | ) 143 | ); 144 | 145 | $this->siteGoalMigrator->migrate( 146 | $this->getBatchProvider( 147 | 'SELECT * FROM ' . $this->sourceDbHelper->prefixTable('goal') . ' WHERE idsite = ' . $this->settings->idSite 148 | ) 149 | ); 150 | 151 | $this->siteUrlMigrator->migrate( 152 | $this->getBatchProvider( 153 | 'SELECT * FROM ' . $this->sourceDbHelper->prefixTable('site_url') . ' WHERE idsite = ' . $this->settings->idSite 154 | ) 155 | ); 156 | } 157 | 158 | private function loadActions() 159 | { 160 | Log::info('Loading existing actions'); 161 | 162 | $this->actionMigrator->loadExistingActions(); 163 | } 164 | 165 | private function migrateLogVisits() 166 | { 167 | Log::info('Migrating log data - visits'); 168 | 169 | $query = 'SELECT * FROM ' . $this->sourceDbHelper->prefixTable('log_visit') . ' WHERE idsite = ' . $this->settings->idSite; 170 | 171 | if ($this->settings->dateFrom) { 172 | $query .= ' AND `visit_last_action_time` >= \'' . $this->settings->dateFrom->format('Y-m-d') . '\''; 173 | } 174 | 175 | if ($this->settings->dateTo) { 176 | $query .= ' AND `visit_last_action_time` < \'' . $this->settings->dateTo->format('Y-m-d') . '\''; 177 | } 178 | 179 | $this->visitMigrator->migrate( 180 | $this->getBatchProvider($query) 181 | ); 182 | } 183 | 184 | private function migrateLogVisitActions() 185 | { 186 | Log::info('Migrating log data - link visit action'); 187 | 188 | $queries = $this->getLogVisitQueriesFor('log_link_visit_action'); 189 | 190 | if (count($queries) > 0) { 191 | $this->visitActionMigrator->migrate($this->getBatchProvider($queries)); 192 | } 193 | } 194 | 195 | private function migrateLogVisitConversions() 196 | { 197 | Log::info('Migrating log data - conversions and conversion items'); 198 | 199 | $queries = $this->getLogVisitQueriesFor('log_conversion'); 200 | $itemQueries = $this->getLogVisitQueriesFor('log_conversion_item'); 201 | 202 | if (count($queries) > 0) { 203 | $this->conversionMigrator->migrate($this->getBatchProvider($queries)); 204 | $this->conversionItemMigrator->migrate($this->getBatchProvider($itemQueries)); 205 | } 206 | } 207 | 208 | private function migrateArchives() 209 | { 210 | Log::info('Migrating archive data'); 211 | 212 | $this->archiveMigrator->migrate($this->settings->idSite, $this->settings->dateFrom, $this->settings->dateTo); 213 | } 214 | 215 | private function getLogVisitQueriesFor($table) 216 | { 217 | $visitIdRanges = $this->visitMigrator->getIdRanges(); 218 | 219 | if (count($visitIdRanges) > 0) { 220 | $baseQuery = "SELECT * FROM " . $this->sourceDbHelper->prefixTable($table) . ' WHERE idvisit IN '; 221 | $queries = array(); 222 | 223 | 224 | foreach ($visitIdRanges as $range) { 225 | $queries[] = $baseQuery . ' (' . implode(', ', $range) . ')'; 226 | } 227 | 228 | return $queries; 229 | } else { 230 | return array(); 231 | } 232 | } 233 | 234 | private function getBatchProvider($query) 235 | { 236 | return new BatchProvider($query, $this->sourceDbHelper, $this->gcHelper, 10000); 237 | } 238 | } 239 | -------------------------------------------------------------------------------- /Commands/MigrateSite.php: -------------------------------------------------------------------------------- 1 | setName('migration:site'); 33 | $this->setDescription('Migrate site between Piwik instances'); 34 | $this->addArgument('idSite', InputArgument::REQUIRED, 'Site id'); 35 | 36 | /** 37 | * Migration options 38 | */ 39 | $this->addOption('skip-archive-data', null, InputOption::VALUE_NONE, 'Skip migration of archived data'); 40 | $this->addOption('skip-log-data', null, InputOption::VALUE_NONE, 'Skip migration of log data'); 41 | 42 | /** 43 | * Database options 44 | */ 45 | $this->addOption('db-host', 'H', InputOption::VALUE_REQUIRED, 'Destination database host'); 46 | $this->addOption('db-username', 'U', InputOption::VALUE_REQUIRED, 'Destination database username'); 47 | $this->addOption('db-password', 'P', InputOption::VALUE_REQUIRED, 'Destination database password'); 48 | $this->addOption('db-name', 'N', InputOption::VALUE_REQUIRED, 'Destination database name'); 49 | $this->addOption('db-prefix', null, InputOption::VALUE_OPTIONAL, 'Destination database table prefix'); 50 | $this->addOption('db-port', null, InputOption::VALUE_REQUIRED, 'Destination database port', '3306'); 51 | 52 | /** 53 | * Visit query options 54 | */ 55 | $this->addOption('date-from', 'F', InputOption::VALUE_REQUIRED, 'Start date from which data should be migrated'); 56 | $this->addOption('date-to', 'T', InputOption::VALUE_REQUIRED, 'Start date from which data should be migrated'); 57 | } 58 | 59 | protected function execute(InputInterface $input, OutputInterface $output) 60 | { 61 | $self = $this; 62 | 63 | // Set memory limit to off 64 | @ini_set('memory_limit', -1); 65 | Piwik::doAsSuperUser(function() use ($input, $output, $self){ 66 | $settings = new MigratorSettings(); 67 | $settings->idSite = $input->getArgument('idSite'); 68 | $settings->site = $self->getSite($settings->idSite); 69 | $settings->dateFrom = $input->getOption('date-from') ? new \DateTime($input->getOption('date-from')) : null; 70 | $settings->dateTo = $input->getOption('date-to') ? new \DateTime($input->getOption('date-to')) : null; 71 | $settings->skipArchiveData = $input->getOption('skip-archive-data'); 72 | $settings->skipLogData = $input->getOption('skip-log-data'); 73 | 74 | $config = Db::getDatabaseConfig(); 75 | $startTime = microtime(true); 76 | 77 | $self->createTargetDatabaseConfig($input, $output, $config); 78 | 79 | $tmpConfig = $config; 80 | $sourceDb = Db::get(); 81 | try { 82 | $targetDb = @Db\Adapter::factory($config['adapter'], $tmpConfig); 83 | } catch (\Exception $e) { 84 | throw new \RuntimeException('Unable to connect to the target database: ' . $e->getMessage(), 0, $e); 85 | } 86 | 87 | $sourceDbHelper = new DBHelper($sourceDb, Db::getDatabaseConfig()); 88 | 89 | $migratorFacade = new Migrator( 90 | $sourceDbHelper, 91 | new DBHelper($targetDb, $config), 92 | GCHelper::getInstance(), 93 | $settings, 94 | new ArchiveLister($sourceDbHelper) 95 | ); 96 | 97 | $migratorFacade->migrate(); 98 | 99 | $endTime = microtime(true); 100 | 101 | Log::debug(sprintf('Time taken: %01.2f sec', $endTime - $startTime)); 102 | Log::debug(sprintf('Peak memory usage: %01.2f MB', memory_get_peak_usage(true) / 1048576)); 103 | }); 104 | } 105 | 106 | 107 | public function getSite($idSite) 108 | { 109 | if (!Site::getSite($idSite)) { 110 | throw new \InvalidArgumentException('idSite is not a valid, no such site found'); 111 | } 112 | 113 | return Db::get()->fetchRow( 114 | "SELECT * FROM " . Common::prefixTable("site") . " WHERE idsite = ?", 115 | $idSite 116 | ); 117 | } 118 | 119 | public function createTargetDatabaseConfig(InputInterface $input, OutputInterface $output, &$config) 120 | { 121 | $notNullValidator = function ($answer) { 122 | if (strlen(trim($answer)) == 0) { 123 | throw new \InvalidArgumentException('This value should not be empty'); 124 | } 125 | 126 | return $answer; 127 | }; 128 | 129 | $dummyValidator = function ($answer) { 130 | return $answer; 131 | }; 132 | 133 | $config['host'] = $this->ensureOptionsIsProvided( 134 | 'db-host', 135 | $input, 136 | $output, 137 | 'Please provide the destination database host', 138 | $notNullValidator, 139 | false, 140 | 'localhost' 141 | ); 142 | 143 | $config['username'] = $this->ensureOptionsIsProvided( 144 | 'db-username', 145 | $input, 146 | $output, 147 | 'Please provide the destination database username', 148 | $notNullValidator 149 | ); 150 | 151 | $config['password'] = $this->ensureOptionsIsProvided( 152 | 'db-password', 153 | $input, 154 | $output, 155 | 'Please provide the destination database password', 156 | $dummyValidator, 157 | false, 158 | null, 159 | true 160 | ); 161 | 162 | $config['dbname'] = $this->ensureOptionsIsProvided( 163 | 'db-name', 164 | $input, 165 | $output, 166 | 'Please provide the destination database name', 167 | $notNullValidator 168 | ); 169 | 170 | $config['port'] = $input->getOption('db-port'); 171 | $config['tables_prefix'] = $input->getOption('db-prefix'); 172 | } 173 | 174 | public function ensureOptionsIsProvided($optionName, InputInterface $input, OutputInterface $output, $question, $validator, $attempts = false, $default = null, $hidden = false) 175 | { 176 | if (!$input->getOption($optionName)) { 177 | return $this->askAndValidate($output, $question, $validator, $attempts, $default, $hidden); 178 | } 179 | 180 | return $input->getOption($optionName); 181 | } 182 | 183 | protected function askAndValidate( 184 | OutputInterface $output, 185 | $question, 186 | $validator, 187 | $attempts = false, 188 | $default = null, 189 | $hidden = false 190 | ) 191 | { 192 | /** 193 | * @var $dialog DialogHelper 194 | */ 195 | $dialog = $this->getHelperSet()->get('dialog'); 196 | $question = '' . $question . (($default) ? " [$default]" : '') . ': '; 197 | 198 | if (!$hidden) { 199 | return $dialog->askAndValidate( 200 | $output, 201 | $question, 202 | $validator, 203 | $attempts, 204 | $default 205 | ); 206 | } else { 207 | return $dialog->askHiddenResponseAndValidate( 208 | $output, 209 | $question, 210 | $validator, 211 | $attempts, 212 | $default 213 | ); 214 | } 215 | } 216 | } 217 | --------------------------------------------------------------------------------