├── .gitignore
├── Func
    ├── Common.php
    └── file
    │   └── area.php
├── README.md
├── composer.json
└── src
    ├── Library
        ├── Db.php
        ├── RabbitMq.php
        └── SwitchData.php
    ├── Models
        ├── Models.php
        ├── Table.php
        └── demo.php
    └── main.php


/.gitignore:
--------------------------------------------------------------------------------
1 | /vendor/
2 | 


--------------------------------------------------------------------------------
/Func/Common.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace dormscript\Data\Func;
 3 | 
 4 | //数据转移过程中常用的函数
 5 | //demo1：地区ID转换
 6 | class Common
 7 | {
 8 |     public static $areaInfo; //地区转换关系存储到数组
 9 |     /**
10 |      * 根据省、市、区ID获取新的ID
11 |      * @param  array $p 省、市、区ID
12 |      * @return [type]    [description]
13 |      */
14 |     public static function getAreaInfo($p)
15 |     {
16 |         $p = array_filter($p);
17 |         if (empty($p)) {
18 |             return array(0, 0, 0);
19 |         }
20 |         if (empty(self::$areaInfo)) {
21 |             include BASEDIR . "/Func/file/area.php";
22 |             foreach ($area as $v) {
23 |                 $id = $v['id'];
24 |                 unset($v['id']);
25 |                 $v                                = array_filter($v);
26 |                 self::$areaInfo[implode("_", $v)] = $id;
27 |             }
28 |             unset($area);
29 |         }
30 |         $key = implode("_", $p);
31 |         if (!isset(self::$areaInfo[$key])) {
32 |             echo "\n 未找到地区对应关系 ：" . implode("-", $p) . "\n";
33 |             array_pop($p);
34 |             return Common::getAreaInfo($p);
35 |         }
36 |         $id = self::$areaInfo[$key];
37 | 
38 |         if (strlen($id) == 6) {
39 |             //将ID转换为省、市、区三个字段
40 |             $provinceId = substr($id, 0, 2) . '0000';
41 |             if (substr($id, 2, 2) == '00') {
42 |                 $cityId = 0;
43 |             } else {
44 |                 $cityId = substr($id, 0, 4) . '00';
45 |             }
46 |             if (substr($id, 4, 2) == '00') {
47 |                 $areaId = 0;
48 |             } else {
49 |                 $areaId = $id;
50 |             }
51 |         } else {
52 |             $provinceId = substr($id, 0, 2) . '0000';
53 |             if (substr($id, 2, 4) == '0000') {
54 |                 $cityId = 0;
55 |             } else {
56 |                 $cityId = substr($id, 0, 6);
57 |             }
58 |             $areaId = $id;
59 |         }
60 |         return array($provinceId, $cityId, $areaId);
61 |     }
62 | }
63 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | ## 应用场景
 2 | 网站升级改版时，如果数据结构发生较大修改，就需要将数据从源数据库迁移到新的数据库，同时数据表名、字段名都需要较大修改。涉及到的修改包括：表名修改、字段名修改，字段内的数据格式转换，数据记录过滤（无效数据不再需要转换到新的数据表），多张表合并到一张表，一张表拆分为多张表。
 3 | 
 4 | ## 解决的问题：
 5 | * 封装数据迁移的公共部分
 6 | * 数据迁移时的表、字段对应关系
 7 | * 提供并行处理
 8 | * 同步数据迁移过程中的增量数据变更（需要借助第三方工具）
 9 | * 提供数据迁移后的后续操作，例如更新memcache缓存，通知第三方全文检索等
10 | * 提供简易的数据迁移进度查看界面，动态调整处理进程数
11 | 
12 | ## 用到的技术：
13 | * PHP
14 | * Swoole
15 | 
16 | ## 适用场景：
17 | * 将源数据库中的表迁移到新的数据库
18 | * 将源数据库中的表进行合并迁移到新数据库
19 | * 将源数据库中的数据按照一定规则处理后写到新的数据库
20 | * 将源数据库中的数据按照一定规则过滤，将过滤后的结果写到新的数据库
21 | 
22 | ## 场景1用法介绍：
23 | 将源数据库的user表迁移到新数据库的newUser.user表
24 | 
25 | 	1：vim config.php，配置源、目标数据库的ip、用户名、密码
26 | 	2：进入Models目录，创建子目录，目录名为源数据库的库名
27 | 	3：进入新创建的目录，创建文件，格式为：{$tableName}.php
28 | 	4：内容见Models\usercenter\user.php
29 | 	5：设置需要多少个进程来处理数据迁移。vim config.php，配置setting
30 | 	5：进入根目录执行：  php main.php
31 | 
32 | ## 场景2用法：
33 | 将源数据库的user表、user_contact合并，迁移到新数据库的newUser.user表
34 | 
35 | 	代码见user1.php
36 | ## 场景3、4用法：
37 | 将源数据库的user表迁移到新数据库的newUser.user表，同时将sex字段的0改为男，1改为女。如果sex为其它值，不迁移此条记录
38 | 
39 | 	代码见user2.php
40 | ## callback的使用：
41 | 	代码见user2.php中的callback函数
42 | **需要注意：callback函数接收到的参数默认是源数据库中的记录。如果需要使用转换后的记录，需要一行设置：**
43 | 	public $callbackRow  = 'desc'; 
44 | ## 附：fieldmap支持的格式：
45 | * 一对一
46 | 
47 | 
48 | 		'username' => 'username'
49 | 		说明：将源表的username字段原样转到新表的username字段
50 | 
51 | * 一对一，格式转换
52 | 		
53 | 		'pid' => 'pid/getNewpid'
54 | 		说明：使用自定义函数将源表的pid字段转换后存入新表的pid字段
55 | 		例 ： 
56 | 		public function getNewPid($p)
57 | 	    {
58 | 	        return ($p + 1000000001);
59 | 	    }
60 | 
61 | * 一对一，格式转换、数据过滤
62 | 
63 | 		'state'  => 'status/getStatus',
64 | 		说明：使用自定义函数getstatus处理status字段。
65 | 		如果getstatus函数返回值为false，当前记录会被忽略。返回其它值，将返回值存入新表
66 | 		public function getstatus () {
67 | 			if(in_array($status, 1,2,3)) {
68 | 				//只迁移状态为1、2、3数据
69 | 				return $status;
70 | 			} else {
71 | 				//其它状态的数据不再迁移
72 | 				return false;
73 | 			}
74 | 		}
75 | * 多对一
76 | 
77 | 		'prokey' => 'prokey,prokeyword/getkey',
78 | 		说明：将两个字段经过getkey函数处理后（返回值是字符串），存入新表的prokey字段
79 | * 多对多
80 | 		
81 | 		'cate1-cate2-cate3' => 'cate1,cate2,cate3/getCate',
82 | 		说明：getcate函数接收1个参数（数组），返回值为一个数组。
83 | * 将新表的字段设默认值
84 | 	
85 | 		'newfieldname' => '4/returnme',
86 | 	
87 | 		
88 | ## 其它用途：
89 | 	可以将此框架做为简单的并行读数据框架，callback中写自己需要做的事情。
90 | 	例：user3.php
91 | 
92 | 
93 | 


--------------------------------------------------------------------------------
/composer.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "dormscript/data-transfer",
 3 |     "description": "数据迁移框架",
 4 |     "type": "library",
 5 |     "require": {
 6 |         "ext-swoole": "^4.1@alpha"
 7 |     },
 8 |     "license": "MIT",
 9 |     "authors": [
10 |         {
11 |             "name": "shuai wen",
12 |             "email": "wenshuaiying@gongchang.com"
13 |         }
14 |     ],
15 |     "autoload": {
16 |         "psr-4": {
17 |             "dormscript\\Data\\": "src/"
18 |         }
19 |     }
20 | }
21 | 


--------------------------------------------------------------------------------
/src/Library/Db.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace dormscript\Data\Library;
 3 | 
 4 | //多进程来调用这个Db。如果A进程创建的mysql连接，B进程是不能使用的，所以需要标记连接是哪个进程创建的
 5 | //其实这种方法是有问题的，但在当前情景下可正常运行
 6 | //
 7 | //
 8 | //以前使用的mysql底层驱动是mysql,后来改成mysqli，部分函数未升级
 9 | //使用静态二维数组dbPool完全没用，一维就可以，稍后修改代码
10 | class Db
11 | {
12 |     public static $dbPool;
13 |     public static function exeSql($dbname, $sql, $taskid = 0)
14 |     {
15 |         global $db;
16 |         $ret = array();
17 | 
18 |         if (!isset(self::$dbPool[getmypid()][$dbname]) || empty(self::$dbPool[getmypid()][$dbname])) {
19 |             $dbConnection = mysqli_connect($db[$dbname]['host'], $db[$dbname]['user'], $db[$dbname]['pswd']) or die("连接 '" . $dbname . "'库失败");
20 |             mysqli_query($dbConnection, "SET NAMES 'UTF8'");
21 |             self::$dbPool[getmypid()][$dbname] = $dbConnection;
22 |         } else {
23 |             $dbConnection = self::$dbPool[getmypid()][$dbname];
24 |         }
25 | 
26 |         if (!$query = mysqli_query($dbConnection, $sql)) {
27 |             throw new \Exception("出錯的SQL：" . $sql . "\t" . mysqli_errno($dbConnection) . ": " . mysqli_error($dbConnection));
28 |         }
29 |         if ($query instanceof \mysqli_result) {
30 |             while ($row = mysqli_fetch_array($query, MYSQLI_ASSOC)) {
31 |                 $ret[] = $row;
32 |             }
33 |             return $ret;
34 |         } else {
35 |             return mysqli_insert_id($dbConnection);
36 |         }
37 |     }
38 |     public static function delDbPool()
39 |     {
40 |         if (empty(self::$dbPool[getmypid()])) {
41 |             return true;
42 |         }
43 |         foreach (self::$dbPool[getmypid()] as $value) {
44 |             mysqli_close($value);
45 |         }
46 |         self::$dbPool[getmypid()] = [];
47 |     }
48 | }
49 | 


--------------------------------------------------------------------------------
/src/Library/RabbitMq.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace dormscript\Data\Library;
 3 | 
 4 | class RabbitMq
 5 | {
 6 |     public $conn;
 7 |     public $channel;
 8 | 
 9 |     public function writeQueue($e_name, $str, $routerKey = '', $isBroad = 0)
10 |     {
11 |         //创建交换机对象
12 |         $ex = new \AMQPExchange($this->channel);
13 |         $ex->setName($e_name);
14 |         //$ex->setType(AMQP_EX_TYPE_FANOUT); //direct类型
15 |         $ex->setType(AMQP_EX_TYPE_DIRECT); //direct类型
16 |         $ex->setFlags(AMQP_DURABLE); //持久化
17 |         //echo "Exchange Status:" . $ex->declare() . "\n";
18 |         //发送消息
19 |         if ($isBroad) {
20 |             $ex->publish($str, $routerKey, AMQP_NOPARAM, array('delivery_mode' => '2'));
21 |         } else {
22 |             $ex->publish($str, $routerKey);
23 |         }
24 |     }
25 |     public function readQueue($q_name)
26 |     {
27 |         $q = new \AMQPQueue($this->channel);
28 |         $q->setName($q_name);
29 |         //$q->setFlags(AMQP_DURABLE);
30 |         //$q->declare();
31 |         //$q->bind('exchange', $bindingkey);
32 |         //消息获取
33 |         $messages = $q->get(AMQP_AUTOACK);
34 |         if ($messages) {
35 |             return $messages->getBody();
36 |         } else {
37 |             return false;
38 |         }
39 |     }
40 | 
41 |     public function reconnect($conn_args)
42 |     {
43 |         //创建连接和channel
44 |         $this->conn = new \AMQPConnection($conn_args);
45 |         try {
46 |             if (!$this->conn->connect()) {
47 |                 die("Cannot connect to the broker!\n");
48 |             }
49 |         } catch (EXCEPTION $e) {
50 |             sleep(300);
51 |             if (!$this->conn->connect()) {
52 |                 die("Cannot connect to the broker!\n");
53 |             }
54 |         }
55 |         $this->channel = new \AMQPChannel($this->conn);
56 |     }
57 |     /**
58 |      * 入RabbitMQ队列
59 |      * @param [type] $exName    [交换机名]
60 |      * @param [type] $routingKey [路由名]
61 |      * @param [type] $value     [队列的值]
62 |      * @param [type] $dbType     [数据库类型,默认为mysql]
63 |      * 按照此规则生成的默认队列名称为 exName_routeKey_dbType;值为value
64 |      */
65 |     public function set($exName, $routingKey, $value, $dbType = 'mysql')
66 |     {
67 |         //创建交换机,设置交换机名
68 |         $ex = new \AMQPExchange($this->channel);
69 |         $ex->setName($exName);
70 |         $ex->setType(AMQP_EX_TYPE_DIRECT); //广播模式
71 |         $ex->setFlags(AMQP_DURABLE); //交换器进行持久化，即 RabbitMQ 重启后会自动重建
72 |         // $ex->declareExchange();
73 |         //设置队列名
74 |         $queue = new \AMQPQueue($this->channel);
75 |         $queue->setName($exName . '_' . $routingKey . '_' . $dbType);
76 |         $queue->setFlags(AMQP_DURABLE); //队列进行持久化，即 RabbitMQ 重启后会自动重建
77 |         $queue->declareQueue();
78 |         //交换机和路由绑定到队列
79 |         $queue->bind($exName, $routingKey);
80 |         //入队列
81 |         if (is_array($value)) {
82 |             $value = json_encode($value);
83 |         }
84 |         $ex->publish($value, $routingKey, AMQP_NOPARAM, array('delivery_mode' => '2'));
85 |     }
86 | }
87 | 


--------------------------------------------------------------------------------
/src/Library/SwitchData.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace dormscript\Data\Library;
  3 | 
  4 | /**
  5 |  * 根据每个表的配置信息将数据进行格式转移、写入目标数据库
  6 |  */
  7 | class SwitchData
  8 | {
  9 |     public $tablename;
 10 |     public $startid;
 11 |     public $endid;
 12 |     public $modelObj; //当前正在操作的表结构
 13 |     public $taskid; //处理当前任务的task进程ID
 14 | 
 15 |     public function run($tablename, $startid, $endid, $taskid, $ser = '')
 16 |     {
 17 |         $this->tablename = $tablename;
 18 |         $this->startid   = $startid;
 19 |         $this->endid     = $endid;
 20 |         $this->taskid    = $taskid;
 21 |         $this->modelObj  = \dormscript\Data\Models\Models::getObj($tablename); //获取Models对象
 22 | 
 23 |         if (!empty($ser)) {
 24 |             $this->modelObj->setSwooleSer($ser);
 25 |         }
 26 |         try {
 27 |             $srcData = $this->readRs(); //从源表中读取数据
 28 |             if (empty($srcData)) {
 29 |                 return true;
 30 |             }
 31 |             $newData = array();
 32 |             foreach ($srcData as $key => $row) {
 33 |                 $newRs = $this->analysisRs($row); //做数据格式转换
 34 |                 if ($newRs !== false) {
 35 |                     $newData[$key] = $newRs;
 36 |                 }
 37 |             }
 38 |             $this->writeRs($newData); //将数据写入目标数据表
 39 |             if ($this->modelObj->callbackRow == 'src') {
 40 |                 $this->modelObj->callback($srcData);
 41 |             } elseif ($this->modelObj->callbackRow == 'desc') {
 42 |                 $this->modelObj->callback($newData);
 43 |             } elseif ($this->modelObj->callbackRow == 'merge') {
 44 |                 $mergeData = array();
 45 |                 foreach ($srcData as $key => $value) {
 46 |                     if (isset($newData[$key])) {
 47 |                         $mergeData[$key] = array_merge($value, $newData[$key]);
 48 |                     }
 49 |                 }
 50 |                 $this->modelObj->callback($mergeData);
 51 |             }
 52 |         } catch (\Exception $e) {
 53 |             error_log("\nerror:" . $e->getMessage(), 3, "error.log");
 54 |             return false;
 55 |         }
 56 |         return true;
 57 |     }
 58 | 
 59 |     /**
 60 |      * 根据配置，获取转移对应关系
 61 |      * @return [type] [description]
 62 |      */
 63 |     public function readRs()
 64 |     {
 65 |         $sql = '';
 66 |         if (!$this->modelObj->srcSql) {
 67 |             $sql = "select * from {$this->modelObj->getTablename()} ";
 68 |         } else {
 69 |             $sql = $this->modelObj->srcSql;
 70 |         }
 71 |         $sql .= " where {$this->modelObj->primaryKey} >= {$this->startid} and {$this->modelObj->primaryKey} < {$this->endid} ";
 72 |         $sql .= 'order by ' . $this->modelObj->primaryKey . ' ASC';
 73 |         $dbType = !empty($this->modelObj->readDbName) ? $this->modelObj->readDbName : 'read';
 74 |         $row    = \dormscript\Data\Library\Db::exeSql($dbType, $sql, $this->taskid);
 75 |         return $row;
 76 |     }
 77 | 
 78 |     /**
 79 |      * 根据字段对应关系将源数据$row转换(处理一行数据)
 80 |      * @param  [type] $row [description]
 81 |      * @return [type]      [description]
 82 |      */
 83 |     public function analysisRs($row)
 84 |     {
 85 |         $Arr = array();
 86 |         foreach ($this->modelObj->fieldMap as $des => $src) {
 87 |             if (strpos($src, '/') !== false) {
 88 |                 $a = explode('/', $src);
 89 |                 if (strpos($a[0], ',')) {
 90 |                     //$par   = '';
 91 |                     $param = explode(',', $a[0]);
 92 |                     foreach ($param as $p) {
 93 |                         if (array_key_exists($p, $row)) {
 94 |                             $par[] = $row[$p];
 95 |                         } else {
 96 |                             $par[] = $p;
 97 |                         }
 98 |                     }
 99 |                     $Arr[$des] = $row[$des] = call_user_func(array($this->modelObj, $a[1]), $par);
100 |                     if ($Arr[$des] === false) {
101 |                         return false;
102 |                     }
103 |                     //自定义函数，返回false，表示跳过当前记录
104 |                 } else {
105 |                     if ($a['0'] == '*') {
106 |                         $Arr[$des] = $row[$des] = call_user_func(array($this->modelObj, $a[1]), $row);
107 |                     } elseif (array_key_exists($a['0'], $row)) {
108 |                         $Arr[$des] = $row[$des] = call_user_func(array($this->modelObj, $a[1]), $row[$a[0]]);
109 |                     } else {
110 |                         $Arr[$des] = $row[$des] = call_user_func(array($this->modelObj, $a[1]), $a[0]);
111 |                     }
112 |                 }
113 |                 if ($Arr[$des] === false) {
114 |                     return false;
115 |                 }
116 |             } else {
117 |                 $Arr[$des] = $row[$des] = $row[$src];
118 |             }
119 |             if (stripos($des, '-')) {
120 |                 $temp = array_combine(explode("-", $des), $Arr[$des]);
121 |                 unset($Arr[$des]);
122 |                 foreach ($temp as $key => $value) {
123 |                     $Arr[$key] = $row[$key] = $value;
124 |                 }
125 |             }
126 |         }
127 |         return $Arr;
128 |     }
129 | 
130 |     /**
131 |      * 将数据写到目标数据库
132 |      * @param  [type] $newData [description]
133 |      * @return [type]          [description]
134 |      */
135 |     public function writeRs($newData)
136 |     {
137 |         if (empty($newData)) {
138 |             return '';
139 |         }
140 |         $fieldarray = array_keys(current($newData));
141 |         $fields     = implode(",", $fieldarray);
142 | 
143 |         $sql = "replace into {$this->modelObj->descTable} " . "(" . $fields . ") VALUES";
144 |         foreach ($newData as $line) {
145 |             $sql .= "\n(";
146 |             foreach ($line as $field) {
147 |                 $sql .= "'" . addslashes($field) . "',";
148 |             }
149 |             $sql = substr($sql, 0, -1) . "),";
150 |         }
151 |         $sql = substr($sql, 0, -1);
152 |         $rs  = \dormscript\Data\Library\Db::exeSql($this->modelObj->writeDbName, $sql, $this->taskid);
153 |     }
154 | }
155 | 


--------------------------------------------------------------------------------
/src/Models/Models.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace dormscript\Data\Models;
 3 | 
 4 | /**
 5 |  * 读取表的对象
 6 |  */
 7 | class Models
 8 | {
 9 |     public static $ModelArr;
10 |     public static function getObj($tablename)
11 |     {
12 |         if (!isset(self::$ModelArr[md5($tablename)])) {
13 |             self::$ModelArr[md5($tablename)] = self::readFile($tablename);
14 |         }
15 |         return self::$ModelArr[md5($tablename)];
16 |     }
17 | 
18 |     //释放资源
19 |     public static function delObj($tablename)
20 |     {
21 |         if (isset(self::$ModelArr[md5($tablename)])) {
22 |             unset(self::$ModelArr[md5($tablename)]);
23 |         }
24 |     }
25 | 
26 |     /**
27 |      * 根据tablename获取Models对象
28 |      * @param  [type] $tablename [description]
29 |      * @return [type]            [description]
30 |      */
31 |     public static function readFile($tablename)
32 |     {
33 |         if (class_exists($tablename)) {
34 |             return new $tablename;
35 |         }
36 |         die("\n表不存在:" . $tablename);
37 |     }
38 | }
39 | 


--------------------------------------------------------------------------------
/src/Models/Table.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace dormscript\Data\Models;
 3 | 
 4 | class Table
 5 | {
 6 |     public $srcTable  = ""; //从哪个表读数据
 7 |     public $descTable = ""; //写入到哪个库的哪个表，中间用.分隔
 8 | 
 9 |     public $primaryKey   = ""; //源表的主键名称
10 |     public $srcSql       = ""; // 从源数据表读数据的SQL,可选项
11 |     public $readDbName   = "read"; //源表所在Mysql服务器名称
12 |     public $writeDbName  = "write"; //目标所有mysql服务器名称
13 |     public $callbackRow  = 'src'; //callback调取源数据 desc(调取新数据)
14 |     protected $swooleSer = ''; //在task中获取server信息
15 | 
16 |     //字段对应关系
17 |     public $fieldMap = array();
18 | 
19 |     //获取数据库表的名称
20 |     public function getTablename()
21 |     {
22 |         if (!empty($this->srcTable)) {
23 |             return $this->srcTable;
24 |         }
25 |         die("\n" . get_class($this) . " 未配置srcTable");
26 |     }
27 |     public function returnme($p)
28 |     {
29 |         return $p;
30 |     }
31 |     public function setSwooleSer($ser)
32 |     {
33 |         $this->swooleSer = $ser;
34 |     }
35 |     public function getSwooleSer()
36 |     {
37 |         return $this->swooleSer;
38 |     }
39 |     public function callback($row)
40 |     {
41 |         return true;
42 |     }
43 | 
44 |     public function int10($p)
45 |     {
46 |         $p = intval($p);
47 |         if ($p > 4294967296) {
48 |             $p = 0;
49 |         }
50 |         return $p;
51 |     }
52 |     public function __call($funname, $arguments)
53 |     {
54 |         return call_user_func_array($funname, $arguments);
55 |     }
56 | }
57 | 


--------------------------------------------------------------------------------
/src/Models/demo.php:
--------------------------------------------------------------------------------
 1 | <?php
 2 | namespace dormscript\Data\Models\{$schemaName};
 3 | 
 4 | class {$modelName} extends \Data\Models\Table
 5 | {
 6 | 
 7 |     public $primaryKey = "{$primaryKey}";
 8 |     public $descTable  = "{$descSchema}.{$descTableName}";
 9 | 
10 |     //字段对应关系
11 |     public $fieldMap = array(
12 |     	{$fieldMap}
13 |     );
14 | }
15 | 


--------------------------------------------------------------------------------
/src/main.php:
--------------------------------------------------------------------------------
  1 | <?php
  2 | namespace dormscript\Data;
  3 | 
  4 | class Main
  5 | {
  6 |     private $serv;
  7 |     private $setting; //设置每个表应该由几个进程来处理
  8 |     private $curId; //每个表当前处理到的位置
  9 |     private $maxId; //每个表的最大ID
 10 |     private $minId; //记录表的最小ID
 11 |     private $taskinfo; //所有task需要处理的表
 12 |     private $step = 100; //每次处理的数据数量
 13 |     private $startTime;
 14 |     public function __construct()
 15 |     {
 16 |         global $setting;
 17 |         $this->startTime                 = time();
 18 |         $this->setting                   = $setting;
 19 |         $this->max_task_num              = 200;
 20 |         list($this->curId, $this->maxId) = $this->getId();
 21 |         $this->minId                     = $this->curId;
 22 | 
 23 |         $this->serv = new \swoole_server("0.0.0.0", 9501);
 24 |         $this->serv->set(array(
 25 |             'worker_num'      => 1,
 26 |             'daemonize'       => false,
 27 |             'max_request'     => 10000,
 28 |             'dispatch_mode'   => 3,
 29 |             'debug_mode'      => 1,
 30 |             'task_worker_num' => $this->max_task_num,
 31 |         ));
 32 |         $this->serv->on('WorkerStart', array($this, 'onWorkerStart'));
 33 |         $this->serv->on('Close', array($this, 'onClose'));
 34 |         $this->serv->on('Receive', array($this, 'onReceive'));
 35 |         $this->serv->on('Task', array($this, 'onTask'));
 36 |         $this->serv->on('Finish', array($this, 'onFinish'));
 37 |         $this->serv->start();
 38 |     }
 39 |     /**
 40 |      * 获取每个表的最大ID，最小ID
 41 |      * @return [type] [description]
 42 |      */
 43 |     public function getId()
 44 |     {
 45 |         $min = $max = array();
 46 |         foreach ($this->setting as $tablename => $taskNum) {
 47 |             $modelObj = Models\Models::getObj($tablename);
 48 |             $dbType   = !empty($modelObj->readDbName) ? $modelObj->readDbName : 'read';
 49 | 
 50 |             //获取表主键名字
 51 |             $realTableName = $modelObj->getTablename();
 52 |             $descSql       = "desc $realTableName";
 53 |             $row           = Library\Db::exeSql($dbType, $descSql, 1);
 54 |             $primiaryKey   = current($row['0']);
 55 | 
 56 |             //查出最小ID
 57 |             $sql             = "select * from $realTableName order by $primiaryKey ASC limit 0,1";
 58 |             $row             = Library\Db::exeSql($dbType, $sql, 1);
 59 |             $min[$tablename] = empty($row['0']) ? 0 : intval(current($row['0']));
 60 | 
 61 |             //查出最大ID
 62 |             $sql             = "select * from $realTableName order by $primiaryKey DESC limit 0,1";
 63 |             $row             = Library\Db::exeSql($dbType, $sql, 1);
 64 |             $max[$tablename] = empty($row['0']) ? 0 : intval(current($row['0']));
 65 | 
 66 |             if ($modelObj->descTable) {
 67 |                 //从目标库中读出相关信息
 68 |                 $descSql     = "desc " . $modelObj->descTable;
 69 |                 $row         = Library\Db::exeSql('write', $descSql, 1);
 70 |                 $primiaryKey = current($row['0']);
 71 | 
 72 |                 $sql = "select * from " . $modelObj->descTable . " order by $primiaryKey DESC limit 0,1";
 73 |                 $row = Library\Db::exeSql('write', $sql, 1);
 74 |                 if (empty($row)) {
 75 |                     $descMaxId = 0;
 76 |                 } else {
 77 |                     $descMaxId = intval(current($row['0']));
 78 |                 }
 79 |                 if ($descMaxId > $min[$tablename]) {
 80 |                     $min[$tablename] = $descMaxId;
 81 |                 }
 82 |             }
 83 |         }
 84 |         return array($min, $max);
 85 |     }
 86 | 
 87 |     public function onWorkerStart($serv, $worker_id)
 88 |     {
 89 |         if ($worker_id != 0) {
 90 |             return '';
 91 |         }
 92 |         sleep(1); //等待task进程启动
 93 |         $taskid = 1;
 94 |         //根据配置的进程数量，启动所有表的数据迁移
 95 |         foreach ($this->setting as $tablename => $maxnum) {
 96 |             //启用多个task进程来处理表
 97 |             for ($i = 0; $i < $maxnum; $i++) {
 98 |                 $this->addTask($tablename, $taskid++);
 99 |             }
100 |         }
101 |     }
102 | 
103 |     public function onClose($serv, $fd, $from_id)
104 |     {
105 |         echo "Client {$fd} close connection\n";
106 |     }
107 | 
108 |     public function onReceive($serv, $fd, $from_id, $str)
109 |     {
110 |         $params = explode("-", $str);
111 |         echo "\n get data:" . $params['0'] . "\t";
112 |         switch ($params['0']) {
113 |             case 'reassign':
114 |                 $ret = $this->reassign();
115 |                 break;
116 |             case 'setCurID':
117 |                 $ret = $this->setCurID($params['1'], $params['2']);
118 |                 break;
119 |             case 'getinfo':
120 |                 $ret = $this->getInfo();
121 |                 break;
122 |             case 'singleData':
123 |                 $ret = $this->singleData($params['1'], $params['2']);
124 |                 break;
125 |         }
126 |         $serv->send($fd, $ret);
127 |     }
128 | 
129 |     public function onTask($serv, $task_id, $from_id, $param)
130 |     {
131 |         //处理数据表tablename中ID>=$startid && $ID<$startid+100的记录
132 |         list($tablename, $startid, $endid, $taskid) = $param;
133 |         echo "\n OnTask: $taskid \t $tablename \t $startid - $endid ";
134 |         $obj = new Library\SwitchData();
135 |         $rs  = $obj->run($tablename, $startid, $endid, $taskid, $serv);
136 |         while (!$rs) {
137 |             //执行出错时，一直重试。关闭数据库连接，sleep 5秒
138 |             Library\Db::delDbPool();
139 |             Models\Models::delObj($tablename);
140 |             echo "\n 处理( $tablename, $startid, $endid )出错，释放所有资源并重新连接mysql \n";
141 |             sleep(5);
142 |             $rs = $obj->run($tablename, $startid, $endid, $taskid);
143 |         }
144 |         if ($startid + 1 == $endid) {
145 |             //增量更新，处理完成后不再重新处理
146 |             return array($taskid);
147 |         }
148 |         return $taskid;
149 |     }
150 | 
151 |     public function onFinish($serv, $task_id, $param)
152 |     {
153 |         if (is_array($param)) {
154 |             //增量
155 |             $taskid                  = current($param);
156 |             $this->taskinfo[$taskid] = '';
157 |         } elseif ($param) {
158 |             echo "\n onFinish : taskid: { $param } ";
159 |             if ($this->taskinfo[$param]) {
160 |                 $this->addTask($this->taskinfo[$param], $param);
161 |             }
162 |         } else {
163 |             echo "\n close mysqli \n";
164 |         }
165 |     }
166 | 
167 |     /**
168 |      * 根据tablename获取数据表迁移到的最大ID
169 |      * @param  [type] $tablename [description]
170 |      * @return [type]            [description]
171 |      */
172 |     private function getcurId($tablename)
173 |     {
174 |         $startid = $endid = 0;
175 |         $tmp     = str_replace(".", "_", $tablename); //对表名转义
176 | 
177 |         if (!isset($this->maxId[$tmp]) || !isset($this->curId[$tmp])) {
178 |             list($this->curId, $this->maxId) = $this->getId();
179 |             $this->minId                     = $this->curId;
180 |         }
181 |         if (!isset($this->curId[$tmp])) {
182 |             $this->curId[$tmp] = 1;
183 |         }
184 |         $startid = $this->curId[$tmp];
185 | 
186 |         if ($startid > $this->maxId[$tmp]) {
187 |             //所有数据已经处理结束
188 |             return false;
189 |         } elseif ($startid + $this->step > $this->maxId[$tmp]) {
190 |             //最后一次取数据不足100条
191 |             $endid             = $this->maxId[$tmp] + 1;
192 |             $this->curId[$tmp] = $endid;
193 |         } else {
194 |             $endid             = $startid + $this->step;
195 |             $this->curId[$tmp] = $endid;
196 |         }
197 |         return array($startid, $endid);
198 |     }
199 | 
200 |     private function addTask($tablename, $taskid)
201 |     {
202 |         $Ids = $this->getcurId($tablename); //获取开始ID
203 |         if ($Ids === false) {
204 |             $this->taskinfo[$taskid] = '';
205 |             return false;
206 |         }
207 |         list($startid, $endid) = $Ids;
208 | 
209 |         echo "\n addTask: { $taskid }";
210 |         $this->taskinfo[$taskid] = $tablename; //taskid这个进程用来处理tablename。在进行进程数量调整之前，taskid不会去处理其它表数据
211 |         $this->serv->task(array($tablename, $startid, $endid, $taskid), $taskid); //投递task任务
212 |     }
213 | 
214 |     /**
215 |      * 处理单条数据
216 |      * @param  string $tablename 库名+表名（.分隔)
217 |      * @param  int $id        主键ID
218 |      * @return [type]            [description]
219 |      */
220 |     private function singleData($tablename, $id)
221 |     {
222 |         //找到一个空闲进程来处理这个请求（只有task400 - task500处理增量）
223 |         for ($i = $this->max_task_num - 100; $i < $this->max_task_num; $i++) {
224 |             if (empty($this->taskinfo[$i])) {
225 |                 $taskid = $i;
226 |                 break;
227 |             }
228 |         }
229 |         //如果未找到空闲进程
230 |         if ($i == $this->max_task_num) {
231 |             return "fail";
232 |         }
233 |         $this->taskinfo[$taskid] = $tablename; //taskid这个进程用来处理tablename。在进行进程数量调整之前，taskid不会去处理其它表数据
234 |         $this->serv->task(array($tablename, $id, $id + 1, $taskid), $taskid); //投递task任务
235 |         return 'OK';
236 |     }
237 |     /**
238 |      * 根据配置文件重新分配task
239 |      * @return [type] [description]
240 |      */
241 |     public function reassign()
242 |     {
243 |         $task_work_id = 0;
244 |         global $setting;
245 |         $this->setting = $setting;
246 |         foreach ($this->setting as $tablename => $maxnum) {
247 |             for ($i = 0; $i < $maxnum; $i++) {
248 |                 $task_work_id += 1;
249 |                 //如果进程正处于休息状态，启动进程
250 |                 if (empty($this->taskinfo[$task_work_id])) {
251 |                     $this->addTask($tablename, $task_work_id);
252 |                 } else {
253 |                     $this->taskinfo[$task_work_id] = $tablename;
254 |                 }
255 |             }
256 |         }
257 |         for ($i = $task_work_id + 1; $i < $this->max_task_num; $i++) {
258 |             $this->taskinfo[$i] = '';
259 |         }
260 |         return json_encode(array_count_values($this->taskinfo));
261 |     }
262 |     public function setCurID($tablename, $Id)
263 |     {
264 |         $tmp               = str_replace(".", "_", $tablename); //对表名转义
265 |         $this->curId[$tmp] = $Id;
266 |         return "setCurId success !";
267 |     }
268 |     /**
269 |      * 获取当前处理的状态
270 |      * @return [type] [description]
271 |      */
272 |     public function getInfo()
273 |     {
274 |         $ret     = '';
275 |         $taskNum = array_count_values($this->taskinfo);
276 |         $ret .= json_encode($taskNum);
277 |         //状态信息：所有表的进度:   tablename    现在ID   最大ID   几个进程在处理
278 |         foreach ($this->setting as $tablename => $v) {
279 |             $status[] = array(
280 |                 'tablename' => $tablename,
281 |                 'curId'     => $this->curId[str_replace('.', '_', $tablename)],
282 |                 'maxId'     => $this->maxId[str_replace('.', '_', $tablename)],
283 |                 'minId'     => $this->minId[str_replace('.', '_', $tablename)],
284 |             );
285 |         }
286 |         $ret .= "\n" . json_encode($status);
287 |         $ret .= "\n" . json_encode(array_filter($this->taskinfo));
288 |         $ret .= "\n" . $this->startTime;
289 |         return $ret;
290 |     }
291 | }
292 | 


--------------------------------------------------------------------------------