├── .idea ├── goTableCheckSum.iml ├── modules.xml ├── vcs.xml └── workspace.xml ├── Incremental └── binlogCheck.go ├── PublicFunc └── ConnectionDataType.go ├── README.md ├── aa ├── aa.go └── main.go ├── checksum ├── DML.go └── chunkSum.go ├── dispose └── tableInit.go ├── flag └── flagHelp.go ├── go.mod ├── go.sum ├── logs ├── log.go └── tableCheckSum.log ├── main ├── goTableCheckSum └── main.go └── mgorm └── ExecQuerySQL └── QueryTableInfo.go /.idea/goTableCheckSum.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 26 | 27 | 32 | 33 | 34 | 35 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 48 | 49 | 50 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 147 | 148 | 157 | 159 | 160 | true 161 | 162 | 163 | 164 | 165 | 166 | file://$PROJECT_DIR$/mgorm/MySQL/MqueryGetChunkVal.go 167 | 2 168 | 170 | 171 | file://C:/Program Files/Go/src/io/io.go 172 | 14 173 | 175 | 176 | file://$PROJECT_DIR$/checksum/chunkSum.go 177 | 89 178 | 180 | 181 | 182 | 183 | -------------------------------------------------------------------------------- /Incremental/binlogCheck.go: -------------------------------------------------------------------------------- 1 | package Incremental 2 | 3 | import ( 4 | "context" 5 | "github.com/siddontang/go-mysql/mysql" 6 | "github.com/siddontang/go-mysql/replication" 7 | "os" 8 | ) 9 | 10 | // Create a binlog syncer with a unique server id, the server id must be different from other MySQL's. 11 | // flavor is mysql or mariadb 12 | func QueryBinlogEvent() { 13 | 14 | cfg := replication.BinlogSyncerConfig{ 15 | ServerID: 1613306, 16 | Flavor: "mysql", 17 | Host: "172.16.50.161", 18 | Port: 3306, 19 | User: "pcms", 20 | Password: "pcms@123", 21 | } 22 | syncer := replication.NewBinlogSyncer(cfg) 23 | // Start sync with specified binlog file and position 24 | streamer, _ := syncer.StartSync(mysql.Position{"mysql-bin.000002", 234}) 25 | //gtidSet := "5a701ab0-61fa-11eb-bf11-080027193a00:1-2,\n6e54398c-5a10-11eb-b356-080027193a00:1-45" 26 | // or you can start a gtid replication like 27 | //streamer, _ := syncer.StartSyncGTID(gtidSet) 28 | // the mysql GTID set likes this "de278ad0-2106-11e4-9f8e-6edd0ca20947:1-2" 29 | // the mariadb GTID set likes this "0-1-100" 30 | for { 31 | ev, _ := streamer.GetEvent(context.Background()) 32 | // Dump event 33 | ev.Dump(os.Stdout) 34 | } 35 | // or we can use a timeout context 36 | //for { 37 | // ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) 38 | // ev, err := streamer.GetEvent(ctx) 39 | // cancel() 40 | // if err == context.DeadlineExceeded { 41 | // // meet timeout 42 | // continue 43 | // } 44 | // ev.Dump(os.Stdout) 45 | //} 46 | } -------------------------------------------------------------------------------- /PublicFunc/ConnectionDataType.go: -------------------------------------------------------------------------------- 1 | package PublicFunc 2 | <<<<<<< HEAD 3 | 4 | import mgorm "goProject/mgorm/ExecQuerySQL" 5 | 6 | func TypeSql(a map[string]*mgorm.Connection, m,o string) (string,string) { 7 | var SstrSql,DstrSql string 8 | //1、判断表明是否相同,列名是否相同，输出相同的表名 9 | if a["source"].DriverName == "mysql" { 10 | SstrSql = m 11 | }else{ 12 | SstrSql = o 13 | } 14 | if a["dest"].DriverName == "mysql" { 15 | DstrSql = m 16 | }else{ 17 | DstrSql = o 18 | } 19 | return SstrSql,DstrSql 20 | } 21 | ======= 22 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MySQL非主从环境下数据一致性校验及修复程序 # 2 | 3 | ---------- 4 | ## Introductory ## 5 | 6 | 主从环境下数据一致性校验经常会用pt-table-checksum工具。但是pt-table-checksum工具对环境依赖较高， 7 | 需要正常的主从环境，这对非主从环境的数据校验就不适用了，但身为DBA针对此类变更最头疼的就是数据迁移 8 | 完成后如何确定源目端数据是否一致，在迁移过程中是否有数据遗漏或数据异常。于是萌发了自己使用go写一个 9 | 离线数据校验工具的想法。 10 | 11 | 1）测试 pt-table-checksum 的缺陷如下： 12 | （1）pt-table-checksum 数据校验使用数据库的CRC32进行校验，针对表中两个列名及数据相同但顺序不同，无法检测出来 13 | （2）pt-table-checksum 依赖主从关系，非主从关系的数据库无法检测 14 | （3）pt-table-checksum 是基于binlog把在主库进行的检查动作，在从库重放一遍 15 | （4）pt-table-checksum 需要安装依赖包，针对平台有限制 16 | 17 | 2）goTableCheckSum 针对 pt-table-checksum 的缺陷改造： 18 | （1）goTableCheckSum 数据校验是有程序进行校验，可以使用CRC32、MD5、SHA1算法进行对数据校验，将数据使用字节流的形式查询校验，规避上述问题 19 | （2）goTableCheckSum 对源目端数据库只会执行select查询数据操作，对源目端的数据库产生的压力较小 20 | （3）goTableCheckSum 支持针对指定的单表或多表进行数据校验、可以指定忽略校验的表 21 | （4）goTableCheckSum 支持指定where条件的数据校验查询，但仅限于单表 22 | （5）goTableCheckSum 支持自定义每次检验数据库的chunk大小。即每次校验多少条行数 23 | （6）goTableCheckSum 支持自定义修复语句的执行方式，是写在文件中还是直接在表中执行 24 | （7）goTableCheckSum 支持针对单表的where条件的数据校验 25 | （8）goTableCheckSum 支持指定检验数据算法CRC32、MD5、HASH1 26 | （9）goTableCheckSum 支持MySQL <-> MySQL、Oracle <-> MySQL之间的异构数据校验 27 | 28 | 3）goTableCheckSum 后续功能更新 29 | （1）增加相关日志输出（debug、info、warning、error） 30 | （2）针对数据校验增加并发操作，降低数据校验时长 31 | （3）增加单表的自定义列数据校验 32 | （4）断点校验 33 | （5）增加针对源库数据库压力的监控，当压力高时自动减少并发及chunk数据量，或暂停校验 34 | （6）增加根据gitd信息读取binlog日志，实现增量数据的校验 35 | 36 | ------ 37 | 38 | ## Download ## 39 | 40 | 你可以从 [这里](https://github.com/ywlianghang/goTableCheckSum/releases) 下载二进制可执行文件，我已经在ubuntu、centos、redhat、windows x64下测试过 41 | 42 | ----- 43 | ## Usage ## 44 | 45 | 假如需要校验oracle数据库，则需要下载oracle相应版本的驱动，例如：待校验的数据库为11-2则需要去下载11-2的驱动,并生效,否则连接Oracle会报错 46 | 47 | 安装Oracle Instant Client 48 | 49 | 从https://www.oracle.com/database/technologies/instant-client/downloads.html下载免费的Basic或Basic Light软件包。 50 | # oracle basic client 51 | instantclient-basic-linux.x64-11.2.0.4.0.zip 52 | # oracle sqlplus 53 | instantclient-sqlplus-linux.x64-11.2.0.4.0.zip 54 | # oracle sdk 55 | instantclient-sdk-linux.x64-11.2.0.4.0.zip 56 | 57 | 配置oracle client并生效 58 | 59 | shell> unzip instantclient-basic-linux.x64-11.2.0.4.0.zip 60 | shell> unzip instantclient-sqlplus-linux.x64-11.2.0.4.0.zip 61 | shell> unzip instantclient-sdk-linux.x64-11.2.0.4.0.zip 62 | shell> mv instantclient_11_2 /usr/local 63 | shell> echo "export LD_LIBRARY_PATH=/usr/local/instantclient_11_2:$LD_LIBRARY_PATH" >>/etc/profile 64 | shell> source /etc/profile 65 | 66 | 工具使用说明 67 | 68 | The source and destination specified by goTableCheckSum cannot be the same, and the data can only be checked offline 69 | NAME: 70 | goTableCheckSum - mysql table data verification 71 | USAGE: 72 | goTableCheckSum [global options] command [command options] [arguments...] 73 | VERSION: 74 | 2.1.1 75 | AUTHOR: 76 | lianghang 77 | COMMANDS: 78 | help, h Shows a list of commands or help for one command 79 | GLOBAL OPTIONS: 80 | --frameworkCode value, -f value The type of the current validation schema. for example: MySQL(source) <-> MySQL(dest) is -f mm or -f=mm,Oracle(source) <-> MySQL(dest) is -f om or -f=om, MySQL(source) <-> Oracle(dest) is -f mo or -f=mo (default: "mm") 81 | --OracleSid value, --osid value The SID required to connect to Oracle. for example：SID is "helowin", -sid helowin or -sid=helowin (default: "NULL") 82 | --source value, -s value Source side database connection information. For example: --source host=127.0.0.1,user=root,password=abc123,port=3306 (default:"NULL") 83 | --dest value, -d value Target database connection information. For example: --dest host=127.0.0.1,user=root,password=abc123,port=3306 (default: "NULL") 84 | --database value, -D value checksum Database name. For example: -D pcms or -D=pcms (default: "NULL") 85 | --table value, -t value checksum table name. For example: --table=a, or --table=a,b... (default: "ALL") 86 | --ignoreTable value, --igt value Ignore a check for a table. For example: --igt=a or --igt=a,b... (default: "NULL") 87 | --character value, --charset value connection database character. For example: --charset=utf8 (default: "utf8") 88 | --chunkSize value, --chunk value How many rows of data are checked at a time. For example: --chunk=1000 or --chunk 1000 (default: "10000") 89 | --datafix value Fix SQL written to a file or executed directly. For example: --datafix=file or --datafix table (default: "file") 90 | --checksum value, --cks value Specifies algorithms for source and target-side data validation.values are CRC32 MD5 SHA1. For example: --checksum=CRC32 or 91 | --checksum MD5 or --checksum=HASH1 (default: "CRC32") 92 | --where value The WHERE condition is used for data validation under a single table. For example: --where "1=1" or --where "id >=10" (default: "NULL") 93 | --help, -h show help 94 | --version, -v print the version 95 | 96 | 97 | -------- 98 | ## Examples ## 99 | 100 | 1)检测单表数据是否相同，不同则产生修复SQL,默认将修复语句写入到/tmp/目录下，以库名_表名.sql为文件名 101 | shell> ./goTableCheckSum -s host=172.16.50.161,user=pcms,password=pcms@123,port=3306 -d 102 | host=172.16.50.162,user=pcms,password=pcms@123,port=3306 -D pcms -t gobench1 103 | 104 | -- database pcms initialization completes,begin initialization table -- 105 | -- Start initial database pcms table gobench1 -- 106 | ** table gobench1 check start ** 107 | Start the repair Delete SQL and write the repair SQL to /tmp/pcms_gobench1.sql 108 | Start the repair Insert SQL and write the repair SQL to /tmp/pcms_gobench1.sql 109 | Start the repair Delete SQL and write the repair SQL to /tmp/pcms_gobench1.sql 110 | Start the repair Insert SQL and write the repair SQL to /tmp/pcms_gobench1.sql 111 | Start the repair Delete SQL and write the repair SQL to /tmp/pcms_gobench1.sql 112 | Start the repair Insert SQL and write the repair SQL to /tmp/pcms_gobench1.sql 113 | ** table gobench1 check completed ** 114 | ** check table gobench1 time is 1.483941625s ** 115 | 116 | 2）检测单库pcms下所有表的数据是否相同，不同则产生修复SQL，并直接在目标库中执行 117 | shell> ./goTableCheckSum -s host=172.16.50.161,user=pcms,password=pcms@123,port=3306 -d 118 | host=172.16.50.162,user=pcms,password=pcms@123,port=3306 -D pcms -datafix table 119 | 120 | -- database pcms initialization completes,begin initialization table -- 121 | -- Start initial database pcms table gobench1 -- 122 | ** table gobench1 check start ** 123 | Start executing Delete SQL statements in the target databases pcms 124 | Start executing Insert SQL statements in the target databases pcms 125 | Start executing Delete SQL statements in the target databases pcms 126 | Start executing Insert SQL statements in the target databases pcms 127 | Start executing Delete SQL statements in the target databases pcms 128 | Start executing Insert SQL statements in the target databases pcms 129 | ** table gobench1 check completed ** 130 | ** check table gobench1 time is 1.633451665s ** 131 | 132 | 3)检测单表下where条件的数据是否相同，不同则产生修复SQL，并在/tmp/目录下生成修复文件 133 | shell> ./goTableCheckSum -s host=172.16.50.161,user=pcms,password=pcms@123,port=3306 -d 134 | host=172.16.50.162,user=pcms,password=pcms@123,port=3306 -D pcms -t gobench1 -datafix file 135 | --where "id <=200000" 136 | 137 | -- database pcms initialization completes,begin initialization table -- 138 | -- Start initial database pcms table gobench1 -- 139 | ** table gobench1 check start ** 140 | Start the repair Delete SQL and write the repair SQL to /tmp/pcms_gobench1.sql 141 | Start the repair Insert SQL and write the repair SQL to /tmp/pcms_gobench1.sql 142 | ** table gobench1 check completed ** 143 | ** check table gobench1 time is 1.836164054s ** 144 | 145 | 4）检测Oracle和MySQL异构下数据是否相同，不同则产生修复SQL，并在/tmp/目录下生成修复文件 146 | shell> ./goTableCheckSum -f om -osid helowin -s host=172.16.50.161,user=pcms,password=pcms,port=1521 -d host=172.16.50.161,user=pcms,password=pcms@123,port=3306 -D pcms 147 | godror WARNING: discrepancy between DBTIMEZONE ("+00:00"=0) and SYSTIMESTAMP ("+08:00"=800) - set connection timezone, see https://github.com/godror/godror/blob/master/doc/timezone.md 148 | -- database pcms initialization completes,begin initialization table -- 149 | -- Start initial database pcms table GOBENCH1 -- 150 | ** table GOBENCH1 check start ** 151 | Start the repair Delete SQL and write the repair SQL to /tmp/pcms_GOBENCH1.sql 152 | Start the repair Insert SQL and write the repair SQL to /tmp/pcms_GOBENCH1.sql 153 | ** table GOBENCH1 check completed ** 154 | ** check table GOBENCH1 time is 1m25.9036516s ** 155 | 156 | -- Start initial database pcms table GOBENCH2 -- 157 | ** table GOBENCH2 check start ** 158 | ** table GOBENCH2 check completed ** 159 | ** check table GOBENCH2 time is 56.8436ms ** 160 | 161 | 162 | 163 | 164 | 165 | 166 | ------- 167 | ## Building ## 168 | 169 | goTableChecksum needs go version > 1.12 for go mod 170 | 171 | shell> git clone https://github.com/ywlianghang/goTableCheckSum.git 172 | shell> cd main 173 | shell> go build -o goTableChecksum main.go 174 | shell> chmod +x goTableChecksum 175 | shell> mv goTableChecksum /usr/bin 176 | 177 | ----- 178 | ## Requirements ## 179 | 180 | 1）待检验表必须有数据，如果没有数据可以使用 --igt 参数忽略该表 181 | 2）待检验表必须有主键 182 | 3）待检验表主键索引必须是int类型，varchar类型暂不支持 183 | 4）MySQL必须开启lower_case_table_names=1 184 | ----- 185 | ## Author ## 186 | 187 | lianghang ywlianghang@gmail.com 188 | -------------------------------------------------------------------------------- /aa/aa.go: -------------------------------------------------------------------------------- 1 | package aa 2 | -------------------------------------------------------------------------------- /aa/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | -------------------------------------------------------------------------------- /checksum/DML.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "bufio" 5 | "database/sql" 6 | "fmt" 7 | "goProject/PublicFunc" 8 | mgorm "goProject/mgorm/ExecQuerySQL" 9 | "os" 10 | "strings" 11 | ) 12 | 13 | type DesTableInfo struct { 14 | dbname string 15 | tablename string 16 | DestConn *sql.DB 17 | } 18 | 19 | func DestInsert(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo,insertData []string) []string{ 20 | var MySQLinsertVal string 21 | var MySQLinsertValByte,OracleDateColumnName []byte 22 | var DestinsertValstring []string 23 | 24 | tmpOracleSelectColumnStringSlice := strings.Split(o.OracleSelectColumn,",") 25 | for k,v := range tmpOracleSelectColumnStringSlice { 26 | if strings.Index(v, "'yyyy-mm-dd hh24:mi:ss')") != -1 { 27 | tmpIndex := k - 1 28 | OracleDateColumnName = append(OracleDateColumnName,strings.Split(tmpOracleSelectColumnStringSlice[tmpIndex],"(")[1]...) 29 | OracleDateColumnName = append(OracleDateColumnName,","...) 30 | } 31 | } 32 | 33 | for _,v := range insertData { 34 | var MySQLinsertDataByte,OracleinsertDataByte []byte 35 | var MySQLsingleInsertVal,OraclesingleInsertVal string 36 | aa := strings.Split(v,"&@") 37 | MySQLinsertDataByte = append(MySQLinsertDataByte,"("...) 38 | for _,k := range aa { 39 | b := strings.Split(k,"&:") 40 | if len(b) > 1 { 41 | if a["dest"].DriverName == "mysql" { 42 | MySQLinsertDataByte = append(MySQLinsertDataByte,"'"...) 43 | MySQLinsertDataByte = append(MySQLinsertDataByte,b[1]...) 44 | MySQLinsertDataByte = append(MySQLinsertDataByte,"'"...) 45 | MySQLinsertDataByte = append(MySQLinsertDataByte,","...) 46 | } 47 | if a["dest"].DriverName == "godror" { 48 | if strings.Index(string(OracleDateColumnName),b[0]) != -1 { 49 | OracleinsertDataByte = append(OracleinsertDataByte, "to_date('"...) 50 | OracleinsertDataByte = append(OracleinsertDataByte, b[1]...) 51 | OracleinsertDataByte = append(OracleinsertDataByte,"','yyyy-mm-dd hh24:mi:ss')"...) 52 | OracleinsertDataByte = append(OracleinsertDataByte,","...) 53 | }else { 54 | OracleinsertDataByte = append(OracleinsertDataByte,"'"...) 55 | OracleinsertDataByte = append(OracleinsertDataByte, b[1]...) 56 | OracleinsertDataByte = append(OracleinsertDataByte,"'"...) 57 | OracleinsertDataByte = append(OracleinsertDataByte,","...) 58 | } 59 | } 60 | } 61 | } 62 | if strings.HasSuffix(string(OracleinsertDataByte),",") && len(string(OracleinsertDataByte)) >1{ 63 | OraclesingleInsertVal = string(OracleinsertDataByte) 64 | OraclesingleInsertVal = OraclesingleInsertVal[:len(OraclesingleInsertVal)-1] 65 | OinsertSql := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s)", o.Tablename,o.MySQLSelectColumn,OraclesingleInsertVal) 66 | DestinsertValstring = append(DestinsertValstring,OinsertSql) 67 | } 68 | if strings.HasSuffix(string(MySQLinsertDataByte),",") { 69 | MySQLsingleInsertVal = string(MySQLinsertDataByte) 70 | MySQLsingleInsertVal = MySQLsingleInsertVal[:len(MySQLsingleInsertVal)-1] 71 | MySQLinsertValByte = append(MySQLinsertValByte,MySQLsingleInsertVal...) 72 | MySQLinsertValByte = append(MySQLinsertValByte,"),"...) 73 | } 74 | } 75 | if strings.HasSuffix(string(MySQLinsertValByte),",") && len(string(MySQLinsertValByte)) >1{ 76 | MySQLinsertVal = string(MySQLinsertValByte) 77 | MySQLinsertVal = MySQLinsertVal[:len(MySQLinsertVal)-1] 78 | MinsertSql := fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) VALUES %s;", o.Database,o.Tablename,o.MySQLSelectColumn,MySQLinsertVal) 79 | DestinsertValstring = append(DestinsertValstring,MinsertSql) 80 | } 81 | return DestinsertValstring 82 | } 83 | 84 | func DestDelete(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo,deleteData []string) string{ //生成delete语句，删除目标端多余的数据，数据有差异也是先删除，后插入 85 | var deleteSql string 86 | var deleteValByte []byte 87 | for _,v := range deleteData { 88 | aa := strings.Split(v,"&@") 89 | for _,k := range aa { 90 | b := strings.Split(k,"&:") 91 | if b[0] == strings.ToUpper(o.ColumnPRI) { 92 | deleteValByte = append(deleteValByte,b[1]...) 93 | deleteValByte = append(deleteValByte,","...) 94 | } 95 | } 96 | } 97 | deleteVal := string(deleteValByte) 98 | if strings.HasSuffix(deleteVal,","){ 99 | deleteVal = deleteVal[:len(deleteVal)-1] 100 | } 101 | MdeleteSql := fmt.Sprintf("DELETE FROM `%s`.`%s` WHERE %s IN (%s);",o.Database,o.Tablename,o.ColumnPRI,deleteVal) 102 | OdeleteSql := fmt.Sprintf("DELETE FROM %s WHERE %s IN (%s)",o.Tablename,o.ColumnPRI,deleteVal) 103 | _,deleteSql = PublicFunc.TypeSql(a,MdeleteSql,OdeleteSql) 104 | return deleteSql 105 | 106 | } 107 | func SqlFile(dbname string,tablename string,sql string){ //在/tmp/下创建数据修复文件，将在目标端数据修复的语句写入到文件中 108 | sqlFile := "/tmp/"+ dbname + "_" + tablename + ".sql" 109 | //sqlFile := "C:\\"+ dbname + "_" + tablename + ".sql" 110 | sfile,err := os.Open(sqlFile) 111 | if err != nil && os.IsNotExist(err){ 112 | sfile,err = os.OpenFile(sqlFile,os.O_WRONLY|os.O_CREATE,0666) 113 | }else { 114 | sfile,err = os.OpenFile(sqlFile,os.O_WRONLY|os.O_APPEND,0666) 115 | } 116 | 117 | if err != nil { 118 | fmt.Printf("open file err=%v\n",err) 119 | } 120 | write := bufio.NewWriter(sfile) 121 | write.WriteString(sql + "\n") 122 | write.Flush() 123 | defer sfile.Close() 124 | 125 | } -------------------------------------------------------------------------------- /checksum/chunkSum.go: -------------------------------------------------------------------------------- 1 | package checksum 2 | 3 | import ( 4 | "crypto/md5" 5 | "crypto/sha1" 6 | "encoding/hex" 7 | "fmt" 8 | "goProject/flag" 9 | mgorm "goProject/mgorm/ExecQuerySQL" 10 | "hash/crc32" 11 | "strings" 12 | ) 13 | 14 | func CRC32(str []string) uint32{ 15 | aa := strings.Join(str,"") 16 | return crc32.ChecksumIEEE([]byte(aa)) 17 | } 18 | 19 | func MD5(str []string) string { 20 | aa := strings.Join(str,"") 21 | c := md5.New() 22 | c.Write([]byte(aa)) 23 | return hex.EncodeToString(c.Sum(nil)) 24 | } 25 | 26 | 27 | func SHA1(str []string) string{ 28 | aa := strings.Join(str,"") 29 | c:=sha1.New() 30 | c.Write([]byte(aa)) 31 | return hex.EncodeToString(c.Sum(nil)) 32 | } 33 | func Arrcmap(src ,dest []string ) []string{ 34 | msrc := make(map[string]byte) //按源数组建索引 35 | mall := make(map[string]byte) //源+目所有元素建索引 36 | var set []string //交集 37 | //1、源数组建立map 38 | for _,v := range src{ 39 | msrc[v] = 0 40 | mall[v] = 0 41 | } 42 | for _,v := range dest{ 43 | l := len(mall) 44 | mall[v] = 1 45 | if l != len(mall){ 46 | l = len(mall) 47 | }else { 48 | set = append(set,v) 49 | } 50 | } 51 | return set 52 | } 53 | 54 | func Arrcmp(src []string, dest []string) ([]string,[]string) { //对比数据 55 | msrc := make(map[string]byte) //按目数组建索引 56 | mall := make(map[string]byte) //源+目所有元素建索引 并集 57 | var set []string //交集 58 | //1.目数组建立map 59 | for _, v := range dest { 60 | msrc[v] = 0 61 | mall[v] = 0 62 | } 63 | 64 | //2.源数组中，存不进去，即重复元素，所有存不进去的集合就是并集 65 | for _, v := range src { 66 | l := len(mall) 67 | mall[v] = 1 68 | if l != len(mall) { //长度变化，即可以存 69 | l = len(mall) 70 | } else { //存不了，进并集 71 | set = append(set, v) 72 | } 73 | } 74 | //3.遍历交集，在并集中找，找到就从并集中删，删完后就是补集（即并-交=所有变化的元素） 75 | for _, v := range set { 76 | delete(mall, v) 77 | } 78 | //4.此时，mall是补集，所有元素去源中找，找到就是删除的，找不到的必定能在目数组中找到，即新加的 79 | var added,deleted []string 80 | for v, _ := range mall { 81 | _, exist := msrc[v] 82 | if exist { 83 | deleted = append(deleted, v) 84 | }else { 85 | added = append(added,v) 86 | } 87 | } 88 | return added,deleted 89 | } 90 | func ColumnsValidation (sour,dest []byte) []string{ //校验表结构相同的表并返回 91 | var aa []string 92 | var bb []string 93 | var soura,desta []string 94 | if strings.Index(string(sour),";") != -1 { 95 | soura = strings.Split(strings.ToUpper(string(sour)), ";") 96 | soura = soura[:len(soura)-1] 97 | aa = soura 98 | } 99 | if strings.Index(string(dest),";") != -1 { 100 | desta = strings.Split(strings.ToUpper(string(dest)),";") 101 | desta = desta[:len(desta)-1] 102 | } 103 | 104 | if CRC32(soura) != CRC32(desta) { 105 | aa = Arrcmap(soura,desta) 106 | } 107 | 108 | if len(sour) !=0 { 109 | for i := range aa { 110 | bb = append(bb, strings.Split(aa[i], ";")[0]) 111 | } 112 | } 113 | return bb 114 | } 115 | 116 | 117 | func ChunkValidation (a map[string]*mgorm.Connection,o *mgorm.SummaryInfo,p *flag.ConnParameter,sour,dest []byte) { 118 | soura := strings.Split(string(sour),"&,") 119 | desta := strings.Split(string(dest),"&,") 120 | var aa,bb []string 121 | if p.CheckSum == "CRC32" || p.CheckSum == "crc32"{ 122 | if CRC32(soura) != CRC32(desta) { 123 | aa,bb = Arrcmp(soura,desta) 124 | } 125 | }else if p.CheckSum == "MD5" || p.CheckSum == "md5"{ 126 | if MD5(soura) != MD5(desta) { 127 | aa,bb = Arrcmp(soura,desta) 128 | } 129 | }else if p.CheckSum == "SHA1" || p.CheckSum == "SHA1"{ 130 | if SHA1(soura) != SHA1(desta) { 131 | aa,bb = Arrcmp(soura,desta) 132 | } 133 | } 134 | 135 | if bb != nil{ 136 | DeleteSql := DestDelete(a,o,bb) 137 | if p.Datafix =="file"{ 138 | fmt.Printf("Start the repair Delete SQL and write the repair SQL to /tmp/%s_%s.sql\n",o.Database,o.Tablename) 139 | //fmt.Printf("Start the repair Delete SQL and write the repair SQL to C:\\%s_%s.sql\n",o.Database,o.Tablename) 140 | SqlFile(o.Database,o.Tablename,DeleteSql) 141 | }else if p.Datafix =="table"{ 142 | fmt.Printf("Start executing Delete SQL statements in the target databases %s\n",o.Database) 143 | a["dest"].SqlExec(DeleteSql,o) 144 | } 145 | } 146 | 147 | if aa != nil { 148 | InsertSql := DestInsert(a,o,aa) 149 | for _,sql := range InsertSql { 150 | if p.Datafix == "file" { 151 | fmt.Printf("Start the repair Insert SQL and write the repair SQL to /tmp/%s_%s.sql\n",o.Database,o.Tablename) 152 | //fmt.Printf("Start the repair Insert SQL and write the repair SQL to C:\\%s_%s.sql\n", o.Database, o.Tablename) 153 | SqlFile(o.Database, o.Tablename, sql) 154 | } else if p.Datafix == "table" { 155 | fmt.Printf("Start executing Insert SQL statements in the target databases %s\n", o.Database) 156 | a["dest"].SqlExec(sql, o) 157 | } 158 | } 159 | } 160 | 161 | } 162 | -------------------------------------------------------------------------------- /dispose/tableInit.go: -------------------------------------------------------------------------------- 1 | package dispose 2 | 3 | import ( 4 | "fmt" 5 | "goProject/PublicFunc" 6 | "goProject/checksum" 7 | "goProject/flag" 8 | mgorm "goProject/mgorm/ExecQuerySQL" 9 | "os" 10 | "strconv" 11 | "strings" 12 | "time" 13 | ) 14 | 15 | type sourceMesh struct{ 16 | source string 17 | dest string 18 | } 19 | type Table struct{ 20 | tablename []sourceMesh 21 | } 22 | 23 | func ConnInitCheck(o *flag.ConnParameter,b *mgorm.SummaryInfo) map[string]*mgorm.Connection { //传参初始化 24 | //判断需要数据校验的表有哪些 25 | flag.ParameterLimits(o) 26 | a := make(map[string]*mgorm.Connection) 27 | var sourceConn, destConn mgorm.Connection 28 | if o.FrameworkCode[0] == 'm' { 29 | DNS := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=%s", o.Suser, o.Spassword, o.Shost, o.Sport,o.Database, o.Charset) 30 | sourceConn = mgorm.Connection{ 31 | DriverName: "mysql", 32 | DataSourceName: DNS, 33 | MaxIdleConns: 10, 34 | MaxOpenConns: 50, 35 | ConnMaxLifetime: 0, 36 | ConnMaxIdleTime: 0,} 37 | } else { 38 | DSN := fmt.Sprintf("user=%s password=%s connectString=%s:%s/%s", o.Suser, o.Spassword, o.Shost, o.Sport, o.OracleSid) 39 | sourceConn = mgorm.Connection{ 40 | DriverName: "godror", 41 | DataSourceName: DSN, 42 | MaxIdleConns: 10, 43 | MaxOpenConns: 50, 44 | ConnMaxLifetime: 0, 45 | ConnMaxIdleTime: 0, 46 | } 47 | } 48 | if o.FrameworkCode[1] == 'o' { 49 | DSN := fmt.Sprintf("user=%s password=%s connectString=%s:%s/%s", o.Duser, o.Dpassword, o.Dhost, o.Dport, o.OracleSid) 50 | destConn = mgorm.Connection{ 51 | DriverName: "godror", 52 | DataSourceName: DSN, 53 | MaxIdleConns: 10, 54 | MaxOpenConns: 50, 55 | ConnMaxLifetime: 0, 56 | ConnMaxIdleTime: 0, 57 | } 58 | } else { 59 | 60 | DNS := fmt.Sprintf("%s:%s@tcp(%s:%s)/%s?charset=%s", o.Duser, o.Dpassword, o.Dhost, o.Dport, o.Database,o.Charset) 61 | destConn = mgorm.Connection{ 62 | DriverName: "mysql", 63 | DataSourceName: DNS, 64 | MaxIdleConns: 10, 65 | MaxOpenConns: 50, 66 | ConnMaxLifetime: 0, 67 | ConnMaxIdleTime: 0,} 68 | } 69 | a["source"] = &sourceConn 70 | a["dest"] = &destConn 71 | b.Database = o.Database 72 | b.Tablename = o.Tablename 73 | b.IgnoreTable = o.IgnoreTable 74 | b.ChunkSize,_ = strconv.Atoi(o.ChunkSize) 75 | return a 76 | } 77 | func GetCheckTableName(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo) []string { 78 | var tableList []string 79 | //1、判断表明是否相同,列名是否相同，输出相同的表名 80 | MstrSql := fmt.Sprintf("show tables from %s;",o.Database) 81 | OstrSql := "SELECT table_NAME FROM USER_TABLES" 82 | SstrSql,DstrSql := PublicFunc.TypeSql(a,MstrSql,OstrSql) 83 | stableList := a["source"].SQLTableNum(SstrSql,o) 84 | dtableList := a["dest"].SQLTableNum(DstrSql,o) 85 | if len(stableList) !=0 && len(dtableList) !=0 { 86 | tableList = checksum.ColumnsValidation(stableList,dtableList) 87 | }else { 88 | fmt.Printf("[error]: The source-side database %s or the target-side database %s is empty without any tables\n", o.Database, o.Database) 89 | os.Exit(1) 90 | } 91 | 92 | if o.Tablename != "ALL" && len(o.Tablename) > 0{ 93 | table := strings.Split(o.Tablename, ",") 94 | var ao []string 95 | for i := range table { 96 | for k := range tableList{ 97 | if table[i] == tableList[k]{ 98 | ao = append(ao,table[i]) 99 | } 100 | } 101 | } 102 | var bo []string 103 | for i := range table{ 104 | for k := range ao{ 105 | if table[i] != ao[k]{ 106 | bo = append(bo,table[i]) 107 | } 108 | } 109 | } 110 | if len(bo) >0 { 111 | fmt.Println("The current output checklist does not exist at the source end. Please check the checklist：",bo) 112 | os.Exit(1) 113 | } 114 | tableList = table 115 | } 116 | // 是否忽略某个表的数据校验 117 | if o.IgnoreTable != "NULL" { 118 | var aa []string 119 | o.IgnoreTable = strings.ToUpper(o.IgnoreTable) 120 | a := strings.Split(o.IgnoreTable, ",") 121 | bmap := make(map[string]int) 122 | for _, v := range tableList { 123 | bmap[v] = 0 124 | } 125 | for _, v := range a { 126 | bmap[v] = 1 127 | } 128 | for k, v := range bmap { 129 | if v == 0 { 130 | aa = append(aa, k) 131 | } 132 | } 133 | tableList = aa 134 | } 135 | o.TableList = tableList 136 | return tableList 137 | } 138 | 139 | func GetCheckColumnType(c []string,a map[string]*mgorm.Connection,o *mgorm.SummaryInfo) { 140 | var tableList []string 141 | //1、判断表明是否相同,列名是否相同，输出相同的表名 142 | var sourColumnList,destColumnList []byte 143 | for _,ta := range c { 144 | MstrSql := fmt.Sprintf("SELECT column_name,data_type,numeric_scale from information_schema.columns where table_schema='%s' and table_name='%s' order by ORDINAL_POSITION;", o.Database, ta) 145 | OstrSql := fmt.Sprintf("SELECT column_name,data_type,data_scale FROM all_tab_cols WHERE table_name ='%s' order by column_id", ta) 146 | SstrSql,DstrSql := PublicFunc.TypeSql(a,MstrSql,OstrSql) 147 | o.Tablename = ta 148 | aa, _ := a["source"].SQLColumnsNum(SstrSql, o) 149 | sourColumnList = append(sourColumnList,aa...) 150 | sourColumnList = append(sourColumnList,";"...) 151 | bb, _ := a["dest"].SQLColumnsNum(DstrSql, o) 152 | destColumnList = append(destColumnList,bb...) 153 | destColumnList = append(destColumnList,";"...) 154 | } 155 | ac := checksum.ColumnsValidation(sourColumnList,destColumnList) 156 | for i:= range ac { 157 | aa := strings.Split(ac[i],":") 158 | tableList = append(tableList,aa[0]) 159 | } 160 | o.TableList = tableList 161 | } 162 | 163 | func GetCheckTablePRI(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo) { 164 | MstrSql := fmt.Sprintf("select COLUMN_NAME from INFORMATION_SCHEMA.COLUMNS where table_schema='%s' and table_name = '%s' and COLUMN_KEY='PRI' and COLUMN_TYPE like '%%int%%';",o.Database,o.Tablename) 165 | OstrSql := fmt.Sprintf("select a.column_name from user_cons_columns a, user_constraints b where a.constraint_name = b.constraint_name and b.constraint_type = 'P' and a.table_name = '%s'",o.Tablename) 166 | SstrSql,_ := PublicFunc.TypeSql(a,MstrSql,OstrSql) 167 | PRI := a["source"].SQLTablePRIColumn(SstrSql) 168 | if len(PRI) <1 { 169 | fmt.Printf("The current table %s does not have a primary key index. Please create a primary key index or use -igt to ignore the table validation\n",o.Tablename) 170 | os.Exit(1) 171 | } 172 | o.ColumnPRI = PRI 173 | } 174 | func GetCheckTableRows(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo) { 175 | MstrSql := fmt.Sprintf("SELECT /* SQL_NO_CACHE */ count(1) FROM `%s`.`%s` FORCE INDEX(PRIMARY);",o.Database,o.Tablename) 176 | OstrSql := fmt.Sprintf("SELECT /* SQL_NO_CACHE */ count(1) FROM %s",o.Tablename) 177 | SstrSql,_ := PublicFunc.TypeSql(a,MstrSql,OstrSql) 178 | RowsCount := a["source"].SQLTableRows(SstrSql,o) 179 | TableRowsCount,_ := strconv.Atoi(RowsCount) 180 | if TableRowsCount <1 { 181 | fmt.Printf("The current source table %s is empty. Use the -igt parameter to ignore data validation for this table",o.Tablename) 182 | os.Exit(1) 183 | } 184 | o.TableRows = TableRowsCount 185 | } 186 | func GetTableFirstIndexVal(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo) { 187 | //返回源表主键索引第一个索引值，以源表信息为主 188 | MstrSql := fmt.Sprintf("SELECT /*!40001 SQL_NO_CACHE */ %s FROM `%s`.`%s` FORCE INDEX(PRIMARY) WHERE %s IS NOT NULL ORDER BY %s LIMIT 1;" ,o.ColumnPRI,o.Database,o.Tablename,o.ColumnPRI,o.ColumnPRI) 189 | OstrSql := fmt.Sprintf("select %s from %s where %s is not null and rownum<=1 order by %s",o.ColumnPRI,o.Tablename,o.ColumnPRI,o.ColumnPRI) 190 | SstrSql,_ := PublicFunc.TypeSql(a,MstrSql,OstrSql) 191 | firstIndexVal := a["source"].SQLTableStartVal(SstrSql,o) 192 | o.TableFirstIndexVal = firstIndexVal 193 | } 194 | 195 | func ComputerJobTask(ea map[string]*mgorm.Connection,o *mgorm.SummaryInfo) { 196 | // 统计总共有多少个job任务 197 | var indexQue []byte 198 | var JobNums int 199 | FirstIndexVal := o.TableFirstIndexVal 200 | rowCount := o.TableRows 201 | ColumnPRI := o.ColumnPRI 202 | indexQue = append(indexQue,FirstIndexVal...) 203 | //计算task任务数量 204 | 205 | JobNums = rowCount / o.ChunkSize 206 | if rowCount % o.ChunkSize != 0{ 207 | JobNums = JobNums + 1 208 | } 209 | //生成每个任务的始末值 210 | if rowCount > o.ChunkSize { 211 | firstIndex,_ := strconv.Atoi(FirstIndexVal) 212 | for i:=1;i< JobNums;i++ { 213 | endrowCown := o.ChunkSize 214 | if o.ChunkSize+firstIndex > rowCount { 215 | o.ChunkSize = rowCount - firstIndex 216 | } 217 | MstrSql := fmt.Sprintf("SELECT /*!40001 SQL_NO_CACHE */ %s FROM `%s`.`%s` FORCE INDEX(`PRIMARY`) WHERE ((`%s` >= %d )) ORDER BY %s LIMIT %d, %d;",ColumnPRI,o.Database,o.Tablename,ColumnPRI,firstIndex,ColumnPRI,o.ChunkSize,2) 218 | OstrSql := fmt.Sprintf("SELECT %s FROM (SELECT %s FROM (SELECT %s FROM %s WHERE %s >= %d and ROWNUM >=1 and ROWNUM <=%d+2 ORDER BY %s ) ORDER BY %s DESC) WHERE ROWNUM <3 order by %s",ColumnPRI,ColumnPRI,ColumnPRI,o.Tablename,ColumnPRI,firstIndex,endrowCown,ColumnPRI,ColumnPRI,ColumnPRI) 219 | SstrSql,_ := PublicFunc.TypeSql(ea,MstrSql,OstrSql) 220 | d := ea["source"].SQLTablePoint(SstrSql,o) 221 | if len(d) >1 { 222 | firstIndex, _ = strconv.Atoi(d[1]) 223 | indexQue = append(indexQue, "-"...) 224 | indexQue = append(indexQue, []byte(d[0])...) 225 | indexQue = append(indexQue, ","...) 226 | indexQue = append(indexQue, []byte(d[1])...) 227 | }else { 228 | indexQue = append(indexQue,"-"...) 229 | indexQue = append(indexQue,[]byte(d[0])...) 230 | } 231 | } 232 | }else { //假如数据量小于chunksize的值时，直接生成始末段 233 | indexQue = append(indexQue,"-"...) 234 | indexQue = append(indexQue,strconv.Itoa(o.ChunkSize)...) 235 | } 236 | o.TableIndexQue = indexQue 237 | } 238 | 239 | func GetSelectColumnDispose (DriverName string,columnSclice []string) string{ 240 | var SelectColumns string 241 | if DriverName == "mysql" { 242 | for i := 0; i < len(columnSclice)-1; i++ { 243 | d := strings.Split(columnSclice[i], ":") 244 | e := d[0] 245 | SelectColumns = SelectColumns + e 246 | SelectColumns = SelectColumns + "," 247 | } 248 | } 249 | if DriverName == "godror" { 250 | for i := 0; i < len(columnSclice)-1; i++ { 251 | d := strings.Split(columnSclice[i], ":") 252 | if strings.ToUpper(d[1]) == "DATE" { 253 | f := "to_char(" + d[0] + ",'yyyy-mm-dd hh24:mi:ss') as " +d[0] 254 | SelectColumns = SelectColumns + f 255 | } else if x, _ := strconv.Atoi(d[2]); x > 0 && x < 9999999999 { 256 | var xx []byte 257 | for i := 1; i <= x; i++ { 258 | xx = append(xx, '0') 259 | } 260 | e := "to_char(" + d[0] + ",'fm9999990." + string(xx) +"') as " +d[0] 261 | SelectColumns = SelectColumns + e 262 | } else { 263 | e := d[0] 264 | SelectColumns = SelectColumns + e 265 | } 266 | SelectColumns = SelectColumns + "," 267 | } 268 | } 269 | if strings.HasSuffix(SelectColumns, ",") { 270 | SelectColumns = SelectColumns[:len(SelectColumns)-1] 271 | } 272 | return SelectColumns 273 | } 274 | func GetSelectColumn(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo) { 275 | MstrSql := fmt.Sprintf("SELECT column_name,data_type,numeric_scale from information_schema.columns where table_schema='%s' and table_name='%s' order by ORDINAL_POSITION;", o.Database, o.Tablename) 276 | OstrSql := fmt.Sprintf("SELECT column_name,data_type,data_scale FROM all_tab_cols WHERE table_name ='%s' order by column_id", o.Tablename) 277 | SstrSql, DstrSql := PublicFunc.TypeSql(a, MstrSql, OstrSql) 278 | _, aa := a["source"].SQLColumnsNum(SstrSql, o) 279 | _, bb := a["dest"].SQLColumnsNum(DstrSql, o) 280 | c := strings.Split(string(aa), "@") 281 | d := strings.Split(string(bb), "@") 282 | SSelectColumns := GetSelectColumnDispose(a["source"].DriverName,c) 283 | DSelectColumns := GetSelectColumnDispose(a["dest"].DriverName,d) 284 | if a["source"].DriverName == "mysql"{ 285 | o.MySQLSelectColumn = SSelectColumns 286 | }else { 287 | o.OracleSelectColumn = SSelectColumns 288 | } 289 | if a["dest"].DriverName == "mysql"{ 290 | o.MySQLSelectColumn = DSelectColumns 291 | }else { 292 | o.OracleSelectColumn = DSelectColumns 293 | } 294 | } 295 | 296 | func ExecCheckSumData(a map[string]*mgorm.Connection,o *mgorm.SummaryInfo,p *flag.ConnParameter) { 297 | c := strings.Split(string(o.TableIndexQue),",") 298 | for _,aa := range c{ 299 | bb := strings.Split(aa,"-") 300 | var first,end string 301 | if cap(bb) <= 1{ 302 | first = bb[0] 303 | end = bb[0] 304 | }else{ 305 | first = bb[0] 306 | end = bb[1] 307 | } 308 | OstrSql := fmt.Sprintf("SELECT /*!40001 SQL_NO_CACHE */ %s FROM %s WHERE %s >= %s and %s <= %s" ,o.OracleSelectColumn,o.Tablename,o.ColumnPRI,first,o.ColumnPRI,end) 309 | MstrSql := fmt.Sprintf("SELECT /*!40001 SQL_NO_CACHE */ %s FROM `%s`.`%s` FORCE INDEX(`PRIMARY`) WHERE %s >= %s and %s <= %s;" ,o.MySQLSelectColumn,o.Database,o.Tablename,o.ColumnPRI,first,o.ColumnPRI,end) 310 | SourceSelectSql,DestSelectSql := PublicFunc.TypeSql(a,MstrSql,OstrSql) 311 | _,sourceDateInfo := a["source"].SQLTableCheckSum(SourceSelectSql,o) 312 | _,destDateInfo := a["dest"].SQLTableCheckSum(DestSelectSql,o) 313 | checksum.ChunkValidation(a,o,p,sourceDateInfo,destDateInfo) 314 | } 315 | 316 | 317 | } 318 | 319 | 320 | 321 | func TableCheckActive(){ 322 | var p flag.ConnParameter 323 | var b mgorm.SummaryInfo 324 | a := ConnInitCheck(&p,&b) 325 | //获取到源目端相同的表，只针对当前这些表进行校验 326 | checksumTableList := GetCheckTableName(a,&b) 327 | //针对源目端相同的表进行表结构校验 328 | GetCheckColumnType(checksumTableList,a,&b) 329 | 330 | fmt.Printf(" -- database %s initialization completes,begin initialization table -- \n", p.Database) 331 | for _,v := range b.TableList{ 332 | start := time.Now() 333 | fmt.Printf(" -- Start initial database %s table %s -- \n", p.Database, v) 334 | fmt.Printf(" ** table %s check start ** \n", v) 335 | 336 | sqlFile := "/tmp/"+ p.Database + "_" + v + ".sql" 337 | //sqlFile := "C:\\" + p.Database + "_" + v + ".sql" 338 | _, err := os.Stat(sqlFile) 339 | if err == nil { 340 | os.Remove(sqlFile) 341 | } 342 | 343 | b.Tablename = v 344 | GetCheckTablePRI(a,&b) 345 | GetCheckTableRows(a,&b) 346 | GetTableFirstIndexVal(a,&b) 347 | ComputerJobTask(a,&b) 348 | GetSelectColumn(a,&b) 349 | ExecCheckSumData(a,&b,&p) 350 | 351 | end := time.Now() 352 | curr := end.Sub(start) 353 | fmt.Printf(" ** table %s check completed ** \n", v) 354 | fmt.Printf(" ** check table %s time is %s ** \n", v, curr) 355 | fmt.Println() 356 | } 357 | } 358 | -------------------------------------------------------------------------------- /flag/flagHelp.go: -------------------------------------------------------------------------------- 1 | package flag 2 | 3 | import ( 4 | "fmt" 5 | "github.com/urfave/cli" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | type ConnParameter struct { 11 | SourceConnection, DestConnection, Database, Charset, Datafix, ChunkSize, Tablename, IgnoreTable, CheckSum, Where, FrameworkCode, OracleSid string 12 | Suser,Spassword,Shost,Sport,Duser,Dpassword,Dhost,Dport string 13 | TableList []string 14 | CheckTableStatus bool 15 | } 16 | 17 | func CliHelp(q *ConnParameter){ 18 | app := cli.NewApp() 19 | app.Name = "goTableCheckSum" //应用名称 20 | app.Usage = "mysql Oracle table data verification" //应用功能说明 21 | app.Author = "lianghang" //作者 22 | app.Email = "ywlianghang@gmail.com" //邮箱 23 | app.Version = "2.1.1" //版本 24 | app.Flags = []cli.Flag{ 25 | cli.StringFlag{ 26 | Name: "frameworkCode,f", //命令名称 27 | Usage: "The type of the current validation schema. for example: MySQL(source) <-> MySQL(dest) is -f mm or -f=mm," + 28 | "Oracle(source) <-> MySQL(dest) is -f om or -f=om, MySQL(source) <-> Oracle(dest) is -f mo or -f=mo", //命令说明 29 | Value: "mm", //默认值 30 | Destination: &q.FrameworkCode, //赋值 31 | }, 32 | cli.StringFlag{ 33 | Name: "OracleSid,osid", //命令名称 34 | Usage: "The SID required to connect to Oracle. for example：SID is \"helowin\", -osid helowin or -osid=helowin", //命令说明 35 | Value: "NULL", //默认值 36 | Destination: &q.OracleSid, //赋值 37 | }, 38 | cli.StringFlag{ 39 | Name: "source,s", //命令名称 40 | Usage: "Source side database connection information. For example: --source host=127.0.0.1,user=root,password=abc123,port=3306", //命令说明 41 | Value: "NULL", //默认值 42 | Destination: &q.SourceConnection, //赋值 43 | }, 44 | cli.StringFlag{ 45 | Name: "dest,d", //命令名称 46 | Usage: "Target database connection information. For example: --dest host=127.0.0.1,user=root,password=abc123,port=3306", //命令说明 47 | Value: "NULL", //默认值 48 | Destination: &q.DestConnection, //赋值 49 | }, 50 | cli.StringFlag{ 51 | Name: "database,D", //命令名称 52 | Usage: "checksum Database name. For example: -D pcms or -D=pcms", //命令说明 53 | Value: "NULL", //默认值 54 | Destination: &q.Database, //赋值 55 | }, 56 | cli.StringFlag{ 57 | Name: "table,t", //命令名称 58 | Usage: "checksum table name. For example: --table=a, or --table=a,b...", //命令说明 59 | Value: "ALL", //默认值 60 | Destination: &q.Tablename, //赋值 61 | }, 62 | cli.StringFlag{ 63 | Name: "ignoreTable,igt", //命令名称 64 | Usage: "Ignore a check for a table. For example: --igt=a or --igt=a,b...", //命令说明 65 | Value: "NULL", //默认值 66 | Destination: &q.IgnoreTable, //赋值 67 | }, 68 | cli.StringFlag{ 69 | Name: "character,charset", //命令名称 70 | Usage: "connection database character. For example: --charset=utf8", //命令说明 71 | Value: "utf8", //默认值 72 | Destination: &q.Charset, //赋值 73 | }, 74 | cli.StringFlag{ 75 | Name: "chunkSize,chunk", //命令名称 76 | Usage: "How many rows of data are checked at a time. For example: --chunk=1000 or --chunk 1000", //命令说明 77 | Value: "10000", //默认值 78 | Destination: &q.ChunkSize, //赋值 79 | }, 80 | cli.StringFlag{ 81 | Name: "datafix", //命令名称 82 | Usage: "Fix SQL written to a file or executed directly. For example: --datafix=file or --datafix table", //命令说明 83 | Value: "file", //默认值 84 | Destination: &q.Datafix, //赋值 85 | }, 86 | cli.StringFlag{ 87 | Name: "checksum,cks", //命令名称 88 | Usage: "Specifies algorithms for source and target-side data validation.values are CRC32 MD5 SHA1. For example: --checksum=CRC32 or --checksum MD5 or --checksum=HASH1", //命令说明 89 | Value: "CRC32", //默认值 90 | Destination: &q.CheckSum, //赋值 91 | }, 92 | cli.StringFlag{ 93 | Name: "where", //命令名称 94 | Usage: "The WHERE condition is used for data validation under a single table. For example: --where \"1=1\" or --where \"id >=10\"", //命令说明 95 | Value: "NULL", //默认值 96 | Destination: &q.Where, //赋值 97 | }, 98 | } 99 | app.Action = func(c *cli.Context) { //应用执行函数 100 | } 101 | app.Run(os.Args) 102 | q.CheckTableStatus = true 103 | 104 | aa := os.Args 105 | for i:= range aa { 106 | if aa[i] == "--help" || aa[i] == "-h" { 107 | os.Exit(1) 108 | } 109 | } 110 | 111 | } 112 | 113 | func ParameterLimits(q *ConnParameter) { 114 | CliHelp(q) 115 | var parametersList string = "crc32,md5,sha1,table,file,mm,om,mo" 116 | if strings.Index(parametersList,strings.ToLower(q.FrameworkCode)) == -1 { 117 | fmt.Println("Incorrect frameworkCode input parameters, please use --help to see the relevant parameters") 118 | os.Exit(1) 119 | } 120 | 121 | if q.SourceConnection != "NULL" { 122 | ab := strings.Split(q.SourceConnection, ",") 123 | for i:= range ab{ 124 | if strings.Index(ab[i],"user=") != -1{ 125 | q.Suser = strings.Split(ab[i],"=")[1] 126 | } 127 | if strings.Index(ab[i],"host=") != -1{ 128 | q.Shost = strings.Split(ab[i],"=")[1] 129 | } 130 | if strings.Index(ab[i],"password=") != -1{ 131 | q.Spassword = strings.Split(ab[i],"=")[1] 132 | } 133 | if strings.Index(ab[i],"port=") != -1{ 134 | q.Sport = strings.Split(ab[i],"=")[1] 135 | } 136 | } 137 | }else { 138 | fmt.Println("Incorrect source database connection input parameters, please use --help to see the relevant parameters") 139 | os.Exit(1) 140 | } 141 | 142 | if q.DestConnection != "NULL" { 143 | ab := strings.Split(q.DestConnection, ",") 144 | for i:= range ab{ 145 | if strings.Index(ab[i],"user=") != -1{ 146 | q.Duser = strings.Split(ab[i],"=")[1] 147 | } 148 | if strings.Index(ab[i],"host=") != -1{ 149 | q.Dhost = strings.Split(ab[i],"=")[1] 150 | } 151 | if strings.Index(ab[i],"password=") != -1{ 152 | q.Dpassword = strings.Split(ab[i],"=")[1] 153 | } 154 | if strings.Index(ab[i],"port=") != -1{ 155 | q.Dport = strings.Split(ab[i],"=")[1] 156 | } 157 | } 158 | }else{ 159 | fmt.Println("Incorrect dest database connection input parameters, please use --help to see the relevant parameters") 160 | os.Exit(1) 161 | } 162 | if q.Database == "NULL"{ 163 | fmt.Println("Incorrect database name input parameters, please use --help to see the relevant parameters") 164 | os.Exit(1) 165 | } 166 | 167 | if q.Shost == q.Dhost && q.Sport == q.Dport{ 168 | fmt.Println("Incorrect Same source end connection address (host and port) input parameters, please use --help to see the relevant parameters") 169 | os.Exit(1) 170 | } 171 | if strings.Index(parametersList,strings.ToLower(q.CheckSum)) == -1 { 172 | fmt.Println("Incorrect CheckSum algorithm input parameters, please use --help to see the relevant parameters") 173 | os.Exit(1) 174 | } 175 | if strings.Index(parametersList,strings.ToLower(q.Datafix)) == -1 { 176 | fmt.Println("Incorrect datafix input parameters, please use --help to see the relevant parameters") 177 | os.Exit(1) 178 | } 179 | 180 | //支持单表下使用where条件进行数据过滤校验 181 | if q.Where != "NULL" { //限制使用where条件的表数量，默认支持单表，多表直接退出 182 | if q.Tablename == "ALL"{ 183 | fmt.Printf("[error]: Use the WHERE function to specify more than just a single table\n") 184 | os.Exit(1) 185 | }else if len(strings.Split(q.Tablename,",")) >1 { 186 | fmt.Printf("[error]: Use the WHERE function to specify more than just a single table\n") 187 | os.Exit(1) 188 | } 189 | } 190 | } 191 | 192 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module goProject 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/go-sql-driver/mysql v1.5.0 7 | github.com/godror/godror v0.23.1 8 | github.com/natefinch/lumberjack v2.0.0+incompatible 9 | github.com/siddontang/go-mysql v1.1.0 10 | github.com/sirupsen/logrus v1.8.0 // indirect 11 | github.com/urfave/cli v1.22.5 12 | go.uber.org/multierr v1.6.0 // indirect 13 | go.uber.org/zap v1.16.0 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 2 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= 3 | github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= 7 | github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= 8 | github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= 9 | github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gGcHOs= 10 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 11 | github.com/godror/godror v0.23.1 h1:JllRZ14r/2+65m8bn+rJJTKjfsYXN/FM2kpUs4TbiB8= 12 | github.com/godror/godror v0.23.1/go.mod h1:wZv/9vPiUib6tkoDl+AZ/QLf5YZgMravZ7jxH2eQWAE= 13 | github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= 14 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 15 | github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= 16 | github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= 17 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 18 | github.com/kortschak/utter v1.0.1/go.mod h1:vSmSjbyrlKjjsL71193LmzBOKgwePk9DH6uFaWHIInc= 19 | github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= 20 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 21 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 22 | github.com/magefile/mage v1.10.0 h1:3HiXzCUY12kh9bIuyXShaVe529fJfyqoVM42o/uom2g= 23 | github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= 24 | github.com/natefinch/lumberjack v2.0.0+incompatible h1:4QJd3OLAMgj7ph+yZTuX13Ld4UpgHp07nNdFX7mqFfM= 25 | github.com/natefinch/lumberjack v2.0.0+incompatible/go.mod h1:Wi9p2TTF5DG5oU+6YfsmYQpsTIOm0B1VNzQg9Mw6nPk= 26 | github.com/pingcap/check v0.0.0-20190102082844-67f458068fc8/go.mod h1:B1+S9LNcuMyLH/4HMTViQOJevkGiik3wW2AN9zb2fNQ= 27 | github.com/pingcap/errors v0.11.0 h1:DCJQB8jrHbQ1VVlMFIrbj2ApScNNotVmkSNplu2yUt4= 28 | github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= 29 | github.com/pingcap/parser v0.0.0-20190506092653-e336082eb825/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= 30 | github.com/pingcap/tipb v0.0.0-20190428032612-535e1abaa330/go.mod h1:RtkHW8WbcNxj8lsbzjaILci01CtYnYbIkQhjyZWrWVI= 31 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 32 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 33 | github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= 34 | github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= 35 | github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 36 | github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= 37 | github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= 38 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 h1:pntxY8Ary0t43dCZ5dqY4YTJCObLY1kIXl0uzMv+7DE= 39 | github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= 40 | github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= 41 | github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= 42 | github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726 h1:xT+JlYxNGqyT+XcU8iUrN18JYed2TvG9yN5ULG2jATM= 43 | github.com/siddontang/go v0.0.0-20180604090527-bdc77568d726/go.mod h1:3yhqj7WBBfRhbBlzyOC3gUxftwsU0u8gqevxwIHQpMw= 44 | github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07 h1:oI+RNwuC9jF2g2lP0u0cVEEZrc/AYBCuFdvwrLWM/6Q= 45 | github.com/siddontang/go-log v0.0.0-20180807004314-8d05993dda07/go.mod h1:yFdBgwXP24JziuRl2NMUahT7nGLNOKi1SIiFxMttVD4= 46 | github.com/siddontang/go-mysql v1.1.0 h1:NfkS1skrPwUd3hsUqhc6jrv24dKTNMANxKRmDsf1fMc= 47 | github.com/siddontang/go-mysql v1.1.0/go.mod h1:+W4RCzesQDI11HvIkaDjS8yM36SpAnGNQ7jmTLn5BnU= 48 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 49 | github.com/sirupsen/logrus v1.8.0 h1:nfhvjKcUMhBMVqbKHJlk5RPrrfYr/NMo3692g0dwfWU= 50 | github.com/sirupsen/logrus v1.8.0/go.mod h1:4GuYW9TZmE769R5STWrRakJc4UqQ3+QQ95fyz7ENv1A= 51 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 52 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 53 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 54 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 55 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 56 | github.com/urfave/cli v1.22.5 h1:lNq9sAHXK2qfdI8W+GRItjCEkI+2oR4d+MEHy1CKXoU= 57 | github.com/urfave/cli v1.22.5/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= 58 | go.uber.org/atomic v1.6.0 h1:Ezj3JGmsOnG1MoRWQkPBsKLe9DwWD9QeXzTRzzldNVk= 59 | go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= 60 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 61 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 62 | go.uber.org/multierr v1.5.0 h1:KCa4XfM8CWFCpxXRGok+Q0SS/0XBhMDbHHGABQLvD2A= 63 | go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= 64 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 65 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 66 | go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= 67 | go.uber.org/zap v1.16.0 h1:uFRZXykJGK9lLY4HtgSw44DnIcAM+kRBP7x5m+NpAOM= 68 | go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= 69 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 70 | golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= 71 | golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= 72 | golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= 73 | golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 74 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 75 | golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 76 | golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 77 | golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= 78 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 79 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 80 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 81 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037 h1:YyJpGZS1sBuBCzLAR1VEpK193GlqGZbnPFnPV/5Rsb4= 82 | golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 83 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 84 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 85 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 86 | golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= 87 | golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= 88 | golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 89 | golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= 90 | golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 91 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= 92 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 93 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 94 | gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 95 | gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= 96 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 97 | honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= 98 | -------------------------------------------------------------------------------- /logs/log.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "github.com/natefinch/lumberjack" 7 | "go.uber.org/zap" 8 | "go.uber.org/zap/zapcore" 9 | ) 10 | 11 | 12 | func initLogger(loglevel string) *zap.Logger{ 13 | hook := lumberjack.Logger{ 14 | Filename: "./logs/tableCheckSum.log", //日志文件路径 15 | MaxSize: 128, //megabytes 16 | MaxBackups: 30, //最多保留300个备份 17 | MaxAge: 7, //days 18 | Compress: true, //是否压缩 disabled by default 19 | } 20 | w := zapcore.AddSync(&hook) 21 | // 设置日志级别，debug可以打印出info，debug，warn，error;info级别可以打印warn，info，error；warn只能打印warn，error 22 | // debug-->info->warn->error 23 | var level zapcore.Level 24 | switch loglevel { 25 | case "debug": 26 | level = zap.DebugLevel 27 | case "info": 28 | level = zap.InfoLevel 29 | case "warn": 30 | level = zap.WarnLevel 31 | case "error": 32 | level = zap.ErrorLevel 33 | default: 34 | level = zap.InfoLevel 35 | } 36 | encoderConfig := zap.NewProductionEncoderConfig() 37 | //时间格式 38 | encoderConfig.EncodeTime = zapcore.ISO8601TimeEncoder 39 | core := zapcore.NewCore( 40 | zapcore.NewConsoleEncoder(encoderConfig), 41 | w, 42 | level, 43 | ) 44 | logger := zap.New(core) 45 | return logger 46 | } 47 | type Test struct { 48 | Name string `json:"name"` 49 | Age int `json:"age"` 50 | } 51 | func main(){ 52 | t := &Test{ 53 | Name:"xiaoming", 54 | Age:12, 55 | } 56 | data,err := json.Marshal(t) 57 | fmt.Println(data) 58 | if err != nil{ 59 | fmt.Println("marshal is failed,err:",err) 60 | } 61 | logger := initLogger("info") 62 | logger.Info("level") 63 | } 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /logs/tableCheckSum.log: -------------------------------------------------------------------------------- 1 | 2021-04-15T00:08:27.307+0800 info level 2 | -------------------------------------------------------------------------------- /main/goTableCheckSum: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ywlianghang/goTableCheckSum/db2211b1b0d6c13cba5abe9aec8e18e48ee65679/main/goTableCheckSum -------------------------------------------------------------------------------- /main/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "goProject/dispose" 5 | ) 6 | 7 | func main(){ 8 | dispose.TableCheckActive() 9 | 10 | //SELECT @rowid:=@rowid+1 as rowid,a FROM pcms.aa, (SELECT @rowid:=0) as init; 11 | //Incremental.QueryBinlogEvent() 12 | 13 | } 14 | 15 | -------------------------------------------------------------------------------- /mgorm/ExecQuerySQL/QueryTableInfo.go: -------------------------------------------------------------------------------- 1 | package mgorm 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | _ "github.com/go-sql-driver/mysql" 7 | _ "github.com/godror/godror" 8 | <<<<<<< HEAD 9 | "log" 10 | ======= 11 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 12 | "os" 13 | "strings" 14 | "time" 15 | ) 16 | 17 | type Connection struct { 18 | DriverName, DataSourceName string 19 | MaxIdleConns,MaxOpenConns int 20 | ConnMaxLifetime,ConnMaxIdleTime time.Duration 21 | } 22 | 23 | type SummaryInfo struct{ 24 | <<<<<<< HEAD 25 | Database,Tablename,StrSql,IgnoreTable string 26 | RowCount,ChunkSize,JobNums,TableRows int 27 | TableIndexQue []byte 28 | ColumnPRI,TableFirstIndexVal string 29 | MySQLSelectColumn,OracleSelectColumn string 30 | TableList []string 31 | ======= 32 | Database,Tablename,StrSql string 33 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 34 | } 35 | 36 | type DB interface { 37 | GetConnection() *sql.DB 38 | } 39 | 40 | type SqlExec interface { 41 | DB 42 | SQLColumnsNum(o *SummaryInfo) ([]byte,[]byte) 43 | SQLTableNum(o *SummaryInfo) ([]byte,bool) 44 | <<<<<<< HEAD 45 | SQLTablePRIColumn(strSql string,o *SummaryInfo) string 46 | SQLTableRows(strSql string,o *SummaryInfo) int 47 | SQLTablePoint(strSql string,o *SummaryInfo) []string 48 | SQLTableCheckSum(strSql string,o *SummaryInfo) ([]string, []byte) 49 | ======= 50 | SQLTablePRIColumn(o *SummaryInfo) (string,bool) 51 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 52 | } 53 | 54 | 55 | 56 | func (con *Connection) GetConnection() *sql.DB { 57 | db,err:=sql.Open(con.DriverName,con.DataSourceName) 58 | if err != nil { 59 | fmt.Errorf("Failed to open to %s database with",con.DataSourceName) 60 | return nil 61 | } 62 | db.SetMaxIdleConns(con.MaxIdleConns) 63 | db.SetMaxOpenConns(con.MaxOpenConns) 64 | db.SetConnMaxLifetime(con.ConnMaxLifetime) 65 | db.SetConnMaxIdleTime(con.ConnMaxIdleTime) 66 | return db 67 | } 68 | 69 | <<<<<<< HEAD 70 | func (con *Connection) SQLColumnsNum(strSql string,o *SummaryInfo) ([]byte,[]byte){ //获取每个表的列信息 71 | ======= 72 | func (con *Connection) SQLColumnsNum(o *SummaryInfo) ([]byte,[]byte){ //获取每个表的列信息 73 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 74 | var columnsList []byte 75 | var columnsInfo []byte 76 | dbconn:=con.GetConnection() 77 | defer dbconn.Close() 78 | <<<<<<< HEAD 79 | stmt,err := dbconn.Prepare(strSql) 80 | rows,err := stmt.Query() 81 | if err != nil { 82 | fmt.Printf("Failed to get column information for the current table %s under the databases %s !\n",o.Database,o.Tablename) 83 | os.Exit(1) 84 | } 85 | columnsList = append(columnsList,o.Tablename...) 86 | columnsList = append(columnsList,":"...) 87 | ======= 88 | 89 | stmt,err := dbconn.Prepare(o.StrSql) 90 | rows,err := stmt.Query() 91 | if err != nil { 92 | fmt.Printf("Failed to get column information for the current table %s under the databases %s !The information is as follows:%s\n",err) 93 | os.Exit(1) 94 | } 95 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 96 | for rows.Next(){ 97 | var columns string 98 | var colDataType string 99 | var numericScale string 100 | rows.Scan(&columns,&colDataType,&numericScale) 101 | <<<<<<< HEAD 102 | columns = strings.ToUpper(columns) 103 | colDataType = strings.ToUpper(colDataType) 104 | numericScale = strings.ToUpper(numericScale) 105 | ======= 106 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 107 | if len(numericScale) == 0 { 108 | numericScale = "9999999999" 109 | } 110 | columnsList = append(columnsList,columns...) 111 | columnsList = append(columnsList,"@"...) 112 | columnsInfo = append(columnsInfo,columns...) 113 | columnsInfo = append(columnsInfo,":"...) 114 | columnsInfo = append(columnsInfo,colDataType...) 115 | columnsInfo = append(columnsInfo,":"...) 116 | columnsInfo = append(columnsInfo,numericScale...) 117 | columnsInfo = append(columnsInfo,"@"...) 118 | } 119 | defer rows.Close() 120 | return columnsList,columnsInfo 121 | } 122 | 123 | <<<<<<< HEAD 124 | func (m *Connection) SQLTableNum(strSql string,o *SummaryInfo) []byte { //获取库下表和列的信息 125 | var tableList []byte 126 | dbconn := m.GetConnection() 127 | defer dbconn.Close() 128 | stmt, err := dbconn.Prepare(strSql) 129 | rows, err := stmt.Query() 130 | if err != nil { 131 | fmt.Println("获取数据库%s的表信息失败！详细信息如下：%s", o.Database, err) 132 | ======= 133 | func (m *Connection) SQLTableNum(o *SummaryInfo) ([]byte,bool) { //获取库下表和列的信息 134 | var tableList []byte 135 | var status bool = true 136 | 137 | dbconn := m.GetConnection() 138 | defer dbconn.Close() 139 | // strSql := "show tables from " + o.Database 140 | stmt, err := dbconn.Prepare(o.StrSql) 141 | rows, err := stmt.Query() 142 | if err != nil { 143 | fmt.Println("获取数据库%s的表信息失败！详细信息如下：%s", o.Database, err) 144 | status = false 145 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 146 | } 147 | for rows.Next() { 148 | var tablename string 149 | rows.Scan(&tablename) 150 | tablename = strings.ToUpper(tablename) 151 | <<<<<<< HEAD 152 | tableList = append(tableList, tablename...) 153 | tableList = append(tableList, ";"...) 154 | } 155 | return tableList 156 | } 157 | 158 | func (m *Connection) SQLTablePRIColumn(strSql string) string{ //初始化数据库，获取当前库下每个表是否有int类型主键 159 | // 获取当前主键信息 160 | var PRIcolumn string 161 | dbconn := m.GetConnection() 162 | defer dbconn.Close() 163 | stmt,err := dbconn.Prepare(strSql) 164 | err = stmt.QueryRow().Scan(&PRIcolumn) 165 | if err != nil { 166 | fmt.Println("Failed to get primary key columns information") 167 | } 168 | return PRIcolumn 169 | } 170 | 171 | func (m *Connection) SQLTableRows(strSql string,o *SummaryInfo) string{ 172 | var rowCount string 173 | dbconn := m.GetConnection() 174 | defer dbconn.Close() 175 | stmt,err := dbconn.Prepare(strSql) 176 | err = stmt.QueryRow().Scan(&rowCount) 177 | if err != nil{ 178 | fmt.Printf("[error]: Failed to query total %s rows for table under current databases %s.The error message is : %s.\n",o.Tablename,o.Database,err) 179 | os.Exit(1) 180 | } 181 | return rowCount 182 | } 183 | 184 | func (m *Connection) SQLTableStartVal(strSql string,o *SummaryInfo) string { //查询源表的当前数据行总数，以源表的信息为准 185 | dbconn := m.GetConnection() 186 | defer dbconn.Close() 187 | var firstIndexPoint string 188 | stmt,err := dbconn.Prepare(strSql) 189 | err = stmt.QueryRow().Scan(&firstIndexPoint) 190 | if err != nil { 191 | fmt.Printf("[error]: Failed to query the table %s in the current databases %s for primary key information. Please check whether the table has data or primary key information of type int.\n",o.Tablename,o.Database) 192 | os.Exit(1) 193 | } 194 | return firstIndexPoint 195 | } 196 | 197 | func (m *Connection) SQLTablePoint(strSql string,o *SummaryInfo) []string{ //对所有chunk的开头、结尾索引节点进行处理，生成字节数组，返回数组 //目前只支持主键为int类型的 198 | var d []string 199 | dbconn := m.GetConnection() 200 | defer dbconn.Close() 201 | stmt,err := dbconn.Prepare(strSql) 202 | rows, err := stmt.Query() 203 | if err != nil { 204 | fmt.Printf("[error]: Failed to query the Chunk beginning and end nodes of the table %s under the current database %s.The error message is: %s.\n", o.Tablename,o.Database,err) 205 | } 206 | for rows.Next() { 207 | var b string 208 | rows.Scan(&b) 209 | d = append(d,b) 210 | 211 | } 212 | return d 213 | } 214 | 215 | 216 | 217 | func (m *Connection) SQLTableCheckSum(strSql string,o *SummaryInfo) ([]string, []byte) { //根据每个chunk的起始节点，生成查询语句，去数据库查询数据，返回数据的字节数组 218 | var result []byte 219 | dbconn := m.GetConnection() 220 | defer dbconn.Close() 221 | stmt,err := dbconn.Prepare(strSql) 222 | rows, err := stmt.Query() 223 | if err != nil { 224 | log.Fatal("[error]: Failed to query date from Source MySQL database. Please check current database status！", err) 225 | } 226 | // 获取列名 227 | columns,_ := rows.Columns() 228 | // 定义一个切片，长度是字段的个数，切片里面的元素类型是sql.RawBytes 229 | values := make([]sql.RawBytes,len(columns)) 230 | //定义一个切片，元素类型是interface{}接口 231 | scanArgs := make([]interface{},len(values)) 232 | for i := range values{ 233 | //把sql.RawBytes类型的地址存进来 234 | scanArgs[i] = &values[i] 235 | } 236 | for rows.Next(){ 237 | rows.Scan(scanArgs...) 238 | for k,col := range values { 239 | result = append(result,columns[k]...) 240 | result = append(result,"&:"...) 241 | result = append(result,col...) 242 | result = append(result,"&@"...) 243 | } 244 | result = append(result,"&,"...) 245 | } 246 | 247 | defer rows.Close() 248 | return columns,result 249 | } 250 | 251 | func (m *Connection) SqlExec(strSql string,o *SummaryInfo){ //执行目标端数据修复语句 252 | dbconn := m.GetConnection() 253 | defer dbconn.Close() 254 | stmt,err := dbconn.Prepare(strSql) 255 | if err != nil { 256 | fmt.Printf("[error]: Failed to query the Chunk beginning and end nodes of the table %s under the current database %s.The error message is: %s.\n", o.Tablename,o.Database,err) 257 | } 258 | _,err = stmt.Exec() 259 | if err != nil{ 260 | fmt.Printf("exec failed, err:%v\n", err) 261 | os.Exit(1) 262 | } 263 | ======= 264 | o.Tablename = tablename 265 | columns, _ := m.SQLColumnsNum(o) 266 | 267 | tableList = append(tableList, tablename...) 268 | tableList = append(tableList, ":"...) 269 | tableList = append(tableList, columns...) 270 | tableList = append(tableList, ";"...) 271 | } 272 | return tableList, status 273 | } 274 | 275 | func (m *Connection) QueryMySQLTablePRIColumn(o *SummaryInfo) (string,bool){ //初始化数据库，获取当前库下每个表是否有int类型主键 276 | // 获取当前主键信息 277 | var status bool = true 278 | var PRIcolumn string 279 | 280 | dbconn := m.GetConnection() 281 | defer dbconn.Close() 282 | 283 | //strSql := "select COLUMN_NAME from INFORMATION_SCHEMA.COLUMNS where table_schema='" + o.Database + "' and table_name = '" + o.Tablename +"' and COLUMN_KEY='PRI' and COLUMN_TYPE like '%int%';" 284 | stmt,err := dbconn.Prepare(o.StrSql) 285 | err = stmt.QueryRow().Scan(&PRIcolumn) 286 | if err != nil { 287 | status = false 288 | } 289 | return PRIcolumn,status 290 | >>>>>>> ce765399f3d2115cfbd3137bdf1c015bc310c867 291 | } --------------------------------------------------------------------------------