├── .gitignore ├── .goreleaser.yaml ├── LICENSE ├── README.md ├── check.sh ├── config.json ├── go.mod ├── go.sum ├── internal ├── alter.go ├── alter_test.go ├── config.go ├── db.go ├── email.go ├── index.go ├── schema.go ├── schema_test.go ├── statics.go ├── sync.go ├── sync_test.go ├── testdata │ ├── alert_1.sql │ ├── result_1.sql │ ├── result_2.sql │ ├── result_3.sql │ ├── result_4.sql │ ├── user_0.sql │ ├── user_1.sql │ ├── user_2.sql │ └── user_4.sql ├── timer.go └── util.go └── main.go /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files, Static and Dynamic libs (Shared Objects) 2 | *.o 3 | *.a 4 | *.so 5 | 6 | # Folders 7 | _obj 8 | _test 9 | 10 | # Architecture specific extensions/prefixes 11 | *.[568vq] 12 | [568vq].out 13 | 14 | *.cgo1.go 15 | *.cgo2.c 16 | _cgo_defun.c 17 | _cgo_gotypes.go 18 | _cgo_export.* 19 | 20 | _testmain.go 21 | 22 | *.exe 23 | *.test 24 | *.prof 25 | log/ 26 | .idea/ 27 | mysql-schema-sync 28 | vendor 29 | mydb_conf.json 30 | db_alter.sql 31 | -------------------------------------------------------------------------------- /.goreleaser.yaml: -------------------------------------------------------------------------------- 1 | project_name: mysql-schema-sync 2 | before: 3 | hooks: 4 | - go mod tidy 5 | builds: 6 | - env: 7 | - CGO_ENABLED=0 8 | goos: 9 | - linux 10 | - darwin 11 | goarch: 12 | - amd64 13 | - arm64 14 | id: mysql-schema-sync 15 | archives: 16 | - replacements: 17 | darwin: Darwin 18 | linux: Linux 19 | amd64: x86_64 20 | 21 | nfpms: 22 | # note that this is an array of nfpm configs 23 | - 24 | id: mysql-schema-sync 25 | package_name: mysql-schema-sync 26 | file_name_template: "{{ .ConventionalFileName }}" 27 | builds: 28 | - mysql-schema-sync 29 | vendor: Drum Roll Inc. 30 | homepage: https://example.com/ 31 | maintainer: Drummer 32 | description: |- 33 | Drum rolls installer package. 34 | Software to create fast and easy drum rolls. 35 | formats: 36 | - deb 37 | - rpm 38 | conflicts: 39 | - svn 40 | - bash 41 | replaces: 42 | - fish 43 | bindir: /usr/bin 44 | epoch: 2 45 | release: 1 46 | section: default 47 | priority: extra 48 | meta: true 49 | 50 | checksum: 51 | name_template: 'checksums.txt' 52 | snapshot: 53 | name_template: "{{ incpatch .Version }}" 54 | changelog: 55 | sort: asc 56 | filters: 57 | exclude: 58 | - '^docs:' 59 | - '^test:' 60 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 du 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # mysql-schema-sync 2 | 3 | MySQL Schema 自动同步工具 4 | 5 | 用于将 `线上` 数据库 Schema **变化**同步到 `本地测试环境`! 6 | 只同步 Schema、不同步数据。 7 | 8 | 支持功能： 9 | 10 | 1. 同步**新表** 11 | 2. 同步**字段** 变动：新增、修改 12 | 3. 同步**索引** 变动：新增、修改 13 | 4. 支持**预览**（只对比不同步变动） 14 | 5. **邮件**通知变动结果 15 | 6. 支持屏蔽更新**表、字段、索引、外键** 16 | 7. 支持本地比线上额外多一些表、字段、索引、外键 17 | 8. 在该项目的基础上修复了比对过程中遇到分区表会终止后续操作的问题，支持分区表，对于分区表，会同步除了分区以外的变更。 18 | 9. 支持每条 ddl 只会执行单个的修改，目的兼容tidb ddl问题 Unsupported multi schema change，通过single_schema_change字段控制，默认关闭。 19 | 20 | ## 安装 21 | 22 | ```bash 23 | go install github.com/hidu/mysql-schema-sync@master 24 | ``` 25 | 26 | ## 配置 27 | 28 | 参考 默认配置文件 config.json 配置同步源、目的地址。 29 | 修改邮件接收人 当运行失败或者有表结构变化的时候你可以收到邮件通知。 30 | 31 | 默认情况不会对多出的**表、字段、索引、外键**删除。若需要删除**字段、索引、外键** 可以使用 `-drop` 参数。 32 | 33 | 配置示例(config.json): 34 | 35 | ``` 36 | cp config.json mydb_conf.json 37 | ``` 38 | 39 | ``` 40 | { 41 | //source：同步源 42 | "source":"test:test@(127.0.0.1:3306)/test_0", 43 | //dest：待同步的数据库 44 | "dest":"test:test@(127.0.0.1:3306)/test_1", 45 | //alter_ignore： 同步时忽略的字段和索引 46 | "alter_ignore":{ 47 | "tb1*":{ 48 | "column":["aaa","a*"], 49 | "index":["aa"], 50 | "foreign":[] 51 | } 52 | }, 53 | // tables: table to check schema,default is all.eg :["order_*","goods"] 54 | "tables":[], 55 | // tables_ignore: table to ignore check schema,default is Null :["order_*","goods"] 56 | "tables_ignore": [], 57 | //有变动或者失败时，邮件接收人 58 | "email":{ 59 | "send_mail":false, 60 | "smtp_host":"smtp.163.com:25", 61 | "from":"xxx@163.com", 62 | "password":"xxx", 63 | "to":"xxx@163.com" 64 | } 65 | } 66 | ``` 67 | 68 | ### JSON 配置项说明 69 | 70 | source: 数据库同步源 71 | dest: 待同步的数据库 72 | tables： 数组，配置需要同步的表，为空则是不限制，eg: ["goods","order_*"] 73 | alter_ignore： 忽略修改的配置，表名为tableName，可以配置 column 和 index，支持通配符 * 74 | email ： 同步完成后发送邮件通知信息 75 | single_schema_change：是否每个ddl只执行单个修改 76 | 77 | ### 运行 78 | 79 | ### 直接运行 80 | 81 | ```shell 82 | ./mysql-schema-sync -conf mydb_conf.json -sync 83 | ``` 84 | 85 | ### 预览并生成变更sql 86 | 87 | ```shell 88 | ./mysql-schema-sync -drop -conf mydb_conf.json 2>/dev/null >db_alter.sql 89 | 90 | ``` 91 | 92 | ### 使用shell调度 93 | 94 | ```shell 95 | bash check.sh 96 | ``` 97 | 98 | 每个json文件配置一个目的数据库，check.sh脚本会依次运行每份配置。 99 | log存储在当前的log目录中。 100 | 101 | ### 自动定时运行 102 | 103 | 添加crontab 任务 104 | 105 | ```shell 106 | 30 * * * * cd /your/path/xxx/ && bash check.sh >/dev/null 2>&1 107 | ``` 108 | 109 | ### 参数说明 110 | 111 | ```shell 112 | mysql-schema-sync [-conf] [-dest] [-source] [-sync] [-drop] 113 | ``` 114 | 115 | 说明： 116 | 117 | ```shell 118 | mysql-schema-sync -help 119 | -conf string 120 | 配置文件名称 121 | -dest string 122 | 待同步的数据库 eg: test@(10.10.0.1:3306)/test_1 123 | 该项不为空时，忽略读入 -conf参数项 124 | -drop 125 | 是否对本地多出的字段和索引进行删除 默认否 126 | -http 127 | 启用web站点显示运行结果报告的地址，如 :8080,默认否 128 | -source string 129 | mysql 同步源,eg test@(127.0.0.1:3306)/test_0 130 | -sync 131 | 是否将修改同步到数据库中去，默认否 132 | -tables string 133 | 待检查同步的数据库表，为空则是全部 134 | eg : product_base,order_* 135 | -single_schema_change 136 | 生成 SQL DDL 语言每条命令是否只会进行单个修改操作，默认否 137 | ``` 138 | -------------------------------------------------------------------------------- /check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -x 3 | cd $(dirname $0) 4 | if [ ! -d "log" ];then 5 | mkdir -p log 6 | fi 7 | 8 | exec 1>>log/sync.log.`date +"%Y%m%d"` 2>&1 9 | 10 | 11 | for f in `ls *.json` 12 | do 13 | ./mysql-schema-sync -conf $f -sync 14 | done 15 | 16 | 17 | cd log/ 18 | DAY_MAX=15 19 | find ./ -type f -name "*.log*" -mtime +$DAY_MAX -------------------------------------------------------------------------------- /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "source":"test:test@(127.0.0.1:3306)/test", 3 | "dest":"test:test@(127.0.0.1:3306)/test_1", 4 | "alter_ignore":{ 5 | "tb1*":{ 6 | "column":["aaa","a*"], 7 | "index":["aa"], 8 | "foreign":[] 9 | } 10 | }, 11 | // tables: table to check schema,default is all.eg :["order_*","goods"] 12 | "tables":[], 13 | // tables_ignore: table to ignore check schema,default is Null :["order_*","goods"] 14 | "tables_ignore":[], 15 | "email":{ 16 | "send_mail":false, 17 | "smtp_host":"smtp.163.com:25", 18 | "from":"xxx@163.com", 19 | "password":"xxx", 20 | "to":"xxx@163.com" 21 | } 22 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/hidu/mysql-schema-sync 2 | 3 | go 1.20 4 | 5 | require ( 6 | github.com/elliotchance/orderedmap v1.4.0 7 | github.com/go-sql-driver/mysql v1.7.1 8 | github.com/stretchr/testify v1.8.4 9 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df 10 | ) 11 | 12 | require ( 13 | github.com/davecgh/go-spew v1.1.1 // indirect 14 | github.com/pmezard/go-difflib v1.0.0 // indirect 15 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect 16 | gopkg.in/yaml.v3 v3.0.1 // indirect 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 2 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 3 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 4 | github.com/elliotchance/orderedmap v1.4.0 h1:wZtfeEONCbx6in1CZyE6bELEt/vFayMvsxqI5SgsR+A= 5 | github.com/elliotchance/orderedmap v1.4.0/go.mod h1:wsDwEaX5jEoyhbs7x93zk2H/qv0zwuhg4inXhDkYqys= 6 | github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI= 7 | github.com/go-sql-driver/mysql v1.7.1/go.mod h1:OXbVy3sEdcQ2Doequ6Z5BW6fXNQTmx+9S1MCJN5yJMI= 8 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 9 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 10 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 11 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 12 | github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= 13 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 14 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc h1:2gGKlE2+asNV9m7xrywl36YYNnBG5ZQ0r/BOOxqPpmk= 15 | gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= 16 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 17 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df h1:n7WqCuqOuCbNr617RXOY0AWRXxgwEyPp2z+p0+hgMuE= 18 | gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= 19 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 20 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 21 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 22 | -------------------------------------------------------------------------------- /internal/alter.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "strings" 7 | ) 8 | 9 | type alterType int 10 | 11 | const ( 12 | alterTypeNo alterType = iota 13 | alterTypeCreate 14 | alterTypeDropTable 15 | alterTypeAlter 16 | ) 17 | 18 | func (at alterType) String() string { 19 | switch at { 20 | case alterTypeNo: 21 | return "not_change" 22 | case alterTypeCreate: 23 | return "create" 24 | case alterTypeDropTable: 25 | return "drop" 26 | case alterTypeAlter: 27 | return "alter" 28 | default: 29 | return "unknown" 30 | } 31 | } 32 | 33 | // TableAlterData 表的变更情况 34 | type TableAlterData struct { 35 | SchemaDiff *SchemaDiff 36 | Table string 37 | Comment string 38 | SQL []string 39 | Type alterType 40 | } 41 | 42 | func (ta *TableAlterData) Split() []*TableAlterData { 43 | rs := make([]*TableAlterData, len(ta.SQL)) 44 | for i := 0; i < len(ta.SQL); i++ { 45 | rs[i] = &TableAlterData{ 46 | SchemaDiff: ta.SchemaDiff, 47 | Table: ta.Table, 48 | Comment: ta.Comment, 49 | Type: ta.Type, 50 | SQL: []string{ta.SQL[i]}, 51 | } 52 | } 53 | return rs 54 | } 55 | 56 | func (ta *TableAlterData) String() string { 57 | relationTables := ta.SchemaDiff.RelationTables() 58 | sqlTpl := ` 59 | -- Table : %s 60 | -- Type : %s 61 | -- RelationTables :%s 62 | -- Comment :%s 63 | -- SQL : 64 | %s 65 | ` 66 | str := fmt.Sprintf(sqlTpl, 67 | ta.Table, 68 | ta.Type, 69 | strings.Join(relationTables, ","), 70 | strings.TrimSpace(ta.Comment), 71 | strings.Join(ta.SQL, "\n"), 72 | ) 73 | return strings.TrimSpace(str) 74 | } 75 | 76 | var autoIncrReg = regexp.MustCompile(`\sAUTO_INCREMENT=[1-9]\d*\s`) 77 | 78 | func fmtTableCreateSQL(sql string) string { 79 | return autoIncrReg.ReplaceAllString(sql, " ") 80 | } 81 | -------------------------------------------------------------------------------- /internal/alter_test.go: -------------------------------------------------------------------------------- 1 | // Copyright(C) 2022 github.com/hidu All Rights Reserved. 2 | // Author: hidu 3 | // Date: 2022/3/11 4 | 5 | package internal 6 | 7 | import ( 8 | "testing" 9 | ) 10 | 11 | func Test_fmtTableCreateSQL(t *testing.T) { 12 | type args struct { 13 | sql string 14 | } 15 | tests := []struct { 16 | name string 17 | args args 18 | want string 19 | }{ 20 | { 21 | name: "del auto_incr", 22 | args: args{ 23 | sql: `CREATE TABLE user ( 24 | id bigint unsigned NOT NULL AUTO_INCREMENT, 25 | email varchar(1000) NOT NULL DEFAULT '', 26 | PRIMARY KEY (id) 27 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb3`, 28 | }, 29 | want: `CREATE TABLE user ( 30 | id bigint unsigned NOT NULL AUTO_INCREMENT, 31 | email varchar(1000) NOT NULL DEFAULT '', 32 | PRIMARY KEY (id) 33 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb3`, 34 | }, 35 | { 36 | name: "del auto_incr 2", 37 | args: args{ 38 | sql: `CREATE TABLE user ( 39 | id bigint unsigned NOT NULL AUTO_INCREMENT, 40 | email varchar(1000) NOT NULL DEFAULT '', 41 | PRIMARY KEY (id) 42 | ) ENGINE=InnoDB AUTO_INCREMENT=4049116 DEFAULT CHARSET=utf8mb4`, 43 | }, 44 | want: `CREATE TABLE user ( 45 | id bigint unsigned NOT NULL AUTO_INCREMENT, 46 | email varchar(1000) NOT NULL DEFAULT '', 47 | PRIMARY KEY (id) 48 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4`, 49 | }, 50 | } 51 | for _, tt := range tests { 52 | t.Run(tt.name, func(t *testing.T) { 53 | if got := fmtTableCreateSQL(tt.args.sql); got != tt.want { 54 | t.Errorf("fmtTableCreateSQL() = %v, want %v", got, tt.want) 55 | } 56 | }) 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /internal/config.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "encoding/json" 5 | "log" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | // Config config struct 11 | type Config struct { 12 | // AlterIgnore 忽略配置， eg: "tb1*":{"column":["aaa","a*"],"index":["aa"],"foreign":[]} 13 | AlterIgnore map[string]*AlterIgnoreTable `json:"alter_ignore"` 14 | 15 | // Email 完成同步后发送同步信息的邮件账号信息 16 | Email *EmailStruct `json:"email"` 17 | 18 | // SourceDSN 同步的源头 19 | SourceDSN string `json:"source"` 20 | 21 | // DestDSN 将被同步 22 | DestDSN string `json:"dest"` 23 | 24 | ConfigPath string 25 | 26 | // Tables 同步表的白名单，若为空，则同步全库 27 | Tables []string `json:"tables"` 28 | 29 | // TablesIgnore 不同步的表 30 | TablesIgnore []string `json:"tables_ignore"` 31 | 32 | // Sync 是否真正的执行同步操作 33 | Sync bool 34 | 35 | // Drop 若目标数据库表比源头多了字段、索引，是否删除 36 | Drop bool 37 | 38 | // HTTPAddress 生成站点报告的地址，如 :8080 39 | HTTPAddress string 40 | 41 | // SingleSchemaChange 生成sql ddl语言每条命令只会进行单个修改操作 42 | SingleSchemaChange bool `json:"single_schema_change"` 43 | } 44 | 45 | func (cfg *Config) String() string { 46 | ds, _ := json.MarshalIndent(cfg, " ", " ") 47 | return string(ds) 48 | } 49 | 50 | // AlterIgnoreTable table's ignore info 51 | type AlterIgnoreTable struct { 52 | Column []string `json:"column"` 53 | Index []string `json:"index"` 54 | 55 | // 外键 56 | ForeignKey []string `json:"foreign"` 57 | } 58 | 59 | // IsIgnoreField isIgnore 60 | func (cfg *Config) IsIgnoreField(table string, name string) bool { 61 | for tableName, dit := range cfg.AlterIgnore { 62 | if simpleMatch(tableName, table, "IsIgnoreField_table") { 63 | for _, col := range dit.Column { 64 | if simpleMatch(col, name, "IsIgnoreField_colum") { 65 | return true 66 | } 67 | } 68 | } 69 | } 70 | return false 71 | } 72 | 73 | // CheckMatchTables check table is match 74 | func (cfg *Config) CheckMatchTables(name string) bool { 75 | // 若没有指定表，则意味对全库进行同步 76 | if len(cfg.Tables) == 0 { 77 | return true 78 | } 79 | for _, tableName := range cfg.Tables { 80 | if simpleMatch(tableName, name, "CheckMatchTables") { 81 | return true 82 | } 83 | } 84 | return false 85 | } 86 | 87 | func (cfg *Config) SetTables(tables []string) { 88 | for _, name := range tables { 89 | name = strings.TrimSpace(name) 90 | if len(name) > 0 { 91 | cfg.Tables = append(cfg.Tables, name) 92 | } 93 | } 94 | } 95 | 96 | // SetTablesIgnore 设置忽略 97 | func (cfg *Config) SetTablesIgnore(tables []string) { 98 | for _, name := range tables { 99 | name = strings.TrimSpace(name) 100 | if len(name) > 0 { 101 | cfg.TablesIgnore = append(cfg.TablesIgnore, name) 102 | } 103 | } 104 | } 105 | 106 | // CheckMatchIgnoreTables check table_Ignore is match 107 | func (cfg *Config) CheckMatchIgnoreTables(name string) bool { 108 | if len(cfg.TablesIgnore) == 0 { 109 | return false 110 | } 111 | for _, tableName := range cfg.TablesIgnore { 112 | if simpleMatch(tableName, name, "CheckMatchTables") { 113 | return true 114 | } 115 | } 116 | return false 117 | } 118 | 119 | // Check check config 120 | func (cfg *Config) Check() { 121 | if len(cfg.SourceDSN) == 0 { 122 | log.Fatal("source DSN is empty") 123 | } 124 | if len(cfg.DestDSN) == 0 { 125 | log.Fatal("dest DSN is empty") 126 | } 127 | // log.Println("config:\n", cfg) 128 | } 129 | 130 | // IsIgnoreIndex is index ignore 131 | func (cfg *Config) IsIgnoreIndex(table string, name string) bool { 132 | for tableName, dit := range cfg.AlterIgnore { 133 | if simpleMatch(tableName, table, "IsIgnoreIndex_table") { 134 | for _, index := range dit.Index { 135 | if simpleMatch(index, name) { 136 | return true 137 | } 138 | } 139 | } 140 | } 141 | return false 142 | } 143 | 144 | // IsIgnoreForeignKey 检查外键是否忽略掉 145 | func (cfg *Config) IsIgnoreForeignKey(table string, name string) bool { 146 | for tableName, dit := range cfg.AlterIgnore { 147 | if simpleMatch(tableName, table, "IsIgnoreForeignKey_table") { 148 | for _, foreignName := range dit.ForeignKey { 149 | if simpleMatch(foreignName, name) { 150 | return true 151 | } 152 | } 153 | } 154 | } 155 | return false 156 | } 157 | 158 | // SendMailFail send fail mail 159 | func (cfg *Config) SendMailFail(errStr string) { 160 | if cfg.Email == nil { 161 | log.Println("email conf is empty,skip send mail") 162 | return 163 | } 164 | _host, _ := os.Hostname() 165 | title := "[mysql-schema-sync][" + _host + "]failed" 166 | body := "error:" + errStr + "
" 167 | body += "host:" + _host + "
" 168 | body += "config-file:" + cfg.ConfigPath + "
" 169 | body += "dest_dsn:" + cfg.DestDSN + "
" 170 | pwd, _ := os.Getwd() 171 | body += "pwd:" + pwd + "
" 172 | cfg.Email.SendMail(title, body) 173 | } 174 | 175 | // LoadConfig load config file 176 | func LoadConfig(confPath string) *Config { 177 | var cfg *Config 178 | err := loadJSONFile(confPath, &cfg) 179 | if err != nil { 180 | log.Fatalln("load json conf:", confPath, "failed:", err) 181 | } 182 | cfg.ConfigPath = confPath 183 | return cfg 184 | } 185 | -------------------------------------------------------------------------------- /internal/db.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "log" 7 | 8 | _ "github.com/go-sql-driver/mysql" // mysql driver 9 | ) 10 | 11 | // MyDb db struct 12 | type MyDb struct { 13 | Db *sql.DB 14 | dbType string 15 | } 16 | 17 | // NewMyDb parse dsn 18 | func NewMyDb(dsn string, dbType string) *MyDb { 19 | db, err := sql.Open("mysql", dsn) 20 | if err != nil { 21 | panic(fmt.Sprintf("connected to db [%s] failed,err=%s", dsn, err)) 22 | } 23 | return &MyDb{ 24 | Db: db, 25 | dbType: dbType, 26 | } 27 | } 28 | 29 | // GetTableNames table names 30 | func (db *MyDb) GetTableNames() []string { 31 | rs, err := db.Query("show table status") 32 | if err != nil { 33 | panic("show tables failed:" + err.Error()) 34 | } 35 | defer rs.Close() 36 | 37 | var tables []string 38 | columns, _ := rs.Columns() 39 | for rs.Next() { 40 | var values = make([]any, len(columns)) 41 | valuePtrs := make([]any, len(columns)) 42 | for i := range columns { 43 | valuePtrs[i] = &values[i] 44 | } 45 | if err := rs.Scan(valuePtrs...); err != nil { 46 | panic("show tables failed when scan," + err.Error()) 47 | } 48 | var valObj = make(map[string]any) 49 | for i, col := range columns { 50 | var v any 51 | val := values[i] 52 | b, ok := val.([]byte) 53 | if ok { 54 | v = string(b) 55 | } else { 56 | v = val 57 | } 58 | valObj[col] = v 59 | } 60 | if valObj["Engine"] != nil { 61 | tables = append(tables, valObj["Name"].(string)) 62 | } 63 | } 64 | return tables 65 | } 66 | 67 | // GetTableSchema table schema 68 | func (db *MyDb) GetTableSchema(name string) (schema string) { 69 | rs, err := db.Query(fmt.Sprintf("show create table `%s`", name)) 70 | if err != nil { 71 | log.Println(err) 72 | return 73 | } 74 | defer rs.Close() 75 | for rs.Next() { 76 | var vname string 77 | if err := rs.Scan(&vname, &schema); err != nil { 78 | panic(fmt.Sprintf("get table %s 's schema failed, %s", name, err)) 79 | } 80 | } 81 | return 82 | } 83 | 84 | // Query execute sql query 85 | func (db *MyDb) Query(query string, args ...any) (*sql.Rows, error) { 86 | log.Println("[SQL]", "["+db.dbType+"]", query, args) 87 | return db.Db.Query(query, args...) 88 | } 89 | -------------------------------------------------------------------------------- /internal/email.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "encoding/base64" 5 | "fmt" 6 | "log" 7 | "strconv" 8 | "strings" 9 | 10 | "gopkg.in/gomail.v2" 11 | ) 12 | 13 | // EmailStruct email conf info 14 | type EmailStruct struct { 15 | SMTPHost string `json:"smtp_host"` 16 | From string `json:"from"` 17 | Password string `json:"password"` 18 | To string `json:"to"` 19 | SendMailAble bool `json:"send_mail"` 20 | } 21 | 22 | const tableStyle = ` 23 | 38 | ` 39 | 40 | // SendMail send mail 41 | func (m *EmailStruct) SendMail(title string, body string) { 42 | if !m.SendMailAble { 43 | log.Println("send email : no") 44 | return 45 | } 46 | if len(m.SMTPHost) == 0 || len(m.From) == 0 || len(m.To) == 0 { 47 | log.Println("smtp_host, from,to is empty") 48 | return 49 | } 50 | addrInfo := strings.Split(m.SMTPHost, ":") 51 | if len(addrInfo) != 2 { 52 | log.Println("smtp_host wrong, eg: host_name:25") 53 | return 54 | } 55 | var sendTo []string 56 | for _, _to := range strings.Split(m.To, ";") { 57 | _to = strings.TrimSpace(_to) 58 | if len(_to) != 0 && strings.Contains(_to, "@") { 59 | sendTo = append(sendTo, _to) 60 | } 61 | } 62 | 63 | if len(sendTo) < 1 { 64 | log.Println("mail receiver is empty") 65 | return 66 | } 67 | 68 | body = mailBody(body) 69 | 70 | msgBody := fmt.Sprintf( 71 | "To: %s\r\n"+ 72 | "Content-Type: text/html;charset=utf-8\r\n"+ 73 | "Subject: =?UTF-8?B? %s ?=\r\n"+ 74 | "\r\n%s", 75 | strings.Join(sendTo, ";"), 76 | base64.StdEncoding.EncodeToString([]byte(title)), 77 | body, 78 | ) 79 | a := gomail.NewMessage() 80 | a.SetHeader("From", m.From) 81 | a.SetHeader("To", sendTo...) // 发送给多个用户 82 | a.SetHeader("Subject", "表结构对比通知") // 设置邮件主题 83 | a.SetBody("text/html", msgBody) // 设置邮件正文 84 | port, _ := strconv.Atoi(addrInfo[1]) 85 | 86 | d := gomail.NewDialer(addrInfo[0], port, m.From, m.Password) 87 | 88 | err := d.DialAndSend(a) 89 | if err == nil { 90 | log.Println("send mail success") 91 | } else { 92 | log.Println("send mail failed, err:", err) 93 | } 94 | } 95 | 96 | func mailBody(body string) string { 97 | body = tableStyle + "\n" + body 98 | body += "
" + 99 | "
Powered by mysql-schema-sync " + Version + "
" 100 | return body 101 | } 102 | -------------------------------------------------------------------------------- /internal/index.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "log" 7 | "regexp" 8 | "strings" 9 | ) 10 | 11 | // DbIndex db index 12 | type DbIndex struct { 13 | IndexType indexType 14 | Name string 15 | SQL string 16 | 17 | // 相关联的表 18 | RelationTables []string 19 | } 20 | 21 | type indexType string 22 | 23 | const ( 24 | indexTypePrimary indexType = "PRIMARY" 25 | indexTypeIndex indexType = "INDEX" 26 | indexTypeForeignKey indexType = "FOREIGN KEY" 27 | ) 28 | 29 | func (idx *DbIndex) alterAddSQL(drop bool) []string { 30 | var alterSQL []string 31 | if drop { 32 | dropSQL := idx.alterDropSQL() 33 | if len(dropSQL) != 0 { 34 | alterSQL = append(alterSQL, dropSQL) 35 | } 36 | } 37 | 38 | switch idx.IndexType { 39 | case indexTypePrimary: 40 | alterSQL = append(alterSQL, "ADD "+idx.SQL) 41 | case indexTypeIndex, indexTypeForeignKey: 42 | alterSQL = append(alterSQL, fmt.Sprintf("ADD %s", idx.SQL)) 43 | default: 44 | log.Fatalln("unknown indexType", idx.IndexType) 45 | } 46 | return alterSQL 47 | } 48 | 49 | func (idx *DbIndex) String() string { 50 | bs, _ := json.MarshalIndent(idx, " ", " ") 51 | return string(bs) 52 | } 53 | 54 | func (idx *DbIndex) alterDropSQL() string { 55 | switch idx.IndexType { 56 | case indexTypePrimary: 57 | return "DROP PRIMARY KEY" 58 | case indexTypeIndex: 59 | return fmt.Sprintf("DROP INDEX `%s`", idx.Name) 60 | case indexTypeForeignKey: 61 | return fmt.Sprintf("DROP FOREIGN KEY `%s`", idx.Name) 62 | default: 63 | log.Fatalln("unknown indexType", idx.IndexType) 64 | } 65 | return "" 66 | } 67 | 68 | func (idx *DbIndex) addRelationTable(table string) { 69 | table = strings.TrimSpace(table) 70 | if len(table) != 0 { 71 | idx.RelationTables = append(idx.RelationTables, table) 72 | } 73 | } 74 | 75 | // 匹配索引字段 76 | var indexReg = regexp.MustCompile(`^([A-Z]+\s)?KEY\s`) 77 | 78 | // 匹配外键 79 | var foreignKeyReg = regexp.MustCompile("^CONSTRAINT `(.+)` FOREIGN KEY.+ REFERENCES `(.+)` ") 80 | 81 | func parseDbIndexLine(line string) *DbIndex { 82 | line = strings.TrimSpace(line) 83 | idx := &DbIndex{ 84 | SQL: line, 85 | RelationTables: []string{}, 86 | } 87 | if strings.HasPrefix(line, "PRIMARY") { 88 | idx.IndexType = indexTypePrimary 89 | idx.Name = "PRIMARY KEY" 90 | return idx 91 | } 92 | 93 | // UNIQUE KEY `idx_a` (`a`) USING HASH COMMENT '注释', 94 | // FULLTEXT KEY `c` (`c`) 95 | // PRIMARY KEY (`d`) 96 | // KEY `idx_e` (`e`), 97 | if indexReg.MatchString(line) { 98 | arr := strings.Split(line, "`") 99 | idx.IndexType = indexTypeIndex 100 | idx.Name = arr[1] 101 | return idx 102 | } 103 | 104 | // CONSTRAINT `busi_table_ibfk_1` FOREIGN KEY (`repo_id`) REFERENCES `repo_table` (`repo_id`) 105 | foreignMatches := foreignKeyReg.FindStringSubmatch(line) 106 | if len(foreignMatches) > 0 { 107 | idx.IndexType = indexTypeForeignKey 108 | idx.Name = foreignMatches[1] 109 | idx.addRelationTable(foreignMatches[2]) 110 | return idx 111 | } 112 | 113 | log.Fatalln("db_index parse failed, unsupported, line:", line) 114 | return nil 115 | } 116 | -------------------------------------------------------------------------------- /internal/schema.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | 8 | "github.com/elliotchance/orderedmap" 9 | ) 10 | 11 | // MySchema table schema 12 | type MySchema struct { 13 | Fields *orderedmap.OrderedMap 14 | IndexAll map[string]*DbIndex 15 | ForeignAll map[string]*DbIndex 16 | SchemaRaw string 17 | } 18 | 19 | func (mys *MySchema) String() string { 20 | if mys.Fields == nil { 21 | return "nil" 22 | } 23 | var buf bytes.Buffer 24 | buf.WriteString("Fields:\n") 25 | for name, v := range mys.Fields.Keys() { 26 | buf.WriteString(fmt.Sprintf(" %v : %s\n", name, v)) 27 | } 28 | 29 | buf.WriteString("Index:\n") 30 | for name, idx := range mys.IndexAll { 31 | buf.WriteString(fmt.Sprintf(" %s : %s\n", name, idx.SQL)) 32 | } 33 | buf.WriteString("ForeignKey:\n") 34 | for name, idx := range mys.ForeignAll { 35 | buf.WriteString(fmt.Sprintf(" %s : %s\n", name, idx.SQL)) 36 | } 37 | return buf.String() 38 | } 39 | 40 | // GetFieldNames table names 41 | func (mys *MySchema) GetFieldNames() []string { 42 | var names []string 43 | for _, name := range mys.Fields.Keys() { 44 | names = append(names, name.(string)) 45 | } 46 | return names 47 | } 48 | 49 | func (mys *MySchema) RelationTables() []string { 50 | tbs := make(map[string]int) 51 | for _, idx := range mys.ForeignAll { 52 | for _, tb := range idx.RelationTables { 53 | tbs[tb] = 1 54 | } 55 | } 56 | var tables []string 57 | for tb := range tbs { 58 | tables = append(tables, tb) 59 | } 60 | return tables 61 | } 62 | 63 | // ParseSchema parse table's schema 64 | func ParseSchema(schema string) *MySchema { 65 | schema = strings.TrimSpace(schema) 66 | lines := strings.Split(schema, "\n") 67 | mys := &MySchema{ 68 | SchemaRaw: schema, 69 | Fields: orderedmap.NewOrderedMap(), 70 | IndexAll: make(map[string]*DbIndex), 71 | ForeignAll: make(map[string]*DbIndex), 72 | } 73 | 74 | for i := 1; i < len(lines)-1; i++ { 75 | line := strings.TrimSpace(lines[i]) 76 | if len(line) == 0 { 77 | continue 78 | } 79 | 80 | line = strings.TrimRight(line, ",") 81 | switch line[0] { 82 | case '`': 83 | index := strings.Index(line[1:], "`") 84 | name := line[1 : index+1] 85 | mys.Fields.Set(name, line) 86 | 87 | case '"': 88 | index := strings.Index(line[1:], "\"") 89 | name := line[1 : index+1] 90 | mys.Fields.Set(name, line) 91 | 92 | default: 93 | idx := parseDbIndexLine(line) 94 | if idx == nil { 95 | continue 96 | } 97 | switch idx.IndexType { 98 | case indexTypeForeignKey: 99 | mys.ForeignAll[idx.Name] = idx 100 | default: 101 | mys.IndexAll[idx.Name] = idx 102 | } 103 | } 104 | } 105 | return mys 106 | } 107 | 108 | type SchemaDiff struct { 109 | Source *MySchema 110 | Dest *MySchema 111 | Table string 112 | } 113 | 114 | func newSchemaDiff(table, source, dest string) *SchemaDiff { 115 | return &SchemaDiff{ 116 | Table: table, 117 | Source: ParseSchema(source), 118 | Dest: ParseSchema(dest), 119 | } 120 | } 121 | 122 | func (sdiff *SchemaDiff) RelationTables() []string { 123 | return sdiff.Source.RelationTables() 124 | } 125 | -------------------------------------------------------------------------------- /internal/schema_test.go: -------------------------------------------------------------------------------- 1 | // Copyright(C) 2022 github.com/fsgo All Rights Reserved. 2 | // Author: hidu 3 | // Date: 2022/9/25 4 | 5 | package internal 6 | 7 | import ( 8 | "os" 9 | "testing" 10 | 11 | "github.com/elliotchance/orderedmap" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func testLoadFile(name string) string { 16 | bf, err := os.ReadFile(name) 17 | if err != nil { 18 | panic("read " + name + " failed:" + err.Error()) 19 | } 20 | return string(bf) 21 | } 22 | 23 | func TestParseSchema(t *testing.T) { 24 | type args struct { 25 | schema string 26 | } 27 | tests := []struct { 28 | name string 29 | args args 30 | want *MySchema 31 | }{ 32 | { 33 | name: "case 1", 34 | args: args{ 35 | schema: testLoadFile("testdata/user_0.sql"), 36 | }, 37 | want: &MySchema{ 38 | Fields: (func() *orderedmap.OrderedMap { 39 | m := orderedmap.NewOrderedMap() 40 | // 不会检查 value 41 | m.Set("id", "`id` bigint unsigned NOT NULL AUTO_INCREMENT,") 42 | m.Set("email", "`email` varchar(1000) NOT NULL DEFAULT '',") 43 | m.Set("register_time", "`register_time` timestamp NOT NULL,") 44 | m.Set("password", "`password` varchar(1000) NOT NULL DEFAULT '',") 45 | m.Set("status", "`status` tinyint unsigned NOT NULL DEFAULT '0',") 46 | return m 47 | })(), 48 | IndexAll: map[string]*DbIndex{ 49 | "PRIMARY KEY": { 50 | Name: "PRIMARY KEY", 51 | SQL: "PRIMARY KEY (`id`)", 52 | IndexType: indexTypePrimary, 53 | }, 54 | }, 55 | }, 56 | }, 57 | { 58 | name: "case 2", 59 | args: args{ 60 | schema: testLoadFile("testdata/user_4.sql"), 61 | }, 62 | want: &MySchema{ 63 | Fields: (func() *orderedmap.OrderedMap { 64 | m := orderedmap.NewOrderedMap() 65 | // 不会检查 value 66 | m.Set("id", "\"id\" bigint unsigned NOT NULL AUTO_INCREMENT,") 67 | m.Set("email", "\"email\" varchar(1000) NOT NULL DEFAULT \"\",") 68 | m.Set("register_time", "\"register_time\" timestamp NOT NULL,") 69 | m.Set("password", "\"password\" varchar(1000) NOT NULL DEFAULT \"\",") 70 | m.Set("status", "\"status\" tinyint unsigned NOT NULL DEFAULT \"0\",") 71 | return m 72 | })(), 73 | IndexAll: map[string]*DbIndex{ 74 | "PRIMARY KEY": { 75 | Name: "PRIMARY KEY", 76 | SQL: "PRIMARY KEY (\"id\")", 77 | IndexType: indexTypePrimary, 78 | }, 79 | }, 80 | }, 81 | }, 82 | } 83 | for _, tt := range tests { 84 | t.Run(tt.name, func(t *testing.T) { 85 | got := ParseSchema(tt.args.schema) 86 | gs := got.String() 87 | ws := tt.want.String() 88 | require.Equal(t, ws, gs) 89 | }) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /internal/statics.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "html" 7 | "log" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | "strconv" 12 | "strings" 13 | ) 14 | 15 | type statics struct { 16 | timer *myTimer 17 | Config *Config 18 | tables []*tableStatics 19 | } 20 | 21 | type tableStatics struct { 22 | timer *myTimer 23 | table string 24 | alter *TableAlterData 25 | alterRet error 26 | schemaAfter string 27 | } 28 | 29 | func newStatics(cfg *Config) *statics { 30 | return &statics{ 31 | timer: newMyTimer(), 32 | tables: make([]*tableStatics, 0), 33 | Config: cfg, 34 | } 35 | } 36 | 37 | func (s *statics) newTableStatics(table string, sd *TableAlterData, index int) *tableStatics { 38 | ts := &tableStatics{ 39 | timer: newMyTimer(), 40 | table: table, 41 | alter: sd, 42 | } 43 | if sd.Type == alterTypeNo { 44 | return ts 45 | } 46 | if s.Config.SingleSchemaChange { 47 | sds := sd.Split() 48 | nts := &tableStatics{} 49 | *nts = *ts 50 | nts.alter = sds[index] 51 | s.tables = append(s.tables, nts) 52 | } else { 53 | s.tables = append(s.tables, ts) 54 | } 55 | return ts 56 | } 57 | 58 | func (s *statics) toHTML() string { 59 | code := "

运行结果

\n" 60 | code += "

Tables

\n" 61 | code += ` 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | ` 71 | for idx, tb := range s.tables { 72 | code += "" 73 | code += "\n" 74 | code += "\n" 75 | code += "\n" 86 | code += "\n" 87 | code += "\n" 88 | } 89 | code += "
序号Table 同步(alter) 结果耗时
" + strconv.Itoa(idx+1) + "" + tb.table + "" 76 | if s.Config.Sync { 77 | if tb.alterRet == nil { 78 | code += "成功" 79 | } else { 80 | code += "失败：" + html.EscapeString(tb.alterRet.Error()) + "" 81 | } 82 | } else { 83 | code += "未同步" 84 | } 85 | code += "" + tb.timer.usedSecond() + "
\n

SQLs

\n
"
 90 | 	for _, tb := range s.tables {
 91 | 		code += ""
 92 | 		code += html.EscapeString(tb.alter.String()) + "\n\n"
 93 | 	}
 94 | 	code += "
\n\n" 95 | 96 | code += "

详情

\n" 97 | code += ` 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | ` 107 | for idx, tb := range s.tables { 108 | code += "" 109 | code += "\n" 110 | code += "\n" 121 | code += "\n" 128 | 129 | code += "\n" 136 | code += "\n" 137 | 138 | code += "\n" 139 | code += "\n" 142 | code += "\n" 147 | code += "\n" 148 | } 149 | code += "
序号Table  
" + strconv.Itoa(idx+1) + "" + tb.table + "

" 111 | if s.Config.Sync { 112 | if tb.alterRet == nil { 113 | code += "成功" 114 | } else { 115 | code += "失败：" + tb.alterRet.Error() + "" 116 | } 117 | } else { 118 | code += "未同步" 119 | } 120 | code += "		数据源 Schema:
" 122 | if len(tb.alter.SchemaDiff.Source.SchemaRaw) == 0 { 123 | code += "在源数据源不存在，在目标数据库存在" 124 | } else { 125 | code += htmlPre(tb.alter.SchemaDiff.Source.SchemaRaw) 126 | } 127 | code += "		目标 Schema:
" 130 | if len(tb.alter.SchemaDiff.Dest.SchemaRaw) == 0 { 131 | code += "不存在" 132 | } else { 133 | code += htmlPre(tb.alter.SchemaDiff.Dest.SchemaRaw) 134 | } 135 | code += "
请在目标库执行如下 SQL:
" 140 | code += htmlPre(strings.Join(tb.alter.SQL, ",")) 141 | code += "		" 143 | if s.Config.Sync { 144 | code += "执行后:
" + htmlPre(tb.schemaAfter) 145 | } 146 | code += "
\n" 150 | return code 151 | } 152 | 153 | func (s *statics) alterFailedNum() int { 154 | n := 0 155 | for _, tb := range s.tables { 156 | if tb.alterRet != nil { 157 | n++ 158 | } 159 | } 160 | return n 161 | } 162 | 163 | func (s *statics) sendMailNotice(cfg *Config) { 164 | alterTotal := len(s.tables) 165 | if alterTotal < 1 { 166 | writeHTMLResult("no table change") 167 | log.Println("no table change, skip send mail") 168 | return 169 | } 170 | title := "[mysql_schema_sync] " + strconv.Itoa(alterTotal) + " tables change [" + dsnSort(cfg.DestDSN) + "]" 171 | body := ` 172 | ` 176 | 177 | if !s.Config.Sync { 178 | title += "[preview]" 179 | body += "所有 SQL 均未执行!\n" 180 | } 181 | 182 | hostName, _ := os.Hostname() 183 | body += "

任务信息

\n
"
184 | 	body += " 数据源：" + dsnSort(cfg.SourceDSN) + "\n"
185 | 	body += "   目标：" + dsnSort(cfg.DestDSN) + "\n"
186 | 	body += " 有变化：" + strconv.Itoa(len(s.tables)) + " 张表/条语句\n"
187 | 	body += "是否同步：" + fmt.Sprintf("%t", s.Config.Sync) + "\n"
188 | 	if s.Config.Sync {
189 | 		fn := s.alterFailedNum()
190 | 		body += "失败数 : " + strconv.Itoa(fn) + "\n"
191 | 		if fn > 0 {
192 | 			title += " [failed=" + strconv.Itoa(fn) + "]"
193 | 		}
194 | 	}
195 | 	body += "\n"
196 | 	body += "  主机名： " + hostName + "\n"
197 | 	body += "开始时间： " + s.timer.start.Format(timeFormatStd) + "\n"
198 | 	body += "截止时间： " + s.timer.end.Format(timeFormatStd) + "\n"
199 | 	body += "运行耗时： " + s.timer.usedSecond() + "\n"
200 | 
201 | 	body += "
\n" 202 | body += s.toHTML() 203 | 204 | writeHTMLResult(body) 205 | if cfg.Email != nil { 206 | cfg.Email.SendMail(title, body) 207 | } 208 | if cfg.HTTPAddress != "" { 209 | startWebServer(cfg.HTTPAddress) 210 | } 211 | } 212 | 213 | func startWebServer(addr string) { 214 | fp := filepath.Join(os.TempDir(), "mysql-schema-sync_last.html") 215 | http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { 216 | bf, err := os.ReadFile(fp) 217 | if err != nil { 218 | http.NotFoundHandler().ServeHTTP(w, r) 219 | return 220 | } 221 | _, _ = w.Write(bf) 222 | }) 223 | log.Printf("http://%s", addr) 224 | log.Println("Press Ctrl-C to terminate the program") 225 | ser := &http.Server{ 226 | Addr: addr, 227 | } 228 | log.Println(ser.ListenAndServe()) 229 | } 230 | 231 | func writeHTMLResult(str string) { 232 | fp := filepath.Join(os.TempDir(), "mysql-schema-sync_last.html") 233 | if len(htmlResultPath) > 0 { 234 | fp = htmlResultPath 235 | } 236 | err := os.WriteFile(fp, []byte(str), 0666) 237 | log.Println("html result:", fp, err) 238 | } 239 | 240 | func init() { 241 | flag.StringVar(&htmlResultPath, "html", "", "html result file path") 242 | } 243 | 244 | var htmlResultPath string 245 | -------------------------------------------------------------------------------- /internal/sync.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "strings" 7 | ) 8 | 9 | // SchemaSync 配置文件 10 | type SchemaSync struct { 11 | Config *Config 12 | SourceDb *MyDb 13 | DestDb *MyDb 14 | } 15 | 16 | // NewSchemaSync 对一个配置进行同步 17 | func NewSchemaSync(config *Config) *SchemaSync { 18 | s := new(SchemaSync) 19 | s.Config = config 20 | s.SourceDb = NewMyDb(config.SourceDSN, "source") 21 | s.DestDb = NewMyDb(config.DestDSN, "dest") 22 | return s 23 | } 24 | 25 | // GetNewTableNames 获取所有新增加的表名 26 | func (sc *SchemaSync) GetNewTableNames() []string { 27 | sourceTables := sc.SourceDb.GetTableNames() 28 | destTables := sc.DestDb.GetTableNames() 29 | 30 | var newTables []string 31 | 32 | for _, name := range sourceTables { 33 | if !inStringSlice(name, destTables) { 34 | newTables = append(newTables, name) 35 | } 36 | } 37 | return newTables 38 | } 39 | 40 | // 合并源数据库和目标数据库的表名 41 | func (sc *SchemaSync) GetTableNames() []string { 42 | sourceTables := sc.SourceDb.GetTableNames() 43 | destTables := sc.DestDb.GetTableNames() 44 | var tables []string 45 | tables = append(tables, destTables...) 46 | for _, name := range sourceTables { 47 | if !inStringSlice(name, tables) { 48 | tables = append(tables, name) 49 | } 50 | } 51 | return tables 52 | } 53 | 54 | // RemoveTableSchemaConfig 删除表创建引擎信息，编码信息，分区信息，已修复同步表结构遇到分区表异常退出问题， 55 | // 对于分区表，只会同步字段，索引，主键，外键的变更 56 | func RemoveTableSchemaConfig(schema string) string { 57 | return strings.Split(schema, "ENGINE")[0] 58 | } 59 | 60 | func (sc *SchemaSync) getAlterDataByTable(table string, cfg *Config) *TableAlterData { 61 | sSchema := sc.SourceDb.GetTableSchema(table) 62 | dSchema := sc.DestDb.GetTableSchema(table) 63 | return sc.getAlterDataBySchema(table, sSchema, dSchema, cfg) 64 | } 65 | 66 | func (sc *SchemaSync) getAlterDataBySchema(table string, sSchema string, dSchema string, cfg *Config) *TableAlterData { 67 | alter := new(TableAlterData) 68 | alter.Table = table 69 | alter.Type = alterTypeNo 70 | alter.SchemaDiff = newSchemaDiff(table, RemoveTableSchemaConfig(sSchema), RemoveTableSchemaConfig(dSchema)) 71 | 72 | if sSchema == dSchema { 73 | return alter 74 | } 75 | if len(sSchema) == 0 { 76 | alter.Type = alterTypeDropTable 77 | alter.Comment = "源数据库不存在，删除目标数据库多余的表" 78 | alter.SQL = append(alter.SQL, fmt.Sprintf("drop table `%s`;", table)) 79 | return alter 80 | } 81 | if len(dSchema) == 0 { 82 | alter.Type = alterTypeCreate 83 | alter.Comment = "目标数据库不存在，创建" 84 | alter.SQL = append(alter.SQL, fmtTableCreateSQL(sSchema)+";") 85 | return alter 86 | } 87 | 88 | diffLines := sc.getSchemaDiff(alter) 89 | if len(diffLines) == 0 { 90 | return alter 91 | } 92 | alter.Type = alterTypeAlter 93 | if cfg.SingleSchemaChange { 94 | for _, line := range diffLines { 95 | ns := fmt.Sprintf("ALTER TABLE `%s`\n%s;", table, line) 96 | alter.SQL = append(alter.SQL, ns) 97 | } 98 | } else { 99 | ns := fmt.Sprintf("ALTER TABLE `%s`\n%s;", table, strings.Join(diffLines, ",\n")) 100 | alter.SQL = append(alter.SQL, ns) 101 | } 102 | 103 | return alter 104 | } 105 | 106 | func (sc *SchemaSync) getSchemaDiff(alter *TableAlterData) []string { 107 | sourceMyS := alter.SchemaDiff.Source 108 | destMyS := alter.SchemaDiff.Dest 109 | table := alter.Table 110 | var beforeFieldName string 111 | var alterLines []string 112 | var fieldCount int = 0 113 | // 比对字段 114 | for el := sourceMyS.Fields.Front(); el != nil; el = el.Next() { 115 | if sc.Config.IsIgnoreField(table, el.Key.(string)) { 116 | log.Printf("ignore column %s.%s", table, el.Key.(string)) 117 | continue 118 | } 119 | var alterSQL string 120 | if destDt, has := destMyS.Fields.Get(el.Key); has { 121 | if el.Value != destDt { 122 | alterSQL = fmt.Sprintf("CHANGE `%s` %s", el.Key, el.Value) 123 | } 124 | beforeFieldName = el.Key.(string) 125 | } else { 126 | if len(beforeFieldName) == 0 { 127 | if fieldCount == 0 { 128 | alterSQL = "ADD " + el.Value.(string) + " FIRST" 129 | } else { 130 | alterSQL = "ADD " + el.Value.(string) 131 | } 132 | } else { 133 | alterSQL = fmt.Sprintf("ADD %s AFTER `%s`", el.Value.(string), beforeFieldName) 134 | } 135 | beforeFieldName = el.Key.(string) 136 | } 137 | 138 | if len(alterSQL) != 0 { 139 | log.Println("[Debug] check column.alter ", fmt.Sprintf("%s.%s", table, el.Key.(string)), "alterSQL=", alterSQL) 140 | alterLines = append(alterLines, alterSQL) 141 | } else { 142 | log.Println("[Debug] check column.alter ", fmt.Sprintf("%s.%s", table, el.Key.(string)), "not change") 143 | } 144 | fieldCount++ 145 | } 146 | 147 | // 源库已经删除的字段 148 | if sc.Config.Drop { 149 | for _, name := range destMyS.Fields.Keys() { 150 | if sc.Config.IsIgnoreField(table, name.(string)) { 151 | log.Printf("ignore column %s.%s", table, name) 152 | continue 153 | } 154 | if _, has := sourceMyS.Fields.Get(name); !has { 155 | alterSQL := fmt.Sprintf("drop `%s`", name) 156 | alterLines = append(alterLines, alterSQL) 157 | log.Println("[Debug] check column.drop ", fmt.Sprintf("%s.%s", table, name), "alterSQL=", alterSQL) 158 | } else { 159 | log.Println("[Debug] check column.drop ", fmt.Sprintf("%s.%s", table, name), "not change") 160 | } 161 | } 162 | } 163 | 164 | // 多余的字段暂不删除 165 | 166 | // 比对索引 167 | for indexName, idx := range sourceMyS.IndexAll { 168 | if sc.Config.IsIgnoreIndex(table, indexName) { 169 | log.Printf("ignore index %s.%s", table, indexName) 170 | continue 171 | } 172 | dIdx, has := destMyS.IndexAll[indexName] 173 | log.Println("[Debug] indexName---->[", fmt.Sprintf("%s.%s", table, indexName), 174 | "] dest_has:", has, "\ndest_idx:", dIdx, "\nsource_idx:", idx) 175 | var alterSQLs []string 176 | if has { 177 | if idx.SQL != dIdx.SQL { 178 | alterSQLs = append(alterSQLs, idx.alterAddSQL(true)...) 179 | } 180 | } else { 181 | alterSQLs = append(alterSQLs, idx.alterAddSQL(false)...) 182 | } 183 | if len(alterSQLs) > 0 { 184 | alterLines = append(alterLines, alterSQLs...) 185 | log.Println("[Debug] check index.alter ", fmt.Sprintf("%s.%s", table, indexName), "alterSQL=", alterSQLs) 186 | } else { 187 | log.Println("[Debug] check index.alter ", fmt.Sprintf("%s.%s", table, indexName), "not change") 188 | } 189 | } 190 | 191 | // drop index 192 | if sc.Config.Drop { 193 | for indexName, dIdx := range destMyS.IndexAll { 194 | if sc.Config.IsIgnoreIndex(table, indexName) { 195 | log.Printf("ignore index %s.%s", table, indexName) 196 | continue 197 | } 198 | var dropSQL string 199 | if _, has := sourceMyS.IndexAll[indexName]; !has { 200 | dropSQL = dIdx.alterDropSQL() 201 | } 202 | 203 | if len(dropSQL) != 0 { 204 | alterLines = append(alterLines, dropSQL) 205 | log.Println("[Debug] check index.drop ", fmt.Sprintf("%s.%s", table, indexName), "alterSQL=", dropSQL) 206 | } else { 207 | log.Println("[Debug] check index.drop ", fmt.Sprintf("%s.%s", table, indexName), " not change") 208 | } 209 | } 210 | } 211 | 212 | // 比对外键 213 | for foreignName, idx := range sourceMyS.ForeignAll { 214 | if sc.Config.IsIgnoreForeignKey(table, foreignName) { 215 | log.Printf("ignore foreignName %s.%s", table, foreignName) 216 | continue 217 | } 218 | dIdx, has := destMyS.ForeignAll[foreignName] 219 | log.Println("[Debug] foreignName---->[", fmt.Sprintf("%s.%s", table, foreignName), 220 | "] dest_has:", has, "\ndest_idx:", dIdx, "\nsource_idx:", idx) 221 | var alterSQLs []string 222 | if has { 223 | if idx.SQL != dIdx.SQL { 224 | alterSQLs = append(alterSQLs, idx.alterAddSQL(true)...) 225 | } 226 | } else { 227 | alterSQLs = append(alterSQLs, idx.alterAddSQL(false)...) 228 | } 229 | if len(alterSQLs) > 0 { 230 | alterLines = append(alterLines, alterSQLs...) 231 | log.Println("[Debug] check foreignKey.alter ", fmt.Sprintf("%s.%s", table, foreignName), "alterSQL=", alterSQLs) 232 | } else { 233 | log.Println("[Debug] check foreignKey.alter ", fmt.Sprintf("%s.%s", table, foreignName), "not change") 234 | } 235 | } 236 | 237 | // drop 外键 238 | if sc.Config.Drop { 239 | for foreignName, dIdx := range destMyS.ForeignAll { 240 | if sc.Config.IsIgnoreForeignKey(table, foreignName) { 241 | log.Printf("ignore foreignName %s.%s", table, foreignName) 242 | continue 243 | } 244 | var dropSQL string 245 | if _, has := sourceMyS.ForeignAll[foreignName]; !has { 246 | log.Println("[Debug] foreignName --->[", fmt.Sprintf("%s.%s", table, foreignName), "]", "didx:", dIdx) 247 | dropSQL = dIdx.alterDropSQL() 248 | } 249 | if len(dropSQL) != 0 { 250 | alterLines = append(alterLines, dropSQL) 251 | log.Println("[Debug] check foreignKey.drop ", fmt.Sprintf("%s.%s", table, foreignName), "alterSQL=", dropSQL) 252 | } else { 253 | log.Println("[Debug] check foreignKey.drop ", fmt.Sprintf("%s.%s", table, foreignName), "not change") 254 | } 255 | } 256 | } 257 | 258 | return alterLines 259 | } 260 | 261 | // SyncSQL4Dest sync schema change 262 | func (sc *SchemaSync) SyncSQL4Dest(sqlStr string, sqls []string) error { 263 | log.Print("Exec_SQL_START:\n>>>>>>\n", sqlStr, "\n<<<<<<<<\n\n") 264 | sqlStr = strings.TrimSpace(sqlStr) 265 | if len(sqlStr) == 0 { 266 | log.Println("sql_is_empty, skip") 267 | return nil 268 | } 269 | t := newMyTimer() 270 | ret, err := sc.DestDb.Query(sqlStr) 271 | 272 | defer func() { 273 | if ret != nil { 274 | err := ret.Close() 275 | if err != nil { 276 | log.Println("close ret error:", err) 277 | return 278 | } 279 | } 280 | }() 281 | 282 | // how to enable allowMultiQueries? 283 | if err != nil && len(sqls) > 1 { 284 | log.Println("exec_mut_query failed, err=", err, ",now exec SQLs foreach") 285 | tx, errTx := sc.DestDb.Db.Begin() 286 | if errTx == nil { 287 | for _, sql := range sqls { 288 | ret, err = tx.Query(sql) 289 | log.Println("query_one:[", sql, "]", err) 290 | if err != nil { 291 | break 292 | } 293 | } 294 | if err == nil { 295 | err = tx.Commit() 296 | } else { 297 | _ = tx.Rollback() 298 | } 299 | } 300 | } 301 | t.stop() 302 | if err != nil { 303 | log.Println("EXEC_SQL_FAILED:", err) 304 | return err 305 | } 306 | log.Println("EXEC_SQL_SUCCESS, used:", t.usedSecond()) 307 | cl, err := ret.Columns() 308 | log.Println("EXEC_SQL_RET:", cl, err) 309 | return err 310 | } 311 | 312 | // CheckSchemaDiff 执行最终的 diff 313 | func CheckSchemaDiff(cfg *Config) { 314 | scs := newStatics(cfg) 315 | defer func() { 316 | scs.timer.stop() 317 | scs.sendMailNotice(cfg) 318 | }() 319 | 320 | sc := NewSchemaSync(cfg) 321 | newTables := sc.GetTableNames() 322 | // log.Println("source db table total:", len(newTables)) 323 | 324 | changedTables := make(map[string][]*TableAlterData) 325 | 326 | for _, table := range newTables { 327 | // log.Printf("Index : %d Table : %s\n", index, table) 328 | if !cfg.CheckMatchTables(table) { 329 | // log.Println("Table:", table, "skip") 330 | continue 331 | } 332 | 333 | if cfg.CheckMatchIgnoreTables(table) { 334 | log.Println("Table:", table, "skipped by ignore") 335 | continue 336 | } 337 | 338 | sd := sc.getAlterDataByTable(table, cfg) 339 | 340 | if sd.Type == alterTypeNo { 341 | log.Println("table:", table, "not change,", sd) 342 | continue 343 | } 344 | 345 | if sd.Type == alterTypeDropTable { 346 | log.Println("skipped table", table, ",only exists in dest's db") 347 | continue 348 | } 349 | 350 | fmt.Println(sd) 351 | fmt.Println("") 352 | relationTables := sd.SchemaDiff.RelationTables() 353 | // fmt.Println("relationTables:",table,relationTables) 354 | 355 | // 将所有有外键关联的单独放 356 | groupKey := "multi" 357 | if len(relationTables) == 0 { 358 | groupKey = "single_" + table 359 | } 360 | if _, has := changedTables[groupKey]; !has { 361 | changedTables[groupKey] = make([]*TableAlterData, 0) 362 | } 363 | changedTables[groupKey] = append(changedTables[groupKey], sd) 364 | } 365 | 366 | log.Println("[Debug] changedTables:", changedTables) 367 | 368 | var countSuccess int 369 | var countFailed int 370 | canRunTypePref := "single" 371 | 372 | // 先执行单个表的 373 | runSync: 374 | for typeName, sds := range changedTables { 375 | if !strings.HasPrefix(typeName, canRunTypePref) { 376 | continue 377 | } 378 | log.Println("runSyncType:", typeName) 379 | var sqls []string 380 | var sts []*tableStatics 381 | for _, sd := range sds { 382 | for index := range sd.SQL { 383 | sql := strings.TrimRight(sd.SQL[index], ";") 384 | sqls = append(sqls, sql) 385 | 386 | st := scs.newTableStatics(sd.Table, sd, index) 387 | sts = append(sts, st) 388 | } 389 | } 390 | 391 | sql := strings.Join(sqls, ";\n") + ";" 392 | var ret error 393 | 394 | if sc.Config.Sync { 395 | ret = sc.SyncSQL4Dest(sql, sqls) 396 | if ret == nil { 397 | countSuccess++ 398 | } else { 399 | countFailed++ 400 | } 401 | } 402 | for _, st := range sts { 403 | st.alterRet = ret 404 | st.schemaAfter = sc.DestDb.GetTableSchema(st.table) 405 | st.timer.stop() 406 | } 407 | } // end for 408 | 409 | // 最后再执行多个表的 alter 410 | if canRunTypePref == "single" { 411 | canRunTypePref = "multi" 412 | goto runSync 413 | } 414 | 415 | if sc.Config.Sync { 416 | log.Println("execute_all_sql_done, success_total:", countSuccess, "failed_total:", countFailed) 417 | } 418 | } 419 | -------------------------------------------------------------------------------- /internal/sync_test.go: -------------------------------------------------------------------------------- 1 | // Copyright(C) 2022 github.com/fsgo All Rights Reserved. 2 | // Author: hidu 3 | // Date: 2022/9/25 4 | 5 | package internal 6 | 7 | import ( 8 | "testing" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestSchemaSync_getAlterDataBySchema(t *testing.T) { 14 | type args struct { 15 | table string 16 | sSchema string 17 | dSchema string 18 | cfg *Config 19 | } 20 | tests := []struct { 21 | name string 22 | sc *SchemaSync 23 | args args 24 | want string 25 | }{ 26 | { 27 | name: "user 0-1", 28 | args: args{ 29 | table: "user", 30 | sSchema: testLoadFile("testdata/user_0.sql"), 31 | dSchema: testLoadFile("testdata/user_1.sql"), 32 | cfg: &Config{}, 33 | }, 34 | sc: &SchemaSync{ 35 | Config: &Config{}, 36 | }, 37 | want: testLoadFile("testdata/result_1.sql"), 38 | }, 39 | { 40 | name: "user 0-1 ssc", 41 | args: args{ 42 | table: "user", 43 | sSchema: testLoadFile("testdata/user_0.sql"), 44 | dSchema: testLoadFile("testdata/user_1.sql"), 45 | cfg: &Config{ 46 | SingleSchemaChange: true, 47 | }, 48 | }, 49 | sc: &SchemaSync{ 50 | Config: &Config{}, 51 | }, 52 | want: testLoadFile("testdata/result_2.sql"), 53 | }, 54 | { 55 | name: "user 0-1 ssc", 56 | args: args{ 57 | table: "user", 58 | sSchema: testLoadFile("testdata/user_0.sql"), 59 | dSchema: testLoadFile("testdata/user_1.sql"), 60 | cfg: &Config{ 61 | SingleSchemaChange: true, 62 | }, 63 | }, 64 | sc: &SchemaSync{ 65 | Config: &Config{}, 66 | }, 67 | want: testLoadFile("testdata/result_2.sql"), 68 | }, 69 | { 70 | name: "user 1-0 ssc", 71 | args: args{ 72 | table: "user", 73 | sSchema: testLoadFile("testdata/user_1.sql"), 74 | dSchema: testLoadFile("testdata/user_0.sql"), 75 | cfg: &Config{ 76 | SingleSchemaChange: true, 77 | }, 78 | }, 79 | sc: &SchemaSync{ 80 | Config: &Config{}, 81 | }, 82 | want: testLoadFile("testdata/result_3.sql"), 83 | }, 84 | { 85 | name: "user 2-0 ssc", 86 | args: args{ 87 | table: "user", 88 | sSchema: testLoadFile("testdata/user_2.sql"), 89 | dSchema: testLoadFile("testdata/user_0.sql"), 90 | cfg: &Config{}, 91 | }, 92 | sc: &SchemaSync{ 93 | Config: &Config{}, 94 | }, 95 | want: testLoadFile("testdata/result_4.sql"), 96 | }, 97 | } 98 | for _, tt := range tests { 99 | t.Run(tt.name, func(t *testing.T) { 100 | got := tt.sc.getAlterDataBySchema(tt.args.table, tt.args.sSchema, tt.args.dSchema, tt.args.cfg) 101 | t.Log("got alter:\n", got.String()) 102 | require.Equal(t, tt.want, got.String()) 103 | }) 104 | } 105 | } 106 | -------------------------------------------------------------------------------- /internal/testdata/alert_1.sql: -------------------------------------------------------------------------------- 1 | 2 | ALTER TABLE `test_1`.`user` DROP COLUMN `email`,CHANGE COLUMN `password` `password` INT NOT NULL DEFAULT 0 ,CHANGE COLUMN `status` `status` INT UNSIGNED NOT NULL DEFAULT 0 ; -------------------------------------------------------------------------------- /internal/testdata/result_1.sql: -------------------------------------------------------------------------------- 1 | -- Table : user 2 | -- Type : alter 3 | -- RelationTables : 4 | -- Comment : 5 | -- SQL : 6 | ALTER TABLE `user` 7 | ADD `register_time` timestamp NOT NULL AFTER `email`, 8 | ADD `password` varchar(1000) NOT NULL DEFAULT '' AFTER `register_time`, 9 | ADD `status` tinyint unsigned NOT NULL DEFAULT '0' AFTER `password`; -------------------------------------------------------------------------------- /internal/testdata/result_2.sql: -------------------------------------------------------------------------------- 1 | -- Table : user 2 | -- Type : alter 3 | -- RelationTables : 4 | -- Comment : 5 | -- SQL : 6 | ALTER TABLE `user` 7 | ADD `register_time` timestamp NOT NULL AFTER `email`; 8 | ALTER TABLE `user` 9 | ADD `password` varchar(1000) NOT NULL DEFAULT '' AFTER `register_time`; 10 | ALTER TABLE `user` 11 | ADD `status` tinyint unsigned NOT NULL DEFAULT '0' AFTER `password`; -------------------------------------------------------------------------------- /internal/testdata/result_3.sql: -------------------------------------------------------------------------------- 1 | -- Table : user 2 | -- Type : not_change 3 | -- RelationTables : 4 | -- Comment : 5 | -- SQL : -------------------------------------------------------------------------------- /internal/testdata/result_4.sql: -------------------------------------------------------------------------------- 1 | -- Table : user 2 | -- Type : alter 3 | -- RelationTables : 4 | -- Comment : 5 | -- SQL : 6 | ALTER TABLE `user` 7 | CHANGE `id` `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 8 | CHANGE `email` `email` varchar(100) NOT NULL DEFAULT '', 9 | CHANGE `password` `password` varchar(255) NOT NULL DEFAULT '', 10 | CHANGE `status` `status` int(10) unsigned NOT NULL DEFAULT 1; -------------------------------------------------------------------------------- /internal/testdata/user_0.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `user` ( 2 | `id` bigint unsigned NOT NULL AUTO_INCREMENT, 3 | `email` varchar(1000) NOT NULL DEFAULT '', 4 | `register_time` timestamp NOT NULL, 5 | `password` varchar(1000) NOT NULL DEFAULT '', 6 | `status` tinyint unsigned NOT NULL DEFAULT '0', 7 | PRIMARY KEY (`id`) 8 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb3 -------------------------------------------------------------------------------- /internal/testdata/user_1.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `user` ( 2 | `id` bigint unsigned NOT NULL AUTO_INCREMENT, 3 | `email` varchar(1000) NOT NULL DEFAULT '', 4 | PRIMARY KEY (`id`) 5 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb3 -------------------------------------------------------------------------------- /internal/testdata/user_2.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE `user` ( 2 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 3 | `email` varchar(100) NOT NULL DEFAULT '', 4 | `register_time` timestamp NOT NULL, 5 | `password` varchar(255) NOT NULL DEFAULT '', 6 | `status` int(10) unsigned NOT NULL DEFAULT 1, 7 | PRIMARY KEY (`id`) 8 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb3 -------------------------------------------------------------------------------- /internal/testdata/user_4.sql: -------------------------------------------------------------------------------- 1 | CREATE TABLE "user" ( 2 | "id" bigint unsigned NOT NULL AUTO_INCREMENT, 3 | "email" varchar(1000) NOT NULL DEFAULT "", 4 | "register_time" timestamp NOT NULL, 5 | "password" varchar(1000) NOT NULL DEFAULT "", 6 | "status" tinyint unsigned NOT NULL DEFAULT "0", 7 | PRIMARY KEY ("id") 8 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8mb3 9 | -------------------------------------------------------------------------------- /internal/timer.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | type myTimer struct { 9 | start time.Time 10 | end time.Time 11 | } 12 | 13 | func newMyTimer() *myTimer { 14 | return &myTimer{ 15 | start: time.Now(), 16 | } 17 | } 18 | 19 | func (mt *myTimer) stop() { 20 | mt.end = time.Now() 21 | } 22 | 23 | func (mt *myTimer) usedSecond() string { 24 | return fmt.Sprintf("%f s", mt.end.Sub(mt.start).Seconds()) 25 | } 26 | -------------------------------------------------------------------------------- /internal/util.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "html" 7 | "log" 8 | "os" 9 | "regexp" 10 | "strings" 11 | ) 12 | 13 | // Version 版本号，格式：更新日期(8位).更新次数(累加) 14 | const Version = "20220925.3" 15 | 16 | // AppURL site 17 | const AppURL = "https://github.com/hidu/mysql-schema-sync/" 18 | 19 | const timeFormatStd string = "2006-01-02 15:04:05" 20 | 21 | // loadJsonFile load json 22 | func loadJSONFile(jsonPath string, val any) error { 23 | bs, err := os.ReadFile(jsonPath) 24 | if err != nil { 25 | return err 26 | } 27 | lines := strings.Split(string(bs), "\n") 28 | var bf bytes.Buffer 29 | for _, line := range lines { 30 | lineNew := strings.TrimSpace(line) 31 | if (len(lineNew) > 0 && lineNew[0] == '#') || (len(lineNew) > 1 && lineNew[0:2] == "//") { 32 | continue 33 | } 34 | bf.WriteString(lineNew) 35 | } 36 | return json.Unmarshal(bf.Bytes(), &val) 37 | } 38 | 39 | func inStringSlice(str string, strSli []string) bool { 40 | for _, v := range strSli { 41 | if str == v { 42 | return true 43 | } 44 | } 45 | return false 46 | } 47 | 48 | func simpleMatch(patternStr string, str string, msg ...string) bool { 49 | str = strings.TrimSpace(str) 50 | patternStr = strings.TrimSpace(patternStr) 51 | if patternStr == str { 52 | log.Println("simple_match:suc,equal", msg, "patternStr:", patternStr, "str:", str) 53 | return true 54 | } 55 | pattern := "^" + strings.ReplaceAll(patternStr, "*", `.*`) + "$" 56 | match, err := regexp.MatchString(pattern, str) 57 | if err != nil { 58 | log.Println("simple_match:error", msg, "patternStr:", patternStr, "pattern:", pattern, "str:", str, "err:", err) 59 | } 60 | // if match { 61 | // log.Println("simple_match:suc", msg, "patternStr:", patternStr, "pattern:", pattern, "str:", str) 62 | // } 63 | return match 64 | } 65 | 66 | func htmlPre(str string) string { 67 | return "
" + html.EscapeString(str) + "
" 68 | } 69 | 70 | func dsnSort(dsn string) string { 71 | i := strings.Index(dsn, "@") 72 | if i < 1 { 73 | return dsn 74 | } 75 | return dsn[i+1:] 76 | } 77 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "os" 8 | "runtime" 9 | "strings" 10 | 11 | "github.com/hidu/mysql-schema-sync/internal" 12 | ) 13 | 14 | var configPath = flag.String("conf", "./mydb_conf.json", "json config file path") 15 | var sync = flag.Bool("sync", false, "sync schema changes to dest's db\non default, only show difference") 16 | var drop = flag.Bool("drop", false, "drop fields,index,foreign key only on dest's table") 17 | var httpAddress = flag.String("http", "", "HTTP service address, eg. :8080") 18 | 19 | var source = flag.String("source", "", "sync from, eg: test@(10.10.0.1:3306)/my_online_db_name\nwhen it is not empty,[-conf] while ignore") 20 | var dest = flag.String("dest", "", "sync to, eg: test@(127.0.0.1:3306)/my_local_db_name") 21 | var tables = flag.String("tables", "", "tables to sync\neg : product_base,order_*") 22 | var tablesIgnore = flag.String("tables_ignore", "", "tables ignore sync\neg : product_base,order_*") 23 | var mailTo = flag.String("mail_to", "", "overwrite config's email.to") 24 | var singleSchemaChange = flag.Bool("single_schema_change", false, "single schema changes ddl command a single schema change") 25 | 26 | func init() { 27 | log.SetFlags(log.Lshortfile | log.Ldate) 28 | df := flag.Usage 29 | flag.Usage = func() { 30 | df() 31 | fmt.Fprintln(os.Stderr, "") 32 | fmt.Fprintln(os.Stderr, "mysql schema sync tools "+internal.Version) 33 | fmt.Fprint(os.Stderr, internal.AppURL+"\n\n") 34 | } 35 | } 36 | 37 | var cfg *internal.Config 38 | 39 | func main() { 40 | flag.Parse() 41 | if len(*source) == 0 { 42 | cfg = internal.LoadConfig(*configPath) 43 | } else { 44 | cfg = new(internal.Config) 45 | cfg.SourceDSN = *source 46 | cfg.DestDSN = *dest 47 | } 48 | cfg.Sync = *sync 49 | cfg.Drop = *drop 50 | cfg.HTTPAddress = *httpAddress 51 | cfg.SingleSchemaChange = *singleSchemaChange 52 | 53 | if len(*mailTo) != 0 && cfg.Email != nil { 54 | cfg.Email.To = *mailTo 55 | } 56 | cfg.SetTables(strings.Split(*tables, ",")) 57 | cfg.SetTablesIgnore(strings.Split(*tablesIgnore, ",")) 58 | 59 | defer (func() { 60 | if re := recover(); re != nil { 61 | log.Println(re) 62 | bf := make([]byte, 4096) 63 | n := runtime.Stack(bf, false) 64 | cfg.SendMailFail(fmt.Sprintf("panic:%s\n trace=%s", re, bf[:n])) 65 | log.Fatalln("panic:", string(bf[:n])) 66 | } 67 | })() 68 | 69 | cfg.Check() 70 | internal.CheckSchemaDiff(cfg) 71 | } 72 | --------------------------------------------------------------------------------