├── cmd └── carpenter │ ├── .gitignore │ ├── version.go │ ├── command │ ├── seed_test.go │ ├── build_test.go │ ├── design_test.go │ ├── export_test.go │ ├── before.go │ ├── util.go │ ├── design.go │ ├── export.go │ ├── build.go │ └── seed.go │ ├── main.go │ ├── release.sh │ └── commands.go ├── dialect └── mysql │ ├── system.go │ ├── util.go │ ├── partition.go │ ├── table.go │ ├── index.go │ ├── seed.go │ └── column.go ├── designer ├── designer.go ├── designer_test.go └── _test │ └── expected.json ├── exporter └── exporter.go ├── LICENSE ├── seeder ├── _test │ └── table1.json ├── seeder.go └── seeder_test.go ├── README.md └── CHANGELOG.md /cmd/carpenter/.gitignore: -------------------------------------------------------------------------------- 1 | *.test -------------------------------------------------------------------------------- /cmd/carpenter/version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | const Name string = "carpenter" 4 | const Version string = "0.6.0" 5 | -------------------------------------------------------------------------------- /cmd/carpenter/command/seed_test.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "testing" 4 | 5 | func TestCmdSeed(t *testing.T) { 6 | // Write your code here 7 | } 8 | -------------------------------------------------------------------------------- /cmd/carpenter/command/build_test.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "testing" 4 | 5 | func TestCmdBuild(t *testing.T) { 6 | // Write your code here 7 | } 8 | -------------------------------------------------------------------------------- /cmd/carpenter/command/design_test.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "testing" 4 | 5 | func TestCmdDesign(t *testing.T) { 6 | // Write your code here 7 | } 8 | -------------------------------------------------------------------------------- /cmd/carpenter/command/export_test.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import "testing" 4 | 5 | func TestCmdExport(t *testing.T) { 6 | // Write your code here 7 | } 8 | -------------------------------------------------------------------------------- /dialect/mysql/system.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import "fmt" 4 | 5 | //ForeignKeyCheck returns `"SET FOREIGN_KEY_CHECKS = %s"` 6 | func ForeignKeyCheck(turnOn bool) string { 7 | v := "0" 8 | if turnOn { 9 | v = "1" 10 | } 11 | return fmt.Sprintf("SET foreign_key_checks = %s", v) 12 | } 13 | -------------------------------------------------------------------------------- /designer/designer.go: -------------------------------------------------------------------------------- 1 | package designer 2 | 3 | import ( 4 | "database/sql" 5 | 6 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 7 | ) 8 | 9 | func Export(db *sql.DB, schema string, tableNames ...string) (mysql.Tables, error) { 10 | return mysql.GetTables(db, schema, tableNames...) 11 | } 12 | -------------------------------------------------------------------------------- /cmd/carpenter/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | 6 | "github.com/codegangsta/cli" 7 | ) 8 | 9 | func main() { 10 | 11 | app := cli.NewApp() 12 | app.Name = Name 13 | app.Version = Version 14 | app.Author = "hatajoe" 15 | app.Email = "hatanaka@cloverlab.jp" 16 | app.Usage = "Carpenter is a tool to manage DB schema and data" 17 | 18 | app.Flags = GlobalFlags 19 | app.Commands = Commands 20 | app.CommandNotFound = CommandNotFound 21 | 22 | app.Run(os.Args) 23 | } 24 | -------------------------------------------------------------------------------- /cmd/carpenter/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | XC_ARCH=${XC_ARCH:-386 amd64} 4 | XC_OS=${XC_OS:-linux darwin windows} 5 | 6 | rm -rf pkg/ 7 | rm -rf pkg-tmp/ 8 | 9 | gox \ 10 | -os="${XC_OS}" \ 11 | -arch="${XC_ARCH}" \ 12 | -output "pkg-tmp/{{.OS}}_{{.Arch}}/{{.Dir}}" 13 | 14 | mkdir pkg/ 15 | for file in `\find ./pkg-tmp -type f`; do 16 | FILE_NAME=${file##*/} 17 | FILE_PATH=${file%/*} 18 | `zip -jq ./pkg/${FILE_NAME%.*}_${FILE_PATH#*/*/}.zip ${file}` 19 | done 20 | 21 | ghr -u dev-cloverlab ${1:-v1.0.0} pkg/ 22 | 23 | rm -rf pkg/ 24 | rm -rf pkg-tmp/ 25 | -------------------------------------------------------------------------------- /exporter/exporter.go: -------------------------------------------------------------------------------- 1 | package exporter 2 | 3 | import ( 4 | "database/sql" 5 | "strings" 6 | 7 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 8 | ) 9 | 10 | func Export(db *sql.DB, schema string, tableName string) (string, error) { 11 | cnk, err := mysql.GetChunk(db, tableName, nil) 12 | if err != nil { 13 | return "", err 14 | } 15 | csv := make([]string, 0, len(cnk.Seeds)+1) 16 | csv = append(csv, strings.Join(cnk.ColumnNames, ",")) 17 | for _, seed := range cnk.Seeds { 18 | cols := make([]string, 0, len(cnk.ColumnNames)) 19 | for i := range seed.ColumnData { 20 | cols = append(cols, seed.ToColumnValue(i)) 21 | } 22 | csv = append(csv, strings.Join(cols, ",")) 23 | } 24 | return strings.Join(csv, "\n"), nil 25 | } 26 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Clover Lab.,inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /cmd/carpenter/command/before.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "database/sql" 5 | "time" 6 | 7 | "fmt" 8 | 9 | "github.com/codegangsta/cli" 10 | _ "github.com/go-sql-driver/mysql" 11 | ) 12 | 13 | var db *sql.DB 14 | var schema string 15 | var verbose bool 16 | var dryrun bool 17 | var maxIdleConns int 18 | var maxOpenConns int 19 | 20 | func Before(c *cli.Context) error { 21 | verbose = c.GlobalBool("verbose") 22 | dryrun = c.GlobalBool("dry-run") 23 | schema = c.GlobalString("schema") 24 | maxIdleConns = c.GlobalInt("max-idle-conns") 25 | maxOpenConns = c.GlobalInt("max-open-conns") 26 | 27 | if len(schema) <= 0 { 28 | return fmt.Errorf("err: Specify required `--schema' option") 29 | } 30 | datasource := c.GlobalString("data-source") 31 | if len(datasource) <= 0 { 32 | return fmt.Errorf("err: Specify required `--data-soruce' option") 33 | } 34 | var err error 35 | db, err = sql.Open("mysql", fmt.Sprintf("%s/%s?charset=utf8", datasource, schema)) 36 | if err != nil { 37 | return fmt.Errorf("err: db.Open is failed for reason %v", err) 38 | } 39 | db.SetMaxIdleConns(maxIdleConns) 40 | db.SetMaxOpenConns(maxOpenConns) 41 | db.SetConnMaxLifetime(time.Minute) 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /cmd/carpenter/command/util.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "strings" 8 | ) 9 | 10 | func getErrorMessages(errs []error) []string { 11 | msg := make([]string, 0, len(errs)) 12 | for _, e := range errs { 13 | msg = append(msg, e.Error()) 14 | } 15 | return msg 16 | } 17 | 18 | func execute(queries []string) error { 19 | for _, query := range queries { 20 | if !dryrun { 21 | if _, err := db.Exec(query); err != nil { 22 | return fmt.Errorf("err: db.Exec `%s' failed for reason %s", query, err) 23 | } 24 | } 25 | if verbose { 26 | fmt.Println(query + ";") 27 | } 28 | } 29 | return nil 30 | } 31 | 32 | func walk(path, ext string) (map[string][]string, error) { 33 | dir, err := ioutil.ReadDir(path) 34 | if err != nil { 35 | return nil, fmt.Errorf("err: ioutil.ReadDir %s for reason %v", path, err) 36 | } 37 | files := []os.FileInfo{} 38 | for _, file := range dir { 39 | filename := file.Name() 40 | pos := strings.LastIndex(filename, ".") 41 | if pos <= 0 { 42 | continue 43 | } 44 | if filename[pos:] != ext { 45 | continue 46 | } 47 | files = append(files, file) 48 | } 49 | if len(files) <= 0 { 50 | return nil, fmt.Errorf("err: No csv files found in %s", path) 51 | } 52 | 53 | filesMap := map[string][]string{} 54 | for _, file := range files { 55 | filename := file.Name() 56 | splited := strings.Split(filename, string(os.PathSeparator)) 57 | table := strings.Split(splited[len(splited)-1], ".")[0] 58 | if _, ok := filesMap[table]; !ok { 59 | filesMap[table] = []string{} 60 | } 61 | filesMap[table] = append(filesMap[table], fmt.Sprintf("%s%s%s", path, string(os.PathSeparator), filename)) 62 | } 63 | return filesMap, nil 64 | } 65 | -------------------------------------------------------------------------------- /dialect/mysql/util.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | type JsonNullInt64 struct { 11 | sql.NullInt64 12 | } 13 | 14 | func (v JsonNullInt64) MarshalJSON() ([]byte, error) { 15 | if v.Valid { 16 | return json.Marshal(v.Int64) 17 | } else { 18 | return json.Marshal(nil) 19 | } 20 | } 21 | 22 | func (v *JsonNullInt64) UnmarshalJSON(data []byte) error { 23 | var x *int64 24 | if err := json.Unmarshal(data, &x); err != nil { 25 | i := sql.NullInt64{} 26 | if err := json.Unmarshal(data, &i); err != nil { 27 | return err 28 | } 29 | v.NullInt64 = i 30 | return nil 31 | } 32 | if x != nil { 33 | v.Valid = true 34 | v.Int64 = *x 35 | } else { 36 | v.Valid = false 37 | } 38 | return nil 39 | } 40 | 41 | type JsonNullString struct { 42 | sql.NullString 43 | } 44 | 45 | func (v JsonNullString) MarshalJSON() ([]byte, error) { 46 | if v.Valid { 47 | return json.Marshal(v.String) 48 | } else { 49 | return json.Marshal(nil) 50 | } 51 | } 52 | 53 | func (v *JsonNullString) UnmarshalJSON(data []byte) error { 54 | var x *string 55 | if err := json.Unmarshal(data, &x); err != nil { 56 | s := sql.NullString{} 57 | if err := json.Unmarshal(data, &s); err != nil { 58 | return err 59 | } 60 | v.NullString = s 61 | return nil 62 | } 63 | if x != nil { 64 | v.Valid = true 65 | v.String = *x 66 | } else { 67 | v.Valid = false 68 | } 69 | return nil 70 | } 71 | 72 | func Quote(name string) string { 73 | return fmt.Sprintf("`%s`", name) 74 | } 75 | 76 | func QuoteMulti(names []string) []string { 77 | res := make([]string, 0, len(names)) 78 | for _, name := range names { 79 | res = append(res, fmt.Sprintf("`%s`", name)) 80 | } 81 | return res 82 | } 83 | 84 | func QuoteString(name string) string { 85 | if strings.Contains(name, "\"") { 86 | name = strings.Replace(name, "\"", "\"\"", -1) 87 | } 88 | return fmt.Sprintf("\"%s\"", name) 89 | } 90 | -------------------------------------------------------------------------------- /cmd/carpenter/command/design.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | 9 | "github.com/codegangsta/cli" 10 | "github.com/dev-cloverlab/carpenter/designer" 11 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 12 | ) 13 | 14 | func CmdDesign(c *cli.Context) { 15 | // Write your code here 16 | dirPath := c.String("dir") 17 | if dirPath == "" { 18 | var err error 19 | dirPath, err = os.Getwd() 20 | if err != nil { 21 | panic(fmt.Errorf("err: os.Getwd failed for reason %s", err)) 22 | } 23 | } 24 | pretty := c.Bool("pretty") 25 | separate := c.Bool("separate") 26 | 27 | tables, err := designer.Export(db, schema) 28 | if err != nil { 29 | panic(fmt.Errorf("err: designer.Export failed for reason %s", err)) 30 | } 31 | 32 | if separate { 33 | for _, table := range tables { 34 | var j []byte 35 | var err error 36 | if pretty { 37 | j, err = json.MarshalIndent(mysql.Tables{table}, "", "\t") 38 | } else { 39 | j, err = json.Marshal(mysql.Tables{table}) 40 | } 41 | if err != nil { 42 | panic(fmt.Errorf("err: json.MarshalIndent is fialed for reason %s", err)) 43 | } 44 | if err := exportJson(dirPath, table.TableName, j); err != nil { 45 | panic(fmt.Errorf("err: exportJson is fialed for reason %s", err)) 46 | } 47 | } 48 | } else { 49 | var j []byte 50 | var err error 51 | if pretty { 52 | j, err = json.MarshalIndent(tables, "", "\t") 53 | } else { 54 | j, err = json.Marshal(tables) 55 | } 56 | if err != nil { 57 | panic(fmt.Errorf("err: json.MarshalIndent is fialed for reason %s", err)) 58 | } 59 | if err := exportJson(dirPath, "tables", j); err != nil { 60 | panic(fmt.Errorf("err: exportJson is fialed for reason %s", err)) 61 | } 62 | } 63 | } 64 | 65 | func exportJson(dirPath, filename string, buf []byte) error { 66 | return ioutil.WriteFile(fmt.Sprintf("%s%s%s.json", dirPath, string(os.PathSeparator), filename), buf, os.ModePerm) 67 | } 68 | -------------------------------------------------------------------------------- /cmd/carpenter/command/export.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "regexp" 9 | "strings" 10 | "sync" 11 | 12 | "github.com/codegangsta/cli" 13 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 14 | "github.com/dev-cloverlab/carpenter/exporter" 15 | ) 16 | 17 | func CmdExport(c *cli.Context) { 18 | // Write your code here 19 | dirPath := c.String("dir") 20 | if dirPath == "" { 21 | var err error 22 | dirPath, err = os.Getwd() 23 | if err != nil { 24 | panic(fmt.Errorf("err: os.Getwd failed for reason %s", err)) 25 | } 26 | } 27 | 28 | re := c.String("regexp") 29 | tableNameRegexp := regexp.MustCompile(re) 30 | tables, err := mysql.GetTables(db, schema) 31 | if err != nil { 32 | panic(fmt.Errorf("err: mysql.GetTables failed for reason %s", err)) 33 | } 34 | 35 | errs := []error{} 36 | errCh := make(chan error) 37 | succeedCh := make(chan string) 38 | doneCh := make(chan bool) 39 | go func() { 40 | for { 41 | select { 42 | case err := <-errCh: 43 | errs = append(errs, err) 44 | case tableName := <-succeedCh: 45 | if verbose { 46 | fmt.Println(tableName) 47 | } 48 | case <-doneCh: 49 | if len(errs) > 0 { 50 | msg := make([]string, 0, len(errs)) 51 | for _, err := range errs { 52 | msg = append(msg, err.Error()) 53 | } 54 | panic(fmt.Errorf("err: %s", strings.Join(msg, "\n"))) 55 | } 56 | return 57 | } 58 | } 59 | }() 60 | 61 | wg := sync.WaitGroup{} 62 | for _, table := range tables { 63 | tableName := table.TableName 64 | if !tableNameRegexp.MatchString(tableName) { 65 | continue 66 | } 67 | wg.Add(1) 68 | go func(d *sql.DB, s, t string) { 69 | defer wg.Done() 70 | csv, err := exporter.Export(d, s, t) 71 | if err != nil { 72 | errCh <- err 73 | return 74 | } 75 | if err := ioutil.WriteFile(fmt.Sprintf("%s%s%s.csv", dirPath, string(os.PathSeparator), t), []byte(csv), os.ModePerm); err != nil { 76 | errCh <- err 77 | return 78 | } 79 | succeedCh <- t 80 | }(db, schema, tableName) 81 | } 82 | wg.Wait() 83 | doneCh <- true 84 | } 85 | -------------------------------------------------------------------------------- /designer/designer_test.go: -------------------------------------------------------------------------------- 1 | package designer 2 | 3 | import ( 4 | "bytes" 5 | "database/sql" 6 | "encoding/json" 7 | "io/ioutil" 8 | "os" 9 | "reflect" 10 | "testing" 11 | 12 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 13 | _ "github.com/go-sql-driver/mysql" 14 | ) 15 | 16 | var ( 17 | db *sql.DB 18 | schema = "carpenter_test" 19 | ) 20 | 21 | func init() { 22 | var err error 23 | db, err = sql.Open("mysql", "root@/") 24 | if err != nil { 25 | panic(err) 26 | } 27 | } 28 | 29 | func TestMain(m *testing.M) { 30 | _, err := db.Exec("CREATE DATABASE IF NOT EXISTS `" + schema + "`") 31 | if err != nil { 32 | panic(err) 33 | } 34 | _, err = db.Exec("USE `" + schema + "`") 35 | if err != nil { 36 | panic(err) 37 | } 38 | code := m.Run() 39 | _, err = db.Exec("drop table if exists `design_test`") 40 | if err != nil { 41 | panic(err) 42 | } 43 | os.Exit(code) 44 | } 45 | 46 | func TestExport(t *testing.T) { 47 | _, err := db.Exec("create table if not exists `design_test` (\n" + 48 | " `id` int(11) unsigned not null auto_increment,\n" + 49 | " `name` varchar(64) not null default'',\n" + 50 | " `email` varchar(255) not null default'',\n" + 51 | " `gender` tinyint(4) not null,\n" + 52 | " `country_code` int(11) not null,\n" + 53 | " `comment` text,\n" + 54 | " `created_at` datetime not null,\n" + 55 | " primary key (`id`),\n" + 56 | " unique key `k1` (`email`),\n" + 57 | " key `k2` (`name`),\n" + 58 | " key `k3` (`gender`,`country_code`)\n" + 59 | ") engine=InnoDB default charset=utf8", 60 | ) 61 | if err != nil { 62 | t.Fatal(err) 63 | } 64 | 65 | buf, err := Export(db, schema, "design_test") 66 | if err != nil { 67 | t.Fatal(err) 68 | } 69 | a, err := json.Marshal(buf) 70 | if err != nil { 71 | t.Error(err) 72 | } 73 | actual := mysql.Tables{} 74 | 75 | decoder := json.NewDecoder(bytes.NewBuffer(a)) 76 | decoder.UseNumber() 77 | if err := decoder.Decode(&actual); err != nil { 78 | t.Error(err) 79 | } 80 | 81 | e, err := ioutil.ReadFile("./_test/expected.json") 82 | if err != nil { 83 | t.Fatal(err) 84 | } 85 | expected := mysql.Tables{} 86 | if err := json.Unmarshal(e, &expected); err != nil { 87 | t.Error(err) 88 | } 89 | 90 | if !reflect.DeepEqual(actual, expected) { 91 | t.Errorf("err: unexpected JSON returned.\nactual: %s\nexpected: %s", a, e) 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /seeder/_test/table1.json: -------------------------------------------------------------------------------- 1 | [{"TableCatalog":"def","TableSchema":"test","TableName":"seed_test","TableType":"BASE TABLE","Engine":"InnoDB","Version":10,"RowFormat":"Dynamic","TableRows":0,"AvgRowLength":0,"DataLength":16384,"MaxDataLength":0,"IndexLength":0,"DataFree":0,"AutoIncrement":{"Int64":0,"Valid":false},"TableCollation":"utf8_general_ci","CheckSum":{"String":"","Valid":false},"CreateOptions":"","TableComment":"","Columns":[{"TableCatalog":"def","TableSchema":"test","TableName":"seed_test","ColumnName":"int","OrdinalPosition":1,"ColumnDefault":{"String":"","Valid":false},"Nullable":"NO","DataType":"int","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":10,"NumericScale":0,"DatatimePrecision":null,"CharacterSetName":null,"CollationName":null,"ColumnType":"int(11)","ColumnKey":"PRI","Extra":"","Privileges":"select,insert,update,references","ColumnComment":"","GenerationExpression":""},{"TableCatalog":"def","TableSchema":"test","TableName":"seed_test","ColumnName":"string","OrdinalPosition":2,"ColumnDefault":{"String":"","Valid":true},"Nullable":"NO","DataType":"varchar","CharacterMaximumLength":255,"CharacterOctetLength":765,"NumericPrecision":null,"NumericScale":null,"DatatimePrecision":null,"CharacterSetName":"utf8","CollationName":"utf8_general_ci","ColumnType":"varchar(255)","ColumnKey":"","Extra":"","Privileges":"select,insert,update,references","ColumnComment":"","GenerationExpression":""},{"TableCatalog":"def","TableSchema":"test","TableName":"seed_test","ColumnName":"time","OrdinalPosition":3,"ColumnDefault":{"String":"","Valid":false},"Nullable":"NO","DataType":"datetime","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":null,"NumericScale":null,"DatatimePrecision":0,"CharacterSetName":null,"CollationName":null,"ColumnType":"datetime","ColumnKey":"","Extra":"","Privileges":"select,insert,update,references","ColumnComment":"","GenerationExpression":""},{"TableCatalog":"def","TableSchema":"test","TableName":"seed_test","ColumnName":"null","OrdinalPosition":4,"ColumnDefault":{"String":"","Valid":false},"Nullable":"YES","DataType":"tinyint","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":3,"NumericScale":0,"DatatimePrecision":null,"CharacterSetName":null,"CollationName":null,"ColumnType":"tinyint(1)","ColumnKey":"","Extra":"","Privileges":"select,insert,update,references","ColumnComment":"","GenerationExpression":""}],"Indices":[[{"Table":"seed_test","NonUniue":0,"KeyName":"PRIMARY","SeqInIndex":1,"ColumnName":"int","Collation":"A","Cardinality":0,"SubPart":null,"Packed":null,"Null":"","IndexType":"BTREE","Comment":"","IndexComment":""}]]}] 2 | -------------------------------------------------------------------------------- /cmd/carpenter/command/build.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "encoding/json" 5 | "fmt" 6 | "io/ioutil" 7 | "strings" 8 | "sync" 9 | 10 | "github.com/codegangsta/cli" 11 | "github.com/dev-cloverlab/carpenter/builder" 12 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 13 | ) 14 | 15 | func CmdBuild(c *cli.Context) { 16 | // Write your code here 17 | dirPath := c.String("dir") 18 | withDrop := c.Bool("with-drop") 19 | queries, errs := makeBuildQueries(dirPath, withDrop) 20 | if len(errs) > 0 { 21 | panic(fmt.Errorf("err: makeQueries failed for reason\n%s", strings.Join(getErrorMessages(errs), "\n"))) 22 | } 23 | if err := execute(queries); err != nil { 24 | panic(fmt.Errorf("err: execute failed for reason %s", err)) 25 | } 26 | } 27 | 28 | func makeBuildQueries(path string, withDrop bool) (queries []string, errs []error) { 29 | files, err := walk(path, ".json") 30 | if err != nil { 31 | return nil, []error{err} 32 | } 33 | new := mysql.Tables{} 34 | for _, file := range files { 35 | tables, err := parseJSON(file[0]) 36 | if err != nil { 37 | return nil, []error{err} 38 | } 39 | new = append(new, tables...) 40 | } 41 | old, err := mysql.GetTables(db, schema) 42 | if err != nil { 43 | return nil, []error{err} 44 | } 45 | errCh := make(chan error) 46 | sqlCh := make(chan []string) 47 | doneCh := make(chan bool) 48 | go func() { 49 | for { 50 | select { 51 | case err := <-errCh: 52 | errs = append(errs, err) 53 | case query := <-sqlCh: 54 | queries = append(queries, query...) 55 | case <-doneCh: 56 | return 57 | } 58 | } 59 | }() 60 | 61 | newMap := new.GroupByTableName() 62 | oldMap := old.GroupByTableName() 63 | tableNames := getTableNames(newMap, oldMap) 64 | wg := &sync.WaitGroup{} 65 | for _, tableName := range tableNames { 66 | oTbl, ok := oldMap[tableName] 67 | if !ok { 68 | oTbl = nil 69 | } 70 | nTbl, ok := newMap[tableName] 71 | if !ok { 72 | nTbl = nil 73 | } 74 | wg.Add(1) 75 | go func(o, n *mysql.Table) { 76 | defer wg.Done() 77 | queries, err := builder.Build(db, o, n, withDrop) 78 | if err != nil { 79 | errCh <- err 80 | return 81 | } 82 | sqlCh <- queries 83 | }(oTbl, nTbl) 84 | } 85 | wg.Wait() 86 | 87 | doneCh <- true 88 | 89 | return queries, errs 90 | } 91 | 92 | func parseJSON(filename string) (mysql.Tables, error) { 93 | buf, err := ioutil.ReadFile(filename) 94 | if err != nil { 95 | return nil, err 96 | } 97 | tables := mysql.Tables{} 98 | if err := json.Unmarshal(buf, &tables); err != nil { 99 | return nil, err 100 | } 101 | return tables, nil 102 | } 103 | 104 | func getTableNames(new, old map[string]*mysql.Table) []string { 105 | tableNames := map[string]struct{}{} 106 | for tableName := range new { 107 | tableNames[tableName] = struct{}{} 108 | } 109 | for tableName := range old { 110 | tableNames[tableName] = struct{}{} 111 | } 112 | ret := make([]string, 0, len(tableNames)) 113 | for name := range tableNames { 114 | ret = append(ret, name) 115 | } 116 | return ret 117 | } 118 | -------------------------------------------------------------------------------- /cmd/carpenter/commands.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | 7 | "github.com/codegangsta/cli" 8 | "github.com/dev-cloverlab/carpenter/cmd/carpenter/command" 9 | ) 10 | 11 | var GlobalFlags = []cli.Flag{ 12 | cli.BoolFlag{ 13 | Name: "verbose, vv", 14 | Hidden: false, 15 | Usage: "show verbose output (default off)", 16 | }, 17 | cli.BoolFlag{ 18 | Name: "dry-run", 19 | Hidden: false, 20 | Usage: "execute as dry-run mode (default off)", 21 | }, 22 | cli.StringFlag{ 23 | Name: "schema, s", 24 | Usage: "database name (required)", 25 | Hidden: false, 26 | }, 27 | cli.StringFlag{ 28 | Name: "data-source, d", 29 | Usage: "data source name like '[username[:password]@][tcp[(address:port)]]' (required)", 30 | Hidden: false, 31 | }, 32 | cli.IntFlag{ 33 | Name: "max-idle-conns, mi", 34 | Usage: "max idel database connection setting", 35 | Hidden: false, 36 | Value: 0, 37 | }, 38 | cli.IntFlag{ 39 | Name: "max-open-conns, mo", 40 | Usage: "max open database connection setting", 41 | Hidden: false, 42 | Value: 8, 43 | }, 44 | } 45 | 46 | var Commands = []cli.Command{ 47 | { 48 | Name: "design", 49 | Usage: "Export table structure as JSON string", 50 | Before: command.Before, 51 | Action: command.CmdDesign, 52 | Flags: []cli.Flag{ 53 | cli.BoolFlag{ 54 | Name: "separate, s", 55 | Usage: "output for each table (default off)", 56 | Hidden: false, 57 | }, 58 | cli.BoolFlag{ 59 | Name: "pretty, p", 60 | Usage: "pretty output (default off)", 61 | Hidden: false, 62 | }, 63 | cli.StringFlag{ 64 | Name: "dir, d", 65 | Usage: "path to export directory (default execution dir)", 66 | Hidden: false, 67 | }, 68 | }, 69 | }, 70 | { 71 | Name: "build", 72 | Usage: "Build(Migrate) table from specified JSON string", 73 | Before: command.Before, 74 | Action: command.CmdBuild, 75 | Flags: []cli.Flag{ 76 | cli.StringFlag{ 77 | Name: "dir, d", 78 | Usage: "path to JSON file directory (required)", 79 | Hidden: false, 80 | }, 81 | cli.BoolFlag{ 82 | Name: "with-drop", 83 | Usage: "drop table when if JSON file does not exist", 84 | Hidden: false, 85 | }, 86 | }, 87 | }, 88 | { 89 | Name: "import", 90 | Usage: "Import CSV to table", 91 | Before: command.Before, 92 | Action: command.CmdSeed, 93 | Flags: []cli.Flag{ 94 | cli.StringFlag{ 95 | Name: "dir, d", 96 | Usage: "path to CSV file directory (required)", 97 | Hidden: false, 98 | }, 99 | cli.BoolFlag{ 100 | Name: "ignore-foreign-key, i", 101 | Usage: "ignore foreign key check", 102 | Hidden: false, 103 | }, 104 | }, 105 | }, 106 | { 107 | Name: "export", 108 | Usage: "Export CSV to table", 109 | Before: command.Before, 110 | Action: command.CmdExport, 111 | Flags: []cli.Flag{ 112 | cli.StringFlag{ 113 | Name: "dir, d", 114 | Usage: "path to export directory (default execution dir)", 115 | Hidden: false, 116 | }, 117 | cli.StringFlag{ 118 | Name: "regexp, r", 119 | Usage: "regular expression for exporting table (default all)", 120 | Hidden: false, 121 | }, 122 | }, 123 | }, 124 | } 125 | 126 | func CommandNotFound(c *cli.Context, command string) { 127 | fmt.Fprintf(os.Stderr, "%s: '%s' is not a %s command. See '%s --help'.", c.App.Name, command, c.App.Name, c.App.Name) 128 | os.Exit(2) 129 | } 130 | -------------------------------------------------------------------------------- /dialect/mysql/partition.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "strings" 7 | ) 8 | 9 | type ( 10 | Partition struct { 11 | TableCatalog string 12 | TableSchema string 13 | TableName string 14 | PartitionName string 15 | SubpartitionName JsonNullString 16 | PartitionOrdinalPosition string 17 | SubpartitionOrdinalPosition JsonNullString 18 | PartitionMethod string 19 | SubpartitionMethod JsonNullString 20 | PartitionExpression string 21 | SubpartitionExpression JsonNullString 22 | PartitionDescription JsonNullString 23 | PartitionComment string 24 | Nodegroup string 25 | TablespaceName JsonNullString 26 | } 27 | Partitions []*Partition 28 | ) 29 | 30 | const ( 31 | PartitionMethodLinearKey = "LINEAR KEY" 32 | PartitionMethodLinearHash = "LINEAR HASH" 33 | PartitionMethodRange = "RANGE COLUMNS" 34 | ) 35 | 36 | func (m Partitions) ToSQL() string { 37 | if m == nil { 38 | return "" 39 | } 40 | // only for support "LINEAR KEY" or "LINEAR HASH" or "RANGE COLUMNS" partition method 41 | switch m[0].PartitionMethod { 42 | case PartitionMethodLinearKey: 43 | return fmt.Sprintf("partition by linear key (%s) partitions %d", m[0].PartitionExpression, len(m)) 44 | case PartitionMethodLinearHash: 45 | return fmt.Sprintf("partition by linear hash (%s) partitions %d", m[0].PartitionExpression, len(m)) 46 | case PartitionMethodRange: 47 | sqls := make([]string, 0, len(m)) 48 | for _, partition := range m { 49 | sqls = append(sqls, fmt.Sprintf("\t\tpartition %s values less than (%s)", partition.PartitionName, partition.PartitionDescription.String)) 50 | } 51 | return fmt.Sprintf("partition by range columns (%s) (\n%s\n\t)", m[0].PartitionExpression, strings.Join(sqls, ",\n")) 52 | default: 53 | return "" 54 | } 55 | } 56 | 57 | func GetPartitions(db *sql.DB, schema string, tableName string) (Partitions, error) { 58 | var rows *sql.Rows 59 | var err error 60 | 61 | selectCols := []string{ 62 | "TABLE_CATALOG", 63 | "TABLE_NAME", 64 | "PARTITION_NAME", 65 | "SUBPARTITION_NAME", 66 | "PARTITION_ORDINAL_POSITION", 67 | "SUBPARTITION_ORDINAL_POSITION", 68 | "PARTITION_METHOD", 69 | "SUBPARTITION_METHOD", 70 | "PARTITION_EXPRESSION", 71 | "SUBPARTITION_EXPRESSION", 72 | "PARTITION_DESCRIPTION", 73 | "PARTITION_COMMENT", 74 | "NODEGROUP", 75 | "TABLESPACE_NAME", 76 | } 77 | query := fmt.Sprintf(`select %s from information_schema.partitions where TABLE_SCHEMA=%s and TABLE_NAME=%s`, strings.Join(selectCols, ","), QuoteString(schema), QuoteString(tableName)) 78 | rows, err = db.Query(query) 79 | if err != nil { 80 | return nil, fmt.Errorf("err: db.Query `%s' failed for reason %s", query, err) 81 | } 82 | defer rows.Close() 83 | 84 | partitions := Partitions{} 85 | for rows.Next() { 86 | partition := &Partition{} 87 | if err := rows.Scan( 88 | &partition.TableCatalog, 89 | &partition.TableName, 90 | &partition.PartitionName, 91 | &partition.SubpartitionName, 92 | &partition.PartitionOrdinalPosition, 93 | &partition.SubpartitionOrdinalPosition, 94 | &partition.PartitionMethod, 95 | &partition.SubpartitionMethod, 96 | &partition.PartitionExpression, 97 | &partition.SubpartitionExpression, 98 | &partition.PartitionDescription, 99 | &partition.PartitionComment, 100 | &partition.Nodegroup, 101 | &partition.TablespaceName, 102 | ); err != nil { 103 | return nil, err 104 | } 105 | partitions = append(partitions, partition) 106 | } 107 | 108 | return partitions, nil 109 | } 110 | -------------------------------------------------------------------------------- /seeder/seeder.go: -------------------------------------------------------------------------------- 1 | package seeder 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "reflect" 7 | 8 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 9 | ) 10 | 11 | var ( 12 | defaultComparisonColumnName string = "id" 13 | ) 14 | 15 | func Seed(db *sql.DB, old, new *mysql.Chunk, compColName *string) (queries []string, err error) { 16 | if old == nil && new == nil { 17 | return []string{}, fmt.Errorf("err: Both pointer of the specified new and old is nil.") 18 | } 19 | if reflect.DeepEqual(old, new) { 20 | return queries, nil 21 | } 22 | ccName := defaultComparisonColumnName 23 | if compColName != nil { 24 | ccName = *compColName 25 | } 26 | if q := willTruncate(old, new); len(q) > 0 { 27 | queries = append(queries, q) 28 | } else { 29 | if q, err := willDelete(old, new, ccName); err != nil { 30 | return []string{}, err 31 | } else if len(q) > 0 { 32 | queries = append(queries, q...) 33 | } 34 | } 35 | if q, err := willReplace(old, new, ccName); err != nil { 36 | return []string{}, err 37 | } else if len(q) > 0 { 38 | queries = append(queries, q...) 39 | } 40 | if q, err := willInsert(old, new, ccName); err != nil { 41 | return []string{}, err 42 | } else if len(q) > 0 { 43 | queries = append(queries, q...) 44 | } 45 | return queries, nil 46 | } 47 | 48 | func willTruncate(old, new *mysql.Chunk) string { 49 | if len(old.Seeds) != 0 && len(new.Seeds) <= 0 { 50 | return old.ToTrancateSQL() 51 | } 52 | return "" 53 | } 54 | 55 | func willInsert(old, new *mysql.Chunk, compColName string) ([]string, error) { 56 | if new == nil { 57 | return []string{}, nil 58 | } 59 | if old == nil && new != nil { 60 | return new.ToInsertSQL(), nil 61 | } 62 | oldMap, err := old.GetSeedGroupBy(compColName) 63 | if err != nil { 64 | return nil, err 65 | } 66 | newMap, err := new.GetSeedGroupBy(compColName) 67 | if err != nil { 68 | return nil, err 69 | } 70 | seeds := mysql.Seeds{} 71 | for k, seed := range newMap { 72 | if _, ok := oldMap[k]; ok { 73 | continue 74 | } 75 | seeds = append(seeds, seed) 76 | } 77 | cnk := mysql.Chunk{ 78 | TableName: new.TableName, 79 | ColumnNames: new.ColumnNames, 80 | Seeds: seeds, 81 | } 82 | return cnk.ToInsertSQL(), nil 83 | } 84 | 85 | func willDelete(old, new *mysql.Chunk, compColName string) ([]string, error) { 86 | if old == nil { 87 | return []string{}, nil 88 | } 89 | if old != nil && new == nil { 90 | return []string{old.ToTrancateSQL()}, nil 91 | } 92 | oldMap, err := old.GetSeedGroupBy(compColName) 93 | if err != nil { 94 | return nil, err 95 | } 96 | newMap, err := new.GetSeedGroupBy(compColName) 97 | if err != nil { 98 | return nil, err 99 | } 100 | seeds := mysql.Seeds{} 101 | for k, seed := range oldMap { 102 | if _, ok := newMap[k]; ok { 103 | continue 104 | } 105 | seeds = append(seeds, seed) 106 | } 107 | cnk := mysql.Chunk{ 108 | TableName: new.TableName, 109 | ColumnNames: new.ColumnNames, 110 | Seeds: seeds, 111 | } 112 | colIdx, err := cnk.GetColumnIndexBy(compColName) 113 | if err != nil { 114 | return nil, err 115 | } 116 | return cnk.ToDeleteSQL(colIdx), nil 117 | } 118 | 119 | func willReplace(old, new *mysql.Chunk, compColName string) ([]string, error) { 120 | if old == nil || new == nil { 121 | return []string{}, nil 122 | } 123 | oldMap, err := old.GetSeedGroupBy(compColName) 124 | if err != nil { 125 | return nil, err 126 | } 127 | newMap, err := new.GetSeedGroupBy(compColName) 128 | if err != nil { 129 | return nil, err 130 | } 131 | seeds := mysql.Seeds{} 132 | for k, seed := range newMap { 133 | if _, ok := oldMap[k]; !ok { 134 | continue 135 | } 136 | if newMap[k].ValueEqual(oldMap[k]) { 137 | continue 138 | } 139 | seeds = append(seeds, seed) 140 | } 141 | cnk := mysql.Chunk{ 142 | TableName: new.TableName, 143 | ColumnNames: new.ColumnNames, 144 | Seeds: seeds, 145 | } 146 | return cnk.ToReplaceSQL(), nil 147 | } 148 | -------------------------------------------------------------------------------- /cmd/carpenter/command/seed.go: -------------------------------------------------------------------------------- 1 | package command 2 | 3 | import ( 4 | "encoding/csv" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strconv" 9 | "strings" 10 | "sync" 11 | 12 | "github.com/codegangsta/cli" 13 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 14 | "github.com/dev-cloverlab/carpenter/seeder" 15 | ) 16 | 17 | func CmdSeed(c *cli.Context) { 18 | // Write your code here 19 | dirPath := c.String("dir") 20 | queries, errs := makeSeedQueries(dirPath, nil) 21 | if len(errs) > 0 { 22 | panic(fmt.Errorf("err: makeSeedQueries failed for reason\n%s", strings.Join(getErrorMessages(errs), "\n"))) 23 | } 24 | 25 | ignoreForeignKey := c.Bool("ignore-foreign-key") 26 | if ignoreForeignKey { 27 | queries = append([]string{mysql.ForeignKeyCheck(false)}, queries...) 28 | 29 | defer func() { 30 | if err := execute([]string{mysql.ForeignKeyCheck(true)}); err != nil { 31 | panic(fmt.Errorf("err: execute failed for reason %s", err)) 32 | } 33 | }() 34 | } 35 | 36 | if err := execute(queries); err != nil { 37 | panic(fmt.Errorf("err: execute failed for reason %s", err)) 38 | } 39 | } 40 | 41 | func makeSeedQueries(path string, colName *string) (queries []string, errs []error) { 42 | files, err := walk(path, ".csv") 43 | if err != nil { 44 | return nil, []error{err} 45 | } 46 | 47 | errCh := make(chan error) 48 | sqlCh := make(chan []string) 49 | doneCh := make(chan bool) 50 | go func() { 51 | for { 52 | select { 53 | case err := <-errCh: 54 | errs = append(errs, err) 55 | case query := <-sqlCh: 56 | queries = append(queries, query...) 57 | case <-doneCh: 58 | return 59 | } 60 | } 61 | }() 62 | 63 | wg := &sync.WaitGroup{} 64 | for tableName, file := range files { 65 | wg.Add(1) 66 | go func(t string, fs []string, c *string) { 67 | defer wg.Done() 68 | 69 | var colNames []string 70 | var err error 71 | seeds := mysql.Seeds{} 72 | for _, f := range fs { 73 | var s mysql.Seeds 74 | colNames, s, err = parseCSV(t, f) 75 | if err != nil { 76 | errCh <- fmt.Errorf("err: parseCSV %s failed for reason %s", t, err) 77 | return 78 | } 79 | seeds = append(seeds, s...) 80 | } 81 | new := makeChunk(t, colNames, seeds) 82 | old, err := mysql.GetChunk(db, t, c) 83 | if err != nil { 84 | errCh <- fmt.Errorf("err: mysql.GetChunk %s failed for reason %s", t, err) 85 | return 86 | } 87 | queries, err := seeder.Seed(db, old, new, c) 88 | if err != nil { 89 | errCh <- fmt.Errorf("err: seeder.Seed %s failed for reason %s", t, err) 90 | } 91 | sqlCh <- queries 92 | }(tableName, file, colName) 93 | } 94 | wg.Wait() 95 | 96 | doneCh <- true 97 | 98 | return queries, errs 99 | } 100 | 101 | func parseCSV(tableName, filename string) (columnNames []string, seeds mysql.Seeds, err error) { 102 | fp, err := os.Open(filename) 103 | if err != nil { 104 | return nil, nil, err 105 | } 106 | defer fp.Close() 107 | 108 | reader := csv.NewReader(fp) 109 | reader.LazyQuotes = true 110 | 111 | for { 112 | record, err := reader.Read() 113 | if err == io.EOF { 114 | break 115 | } 116 | if err != nil { 117 | return nil, nil, err 118 | } 119 | if len(columnNames) <= 0 { 120 | columnNames = record 121 | } else { 122 | columnData := make([]interface{}, 0, len(record)) 123 | for _, r := range record { 124 | var v interface{} 125 | var err error 126 | if r == "NULL" || r == "null" || r == "Null" { 127 | v = nil 128 | } else if v, err = strconv.ParseFloat(r, 64); err != nil { 129 | v = r 130 | } else { 131 | if len(string(r)) > 1 && string(string(r)[0]) == "0" { 132 | v = string(r) 133 | } 134 | } 135 | columnData = append(columnData, v) 136 | } 137 | seeds = append(seeds, mysql.Seed{ 138 | ColumnData: columnData, 139 | }) 140 | } 141 | } 142 | return columnNames, seeds, nil 143 | } 144 | 145 | func makeChunk(tableName string, columnNames []string, seeds mysql.Seeds) *mysql.Chunk { 146 | return &mysql.Chunk{ 147 | TableName: tableName, 148 | ColumnNames: columnNames, 149 | Seeds: seeds, 150 | } 151 | } 152 | -------------------------------------------------------------------------------- /designer/_test/expected.json: -------------------------------------------------------------------------------- 1 | [{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","TableType":"BASE TABLE","Engine":"InnoDB","Version":10,"RowFormat":"Dynamic","TableCollation":"utf8_general_ci","CheckSum":null,"CreateOptions":"","TableComment":"","Columns":[{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"id","OrdinalPosition":1,"ColumnDefault":null,"Nullable":"NO","DataType":"int","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":10,"NumericScale":0,"CharacterSetName":null,"CollationName":null,"ColumnType":"int(11) unsigned","ColumnKey":"PRI","Extra":"auto_increment","Privileges":"select,insert,update,references","ColumnComment":""},{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"name","OrdinalPosition":2,"ColumnDefault":"","Nullable":"NO","DataType":"varchar","CharacterMaximumLength":64,"CharacterOctetLength":192,"NumericPrecision":null,"NumericScale":null,"CharacterSetName":"utf8","CollationName":"utf8_general_ci","ColumnType":"varchar(64)","ColumnKey":"MUL","Extra":"","Privileges":"select,insert,update,references","ColumnComment":""},{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"email","OrdinalPosition":3,"ColumnDefault":"","Nullable":"NO","DataType":"varchar","CharacterMaximumLength":255,"CharacterOctetLength":765,"NumericPrecision":null,"NumericScale":null,"CharacterSetName":"utf8","CollationName":"utf8_general_ci","ColumnType":"varchar(255)","ColumnKey":"UNI","Extra":"","Privileges":"select,insert,update,references","ColumnComment":""},{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"gender","OrdinalPosition":4,"ColumnDefault":null,"Nullable":"NO","DataType":"tinyint","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":3,"NumericScale":0,"CharacterSetName":null,"CollationName":null,"ColumnType":"tinyint(4)","ColumnKey":"MUL","Extra":"","Privileges":"select,insert,update,references","ColumnComment":""},{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"country_code","OrdinalPosition":5,"ColumnDefault":null,"Nullable":"NO","DataType":"int","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":10,"NumericScale":0,"CharacterSetName":null,"CollationName":null,"ColumnType":"int(11)","ColumnKey":"","Extra":"","Privileges":"select,insert,update,references","ColumnComment":""},{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"comment","OrdinalPosition":6,"ColumnDefault":null,"Nullable":"YES","DataType":"text","CharacterMaximumLength":65535,"CharacterOctetLength":65535,"NumericPrecision":null,"NumericScale":null,"CharacterSetName":"utf8","CollationName":"utf8_general_ci","ColumnType":"text","ColumnKey":"","Extra":"","Privileges":"select,insert,update,references","ColumnComment":""},{"TableCatalog":"def","TableSchema":"carpenter_test","TableName":"design_test","ColumnName":"created_at","OrdinalPosition":7,"ColumnDefault":null,"Nullable":"NO","DataType":"datetime","CharacterMaximumLength":null,"CharacterOctetLength":null,"NumericPrecision":null,"NumericScale":null,"CharacterSetName":null,"CollationName":null,"ColumnType":"datetime","ColumnKey":"","Extra":"","Privileges":"select,insert,update,references","ColumnComment":""}],"Indices":[[{"Table":"design_test","NonUniue":0,"KeyName":"PRIMARY","SeqInIndex":1,"ColumnName":"id","Collation":"A","SubPart":null,"Packed":null,"Null":"","IndexType":"BTREE","Comment":"","IndexComment":""}],[{"Table":"design_test","NonUniue":0,"KeyName":"k1","SeqInIndex":1,"ColumnName":"email","Collation":"A","SubPart":null,"Packed":null,"Null":"","IndexType":"BTREE","Comment":"","IndexComment":""}],[{"Table":"design_test","NonUniue":1,"KeyName":"k2","SeqInIndex":1,"ColumnName":"name","Collation":"A","SubPart":null,"Packed":null,"Null":"","IndexType":"BTREE","Comment":"","IndexComment":""}],[{"Table":"design_test","NonUniue":1,"KeyName":"k3","SeqInIndex":1,"ColumnName":"gender","Collation":"A","SubPart":null,"Packed":null,"Null":"","IndexType":"BTREE","Comment":"","IndexComment":""},{"Table":"design_test","NonUniue":1,"KeyName":"k3","SeqInIndex":2,"ColumnName":"country_code","Collation":"A","SubPart":null,"Packed":null,"Null":"","IndexType":"BTREE","Comment":"","IndexComment":""}]],"Partitions":null}] 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # carpenter 2 | 3 | [![GitHub release](https://img.shields.io/github/release/dev-cloverlab/carpenter.svg?style=flat-square)](https://github.com/dev-cloverlab/carpenter) 4 | [![license](https://img.shields.io/github/license/dev-cloverlab/carpenter.svg?style=flat-square)](https://github.com/dev-cloverlab/carpenter) 5 | 6 | carpenter is a tool to manage DB schema and data inspired by [naoina/migu](https://github.com/naoina/migu). 7 | By using this, you can manage the database structures and data as text (JSON, CSV).  8 | carpenter can restore the database structure and data from text, or can export them to text in easy. 9 | 10 | **supported databases are MySQL|MariaDB only currently** 11 | 12 | ## Install 13 | 14 | ``` 15 | % brew tap dev-cloverlab/carpenter 16 | % brew install carpenter 17 | ``` 18 | 19 | for Gophers. 20 | 21 | ``` 22 | % go get -u github.com/dev-cloverlab/carpenter 23 | ``` 24 | 25 | # How to use 26 | 27 | carpenter has four simple commands are classified database `structure` and `data`. For each command is also available to use as indivisually. 28 | 29 | ## Commands for structure 30 | 31 | ### design 32 | 33 | `design` command can export database structure as JSON. By doing below, exports JSON file named `table.json` to current directory. 34 | 35 | ``` 36 | % carpenter -s test -d "root:@tcp(127.0.0.1:3306)" design -d ./ 37 | ``` 38 | 39 | When you want to separate files for each tables, you can set `-s` option. 40 | 41 | options: 42 | 43 | - `-s` export JSON files are separated for each table (default off) 44 | - `-p` pretty output (default off) 45 | - `-d` export directory path 46 | 47 | Each option has alternative long name. Please see the help for details. 48 | 49 | ### build 50 | 51 | `build` command can restore database structure from JSON files. By doing below, generate the difference SQLs between tables and JSON files and execute them. 52 | 53 | ``` 54 | % carpenter -s test -d "root:@tcp(127.0.0.1:3306)" build -d . 55 | ``` 56 | 57 | When you want to just show the generated SQLs, you can set `--dry-run` global option. 58 | 59 | ## Commands for data 60 | 61 | ### export 62 | 63 | `export` command can export data as CSV files. By doing below, export data as CSV files for each table. 64 | 65 | ``` 66 | % carpenter -s test -d "root:@tcp(127.0.0.1:3306)" export -d . 67 | ``` 68 | 69 | When you want to select exporting tables, you can set regular expression to `-r` option like below. 70 | 71 | ``` 72 | % carpenter -s test -d "root:@tcp(127.0.0.1:3306)" export -r "^master_*$" -d . 73 | ``` 74 | 75 | ### import 76 | 77 | `import` command can import CSV files to tables. By doing below, generate the difference SQLs between tables and CSV files and execute them. 78 | 79 | ``` 80 | % carpenter -s test -d "root:@tcp(127.0.0.1:3306)" import -d . 81 | ``` 82 | 83 | When you want to just show the generated SQLs, you can set `--dry-run` global option. 84 | 85 | ## Architecture 86 | 87 | Explain how carpenter syncronizes text and database. 88 | 89 | MySQL(MariaDB) has information table that has table, column, index and partition information. carpenter refers that and translate it to JSON, and unmarshal it to struct. Both of structs that are made from database and files can compare each field type and etc. When some difference are found for each field, carpenter generates SQLs for resolve differences. 90 | 91 | For example: 92 | 93 | ``` 94 | // about member table 95 | 96 | // database 97 | +-------+--------------+------+ 98 | | Field | Type | Null | 99 | +-------+--------------+------+ 100 | | name | varchar(255) | NO | 101 | | email | varchar(255) | NO | 102 | +-------+--------------+------+ 103 | 104 | // file 105 | +--------+--------------+------+ 106 | | Field | Type | Null | 107 | +--------+--------------+------+ 108 | | name | varchar(255) | NO | 109 | | email | varchar(255) | NO | 110 | | gender | tinyint(4) | NO | 111 | +--------+--------------+------+ 112 | ``` 113 | 114 | To generate this. 115 | 116 | ```sql 117 | alter table `member` add `gender` tinyint(4) not null after `email` 118 | ``` 119 | 120 | carpenter can generate SQLs at various scenes like: 121 | 122 | - CREATE 123 | - DROP 124 | - ALTER 125 | - INSERT 126 | - REPLACE 127 | - DELETE 128 | 129 | These SQLs are generated by difference of both information structs. 130 | 131 | ## Contribution 132 | 133 | 1. Fork ([https://github.com/dev-cloverlab/carpenter/fork](https://github.com/dev-cloverlab/carpenter/fork)) 134 | 1. Create a feature branch 135 | 1. Commit your changes 136 | 1. Rebase your local changes against the master branch 137 | 1. Run test suite with the `go test ./...` command and confirm that it passes 138 | 1. Run `gofmt -s` 139 | 1. Create a new Pull Request 140 | 141 | ## Author 142 | 143 | [@hatajoe](https://twitter.com/hatajoe) 144 | 145 | ## Licence 146 | 147 | MIT 148 | -------------------------------------------------------------------------------- /dialect/mysql/table.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "sort" 7 | "strings" 8 | ) 9 | 10 | type ( 11 | Table struct { 12 | TableCatalog string 13 | TableSchema string 14 | TableName string 15 | TableType string 16 | Engine string 17 | Version int 18 | RowFormat string 19 | TableCollation string 20 | CheckSum JsonNullString 21 | CreateOptions string 22 | TableComment string 23 | Columns Columns 24 | Indices Indices 25 | Partitions Partitions 26 | } 27 | Tables []*Table 28 | ) 29 | 30 | var ( 31 | createSQLFmt string = `create table if not exists %s ( 32 | %s 33 | ) engine=%s default charset=%s %s` 34 | dropSQLFmt string = `drop table if exists %s` 35 | alterSQLFmt string = `alter table %s %s 36 | %s` 37 | ) 38 | 39 | func (m *Table) IsPartitioned() bool { 40 | return m.CreateOptions == "partitioned" 41 | } 42 | 43 | func (m *Table) GetFormatedTableName() string { 44 | return Quote(m.TableName) 45 | } 46 | 47 | func (m *Table) GetCharset() string { 48 | seg := strings.Split(m.TableCollation, "_") 49 | return seg[0] 50 | } 51 | 52 | func (m *Table) ToAlterSQL(sqls []string, partitionSql string) string { 53 | if len(sqls) <= 0 { 54 | return "" 55 | } 56 | return fmt.Sprintf(alterSQLFmt, m.GetFormatedTableName(), strings.Join(sqls, ",\n "), partitionSql) 57 | } 58 | 59 | func (m Tables) GetFormatedTableNames() []string { 60 | names := make([]string, 0, len(m)) 61 | for _, table := range m { 62 | names = append(names, table.GetFormatedTableName()) 63 | } 64 | return names 65 | } 66 | 67 | func (m *Table) ToCreateSQL() string { 68 | columnSQLs := m.Columns.ToSQL() 69 | indexSQLs := m.Indices.ToSQL() 70 | partitionSQL := m.Partitions.ToSQL() 71 | sqls := make([]string, 0, len(columnSQLs)+len(indexSQLs)) 72 | sqls = append(columnSQLs, indexSQLs...) 73 | return fmt.Sprintf(createSQLFmt, m.GetFormatedTableName(), strings.Join(sqls, ",\n "), m.Engine, m.GetCharset(), partitionSQL) 74 | } 75 | 76 | func (m *Table) ToDropSQL() string { 77 | return fmt.Sprintf(dropSQLFmt, m.GetFormatedTableName()) 78 | } 79 | 80 | func (m *Table) ToConvertCharsetSQL() string { 81 | return fmt.Sprintf("convert to character set %s", m.GetCharset()) 82 | } 83 | 84 | func (m Tables) Contains(t *Table) bool { 85 | for _, v := range m { 86 | if v.TableName == t.TableName { 87 | return true 88 | } 89 | } 90 | return false 91 | } 92 | 93 | func (m Tables) GroupByTableName() map[string]*Table { 94 | nameMap := make(map[string]*Table, len(m)) 95 | for _, table := range m { 96 | nameMap[table.TableName] = table 97 | } 98 | return nameMap 99 | } 100 | 101 | func (m Tables) GetSortedTableNames() []string { 102 | names := make([]string, 0, len(m)) 103 | for _, table := range m { 104 | names = append(names, table.TableName) 105 | } 106 | sort.Strings(names) 107 | return names 108 | } 109 | 110 | func GetTables(db *sql.DB, schema string, tableNames ...string) (Tables, error) { 111 | var rows *sql.Rows 112 | var err error 113 | 114 | selectCols := []string{ 115 | "TABLE_CATALOG", 116 | "TABLE_SCHEMA", 117 | "TABLE_NAME", 118 | "TABLE_TYPE", 119 | "ENGINE", 120 | "VERSION", 121 | "ROW_FORMAT", 122 | "TABLE_COLLATION", 123 | "CHECKSUM", 124 | "CREATE_OPTIONS", 125 | "TABLE_COMMENT", 126 | } 127 | query := fmt.Sprintf(`select %s from information_schema.tables where TABLE_SCHEMA=%s`, strings.Join(selectCols, ","), QuoteString(schema)) 128 | if len(tableNames) > 0 { 129 | tn := make([]string, 0, len(tableNames)) 130 | for _, t := range tableNames { 131 | tn = append(tn, QuoteString(t)) 132 | } 133 | query = fmt.Sprintf(`select %s from information_schema.tables where TABLE_SCHEMA=%s and TABLE_NAME in (%s)`, strings.Join(selectCols, ","), QuoteString(schema), strings.Join(tn, ",")) 134 | } 135 | rows, err = db.Query(query) 136 | if err != nil { 137 | return nil, fmt.Errorf("err: db.Query `%s' failed for reason %s", query, err) 138 | } 139 | defer rows.Close() 140 | 141 | tables := []*Table{} 142 | for rows.Next() { 143 | table := &Table{} 144 | if err := rows.Scan( 145 | &table.TableCatalog, 146 | &table.TableSchema, 147 | &table.TableName, 148 | &table.TableType, 149 | &table.Engine, 150 | &table.Version, 151 | &table.RowFormat, 152 | &table.TableCollation, 153 | &table.CheckSum, 154 | &table.CreateOptions, 155 | &table.TableComment, 156 | ); err != nil { 157 | return nil, err 158 | } 159 | tables = append(tables, table) 160 | } 161 | columns, err := GetColumns(db, schema) 162 | if err != nil { 163 | return nil, err 164 | } 165 | for i, table := range tables { 166 | indices, err := GetIndices(db, table.TableName) 167 | if err != nil { 168 | return nil, err 169 | } 170 | c := []*Column{} 171 | for _, v := range columns { 172 | if table.TableName != v.TableName { 173 | continue 174 | } 175 | c = append(c, v) 176 | } 177 | if table.IsPartitioned() { 178 | partitions, err := GetPartitions(db, table.TableSchema, table.TableName) 179 | if err != nil { 180 | return nil, err 181 | } 182 | tables[i].Partitions = partitions 183 | } 184 | tables[i].Columns = c 185 | tables[i].Indices = indices 186 | } 187 | return tables, nil 188 | } 189 | -------------------------------------------------------------------------------- /dialect/mysql/index.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "sort" 7 | "strings" 8 | ) 9 | 10 | type ( 11 | IndexColumn struct { 12 | Table string 13 | NonUniue int8 14 | KeyName string 15 | SeqInIndex int32 16 | ColumnName string 17 | Collation string 18 | SubPart JsonNullString 19 | Packed JsonNullString 20 | Null string 21 | IndexType string 22 | Comment string 23 | IndexComment string 24 | } 25 | Index []IndexColumn 26 | Indices []Index 27 | ) 28 | 29 | func (m Index) IsPrimaryKey() bool { 30 | return m[0].KeyName == "PRIMARY" 31 | } 32 | 33 | func (m Index) IsUniqueKey() bool { 34 | return m[0].NonUniue != 1 35 | } 36 | 37 | func (m Index) GetKeyName() string { 38 | return m[0].KeyName 39 | } 40 | 41 | func (m Index) ColumnNames() []string { 42 | names := make([]string, 0, len(m)) 43 | for _, info := range m { 44 | names = append(names, Quote(info.ColumnName)) 45 | } 46 | return names 47 | } 48 | 49 | func (m Index) KeyNamesWithSubPart() []string { 50 | names := make([]string, 0, len(m)) 51 | for _, info := range m { 52 | if info.SubPart.Valid { 53 | names = append(names, fmt.Sprintf("%s(%s)", Quote(info.ColumnName), info.SubPart.String)) 54 | } else { 55 | names = append(names, Quote(info.ColumnName)) 56 | } 57 | } 58 | return names 59 | } 60 | 61 | func (m Indices) ToSQL() []string { 62 | indices := m.getSortedIndices(m.GetSortedKeys()) 63 | indexSQLs := make([]string, 0, len(m)) 64 | for _, index := range indices { 65 | sql := "" 66 | comment := index[0].Comment 67 | switch { 68 | case index.IsPrimaryKey(): 69 | sql = fmt.Sprintf("primary key (%s)", strings.Join(index.ColumnNames(), ",")) 70 | case !index.IsPrimaryKey() && index.IsUniqueKey(): 71 | sql = fmt.Sprintf("unique key %s (%s)", Quote(index.GetKeyName()), strings.Join(index.ColumnNames(), ",")) 72 | default: 73 | sql = fmt.Sprintf("key %s (%s)", Quote(index.GetKeyName()), strings.Join(index.KeyNamesWithSubPart(), ",")) 74 | } 75 | if comment != "" { 76 | sql = fmt.Sprintf("%s comment %s", sql, QuoteString(comment)) 77 | } 78 | indexSQLs = append(indexSQLs, sql) 79 | } 80 | return indexSQLs 81 | } 82 | 83 | func (m Indices) ToAddSQL() []string { 84 | sqls := m.ToSQL() 85 | for i, sql := range sqls { 86 | sqls[i] = fmt.Sprintf("add %s", sql) 87 | } 88 | return sqls 89 | } 90 | 91 | func (m Indices) ToDropSQL() []string { 92 | idxMap := map[string]struct{}{} 93 | for _, index := range m { 94 | idxMap[index.GetKeyName()] = struct{}{} 95 | } 96 | sqls := make([]string, 0, len(idxMap)) 97 | for keyName := range idxMap { 98 | sqls = append(sqls, fmt.Sprintf("drop key %s", Quote(keyName))) 99 | } 100 | return sqls 101 | } 102 | 103 | func (m Indices) GroupByKeyName() map[string]Indices { 104 | nameMap := make(map[string]Indices, len(m)) 105 | for _, index := range m { 106 | if _, ok := nameMap[index.GetKeyName()]; !ok { 107 | nameMap[index.GetKeyName()] = Indices{} 108 | } 109 | nameMap[index.GetKeyName()] = append(nameMap[index.GetKeyName()], index) 110 | } 111 | return nameMap 112 | } 113 | 114 | func (m Indices) GetSortedKeys() []string { 115 | keys := make([]string, 0, len(m)) 116 | for _, index := range m { 117 | keys = append(keys, index.GetKeyName()) 118 | } 119 | sort.Strings(keys) 120 | return keys 121 | } 122 | 123 | func (m Indices) getSortedIndices(keys []string) Indices { 124 | idxMap := map[string]Index{} 125 | for _, index := range m { 126 | idxMap[index.GetKeyName()] = index 127 | } 128 | indices := make(Indices, 0, len(idxMap)) 129 | for _, key := range keys { 130 | index := idxMap[key] 131 | if index.IsPrimaryKey() { 132 | indices = append(indices, index) 133 | } 134 | } 135 | for _, key := range keys { 136 | index := idxMap[key] 137 | if !index.IsPrimaryKey() && index.IsUniqueKey() { 138 | indices = append(indices, index) 139 | } 140 | } 141 | for _, key := range keys { 142 | index := idxMap[key] 143 | if !index.IsPrimaryKey() && !index.IsUniqueKey() { 144 | indices = append(indices, index) 145 | } 146 | } 147 | return indices 148 | } 149 | 150 | func GetIndices(db *sql.DB, table string) (Indices, error) { 151 | query := fmt.Sprintf("show index from `%s`", table) 152 | rows, err := db.Query(query) 153 | if err != nil { 154 | return nil, fmt.Errorf("err: db.Query faild `%s' for reason %s", query, err) 155 | } 156 | defer rows.Close() 157 | 158 | dummy := JsonNullInt64{} 159 | idxMap := map[string]Index{} 160 | for rows.Next() { 161 | idxCol := IndexColumn{} 162 | if err := rows.Scan( 163 | &idxCol.Table, 164 | &idxCol.NonUniue, 165 | &idxCol.KeyName, 166 | &idxCol.SeqInIndex, 167 | &idxCol.ColumnName, 168 | &idxCol.Collation, 169 | &dummy, 170 | &idxCol.SubPart, 171 | &idxCol.Packed, 172 | &idxCol.Null, 173 | &idxCol.IndexType, 174 | &idxCol.IndexComment, 175 | &idxCol.Comment, 176 | ); err != nil { 177 | return nil, err 178 | } 179 | if _, ok := idxMap[idxCol.KeyName]; !ok { 180 | idxMap[idxCol.KeyName] = Index{} 181 | } 182 | idxMap[idxCol.KeyName] = append(idxMap[idxCol.KeyName], idxCol) 183 | } 184 | indices := make(Indices, 0, len(idxMap)) 185 | for _, index := range idxMap { 186 | indices = append(indices, index) 187 | } 188 | return indices.getSortedIndices(indices.GetSortedKeys()), nil 189 | } 190 | -------------------------------------------------------------------------------- /seeder/seeder_test.go: -------------------------------------------------------------------------------- 1 | package seeder 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "reflect" 10 | "testing" 11 | "time" 12 | 13 | "github.com/dev-cloverlab/carpenter/builder" 14 | "github.com/dev-cloverlab/carpenter/dialect/mysql" 15 | _ "github.com/go-sql-driver/mysql" 16 | ) 17 | 18 | var ( 19 | db *sql.DB 20 | schema = "carpenter_test" 21 | ) 22 | 23 | func init() { 24 | var err error 25 | db, err = sql.Open("mysql", "root@/") 26 | if err != nil { 27 | panic(err) 28 | } 29 | } 30 | 31 | func TestMain(m *testing.M) { 32 | _, err := db.Exec("CREATE DATABASE IF NOT EXISTS `" + schema + "`") 33 | if err != nil { 34 | panic(err) 35 | } 36 | _, err = db.Exec("USE `" + schema + "`") 37 | if err != nil { 38 | panic(err) 39 | } 40 | new, err := getTables("./_test/table1.json") 41 | if err != nil { 42 | panic(err) 43 | } 44 | createSQL, err := builder.Build(db, nil, new[0], true) 45 | if err != nil { 46 | panic(err) 47 | } 48 | for _, sql := range createSQL { 49 | if _, err := db.Exec(sql); err != nil { 50 | panic(err) 51 | } 52 | } 53 | code := m.Run() 54 | _, err = db.Exec("drop table if exists `seed_test`") 55 | if err != nil { 56 | panic(err) 57 | } 58 | os.Exit(code) 59 | } 60 | 61 | func TestInsert(t *testing.T) { 62 | colName := "int" 63 | oldChunk, err := mysql.GetChunk(db, "seed_test", &colName) 64 | if err != nil { 65 | t.Fatal(err) 66 | } 67 | now := time.Now().Format(mysql.TimeFmt) 68 | newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{ 69 | makeSeed([]interface{}{float64(10), "stringA", now, nil}), 70 | makeSeed([]interface{}{float64(20), "stringB", now, nil}), 71 | }) 72 | 73 | expected := []string{ 74 | "insert into `seed_test`(`int`,`string`,`time`,`null`)\n" + 75 | "values\n" + 76 | fmt.Sprintf("(10,\"stringA\",\"%v\",null),\n", now) + 77 | fmt.Sprintf("(20,\"stringB\",\"%v\",null)", now), 78 | } 79 | compString := "int" 80 | actual, err := Seed(db, oldChunk, newChunk, &compString) 81 | if err != nil { 82 | t.Fatal(err) 83 | } 84 | if !reflect.DeepEqual(actual, expected) { 85 | t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected) 86 | } 87 | for _, sql := range actual { 88 | if _, err := db.Exec(sql); err != nil { 89 | t.Fatal(err) 90 | } 91 | } 92 | } 93 | 94 | func TestReplace(t *testing.T) { 95 | colName := "int" 96 | oldChunk, err := mysql.GetChunk(db, "seed_test", &colName) 97 | if err != nil { 98 | t.Fatal(err) 99 | } 100 | now := time.Now().Format(mysql.TimeFmt) 101 | newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{ 102 | makeSeed([]interface{}{float64(10), "stringC", now, nil}), 103 | makeSeed([]interface{}{float64(20), "stringB", now, nil}), 104 | }) 105 | 106 | expected := []string{ 107 | "replace into `seed_test`(`int`,`string`,`time`,`null`)\n" + 108 | "values\n" + 109 | fmt.Sprintf("(10,\"stringC\",\"%v\",null)", now), 110 | } 111 | compString := "int" 112 | actual, err := Seed(db, oldChunk, newChunk, &compString) 113 | if err != nil { 114 | t.Fatal(err) 115 | } 116 | if !reflect.DeepEqual(actual, expected) { 117 | t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected) 118 | } 119 | for _, sql := range actual { 120 | if _, err := db.Exec(sql); err != nil { 121 | t.Fatal(err) 122 | } 123 | } 124 | } 125 | 126 | func TestDelete(t *testing.T) { 127 | colName := "int" 128 | oldChunk, err := mysql.GetChunk(db, "seed_test", &colName) 129 | if err != nil { 130 | t.Fatal(err) 131 | } 132 | now := time.Now().Format(mysql.TimeFmt) 133 | newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{ 134 | makeSeed([]interface{}{float64(20), "stringB", now, nil}), 135 | }) 136 | 137 | expected := []string{ 138 | "delete from `seed_test` where `int` in (\n" + 139 | "10\n" + 140 | ")", 141 | } 142 | compString := "int" 143 | actual, err := Seed(db, oldChunk, newChunk, &compString) 144 | if err != nil { 145 | t.Fatal(err) 146 | } 147 | if !reflect.DeepEqual(actual, expected) { 148 | t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected) 149 | } 150 | for _, sql := range actual { 151 | if _, err := db.Exec(sql); err != nil { 152 | t.Fatal(err) 153 | } 154 | } 155 | } 156 | 157 | func TestTruncate(t *testing.T) { 158 | colName := "int" 159 | oldChunk, err := mysql.GetChunk(db, "seed_test", &colName) 160 | if err != nil { 161 | t.Fatal(err) 162 | } 163 | newChunk := makeChunk("seed_test", oldChunk.ColumnNames, mysql.Seeds{}) 164 | 165 | expected := []string{ 166 | "truncate table `seed_test`", 167 | } 168 | compString := "int" 169 | actual, err := Seed(db, oldChunk, newChunk, &compString) 170 | if err != nil { 171 | t.Fatal(err) 172 | } 173 | if !reflect.DeepEqual(actual, expected) { 174 | t.Fatalf("err: create: unexpected SQL returned.\nactual:\n%s\nexpected:\n%s\n", actual, expected) 175 | } 176 | for _, sql := range actual { 177 | if _, err := db.Exec(sql); err != nil { 178 | t.Fatal(err) 179 | } 180 | } 181 | } 182 | 183 | func makeSeed(columnData []interface{}) mysql.Seed { 184 | return mysql.Seed{ 185 | ColumnData: columnData, 186 | } 187 | } 188 | 189 | func makeChunk(tableName string, columnNames []string, seeds mysql.Seeds) *mysql.Chunk { 190 | return &mysql.Chunk{ 191 | TableName: tableName, 192 | ColumnNames: columnNames, 193 | Seeds: seeds, 194 | } 195 | } 196 | 197 | func getTables(filename string) (mysql.Tables, error) { 198 | buf, err := ioutil.ReadFile(filename) 199 | if err != nil { 200 | return nil, err 201 | } 202 | tables := mysql.Tables{} 203 | if err := json.Unmarshal(buf, &tables); err != nil { 204 | return nil, err 205 | } 206 | return tables, nil 207 | } 208 | -------------------------------------------------------------------------------- /dialect/mysql/seed.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "encoding/json" 6 | "fmt" 7 | "strconv" 8 | "strings" 9 | "time" 10 | ) 11 | 12 | type ( 13 | Chunk struct { 14 | TableName string 15 | ColumnNames []string 16 | Seeds Seeds 17 | } 18 | 19 | Seeds []Seed 20 | 21 | Seed struct { 22 | ColumnData []interface{} 23 | } 24 | ) 25 | 26 | var ( 27 | TimeFmt string = "2006-01-02 15:04:05" 28 | defaultBulkSize int = 2000 29 | trancateSQLFmt string = `truncate table %s` 30 | insertSQLFmt string = `insert into %s(%s) 31 | values 32 | %s` 33 | deleteSQLFmt string = `delete from %s where %s in ( 34 | %s 35 | )` 36 | replaceSQLFmt string = `replace into %s(%s) 37 | values 38 | %s` 39 | ) 40 | 41 | func (m *Chunk) GetColumnIndexBy(columnName string) (int, error) { 42 | for i, cName := range m.ColumnNames { 43 | if columnName == cName { 44 | return i, nil 45 | } 46 | } 47 | return 0, fmt.Errorf("err: Specified columnName `%s' is not found in this table %s", columnName, m.TableName) 48 | } 49 | 50 | func (m *Chunk) GetSeedGroupBy(columnName string) (map[interface{}]Seed, error) { 51 | colIdx, err := m.GetColumnIndexBy(columnName) 52 | if err != nil { 53 | return nil, err 54 | } 55 | cdMap := make(map[interface{}]Seed, len(m.Seeds)) 56 | for _, seed := range m.Seeds { 57 | if len(seed.ColumnData) <= 0 { 58 | continue 59 | } 60 | cdMap[seed.ColumnData[colIdx]] = seed 61 | } 62 | return cdMap, nil 63 | } 64 | 65 | func (m *Chunk) GetFormatedTableName() string { 66 | return Quote(m.TableName) 67 | } 68 | 69 | func (m *Chunk) ToTrancateSQL() string { 70 | return fmt.Sprintf(trancateSQLFmt, m.GetFormatedTableName()) 71 | } 72 | 73 | func (m *Chunk) ToInsertSQL() []string { 74 | columnStr := strings.Join(QuoteMulti(m.ColumnNames), ",") 75 | 76 | seedSize := defaultBulkSize 77 | if seedSize > len(m.Seeds) { 78 | seedSize = len(m.Seeds) 79 | } 80 | queries := make([]string, 0, int(len(m.Seeds)/defaultBulkSize)+1) 81 | seeds := Seeds{} 82 | for _, seed := range m.Seeds { 83 | seeds = append(seeds, seed) 84 | if len(seeds) >= defaultBulkSize { 85 | queries = append(queries, fmt.Sprintf(insertSQLFmt, m.GetFormatedTableName(), columnStr, strings.Join(seeds.ToValueSQL(), ",\n"))) 86 | seeds = Seeds{} 87 | } 88 | } 89 | if len(seeds) > 0 { 90 | queries = append(queries, fmt.Sprintf(insertSQLFmt, m.GetFormatedTableName(), columnStr, strings.Join(seeds.ToValueSQL(), ",\n"))) 91 | } 92 | return queries 93 | } 94 | 95 | func (m *Chunk) ToDeleteSQL(colIdx int) []string { 96 | columnStr := Quote(m.ColumnNames[colIdx]) 97 | seedSize := defaultBulkSize 98 | if seedSize > len(m.Seeds) { 99 | seedSize = len(m.Seeds) 100 | } 101 | queries := make([]string, 0, int(len(m.Seeds)/defaultBulkSize)+1) 102 | seeds := Seeds{} 103 | for _, seed := range m.Seeds { 104 | seeds = append(seeds, seed) 105 | if len(seeds) >= defaultBulkSize { 106 | queries = append(queries, fmt.Sprintf(deleteSQLFmt, m.GetFormatedTableName(), columnStr, strings.Join(seeds.ToColumnValues(colIdx), ",\n"))) 107 | seeds = Seeds{} 108 | } 109 | } 110 | if len(seeds) > 0 { 111 | queries = append(queries, fmt.Sprintf(deleteSQLFmt, m.GetFormatedTableName(), columnStr, strings.Join(seeds.ToColumnValues(colIdx), ",\n"))) 112 | } 113 | return queries 114 | } 115 | 116 | func (m *Chunk) ToReplaceSQL() []string { 117 | columnStr := strings.Join(QuoteMulti(m.ColumnNames), ",") 118 | 119 | seedSize := defaultBulkSize 120 | if seedSize > len(m.Seeds) { 121 | seedSize = len(m.Seeds) 122 | } 123 | queries := make([]string, 0, int(len(m.Seeds)/defaultBulkSize)+1) 124 | seeds := Seeds{} 125 | for _, seed := range m.Seeds { 126 | seeds = append(seeds, seed) 127 | if len(seeds) >= defaultBulkSize { 128 | queries = append(queries, fmt.Sprintf(replaceSQLFmt, m.GetFormatedTableName(), columnStr, strings.Join(seeds.ToValueSQL(), ","))) 129 | seeds = Seeds{} 130 | } 131 | } 132 | if len(seeds) > 0 { 133 | queries = append(queries, fmt.Sprintf(replaceSQLFmt, m.GetFormatedTableName(), columnStr, strings.Join(seeds.ToValueSQL(), ","))) 134 | } 135 | return queries 136 | } 137 | 138 | func (m Seeds) ToValueSQL() []string { 139 | sqls := make([]string, 0, len(m)) 140 | for _, seed := range m { 141 | sqls = append(sqls, fmt.Sprintf("(%s)", seed.ToValueSQL())) 142 | } 143 | return sqls 144 | } 145 | 146 | func (m Seed) ToValueSQL() string { 147 | str := make([]string, 0, len(m.ColumnData)) 148 | for _, data := range m.ColumnData { 149 | str = append(str, toString(data)) 150 | } 151 | return strings.Join(str, ",") 152 | } 153 | 154 | func (m Seeds) ToColumnValues(colIdx int) []string { 155 | values := make([]string, 0, len(m)) 156 | for _, seed := range m { 157 | values = append(values, seed.ToColumnValue(colIdx)) 158 | } 159 | return values 160 | } 161 | 162 | func (m Seed) ToColumnValue(colIdx int) string { 163 | return toString(m.ColumnData[colIdx]) 164 | } 165 | 166 | func (m Seed) ValueEqual(seed Seed) bool { 167 | cnt := len(m.ColumnData) 168 | for i := 0; i < cnt; i++ { 169 | if toString(m.ColumnData[i]) != toString(seed.ColumnData[i]) { 170 | return false 171 | } 172 | } 173 | return true 174 | } 175 | 176 | func toString(data interface{}) (str string) { 177 | switch data.(type) { 178 | case nil: 179 | str = "null" 180 | case string: 181 | str = QuoteString(data.(string)) 182 | case time.Time: 183 | str = QuoteString(data.(time.Time).Format(TimeFmt)) 184 | default: 185 | str = fmt.Sprintf("%v", data) 186 | } 187 | return str 188 | } 189 | 190 | func GetChunk(db *sql.DB, table string, colName *string) (*Chunk, error) { 191 | cntCol := "*" 192 | if colName != nil { 193 | cntCol = Quote(*colName) 194 | } 195 | res, err := db.Exec(fmt.Sprintf("select count(%s) from %s", cntCol, Quote(table))) 196 | if err != nil { 197 | return nil, err 198 | } 199 | cnt, err := res.RowsAffected() 200 | if err != nil { 201 | return nil, err 202 | } 203 | rows, err := db.Query(fmt.Sprintf("select * from %s", table)) 204 | if err != nil { 205 | return nil, err 206 | } 207 | defer rows.Close() 208 | columns, err := rows.Columns() 209 | if err != nil { 210 | return nil, err 211 | } 212 | cnk := &Chunk{ 213 | TableName: table, 214 | ColumnNames: columns, 215 | Seeds: make(Seeds, 0, cnt), 216 | } 217 | colLen := len(columns) 218 | holderPtrs := make([]interface{}, colLen) 219 | for rows.Next() { 220 | holders := make([]interface{}, colLen) 221 | for i := range columns { 222 | holderPtrs[i] = &holders[i] 223 | } 224 | if err := rows.Scan(holderPtrs...); err != nil { 225 | return nil, err 226 | } 227 | for i := range columns { 228 | var v interface{} 229 | var err error 230 | if b, ok := holders[i].([]byte); ok { 231 | if v, err = strconv.ParseFloat(string(json.Number(string(b))), 64); err != nil { 232 | v = string(b) 233 | if v == "0000-00-00 00:00:00" || v == "0000-00-00" { 234 | v = "" 235 | } 236 | } else { 237 | if len(string(b)) > 1 && string(string(b)[0]) == "0" { 238 | v = string(b) 239 | } 240 | } 241 | } else { 242 | v = holders[i] 243 | } 244 | holders[i] = v 245 | } 246 | cnk.Seeds = append(cnk.Seeds, Seed{ 247 | ColumnData: holders, 248 | }) 249 | } 250 | return cnk, nil 251 | } 252 | -------------------------------------------------------------------------------- /dialect/mysql/column.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "reflect" 7 | "sort" 8 | "strings" 9 | "regexp" 10 | ) 11 | 12 | type ( 13 | Column struct { 14 | TableCatalog string 15 | TableSchema string 16 | TableName string 17 | ColumnName string 18 | OrdinalPosition int32 19 | ColumnDefault JsonNullString 20 | Nullable string 21 | DataType string 22 | CharacterMaximumLength JsonNullInt64 23 | CharacterOctetLength JsonNullInt64 24 | NumericPrecision JsonNullInt64 25 | NumericScale JsonNullInt64 26 | CharacterSetName JsonNullString 27 | CollationName JsonNullString 28 | ColumnType string 29 | ColumnKey string 30 | Extra JsonNullString 31 | Privileges string 32 | ColumnComment string 33 | } 34 | Columns []*Column 35 | ) 36 | 37 | func (m *Column) IsPrimary() bool { 38 | return m.ColumnKey == "PRI" 39 | } 40 | 41 | func (m *Column) IsUnique() bool { 42 | return m.ColumnKey == "UNI" 43 | } 44 | 45 | func (m *Column) IsMul() bool { 46 | return m.ColumnKey == "MUL" 47 | } 48 | 49 | func (m *Column) IsNullable() bool { 50 | return m.Nullable == "YES" 51 | } 52 | 53 | func (m *Column) HasDefault() bool { 54 | return m.ColumnDefault.Valid 55 | } 56 | 57 | func (m *Column) HasExtra() bool { 58 | return m.Extra.Valid 59 | } 60 | 61 | func (m *Column) HasCharacterSetName() bool { 62 | return m.CharacterSetName.Valid 63 | } 64 | 65 | func (m *Column) HasComment() bool { 66 | return m.ColumnComment != "" 67 | } 68 | 69 | func (m *Column) HasDefaultValueNull() bool { 70 | if r, err := regexp.MatchString(`(?i)null`, m.ColumnDefault.String); !r || err != nil { 71 | return false 72 | } 73 | return true 74 | } 75 | 76 | func (m *Column) FormatDefault() string { 77 | if m.HasDefaultValueNull() { 78 | return m.ColumnDefault.String 79 | } 80 | 81 | var def string 82 | switch m.DataType { 83 | case "char", "varchar", "tinyblob", "blob", "mediumblob", "longblob", "tinytext", "text", "mediumtext", "longtext", "date": 84 | def = QuoteString(m.ColumnDefault.String) 85 | case "datetime": 86 | def = QuoteString(m.ColumnDefault.String) 87 | if m.ColumnDefault.String == "CURRENT_TIMESTAMP" { 88 | def = m.ColumnDefault.String 89 | } 90 | default: 91 | def = m.ColumnDefault.String 92 | } 93 | return def 94 | } 95 | 96 | func (m *Column) FormatExtra() string { 97 | return m.Extra.String 98 | } 99 | 100 | func (m *Column) CompareCharacterSet(col *Column) bool { 101 | return reflect.DeepEqual(m.CharacterSetName, col.CharacterSetName) 102 | } 103 | 104 | func (m *Column) CompareCollation(col *Column) bool { 105 | return reflect.DeepEqual(m.CollationName, col.CollationName) 106 | } 107 | 108 | func (m *Column) ToSQL() string { 109 | token := []string{Quote(m.ColumnName), m.ColumnType} 110 | if !m.IsNullable() { 111 | token = append(token, "not null") 112 | } 113 | if m.HasDefault() { 114 | token = append(token, "default", m.FormatDefault()) 115 | } 116 | if m.HasExtra() { 117 | token = append(token, m.FormatExtra()) 118 | } 119 | if m.HasComment() { 120 | token = append(token, "comment", QuoteString(m.ColumnComment)) 121 | } 122 | return strings.Join(token, " ") 123 | } 124 | 125 | func (m *Column) ToAddSQL(pos string) string { 126 | return fmt.Sprintf("add %s %s", m.ToSQL(), pos) 127 | } 128 | 129 | func (m *Column) ToDropSQL() string { 130 | return fmt.Sprintf("drop %s", Quote(m.ColumnName)) 131 | } 132 | 133 | func (m *Column) ToModifySQL() string { 134 | return fmt.Sprintf("modify %s", m.ToSQL()) 135 | } 136 | 137 | func (m *Column) ToModifyCharsetSQL() string { 138 | return fmt.Sprintf("modify %s %s %s", Quote(m.ColumnName), m.ColumnType, fmt.Sprintf("character set %s collate %s", m.CharacterSetName.String, m.CollationName.String)) 139 | } 140 | 141 | func (m Columns) ToSQL() []string { 142 | sqls := make([]string, 0, len(m)) 143 | for _, col := range m { 144 | sqls = append(sqls, col.ToSQL()) 145 | } 146 | return sqls 147 | } 148 | 149 | func (m *Column) AppendPos(all Columns) string { 150 | if n := all.GetBeforeColumn(m); n != nil { 151 | return fmt.Sprintf("after %s", Quote(n.ColumnName)) 152 | } 153 | return "first" 154 | } 155 | 156 | func (m Columns) GetBeforeColumn(col *Column) *Column { 157 | search := col.OrdinalPosition - 1 158 | for _, c := range m { 159 | if c.OrdinalPosition == search { 160 | return c 161 | } 162 | } 163 | return nil 164 | } 165 | 166 | func (m Columns) ToAddSQL(all Columns) []string { 167 | sqls := make([]string, 0, len(m)) 168 | for _, col := range m { 169 | sqls = append(sqls, col.ToAddSQL(col.AppendPos(all))) 170 | } 171 | return sqls 172 | } 173 | 174 | func (m Columns) ToDropSQL() []string { 175 | sqls := make([]string, 0, len(m)) 176 | for _, col := range m { 177 | sqls = append(sqls, col.ToDropSQL()) 178 | } 179 | return sqls 180 | } 181 | 182 | func (m Columns) Contains(c *Column) bool { 183 | for _, v := range m { 184 | if v.ColumnName == c.ColumnName { 185 | return true 186 | } 187 | } 188 | return false 189 | } 190 | 191 | func (m Columns) GroupByColumnName() map[string]*Column { 192 | nameMap := make(map[string]*Column, len(m)) 193 | for _, column := range m { 194 | nameMap[column.ColumnName] = column 195 | } 196 | return nameMap 197 | } 198 | 199 | func (m Columns) GetSortedColumnNames() []string { 200 | names := make([]string, 0, len(m)) 201 | sort.Slice(m, func(i, j int) bool { 202 | return m[i].OrdinalPosition < m[j].OrdinalPosition 203 | }) 204 | for _, column := range m { 205 | names = append(names, column.ColumnName) 206 | } 207 | return names 208 | } 209 | 210 | func GetColumns(db *sql.DB, schema string) ([]*Column, error) { 211 | selectCols := []string{ 212 | "TABLE_CATALOG", 213 | "TABLE_SCHEMA", 214 | "TABLE_NAME", 215 | "COLUMN_NAME", 216 | "ORDINAL_POSITION", 217 | "COLUMN_DEFAULT", 218 | "IS_NULLABLE", 219 | "DATA_TYPE", 220 | "CHARACTER_MAXIMUM_LENGTH", 221 | "CHARACTER_OCTET_LENGTH", 222 | "NUMERIC_PRECISION", 223 | "NUMERIC_SCALE", 224 | "CHARACTER_SET_NAME", 225 | "COLLATION_NAME", 226 | "COLUMN_TYPE", 227 | "COLUMN_KEY", 228 | "EXTRA", 229 | "PRIVILEGES", 230 | "COLUMN_COMMENT", 231 | } 232 | query := fmt.Sprintf(`select %s from information_schema.columns where TABLE_SCHEMA="%s"`, strings.Join(selectCols, ","), schema) 233 | rows, err := db.Query(query) 234 | if err != nil { 235 | return nil, fmt.Errorf("err: db.Query failed `%s' for reason %s", query, err) 236 | } 237 | defer rows.Close() 238 | 239 | columns := []*Column{} 240 | for rows.Next() { 241 | column := &Column{} 242 | if err := rows.Scan( 243 | &column.TableCatalog, 244 | &column.TableSchema, 245 | &column.TableName, 246 | &column.ColumnName, 247 | &column.OrdinalPosition, 248 | &column.ColumnDefault, 249 | &column.Nullable, 250 | &column.DataType, 251 | &column.CharacterMaximumLength, 252 | &column.CharacterOctetLength, 253 | &column.NumericPrecision, 254 | &column.NumericScale, 255 | &column.CharacterSetName, 256 | &column.CollationName, 257 | &column.ColumnType, 258 | &column.ColumnKey, 259 | &column.Extra, 260 | &column.Privileges, 261 | &column.ColumnComment, 262 | ); err != nil { 263 | return nil, err 264 | } 265 | columns = append(columns, column) 266 | } 267 | return columns, nil 268 | } 269 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.6.0 (2018-07-05) 2 | 3 | ### Added 4 | 5 | - drop table only for if JSON file exist (as a default) 6 | 7 | ### Deprecated 8 | 9 | - Nothing 10 | 11 | ### Removed 12 | 13 | - Nothing 14 | 15 | ### Fixed 16 | 17 | - Nothing 18 | 19 | 20 | ## 0.5.4 (2018-02-15) 21 | 22 | ### Added 23 | 24 | - Added two global command line options which related database connection 25 | - see also the discuss: #32 26 | - --max-idle-conns 27 | - --max-open-conns 28 | 29 | ### Deprecated 30 | 31 | - Nothing 32 | 33 | ### Removed 34 | 35 | - Nothing 36 | 37 | ### Fixed 38 | 39 | - Nothing 40 | 41 | 42 | ## 0.5.3 (2018-01-13) 43 | 44 | ### Added 45 | 46 | - Nothing 47 | 48 | ### Deprecated 49 | 50 | - Nothing 51 | 52 | ### Removed 53 | 54 | - Nothing 55 | 56 | ### Fixed 57 | 58 | - Fixed the importing bug that when column contains 0 value at the beginning of the string 59 | 60 | 61 | ## 0.5.2 (2018-01-09) 62 | 63 | ### Added 64 | 65 | - Nothing 66 | 67 | ### Deprecated 68 | 69 | - Nothing 70 | 71 | ### Removed 72 | 73 | - Nothing 74 | 75 | ### Fixed 76 | 77 | - Fixed the exporting bug that when column contains 0 value at the beginning of the string 78 | 79 | ## 0.5.1 (2017-11-19) 80 | 81 | Carpenter 0.5.1 has been released with some fixes. 82 | 83 | ### Added 84 | 85 | - Nothing 86 | 87 | ### Deprecated 88 | 89 | - Nothing 90 | 91 | ### Removed 92 | 93 | - Nothing 94 | 95 | ### Fixed 96 | 97 | - Solved to escape value that contained double quote 98 | - Fixed the wrong time format that was 0000-00-00 when zero value 99 | 100 | ## 0.5.0 (2017-11-14) 101 | 102 | Carpenter 0.5.0 has been released. 103 | 104 | Note: 105 | This version has to become impossible to modify column positions. 106 | But adding column is adjusted to expecting position. 107 | 108 | ### Added 109 | 110 | - Nothing 111 | 112 | ### Deprecated 113 | 114 | - Nothing 115 | 116 | ### Removed 117 | 118 | - It's to become impossible to modify column positions when alter columns. 119 | 120 | ### Fixed 121 | 122 | - Fixed that added column position to expected 123 | 124 | 125 | ## 0.4.9 (2017-11-10) 126 | 127 | ### Added 128 | 129 | - Modified MySQL connection settings (no idle connections) 130 | 131 | ### Deprecated 132 | 133 | - Nothing 134 | 135 | ### Removed 136 | 137 | - Nothing 138 | 139 | ### Fixed 140 | 141 | - Fixed the SQL syntax error caused by column collation comparing 142 | 143 | 144 | ## 0.4.8 (2017-10-05) 145 | 146 | ### Added 147 | 148 | - Nothing 149 | 150 | ### Deprecated 151 | 152 | - Nothing 153 | 154 | ### Removed 155 | 156 | - Nothing 157 | 158 | ### Fixed 159 | 160 | - Fixed the SQL syntax error when change column data type varchar to any 161 | 162 | 163 | ## 0.4.7 (2017-09-22) 164 | 165 | ### Added 166 | 167 | - Support Extra field of column information (e.g, auto_increment, ON UPDATE CURRENT_TIMESTAMP) 168 | 169 | ### Deprecated 170 | 171 | - Nothing 172 | 173 | ### Removed 174 | 175 | - Nothing 176 | 177 | ### Fixed 178 | 179 | - Fixed that "CURRENT_TIMESTAMP" behaves as string 180 | 181 | 182 | ## 0.4.6 (2017-07-13) 183 | 184 | Bug fix 185 | 186 | ### Added 187 | 188 | - Nothing 189 | 190 | ### Deprecated 191 | 192 | - Nothing 193 | 194 | ### Removed 195 | 196 | - Nothing 197 | 198 | ### Fixed 199 | 200 | - Fixed the bug that unexpected differences ocurrured 201 | 202 | ## 0.4.5 (2017-07-07) 203 | 204 | Bug fix 205 | 206 | ### Added 207 | 208 | - Nothing 209 | 210 | ### Deprecated 211 | 212 | - Nothing 213 | 214 | ### Removed 215 | 216 | - Nothing 217 | 218 | ### Fixed 219 | 220 | - Fixed a bug that syntax error occurred when SQL contains partition queries 221 | 222 | ## 0.4.4 (2017-03-29) 223 | 224 | Bug fix 225 | 226 | ### Added 227 | 228 | - Nothing 229 | 230 | ### Deprecated 231 | 232 | - Nothing 233 | 234 | ### Removed 235 | 236 | - Nothing 237 | 238 | ### Fixed 239 | 240 | - Fixed a bug that will get a difference when contains some partition information table columns 241 | 242 | ## 0.4.3 (2017-03-13) 243 | 244 | Support partition table 245 | 246 | ### Added 247 | 248 | - support partition table 249 | - only "LINEAR KEY", "LINEAR HASH" and "RANGE COLUMNS" are supported 250 | - supported alter only. drop and remove is not supported 251 | 252 | ### Deprecated 253 | 254 | - Nothing 255 | 256 | ### Removed 257 | 258 | - Nothing 259 | 260 | ### Fixed 261 | 262 | - Fixed a bug that will get a difference when table names contain uppercase letters 263 | 264 | ## 0.4.2 (2017-02-10) 265 | 266 | Minor feature released 267 | 268 | ### Added 269 | 270 | - resolve column position modification 271 | 272 | ### Deprecated 273 | 274 | - Nothing 275 | 276 | ### Removed 277 | 278 | - Nothing 279 | 280 | ### Fixed 281 | 282 | - Fix SEGV error 283 | 284 | 285 | ## 0.4.1 (2017-02-08) 286 | 287 | Add migrate table collation 288 | 289 | ### Added 290 | 291 | - build 292 | - sync table and column character set 293 | 294 | ### Deprecated 295 | 296 | - Nothing 297 | 298 | ### Removed 299 | 300 | - Nothing 301 | 302 | ### Fixed 303 | 304 | - Nothing 305 | 306 | 307 | ## 0.4.0 (2017-01-27) 308 | 309 | Revert version 310 | 311 | ### Added 312 | 313 | - Nothing 314 | 315 | ### Deprecated 316 | 317 | - Nothing 318 | 319 | ### Removed 320 | 321 | - revert 0.2.4 322 | 323 | ### Fixed 324 | 325 | - Nothing 326 | 327 | 328 | ## 0.3.1 (2016-12-12) 329 | 330 | Bug fix 331 | 332 | ### Added 333 | 334 | - Nothing 335 | 336 | ### Deprecated 337 | 338 | - Nothing 339 | 340 | ### Removed 341 | 342 | - design command: STDOUT output now removed 343 | 344 | ### Fixed 345 | 346 | - Enables specification of design option `-s` 347 | 348 | ## 0.3.0 (2016-12-12) 349 | 350 | - design command: Change export format 351 | - design command: Add `-s` option 352 | - Fix test 353 | - Fix bug 354 | 355 | ### Added 356 | 357 | - Add separate option to design command 358 | - Change STDOUT output to files for each tables (if `-s` option specified) 359 | 360 | ### Deprecated 361 | 362 | - Nothing 363 | 364 | ### Removed 365 | 366 | - design command: STDOUT output now removed 367 | 368 | ### Fixed 369 | 370 | - Type translation bug fixed 371 | - Test 372 | 373 | 374 | ## 0.2.6 (2016-12-02) 375 | 376 | - Bug Fix 377 | 378 | ### Added 379 | 380 | - Nothing 381 | 382 | ### Deprecated 383 | 384 | - Nothing 385 | 386 | ### Removed 387 | 388 | - Nothing 389 | 390 | ### Fixed 391 | 392 | - Support bigint export 393 | 394 | 395 | ## 0.2.5 (2016-12-01) 396 | 397 | - Bug Fix 398 | 399 | ### Added 400 | 401 | - Nothing 402 | 403 | ### Deprecated 404 | 405 | - Nothing 406 | 407 | ### Removed 408 | 409 | - Nothing 410 | 411 | ### Fixed 412 | 413 | - Support index comment 414 | 415 | 416 | ## 0.2.4 (2016-11-29) 417 | 418 | - Bug Fix 419 | 420 | ### Added 421 | 422 | - Nothing 423 | 424 | ### Deprecated 425 | 426 | - Nothing 427 | 428 | ### Removed 429 | 430 | - Nothing 431 | 432 | ### Fixed 433 | 434 | - Unescape double escape string 435 | 436 | 437 | ## 0.2.3 (2016-11-28) 438 | 439 | - Bug Fix 440 | 441 | ### Added 442 | 443 | - Nothing 444 | 445 | ### Deprecated 446 | 447 | - Nothing 448 | 449 | ### Removed 450 | 451 | - Nothing 452 | 453 | ### Fixed 454 | 455 | - Do not compare privileges 456 | 457 | 458 | ## 0.2.2 (2016-11-28) 459 | 460 | - Bug Fix 461 | 462 | ### Added 463 | 464 | - Nothing 465 | 466 | ### Deprecated 467 | 468 | - Nothing 469 | 470 | ### Removed 471 | 472 | - Nothing 473 | 474 | ### Fixed 475 | 476 | - Fix error when using MySQL5.5 477 | 478 | 479 | ## 0.2.1 (2016-11-26) 480 | 481 | - Bug Fix 482 | 483 | ### Added 484 | 485 | - Nothing 486 | 487 | ### Deprecated 488 | 489 | - Nothing 490 | 491 | ### Removed 492 | 493 | - Nothing 494 | 495 | ### Fixed 496 | 497 | - Always a difference between schemas with different names 498 | 499 | 500 | ## 0.2.0 (2016-11-21) 501 | 502 | - Add export subcommand 503 | - Rename subcommand export to design 504 | 505 | ### Added 506 | 507 | - Added subcommand to export table data as CSV 508 | 509 | ### Deprecated 510 | 511 | - Nothing 512 | 513 | ### Removed 514 | 515 | - Nothing 516 | 517 | ### Fixed 518 | 519 | - Adjust the maximum number of MySQL connections 520 | 521 | 522 | ## 0.1.1 (2016-11-15) 523 | 524 | Bug fixed 525 | 526 | ### Added 527 | 528 | - Nothing 529 | 530 | ### Deprecated 531 | 532 | - Nothing 533 | 534 | ### Removed 535 | 536 | - Nothing 537 | 538 | ### Fixed 539 | 540 | - Quote the default value that needed to be quoted 541 | 542 | ## 0.1.0 (2016-11-15) 543 | 544 | Initial release 545 | 546 | ### Added 547 | 548 | - Add Fundamental features 549 | 550 | ### Deprecated 551 | 552 | - Nothing 553 | 554 | ### Removed 555 | 556 | - Nothing 557 | 558 | ### Fixed 559 | 560 | - Nothing 561 | --------------------------------------------------------------------------------