├── .gitignore ├── .idea ├── vcs.xml ├── modules.xml ├── db-backuper.iml └── workspace.xml ├── main.go ├── README.md ├── go.mod ├── config.example.yaml ├── tool └── tool.go ├── config ├── struct.go └── config.go ├── db └── db.go ├── backup └── cos.go ├── core └── core.go └── go.sum /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | config.yaml 3 | backups -------------------------------------------------------------------------------- /.idea/vcs.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/soxft/db-backuper/config" 8 | "github.com/soxft/db-backuper/core" 9 | ) 10 | 11 | func main() { 12 | log.SetOutput(os.Stdout) 13 | config.Init() 14 | core.Run() 15 | } 16 | -------------------------------------------------------------------------------- /.idea/modules.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | -------------------------------------------------------------------------------- /.idea/db-backuper.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # db_backuper 2 | > A simple tool to backup your database 3 | 4 | ## usage 5 | 6 | > db_backuper -h 7 | 8 | ``` 9 | Usage of db_backuper: 10 | -c string 11 | specify config file path (default "config.yaml") 12 | ``` 13 | 14 | ## support type 15 | 16 | ### database 17 | 18 | - mysql 19 | 20 | ### cloud storage 21 | 22 | - Tencent Cloud COS -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/soxft/db-backuper 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/clbanning/mxj v1.8.4 // indirect 7 | github.com/google/go-querystring v1.1.0 // indirect 8 | github.com/mitchellh/mapstructure v1.5.0 // indirect 9 | github.com/mozillazg/go-httpheader v0.3.1 // indirect 10 | github.com/robfig/cron/v3 v3.0.1 // indirect 11 | github.com/tencentyun/cos-go-sdk-v5 v0.7.40 // indirect 12 | go.uber.org/atomic v1.7.0 // indirect 13 | go.uber.org/multierr v1.6.0 // indirect 14 | go.uber.org/zap v1.24.0 // indirect 15 | gopkg.in/yaml.v2 v2.4.0 // indirect 16 | ) 17 | -------------------------------------------------------------------------------- /config.example.yaml: -------------------------------------------------------------------------------- 1 | Local: 2 | Dir: "/root/backups/" # 本地备份目录, 绝对路径, 必须指定 3 | MaxFileNum: 10 4 | Mysql: 5 | urlshorter: 6 | Host: localhost 7 | Port: 3306 8 | User: root 9 | Pass: rootpwd 10 | Db: urlshorter 11 | Cron: "* * * * *" 12 | BackupTo: 13 | - local 14 | - cos 15 | blog: 16 | Host: localhost 17 | Port: 3306 18 | User: root 19 | Pass: rootpwd 20 | Db: blog 21 | Cron: "* * * * *" 22 | BackupTo: 23 | - cos 24 | Cos: 25 | Region: ap-hongkong 26 | Bucket: example-1000000000 27 | Secret: 28 | ID: SecretID 29 | Key: SecretKey 30 | Path: /backup/ 31 | MaxFileNum: 10 -------------------------------------------------------------------------------- /tool/tool.go: -------------------------------------------------------------------------------- 1 | package tool 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | ) 7 | 8 | func PathExists(path string) bool { 9 | _, err := os.Stat(path) 10 | if err == nil { 11 | return true 12 | } 13 | 14 | if os.IsNotExist(err) { 15 | return false 16 | } 17 | return false 18 | } 19 | 20 | // DeleteLocal delete oldest file if there are more than maxNum 21 | func DeleteLocal(path string, maxNum int) error { 22 | // 读取 path 下的文件列表 23 | if fileList, err := os.ReadDir(path); errors.Is(err, nil) { 24 | for _, file := range fileList { 25 | if getDirFileNum(path) <= maxNum { 26 | break 27 | } 28 | _ = os.Remove(path + file.Name()) 29 | } 30 | } else { 31 | return err 32 | } 33 | return nil 34 | } 35 | 36 | func getDirFileNum(path string) int { 37 | if list, err := os.ReadDir(path); errors.Is(err, nil) { 38 | return len(list) 39 | } 40 | return 0 41 | } 42 | -------------------------------------------------------------------------------- /config/struct.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | type CStruct struct { 4 | Local LocalStruct `yaml:"Local"` 5 | Mysql map[string]MysqlStruct `yaml:"Mysql"` 6 | Cos CosStruct `yaml:"Cos"` 7 | } 8 | 9 | type LocalStruct struct { 10 | Dir string `yaml:"Dir"` 11 | MaxFileNum int `yaml:"MaxFileNum"` 12 | } 13 | 14 | type MysqlStruct struct { 15 | Host string `yaml:"Host"` 16 | Port string `yaml:"Port"` 17 | User string `yaml:"User"` 18 | Pass string `yaml:"Pass"` 19 | Db string `yaml:"Db"` 20 | Cron string `yaml:"Cron"` 21 | BackupTo []string `yaml:"BackupTo"` 22 | } 23 | 24 | type CosStruct struct { 25 | Region string `yaml:"Region"` 26 | Bucket string `yaml:"Bucket"` 27 | Secret struct { 28 | Id string `yaml:"ID"` 29 | Key string `yaml:"Key"` 30 | } `yaml:"Secret"` 31 | Path string `yaml:"Path"` 32 | MaxFileNum int `yaml:"MaxFileNum"` 33 | } 34 | -------------------------------------------------------------------------------- /config/config.go: -------------------------------------------------------------------------------- 1 | package config 2 | 3 | import ( 4 | "flag" 5 | "io" 6 | "log" 7 | "os" 8 | 9 | "gopkg.in/yaml.v2" 10 | ) 11 | 12 | var ( 13 | C *CStruct 14 | configPath string 15 | 16 | Local LocalStruct 17 | Mysql map[string]MysqlStruct 18 | Cos CosStruct 19 | ) 20 | 21 | func Init() { 22 | flag.StringVar(&configPath, "c", "config.yaml", "specify config file path") 23 | flag.Parse() 24 | 25 | file, err := os.Open(configPath) 26 | if err != nil { 27 | log.Fatalf("Error opening config file: %v", err) 28 | } 29 | 30 | cRaw, err := io.ReadAll(file) 31 | if err != nil { 32 | log.Fatalf("Error opening config file: %v", err) 33 | } 34 | 35 | C = &CStruct{} 36 | err = yaml.Unmarshal(cRaw, C) 37 | if err != nil { 38 | log.Fatalf("Error parsing config file: %v", err) 39 | } 40 | 41 | Local = C.Local 42 | Mysql = C.Mysql 43 | Cos = C.Cos 44 | 45 | // check if path ends with "/" 46 | if Cos.Path[len(Cos.Path)-1:] != "/" { 47 | Cos.Path += "/" 48 | } 49 | // not start with "/" 50 | if Cos.Path[0:1] == "/" { 51 | Cos.Path = Cos.Path[1:] 52 | } 53 | // check if path ends with "/" 54 | if Local.Dir[len(Local.Dir)-1:] != "/" { 55 | Local.Dir += "/" 56 | } 57 | 58 | // log.Println("Config loaded", C) 59 | log.Printf("Config loaded") 60 | } 61 | -------------------------------------------------------------------------------- /db/db.go: -------------------------------------------------------------------------------- 1 | package db 2 | 3 | import ( 4 | "errors" 5 | "io" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "time" 10 | 11 | "github.com/soxft/db-backuper/tool" 12 | ) 13 | 14 | func MysqlDump(host, port, user, password, databaseName, sqlPath string) (string, error) { 15 | // check if sqlPath dir exists 16 | if !tool.PathExists(sqlPath) { 17 | return "", errors.New("sqlPath does not exist") 18 | } 19 | 20 | var cmd *exec.Cmd 21 | 22 | backupPath := sqlPath + "db_" + databaseName + "_" + time.Now().Format("060102_150405") + ".sql" 23 | 24 | cmd = exec.Command("mysqldump", "--opt", "-h"+host, "-P"+port, "-u"+user, "-p"+password, databaseName, "--result-file="+backupPath) 25 | 26 | //stdout, _ := cmd.StdoutPipe() 27 | // defer stdout.Close() 28 | 29 | stderr, _ := cmd.StderrPipe() 30 | defer stderr.Close() 31 | 32 | if err := cmd.Start(); err != nil { 33 | log.Println(err) 34 | return "", err 35 | } 36 | 37 | stderrContent, _ := io.ReadAll(stderr) 38 | 39 | // wait for command to finish 40 | cmd.Wait() 41 | 42 | // check if the backup file is created or if file is 0 bytes 43 | if fi, err := os.Stat(backupPath); err == nil { 44 | if fi.Size() == 0 { 45 | // log.Println("Backup file is 0 bytes") 46 | os.Remove(backupPath) 47 | return "", errors.New(string(stderrContent)) 48 | } 49 | } else { 50 | return "", err 51 | } 52 | 53 | return backupPath, nil 54 | } 55 | -------------------------------------------------------------------------------- /backup/cos.go: -------------------------------------------------------------------------------- 1 | package backup 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "log" 7 | "net/http" 8 | "net/url" 9 | 10 | "github.com/soxft/db-backuper/config" 11 | "github.com/tencentyun/cos-go-sdk-v5" 12 | ) 13 | 14 | // ToCos 15 | // @Param flocation: local file location 16 | // @Param dbname: database name 17 | func ToCos(flocation, dbname string) (string, error) { 18 | bucketURL, _ := url.Parse("https://" + config.Cos.Bucket + ".cos." + config.Cos.Region + ".myqcloud.com") 19 | b := &cos.BaseURL{BucketURL: bucketURL} 20 | 21 | client := cos.NewClient(b, &http.Client{ 22 | Transport: &cos.AuthorizationTransport{ 23 | SecretID: config.Cos.Secret.Id, 24 | SecretKey: config.Cos.Secret.Key, 25 | }, 26 | }) 27 | 28 | // check Bucket exist 29 | ok, err := client.Bucket.IsExist(context.Background()) 30 | 31 | if err != nil { 32 | return "", err 33 | } else if !ok { 34 | return "", errors.New("bucket does not exist") 35 | } 36 | 37 | // upload 38 | remotePath := config.Cos.Path + dbname + "/" 39 | remoteFullPath := remotePath + flocation[len(config.Local.Dir):] 40 | _, err = client.Object.PutFromFile(context.Background(), remoteFullPath, flocation, nil) 41 | if err != nil { 42 | return "", err 43 | } 44 | 45 | _ = removeCosMax(remotePath, config.Cos.MaxFileNum) 46 | return remoteFullPath, nil 47 | } 48 | 49 | // removeMax 删除 cos 上 指定路径下超过 max 个文件 50 | func removeCosMax(remotePath string, max int) error { 51 | bucketURL, _ := url.Parse("https://" + config.Cos.Bucket + ".cos." + config.Cos.Region + ".myqcloud.com") 52 | b := &cos.BaseURL{BucketURL: bucketURL} 53 | 54 | client := cos.NewClient(b, &http.Client{ 55 | Transport: &cos.AuthorizationTransport{ 56 | SecretID: config.Cos.Secret.Id, 57 | SecretKey: config.Cos.Secret.Key, 58 | }, 59 | }) 60 | 61 | opt := &cos.BucketGetOptions{ 62 | Prefix: remotePath, 63 | MaxKeys: 1000, 64 | } 65 | res, _, err := client.Bucket.Get(context.Background(), opt) 66 | if err != nil { 67 | return err 68 | } 69 | 70 | num := len(res.Contents) 71 | for _, v := range res.Contents { 72 | if num > max { 73 | _, _ = client.Object.Delete(context.Background(), v.Key) 74 | log.Println("remove cos file:", v.Key) 75 | num-- 76 | } else { 77 | break 78 | } 79 | } 80 | return nil 81 | } 82 | -------------------------------------------------------------------------------- /core/core.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "os/signal" 7 | "syscall" 8 | 9 | "github.com/robfig/cron/v3" 10 | "github.com/soxft/db-backuper/backup" 11 | "github.com/soxft/db-backuper/config" 12 | "github.com/soxft/db-backuper/db" 13 | "github.com/soxft/db-backuper/tool" 14 | ) 15 | 16 | func Run() { 17 | c := cron.New() 18 | 19 | for k, v := range config.Mysql { 20 | if _, err := c.AddFunc(v.Cron, cronFunc(k, v)); err != nil { 21 | log.Fatalf("%s > Add Cron error: %v", k, err) 22 | } else { 23 | log.Printf("%s > Cron added: %s", k, v.Cron) 24 | } 25 | } 26 | 27 | c.Start() 28 | 29 | // wait for interrupt signal to gracefully shutdown the server with 30 | sig := make(chan os.Signal, 1) 31 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) 32 | <-sig 33 | c.Stop() 34 | log.Println("Bye! :)") 35 | } 36 | 37 | func cronFunc(k string, v config.MysqlStruct) func() { 38 | return func() { 39 | go run(k, v) 40 | } 41 | } 42 | 43 | // backup main func 44 | func run(name string, info config.MysqlStruct) { 45 | defer func() { 46 | // recover 47 | if err := recover(); err != nil { 48 | log.Printf("%s > Backup error: %v", name, err) 49 | } 50 | }() 51 | 52 | if info.BackupTo == nil { 53 | log.Printf("%s > BackupTo is empty", name) 54 | return 55 | } 56 | 57 | if location, err := db.MysqlDump(info.Host, info.Port, info.User, info.Pass, info.Db, config.Local.Dir); err != nil { 58 | log.Printf("%s > Backup error: %v", name, err) 59 | } else { 60 | log.Printf("%s > Backup created: %s", name, location) 61 | 62 | if isMethodContains(info.BackupTo, "cos") { 63 | if rlocation, err := backup.ToCos(location, info.Db); err != nil { 64 | log.Printf("%s > cos upload error: %v", name, err) 65 | } else { 66 | log.Printf("%s > cos upload success: %s", name, rlocation) 67 | } 68 | } 69 | 70 | if !isMethodContains(info.BackupTo, "local") { 71 | _ = os.Remove(location) 72 | log.Printf("%s > local backup removed: %s", name, location) 73 | } 74 | 75 | // remove local backup files if max file num is set 76 | tool.DeleteLocal(config.Local.Dir, config.Local.MaxFileNum) 77 | } 78 | } 79 | 80 | // isMethodContains check if method is in list 81 | func isMethodContains(list []string, method string) bool { 82 | for _, v := range list { 83 | if v == method { 84 | return true 85 | } 86 | } 87 | return false 88 | } 89 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= 2 | github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I= 3 | github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 6 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 7 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 8 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= 9 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= 10 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= 11 | github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 12 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= 13 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= 14 | github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= 15 | github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok= 16 | github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA= 17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 18 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= 19 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= 20 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 22 | github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= 23 | github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4= 24 | github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM= 25 | github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= 26 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw= 27 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= 28 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4= 29 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= 30 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= 31 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= 32 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 33 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 34 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 35 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 36 | -------------------------------------------------------------------------------- /.idea/workspace.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 6 | 7 | 8 | 9 | 10 | 15 | 16 | 17 | 19 | 20 | 22 | 23 | 24 | 27 | { 28 | "keyToString": { 29 | "ASKED_ADD_EXTERNAL_FILES": "true", 30 | "ASKED_SHARE_PROJECT_CONFIGURATION_FILES": "true", 31 | "RunOnceActivity.OpenProjectViewOnStart": "true", 32 | "RunOnceActivity.ShowReadmeOnStart": "true", 33 | "RunOnceActivity.go.formatter.settings.were.checked": "true", 34 | "RunOnceActivity.go.migrated.go.modules.settings": "true", 35 | "RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true", 36 | "WebServerToolWindowFactoryState": "false", 37 | "go.import.settings.migrated": "true", 38 | "go.sdk.automatically.set": "true", 39 | "last_opened_file_path": "/Users/xcsoft/store/project/golang/db-backuper", 40 | "node.js.detected.package.eslint": "true", 41 | "node.js.selected.package.eslint": "(autodetect)", 42 | "settings.editor.selected.configurable": "preferences.lookFeel" 43 | } 44 | } 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 61 | 62 | 63 | 64 | 66 | 67 | true 68 | 69 | --------------------------------------------------------------------------------