├── .gitignore
├── .idea
├── vcs.xml
├── modules.xml
├── db-backuper.iml
└── workspace.xml
├── main.go
├── README.md
├── go.mod
├── config.example.yaml
├── tool
└── tool.go
├── config
├── struct.go
└── config.go
├── db
└── db.go
├── backup
└── cos.go
├── core
└── core.go
└── go.sum
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | config.yaml
3 | backups
--------------------------------------------------------------------------------
/.idea/vcs.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
--------------------------------------------------------------------------------
/main.go:
--------------------------------------------------------------------------------
1 | package main
2 |
3 | import (
4 | "log"
5 | "os"
6 |
7 | "github.com/soxft/db-backuper/config"
8 | "github.com/soxft/db-backuper/core"
9 | )
10 |
11 | func main() {
12 | log.SetOutput(os.Stdout)
13 | config.Init()
14 | core.Run()
15 | }
16 |
--------------------------------------------------------------------------------
/.idea/modules.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
--------------------------------------------------------------------------------
/.idea/db-backuper.iml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # db_backuper
2 | > A simple tool to backup your database
3 |
4 | ## usage
5 |
6 | > db_backuper -h
7 |
8 | ```
9 | Usage of db_backuper:
10 | -c string
11 | specify config file path (default "config.yaml")
12 | ```
13 |
14 | ## support type
15 |
16 | ### database
17 |
18 | - mysql
19 |
20 | ### cloud storage
21 |
22 | - Tencent Cloud COS
--------------------------------------------------------------------------------
/go.mod:
--------------------------------------------------------------------------------
1 | module github.com/soxft/db-backuper
2 |
3 | go 1.19
4 |
5 | require (
6 | github.com/clbanning/mxj v1.8.4 // indirect
7 | github.com/google/go-querystring v1.1.0 // indirect
8 | github.com/mitchellh/mapstructure v1.5.0 // indirect
9 | github.com/mozillazg/go-httpheader v0.3.1 // indirect
10 | github.com/robfig/cron/v3 v3.0.1 // indirect
11 | github.com/tencentyun/cos-go-sdk-v5 v0.7.40 // indirect
12 | go.uber.org/atomic v1.7.0 // indirect
13 | go.uber.org/multierr v1.6.0 // indirect
14 | go.uber.org/zap v1.24.0 // indirect
15 | gopkg.in/yaml.v2 v2.4.0 // indirect
16 | )
17 |
--------------------------------------------------------------------------------
/config.example.yaml:
--------------------------------------------------------------------------------
1 | Local:
2 | Dir: "/root/backups/" # 本地备份目录, 绝对路径, 必须指定
3 | MaxFileNum: 10
4 | Mysql:
5 | urlshorter:
6 | Host: localhost
7 | Port: 3306
8 | User: root
9 | Pass: rootpwd
10 | Db: urlshorter
11 | Cron: "* * * * *"
12 | BackupTo:
13 | - local
14 | - cos
15 | blog:
16 | Host: localhost
17 | Port: 3306
18 | User: root
19 | Pass: rootpwd
20 | Db: blog
21 | Cron: "* * * * *"
22 | BackupTo:
23 | - cos
24 | Cos:
25 | Region: ap-hongkong
26 | Bucket: example-1000000000
27 | Secret:
28 | ID: SecretID
29 | Key: SecretKey
30 | Path: /backup/
31 | MaxFileNum: 10
--------------------------------------------------------------------------------
/tool/tool.go:
--------------------------------------------------------------------------------
1 | package tool
2 |
3 | import (
4 | "errors"
5 | "os"
6 | )
7 |
8 | func PathExists(path string) bool {
9 | _, err := os.Stat(path)
10 | if err == nil {
11 | return true
12 | }
13 |
14 | if os.IsNotExist(err) {
15 | return false
16 | }
17 | return false
18 | }
19 |
20 | // DeleteLocal delete oldest file if there are more than maxNum
21 | func DeleteLocal(path string, maxNum int) error {
22 | // 读取 path 下的文件列表
23 | if fileList, err := os.ReadDir(path); errors.Is(err, nil) {
24 | for _, file := range fileList {
25 | if getDirFileNum(path) <= maxNum {
26 | break
27 | }
28 | _ = os.Remove(path + file.Name())
29 | }
30 | } else {
31 | return err
32 | }
33 | return nil
34 | }
35 |
36 | func getDirFileNum(path string) int {
37 | if list, err := os.ReadDir(path); errors.Is(err, nil) {
38 | return len(list)
39 | }
40 | return 0
41 | }
42 |
--------------------------------------------------------------------------------
/config/struct.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | type CStruct struct {
4 | Local LocalStruct `yaml:"Local"`
5 | Mysql map[string]MysqlStruct `yaml:"Mysql"`
6 | Cos CosStruct `yaml:"Cos"`
7 | }
8 |
9 | type LocalStruct struct {
10 | Dir string `yaml:"Dir"`
11 | MaxFileNum int `yaml:"MaxFileNum"`
12 | }
13 |
14 | type MysqlStruct struct {
15 | Host string `yaml:"Host"`
16 | Port string `yaml:"Port"`
17 | User string `yaml:"User"`
18 | Pass string `yaml:"Pass"`
19 | Db string `yaml:"Db"`
20 | Cron string `yaml:"Cron"`
21 | BackupTo []string `yaml:"BackupTo"`
22 | }
23 |
24 | type CosStruct struct {
25 | Region string `yaml:"Region"`
26 | Bucket string `yaml:"Bucket"`
27 | Secret struct {
28 | Id string `yaml:"ID"`
29 | Key string `yaml:"Key"`
30 | } `yaml:"Secret"`
31 | Path string `yaml:"Path"`
32 | MaxFileNum int `yaml:"MaxFileNum"`
33 | }
34 |
--------------------------------------------------------------------------------
/config/config.go:
--------------------------------------------------------------------------------
1 | package config
2 |
3 | import (
4 | "flag"
5 | "io"
6 | "log"
7 | "os"
8 |
9 | "gopkg.in/yaml.v2"
10 | )
11 |
12 | var (
13 | C *CStruct
14 | configPath string
15 |
16 | Local LocalStruct
17 | Mysql map[string]MysqlStruct
18 | Cos CosStruct
19 | )
20 |
21 | func Init() {
22 | flag.StringVar(&configPath, "c", "config.yaml", "specify config file path")
23 | flag.Parse()
24 |
25 | file, err := os.Open(configPath)
26 | if err != nil {
27 | log.Fatalf("Error opening config file: %v", err)
28 | }
29 |
30 | cRaw, err := io.ReadAll(file)
31 | if err != nil {
32 | log.Fatalf("Error opening config file: %v", err)
33 | }
34 |
35 | C = &CStruct{}
36 | err = yaml.Unmarshal(cRaw, C)
37 | if err != nil {
38 | log.Fatalf("Error parsing config file: %v", err)
39 | }
40 |
41 | Local = C.Local
42 | Mysql = C.Mysql
43 | Cos = C.Cos
44 |
45 | // check if path ends with "/"
46 | if Cos.Path[len(Cos.Path)-1:] != "/" {
47 | Cos.Path += "/"
48 | }
49 | // not start with "/"
50 | if Cos.Path[0:1] == "/" {
51 | Cos.Path = Cos.Path[1:]
52 | }
53 | // check if path ends with "/"
54 | if Local.Dir[len(Local.Dir)-1:] != "/" {
55 | Local.Dir += "/"
56 | }
57 |
58 | // log.Println("Config loaded", C)
59 | log.Printf("Config loaded")
60 | }
61 |
--------------------------------------------------------------------------------
/db/db.go:
--------------------------------------------------------------------------------
1 | package db
2 |
3 | import (
4 | "errors"
5 | "io"
6 | "log"
7 | "os"
8 | "os/exec"
9 | "time"
10 |
11 | "github.com/soxft/db-backuper/tool"
12 | )
13 |
14 | func MysqlDump(host, port, user, password, databaseName, sqlPath string) (string, error) {
15 | // check if sqlPath dir exists
16 | if !tool.PathExists(sqlPath) {
17 | return "", errors.New("sqlPath does not exist")
18 | }
19 |
20 | var cmd *exec.Cmd
21 |
22 | backupPath := sqlPath + "db_" + databaseName + "_" + time.Now().Format("060102_150405") + ".sql"
23 |
24 | cmd = exec.Command("mysqldump", "--opt", "-h"+host, "-P"+port, "-u"+user, "-p"+password, databaseName, "--result-file="+backupPath)
25 |
26 | //stdout, _ := cmd.StdoutPipe()
27 | // defer stdout.Close()
28 |
29 | stderr, _ := cmd.StderrPipe()
30 | defer stderr.Close()
31 |
32 | if err := cmd.Start(); err != nil {
33 | log.Println(err)
34 | return "", err
35 | }
36 |
37 | stderrContent, _ := io.ReadAll(stderr)
38 |
39 | // wait for command to finish
40 | cmd.Wait()
41 |
42 | // check if the backup file is created or if file is 0 bytes
43 | if fi, err := os.Stat(backupPath); err == nil {
44 | if fi.Size() == 0 {
45 | // log.Println("Backup file is 0 bytes")
46 | os.Remove(backupPath)
47 | return "", errors.New(string(stderrContent))
48 | }
49 | } else {
50 | return "", err
51 | }
52 |
53 | return backupPath, nil
54 | }
55 |
--------------------------------------------------------------------------------
/backup/cos.go:
--------------------------------------------------------------------------------
1 | package backup
2 |
3 | import (
4 | "context"
5 | "errors"
6 | "log"
7 | "net/http"
8 | "net/url"
9 |
10 | "github.com/soxft/db-backuper/config"
11 | "github.com/tencentyun/cos-go-sdk-v5"
12 | )
13 |
14 | // ToCos
15 | // @Param flocation: local file location
16 | // @Param dbname: database name
17 | func ToCos(flocation, dbname string) (string, error) {
18 | bucketURL, _ := url.Parse("https://" + config.Cos.Bucket + ".cos." + config.Cos.Region + ".myqcloud.com")
19 | b := &cos.BaseURL{BucketURL: bucketURL}
20 |
21 | client := cos.NewClient(b, &http.Client{
22 | Transport: &cos.AuthorizationTransport{
23 | SecretID: config.Cos.Secret.Id,
24 | SecretKey: config.Cos.Secret.Key,
25 | },
26 | })
27 |
28 | // check Bucket exist
29 | ok, err := client.Bucket.IsExist(context.Background())
30 |
31 | if err != nil {
32 | return "", err
33 | } else if !ok {
34 | return "", errors.New("bucket does not exist")
35 | }
36 |
37 | // upload
38 | remotePath := config.Cos.Path + dbname + "/"
39 | remoteFullPath := remotePath + flocation[len(config.Local.Dir):]
40 | _, err = client.Object.PutFromFile(context.Background(), remoteFullPath, flocation, nil)
41 | if err != nil {
42 | return "", err
43 | }
44 |
45 | _ = removeCosMax(remotePath, config.Cos.MaxFileNum)
46 | return remoteFullPath, nil
47 | }
48 |
49 | // removeMax 删除 cos 上 指定路径下超过 max 个文件
50 | func removeCosMax(remotePath string, max int) error {
51 | bucketURL, _ := url.Parse("https://" + config.Cos.Bucket + ".cos." + config.Cos.Region + ".myqcloud.com")
52 | b := &cos.BaseURL{BucketURL: bucketURL}
53 |
54 | client := cos.NewClient(b, &http.Client{
55 | Transport: &cos.AuthorizationTransport{
56 | SecretID: config.Cos.Secret.Id,
57 | SecretKey: config.Cos.Secret.Key,
58 | },
59 | })
60 |
61 | opt := &cos.BucketGetOptions{
62 | Prefix: remotePath,
63 | MaxKeys: 1000,
64 | }
65 | res, _, err := client.Bucket.Get(context.Background(), opt)
66 | if err != nil {
67 | return err
68 | }
69 |
70 | num := len(res.Contents)
71 | for _, v := range res.Contents {
72 | if num > max {
73 | _, _ = client.Object.Delete(context.Background(), v.Key)
74 | log.Println("remove cos file:", v.Key)
75 | num--
76 | } else {
77 | break
78 | }
79 | }
80 | return nil
81 | }
82 |
--------------------------------------------------------------------------------
/core/core.go:
--------------------------------------------------------------------------------
1 | package core
2 |
3 | import (
4 | "log"
5 | "os"
6 | "os/signal"
7 | "syscall"
8 |
9 | "github.com/robfig/cron/v3"
10 | "github.com/soxft/db-backuper/backup"
11 | "github.com/soxft/db-backuper/config"
12 | "github.com/soxft/db-backuper/db"
13 | "github.com/soxft/db-backuper/tool"
14 | )
15 |
16 | func Run() {
17 | c := cron.New()
18 |
19 | for k, v := range config.Mysql {
20 | if _, err := c.AddFunc(v.Cron, cronFunc(k, v)); err != nil {
21 | log.Fatalf("%s > Add Cron error: %v", k, err)
22 | } else {
23 | log.Printf("%s > Cron added: %s", k, v.Cron)
24 | }
25 | }
26 |
27 | c.Start()
28 |
29 | // wait for interrupt signal to gracefully shutdown the server with
30 | sig := make(chan os.Signal, 1)
31 | signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
32 | <-sig
33 | c.Stop()
34 | log.Println("Bye! :)")
35 | }
36 |
37 | func cronFunc(k string, v config.MysqlStruct) func() {
38 | return func() {
39 | go run(k, v)
40 | }
41 | }
42 |
43 | // backup main func
44 | func run(name string, info config.MysqlStruct) {
45 | defer func() {
46 | // recover
47 | if err := recover(); err != nil {
48 | log.Printf("%s > Backup error: %v", name, err)
49 | }
50 | }()
51 |
52 | if info.BackupTo == nil {
53 | log.Printf("%s > BackupTo is empty", name)
54 | return
55 | }
56 |
57 | if location, err := db.MysqlDump(info.Host, info.Port, info.User, info.Pass, info.Db, config.Local.Dir); err != nil {
58 | log.Printf("%s > Backup error: %v", name, err)
59 | } else {
60 | log.Printf("%s > Backup created: %s", name, location)
61 |
62 | if isMethodContains(info.BackupTo, "cos") {
63 | if rlocation, err := backup.ToCos(location, info.Db); err != nil {
64 | log.Printf("%s > cos upload error: %v", name, err)
65 | } else {
66 | log.Printf("%s > cos upload success: %s", name, rlocation)
67 | }
68 | }
69 |
70 | if !isMethodContains(info.BackupTo, "local") {
71 | _ = os.Remove(location)
72 | log.Printf("%s > local backup removed: %s", name, location)
73 | }
74 |
75 | // remove local backup files if max file num is set
76 | tool.DeleteLocal(config.Local.Dir, config.Local.MaxFileNum)
77 | }
78 | }
79 |
80 | // isMethodContains check if method is in list
81 | func isMethodContains(list []string, method string) bool {
82 | for _, v := range list {
83 | if v == method {
84 | return true
85 | }
86 | }
87 | return false
88 | }
89 |
--------------------------------------------------------------------------------
/go.sum:
--------------------------------------------------------------------------------
1 | github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM=
2 | github.com/clbanning/mxj v1.8.4 h1:HuhwZtbyvyOw+3Z1AowPkU87JkJUSv751ELWaiTpj8I=
3 | github.com/clbanning/mxj v1.8.4/go.mod h1:BVjHeAH+rl9rs6f+QIpeRl0tfu10SXn1pUSa5PVGJng=
4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
5 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
6 | github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
7 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
8 | github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
9 | github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
10 | github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
11 | github.com/mitchellh/mapstructure v1.4.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
12 | github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
13 | github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
14 | github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60=
15 | github.com/mozillazg/go-httpheader v0.3.1 h1:IRP+HFrMX2SlwY9riuio7raffXUpzAosHtZu25BSJok=
16 | github.com/mozillazg/go-httpheader v0.3.1/go.mod h1:PuT8h0pw6efvp8ZeUec1Rs7dwjK08bt6gKSReGMqtdA=
17 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
18 | github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
19 | github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
20 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
21 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
22 | github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.194/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y=
23 | github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.194/go.mod h1:yrBKWhChnDqNz1xuXdSbWXG56XawEq0G5j1lg4VwBD4=
24 | github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4O8IB2ozzxM=
25 | github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw=
26 | go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
27 | go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
28 | go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
29 | go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
30 | go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
31 | go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
32 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
33 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
34 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
35 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
36 |
--------------------------------------------------------------------------------
/.idea/workspace.xml:
--------------------------------------------------------------------------------
1 |
2 |
3 |
4 |
5 |
6 |
7 |
8 |
9 |
10 |
11 |
12 |
13 |
14 |
15 |
16 |
17 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 | {
28 | "keyToString": {
29 | "ASKED_ADD_EXTERNAL_FILES": "true",
30 | "ASKED_SHARE_PROJECT_CONFIGURATION_FILES": "true",
31 | "RunOnceActivity.OpenProjectViewOnStart": "true",
32 | "RunOnceActivity.ShowReadmeOnStart": "true",
33 | "RunOnceActivity.go.formatter.settings.were.checked": "true",
34 | "RunOnceActivity.go.migrated.go.modules.settings": "true",
35 | "RunOnceActivity.go.modules.go.list.on.any.changes.was.set": "true",
36 | "WebServerToolWindowFactoryState": "false",
37 | "go.import.settings.migrated": "true",
38 | "go.sdk.automatically.set": "true",
39 | "last_opened_file_path": "/Users/xcsoft/store/project/golang/db-backuper",
40 | "node.js.detected.package.eslint": "true",
41 | "node.js.selected.package.eslint": "(autodetect)",
42 | "settings.editor.selected.configurable": "preferences.lookFeel"
43 | }
44 | }
45 |
46 |
47 |
48 |
49 |
50 |
51 |
52 |
53 |
54 |
55 |
56 |
57 |
58 |
59 |
60 |
61 |
62 |
63 |
64 |
65 |
66 |
67 | true
68 |
69 |
--------------------------------------------------------------------------------