├── .gitignore ├── .travis.yml ├── Dockerfile ├── LICENSE ├── README.md ├── common ├── app.go ├── app_backup_parse_config.go ├── app_do_backup.go ├── app_do_restore.go ├── app_setup_test.go ├── config_backup.go ├── config_module.go ├── config_restore.go ├── runner.go └── runner_backup.go ├── core ├── module.go ├── module_error.go ├── module_group.go ├── module_type.go └── registry.go ├── docker-compose.yml ├── docker ├── mongo │ ├── Dockerfile │ ├── data │ │ ├── link.bson │ │ ├── link.metadata.json │ │ ├── test.bson │ │ └── test.metadata.json │ └── restore.sh └── ssh │ ├── Dockerfile │ ├── entrypoint.sh │ ├── ssh_config │ ├── sshd_config │ └── user.sh ├── docker_push.sh ├── docs ├── logo.svg └── md-logo.png ├── go.mod ├── go.sum ├── main.go ├── modules ├── backup │ ├── compress │ │ ├── gzip │ │ │ ├── config.go │ │ │ ├── gzip.go │ │ │ └── gzip_test.go │ │ └── lz4 │ │ │ ├── config.go │ │ │ ├── lz4.go │ │ │ ├── lz4_test.go │ │ │ └── test.txt │ ├── encrypt │ │ └── aesgcm │ │ │ ├── aesgcm.go │ │ │ ├── aesgcm_test.go │ │ │ └── config.go │ ├── input │ │ ├── consul │ │ │ ├── config.go │ │ │ └── consul.go │ │ ├── etcd │ │ │ ├── config.go │ │ │ ├── etcd.go │ │ │ └── etcd_test.go │ │ ├── etcdv3 │ │ │ ├── config.go │ │ │ ├── etcdv3.go │ │ │ └── etcdv3_test.go │ │ ├── local │ │ │ ├── config.go │ │ │ ├── local.go │ │ │ ├── local_test.go │ │ │ └── test.file │ │ ├── mongodb │ │ │ ├── config.go │ │ │ ├── mongo.go │ │ │ └── mongo_test.go │ │ ├── mysql │ │ │ ├── config.go │ │ │ ├── mysql.go │ │ │ ├── mysql_test.go │ │ │ └── templates.go │ │ ├── mysqldump │ │ │ ├── config.go │ │ │ ├── mysqldump.go │ │ │ └── mysqldump_test.go │ │ ├── postgresql │ │ │ ├── config.go │ │ │ ├── postgresql.go │ │ │ ├── postgresql_test.go │ │ │ └── templates.go │ │ └── tar │ │ │ ├── config.go │ │ │ ├── tar.go │ │ │ ├── tar_test.go │ │ │ └── target │ │ │ └── test.file │ └── output │ │ ├── gcp │ │ ├── config.go │ │ ├── gcp.go │ │ └── gcp_test.go │ │ ├── http │ │ ├── config.go │ │ ├── http.go │ │ └── http_test.go │ │ ├── local │ │ ├── config.go │ │ ├── local.go │ │ └── local_test.go │ │ ├── s3 │ │ ├── config.go │ │ ├── s3.go │ │ └── s3_test.go │ │ └── scp │ │ ├── README.md │ │ ├── config.go │ │ ├── scp.go │ │ └── scp_test.go ├── global │ ├── connect │ │ └── ssh │ │ │ ├── config.go │ │ │ ├── ssh.go │ │ │ ├── ssh_test.go │ │ │ └── tunnel.go │ └── notifier │ │ ├── awsses │ │ ├── awsses.go │ │ ├── awsses_test.go │ │ └── config.go │ │ ├── awssqs │ │ ├── awssqs.go │ │ ├── awssqs_test.go │ │ └── config.go │ │ ├── email │ │ ├── config.go │ │ ├── email.go │ │ └── email_test.go │ │ ├── kafka │ │ ├── config.go │ │ ├── kafka.go │ │ └── kafka_test.go │ │ ├── nats │ │ ├── config.go │ │ ├── nats.go │ │ └── nats_test.go │ │ ├── nsq │ │ ├── config.go │ │ ├── nsq.go │ │ └── nsq_test.go │ │ ├── pagerduty │ │ ├── config.go │ │ ├── pagerduty.go │ │ └── pagerduty_test.go │ │ ├── pushbullet │ │ ├── config.go │ │ ├── pushbullet.go │ │ └── pushbullet_test.go │ │ ├── rabbitmq │ │ ├── config.go │ │ ├── rabbitmq.go │ │ └── rabbitmq_test.go │ │ ├── slack │ │ ├── config.go │ │ ├── slack.go │ │ └── slack_test.go │ │ ├── telegram │ │ ├── config.go │ │ ├── telegram.go │ │ └── telegram_test.go │ │ ├── twilio │ │ ├── config.go │ │ ├── twilio.go │ │ └── twilio_test.go │ │ └── webcallback │ │ ├── config.go │ │ ├── webcallback.go │ │ └── webcallback_test.go └── restore │ ├── decompress │ ├── gzip │ │ ├── gzip.go │ │ └── gzip_test.go │ └── lz4 │ │ ├── config.go │ │ ├── lz4.go │ │ ├── lz4_test.go │ │ └── test.txt │ ├── decrypt │ └── aesgcm │ │ ├── aesgcm.go │ │ ├── aesgcm_test.go │ │ └── config.go │ └── output │ ├── mysql │ ├── config.go │ ├── mysql.go │ └── mysql_test.go │ └── postgresql │ ├── config.go │ ├── postgresql.go │ └── postgresql_test.go └── samples ├── config_backup_example.yml ├── mysql.sql └── postgres.sql /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /.vagrant 3 | *.iml 4 | *.sw[p,o] 5 | bin 6 | *.out 7 | /mysql 8 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | 3 | services: 4 | - docker 5 | 6 | sudo: false 7 | 8 | os: 9 | - linux 10 | 11 | env: 12 | global: 13 | - GO111MODULE=on 14 | - PRODUCT=copybird 15 | 16 | go: 17 | - 1.12.x 18 | dist: xenial 19 | 20 | before_install: 21 | - go get golang.org/x/tools/cmd/cover 22 | - go get github.com/mattn/goveralls 23 | 24 | install: 25 | # Core testing install 26 | - docker-compose build 27 | - docker-compose up -d 28 | 29 | script: 30 | - go test -v -covermode=count -coverprofile=coverage.out ./... 31 | - $HOME/gopath/bin/goveralls -coverprofile=coverage.out -service=travis-ci -repotoken $COVERALLS_TOKEN 32 | - GOOS=linux GOARCH=${TRAVIS_CPU_ARCH} go build -o bin/${PRODUCT}-${TRAVIS_TAG} 33 | - GOOS=darwin GOARCH=${TRAVIS_CPU_ARCH} go build -o bin/${PRODUCT}-${TRAVIS_TAG}_osx 34 | - docker build -t copybird/${PRODUCT}:latest . 35 | 36 | before_deploy: 37 | - docker tag copybird/${PRODUCT}:latest copybird/${PRODUCT}:${TRAVIS_TAG} 38 | # - tar czvf build/${PRODUCT}-${TRAVIS_TAG}.linux-${TRAVIS_CPU_ARCH}.tar.gz bin/${PRODUCT} 39 | # - tar czvf build/${PRODUCT}-${TRAVIS_TAG}.osx-${TRAVIS_CPU_ARCH}.tar.gz bin/${PRODUCT}_osx 40 | 41 | deploy: 42 | - provider: releases 43 | api_key: ${GITHUB_OAUTH_TOKEN} 44 | file: 45 | - bin/${PRODUCT}-${TRAVIS_TAG} 46 | - bin/${PRODUCT}-${TRAVIS_TAG}_osx 47 | # - build/${PRODUCT}-${TRAVIS_TAG}.linux-${TRAVIS_CPU_ARCH}.tar.gz 48 | # - build/${PRODUCT}-${TRAVIS_TAG}.osx-${TRAVIS_CPU_ARCH}.tar.gz 49 | skip_cleanup: true 50 | on: 51 | tags: true 52 | - provider: script 53 | script: bash docker_push.sh ${TRAVIS_TAG} 54 | skip_cleanup: true 55 | on: 56 | tags: true 57 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | # build stage 2 | FROM golang:alpine AS build-env 3 | 4 | RUN echo "@edge http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && apk --no-cache add ca-certificates dumb-init@edge openssl curl git 5 | ADD . /src 6 | RUN cd /src && go build -o copybird 7 | 8 | FROM alpine 9 | RUN echo "@edge http://nl.alpinelinux.org/alpine/edge/testing" >> /etc/apk/repositories && apk --no-cache add ca-certificates dumb-init@edge openssl 10 | COPY --from=build-env /src/copybird /copybird 11 | ENTRYPOINT /copybird 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |
2 | Copybird 3 |
4 | 5 | # Copybird 6 | 7 | [![Developed by Mad Devs](https://maddevs.io/badge-dark.svg)](https://maddevs.io/) 8 | [![Project Status: WIP – Initial development is in progress, but there has not yet been a stable, usable release suitable for the public.](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip) 9 | [![](https://images.microbadger.com/badges/version/copybird/copybird.svg)](https://microbadger.com/images/copybird/copybird) 10 | [![](https://images.microbadger.com/badges/image/copybird/copybird.svg)](https://microbadger.com/images/copybird/copybird) 11 | [![](https://godoc.org/github.com/copybird/copybird?status.svg)](http://godoc.org/github.com/copybird/copybird) 12 | [![GitHub release](https://img.shields.io/github/release/copybird/copybird/all.svg?style=flat-square)](https://github.com/copybird/copybird/releases) 13 | 14 | ![](https://travis-ci.org/copybird/copybird.svg?branch=master) 15 | [![Coverage Status](https://coveralls.io/repos/github/copybird/copybird/badge.svg)](https://coveralls.io/github/copybird/copybird) 16 | [![Go Report Card](https://goreportcard.com/badge/github.com/copybird/copybird)](https://goreportcard.com/report/github.com/copybird/copybird) 17 | [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) 18 | 19 | ## About 20 | 21 | Copybird is open-source **cloud-native** universal backup tool for databases and files. 22 | 23 | It allows you to: 24 | 1. Create database backup 25 | 2. Compress backup stream 26 | 3. Encrypt backup stream 27 | 4. Send it to various destinations fast and secure 28 | 5. Get notification about backup status in messagers and notification services 29 | 6. Enjoy simple backup as a service with k8s backup controller 30 | 31 | Backup process not using local storage for temp files. 32 | 33 | learn more at [copybird.org](https://copybird.org). Note that this repository is in Work in Progress status. Feel free to contribure. Read more about contributing below. 34 | 35 | ## Databases 36 | Currently Copybird supports the following databases: 37 | - MySQL 38 | - Postgres 39 | - MongoDB 40 | - Etcd (v2 and v3 API) 41 | 42 | ## Compression 43 | Copybird compresses with the following tools: 44 | - gzip 45 | - lz4 46 | 47 | ## Encryption 48 | Copybird uses AES-GCM for Efficient Authenticated Encryption 49 | 50 | ## Output 51 | Copybird can deliver encrypted compressed backup to the following destinations: 52 | - store the file locally 53 | - save it on [GCP](https://cloud.google.com/‎) 54 | - save it on [S3](https://aws.amazon.com/s3/) 55 | - send over HTTP 56 | - send over SCP 57 | 58 | ## Notification services 59 | Copybird currently supports the following notification services: 60 | 61 | - Slack 62 | - Telegram 63 | - AWS SES 64 | - AWS SQS 65 | - get notificatoin on email 66 | - Kafka 67 | - Nats 68 | - Create issue in PagerDuty 69 | - Pushbullet 70 | - RabbitMQ 71 | - Twilio 72 | - Webcallback 73 | 74 | If you would like to add additional service, please submit an issue with feature request or add it yourself and send a Pull Request. 75 | 76 | ## How to Run the tool 77 | There are different ways you can use this tool: 78 | 79 | ### Run locally 80 | First get the source code on your machine 81 | ``` 82 | go get -u github.com/copybird/copybird 83 | ``` 84 | Then run it with `go run main.go` to see helpers for various optional parameters 85 | Example creating MySQL dump: 86 | ``` 87 | go run -v main.go backup -i 'mysql::dsn=root:root@tcp(localhost:3306)/test' -o local::file=dump.sql 88 | ``` 89 | 90 | ### Run with Docker 91 | Run `docker run copybird/copybird` to see the available optional parameters 92 | 93 | ### Use Backup Custom Controller/Operator for k8s 94 | 95 | First create custom resource definition in your cluster: 96 | ``` 97 | kubectl apply -f operator/crd/crd.yaml 98 | ``` 99 | 100 | To run the controller: 101 | ``` 102 | go run main.go operator 103 | ``` 104 | 105 | And then in a separate shell, create custom resource: 106 | ``` 107 | kubectl create -f operator/example/backup-example.yaml 108 | ``` 109 | As output you get the following logs when creating, updating or deleting custom resource: 110 | ``` 111 | INFO[0000] Successfully constructed k8s client 112 | INFO[0000] Starting Foo controller 113 | INFO[0000] Waiting for informer caches to sync 114 | INFO[0001] Starting workers 115 | INFO[0001] Started workers 116 | ``` 117 | You can modify example file as you wish to get proper configuration for your jobs 118 | 119 | ## Tests 120 | 121 | To run tests against MySQL module proceed with the following commands: 122 | ``` 123 | docker run --name test_mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=test -d percona:latest 124 | docker exec -i test_mysql mysql -uroot -proot test < samples/mysql.sql 125 | cd modules/backup/input/mysql/ 126 | go test -v -cover 127 | ``` 128 | To clean up after you finish with tests: 129 | ``` 130 | docker kill test_mysql 131 | docker rm test_mysql 132 | ``` 133 | To run tests against MySQLDump module, first make sure that you have `mysqldump` binary 134 | available in `$PATH` and then proceed with the following commands: 135 | ``` 136 | docker run --name test_mysql -p 3306:3306 -e MYSQL_ROOT_PASSWORD=root -e MYSQL_DATABASE=test -d percona:latest 137 | docker exec -i test_mysql mysql -hlocalhost -uroot -proot test < samples/mysql.sql 138 | cd modules/backup/input/mysqldump/ 139 | go test -v -cover 140 | ``` 141 | To clean up after you finish with tests: 142 | ``` 143 | docker kill test_mysql 144 | docker rm test_mysql 145 | ``` 146 | To run tests against Postgres module proceed with the following commands: 147 | ``` 148 | docker run --name test_postgres -p 5432:5432 -e POSTGRES_PASSWORD=postgres -e POSTGRES_DB=test -d postgres:latest 149 | docker exec -i test_postgres psql -U postgres test < samples/postgres.sql 150 | cd modules/backup/input/postgresql/ 151 | go test -v -cover 152 | ``` 153 | To clean up after you finish with tests: 154 | ``` 155 | docker kill test_postgres 156 | docker rm test_postgres 157 | ``` 158 | 159 | 160 | ## Contributing 161 | Pull requests are more than welcomed. For major changes, please open an issue first to discuss what you would like to change. 162 | 163 | Before submission of pull request make sure you pulled recent updates, included tests for your code that covers at least the core functionality and you submitted a desciptive issue that will be fixed with your pull request. Do not forget to mention the issue in the pull request. 164 | 165 | Project started by Artem Andreenko, Andrew Minkin, and Mad Devs. 166 | 167 |
168 |

Built with Mad Devs support for the community

169 | 170 |
171 | 172 | P.S. Love distributed and mesh? Meshbird. 173 | 174 | -------------------------------------------------------------------------------- /common/app.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "github.com/copybird/copybird/core" 5 | "github.com/copybird/copybird/modules/backup/compress/gzip" 6 | "github.com/copybird/copybird/modules/backup/compress/lz4" 7 | "github.com/copybird/copybird/modules/backup/encrypt/aesgcm" 8 | "github.com/copybird/copybird/modules/backup/input/mongodb" 9 | "github.com/copybird/copybird/modules/backup/input/mysql" 10 | "github.com/copybird/copybird/modules/backup/input/mysqldump" 11 | postgres "github.com/copybird/copybird/modules/backup/input/postgresql" 12 | "github.com/copybird/copybird/modules/backup/output/gcp" 13 | "github.com/copybird/copybird/modules/backup/output/http" 14 | "github.com/copybird/copybird/modules/backup/output/local" 15 | "github.com/copybird/copybird/modules/backup/output/s3" 16 | "github.com/copybird/copybird/modules/backup/output/scp" 17 | "github.com/spf13/cobra" 18 | ) 19 | 20 | type App struct { 21 | cmdRoot *cobra.Command 22 | cmdBackup *cobra.Command 23 | cmdRestore *cobra.Command 24 | vars map[string]interface{} 25 | } 26 | 27 | func NewApp() *App { 28 | return &App{ 29 | vars: make(map[string]interface{}), 30 | cmdRoot: &cobra.Command{}, 31 | cmdBackup: &cobra.Command{}, 32 | cmdRestore: &cobra.Command{}, 33 | } 34 | } 35 | 36 | func (a *App) Run() error { 37 | a.registerModules() 38 | var rootCmd = &cobra.Command{ 39 | Use: "copybird", 40 | SilenceUsage: true, 41 | } 42 | a.cmdBackup = &cobra.Command{ 43 | Use: "backup", 44 | Short: "Start new backup", 45 | Long: ``, 46 | Args: cobra.MinimumNArgs(0), 47 | RunE: cmdCallback(a.DoBackup), 48 | } 49 | a.cmdRestore = &cobra.Command{ 50 | Use: "restore", 51 | Short: "Start new restore", 52 | Long: ``, 53 | Args: cobra.MinimumNArgs(0), 54 | RunE: cmdCallback(a.DoRestore), 55 | } 56 | rootCmd.AddCommand(a.cmdBackup) 57 | rootCmd.AddCommand(a.cmdRestore) 58 | a.Setup() 59 | return rootCmd.Execute() 60 | } 61 | 62 | func cmdCallback(f func() error) func(cmd *cobra.Command, args []string) error { 63 | return func(cmd *cobra.Command, args []string) error { 64 | return f() 65 | } 66 | } 67 | 68 | func (a *App) registerModules() { 69 | core.RegisterModule(&mysql.BackupInputMysql{}) 70 | core.RegisterModule(&mysqldump.BackupInputMysqlDump{}) 71 | core.RegisterModule(&postgres.BackupInputPostgresql{}) 72 | core.RegisterModule(&mongodb.BackupInputMongodb{}) 73 | core.RegisterModule(&gzip.BackupCompressGzip{}) 74 | core.RegisterModule(&lz4.BackupCompressLz4{}) 75 | core.RegisterModule(&aesgcm.BackupEncryptAesgcm{}) 76 | core.RegisterModule(&gcp.BackupOutputGcp{}) 77 | core.RegisterModule(&http.BackupOutputHttp{}) 78 | core.RegisterModule(&local.BackupOutputLocal{}) 79 | core.RegisterModule(&s3.BackupOutputS3{}) 80 | core.RegisterModule(&scp.BackupOutputScp{}) 81 | } 82 | 83 | func (a *App) Setup() error { 84 | 85 | a.addFlagString(a.cmdBackup, "config", "f", "", "") 86 | a.addFlagString(a.cmdBackup, "connect", "c", "", "") 87 | a.addFlagString(a.cmdBackup, "input", "i", "", "(required)") 88 | a.addFlagString(a.cmdBackup, "compress", "z", "", "") 89 | a.addFlagString(a.cmdBackup, "encrypt", "e", "", "") 90 | a.addFlagString(a.cmdBackup, "output", "o", "", "(required)") 91 | a.addFlagStrings(a.cmdBackup, "notifier", "n", "") 92 | 93 | a.addFlagString(a.cmdRestore, "config", "f", "", "") 94 | a.addFlagString(a.cmdRestore, "connect", "c", "", "") 95 | a.addFlagString(a.cmdRestore, "input", "i", "", "(required)") 96 | a.addFlagString(a.cmdRestore, "decompress", "z", "", "") 97 | a.addFlagString(a.cmdRestore, "decrypt", "e", "", "") 98 | a.addFlagString(a.cmdRestore, "output", "o", "", "(required)") 99 | a.addFlagStrings(a.cmdRestore, "notifier", "n", "") 100 | 101 | return nil 102 | } 103 | 104 | func (a *App) addFlagString(cmd *cobra.Command, name, shortName, defaultValue, comment string) { 105 | a.vars[name] = cmd.Flags().StringP(name, shortName, defaultValue, comment) 106 | } 107 | 108 | func (a *App) addFlagStrings(cmd *cobra.Command, name, shortName, comment string) { 109 | a.vars[name] = cmd.Flags().StringArrayP(name, shortName, nil, comment) 110 | } 111 | -------------------------------------------------------------------------------- /common/app_backup_parse_config.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | //func (a *App) backupParseConfig(configPath string) (*ConfigBackup, error) { 4 | // configFile, err := os.Open(configPath) 5 | // if err != nil { 6 | // return nil, err 7 | // } 8 | // var cfg ConfigBackup 9 | // err = yaml.NewDecoder(configFile).Decode(&cfg) 10 | // if err != nil { 11 | // return nil, err 12 | // } 13 | // if cfg.Input == nil { 14 | // return nil, fmt.Errorf("need input module") 15 | // } 16 | // moduleInput := a.modulesBackup[cfg.Input.Type] 17 | // if moduleInput == nil { 18 | // return nil, fmt.Errorf("backup input module %s not found", cfg.Input.Type) 19 | // } 20 | // return &cfg, nil 21 | //} 22 | -------------------------------------------------------------------------------- /common/app_do_backup.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "log" 8 | "os" 9 | "reflect" 10 | "strconv" 11 | "strings" 12 | "time" 13 | 14 | "github.com/copybird/copybird/core" 15 | "github.com/iancoleman/strcase" 16 | "github.com/kelseyhightower/envconfig" 17 | "golang.org/x/sync/errgroup" 18 | ) 19 | 20 | const ( 21 | envInput = "COPYBIRD_INPUT" 22 | envOutput = "COPYBIRD_OUTPUT" 23 | envCompress = "COPYBIRD_COMPRESS" 24 | envEncrypt = "COPYBIRD_ENCRYPT" 25 | ) 26 | 27 | func (a *App) DoBackup() error { 28 | var mInput, mCompress, mEncrypt, mOutput core.Module 29 | var err error 30 | // Read CLI args 31 | lFlags := a.cmdBackup.LocalFlags() 32 | mInputArgs := lFlags.Lookup("input") 33 | if mInputArgs == nil { 34 | mInputArgs = lFlags.ShorthandLookup("i") 35 | } 36 | mCompressArgs := lFlags.Lookup("compress") 37 | if mCompressArgs == nil { 38 | mCompressArgs = lFlags.ShorthandLookup("z") 39 | } 40 | mEncryptArgs := lFlags.Lookup("encrypt") 41 | if mEncryptArgs == nil { 42 | mEncryptArgs = lFlags.ShorthandLookup("e") 43 | } 44 | mOutputArgs := lFlags.Lookup("output") 45 | if mOutputArgs == nil { 46 | mOutputArgs = lFlags.ShorthandLookup("o") 47 | } 48 | 49 | // Load modules 50 | if mInputArgs != nil && mInputArgs.Value.String() != "" { 51 | mInput, err = loadModuleFromArgs(core.ModuleGroupBackup, core.ModuleTypeInput, mInputArgs.Value.String()) 52 | if err != nil { 53 | return err 54 | } 55 | } else { 56 | mInput, err = loadModuleFromEnv(core.ModuleGroupBackup, core.ModuleTypeInput, envInput) 57 | if err != nil { 58 | return fmt.Errorf("can't get input module config: %s", err) 59 | } 60 | } 61 | 62 | if mOutputArgs != nil && mOutputArgs.Value.String() != "" { 63 | mOutput, err = loadModuleFromArgs(core.ModuleGroupBackup, core.ModuleTypeOutput, mOutputArgs.Value.String()) 64 | if err != nil { 65 | return err 66 | } 67 | } else { 68 | mOutput, err = loadModuleFromEnv(core.ModuleGroupBackup, core.ModuleTypeOutput, envOutput) 69 | if err != nil { 70 | return fmt.Errorf("can't get output module config: %s", err) 71 | } 72 | } 73 | 74 | if mCompressArgs != nil && mCompressArgs.Value.String() != "" { 75 | mCompress, err = loadModuleFromArgs(core.ModuleGroupBackup, core.ModuleTypeCompress, mCompressArgs.Value.String()) 76 | if err != nil { 77 | return err 78 | } 79 | } else { 80 | mCompress, err = loadModuleFromEnv(core.ModuleGroupBackup, core.ModuleTypeOutput, envCompress) 81 | } 82 | 83 | if mEncryptArgs != nil && mEncryptArgs.Value.String() != "" { 84 | mEncrypt, err = loadModuleFromArgs(core.ModuleGroupBackup, core.ModuleTypeEncrypt, mEncryptArgs.Value.String()) 85 | if err != nil { 86 | return err 87 | } 88 | } else { 89 | mEncrypt, err = loadModuleFromEnv(core.ModuleGroupBackup, core.ModuleTypeOutput, envEncrypt) 90 | } 91 | 92 | // Run pipeline 93 | eg, ctx := errgroup.WithContext(context.Background()) 94 | nextReader, nextWriter := io.Pipe() 95 | 96 | eg.Go(runModule(ctx, mInput, nextWriter, nil)) 97 | if mCompress != nil { 98 | interimReader, interimWriter := io.Pipe() 99 | eg.Go(runModule(ctx, mCompress, interimWriter, nextReader)) 100 | nextReader = interimReader 101 | } 102 | if mEncrypt != nil { 103 | interimReader, interimWriter := io.Pipe() 104 | eg.Go(runModule(ctx, mEncrypt, interimWriter, nextReader)) 105 | nextReader = interimReader 106 | } 107 | eg.Go(runModule(ctx, mOutput, nil, nextReader)) 108 | 109 | return eg.Wait() 110 | } 111 | 112 | func loadModuleFromArgs(mGroup core.ModuleGroup, mType core.ModuleType, args string) (core.Module, error) { 113 | name, params := parseArgs(args) 114 | module := core.GetModule(mGroup, mType, name) 115 | if module == nil { 116 | return nil, fmt.Errorf("module %s/%s not found", mType, name) 117 | } 118 | config := module.GetConfig() 119 | loadConfig(config, params) 120 | log.Printf("module %s/%s config: %+v", mType, name, config) 121 | if err := module.InitModule(config); err != nil { 122 | return nil, fmt.Errorf("init module %s/%s err: %s", mType, name, err) 123 | } 124 | return module, nil 125 | } 126 | 127 | func loadConfig(cfg interface{}, params map[string]string) error { 128 | cfgValue := reflect.Indirect(reflect.ValueOf(cfg)) 129 | cfgType := cfgValue.Type() 130 | 131 | for pName, pValue := range params { 132 | for i := 0; i < cfgType.NumField(); i++ { 133 | fieldValue := cfgValue.Field(i) 134 | fieldType := cfgType.Field(i) 135 | if strcase.ToSnake(fieldType.Name) == pName { 136 | switch fieldType.Type.Kind() { 137 | case reflect.String: 138 | fieldValue.SetString(pValue) 139 | case reflect.Int: 140 | val, err := strconv.ParseInt(pValue, 10, 63) 141 | if err != nil { 142 | return err 143 | } 144 | fieldValue.SetInt(val) 145 | case reflect.Bool: 146 | val, err := strconv.ParseBool(pValue) 147 | if err != nil { 148 | return err 149 | } 150 | fieldValue.SetBool(val) 151 | default: 152 | return fmt.Errorf("unsupported config param type: %s %s", 153 | pName, 154 | fieldType.Type.Kind().String()) 155 | } 156 | } 157 | } 158 | } 159 | return nil 160 | } 161 | 162 | func runModule(ctx context.Context, module core.Module, writer io.WriteCloser, reader io.ReadCloser) func() error { 163 | return func() error { 164 | defer func() { 165 | if writer != nil { 166 | writer.Close() 167 | } 168 | if reader != nil { 169 | reader.Close() 170 | } 171 | }() 172 | t := time.Now() 173 | err := module.InitPipe(writer, reader) 174 | if err != nil { 175 | return fmt.Errorf("module %s/%s pipe initialization err: %s", module.GetType(), module.GetName(), err) 176 | } 177 | err = module.Run(ctx) 178 | if err != nil { 179 | return fmt.Errorf("module %s/%s execution err: %s", module.GetType(), module.GetName(), err) 180 | } 181 | log.Printf("module %s/%s done by %.2fms", module.GetType(), module.GetName(), time.Since(t).Seconds()*1000) 182 | return nil 183 | } 184 | } 185 | 186 | func parseArgs(args string) (string, map[string]string) { 187 | var moduleName string 188 | var moduleParams = make(map[string]string) 189 | 190 | parts := strings.Split(args, "::") 191 | if len(parts) == 0 { 192 | return "", nil 193 | } 194 | moduleName = parts[0] 195 | for _, param := range parts[1:] { 196 | paramParts := strings.Split(param, "=") 197 | if len(paramParts) == 2 && paramParts[0] != "" { 198 | moduleParams[paramParts[0]] = paramParts[1] 199 | } 200 | } 201 | 202 | return moduleName, moduleParams 203 | } 204 | 205 | func loadModuleFromEnv(mGroup core.ModuleGroup, mType core.ModuleType, envPrefix string) (core.Module, error) { 206 | mName, defined := os.LookupEnv(envPrefix) 207 | if !defined { 208 | return nil, fmt.Errorf("%s/%s component not defined", mGroup, mType) 209 | } 210 | module := core.GetModule(mGroup, mType, mName) 211 | if module == nil { 212 | return nil, fmt.Errorf("module %s/%s not found", mType, mName) 213 | } 214 | config := module.GetConfig() 215 | err := envconfig.Process(envPrefix, config) 216 | if err != nil { 217 | return nil, fmt.Errorf("module %s/%s env parse err: %s", mType, mName, err) 218 | } 219 | log.Printf("module %s/%s config: %+v", mType, mName, config) 220 | if err := module.InitModule(config); err != nil { 221 | return nil, fmt.Errorf("init module %s/%s err: %s", mType, mName, err) 222 | } 223 | return module, nil 224 | } 225 | -------------------------------------------------------------------------------- /common/app_do_restore.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | func (a *App) DoRestore() error { 4 | return nil 5 | } 6 | 7 | -------------------------------------------------------------------------------- /common/app_setup_test.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "testing" 5 | 6 | "gotest.tools/assert" 7 | ) 8 | 9 | func TestAppSetup(t *testing.T) { 10 | app := NewApp() 11 | assert.NilError(t, app.Setup()) 12 | } 13 | -------------------------------------------------------------------------------- /common/config_backup.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type ConfigBackup struct { 4 | Connect *ConfigModule `yaml:'connect'` 5 | Input *ConfigModule `yaml:'input'` 6 | Compress *ConfigModule `yaml:'compress'` 7 | Encrypt *ConfigModule `yaml:'encrypt'` 8 | Outputs []*ConfigModule `yaml:'outputs'` 9 | Notifiers []*ConfigModule `yaml:'notifiers'` 10 | } 11 | -------------------------------------------------------------------------------- /common/config_module.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type ConfigModule struct { 4 | Type string `yaml:'type'` 5 | Config interface{} `yaml:'config'` 6 | } 7 | -------------------------------------------------------------------------------- /common/config_restore.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type ConfigRestore struct { 4 | Connect ConfigModule 5 | Input ConfigModule 6 | Decompress ConfigModule 7 | Decrypt ConfigModule 8 | Output ConfigModule 9 | Notifiers []ConfigModule 10 | } 11 | -------------------------------------------------------------------------------- /common/runner.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log" 7 | "sync" 8 | 9 | "github.com/copybird/copybird/core" 10 | ) 11 | 12 | type Runner struct { 13 | moduleInput core.Module 14 | moduleCompress core.Module 15 | moduleEncrypt core.Module 16 | moduleOutput core.Module 17 | moduleNotifiers []core.Module 18 | } 19 | 20 | func (r *Runner) Run() { 21 | wg := sync.WaitGroup{} 22 | // for input and output 23 | wg.Add(2) 24 | // for compress 25 | if r.moduleCompress != nil { 26 | wg.Add(1) 27 | } 28 | // for encrypt 29 | if r.moduleEncrypt != nil { 30 | wg.Add(1) 31 | } 32 | chanError := make(chan error, 1000) 33 | nextReader, nextWriter := io.Pipe() 34 | go r.runModule(r.moduleInput, nextWriter, nil, &wg, chanError) 35 | if r.moduleCompress != nil { 36 | _nextReader, _nextWriter := io.Pipe() 37 | go r.runModule(r.moduleCompress, _nextWriter, nextReader, &wg, chanError) 38 | nextReader = _nextReader 39 | } 40 | if r.moduleEncrypt != nil { 41 | _nextReader, _nextWriter := io.Pipe() 42 | go r.runModule(r.moduleEncrypt, _nextWriter, nextReader, &wg, chanError) 43 | nextReader = _nextReader 44 | } 45 | go r.runModule(r.moduleOutput, nil, nextReader, &wg, chanError) 46 | wg.Wait() 47 | for { 48 | err, ok := <-chanError 49 | if !ok { 50 | break 51 | } 52 | log.Printf("err: %s", err) 53 | } 54 | } 55 | 56 | func (r *Runner) runModule(module core.Module, writer io.Writer, reader io.Reader, wg *sync.WaitGroup, chanError chan error) { 57 | defer wg.Done() 58 | err := module.InitPipe(writer, reader) 59 | if err != nil { 60 | chanError <- &core.ModuleError{Module: module, Err: err} 61 | return 62 | } 63 | err = module.Run(context.TODO()) 64 | if err != nil { 65 | chanError <- &core.ModuleError{Module: module, Err: err} 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /common/runner_backup.go: -------------------------------------------------------------------------------- 1 | package common 2 | 3 | type RunnerBackup struct { 4 | app *App 5 | config *ConfigBackup 6 | } 7 | 8 | func NewRunnerBackup(app *App, config *ConfigBackup) *RunnerBackup { 9 | return &RunnerBackup{ 10 | app: app, 11 | config: config, 12 | } 13 | } 14 | 15 | func (rb *RunnerBackup) Run() error { 16 | return nil 17 | } 18 | -------------------------------------------------------------------------------- /core/module.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "context" 5 | "io" 6 | ) 7 | 8 | type Module interface { 9 | GetName() string 10 | GetGroup() ModuleGroup 11 | GetType() ModuleType 12 | GetConfig() interface{} 13 | InitModule(cfg interface{}) error 14 | Run(ctx context.Context) error 15 | Close() error 16 | InitPipe(w io.Writer, r io.Reader) error 17 | } 18 | -------------------------------------------------------------------------------- /core/module_error.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type ModuleError struct { 8 | Module Module 9 | Err error 10 | } 11 | 12 | func (me ModuleError) Error() string { 13 | return fmt.Sprintf("module %s err: %s", me.Module.GetName(), me.Err) 14 | } 15 | -------------------------------------------------------------------------------- /core/module_group.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type ModuleGroup string 4 | 5 | const ( 6 | ModuleGroupBackup ModuleGroup = "backup" 7 | ModuleGroupRestore ModuleGroup = "restore" 8 | ModuleGroupGlobal ModuleGroup = "global" 9 | ) 10 | -------------------------------------------------------------------------------- /core/module_type.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | type ModuleType string 4 | 5 | const ( 6 | ModuleTypeInput ModuleType = "input" 7 | ModuleTypeOutput ModuleType = "output" 8 | ModuleTypeCompress ModuleType = "compress" 9 | ModuleTypeDecompress ModuleType = "decompress" 10 | ModuleTypeEncrypt ModuleType = "encrypt" 11 | ModuleTypeDecrypt ModuleType = "decrypt" 12 | ) 13 | -------------------------------------------------------------------------------- /core/registry.go: -------------------------------------------------------------------------------- 1 | package core 2 | 3 | import ( 4 | "sync" 5 | ) 6 | 7 | var _modules = []Module{} 8 | var _modulesMutex = sync.Mutex{} 9 | 10 | func RegisterModule(module Module) { 11 | _modulesMutex.Lock() 12 | _modules = append(_modules, module) 13 | _modulesMutex.Unlock() 14 | } 15 | 16 | func GetModule(moduleGroup ModuleGroup, moduleType ModuleType, name string) Module { 17 | _modulesMutex.Lock() 18 | defer _modulesMutex.Unlock() 19 | for _, module := range _modules { 20 | if module.GetName() == name && module.GetGroup() == moduleGroup && module.GetType() == moduleType { 21 | return module 22 | } 23 | } 24 | return nil 25 | } -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | 3 | services: 4 | mysql: 5 | image: mysql:5.7 6 | restart: always 7 | environment: 8 | MYSQL_DATABASE: test 9 | MYSQL_USER: user 10 | MYSQL_PASSWORD: user 11 | MYSQL_ROOT_PASSWORD: root 12 | ports: 13 | - '3306:3306' 14 | volumes: 15 | - ./samples/mysql.sql:/docker-entrypoint-initdb.d/init.sql 16 | 17 | mongo: 18 | build: 19 | context: docker/mongo 20 | environment: 21 | MONGO_INITDB_DATABASE: test 22 | ports: 23 | - '27017:27017' 24 | 25 | nats: 26 | image: nats:latest 27 | restart: always 28 | container_name: nats 29 | ports: 30 | - '4222:4222' 31 | 32 | zookeeper: 33 | image: wurstmeister/zookeeper:3.4.6 34 | ports: 35 | - "2181:2181" 36 | 37 | kafka: 38 | image: wurstmeister/kafka:2.11-2.0.0 39 | depends_on: 40 | - zookeeper 41 | ports: 42 | - "9092:9092" 43 | environment: 44 | KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://localhost:9092 45 | KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092 46 | KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 47 | 48 | rabbitmq: 49 | image: rabbitmq:latest 50 | restart: always 51 | container_name: rabbitmq 52 | ports: 53 | - '5672:5672' 54 | 55 | postgres: 56 | image: library/postgres:11.4 57 | restart: always 58 | ports: 59 | - "5432:5432" 60 | environment: 61 | POSTGRES_PASSWORD: postgres 62 | POSTGRES_DB: test 63 | volumes: 64 | - ./samples/postgres.sql:/docker-entrypoint-initdb.d/postgres.sql 65 | 66 | ssh: 67 | build: 68 | context: docker/ssh 69 | args: 70 | SSH_MASTER_USER: user 71 | SSH_MASTER_PASS: user 72 | ports: 73 | - "2222:22" 74 | 75 | consul-agent-1: &consul-agent 76 | image: consul:latest 77 | container_name: consul-agent-1 78 | networks: 79 | - consul-demo 80 | command: "agent -retry-join consul-server-bootstrap -client 0.0.0.0" 81 | 82 | consul-agent-2: 83 | <<: *consul-agent 84 | container_name: consul-agent-2 85 | 86 | consul-agent-3: 87 | <<: *consul-agent 88 | container_name: consul-agent-3 89 | 90 | consul-server-1: &consul-server 91 | <<: *consul-agent 92 | command: "agent -server -retry-join consul-server-bootstrap -client 0.0.0.0" 93 | container_name: consul-server-1 94 | 95 | consul-server-2: 96 | <<: *consul-server 97 | container_name: consul-server-2 98 | 99 | consul-server-bootstrap: 100 | <<: *consul-agent 101 | ports: 102 | - "8400:8400" 103 | - "8500:8500" 104 | - "8600:8600" 105 | - "8600:8600/udp" 106 | command: "agent -server -bootstrap-expect 3 -ui -client 0.0.0.0" 107 | container_name: consul-server-bootstrap 108 | etcd1: 109 | image: quay.io/coreos/etcd:v3.2.5 110 | networks: 111 | - etcd 112 | ports: 113 | - 23791:2379 114 | - 23801:2380 115 | volumes: 116 | - ./certs/:/srv/ 117 | - /srv/docker/etcd:/etcd-data 118 | environment: 119 | ETCD_NAME: node1 120 | ETCD_DATA_DIR: /etcd-data/etcd1.etcd 121 | ETCDCTL_API: 3 122 | ETCD_DEBUG: 1 123 | ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd1:2380 124 | ETCD_INITIAL_CLUSTER: node3=http://etcd3:2380,node1=http://etcd1:2380,node2=http://etcd2:2380 125 | ETCD_INITIAL_CLUSTER_STATE: new 126 | ETCD_INITIAL_CLUSTER_TOKEN: etcd-ftw 127 | ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 128 | ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 129 | ETCD_ADVERTISE_CLIENT_URLS: http://etcd1:2379 130 | etcd2: 131 | image: quay.io/coreos/etcd:v3.2.5 132 | networks: 133 | - etcd 134 | ports: 135 | - 23792:2379 136 | - 23802:2380 137 | volumes: 138 | - ./certs/:/srv/ 139 | - /srv/docker/etcd:/etcd-data 140 | environment: 141 | ETCD_NAME: node2 142 | ETCD_DATA_DIR: /etcd-data/etcd2.etcd 143 | ETCDCTL_API: 3 144 | ETCD_DEBUG: 1 145 | ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd2:2380 146 | ETCD_INITIAL_CLUSTER: node3=http://etcd3:2380,node1=http://etcd1:2380,node2=http://etcd2:2380 147 | ETCD_INITIAL_CLUSTER_STATE: new 148 | ETCD_INITIAL_CLUSTER_TOKEN: etcd-ftw 149 | ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 150 | ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 151 | ETCD_ADVERTISE_CLIENT_URLS: http://etcd2:2379 152 | etcd3: 153 | image: quay.io/coreos/etcd:v3.2.5 154 | networks: 155 | - etcd 156 | ports: 157 | - 23793:2379 158 | - 23803:2380 159 | volumes: 160 | - ./certs/:/srv/ 161 | - /srv/docker/etcd:/etcd-data 162 | environment: 163 | ETCD_NAME: node3 164 | ETCD_DATA_DIR: /etcd-data/etcd3.etcd 165 | ETCDCTL_API: 3 166 | ETCD_DEBUG: 1 167 | ETCD_INITIAL_ADVERTISE_PEER_URLS: http://etcd3:2380 168 | ETCD_INITIAL_CLUSTER: node3=http://etcd3:2380,node1=http://etcd1:2380,node2=http://etcd2:2380 169 | ETCD_INITIAL_CLUSTER_STATE: new 170 | ETCD_INITIAL_CLUSTER_TOKEN: etcd-ftw 171 | ETCD_LISTEN_CLIENT_URLS: http://0.0.0.0:2379 172 | ETCD_LISTEN_PEER_URLS: http://0.0.0.0:2380 173 | ETCD_ADVERTISE_CLIENT_URLS: http://etcd3:2379 174 | 175 | skydns: 176 | image: skynetservices/skydns 177 | networks: 178 | - etcd 179 | ports: 180 | - "5354:5354" 181 | - "5354:5354/udp" 182 | 183 | environment: 184 | ETCD_MACHINES: http://etcd1:2379 185 | 186 | networks: 187 | etcd: 188 | consul-demo: 189 | -------------------------------------------------------------------------------- /docker/mongo/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mongo:3.4 2 | 3 | COPY data /tmp/dump 4 | 5 | CMD mongod --fork --logpath /var/log/mongodb.log; \ 6 | mongorestore -d ${MONGO_INITDB_DATABASE} /tmp/dump/; \ 7 | mongod --shutdown; \ 8 | docker-entrypoint.sh mongod 9 | -------------------------------------------------------------------------------- /docker/mongo/data/link.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/copybird/copybird/b2d74e5905a96b9974e67233a972074eab131937/docker/mongo/data/link.bson -------------------------------------------------------------------------------- /docker/mongo/data/link.metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "options": {}, 3 | "indexes": [ 4 | { 5 | "v": 2, 6 | "key": { 7 | "_id": 1 8 | }, 9 | "name": "_id_", 10 | "ns": "datagen_it_test.link" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /docker/mongo/data/test.bson: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/copybird/copybird/b2d74e5905a96b9974e67233a972074eab131937/docker/mongo/data/test.bson -------------------------------------------------------------------------------- /docker/mongo/data/test.metadata.json: -------------------------------------------------------------------------------- 1 | { 2 | "options": {}, 3 | "indexes": [ 4 | { 5 | "v": 2, 6 | "key": { 7 | "_id": 1 8 | }, 9 | "name": "_id_", 10 | "ns": "datagen_it_test.test" 11 | } 12 | ] 13 | } -------------------------------------------------------------------------------- /docker/mongo/restore.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Execute restore in the background after 5s 4 | # https://docs.docker.com/engine/reference/run/#detached--d 5 | sleep 5 && mongorestore /dump & 6 | 7 | # Keep mongod in the foreground, otherwise the container will stop 8 | docker-entrypoint.sh mongod 9 | -------------------------------------------------------------------------------- /docker/ssh/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:9.5 2 | 3 | ARG SSH_MASTER_USER 4 | ARG SSH_MASTER_PASS 5 | 6 | RUN apt-get update \ 7 | && apt-get install -y --no-install-recommends \ 8 | nano \ 9 | sudo \ 10 | openssh-server 11 | 12 | COPY ssh_config /etc/ssh/ssh_config 13 | COPY sshd_config /etc/ssh/sshd_config 14 | 15 | COPY user.sh /usr/local/bin/user.sh 16 | RUN chmod +x /usr/local/bin/user.sh 17 | RUN /usr/local/bin/user.sh 18 | 19 | COPY entrypoint.sh /usr/local/bin/entrypoint.sh 20 | RUN chmod +x /usr/local/bin/entrypoint.sh 21 | 22 | ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] 23 | 24 | CMD tail -f /dev/null 25 | -------------------------------------------------------------------------------- /docker/ssh/entrypoint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | printf "\n\033[0;44m---> Starting the SSH server.\033[0m\n" 5 | 6 | service ssh start 7 | service ssh status 8 | 9 | exec "$@" 10 | -------------------------------------------------------------------------------- /docker/ssh/ssh_config: -------------------------------------------------------------------------------- 1 | # Prevents "Are you sure you want to continue connecting (yes/no)?" question while connecting to the server. 2 | # # The host IP below is the client machine where the ssh command is issued from. 3 | # # Host 192.168.99.* 4 | # # StrictHostKeyChecking no 5 | # # UserKnownHostsFile=/dev/null 6 | # 7 | Host * 8 | HashKnownHosts yes 9 | GSSAPIAuthentication yes 10 | -------------------------------------------------------------------------------- /docker/ssh/sshd_config: -------------------------------------------------------------------------------- 1 | ChallengeResponseAuthentication no 2 | # UsePAM yes # Prints login information 3 | PrintMotd no 4 | X11Forwarding no 5 | AllowTcpForwarding no 6 | AllowAgentForwarding no 7 | PermitTunnel no 8 | Subsystem sftp /usr/lib/openssh/sftp-server 9 | -------------------------------------------------------------------------------- /docker/ssh/user.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | printf "\n\033[0;44m---> Creating SSH master user.\033[0m\n" 5 | 6 | useradd -m -d /home/${SSH_MASTER_USER} -G ssh ${SSH_MASTER_USER} -s /bin/bash 7 | echo "${SSH_MASTER_USER}:${SSH_MASTER_PASS}" | chpasswd 8 | echo 'PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin"' >> /home/${SSH_MASTER_USER}/.profile 9 | 10 | echo "${SSH_MASTER_USER} ALL=NOPASSWD:/bin/rm" >> /etc/sudoers 11 | echo "${SSH_MASTER_USER} ALL=NOPASSWD:/bin/mkdir" >> /etc/sudoers 12 | echo "${SSH_MASTER_USER} ALL=NOPASSWD:/bin/chown" >> /etc/sudoers 13 | echo "${SSH_MASTER_USER} ALL=NOPASSWD:/usr/sbin/useradd" >> /etc/sudoers 14 | echo "${SSH_MASTER_USER} ALL=NOPASSWD:/usr/sbin/deluser" >> /etc/sudoers 15 | echo "${SSH_MASTER_USER} ALL=NOPASSWD:/usr/sbin/chpasswd" >> /etc/sudoers 16 | 17 | exec "$@" 18 | -------------------------------------------------------------------------------- /docker_push.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin 4 | docker push copybird/copybird:${1} -------------------------------------------------------------------------------- /docs/logo.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 5 | 10 | 12 | 13 | 15 | 16 | 17 | 20 | 23 | 25 | 27 | 29 | 31 | 32 | -------------------------------------------------------------------------------- /docs/md-logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/copybird/copybird/b2d74e5905a96b9974e67233a972074eab131937/docs/md-logo.png -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/copybird/copybird 2 | 3 | go 1.12 4 | 5 | require ( 6 | cloud.google.com/go v0.40.0 7 | github.com/PagerDuty/go-pagerduty v0.0.0-20190503230806-cf1437c7c8d6 8 | github.com/Shopify/sarama v1.22.1 9 | github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6 // indirect 10 | github.com/aws/aws-sdk-go v1.20.6 11 | github.com/coreos/bbolt v1.3.3 // indirect 12 | github.com/coreos/etcd v3.3.10+incompatible 13 | github.com/coreos/go-etcd v2.0.0+incompatible // indirect 14 | github.com/coreos/go-semver v0.2.0 // indirect 15 | github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect 16 | github.com/cpuguy83/go-md2man v1.0.10 // indirect 17 | github.com/davecgh/go-spew v1.1.1 18 | github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect 19 | github.com/etcd-io/etcd v3.3.13+incompatible 20 | github.com/go-sql-driver/mysql v1.4.1 21 | github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible 22 | github.com/go-yaml/yaml v2.1.0+incompatible // indirect 23 | github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9 // indirect 24 | github.com/google/go-querystring v1.0.0 // indirect 25 | github.com/gorilla/schema v1.1.0 // indirect 26 | github.com/gorilla/websocket v1.4.1 // indirect 27 | github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect 28 | github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect 29 | github.com/grpc-ecosystem/grpc-gateway v1.12.1 // indirect 30 | github.com/iancoleman/strcase v0.0.0-20190422225806-e506e3ef7365 31 | github.com/imdario/mergo v0.3.7 // indirect 32 | github.com/inconshreveable/mousetrap v1.0.0 // indirect 33 | github.com/jarcoal/httpmock v1.0.4 34 | github.com/jonboulle/clockwork v0.1.0 // indirect 35 | github.com/kelseyhightower/envconfig v1.4.0 36 | github.com/knative/pkg v0.0.0-20190621220722-c2cd40c1c217 37 | github.com/kr/fs v0.1.0 // indirect 38 | github.com/lib/pq v1.1.1 39 | github.com/mitchellh/go-homedir v1.1.0 // indirect 40 | github.com/mitchellh/mapstructure v1.1.2 // indirect 41 | github.com/nats-io/gnatsd v1.4.1 // indirect 42 | github.com/nats-io/go-nats v1.7.2 43 | github.com/nats-io/nkeys v0.0.2 // indirect 44 | github.com/nats-io/nuid v1.0.1 // indirect 45 | github.com/pierrec/lz4 v2.0.5+incompatible 46 | github.com/pkg/sftp v1.10.0 47 | github.com/prometheus/client_golang v1.2.1 // indirect 48 | github.com/sfreiberg/gotwilio v0.0.0-20190522212351-14c666f1d505 49 | github.com/soheilhy/cmux v0.1.4 // indirect 50 | github.com/spf13/cast v1.3.0 // indirect 51 | github.com/spf13/cobra v0.0.3 52 | github.com/spf13/viper v1.2.1 // indirect 53 | github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 54 | github.com/stretchr/testify v1.4.0 55 | github.com/technoweenie/multipartstreamer v1.0.1 // indirect 56 | github.com/tidwall/pretty v1.0.0 // indirect 57 | github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect 58 | github.com/ugorji/go v1.1.1 // indirect 59 | github.com/xconstruct/go-pushbullet v0.0.0-20171206132031-67759df45fbb 60 | github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect 61 | github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77 // indirect 62 | github.com/xwb1989/sqlparser v0.0.0-20180606152119-120387863bf2 63 | go.etcd.io/bbolt v1.3.3 // indirect 64 | go.etcd.io/etcd v3.3.13+incompatible // indirect 65 | go.mongodb.org/mongo-driver v1.0.3 66 | go.uber.org/zap v1.13.0 // indirect 67 | golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5 68 | golang.org/x/sync v0.0.0-20190423024810-112230192c58 69 | google.golang.org/api v0.6.0 70 | google.golang.org/appengine v1.6.1 // indirect 71 | gopkg.in/yaml.v3 v3.0.0-20190502103701-55513cacd4ae // indirect 72 | gotest.tools v2.2.0+incompatible 73 | k8s.io/apimachinery v0.0.0-20191123233150-4c4803ed55e3 // indirect 74 | ) 75 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | 7 | "github.com/copybird/copybird/common" 8 | ) 9 | 10 | func main() { 11 | app := common.NewApp() 12 | if err := app.Run(); err != nil { 13 | log.Printf("run err: %s", err) 14 | os.Exit(1) 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /modules/backup/compress/gzip/config.go: -------------------------------------------------------------------------------- 1 | package gzip 2 | 3 | type Config struct { 4 | Level int 5 | } 6 | -------------------------------------------------------------------------------- /modules/backup/compress/gzip/gzip.go: -------------------------------------------------------------------------------- 1 | package gzip 2 | 3 | import ( 4 | "compress/gzip" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "io" 9 | 10 | "github.com/copybird/copybird/core" 11 | ) 12 | 13 | const MODULE_NAME = "gzip" 14 | const MODULE_GROUP = "backup" 15 | const MODULE_TYPE = "compress" 16 | 17 | type BackupCompressGzip struct { 18 | core.Module 19 | reader io.Reader 20 | writer io.Writer 21 | level int 22 | } 23 | 24 | func (m *BackupCompressGzip) GetName() string { 25 | return MODULE_NAME 26 | } 27 | 28 | func (m *BackupCompressGzip) GetGroup() core.ModuleGroup { 29 | return MODULE_GROUP 30 | } 31 | 32 | func (m *BackupCompressGzip) GetType() core.ModuleType { 33 | return MODULE_TYPE 34 | } 35 | 36 | func (m *BackupCompressGzip) GetConfig() interface{} { 37 | return &Config{ 38 | Level: 3, 39 | } 40 | } 41 | 42 | func (m *BackupCompressGzip) InitModule(_cfg interface{}) error { 43 | cfg := _cfg.(*Config) 44 | level := cfg.Level 45 | if level < -1 || level > 9 { 46 | return errors.New("compression level must be between -1 and 9") 47 | } 48 | m.level = level 49 | return nil 50 | } 51 | 52 | func (m *BackupCompressGzip) Run(ctx context.Context) error { 53 | gw, err := gzip.NewWriterLevel(m.writer, m.level) 54 | if err != nil { 55 | return fmt.Errorf("cant start gzip writer with error: %s", err) 56 | } 57 | defer gw.Close() 58 | 59 | _, err = io.Copy(gw, m.reader) 60 | if err != nil { 61 | return fmt.Errorf("copy error: %s", err) 62 | } 63 | return nil 64 | } 65 | 66 | func (m *BackupCompressGzip) Close() error { 67 | return nil 68 | } 69 | 70 | func (m *BackupCompressGzip) InitPipe(w io.Writer, r io.Reader) error { 71 | m.reader = r 72 | m.writer = w 73 | return nil 74 | } 75 | -------------------------------------------------------------------------------- /modules/backup/compress/gzip/gzip_test.go: -------------------------------------------------------------------------------- 1 | package gzip 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "context" 7 | "io" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | var compressor BackupCompressGzip 14 | var cfg Config 15 | 16 | func TestCompress_InitCompress_Default_Compress(t *testing.T) { 17 | cfg.Level = -1 18 | err := compressor.InitModule(&cfg) 19 | assert.Equal(t, err, nil) 20 | assert.Equal(t, compressor.level, -1) 21 | } 22 | 23 | func TestCompress_InitCompress_Compress_Level_Out_Of_range(t *testing.T) { 24 | cfg.Level = 10 25 | err := compressor.InitModule(&cfg) 26 | assert.NotEqual(t, err, nil) 27 | } 28 | 29 | func TestCompress_Run_Success_Compress(t *testing.T) { 30 | cfg.Level = -1 31 | 32 | rb := bytes.NewReader([]byte("hello, world.")) 33 | wb := new(bytes.Buffer) 34 | 35 | _ = compressor.InitModule(&cfg) 36 | _ = compressor.InitPipe(wb, rb) 37 | err := compressor.Run(context.TODO()) 38 | assert.Equal(t, err, nil) 39 | 40 | var buff2 = new(bytes.Buffer) 41 | gr, err := gzip.NewReader(wb) 42 | defer gr.Close() 43 | 44 | _, err = io.Copy(buff2, gr) 45 | assert.Equal(t, err, nil) 46 | assert.Equal(t, buff2.String(), "hello, world.") 47 | } 48 | -------------------------------------------------------------------------------- /modules/backup/compress/lz4/config.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | type Config struct { 4 | Level int 5 | } 6 | -------------------------------------------------------------------------------- /modules/backup/compress/lz4/lz4.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/copybird/copybird/core" 10 | 11 | "github.com/pierrec/lz4" 12 | ) 13 | 14 | var ( 15 | errCompLevel = errors.New("compression level must be between -1 and 9") 16 | errNotCompressible = errors.New("is not compressible") 17 | ) 18 | 19 | const GROUP_NAME = "backup" 20 | const TYPE_NAME = "compress" 21 | const MODULE_NAME = "lz4" 22 | 23 | // BackupCompressLz4 represents ... 24 | type BackupCompressLz4 struct { 25 | core.Module 26 | reader io.Reader 27 | writer io.Writer 28 | level int 29 | } 30 | 31 | func (m *BackupCompressLz4) GetGroup() core.ModuleGroup { 32 | return GROUP_NAME 33 | } 34 | 35 | func (m *BackupCompressLz4) GetType() core.ModuleType { 36 | return TYPE_NAME 37 | } 38 | 39 | func (m *BackupCompressLz4) GetName() string { 40 | return MODULE_NAME 41 | } 42 | 43 | func (m *BackupCompressLz4) GetConfig() interface{} { 44 | return &Config{Level: 2} 45 | } 46 | 47 | func (m *BackupCompressLz4) InitPipe(w io.Writer, r io.Reader) error { 48 | m.reader = r 49 | m.writer = w 50 | return nil 51 | } 52 | 53 | func (m *BackupCompressLz4) InitModule(_cfg interface{}) error { 54 | cfg := _cfg.(*Config) 55 | if cfg.Level < -1 || cfg.Level > 9 { 56 | return errCompLevel 57 | } 58 | m.level = cfg.Level 59 | return nil 60 | } 61 | 62 | func (m *BackupCompressLz4) Run(ctx context.Context) error { 63 | lw := lz4.NewWriter(m.writer) 64 | lw.Header = lz4.Header{CompressionLevel: m.level} 65 | defer lw.Close() 66 | 67 | _, err := io.Copy(lw, m.reader) 68 | if err != nil { 69 | return fmt.Errorf("copy error: %s", err) 70 | } 71 | return nil 72 | } 73 | 74 | // Close closes compressor 75 | func (m *BackupCompressLz4) Close() error { 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /modules/backup/compress/lz4/lz4_test.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "testing" 8 | 9 | "github.com/pierrec/lz4" 10 | "gotest.tools/assert" 11 | ) 12 | 13 | func TestCompressLZ4(t *testing.T) { 14 | type args struct { 15 | level int 16 | input string 17 | } 18 | 19 | tests := []struct { 20 | name string 21 | args args 22 | want int32 23 | wantErr bool 24 | }{ 25 | // TODO: Add more test cases. 26 | { 27 | name: "invalid-level", 28 | args: args{level: -2, input: "hello, world."}, 29 | wantErr: true, 30 | }, 31 | { 32 | name: "valid-level", 33 | args: args{level: 0, input: "hello, world."}, 34 | wantErr: false, 35 | }, 36 | } 37 | 38 | for _, tt := range tests { 39 | t.Run(tt.name, func(t *testing.T) { 40 | comp := BackupCompressLz4{} 41 | 42 | rb := bytes.NewReader([]byte("hello, world.")) 43 | wb := new(bytes.Buffer) 44 | 45 | assert.Assert(t, comp.GetConfig() != nil) 46 | assert.NilError(t, comp.InitPipe(wb, rb)) 47 | err := comp.InitModule(&Config{Level: tt.args.level}) 48 | if !tt.wantErr && err != nil { 49 | t.Errorf("Compress.BackupCompressLz4() result = %v, want result %v", err, tt.wantErr) 50 | return 51 | } 52 | 53 | err = comp.Run(context.TODO()) 54 | if err != nil { 55 | panic(err) 56 | } 57 | 58 | var buff2 = new(bytes.Buffer) 59 | gr := lz4.NewReader(wb) 60 | _, err = io.Copy(buff2, gr) 61 | assert.Equal(t, err, nil) 62 | assert.Equal(t, buff2.String(), "hello, world.") 63 | 64 | assert.NilError(t, comp.Close()) 65 | }) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /modules/backup/compress/lz4/test.txt: -------------------------------------------------------------------------------- 1 | ssdmnfbsnmdfbmsdnfbsmdnfbdsf -------------------------------------------------------------------------------- /modules/backup/encrypt/aesgcm/aesgcm.go: -------------------------------------------------------------------------------- 1 | package aesgcm 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "crypto/aes" 7 | "crypto/cipher" 8 | "crypto/rand" 9 | "encoding/binary" 10 | "encoding/hex" 11 | "errors" 12 | "fmt" 13 | "io" 14 | 15 | "github.com/copybird/copybird/core" 16 | ) 17 | 18 | const GROUP_NAME = "backup" 19 | const TYPE_NAME = "encrypt" 20 | const MODULE_NAME = "aesgcm" 21 | const BUF_SIZE = 4096 22 | 23 | type BackupEncryptAesgcm struct { 24 | core.Module 25 | reader io.Reader 26 | writer io.Writer 27 | gcm cipher.AEAD 28 | bufReader *bufio.Reader 29 | } 30 | 31 | func (m *BackupEncryptAesgcm) GetGroup() core.ModuleGroup { 32 | return GROUP_NAME 33 | } 34 | 35 | func (m *BackupEncryptAesgcm) GetType() core.ModuleType { 36 | return TYPE_NAME 37 | } 38 | 39 | func (m *BackupEncryptAesgcm) GetName() string { 40 | return MODULE_NAME 41 | } 42 | 43 | func (m *BackupEncryptAesgcm) GetConfig() interface{} { 44 | return &Config{} 45 | } 46 | 47 | func (m *BackupEncryptAesgcm) InitPipe(w io.Writer, r io.Reader) error { 48 | m.reader = r 49 | m.writer = w 50 | m.bufReader = bufio.NewReaderSize(m.reader, 4096) 51 | return nil 52 | } 53 | 54 | func (m *BackupEncryptAesgcm) InitModule(_cfg interface{}) error { 55 | cfg := _cfg.(*Config) 56 | 57 | if cfg.Key == "" { 58 | return errors.New("need key") 59 | } 60 | key, err := hex.DecodeString(cfg.Key) 61 | if err != nil { 62 | return fmt.Errorf("cipher key hex decode err: %s", err) 63 | } 64 | block, err := aes.NewCipher(key) 65 | if err != nil { 66 | return fmt.Errorf("cipher init err: %s", err) 67 | } 68 | 69 | m.gcm, err = cipher.NewGCM(block) 70 | if err != nil { 71 | return fmt.Errorf("aes gcm init err: %s", err) 72 | } 73 | return nil 74 | } 75 | 76 | func (m *BackupEncryptAesgcm) Run(ctx context.Context) error { 77 | var err error 78 | var n int 79 | 80 | nonce := make([]byte, 12) 81 | 82 | originaData := make([]byte, BUF_SIZE) 83 | encryptedData := make([]byte, BUF_SIZE+m.gcm.Overhead()) 84 | 85 | for { 86 | n, err = m.reader.Read(originaData) 87 | if err != nil { 88 | if err == io.EOF { 89 | break 90 | } 91 | return fmt.Errorf("read err: %s", err) 92 | } 93 | 94 | if err = binary.Write(m.writer, binary.LittleEndian, int32(n)); err != nil { 95 | return err 96 | } 97 | 98 | if _, err = io.ReadFull(rand.Reader, nonce); err != nil { 99 | return fmt.Errorf("nonce generate err: %s", err) 100 | } 101 | 102 | if _, err = m.writer.Write(nonce); err != nil { 103 | return fmt.Errorf("nonce write err: %s", err) 104 | } 105 | 106 | m.gcm.Seal(encryptedData, nonce, encryptedData, nil) 107 | _, err = m.writer.Write(encryptedData) 108 | if err != nil { 109 | return fmt.Errorf("write err: %s", err) 110 | } 111 | } 112 | return nil 113 | } 114 | 115 | func (m *BackupEncryptAesgcm) Close() error { 116 | return nil 117 | } 118 | -------------------------------------------------------------------------------- /modules/backup/encrypt/aesgcm/aesgcm_test.go: -------------------------------------------------------------------------------- 1 | package aesgcm 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "gotest.tools/assert" 9 | ) 10 | 11 | func TestEncryptionAESGCM(t *testing.T) { 12 | key := "666f6f6261726b657973616d706c655f" 13 | enc := &BackupEncryptAesgcm{} 14 | 15 | bufInput := bytes.NewBuffer([]byte("hello world")) 16 | bufOutput := &bytes.Buffer{} 17 | assert.Assert(t, enc.GetConfig() != nil) 18 | assert.NilError(t, enc.InitPipe(bufOutput, bufInput)) 19 | assert.NilError(t, enc.InitModule(&Config{Key: key})) 20 | assert.NilError(t, enc.Run(context.TODO())) 21 | assert.NilError(t, enc.Close()) 22 | } 23 | -------------------------------------------------------------------------------- /modules/backup/encrypt/aesgcm/config.go: -------------------------------------------------------------------------------- 1 | package aesgcm 2 | 3 | type Config struct { 4 | Key string 5 | } 6 | -------------------------------------------------------------------------------- /modules/backup/input/consul/config.go: -------------------------------------------------------------------------------- 1 | package consul 2 | 3 | type Config struct{} 4 | -------------------------------------------------------------------------------- /modules/backup/input/consul/consul.go: -------------------------------------------------------------------------------- 1 | package consul 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/copybird/copybird/core" 8 | ) 9 | 10 | // Module Constants 11 | const ( 12 | GroupName = "backup" 13 | TypeName = "input" 14 | ModuleName = "consul" 15 | ) 16 | 17 | type ( 18 | // BackupInputConsul is struct storing inner properties for mysql backups 19 | BackupInputConsul struct { 20 | core.Module 21 | reader io.Reader 22 | writer io.Writer 23 | config *Config 24 | } 25 | ) 26 | 27 | // GetGroup returns group 28 | func (b *BackupInputConsul) GetGroup() core.ModuleGroup { 29 | return GroupName 30 | } 31 | 32 | // GetType returns type 33 | func (b *BackupInputConsul) GetType() core.ModuleType { 34 | return TypeName 35 | } 36 | 37 | // GetName returns name of module 38 | func (b *BackupInputConsul) GetName() string { 39 | return ModuleName 40 | } 41 | 42 | // GetConfig returns config of module 43 | func (b *BackupInputConsul) GetConfig() interface{} { 44 | return &Config{} 45 | } 46 | 47 | // InitPipe initializes pipe 48 | func (b *BackupInputConsul) InitPipe(w io.Writer, r io.Reader) error { 49 | b.reader = r 50 | b.writer = w 51 | return nil 52 | } 53 | 54 | // InitModule initializes module 55 | func (b *BackupInputConsul) InitModule(cfg interface{}) error { 56 | b.config = cfg.(*Config) 57 | return nil 58 | } 59 | 60 | // Run dumps database 61 | func (b *BackupInputConsul) Run(ctx context.Context) error { 62 | return nil 63 | } 64 | 65 | // Close closes ... 66 | func (b *BackupInputConsul) Close() error { 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /modules/backup/input/etcd/config.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | type Config struct { 4 | Endpoints []string 5 | Key string 6 | } 7 | -------------------------------------------------------------------------------- /modules/backup/input/etcd/etcd.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | "sort" 8 | 9 | "github.com/copybird/copybird/core" 10 | "github.com/etcd-io/etcd/client" 11 | ) 12 | 13 | // Module Constants 14 | const ( 15 | GroupName = "backup" 16 | TypeName = "input" 17 | ModuleName = "etcd" 18 | ) 19 | 20 | type ( 21 | // BackupInputEtcd is struct storing inner properties for mysql backups 22 | BackupInputEtcd struct { 23 | core.Module 24 | reader io.Reader 25 | writer io.Writer 26 | config *Config 27 | api client.KeysAPI 28 | } 29 | ) 30 | 31 | // GetGroup returns group 32 | func (b *BackupInputEtcd) GetGroup() core.ModuleGroup { 33 | return GroupName 34 | } 35 | 36 | // GetType returns type 37 | func (b *BackupInputEtcd) GetType() core.ModuleType { 38 | return TypeName 39 | } 40 | 41 | // GetName returns name of module 42 | func (b *BackupInputEtcd) GetName() string { 43 | return ModuleName 44 | } 45 | 46 | // GetConfig returns config of module 47 | func (b *BackupInputEtcd) GetConfig() interface{} { 48 | return &Config{} 49 | } 50 | 51 | // InitPipe initializes pipe 52 | func (b *BackupInputEtcd) InitPipe(w io.Writer, r io.Reader) error { 53 | b.reader = r 54 | b.writer = w 55 | return nil 56 | } 57 | 58 | // InitModule initializes module 59 | func (b *BackupInputEtcd) InitModule(cfg interface{}) error { 60 | b.config = cfg.(*Config) 61 | c, err := client.New(client.Config{ 62 | Endpoints: b.config.Endpoints, 63 | }) 64 | if err != nil { 65 | return err 66 | } 67 | b.api = client.NewKeysAPI(c) 68 | return nil 69 | } 70 | 71 | // Run dumps database 72 | func (b *BackupInputEtcd) Run(ctx context.Context) error { 73 | resp, err := b.api.Get(context.TODO(), b.config.Key, &client.GetOptions{Recursive: true}) 74 | if err != nil { 75 | return err 76 | } 77 | sort.Sort(resp.Node.Nodes) 78 | j := json.NewEncoder(b.writer) 79 | return j.Encode(resp.Node.Nodes) 80 | } 81 | 82 | // Close closes ... 83 | func (b *BackupInputEtcd) Close() error { 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /modules/backup/input/etcd/etcd_test.go: -------------------------------------------------------------------------------- 1 | package etcd 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/etcd-io/etcd/client" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestEtcdBackup(t *testing.T) { 13 | b := &BackupInputEtcd{} 14 | assert.Equal(t, &Config{}, b.GetConfig()) 15 | conf := &Config{Endpoints: []string{"http://127.0.0.1:23791"}, Key: "/test"} 16 | assert.NoError(t, b.InitModule(conf)) 17 | wr := &bytes.Buffer{} 18 | assert.NoError(t, b.InitPipe(wr, nil)) 19 | c, err := client.New(client.Config{Endpoints: conf.Endpoints, Transport: client.DefaultTransport}) 20 | assert.NoError(t, err) 21 | a := client.NewKeysAPI(c) 22 | o := client.SetOptions{Dir: true} 23 | a.Set(context.TODO(), conf.Key, "", &o) 24 | a.Set(context.TODO(), conf.Key+"/test", "World", nil) 25 | a.Set(context.TODO(), conf.Key+"/dir", "Hey", &o) 26 | a.Set(context.TODO(), conf.Key+"/dir/name", "Value", nil) 27 | assert.NoError(t, b.Run(context.TODO())) 28 | a.Delete(context.TODO(), conf.Key, &client.DeleteOptions{Recursive: true}) 29 | 30 | } 31 | -------------------------------------------------------------------------------- /modules/backup/input/etcdv3/config.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | type Config struct { 4 | Endpoints []string 5 | Key string 6 | } 7 | -------------------------------------------------------------------------------- /modules/backup/input/etcdv3/etcdv3.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "io" 7 | 8 | "github.com/copybird/copybird/core" 9 | "github.com/coreos/etcd/clientv3" 10 | "github.com/davecgh/go-spew/spew" 11 | ) 12 | 13 | // Module Constants 14 | const ( 15 | GroupName = "backup" 16 | TypeName = "input" 17 | ModuleName = "etcdv3" 18 | ) 19 | 20 | type ( 21 | // BackupInputEtcd is struct storing inner properties for mysql backups 22 | BackupInputEtcd struct { 23 | core.Module 24 | reader io.Reader 25 | writer io.Writer 26 | config *Config 27 | api clientv3.KV 28 | } 29 | ) 30 | 31 | // GetGroup returns group 32 | func (b *BackupInputEtcd) GetGroup() core.ModuleGroup { 33 | return GroupName 34 | } 35 | 36 | // GetType returns type 37 | func (b *BackupInputEtcd) GetType() core.ModuleType { 38 | return TypeName 39 | } 40 | 41 | // GetName returns name of module 42 | func (b *BackupInputEtcd) GetName() string { 43 | return ModuleName 44 | } 45 | 46 | // GetConfig returns config of module 47 | func (b *BackupInputEtcd) GetConfig() interface{} { 48 | return &Config{} 49 | } 50 | 51 | // InitPipe initializes pipe 52 | func (b *BackupInputEtcd) InitPipe(w io.Writer, r io.Reader) error { 53 | b.reader = r 54 | b.writer = w 55 | return nil 56 | } 57 | 58 | // InitModule initializes module 59 | func (b *BackupInputEtcd) InitModule(cfg interface{}) error { 60 | b.config = cfg.(*Config) 61 | c, err := clientv3.New(clientv3.Config{ 62 | Endpoints: b.config.Endpoints, 63 | }) 64 | if err != nil { 65 | return err 66 | } 67 | b.api = clientv3.NewKV(c) 68 | return nil 69 | } 70 | 71 | // Run dumps database 72 | func (b *BackupInputEtcd) Run(ctx context.Context) error { 73 | resp, err := b.api.Get(context.TODO(), b.config.Key) 74 | if err != nil { 75 | return err 76 | } 77 | spew.Dump(resp.Kvs) 78 | j := json.NewEncoder(b.writer) 79 | return j.Encode(resp.Kvs) 80 | } 81 | 82 | // Close closes ... 83 | func (b *BackupInputEtcd) Close() error { 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /modules/backup/input/etcdv3/etcdv3_test.go: -------------------------------------------------------------------------------- 1 | package etcdv3 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/coreos/etcd/clientv3" 9 | "github.com/stretchr/testify/assert" 10 | ) 11 | 12 | func TestEtcdBackup(t *testing.T) { 13 | b := &BackupInputEtcd{} 14 | assert.Equal(t, &Config{}, b.GetConfig()) 15 | conf := &Config{Endpoints: []string{"http://127.0.0.1:23791"}, Key: "/test"} 16 | assert.NoError(t, b.InitModule(conf)) 17 | wr := &bytes.Buffer{} 18 | assert.NoError(t, b.InitPipe(wr, nil)) 19 | c, err := clientv3.New(clientv3.Config{Endpoints: conf.Endpoints}) 20 | assert.NoError(t, err) 21 | a := clientv3.NewKV(c) 22 | a.Put(context.TODO(), conf.Key, "") 23 | a.Put(context.TODO(), conf.Key+"/test", "World") 24 | a.Put(context.TODO(), conf.Key+"/dir", "Hey") 25 | a.Put(context.TODO(), conf.Key+"/dir/name", "Value") 26 | assert.NoError(t, b.Run(context.TODO())) 27 | a.Delete(context.TODO(), conf.Key, clientv3.WithPrefix()) 28 | 29 | } 30 | -------------------------------------------------------------------------------- /modules/backup/input/local/config.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | type Config struct { 4 | Filename string 5 | } 6 | -------------------------------------------------------------------------------- /modules/backup/input/local/local.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | 8 | "github.com/copybird/copybird/core" 9 | ) 10 | 11 | // Module Constants 12 | const ( 13 | GroupName = "backup" 14 | TypeName = "input" 15 | ModuleName = "local" 16 | ) 17 | 18 | type ( 19 | // BackupInputLocal is struct storing inner properties for mysql backups 20 | BackupInputLocal struct { 21 | core.Module 22 | reader io.Reader 23 | writer io.Writer 24 | config *Config 25 | } 26 | ) 27 | 28 | // GetGroup returns group 29 | func (b *BackupInputLocal) GetGroup() core.ModuleGroup { 30 | return GroupName 31 | } 32 | 33 | // GetType returns type 34 | func (b *BackupInputLocal) GetType() core.ModuleType { 35 | return TypeName 36 | } 37 | 38 | // GetName returns name of module 39 | func (b *BackupInputLocal) GetName() string { 40 | return ModuleName 41 | } 42 | 43 | // GetConfig returns config of module 44 | func (b *BackupInputLocal) GetConfig() interface{} { 45 | return &Config{} 46 | } 47 | 48 | // InitPipe initializes pipe 49 | func (b *BackupInputLocal) InitPipe(w io.Writer, r io.Reader) error { 50 | b.reader = r 51 | b.writer = w 52 | return nil 53 | } 54 | 55 | // InitModule initializes module 56 | func (b *BackupInputLocal) InitModule(cfg interface{}) error { 57 | b.config = cfg.(*Config) 58 | return nil 59 | } 60 | 61 | // Run dumps database 62 | func (b *BackupInputLocal) Run(ctx context.Context) error { 63 | f, err := os.Open(b.config.Filename) 64 | if err != nil { 65 | return err 66 | } 67 | if _, err := io.Copy(b.writer, f); err != nil { 68 | return err 69 | } 70 | return nil 71 | } 72 | 73 | // Close closes ... 74 | func (b *BackupInputLocal) Close() error { 75 | return nil 76 | } 77 | -------------------------------------------------------------------------------- /modules/backup/input/local/local_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestLocalInput(t *testing.T) { 12 | b := &BackupInputLocal{} 13 | wr := &bytes.Buffer{} 14 | assert.NoError(t, b.InitPipe(wr, nil)) 15 | assert.NoError(t, b.InitModule(&Config{Filename: "test.file"})) 16 | assert.Equal(t, &Config{}, b.GetConfig()) 17 | assert.NoError(t, b.Run(context.TODO())) 18 | assert.Equal(t, "test\n", wr.String()) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /modules/backup/input/local/test.file: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /modules/backup/input/mongodb/config.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | type ( 4 | 5 | // MongoConfig represents configuration 6 | // for mongo backup 7 | MongoConfig struct { 8 | DSN string 9 | } 10 | ) 11 | -------------------------------------------------------------------------------- /modules/backup/input/mongodb/mongo.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "fmt" 7 | "io" 8 | "time" 9 | 10 | "github.com/copybird/copybird/core" 11 | 12 | "go.mongodb.org/mongo-driver/bson" 13 | "go.mongodb.org/mongo-driver/mongo" 14 | "go.mongodb.org/mongo-driver/mongo/options" 15 | "go.mongodb.org/mongo-driver/x/bsonx" 16 | ) 17 | 18 | // Module Constants 19 | const ( 20 | groupName = "backup" 21 | typeName = "input" 22 | moduleName = "mongodb" 23 | ) 24 | 25 | var ( 26 | timeout = time.Second * 2 27 | header = `{"database":"%s","collection":"%s","time":"%d"}` 28 | ) 29 | 30 | type ( 31 | // BackupInputMongodb represents struct for dumping mongo 32 | BackupInputMongodb struct { 33 | core.Module 34 | config *MongoConfig 35 | reader io.Reader 36 | writer io.Writer 37 | conn *mongo.Client 38 | rootCtx context.Context 39 | cancelFn func() 40 | dbCount int 41 | collCount int 42 | docCount int64 43 | bytesOut uint64 44 | recCount uint64 45 | startTimestamp time.Time 46 | } 47 | ) 48 | 49 | // SetDefaultTimeout sets default timeout used for operations 50 | func SetDefaultTimeout(t time.Duration) { 51 | timeout = t 52 | } 53 | 54 | // InitPipe initializes pipes 55 | func (m *BackupInputMongodb) InitPipe(w io.Writer, r io.Reader) error { 56 | m.reader = r 57 | m.writer = w 58 | 59 | return nil 60 | } 61 | 62 | // InitModule initializes module 63 | func (m *BackupInputMongodb) InitModule(cfg interface{}) error { 64 | conf, ok := cfg.(*MongoConfig) 65 | if !ok { 66 | return fmt.Errorf("config type mismatch, expected: %T actual: %T", m.config, cfg) 67 | } 68 | m.config = conf 69 | 70 | rCtx, cancel := context.WithCancel(context.Background()) 71 | m.cancelFn = cancel 72 | m.rootCtx = rCtx 73 | 74 | ctx, cancel := context.WithTimeout(m.rootCtx, timeout) 75 | defer cancel() 76 | 77 | conn, err := mongo.Connect(ctx, options.Client().ApplyURI(m.config.DSN)) 78 | if err != nil { 79 | return err 80 | } 81 | 82 | m.conn = conn 83 | return nil 84 | } 85 | 86 | // Run runs module 87 | func (m *BackupInputMongodb) Run(ctx context.Context) error { 88 | m.startTimestamp = time.Now() 89 | 90 | dbs, err := m.getDatabases(m.rootCtx) 91 | if err != nil { 92 | return fmt.Errorf("unable to fetch databases: %v", err) 93 | } 94 | 95 | for _, db := range dbs { 96 | colls, err := m.getCollections(m.rootCtx, db) 97 | if err != nil { 98 | return fmt.Errorf("unable to fetch collectioins: %v", err) 99 | } 100 | 101 | for _, coll := range colls { 102 | if err := m.exportCollection(db, coll); err != nil { 103 | return fmt.Errorf("unable to export collection: %v", err) 104 | } 105 | m.collCount++ 106 | } 107 | m.dbCount++ 108 | } 109 | _ = fmt.Sprintf("exported [databases: %d collections: %d documents: %d bytes out: %d duration: %v]", m.dbCount, m.collCount, m.bytesOut, m.docCount, time.Since(m.startTimestamp).Seconds()) 110 | m.cancelFn() 111 | return nil 112 | } 113 | 114 | // Close disconnects from the server 115 | func (m *BackupInputMongodb) Close() error { 116 | m.cancelFn() 117 | return m.conn.Disconnect(m.rootCtx) 118 | } 119 | 120 | func (m *BackupInputMongodb) getDatabases(ctx context.Context) ([]string, error) { 121 | return m.conn.ListDatabaseNames(ctx, bsonx.Doc{}) 122 | } 123 | 124 | func (m *BackupInputMongodb) getCollections(ctx context.Context, dbName string) ([]string, error) { 125 | var colls []string 126 | 127 | collections, err := m.conn.Database(dbName).ListCollections(m.rootCtx, bson.M{}) 128 | if err != nil { 129 | return colls, err 130 | } 131 | 132 | for collections.Next(m.rootCtx) { 133 | colNameRaw := collections.Current.Lookup("name") 134 | colName, ok := colNameRaw.StringValueOK() 135 | if !ok { 136 | return colls, fmt.Errorf("invalid collection name: %v", colNameRaw) 137 | } 138 | colls = append(colls, colName) 139 | } 140 | return colls, nil 141 | 142 | } 143 | 144 | func (m *BackupInputMongodb) exportCollection(dbName, collName string) error { 145 | curr, err := m.conn.Database(dbName).Collection(collName).Find(m.rootCtx, bson.D{}) 146 | if err != nil { 147 | return fmt.Errorf("unable to fetch documents :%v", err) 148 | } 149 | defer curr.Close(m.rootCtx) 150 | 151 | if _, err = m.writer.Write([]byte(fmt.Sprintf(header, dbName, collName, time.Now().UnixNano()) + "\n")); err != nil { 152 | return fmt.Errorf("unable to write header: %v", err) 153 | } 154 | 155 | for curr.Next(m.rootCtx) { 156 | var res bson.M 157 | if err := curr.Decode(&res); err != nil { 158 | return fmt.Errorf("unable to decode document: %v", err) 159 | } 160 | 161 | data, err := json.Marshal(res) 162 | if err != nil { 163 | return fmt.Errorf("unable to marshal document: %v", err) 164 | } 165 | data = append(data, "\n"...) 166 | n, err := m.writer.Write(data) 167 | if err != nil { 168 | return fmt.Errorf("unable to write document data: %v", err) 169 | } 170 | m.bytesOut += uint64(n) 171 | m.docCount++ 172 | if n != len(data) { 173 | return fmt.Errorf("expected write: %d actual: %d", len(data), n) 174 | } 175 | } 176 | 177 | return nil 178 | } 179 | 180 | // GetGroup returns group 181 | func (m *BackupInputMongodb) GetGroup() core.ModuleGroup { return groupName } 182 | 183 | // GetType returns type 184 | func (m *BackupInputMongodb) GetType() core.ModuleType { return typeName } 185 | 186 | // GetName returns name 187 | func (m *BackupInputMongodb) GetName() string { return moduleName } 188 | 189 | // GetConfig returns config 190 | func (m *BackupInputMongodb) GetConfig() interface{} { return &MongoConfig{} } 191 | -------------------------------------------------------------------------------- /modules/backup/input/mongodb/mongo_test.go: -------------------------------------------------------------------------------- 1 | package mongodb 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestMongoBackup2(t *testing.T) { 13 | d := &BackupInputMongodb{} 14 | c := d.GetConfig().(*MongoConfig) 15 | c.DSN = "mongodb://127.0.0.1:27017" 16 | require.NoError(t, d.InitModule(c)) 17 | names, err := d.getDatabases(context.TODO()) 18 | assert.NoError(t, err) 19 | assert.Equal(t, []string{"admin", "local", "test"}, names) 20 | collections, err := d.getCollections(context.TODO(), "test") 21 | assert.NoError(t, err) 22 | assert.Equal(t, []string{"link", "test"}, collections) 23 | 24 | } 25 | 26 | func TestExportCollection(t *testing.T) { 27 | d := &BackupInputMongodb{} 28 | c := d.GetConfig().(*MongoConfig) 29 | c.DSN = "mongodb://127.0.0.1:27017" 30 | require.NoError(t, d.InitModule(c)) 31 | tmpFile, err := os.Create("./export.txt") 32 | if !assert.NoError(t, err, "tmp file") { 33 | t.Fail() 34 | } 35 | 36 | d.writer = tmpFile 37 | 38 | assert.NoError(t, d.exportCollection("admin", "system.version"), "export collection") 39 | tmpFile.Close() 40 | // os.Remove(tmpFile.Name()) 41 | } 42 | -------------------------------------------------------------------------------- /modules/backup/input/mysql/config.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | // MySQLConfig stores configuration for MySQL backups 4 | type MySQLConfig struct { 5 | DSN string 6 | } 7 | -------------------------------------------------------------------------------- /modules/backup/input/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "strconv" 11 | "strings" 12 | "text/template" 13 | "time" 14 | 15 | "github.com/copybird/copybird/core" 16 | 17 | _ "github.com/go-sql-driver/mysql" 18 | ) 19 | 20 | // Module Constants 21 | const GroupName = "backup" 22 | const TypeName = "input" 23 | const ModuleName = "mysql" 24 | 25 | type ( 26 | // BackupInputMysql is struct storing inner properties for mysql backups 27 | BackupInputMysql struct { 28 | core.Module 29 | conn *sql.Tx 30 | headerTemplate *template.Template 31 | footerTemplate *template.Template 32 | tableTemplate *template.Template 33 | endTableTemplate *template.Template 34 | config *MySQLConfig 35 | reader io.Reader 36 | writer io.Writer 37 | } 38 | dumpHeader struct { 39 | Version string 40 | } 41 | dumpFooter struct { 42 | EndTime string 43 | } 44 | table struct { 45 | Name string 46 | Schema string 47 | Data string 48 | } 49 | ) 50 | 51 | // GetGroup returns group 52 | func (m *BackupInputMysql) GetGroup() core.ModuleGroup { 53 | return GroupName 54 | } 55 | 56 | // GetType returns type 57 | func (m *BackupInputMysql) GetType() core.ModuleType { 58 | return TypeName 59 | } 60 | 61 | // GetName returns name of module 62 | func (m *BackupInputMysql) GetName() string { 63 | return ModuleName 64 | } 65 | 66 | // GetConfig returns config of module 67 | func (m *BackupInputMysql) GetConfig() interface{} { 68 | return &MySQLConfig{ 69 | DSN: "root:root@tcp(localhost:3306)/test", 70 | } 71 | } 72 | 73 | // InitPipe initializes pipe 74 | func (m *BackupInputMysql) InitPipe(w io.Writer, r io.Reader) error { 75 | m.reader = r 76 | m.writer = w 77 | return nil 78 | } 79 | 80 | // InitModule initializes module 81 | func (m *BackupInputMysql) InitModule(cfg interface{}) error { 82 | m.config = cfg.(*MySQLConfig) 83 | conn, err := sql.Open("mysql", m.config.DSN) 84 | if err != nil { 85 | return err 86 | } 87 | if err := conn.Ping(); err != nil { 88 | return err 89 | } 90 | tx, err := conn.Begin() 91 | if err != nil { 92 | return err 93 | } 94 | m.conn = tx 95 | t, err := template.New("headerTemplate").Parse(headerTemplate) 96 | if err != nil { 97 | return err 98 | } 99 | m.headerTemplate = t 100 | t, err = template.New("footerTemplate").Parse(footerTemplate) 101 | if err != nil { 102 | return err 103 | } 104 | m.footerTemplate = t 105 | t, err = template.New("tableTemplate").Parse(tableTemplate) 106 | if err != nil { 107 | return err 108 | } 109 | m.tableTemplate = t 110 | t, err = template.New("endTableTemplate").Parse(endTableTemplate) 111 | if err != nil { 112 | return err 113 | } 114 | m.endTableTemplate = t 115 | return nil 116 | } 117 | 118 | // Run dumps database 119 | func (m *BackupInputMysql) Run(ctx context.Context) error { 120 | return m.dumpDatabase() 121 | } 122 | 123 | // Close closes ... 124 | func (m *BackupInputMysql) Close() error { 125 | return nil 126 | } 127 | func (m *BackupInputMysql) getTables() ([]string, error) { 128 | tables := []string{} 129 | rows, err := m.conn.Query("SHOW TABLES") 130 | if err != nil { 131 | return tables, err 132 | } 133 | defer rows.Close() 134 | for rows.Next() { 135 | var table sql.NullString 136 | if err := rows.Scan(&table); err != nil { 137 | return tables, err 138 | } 139 | tables = append(tables, table.String) 140 | } 141 | return tables, rows.Err() 142 | } 143 | 144 | func (m *BackupInputMysql) getTableSchema(name string) (string, error) { 145 | q := fmt.Sprintf("SHOW CREATE TABLE %s", name) 146 | var returnTable sql.NullString 147 | var sqlTable sql.NullString 148 | if err := m.conn.QueryRow(q).Scan(&returnTable, &sqlTable); err != nil { 149 | return "", err 150 | } 151 | if returnTable.String != name { 152 | return "", errors.New("wrong table returned") 153 | } 154 | return sqlTable.String, nil 155 | } 156 | 157 | func (m *BackupInputMysql) writeTableData(name string) error { 158 | q := fmt.Sprintf("SELECT * FROM %s", name) 159 | rows, err := m.conn.Query(q) 160 | if err != nil { 161 | return err 162 | } 163 | defer rows.Close() 164 | rowsCount := 0 165 | for rows.Next() { 166 | rowsCount++ 167 | } 168 | if rowsCount == 0 { 169 | return nil 170 | } 171 | rows, err = m.conn.Query(q) 172 | if err != nil { 173 | return err 174 | } 175 | defer rows.Close() 176 | columns, err := rows.Columns() 177 | if err != nil { 178 | return err 179 | } 180 | if len(columns) == 0 { 181 | return fmt.Errorf("no columns in table %s", name) 182 | } 183 | if _, err := io.WriteString(m.writer, fmt.Sprintf("INSERT INTO `%s` VALUES ", name)); err != nil { 184 | return err 185 | } 186 | currCount := 0 187 | for rows.Next() { 188 | currCount++ 189 | scanData := make([]sql.RawBytes, len(columns)) 190 | pointers := make([]interface{}, len(columns)) 191 | for i := range scanData { 192 | pointers[i] = &scanData[i] 193 | } 194 | if err := rows.Scan(pointers...); err != nil { 195 | return err 196 | } 197 | rowData := make([]string, len(columns)) 198 | for i, v := range scanData { 199 | if v != nil { 200 | if _, err := strconv.Atoi(string(v)); err == nil { 201 | rowData[i] = string(v) 202 | } else { 203 | rowData[i] = fmt.Sprintf("'%s'", strings.Replace(string(v), "'", "\\'", -1)) 204 | } 205 | } else { 206 | rowData[i] = "NULL" 207 | } 208 | json.Unmarshal(v, pointers) 209 | } 210 | _, err = io.WriteString(m.writer, fmt.Sprintf("(%s)", strings.Join(rowData, ","))) 211 | if err != nil { 212 | return err 213 | } 214 | if currCount == rowsCount { 215 | if _, err := io.WriteString(m.writer, ";"); err != nil { 216 | return err 217 | } 218 | } else { 219 | if _, err := io.WriteString(m.writer, ","); err != nil { 220 | return err 221 | } 222 | } 223 | 224 | } 225 | return rows.Err() 226 | 227 | } 228 | func (m *BackupInputMysql) dumpDatabase() error { 229 | version, err := m.getServerVersion() 230 | if err != nil { 231 | return err 232 | } 233 | if err := m.headerTemplate.Execute(m.writer, dumpHeader{Version: version}); err != nil { 234 | return err 235 | } 236 | tables, err := m.getTables() 237 | if err != nil { 238 | return err 239 | } 240 | for _, tableName := range tables { 241 | var table table 242 | table.Name = tableName 243 | schema, err := m.getTableSchema(tableName) 244 | if err != nil { 245 | return err 246 | } 247 | table.Schema = schema 248 | if err := m.tableTemplate.Execute(m.writer, table); err != nil { 249 | return err 250 | } 251 | if err := m.writeTableData(tableName); err != nil { 252 | return err 253 | } 254 | 255 | if err := m.endTableTemplate.Execute(m.writer, table); err != nil { 256 | return err 257 | } 258 | } 259 | return m.footerTemplate.Execute(m.writer, dumpFooter{EndTime: time.Now().String()}) 260 | } 261 | func (m *BackupInputMysql) getServerVersion() (string, error) { 262 | var version sql.NullString 263 | if err := m.conn.QueryRow("SELECT version()").Scan(&version); err != nil { 264 | return version.String, nil 265 | } 266 | return version.String, nil 267 | } 268 | -------------------------------------------------------------------------------- /modules/backup/input/mysql/mysql_test.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | const authorsSchema = "CREATE TABLE `authors` (\n" + 14 | " `id` int(11) NOT NULL AUTO_INCREMENT,\n" + 15 | " `first_name` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" + 16 | " `last_name` varchar(50) COLLATE utf8_unicode_ci NOT NULL,\n" + 17 | " `email` varchar(100) COLLATE utf8_unicode_ci NOT NULL,\n" + 18 | " `birthdate` date NOT NULL,\n" + 19 | " `added` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP,\n" + 20 | " PRIMARY KEY (`id`),\n" + 21 | " UNIQUE KEY `email` (`email`)\n" + 22 | ") ENGINE=InnoDB AUTO_INCREMENT=101 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci" 23 | 24 | func TestGetTables(t *testing.T) { 25 | m := &BackupInputMysql{} 26 | c := m.GetConfig().(*MySQLConfig) 27 | c.DSN = "root:root@tcp(localhost:3306)/test" 28 | 29 | require.NoError(t, m.InitModule(c)) 30 | f, err := os.Create("dump.sql") 31 | assert.NoError(t, err) 32 | assert.NoError(t, m.InitPipe(f, nil)) 33 | tables, err := m.getTables() 34 | require.NoError(t, err) 35 | assert.Equal(t, 1, len(tables)) 36 | assert.Equal(t, "authors", tables[0]) 37 | tableSchema, err := m.getTableSchema(tables[0]) 38 | assert.NoError(t, err) 39 | assert.Equal(t, authorsSchema, tableSchema) 40 | err = m.Run(context.TODO()) 41 | assert.NoError(t, err) 42 | assert.NoError(t, os.Remove("dump.sql")) 43 | } 44 | 45 | func TestMysqlDump(t *testing.T) { 46 | m := &BackupInputMysql{} 47 | 48 | c := m.GetConfig().(*MySQLConfig) 49 | c.DSN = "root:root@tcp(localhost:3306)/test" 50 | 51 | require.NoError(t, m.InitModule(c)) 52 | buf := bytes.Buffer{} 53 | require.NoError(t, m.InitPipe(&buf, nil)) 54 | assert.NoError(t, m.Run(context.TODO())) 55 | t.Log(buf.String()) 56 | } 57 | -------------------------------------------------------------------------------- /modules/backup/input/mysql/templates.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | const ( 4 | headerTemplate = ` 5 | -- 6 | -- ------------------------------------------------------ 7 | -- Server version {{ .Version }} 8 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 9 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 10 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 11 | /*!40101 SET NAMES utf8mb4 */; 12 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 13 | /*!40103 SET TIME_ZONE='+00:00' */; 14 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 15 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 16 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 17 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 18 | ` 19 | tableTemplate = ` 20 | -- 21 | -- Table structure for table {{ .Name }} 22 | -- 23 | DROP TABLE IF EXISTS {{ .Name }}; 24 | /*!40101 SET @saved_cs_client = @@character_set_client */; 25 | /*!40101 SET character_set_client = utf8mb4 */; 26 | {{ .Schema }}; 27 | /*!40101 SET character_set_client = @saved_cs_client */; 28 | -- 29 | -- Dumping data for table {{ .Name }} 30 | -- 31 | LOCK TABLES {{ .Name }} WRITE; 32 | /*!40000 ALTER TABLE {{ .Name }} DISABLE KEYS */; 33 | ` 34 | endTableTemplate = ` 35 | /*!40000 ALTER TABLE {{ .Name }} ENABLE KEYS */; 36 | UNLOCK TABLES; 37 | ` 38 | footerTemplate = "-- Dump completed on {{ .EndTime }}" 39 | ) 40 | -------------------------------------------------------------------------------- /modules/backup/input/mysqldump/config.go: -------------------------------------------------------------------------------- 1 | package mysqldump 2 | 3 | // MySQLDumpConfig stores configuration for Mysqldump utility 4 | type MySQLDumpConfig struct { 5 | Host string 6 | Port string 7 | Username string 8 | Password string 9 | Database string 10 | Routines bool 11 | Events bool 12 | Triggers bool 13 | SingleTransaction bool 14 | ColumnStatistics bool 15 | } 16 | -------------------------------------------------------------------------------- /modules/backup/input/mysqldump/mysqldump.go: -------------------------------------------------------------------------------- 1 | package mysqldump 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "io" 7 | "os" 8 | "os/exec" 9 | 10 | "github.com/copybird/copybird/core" 11 | 12 | _ "github.com/go-sql-driver/mysql" 13 | ) 14 | 15 | // Module Constants 16 | const GroupName = "backup" 17 | const TypeName = "input" 18 | const ModuleName = "mysqldump" 19 | 20 | type ( 21 | // BackupInputMysqlDump is struct storing inner properties for mysql backups 22 | BackupInputMysqlDump struct { 23 | core.Module 24 | command string 25 | args []string 26 | config *MySQLDumpConfig 27 | reader io.Reader 28 | writer io.Writer 29 | } 30 | dumpHeader struct { 31 | Version string 32 | } 33 | dumpFooter struct { 34 | EndTime string 35 | } 36 | table struct { 37 | Name string 38 | Schema string 39 | Data string 40 | } 41 | ) 42 | 43 | // GetGroup returns group 44 | func (m *BackupInputMysqlDump) GetGroup() core.ModuleGroup { 45 | return GroupName 46 | } 47 | 48 | // GetType returns type 49 | func (m *BackupInputMysqlDump) GetType() core.ModuleType { 50 | return TypeName 51 | } 52 | 53 | // GetName returns name of module 54 | func (m *BackupInputMysqlDump) GetName() string { 55 | return ModuleName 56 | } 57 | 58 | // GetConfig returns config of module 59 | func (m *BackupInputMysqlDump) GetConfig() interface{} { 60 | return &MySQLDumpConfig{ 61 | Host: "127.0.0.1", 62 | Port: "3306", 63 | Username: "root", 64 | Password: "root", 65 | Database: "test", 66 | Routines: true, 67 | Events: true, 68 | Triggers: true, 69 | SingleTransaction: true, 70 | ColumnStatistics: false, 71 | } 72 | } 73 | 74 | // InitPipe initializes pipe 75 | func (m *BackupInputMysqlDump) InitPipe(w io.Writer, r io.Reader) error { 76 | m.reader = r 77 | m.writer = w 78 | return nil 79 | } 80 | 81 | // InitModule initializes module 82 | func (m *BackupInputMysqlDump) InitModule(cfg interface{}) error { 83 | m.config = cfg.(*MySQLDumpConfig) 84 | 85 | command, err := exec.LookPath("mysqldump") 86 | if err != nil { 87 | return fmt.Errorf("%s cannot be found", command) 88 | } 89 | args := []string{ 90 | fmt.Sprintf("-h%s", m.config.Host), 91 | fmt.Sprintf("-P%s", m.config.Port), 92 | fmt.Sprintf("-u%s", m.config.Username), 93 | fmt.Sprintf("-p%s", m.config.Password), 94 | fmt.Sprintf("--triggers=%t", m.config.Triggers), 95 | fmt.Sprintf("--routines=%t", m.config.Routines), 96 | fmt.Sprintf("--events=%t", m.config.Events), 97 | fmt.Sprintf("--single-transaction=%t", m.config.SingleTransaction), 98 | // fmt.Sprintf("--column-statistics=%t", m.config.ColumnStatistics), 99 | m.config.Database, 100 | } 101 | 102 | m.command = command 103 | m.args = args 104 | return nil 105 | } 106 | 107 | // Run dumps database 108 | func (m *BackupInputMysqlDump) Run(ctx context.Context) error { 109 | cmd := exec.Command(m.command, m.args...) 110 | cmd.Stdout = m.writer 111 | cmd.Stderr = os.Stderr 112 | return cmd.Run() 113 | } 114 | -------------------------------------------------------------------------------- /modules/backup/input/mysqldump/mysqldump_test.go: -------------------------------------------------------------------------------- 1 | package mysqldump 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestMysqlDumpRun(t *testing.T) { 13 | m := &BackupInputMysqlDump{} 14 | c := m.GetConfig().(*MySQLDumpConfig) 15 | c.Host = "127.0.0.1" 16 | c.Port = "3306" 17 | c.Username = "root" 18 | c.Password = "root" 19 | c.Database = "test" 20 | 21 | require.NoError(t, m.InitModule(c)) 22 | buf := bytes.Buffer{} 23 | assert.NoError(t, m.InitPipe(&buf, nil)) 24 | assert.NoError(t, m.Run(context.TODO())) 25 | t.Log(buf.String()) 26 | } 27 | -------------------------------------------------------------------------------- /modules/backup/input/postgresql/config.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | var defaultSchemaName = "public" 4 | 5 | // Config stores configuration for PostgreSQL backups 6 | type ( 7 | Config struct { 8 | DSN string 9 | } 10 | 11 | tableScheme struct { 12 | columnName, columnDefault, dataType, characterMaximumLength, isNullable, constraintName, constraintType, sequence string 13 | } 14 | sequenceScheme struct { 15 | name string 16 | } 17 | ) 18 | -------------------------------------------------------------------------------- /modules/backup/input/postgresql/postgresql.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "encoding/json" 7 | "strconv" 8 | "strings" 9 | "time" 10 | 11 | "fmt" 12 | "io" 13 | 14 | "text/template" 15 | 16 | "github.com/copybird/copybird/core" 17 | _ "github.com/lib/pq" 18 | ) 19 | 20 | // Module Constants 21 | const GROUP_NAME = "backup" 22 | const TYPE_NAME = "input" 23 | const moduleName = "postgresql" 24 | 25 | type ( 26 | // BackupInputPostgresql is struct storing inner properties for mysql backups 27 | BackupInputPostgresql struct { 28 | core.Module 29 | conn *sql.DB 30 | data dbDump 31 | template *template.Template 32 | config *Config 33 | reader io.Reader 34 | writer io.Writer 35 | } 36 | dbDump struct { 37 | Version string 38 | Tables []table 39 | EndTime string 40 | DBScheme string 41 | } 42 | table struct { 43 | Name string 44 | Schema string 45 | SequenceScheme string 46 | Data string 47 | DBScheme string 48 | } 49 | ) 50 | 51 | // GetGroup returns group 52 | func (m *BackupInputPostgresql) GetGroup() core.ModuleGroup { 53 | return GROUP_NAME 54 | } 55 | 56 | // GetType returns type 57 | func (m *BackupInputPostgresql) GetType() core.ModuleType { 58 | return TYPE_NAME 59 | } 60 | 61 | // GetName returns name of module 62 | func (m *BackupInputPostgresql) GetName() string { 63 | return moduleName 64 | } 65 | 66 | // GetConfig returns Config of module 67 | func (m *BackupInputPostgresql) GetConfig() interface{} { 68 | return &Config{} 69 | } 70 | 71 | // InitPipe initializes pipe 72 | func (m *BackupInputPostgresql) InitPipe(w io.Writer, r io.Reader) error { 73 | m.reader = r 74 | m.writer = w 75 | return nil 76 | } 77 | 78 | // InitModule initializes module 79 | func (m *BackupInputPostgresql) InitModule(cfg interface{}) error { 80 | m.config = cfg.(*Config) 81 | conn, err := sql.Open("postgres", m.config.DSN) 82 | if err != nil { 83 | return err 84 | } 85 | if err := conn.Ping(); err != nil { 86 | return err 87 | } 88 | m.conn = conn 89 | return nil 90 | } 91 | 92 | // Run dumps database 93 | func (m *BackupInputPostgresql) Run(ctx context.Context) error { 94 | 95 | if err := m.dumpDatabase(); err != nil { 96 | return err 97 | } 98 | if err := m.template.Execute(m.writer, m.data); err != nil { 99 | return err 100 | } 101 | return nil 102 | } 103 | 104 | // Close closes ... 105 | func (m *BackupInputPostgresql) Close() error { 106 | return nil 107 | } 108 | func (m *BackupInputPostgresql) getTables() ([]string, error) { 109 | var ( 110 | tables []string 111 | tableType = "BASE TABLE" 112 | ) 113 | rows, err := m.conn.Query(fmt.Sprintf("SELECT table_name FROM information_schema.tables WHERE table_schema='%s' AND table_type='%s'", defaultSchemaName, tableType)) 114 | if err != nil { 115 | return tables, err 116 | } 117 | defer rows.Close() 118 | 119 | for rows.Next() { 120 | var table sql.NullString 121 | if err := rows.Scan(&table); err != nil { 122 | return tables, err 123 | } 124 | tables = append(tables, table.String) 125 | } 126 | return tables, rows.Err() 127 | } 128 | 129 | func (d *BackupInputPostgresql) getTableSchema(tableName string) ([]tableScheme, []sequenceScheme, error) { 130 | 131 | var ( 132 | columns []tableScheme 133 | sequence []sequenceScheme 134 | ) 135 | rows, err := d.conn.Query(` 136 | select cln.table_name, 137 | cln.column_name, 138 | cln.column_default, 139 | cln.data_type, 140 | cln.character_maximum_length, 141 | cln.is_nullable, 142 | tc.constraint_name, 143 | tc.constraint_type, 144 | pg_get_serial_sequence($2, cln.column_name) AS sequence 145 | from INFORMATION_SCHEMA.COLUMNS cln 146 | LEFT JOIN INFORMATION_SCHEMA.constraint_column_usage ctu ON ctu.table_schema = cln.table_schema 147 | AND ctu.column_name = cln.column_name 148 | and ctu.table_name = cln.table_name 149 | LEFT JOIN INFORMATION_SCHEMA.table_constraints tc ON tc.constraint_schema = cln.table_schema 150 | and tc.table_name = cln.table_name 151 | and tc.constraint_name = ctu.constraint_name 152 | where cln.table_schema = $1 153 | and cln.table_name = $2`, defaultSchemaName, tableName) 154 | if err != nil { 155 | return columns, sequence, err 156 | } 157 | defer rows.Close() 158 | 159 | for rows.Next() { 160 | var tableName, columnName, columnDefault, dataType, characterMaximumLength, isNullable, constraintName, constraintType, sequenceName sql.NullString 161 | if err := rows.Scan(&tableName, &columnName, &columnDefault, &dataType, &characterMaximumLength, &isNullable, &constraintName, &constraintType, &sequenceName); err != nil { 162 | return columns, sequence, err 163 | } 164 | columns = append(columns, tableScheme{ 165 | columnName: columnName.String, 166 | columnDefault: columnDefault.String, 167 | dataType: dataType.String, 168 | characterMaximumLength: characterMaximumLength.String, 169 | isNullable: isNullable.String, 170 | constraintName: constraintName.String, 171 | constraintType: constraintType.String, 172 | sequence: sequenceName.String, 173 | }) 174 | sequence = append(sequence, sequenceScheme{name: sequenceName.String}) 175 | } 176 | 177 | return columns, sequence, nil 178 | } 179 | 180 | func (d *BackupInputPostgresql) tableSequenceDump(tableName string, schemas []sequenceScheme) string { 181 | var sequence []string 182 | for _, schema := range schemas { 183 | if schema.name != "" { 184 | sequence = append(sequence, fmt.Sprintf("drop sequence IF EXISTS %s;\ncreate sequence %s;", schema.name, schema.name)) 185 | } 186 | } 187 | return fmt.Sprintf(strings.Join(sequence, ";")) 188 | } 189 | 190 | func (d *BackupInputPostgresql) tableSchemeDump(tableName string, schemas []tableScheme) string { 191 | 192 | var tableColumns []string 193 | for _, schema := range schemas { 194 | var defaultValue, isNull, constraint string 195 | var columnType = schema.dataType 196 | if schema.columnDefault != "" { 197 | defaultValue = fmt.Sprintf("default %s", schema.columnDefault) 198 | } 199 | 200 | if schema.characterMaximumLength != "" { 201 | columnType = fmt.Sprintf("%s(%s)", schema.dataType, schema.characterMaximumLength) 202 | } 203 | 204 | if schema.isNullable == "NO" { 205 | isNull = "not null" 206 | } 207 | 208 | if schema.constraintName != "" { 209 | constraint = fmt.Sprintf("constraint %s %s", schema.constraintName, schema.constraintType) 210 | } 211 | tableColumns = append(tableColumns, fmt.Sprintf("%s %s %s %s %s", schema.columnName, columnType, defaultValue, isNull, constraint)) 212 | } 213 | 214 | return fmt.Sprintf("CREATE TABLE %s (%s);", tableName, strings.Join(tableColumns, ",")) 215 | } 216 | 217 | func (d *BackupInputPostgresql) getTableData(name string) (string, error) { 218 | 219 | rows, err := d.conn.Query(fmt.Sprintf(`SELECT * FROM %s`, name)) 220 | if err != nil { 221 | return "", err 222 | } 223 | defer rows.Close() 224 | 225 | columns, err := rows.Columns() 226 | if err != nil { 227 | return "", err 228 | } 229 | 230 | if len(columns) == 0 { 231 | return "", fmt.Errorf("no columns in table %s", name) 232 | } 233 | 234 | var data []string 235 | for rows.Next() { 236 | var ( 237 | scanData = make([]sql.RawBytes, len(columns)) 238 | pointers = make([]interface{}, len(columns)) 239 | ) 240 | 241 | for i := range scanData { 242 | pointers[i] = &scanData[i] 243 | } 244 | 245 | if err := rows.Scan(pointers...); err != nil { 246 | return "", err 247 | } 248 | 249 | rowData := make([]string, len(columns)) 250 | for i, v := range scanData { 251 | if v != nil { 252 | if _, err := strconv.Atoi(string(v)); err == nil { 253 | rowData[i] = string(v) 254 | } else { 255 | rowData[i] = fmt.Sprintf("'%s'", strings.Replace(string(v), "'", "\\'", -1)) 256 | } 257 | } else { 258 | rowData[i] = "NULL" 259 | } 260 | json.Unmarshal(v, pointers) 261 | } 262 | data = append(data, fmt.Sprintf("(%s)", strings.Join(rowData, ","))) 263 | 264 | } 265 | return strings.Join(data, ","), rows.Err() 266 | 267 | } 268 | func (d *BackupInputPostgresql) dumpDatabase() error { 269 | var dump dbDump 270 | version, err := d.getServerVersion() 271 | if err != nil { 272 | return err 273 | } 274 | 275 | tables, err := d.getTables() 276 | if err != nil { 277 | return err 278 | } 279 | 280 | dump.Version = version 281 | dump.DBScheme = defaultSchemaName 282 | dump.EndTime = time.Now().String() 283 | 284 | for _, tableName := range tables { 285 | var table = table{ 286 | Name: tableName, 287 | DBScheme: defaultSchemaName, 288 | } 289 | 290 | tableSchema, sequenceSchema, err := d.getTableSchema(tableName) 291 | if err != nil { 292 | return err 293 | } 294 | 295 | data, err := d.getTableData(tableName) 296 | if err != nil { 297 | return err 298 | } 299 | 300 | table.Schema = d.tableSchemeDump(tableName, tableSchema) 301 | table.SequenceScheme = d.tableSequenceDump(tableName, sequenceSchema) 302 | table.Data = data 303 | dump.Tables = append(dump.Tables, table) 304 | } 305 | 306 | t, err := template.New("postgresqlbackup").Parse(dumpTemplate) 307 | if err != nil { 308 | return err 309 | } 310 | 311 | d.template = t 312 | d.data = dump 313 | return nil 314 | 315 | } 316 | func (d *BackupInputPostgresql) getServerVersion() (string, error) { 317 | var version sql.NullString 318 | if err := d.conn.QueryRow("SELECT version()").Scan(&version); err != nil { 319 | return version.String, nil 320 | } 321 | return version.String, nil 322 | } 323 | -------------------------------------------------------------------------------- /modules/backup/input/postgresql/postgresql_test.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | const authorsSchema = `CREATE TABLE authors (id integer default nextval('authors_id_seq'::regclass) not null constraint authors_pk PRIMARY KEY,first_name character varying(100) not null ,last_name character varying(100) not null ,email character varying(100) ,created timestamp without time zone default now() not null );` 14 | const sequenceSchema = `drop sequence IF EXISTS public.authors_id_seq; 15 | create sequence public.authors_id_seq;` 16 | const authorsData = "(1,'test','test','te@asd.ru','2019-06-22T13:26:37.078767Z'),(2,'vanya','ivanov',NULL,'2019-06-22T14:45:54.81458Z')" 17 | 18 | func TestGetTables(t *testing.T) { 19 | d := &BackupInputPostgresql{} 20 | c := d.GetConfig().(*Config) 21 | c.DSN = "host=127.0.0.1 port=5432 user=postgres password=postgres dbname=test sslmode=disable" 22 | 23 | f, err := os.Create("dump.sql") 24 | assert.NoError(t, err) 25 | require.NoError(t, d.InitModule(c)) 26 | assert.NoError(t, d.InitPipe(f, nil)) 27 | 28 | tables, err := d.getTables() 29 | assert.NoError(t, err) 30 | assert.Equal(t, "authors", tables[0]) 31 | assert.Equal(t, "posts", tables[1]) 32 | 33 | tableSchema, seqSchema, err := d.getTableSchema(tables[0]) 34 | fmt.Println("err", err, tableSchema, seqSchema) 35 | 36 | assert.NoError(t, err) 37 | assert.Equal(t, authorsSchema, d.tableSchemeDump(tables[0], tableSchema)) 38 | assert.Equal(t, sequenceSchema, d.tableSequenceDump(tables[0], seqSchema)) 39 | 40 | data, err := d.getTableData(tables[0]) 41 | assert.NoError(t, err) 42 | assert.Equal(t, authorsData, data) 43 | 44 | assert.NoError(t, d.dumpDatabase()) 45 | 46 | assert.NoError(t, d.Run(context.TODO())) 47 | 48 | assert.NoError(t, os.Remove("dump.sql")) 49 | } 50 | -------------------------------------------------------------------------------- /modules/backup/input/postgresql/templates.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | const dumpTemplate = ` 4 | -- 5 | -- PostgreSQL database dump 6 | -- 7 | 8 | -- Dumped from database version {{ .Version }} 9 | 10 | SET statement_timeout = 0; 11 | SET lock_timeout = 0; 12 | SET idle_in_transaction_session_timeout = 0; 13 | SET client_encoding = 'UTF8'; 14 | SET standard_conforming_strings = on; 15 | SELECT pg_catalog.set_config('search_path', '{{ .DBScheme }}', false); 16 | SET check_function_bodies = false; 17 | SET client_min_messages = warning; 18 | SET row_security = off; 19 | 20 | SET default_tablespace = ''; 21 | SET default_with_oids = false; 22 | 23 | {{range .Tables}} 24 | 25 | DROP TABLE IF EXISTS {{ .Name }}; 26 | -- 27 | -- Name: {{ .Name }}; Type: TABLE; Schema: {{ .DBScheme }}; Owner: - 28 | -- 29 | {{ .SequenceScheme }} 30 | 31 | -- 32 | -- Name: {{ .Name }}; Type: SEQUENCE; Schema: {{ .DBScheme }}; Owner: - 33 | -- 34 | 35 | {{ .Schema }} 36 | 37 | -- 38 | -- Data for Name: {{ .Name }}; Type: TABLE DATA; Schema: {{ .DBScheme }}; Owner: - 39 | -- 40 | 41 | {{ if .Data }} 42 | INSERT INTO {{ .Name }} VALUES {{ .Data }}; 43 | {{ end }} 44 | 45 | {{ end }} 46 | 47 | -- Dump completed on {{ .EndTime }} 48 | ` 49 | -------------------------------------------------------------------------------- /modules/backup/input/tar/config.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | type Config struct { 4 | DirectoryPath string 5 | } 6 | -------------------------------------------------------------------------------- /modules/backup/input/tar/tar.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | import ( 4 | "archive/tar" 5 | "context" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | 11 | "github.com/copybird/copybird/core" 12 | "github.com/davecgh/go-spew/spew" 13 | ) 14 | 15 | // Module Constants 16 | const ( 17 | GroupName = "backup" 18 | TypeName = "input" 19 | ModuleName = "tar" 20 | ) 21 | 22 | type ( 23 | // BackupInputTar is struct storing inner properties for mysql backups 24 | BackupInputTar struct { 25 | core.Module 26 | reader io.Reader 27 | writer io.Writer 28 | config *Config 29 | } 30 | ) 31 | 32 | // GetGroup returns group 33 | func (b *BackupInputTar) GetGroup() core.ModuleGroup { 34 | return GroupName 35 | } 36 | 37 | // GetType returns type 38 | func (b *BackupInputTar) GetType() core.ModuleType { 39 | return TypeName 40 | } 41 | 42 | // GetName returns name of module 43 | func (b *BackupInputTar) GetName() string { 44 | return ModuleName 45 | } 46 | 47 | // GetConfig returns config of module 48 | func (b *BackupInputTar) GetConfig() interface{} { 49 | return &Config{} 50 | } 51 | 52 | // InitPipe initializes pipe 53 | func (b *BackupInputTar) InitPipe(w io.Writer, r io.Reader) error { 54 | b.reader = r 55 | b.writer = w 56 | return nil 57 | } 58 | 59 | // InitModule initializes module 60 | func (b *BackupInputTar) InitModule(cfg interface{}) error { 61 | b.config = cfg.(*Config) 62 | return nil 63 | } 64 | 65 | // Run dumps database 66 | func (b *BackupInputTar) Run(ctx context.Context) error { 67 | if _, err := os.Stat(b.config.DirectoryPath); err != nil { 68 | return err 69 | } 70 | tw := tar.NewWriter(b.writer) 71 | defer tw.Close() 72 | return filepath.Walk(b.config.DirectoryPath, func(file string, fi os.FileInfo, err error) error { 73 | spew.Dump(file) 74 | 75 | if err != nil { 76 | return err 77 | } 78 | if fi.IsDir() { 79 | return nil 80 | } 81 | 82 | header, err := tar.FileInfoHeader(fi, fi.Name()) 83 | if err != nil { 84 | return err 85 | } 86 | 87 | header.Name = strings.TrimPrefix(strings.Replace(file, b.config.DirectoryPath, "", -1), string(filepath.Separator)) 88 | 89 | if err := tw.WriteHeader(header); err != nil { 90 | return err 91 | } 92 | 93 | f, err := os.Open(file) 94 | if err != nil { 95 | return err 96 | } 97 | defer f.Close() 98 | 99 | if _, err := io.Copy(tw, f); err != nil { 100 | return err 101 | } 102 | 103 | return nil 104 | }) 105 | return nil 106 | } 107 | 108 | // Close closes ... 109 | func (b *BackupInputTar) Close() error { 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /modules/backup/input/tar/tar_test.go: -------------------------------------------------------------------------------- 1 | package tar 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | func TestLocalInput(t *testing.T) { 12 | wr := &bytes.Buffer{} 13 | b := &BackupInputTar{} 14 | assert.NoError(t, b.InitPipe(wr, nil)) 15 | assert.NoError(t, b.InitModule(&Config{DirectoryPath: "target"})) 16 | assert.Equal(t, &Config{}, b.GetConfig()) 17 | assert.NoError(t, b.Run(context.TODO())) 18 | assert.NotNil(t, wr.Bytes()) 19 | 20 | } 21 | -------------------------------------------------------------------------------- /modules/backup/input/tar/target/test.file: -------------------------------------------------------------------------------- 1 | test 2 | -------------------------------------------------------------------------------- /modules/backup/output/gcp/config.go: -------------------------------------------------------------------------------- 1 | package gcp 2 | 3 | type Config struct { 4 | AuthFile string 5 | Bucket string 6 | File string 7 | } 8 | -------------------------------------------------------------------------------- /modules/backup/output/gcp/gcp.go: -------------------------------------------------------------------------------- 1 | package gcp 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github.com/copybird/copybird/core" 7 | "io" 8 | 9 | "cloud.google.com/go/storage" 10 | "google.golang.org/api/option" 11 | ) 12 | 13 | // Module Constants 14 | const GROUP_NAME = "backup" 15 | const TYPE_NAME = "output" 16 | const MODULE_NAME = "gcp" 17 | 18 | type BackupOutputGcp struct { 19 | core.Module 20 | ctx context.Context 21 | reader io.Reader 22 | writer io.Writer 23 | client *storage.Client 24 | bucket *storage.BucketHandle 25 | config *Config 26 | } 27 | 28 | func (m *BackupOutputGcp) GetGroup() core.ModuleGroup { 29 | return GROUP_NAME 30 | } 31 | 32 | func (m *BackupOutputGcp) GetType() core.ModuleType { 33 | return TYPE_NAME 34 | } 35 | 36 | func (m *BackupOutputGcp) GetName() string { 37 | return MODULE_NAME 38 | } 39 | 40 | func (m *BackupOutputGcp) GetConfig() interface{} { 41 | return &Config{} 42 | } 43 | 44 | func (m *BackupOutputGcp) InitPipe(w io.Writer, r io.Reader) error { 45 | m.reader = r 46 | m.writer = w 47 | return nil 48 | } 49 | 50 | func (m *BackupOutputGcp) InitModule(_config interface{}) error { 51 | m.config = _config.(*Config) 52 | 53 | if m.config.AuthFile == "" { 54 | return errors.New("need auth_file") 55 | } 56 | if m.config.Bucket == "" { 57 | return errors.New("need bucker") 58 | } 59 | if m.config.File == "" { 60 | return errors.New("need file") 61 | } 62 | 63 | m.ctx = context.Background() 64 | 65 | switch { 66 | case m.config.AuthFile != "": 67 | client, err := storage.NewClient(m.ctx, option.WithCredentialsFile(m.config.AuthFile)) 68 | if err != nil { 69 | return err 70 | } 71 | m.client = client 72 | default: 73 | client, err := storage.NewClient(m.ctx) 74 | if err != nil { 75 | return err 76 | } 77 | m.client = client 78 | } 79 | 80 | m.bucket = m.client.Bucket(m.config.Bucket) 81 | // check if the bucket exists 82 | if _, err := m.bucket.Attrs(m.ctx); err != nil { 83 | return err 84 | } 85 | 86 | return nil 87 | } 88 | 89 | func (m *BackupOutputGcp) Run(ctx context.Context) error { 90 | 91 | obj := m.bucket.Object(m.config.File) 92 | w := obj.NewWriter(m.ctx) 93 | if _, err := io.Copy(w, m.reader); err != nil { 94 | return err 95 | } 96 | 97 | if err := w.Close(); err != nil { 98 | return err 99 | } 100 | 101 | _, err := obj.Attrs(m.ctx) 102 | return err 103 | } 104 | 105 | func (m *BackupOutputGcp) Close() error { 106 | m.client.Close() 107 | return nil 108 | } 109 | -------------------------------------------------------------------------------- /modules/backup/output/gcp/gcp_test.go: -------------------------------------------------------------------------------- 1 | package gcp 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestGetName(t *testing.T) { 11 | gcp := &BackupOutputGcp{} 12 | name := gcp.GetName() 13 | require.Equal(t, "gcp", name) 14 | } 15 | 16 | func TestGetConfig(t *testing.T) { 17 | gcp := &BackupOutputGcp{} 18 | conf := gcp.GetConfig() 19 | require.Equal(t, &Config{}, conf) 20 | } 21 | 22 | func TestInitPipe(t *testing.T) { 23 | gcp := &BackupOutputGcp{} 24 | bufInput := bytes.NewBuffer([]byte("hello world")) 25 | bufOutput := &bytes.Buffer{} 26 | require.NoError(t, gcp.InitPipe(bufOutput, bufInput)) 27 | } 28 | func TestInitModule(t *testing.T) { 29 | gcp := &BackupOutputGcp{} 30 | 31 | err := gcp.InitModule(&Config{AuthFile: ""}) 32 | require.Error(t, err, "Should fail to find credentials") 33 | 34 | err = gcp.InitModule(&Config{AuthFile: "creds.json"}) 35 | require.Error(t, err, "credentials file is missing") 36 | } 37 | -------------------------------------------------------------------------------- /modules/backup/output/http/config.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | type Config struct { 4 | TargetUrl string 5 | } 6 | -------------------------------------------------------------------------------- /modules/backup/output/http/http.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "net/http" 7 | 8 | "github.com/copybird/copybird/core" 9 | ) 10 | 11 | // Module Constants 12 | const GROUP_NAME = "backup" 13 | const TYPE_NAME = "output" 14 | const MODULE_NAME = "http" 15 | 16 | type BackupOutputHttp struct { 17 | core.Module 18 | reader io.Reader 19 | writer io.Writer 20 | config *Config 21 | } 22 | 23 | func (m *BackupOutputHttp) GetGroup() core.ModuleGroup { 24 | return GROUP_NAME 25 | } 26 | 27 | func (m *BackupOutputHttp) GetType() core.ModuleType { 28 | return TYPE_NAME 29 | } 30 | 31 | func (m *BackupOutputHttp) GetName() string { 32 | return MODULE_NAME 33 | } 34 | 35 | func (m *BackupOutputHttp) GetConfig() interface{} { 36 | return &Config{} 37 | } 38 | 39 | func (m *BackupOutputHttp) InitPipe(w io.Writer, r io.Reader) error { 40 | m.reader = r 41 | m.writer = w 42 | return nil 43 | } 44 | 45 | func (m *BackupOutputHttp) InitModule(cfg interface{}) error { 46 | m.config = cfg.(*Config) 47 | 48 | return nil 49 | } 50 | 51 | func (m *BackupOutputHttp) Run(ctx context.Context) error { 52 | resp, err := http.Post(m.config.TargetUrl, "application/json", m.reader) 53 | if err != nil { 54 | return err 55 | } 56 | 57 | defer resp.Body.Close() 58 | 59 | return nil 60 | } 61 | 62 | func (m *BackupOutputHttp) Close() error { 63 | return nil 64 | } 65 | -------------------------------------------------------------------------------- /modules/backup/output/http/http_test.go: -------------------------------------------------------------------------------- 1 | package http 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestGetName(t *testing.T) { 12 | h := &BackupOutputHttp{} 13 | require.Equal(t, "http", h.GetName()) 14 | } 15 | 16 | func TestGetConfig(t *testing.T) { 17 | h := &BackupOutputHttp{} 18 | require.Equal(t, &Config{}, h.GetConfig()) 19 | } 20 | 21 | func TestInitModule(t *testing.T) { 22 | h := &BackupOutputHttp{} 23 | err := h.InitModule(&Config{TargetUrl: "https://test.com"}) 24 | require.NoError(t, err, "should not be any error here") 25 | } 26 | 27 | func TestRun(t *testing.T) { 28 | h := &BackupOutputHttp{} 29 | conf := &Config{ 30 | TargetUrl: "https://test.com", 31 | } 32 | err := h.InitModule(conf) 33 | require.NoError(t, err) 34 | err = h.Run(context.TODO()) 35 | assert.NoError(t, err) 36 | } 37 | -------------------------------------------------------------------------------- /modules/backup/output/local/config.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | type Config struct { 4 | File string 5 | DefaultMask int 6 | } 7 | -------------------------------------------------------------------------------- /modules/backup/output/local/local.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "os" 7 | 8 | "github.com/copybird/copybird/core" 9 | ) 10 | 11 | // Module Constants 12 | const GROUP_NAME = "backup" 13 | const TYPE_NAME = "output" 14 | const MODULE_NAME = "local" 15 | 16 | type BackupOutputLocal struct { 17 | core.Module 18 | reader io.Reader 19 | writer io.Writer 20 | config *Config 21 | } 22 | 23 | func (m *BackupOutputLocal) GetGroup() core.ModuleGroup { 24 | return GROUP_NAME 25 | } 26 | 27 | func (m *BackupOutputLocal) GetType() core.ModuleType { 28 | return TYPE_NAME 29 | } 30 | 31 | func (m *BackupOutputLocal) GetName() string { 32 | return MODULE_NAME 33 | } 34 | 35 | func (m *BackupOutputLocal) GetConfig() interface{} { 36 | return &Config{ 37 | File: "output", 38 | DefaultMask: os.O_APPEND | os.O_CREATE | os.O_WRONLY, 39 | } 40 | } 41 | 42 | func (m *BackupOutputLocal) InitPipe(w io.Writer, r io.Reader) error { 43 | m.reader = r 44 | m.writer = w 45 | return nil 46 | } 47 | 48 | func (m *BackupOutputLocal) InitModule(_config interface{}) error { 49 | m.config = _config.(*Config) 50 | return nil 51 | } 52 | 53 | func (m *BackupOutputLocal) Run(ctx context.Context) error { 54 | 55 | // If the file doesn't exist, create it, or append to the file 56 | f, err := os.OpenFile(m.config.File, m.config.DefaultMask, 0644) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | defer f.Close() 62 | 63 | _, err = io.Copy(f, m.reader) 64 | return err 65 | } 66 | 67 | func (m *BackupOutputLocal) Close() error { 68 | return nil 69 | } 70 | -------------------------------------------------------------------------------- /modules/backup/output/local/local_test.go: -------------------------------------------------------------------------------- 1 | package local 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "os" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestGetName(t *testing.T) { 13 | l := &BackupOutputLocal{} 14 | require.Equal(t, "local", l.GetName()) 15 | } 16 | 17 | func TestGetConfig(t *testing.T) { 18 | l := &BackupOutputLocal{} 19 | conf := l.GetConfig() 20 | require.Equal(t, &Config{ 21 | DefaultMask: os.O_APPEND | os.O_CREATE | os.O_WRONLY, 22 | File: "output", 23 | }, conf) 24 | } 25 | 26 | func TestInitPipe(t *testing.T) { 27 | l := &BackupOutputLocal{} 28 | bufInput := bytes.NewBuffer([]byte("hello world")) 29 | bufOutput := &bytes.Buffer{} 30 | require.NoError(t, l.InitPipe(bufOutput, bufInput)) 31 | } 32 | 33 | func TestRun(t *testing.T) { 34 | l := &BackupOutputLocal{} 35 | bufInput := bytes.NewBuffer([]byte("hello world")) 36 | bufOutput := &bytes.Buffer{} 37 | require.NoError(t, l.InitPipe(bufOutput, bufInput)) 38 | conf := &Config{ 39 | DefaultMask: os.O_APPEND | os.O_CREATE | os.O_WRONLY, 40 | File: "test.txt", 41 | } 42 | err := l.InitModule(conf) 43 | require.NoError(t, err) 44 | err = l.Run(context.TODO()) 45 | require.NoError(t, err) 46 | os.Remove("test.txt") 47 | } 48 | 49 | func TestInitModule(t *testing.T) { 50 | l := &BackupOutputLocal{} 51 | err := l.InitModule(&Config{File: "test.sql"}) 52 | require.NoError(t, err, "should not be any error here") 53 | } 54 | -------------------------------------------------------------------------------- /modules/backup/output/s3/config.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | type Config struct { 4 | Region string 5 | AccessKeyID string 6 | SecretAccessKey string 7 | Bucket string 8 | FileName string 9 | } 10 | -------------------------------------------------------------------------------- /modules/backup/output/s3/s3.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/credentials" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/s3/s3manager" 11 | "github.com/copybird/copybird/core" 12 | ) 13 | 14 | // Module Constants 15 | const GROUP_NAME = "backup" 16 | const TYPE_NAME = "output" 17 | const MODULE_NAME = "s3" 18 | 19 | type BackupOutputS3 struct { 20 | core.Module 21 | reader io.Reader 22 | writer io.Writer 23 | session *session.Session 24 | config *Config 25 | } 26 | 27 | func (m *BackupOutputS3) GetGroup() core.ModuleGroup { 28 | return GROUP_NAME 29 | } 30 | 31 | func (m *BackupOutputS3) GetType() core.ModuleType { 32 | return TYPE_NAME 33 | } 34 | 35 | func (m *BackupOutputS3) GetName() string { 36 | return MODULE_NAME 37 | } 38 | 39 | func (m *BackupOutputS3) GetConfig() interface{} { 40 | return &Config{} 41 | } 42 | 43 | func (m *BackupOutputS3) InitPipe(w io.Writer, r io.Reader) error { 44 | m.reader = r 45 | m.writer = w 46 | return nil 47 | } 48 | 49 | func (m *BackupOutputS3) InitModule(_config interface{}) error { 50 | m.config = _config.(*Config) 51 | session, err := session.NewSession(&aws.Config{ 52 | Region: aws.String(m.config.Region), 53 | Credentials: credentials.NewStaticCredentials(m.config.AccessKeyID, m.config.SecretAccessKey, ""), 54 | }) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | m.session = session 60 | return nil 61 | } 62 | 63 | func (m *BackupOutputS3) Run(ctx context.Context) error { 64 | 65 | svc := s3manager.NewUploader(m.session) 66 | 67 | input := &s3manager.UploadInput{ 68 | Bucket: aws.String(m.config.Bucket), 69 | Key: aws.String(m.config.FileName), 70 | Body: m.reader, 71 | } 72 | 73 | _, err := svc.Upload(input) 74 | if err != nil { 75 | return err 76 | } 77 | return nil 78 | } 79 | 80 | func (m *BackupOutputS3) Close() error { 81 | return nil 82 | } 83 | -------------------------------------------------------------------------------- /modules/backup/output/s3/s3_test.go: -------------------------------------------------------------------------------- 1 | package s3 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestGetName(t *testing.T) { 11 | s := &BackupOutputS3{} 12 | require.Equal(t, "s3", s.GetName()) 13 | } 14 | 15 | func TestGetConfig(t *testing.T) { 16 | s := &BackupOutputS3{} 17 | require.Equal(t, &Config{}, s.GetConfig()) 18 | } 19 | 20 | func TestInitPipe(t *testing.T) { 21 | s := &BackupOutputS3{} 22 | bufInput := bytes.NewBuffer([]byte("hello world")) 23 | bufOutput := &bytes.Buffer{} 24 | require.NoError(t, s.InitPipe(bufOutput, bufInput)) 25 | } 26 | 27 | func TestInitModule(t *testing.T) { 28 | s := &BackupOutputS3{} 29 | err := s.InitModule(&Config{Region: "us-east-1"}) 30 | require.NoError(t, err, "should not be any error here") 31 | } 32 | -------------------------------------------------------------------------------- /modules/backup/output/scp/README.md: -------------------------------------------------------------------------------- 1 | before tests we should run `ssh 127.0.0.1` and accept invitation 2 | -------------------------------------------------------------------------------- /modules/backup/output/scp/config.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | type Config struct { 4 | Addr string 5 | Port int 6 | User string 7 | Password string 8 | FileName string 9 | PathToKey string 10 | PrivateKeyPassword string 11 | } 12 | -------------------------------------------------------------------------------- /modules/backup/output/scp/scp.go: -------------------------------------------------------------------------------- 1 | package scp 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "github.com/copybird/copybird/core" 14 | 15 | "github.com/pkg/sftp" 16 | "golang.org/x/crypto/ssh" 17 | ) 18 | 19 | // Module Constants 20 | const GROUP_NAME = "backup" 21 | const TYPE_NAME = "output" 22 | const MODULE_NAME = "scp" 23 | 24 | type BackupOutputScp struct { 25 | core.Module 26 | reader io.Reader 27 | writer io.Writer 28 | config *Config 29 | sess ssh.Session 30 | client *sftp.Client 31 | } 32 | 33 | func (m *BackupOutputScp) GetGroup() core.ModuleGroup { 34 | return GROUP_NAME 35 | } 36 | 37 | func (m *BackupOutputScp) GetType() core.ModuleType { 38 | return TYPE_NAME 39 | } 40 | 41 | func (m *BackupOutputScp) GetName() string { 42 | return MODULE_NAME 43 | } 44 | 45 | func (m *BackupOutputScp) GetConfig() interface{} { 46 | return &Config{} 47 | } 48 | 49 | func (m *BackupOutputScp) InitPipe(w io.Writer, r io.Reader) error { 50 | m.reader = r 51 | m.writer = w 52 | return nil 53 | } 54 | 55 | func (m *BackupOutputScp) InitModule(_config interface{}) error { 56 | m.config = _config.(*Config) 57 | 58 | // get host public key 59 | hostKey, err := getHostKey(m.config.Addr) 60 | if err != nil { 61 | return err 62 | } 63 | 64 | //TODO maybe also check for nil hostkey 65 | var clientConfig *ssh.ClientConfig 66 | 67 | if m.config.PathToKey != "" { 68 | priv, err := ioutil.ReadFile(m.config.PathToKey) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | signer, err := ssh.ParsePrivateKey([]byte(priv)) 74 | if err != nil && err.Error() != "ssh: cannot decode encrypted private keys" { 75 | return err 76 | } 77 | 78 | if err.Error() == "ssh: cannot decode encrypted private keys" { 79 | signer, err = ssh.ParsePrivateKeyWithPassphrase(priv, []byte(m.config.PrivateKeyPassword)) 80 | if err != nil { 81 | return err 82 | } 83 | } 84 | 85 | clientConfig = &ssh.ClientConfig{ 86 | User: m.config.User, 87 | Auth: []ssh.AuthMethod{ 88 | ssh.PublicKeys(signer), 89 | }, 90 | HostKeyCallback: ssh.FixedHostKey(hostKey), 91 | } 92 | 93 | } else { 94 | clientConfig = &ssh.ClientConfig{ 95 | User: m.config.User, 96 | Auth: []ssh.AuthMethod{ 97 | ssh.Password(m.config.Password), 98 | }, 99 | HostKeyCallback: ssh.FixedHostKey(hostKey), 100 | } 101 | } 102 | 103 | conn, err := ssh.Dial("tcp", fmt.Sprintf("%s:%d", m.config.Addr, m.config.Port), clientConfig) 104 | if err != nil { 105 | return fmt.Errorf("Failed to dial: " + err.Error()) 106 | } 107 | defer conn.Close() 108 | 109 | // create new SFTP client 110 | client, err := sftp.NewClient(conn) 111 | if err != nil { 112 | return err 113 | } 114 | m.client = client 115 | return nil 116 | } 117 | 118 | func (m *BackupOutputScp) Run(ctx context.Context) error { 119 | 120 | // create destination file 121 | dstFile, err := m.client.Create(m.config.FileName) 122 | if err != nil { 123 | return err 124 | } 125 | defer dstFile.Close() 126 | 127 | // copy bytes from reader to destination file 128 | _, err = io.Copy(dstFile, m.reader) 129 | return err 130 | } 131 | 132 | func (m *BackupOutputScp) Close() error { 133 | m.client.Close() 134 | return nil 135 | } 136 | 137 | func getHostKey(host string) (ssh.PublicKey, error) { 138 | // parse OpenSSH known_hosts file 139 | // ssh or use ssh-keyscan to get initial key 140 | file, err := os.Open(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts")) 141 | if err != nil { 142 | return nil, err 143 | } 144 | defer file.Close() 145 | 146 | scanner := bufio.NewScanner(file) 147 | var hostKey ssh.PublicKey 148 | for scanner.Scan() { 149 | fields := strings.Split(scanner.Text(), " ") 150 | if len(fields) != 3 { 151 | continue 152 | } 153 | if strings.Contains(fields[0], host) { 154 | var err error 155 | hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes()) 156 | if err != nil { 157 | return nil, fmt.Errorf("error parsing %q: %v", fields[2], err) 158 | } 159 | break 160 | } 161 | } 162 | 163 | return hostKey, nil 164 | } 165 | -------------------------------------------------------------------------------- /modules/backup/output/scp/scp_test.go: -------------------------------------------------------------------------------- 1 | // +build disabled 2 | 3 | package scp 4 | 5 | import ( 6 | "bytes" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/require" 10 | ) 11 | 12 | func TestGetName(t *testing.T) { 13 | s := &BackupOutputScp{} 14 | require.Equal(t, "scp", s.GetName()) 15 | } 16 | 17 | func TestGetConfig(t *testing.T) { 18 | s := &BackupOutputScp{} 19 | require.Equal(t, &Config{}, s.GetConfig()) 20 | } 21 | 22 | func TestInitPipe(t *testing.T) { 23 | s := &BackupOutputScp{} 24 | bufInput := bytes.NewBuffer([]byte("hello world")) 25 | bufOutput := &bytes.Buffer{} 26 | require.NoError(t, s.InitPipe(bufOutput, bufInput)) 27 | } 28 | 29 | func TestInitModule(t *testing.T) { 30 | s := &BackupOutputScp{} 31 | err := s.InitModule(&Config{ 32 | Addr: "127.0.0.1", 33 | Port: 2222, 34 | User: "user", 35 | Password: "user", 36 | }) 37 | require.NoError(t, err) 38 | } 39 | -------------------------------------------------------------------------------- /modules/global/connect/ssh/config.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | type Config struct { 4 | LocalEndpointHost string 5 | LocalEndpointPort int 6 | 7 | ServerEndpointHost string 8 | ServerEndpointPort int 9 | 10 | RemoteEndpointHost string 11 | RemoteEndpointPort int 12 | 13 | RemoteUser string 14 | 15 | // Full path to id_rsa key 16 | KeyPath string 17 | } 18 | -------------------------------------------------------------------------------- /modules/global/connect/ssh/ssh.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | import ( 4 | "bufio" 5 | "context" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | 13 | "golang.org/x/crypto/ssh" 14 | ) 15 | 16 | const MODULE_NAME = "ssh" 17 | const MODULE_GROUP = "global" 18 | const MODULE_TYPE = "connect" 19 | 20 | type GlobalConnectSsh struct { 21 | reader io.Reader 22 | writer io.Writer 23 | config *Config 24 | tunnel *SSHtunnel 25 | } 26 | 27 | func (m *GlobalConnectSsh) GetName() string { 28 | return MODULE_NAME 29 | } 30 | 31 | func (m *GlobalConnectSsh) GetGroup() string { 32 | return MODULE_GROUP 33 | } 34 | 35 | func (m *GlobalConnectSsh) GetType() string { 36 | return MODULE_TYPE 37 | } 38 | 39 | func (m *GlobalConnectSsh) GetConfig() interface{} { 40 | return &Config{} 41 | } 42 | 43 | func (m *GlobalConnectSsh) InitPipe(w io.Writer, r io.Reader) error { 44 | m.reader = r 45 | m.writer = w 46 | return nil 47 | } 48 | 49 | func (m *GlobalConnectSsh) InitModule(_cfg interface{}) error { 50 | m.config = _cfg.(*Config) 51 | 52 | // Local machine tunnel output 53 | localEndpoint := &Endpoint{ 54 | Host: m.config.LocalEndpointHost, 55 | Port: m.config.LocalEndpointPort, 56 | } 57 | 58 | // External server 59 | serverEndpoint := &Endpoint{ 60 | Host: m.config.ServerEndpointHost, 61 | Port: m.config.ServerEndpointPort, 62 | } 63 | 64 | // External machine tunnel input 65 | remoteEndpoint := &Endpoint{ 66 | Host: m.config.RemoteEndpointHost, 67 | Port: m.config.RemoteEndpointPort, 68 | } 69 | 70 | // get host public key 71 | hostKey, err := getHostKey(m.config.ServerEndpointHost) 72 | if err != nil { 73 | return err 74 | } 75 | 76 | key, err := ioutil.ReadFile(m.config.KeyPath) 77 | if err != nil { 78 | return err 79 | } 80 | signer, err := ssh.ParsePrivateKey(key) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | sshConfig := &ssh.ClientConfig{ 86 | User: m.config.RemoteUser, 87 | Auth: []ssh.AuthMethod{ 88 | ssh.PublicKeys(signer), 89 | }, 90 | HostKeyCallback: ssh.FixedHostKey(hostKey), 91 | } 92 | 93 | tunnel := &SSHtunnel{ 94 | Config: sshConfig, 95 | Local: localEndpoint, 96 | Server: serverEndpoint, 97 | Remote: remoteEndpoint, 98 | } 99 | m.tunnel = tunnel 100 | 101 | return nil 102 | } 103 | 104 | func (m *GlobalConnectSsh) Run(ctx context.Context) error { 105 | return m.tunnel.Start() 106 | } 107 | 108 | func (m *GlobalConnectSsh) Close() error { 109 | return m.tunnel.Stop() 110 | } 111 | 112 | func getHostKey(host string) (ssh.PublicKey, error) { 113 | // parse OpenSSH known_hosts file 114 | // ssh or use ssh-keyscan to get initial key 115 | file, err := os.Open(filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts")) 116 | if err != nil { 117 | return nil, err 118 | } 119 | defer file.Close() 120 | 121 | scanner := bufio.NewScanner(file) 122 | var hostKey ssh.PublicKey 123 | for scanner.Scan() { 124 | fields := strings.Split(scanner.Text(), " ") 125 | if len(fields) != 3 { 126 | continue 127 | } 128 | if strings.Contains(fields[0], host) { 129 | var err error 130 | hostKey, _, _, _, err = ssh.ParseAuthorizedKey(scanner.Bytes()) 131 | if err != nil { 132 | return nil, fmt.Errorf("error parsing %q: %v", fields[2], err) 133 | } 134 | break 135 | } 136 | } 137 | return hostKey, nil 138 | } 139 | -------------------------------------------------------------------------------- /modules/global/connect/ssh/ssh_test.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | import ( 4 | "github.com/stretchr/testify/require" 5 | "testing" 6 | ) 7 | 8 | var gssh GlobalConnectSsh 9 | var config = Config { 10 | LocalEndpointHost: "127.0.0.1", 11 | LocalEndpointPort: 8080, 12 | ServerEndpointHost: "127.0.0.2", 13 | ServerEndpointPort: 22, 14 | RemoteEndpointHost: "127.0.0.1", 15 | RemoteEndpointPort: 8080, 16 | RemoteUser: "root", 17 | KeyPath: "", 18 | } 19 | 20 | func TestGlobalConnectSsh_GetName(t *testing.T) { 21 | name := gssh.GetName() 22 | require.Equal(t, "ssh", name) 23 | } 24 | 25 | func TestGlobalConnectSsh_GetGroup(t *testing.T) { 26 | group := gssh.GetGroup() 27 | require.Equal(t, "global", group) 28 | } 29 | 30 | func TestGlobalConnectSsh_GetType(t *testing.T) { 31 | moduleType := gssh.GetType() 32 | require.Equal(t, "connect", moduleType) 33 | } 34 | 35 | // TODO: Это пока не работает, походу есть пробелма в глобальной логике получения конфигов. 36 | //func TestGlobalConnectSsh_GetConfig(t *testing.T) { 37 | // conf := gssh.GetConfig() 38 | // assert.Equal(t, conf, &config) 39 | //} 40 | 41 | // TODO: Тут надо как-то замокать ключ. 42 | //func TestGlobalConnectSsh_InitModule(t *testing.T) { 43 | // err := gssh.InitModule(&config) 44 | // assert.Equal(t, err, nil) 45 | // assert.Equal(t, gssh.tunnel.Config.User, config.RemoteUser) 46 | //} 47 | -------------------------------------------------------------------------------- /modules/global/connect/ssh/tunnel.go: -------------------------------------------------------------------------------- 1 | package ssh 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "net" 7 | 8 | "golang.org/x/crypto/ssh" 9 | ) 10 | 11 | type Endpoint struct { 12 | Host string 13 | Port int 14 | } 15 | 16 | func (endpoint *Endpoint) String() string { 17 | return fmt.Sprintf("%s:%d", endpoint.Host, endpoint.Port) 18 | } 19 | 20 | type SSHtunnel struct { 21 | Local *Endpoint 22 | Server *Endpoint 23 | Remote *Endpoint 24 | Listener net.Listener 25 | 26 | Config *ssh.ClientConfig 27 | } 28 | 29 | func (tunnel *SSHtunnel) Start() error { 30 | listener, err := net.Listen("tcp", tunnel.Local.String()) 31 | if err != nil { 32 | return err 33 | } 34 | tunnel.Listener = listener 35 | 36 | defer tunnel.Listener.Close() 37 | 38 | for { 39 | conn, err := tunnel.Listener.Accept() 40 | if err != nil { 41 | return err 42 | } 43 | go tunnel.forward(conn) 44 | } 45 | } 46 | 47 | func (tunnel *SSHtunnel) forward(localConn net.Conn) { 48 | serverConn, err := ssh.Dial("tcp", tunnel.Server.String(), tunnel.Config) 49 | if err != nil { 50 | fmt.Printf("Server dial error: %s\n", err) 51 | return 52 | } 53 | 54 | remoteConn, err := serverConn.Dial("tcp", tunnel.Remote.String()) 55 | if err != nil { 56 | fmt.Printf("Remote dial error: %s\n", err) 57 | return 58 | } 59 | 60 | copyConn := func(writer, reader net.Conn) { 61 | _, err := io.Copy(writer, reader) 62 | if err != nil { 63 | fmt.Printf("io.Copy error: %s", err) 64 | } 65 | } 66 | 67 | go copyConn(localConn, remoteConn) 68 | go copyConn(remoteConn, localConn) 69 | } 70 | 71 | func (tunnel *SSHtunnel) Stop() error { 72 | return tunnel.Listener.Close() 73 | } 74 | -------------------------------------------------------------------------------- /modules/global/notifier/awsses/awsses.go: -------------------------------------------------------------------------------- 1 | package awsses 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/session" 9 | "github.com/aws/aws-sdk-go/service/ses" 10 | "github.com/copybird/copybird/core" 11 | ) 12 | 13 | const ( 14 | GROUP_NAME = "global" 15 | TYPE_NAME = "notifier" 16 | MODULE_NAME = "awsses" 17 | ) 18 | 19 | type GlobalNotifierAwsses struct { 20 | core.Module 21 | config *Config 22 | simem *ses.SES // simple email service 23 | seInput *ses.SendEmailInput 24 | reader io.Reader 25 | writer io.Writer 26 | } 27 | 28 | func (m *GlobalNotifierAwsses) GetGroup() core.ModuleGroup { 29 | return GROUP_NAME 30 | } 31 | 32 | func (m *GlobalNotifierAwsses) GetType() core.ModuleType { 33 | return TYPE_NAME 34 | } 35 | 36 | func (m *GlobalNotifierAwsses) GetName() string { 37 | return MODULE_NAME 38 | } 39 | 40 | func (m *GlobalNotifierAwsses) GetConfig() interface{} { 41 | return &Config{} 42 | } 43 | 44 | func (m *GlobalNotifierAwsses) InitPipe(w io.Writer, r io.Reader) error { 45 | m.reader = r 46 | m.writer = w 47 | return nil 48 | } 49 | 50 | func (m *GlobalNotifierAwsses) InitModule(_cfg interface{}) error { 51 | m.config = _cfg.(*Config) 52 | 53 | // Create a new session in the config region. 54 | sess, err := session.NewSession(&aws.Config{ 55 | Region: aws.String(m.config.Region)}, 56 | ) 57 | if err != nil { 58 | return err 59 | } 60 | 61 | // Create an SES session. 62 | svc := ses.New(sess) 63 | m.simem = svc 64 | 65 | input := &ses.SendEmailInput{ 66 | Destination: &ses.Destination{ 67 | CcAddresses: []*string{}, 68 | ToAddresses: []*string{ 69 | aws.String(m.config.Recipient), 70 | }, 71 | }, 72 | Message: &ses.Message{ 73 | Body: &ses.Body{ 74 | Html: &ses.Content{ 75 | Charset: aws.String(m.config.Charset), 76 | Data: aws.String(m.config.HTMLbody), 77 | }, 78 | Text: &ses.Content{ 79 | Charset: aws.String(m.config.Charset), 80 | Data: aws.String(m.config.Textbody), 81 | }, 82 | }, 83 | Subject: &ses.Content{ 84 | Charset: aws.String(m.config.Charset), 85 | Data: aws.String(m.config.Subject), 86 | }, 87 | }, 88 | Source: aws.String(m.config.Sender), 89 | // TODO: Uncomment to use a configuration set 90 | //ConfigurationSetName: aws.String(ConfigurationSet), 91 | } 92 | m.seInput = input 93 | 94 | return nil 95 | } 96 | 97 | func (m *GlobalNotifierAwsses) Run(ctx context.Context) error { 98 | 99 | // Attempt to send the email. 100 | _, err := m.simem.SendEmail(m.seInput) 101 | 102 | // Display error messages if they occur. 103 | if err != nil { 104 | return err 105 | } 106 | 107 | return nil 108 | } 109 | 110 | // Close closes compressor 111 | func (m *GlobalNotifierAwsses) Close() error { 112 | return nil 113 | } 114 | -------------------------------------------------------------------------------- /modules/global/notifier/awsses/awsses_test.go: -------------------------------------------------------------------------------- 1 | package awsses 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | var ( 11 | noCredErr = "NoCredentialProviders: no valid providers in chain. Deprecated.\n\tFor verbose messaging see aws.Config.CredentialsChainVerboseErrors" 12 | ) 13 | 14 | func TestAwsSes_NoCredentialProvErrs(t *testing.T) { 15 | conf := &Config{ 16 | Region: "us-west-2", 17 | } 18 | 19 | as := &GlobalNotifierAwsses{} 20 | assert.Assert(t, as.GetConfig() != nil) 21 | assert.NilError(t, as.InitModule(conf)) 22 | err := as.Run(context.TODO()) 23 | assert.Error(t, err, noCredErr) 24 | } 25 | 26 | func TestAwsSes_WithCredential(t *testing.T) { 27 | conf := &Config{ 28 | Region: "us-west-2", 29 | Sender: "sender@example.com", 30 | Recipient: "recipient@example.com", 31 | Subject: "Amazon SES Test (AWS SDK for Go)", 32 | HTMLbody: "Test", 33 | Textbody: "This email was sent with Amazon SES using the AWS SDK for Go.", 34 | Charset: "UTF-8", 35 | } 36 | 37 | as := &GlobalNotifierAwsses{} 38 | assert.Assert(t, as.GetConfig() != nil) 39 | assert.NilError(t, as.InitModule(conf)) 40 | err := as.Run(context.TODO()) 41 | assert.Error(t, err, noCredErr) 42 | } 43 | -------------------------------------------------------------------------------- /modules/global/notifier/awsses/config.go: -------------------------------------------------------------------------------- 1 | package awsses 2 | 3 | // Config for sending a Message to an Email Address in Amazon SES 4 | type Config struct { 5 | Region string // AWS Region for Amazon SES 6 | Sender string // This address must be verified with Amazon SES. 7 | Recipient string // If your account is still in the sandbox, this address must be verified. 8 | Subject string // The subject line for the email. 9 | HTMLbody string // The HTML body for the email. 10 | Textbody string // The email body for recipients with non-HTML email clients. 11 | Charset string // The character encoding for the email. 12 | } 13 | -------------------------------------------------------------------------------- /modules/global/notifier/awssqs/awssqs.go: -------------------------------------------------------------------------------- 1 | package awssqs 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/aws/aws-sdk-go/aws" 8 | "github.com/aws/aws-sdk-go/aws/credentials" 9 | "github.com/aws/aws-sdk-go/aws/session" 10 | "github.com/aws/aws-sdk-go/service/sqs" 11 | "github.com/aws/aws-sdk-go/service/sqs/sqsiface" 12 | "github.com/copybird/copybird/core" 13 | "github.com/knative/pkg/cloudevents" 14 | ) 15 | 16 | const ( 17 | MODULE_NAME = "awssqs" 18 | GROUP_NAME = "global" 19 | TYPE_NAME = "notifier" 20 | ) 21 | 22 | func (m *GlobalNotifierAWSSQS) GetGroup() core.ModuleGroup { 23 | return GROUP_NAME 24 | } 25 | 26 | func (m *GlobalNotifierAWSSQS) GetType() core.ModuleType { 27 | return TYPE_NAME 28 | } 29 | 30 | type GlobalNotifierAWSSQS struct { 31 | core.Module 32 | config *Config 33 | reader io.Reader 34 | writer io.Writer 35 | } 36 | 37 | func (m *GlobalNotifierAWSSQS) GetName() string { 38 | return MODULE_NAME 39 | } 40 | 41 | func (m *GlobalNotifierAWSSQS) InitPipe(w io.Writer, r io.Reader) error { 42 | m.reader = r 43 | m.writer = w 44 | return nil 45 | } 46 | 47 | func (m *GlobalNotifierAWSSQS) InitModule(_cfg interface{}) error { 48 | m.config = _cfg.(*Config) 49 | return nil 50 | } 51 | 52 | func (m *GlobalNotifierAWSSQS) Run(ctx context.Context) error { 53 | if err := m.config.NotifyAWSSQS(); err != nil { 54 | return err 55 | } 56 | 57 | return nil 58 | } 59 | 60 | func (m *GlobalNotifierAWSSQS) GetConfig() interface{} { 61 | return &Config{} 62 | } 63 | 64 | func (m *GlobalNotifierAWSSQS) Close() error { 65 | return nil 66 | } 67 | 68 | type Clients struct { 69 | SQS sqsiface.SQSAPI 70 | CloudEvents *cloudevents.Client 71 | } 72 | 73 | func (c *Config) NotifyAWSSQS() error { 74 | sess, err := session.NewSession(&aws.Config{ 75 | Region: aws.String(c.Region), 76 | Credentials: credentials.NewStaticCredentials(c.AccountAccessKeyID, c.AccountSecretAccessKey, ""), 77 | MaxRetries: aws.Int(5), 78 | }) 79 | 80 | if err != nil { 81 | return err 82 | } 83 | 84 | sqsClient := sqs.New(sess) 85 | 86 | queueUrls, err := sqsClient.ListQueues(&sqs.ListQueuesInput{QueueNamePrefix: aws.String(c.Queues)}) 87 | 88 | if err != nil { 89 | return err 90 | } 91 | 92 | sendMessage := &sqs.SendMessageInput{ 93 | MessageBody: aws.String(c.MessageBody), 94 | QueueUrl: queueUrls.QueueUrls[0], 95 | DelaySeconds: aws.Int64(3), 96 | } 97 | 98 | _, err = sqsClient.SendMessage(sendMessage) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | return nil 104 | } 105 | -------------------------------------------------------------------------------- /modules/global/notifier/awssqs/awssqs_test.go: -------------------------------------------------------------------------------- 1 | package awssqs 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestGetName(t *testing.T) { 12 | g := &GlobalNotifierAWSSQS{} 13 | require.Equal(t, MODULE_NAME, g.GetName()) 14 | } 15 | 16 | func TestGetConfig(t *testing.T) { 17 | g := &GlobalNotifierAWSSQS{} 18 | require.Equal(t, &Config{}, g.GetConfig()) 19 | } 20 | func TestClose(t *testing.T) { 21 | g := &GlobalNotifierAWSSQS{} 22 | assert.Equal(t, nil, g.Close()) 23 | } 24 | 25 | func TestInitPipe(t *testing.T) { 26 | g := &GlobalNotifierAWSSQS{} 27 | bufInput := bytes.NewBuffer([]byte("hello world")) 28 | bufOutput := &bytes.Buffer{} 29 | require.NoError(t, g.InitPipe(bufOutput, bufInput)) 30 | } 31 | -------------------------------------------------------------------------------- /modules/global/notifier/awssqs/config.go: -------------------------------------------------------------------------------- 1 | package awssqs 2 | 3 | type Config struct { 4 | Region string 5 | AccountAccessKeyID string 6 | AccountSecretAccessKey string 7 | Queues string 8 | MessageBody string 9 | } 10 | -------------------------------------------------------------------------------- /modules/global/notifier/email/config.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | type Config struct { 4 | MailerUser string 5 | MailerPassword string 6 | MailTo string 7 | } 8 | -------------------------------------------------------------------------------- /modules/global/notifier/email/email.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "net/smtp" 7 | 8 | "github.com/copybird/copybird/core" 9 | ) 10 | 11 | const GROUP_NAME = "global" 12 | const TYPE_NAME = "notifier" 13 | const MODULE_NAME = "email" 14 | 15 | type GlobalNotifierEmail struct { 16 | core.Module 17 | Config *Config 18 | } 19 | 20 | func (e *GlobalNotifierEmail) GetGroup() core.ModuleGroup { 21 | return GROUP_NAME 22 | } 23 | 24 | func (e *GlobalNotifierEmail) GetType() core.ModuleType { 25 | return TYPE_NAME 26 | } 27 | 28 | func (e *GlobalNotifierEmail) GetName() string { 29 | return MODULE_NAME 30 | } 31 | 32 | func (e *GlobalNotifierEmail) GetConfig() interface{} { 33 | return &Config{} 34 | } 35 | 36 | func (e *GlobalNotifierEmail) InitModule(_cfg interface{}) error { 37 | e.Config = _cfg.(*Config) 38 | return nil 39 | } 40 | 41 | func (e *GlobalNotifierEmail) Run(ctx context.Context) error { 42 | if err := e.SendEmail(); err != nil { 43 | return err 44 | } 45 | 46 | return nil 47 | } 48 | 49 | func (e *GlobalNotifierEmail) Close() error { 50 | return nil 51 | } 52 | 53 | func (e *GlobalNotifierEmail) SendEmail() error { 54 | 55 | from := e.Config.MailerUser 56 | pass := e.Config.MailerPassword 57 | to := e.Config.MailTo 58 | body := "Dump created successfully" 59 | subject := "Dump" 60 | 61 | header := "" 62 | header += fmt.Sprintf("From: %s\r\n", from) 63 | header += fmt.Sprintf("To: %s\r\n", to) 64 | header += fmt.Sprint("MIME-Version: 1.0\r\n") 65 | header += fmt.Sprint("Content-type: text/html\r\n") 66 | header += fmt.Sprintf("Subject: %s\r\n", subject) 67 | header += "\r\n" + body + "\r\n" 68 | 69 | err := smtp.SendMail("smtp.gmail.com:587", 70 | smtp.PlainAuth("", from, pass, "smtp.gmail.com"), 71 | from, []string{to}, []byte(header)) 72 | 73 | if err != nil { 74 | return err 75 | } 76 | 77 | return nil 78 | } 79 | -------------------------------------------------------------------------------- /modules/global/notifier/email/email_test.go: -------------------------------------------------------------------------------- 1 | package email 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/assert" 7 | ) 8 | 9 | func TestSendEmail(t *testing.T) { 10 | 11 | testCase := []struct { 12 | MailerUser string 13 | MailerPassword string 14 | MailTo string 15 | }{ 16 | {"c0pybird0@gmail.com", "pas$$w0rd", "example.com"}, 17 | } 18 | 19 | for _, tc := range testCase { 20 | g := &GlobalNotifierEmail{} 21 | assert.NotNil(t, g.GetConfig()) 22 | assert.NoError(t, g.InitModule(&Config{MailerUser: tc.MailerUser, MailerPassword: tc.MailerPassword, MailTo: tc.MailTo})) 23 | assert.Error(t, g.SendEmail()) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /modules/global/notifier/kafka/config.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | type Config struct { 4 | MaxRetry int 5 | BrokerList []string 6 | Topic string 7 | Message string 8 | } 9 | -------------------------------------------------------------------------------- /modules/global/notifier/kafka/kafka.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "io" 5 | "context" 6 | 7 | "github.com/Shopify/sarama" 8 | 9 | "github.com/copybird/copybird/core" 10 | ) 11 | 12 | const ( 13 | GROUP_NAME = "global" 14 | TYPE_NAME = "notifier" 15 | MODULE_NAME = "kafka" 16 | ) 17 | 18 | // GlobalNotifieKafka represends ... 19 | type GlobalNotifieKafka struct { 20 | core.Module 21 | config *Config 22 | conn sarama.SyncProducer 23 | reader io.Reader 24 | writer io.Writer 25 | } 26 | 27 | // GetGroup returns group of the module 28 | func (m *GlobalNotifieKafka) GetGroup() core.ModuleGroup { 29 | return GROUP_NAME 30 | } 31 | 32 | // GetType returns group of the module 33 | func (m *GlobalNotifieKafka) GetType() core.ModuleType { 34 | return TYPE_NAME 35 | } 36 | 37 | // GetName returns name of the module 38 | func (m *GlobalNotifieKafka) GetName() string { 39 | return MODULE_NAME 40 | } 41 | 42 | // GetConfig returns module config 43 | func (m *GlobalNotifieKafka) GetConfig() interface{} { 44 | return &Config{} 45 | } 46 | 47 | // InitPipe initializes pipe 48 | func (m *GlobalNotifieKafka) InitPipe(w io.Writer, r io.Reader) error { 49 | m.reader = r 50 | m.writer = w 51 | return nil 52 | } 53 | 54 | // InitModule initializes module 55 | func (m *GlobalNotifieKafka) InitModule(_cfg interface{}) error { 56 | m.config = _cfg.(*Config) 57 | config := sarama.NewConfig() 58 | config.Producer.RequiredAcks = sarama.WaitForAll 59 | config.Producer.Retry.Max = m.config.MaxRetry 60 | config.Producer.Return.Successes = true 61 | conn, err := sarama.NewSyncProducer(m.config.BrokerList, config) 62 | if err != nil { 63 | return err 64 | } 65 | m.conn = conn 66 | 67 | return nil 68 | } 69 | 70 | // Run runs module 71 | func (m *GlobalNotifieKafka) Run(ctx context.Context) error { 72 | msg := &sarama.ProducerMessage{ 73 | Topic: m.config.Topic, 74 | Value: sarama.StringEncoder(m.config.Message), 75 | } 76 | _, _, err := m.conn.SendMessage(msg) 77 | return err 78 | 79 | } 80 | 81 | // Close closes compressor 82 | func (m *GlobalNotifieKafka) Close() error { 83 | return nil 84 | } 85 | -------------------------------------------------------------------------------- /modules/global/notifier/kafka/kafka_test.go: -------------------------------------------------------------------------------- 1 | package kafka 2 | 3 | import ( 4 | "testing" 5 | "context" 6 | 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func TestKafka(t *testing.T) { 11 | k := &GlobalNotifieKafka{} 12 | c := k.GetConfig().(*Config) 13 | assert.NotNil(t, c) 14 | c.BrokerList = []string{"localhost:9092", "localhost:9092"} 15 | c.Topic = "hello" 16 | c.Message = "world" 17 | assert.NoError(t, k.InitModule(c)) 18 | assert.NoError(t, k.Run(context.TODO())) 19 | } 20 | -------------------------------------------------------------------------------- /modules/global/notifier/nats/config.go: -------------------------------------------------------------------------------- 1 | package nats 2 | 3 | type Config struct { 4 | NATSURL string 5 | Msg string 6 | Topic string 7 | } 8 | -------------------------------------------------------------------------------- /modules/global/notifier/nats/nats.go: -------------------------------------------------------------------------------- 1 | package nats 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "github.com/copybird/copybird/core" 7 | "io" 8 | 9 | "github.com/nats-io/go-nats" 10 | ) 11 | 12 | const ( 13 | GROUP_NAME = "global" 14 | TYPE_NAME = "notifier" 15 | MODULE_NAME = "nats" 16 | ) 17 | 18 | var ( 19 | errNats = errors.New("NATS not connected") 20 | errNatsEmptyTopic = errors.New("NATS empty topic") 21 | ) 22 | 23 | type GlobalNotifierNats struct { 24 | core.Module 25 | config *Config 26 | conn *nats.Conn 27 | reader io.Reader 28 | writer io.Writer 29 | } 30 | 31 | func (m *GlobalNotifierNats) GetGroup() core.ModuleGroup { 32 | return GROUP_NAME 33 | } 34 | 35 | func (m *GlobalNotifierNats) GetType() core.ModuleType { 36 | return TYPE_NAME 37 | } 38 | 39 | func (m *GlobalNotifierNats) GetName() string { 40 | return MODULE_NAME 41 | } 42 | 43 | func (m *GlobalNotifierNats) GetConfig() interface{} { 44 | return &Config{} 45 | } 46 | 47 | func (m *GlobalNotifierNats) InitPipe(w io.Writer, r io.Reader) error { 48 | m.reader = r 49 | m.writer = w 50 | return nil 51 | } 52 | 53 | func (m *GlobalNotifierNats) InitModule(_cfg interface{}) error { 54 | m.config = _cfg.(*Config) 55 | 56 | if m.config.Topic == "" { 57 | return errNatsEmptyTopic 58 | } 59 | 60 | natsConn, err := nats.Connect(m.config.NATSURL) 61 | if err != nil { 62 | return err 63 | } 64 | m.conn = natsConn 65 | 66 | return nil 67 | } 68 | 69 | func (m *GlobalNotifierNats) Run(ctx context.Context) error { 70 | return m.conn.Publish(m.config.Topic, []byte(m.config.Msg)) 71 | } 72 | 73 | // Close closes compressor 74 | func (m *GlobalNotifierNats) Close() error { 75 | m.conn.Close() 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /modules/global/notifier/nats/nats_test.go: -------------------------------------------------------------------------------- 1 | package nats 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "gotest.tools/assert" 8 | ) 9 | 10 | func TestNats_InvalidConn(t *testing.T) { 11 | conf := &Config{ 12 | NATSURL: "0.0.0.0:4223", 13 | Topic: "test.topic", 14 | Msg: "Test", 15 | } 16 | 17 | n := &GlobalNotifierNats{} 18 | assert.Assert(t, n.GetConfig() != nil) 19 | err := n.InitModule(conf) 20 | assert.Error(t, err, "nats: no servers available for connection") 21 | } 22 | 23 | func TestNats_ValidConn(t *testing.T) { 24 | conf := &Config{ 25 | NATSURL: "0.0.0.0:4222", 26 | Topic: "test.topic", 27 | Msg: "Test", 28 | } 29 | 30 | n := GlobalNotifierNats{} 31 | assert.Assert(t, n.GetConfig() != nil) 32 | err := n.InitModule(conf) 33 | if err != nil { 34 | t.Errorf("TestNats: %v", err) 35 | } 36 | 37 | err = n.Run(context.TODO()) 38 | if err != nil { 39 | t.Errorf("TestNats: %v", err) 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /modules/global/notifier/nsq/config.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | type Config struct { 4 | TopicName string 5 | Message string 6 | } 7 | -------------------------------------------------------------------------------- /modules/global/notifier/nsq/nsq.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | 12 | "github.com/copybird/copybird/core" 13 | ) 14 | 15 | const ( 16 | MODULE_NAME = "nsq" 17 | GROUP_NAME = "global" 18 | TYPE_NAME = "notifier" 19 | HeaderContentType = "Content-Type" 20 | MIMEApplicationJSON = "application/json" 21 | NSQUrlSite = "http://127.0.0.1:4151/pub?topic" 22 | ) 23 | 24 | const () 25 | 26 | func (m *GlobalNotifierNSQ) GetGroup() core.ModuleGroup { 27 | return GROUP_NAME 28 | } 29 | 30 | func (m *GlobalNotifierNSQ) GetType() core.ModuleType { 31 | return TYPE_NAME 32 | } 33 | 34 | type GlobalNotifierNSQ struct { 35 | core.Module 36 | config *Config 37 | reader io.Reader 38 | writer io.Writer 39 | } 40 | 41 | func (m *GlobalNotifierNSQ) GetName() string { 42 | return MODULE_NAME 43 | } 44 | 45 | func (m *GlobalNotifierNSQ) InitPipe(w io.Writer, r io.Reader) error { 46 | m.reader = r 47 | m.writer = w 48 | return nil 49 | } 50 | 51 | func (m *GlobalNotifierNSQ) InitModule(_cfg interface{}) error { 52 | m.config = _cfg.(*Config) 53 | return nil 54 | } 55 | 56 | func (m *GlobalNotifierNSQ) Run(ctx context.Context) error { 57 | if err := m.config.NotifyNSQ(); err != nil { 58 | return err 59 | } 60 | 61 | return nil 62 | } 63 | 64 | func (m *GlobalNotifierNSQ) GetConfig() interface{} { 65 | return &Config{} 66 | } 67 | 68 | func (m *GlobalNotifierNSQ) Close() error { 69 | return nil 70 | } 71 | 72 | type NSQMessage struct { 73 | Message string `json:"message"` 74 | } 75 | 76 | func (c *Config) NotifyNSQ() error { 77 | urls := fmt.Sprintf("%s=%s", NSQUrlSite, c.TopicName) 78 | 79 | message, err := json.Marshal(NSQMessage{Message: c.Message}) 80 | 81 | if err != nil { 82 | return err 83 | } 84 | 85 | client := &http.Client{} 86 | 87 | req, err := http.NewRequest(http.MethodPost, urls, bytes.NewBuffer(message)) 88 | 89 | if err != nil { 90 | return err 91 | } 92 | 93 | req.Header.Set(HeaderContentType, MIMEApplicationJSON) 94 | 95 | resp, err := client.Do(req) 96 | 97 | if err != nil { 98 | return err 99 | } 100 | 101 | if resp.StatusCode != http.StatusOK { 102 | statusCode := fmt.Sprintf("%v", resp.StatusCode) 103 | return errors.New("StatusCode: " + statusCode) 104 | } 105 | 106 | return nil 107 | } 108 | -------------------------------------------------------------------------------- /modules/global/notifier/nsq/nsq_test.go: -------------------------------------------------------------------------------- 1 | package nsq 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/jarcoal/httpmock" 11 | "github.com/stretchr/testify/assert" 12 | "github.com/stretchr/testify/require" 13 | ) 14 | 15 | func TestGetName(t *testing.T) { 16 | n := &GlobalNotifierNSQ{} 17 | require.Equal(t, MODULE_NAME, n.GetName()) 18 | } 19 | 20 | func TestGetConfig(t *testing.T) { 21 | n := &GlobalNotifierNSQ{} 22 | require.Equal(t, &Config{}, n.GetConfig()) 23 | } 24 | func TestClose(t *testing.T) { 25 | n := &GlobalNotifierNSQ{} 26 | assert.Equal(t, nil, n.Close()) 27 | } 28 | 29 | func TestInitPipe(t *testing.T) { 30 | n := &GlobalNotifierNSQ{} 31 | bufInput := bytes.NewBuffer([]byte("hello world")) 32 | bufOutput := &bytes.Buffer{} 33 | require.NoError(t, n.InitPipe(bufOutput, bufInput)) 34 | } 35 | 36 | func TestNotifySlackChannel(t *testing.T) { 37 | 38 | n := &GlobalNotifierNSQ{} 39 | httpmock.Activate() 40 | defer httpmock.DeactivateAndReset() 41 | 42 | testCase := []struct { 43 | Responder httpmock.Responder 44 | TopicName string 45 | Message string 46 | Error error 47 | }{ 48 | {httpmock.NewStringResponder(200, "{}"), "hi", "hi", nil}, 49 | {httpmock.NewStringResponder(400, "{}"), "ss", "ss", errors.New("StatusCode: 400")}, 50 | } 51 | 52 | for _, tt := range testCase { 53 | urls := fmt.Sprintf("%s=%s", NSQUrlSite, tt.TopicName) 54 | httpmock.RegisterResponder("POST", urls, tt.Responder) 55 | assert.NoError(t, n.InitModule(&Config{TopicName: tt.TopicName, Message: tt.Message})) 56 | err := n.Run(context.TODO()) 57 | assert.Equal(t, tt.Error, err) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /modules/global/notifier/pagerduty/config.go: -------------------------------------------------------------------------------- 1 | package pagerduty 2 | 3 | type Config struct { 4 | AuthToken string 5 | From string //use user email in this field 6 | } 7 | -------------------------------------------------------------------------------- /modules/global/notifier/pagerduty/pagerduty.go: -------------------------------------------------------------------------------- 1 | package pagerduty 2 | 3 | import ( 4 | "context" 5 | 6 | "github.com/PagerDuty/go-pagerduty" 7 | "github.com/copybird/copybird/core" 8 | ) 9 | 10 | const GROUP_NAME = "global" 11 | const TYPE_NAME = "notifier" 12 | const MODULE_NAME = "pagerduty" 13 | 14 | type GlobalNotifierPagerDuty struct { 15 | core.Module 16 | config *Config 17 | client *pagerduty.Client 18 | } 19 | 20 | func (m *GlobalNotifierPagerDuty) GetGroup() core.ModuleGroup { 21 | return GROUP_NAME 22 | } 23 | 24 | func (m *GlobalNotifierPagerDuty) GetType() core.ModuleType { 25 | return TYPE_NAME 26 | } 27 | 28 | func (m *GlobalNotifierPagerDuty) GetName() string { 29 | return MODULE_NAME 30 | } 31 | 32 | func (m *GlobalNotifierPagerDuty) GetConfig() interface{} { 33 | return &Config{} 34 | } 35 | 36 | func (m *GlobalNotifierPagerDuty) InitModule(_conf interface{}) error { 37 | conf := _conf.(Config) 38 | m.config = &conf 39 | m.client = pagerduty.NewClient(m.config.AuthToken) 40 | return nil 41 | } 42 | 43 | func (m *GlobalNotifierPagerDuty) Run(ctx context.Context) error { 44 | _, err := m.client.CreateIncident(m.config.From, &pagerduty.CreateIncident{Incident: pagerduty.CreateIncidentOptions{ 45 | Type: "dump creation status", 46 | Title: "Test", 47 | Service: pagerduty.APIReference{ 48 | ID: "P4B73MT", 49 | Type: "service_reference", 50 | }, 51 | Body: pagerduty.APIDetails{ 52 | Type: "dump creation failed", 53 | Details: "Error message that goes along with fail", 54 | }, 55 | }}) 56 | return err 57 | } 58 | 59 | func (m *GlobalNotifierPagerDuty) Close() error { 60 | return nil 61 | } 62 | -------------------------------------------------------------------------------- /modules/global/notifier/pagerduty/pagerduty_test.go: -------------------------------------------------------------------------------- 1 | package pagerduty 2 | 3 | import ( 4 | "context" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/require" 8 | ) 9 | 10 | func TestGetName(t *testing.T) { 11 | n := &GlobalNotifierPagerDuty{} 12 | require.Equal(t, "pagerduty", n.GetName()) 13 | } 14 | 15 | func TestGetConfig(t *testing.T) { 16 | n := &GlobalNotifierPagerDuty{} 17 | require.Equal(t, &Config{}, n.GetConfig()) 18 | } 19 | 20 | func TestInitModule(t *testing.T) { 21 | n := &GlobalNotifierPagerDuty{} 22 | err := n.InitModule(Config{}) 23 | require.NoError(t, err, "should not be any error here") 24 | } 25 | 26 | func TestRun(t *testing.T) { 27 | n := &GlobalNotifierPagerDuty{} 28 | err := n.InitModule(Config{ 29 | AuthToken: "insert auth token here", 30 | From: "example@example.com", 31 | }) 32 | require.NoError(t, err, "should not be any error here") 33 | err = n.Run(context.TODO()) 34 | require.Error(t, err) 35 | 36 | } 37 | -------------------------------------------------------------------------------- /modules/global/notifier/pushbullet/config.go: -------------------------------------------------------------------------------- 1 | package pushbullet 2 | 3 | type Config struct { 4 | APIKey string 5 | PhoneNumber string 6 | MessageTitle string 7 | MessageBody string 8 | } 9 | -------------------------------------------------------------------------------- /modules/global/notifier/pushbullet/pushbullet.go: -------------------------------------------------------------------------------- 1 | package pushbullet 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/copybird/copybird/core" 8 | 9 | "github.com/xconstruct/go-pushbullet" 10 | ) 11 | 12 | const GROUP_NAME = "global" 13 | const TYPE_NAME = "notifier" 14 | const MODULE_NAME = "pushbullet" 15 | 16 | type GlobalNotifierPushbullet struct { 17 | core.Module 18 | config *Config 19 | reader io.Reader 20 | writer io.Writer 21 | } 22 | 23 | type Message struct { 24 | Text string `json:"text"` 25 | } 26 | 27 | func (m *GlobalNotifierPushbullet) GetGroup() core.ModuleGroup { 28 | return GROUP_NAME 29 | } 30 | 31 | func (m *GlobalNotifierPushbullet) GetType() core.ModuleType { 32 | return TYPE_NAME 33 | } 34 | 35 | func (m *GlobalNotifierPushbullet) GetName() string { 36 | return MODULE_NAME 37 | } 38 | 39 | func (m *GlobalNotifierPushbullet) InitPipe(w io.Writer, r io.Reader) error { 40 | m.reader = r 41 | m.writer = w 42 | return nil 43 | } 44 | 45 | func (m *GlobalNotifierPushbullet) InitModule(_cfg interface{}) error { 46 | m.config = _cfg.(*Config) 47 | return nil 48 | } 49 | 50 | func (m *GlobalNotifierPushbullet) Run(ctx context.Context) error { 51 | if err := m.config.NotifyPushbulletChannel(); err != nil { 52 | return err 53 | } 54 | 55 | return nil 56 | } 57 | func (m *GlobalNotifierPushbullet) GetConfig() interface{} { 58 | return &Config{} 59 | } 60 | 61 | func (m *GlobalNotifierPushbullet) Close() error { 62 | return nil 63 | } 64 | 65 | func (c *Config) NotifyPushbulletChannel() error { 66 | pb := pushbullet.New(c.APIKey) 67 | devs, err := pb.Devices() 68 | if err != nil { 69 | return err 70 | } 71 | 72 | err = pb.PushNote(devs[0].Iden, c.MessageTitle, c.MessageBody) 73 | if err != nil { 74 | return err 75 | } 76 | 77 | //SMS test 78 | user, err := pb.Me() 79 | if err != nil { 80 | return err 81 | } 82 | 83 | err = pb.PushSMS(user.Iden, devs[0].Iden, c.PhoneNumber, "Sms text") 84 | if err != nil { 85 | return err 86 | } 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /modules/global/notifier/pushbullet/pushbullet_test.go: -------------------------------------------------------------------------------- 1 | // +build disabled 2 | 3 | package pushbullet 4 | 5 | import ( 6 | "bytes" 7 | "testing" 8 | 9 | "github.com/stretchr/testify/assert" 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | func TestGetName(t *testing.T) { 14 | n := GlobalNotifierPushbullet{} 15 | require.Equal(t, MODULE_NAME, n.GetName()) 16 | } 17 | 18 | func TestGetConfig(t *testing.T) { 19 | n := GlobalNotifierPushbullet{} 20 | require.Equal(t, &Config{}, n.GetConfig()) 21 | } 22 | func TestClose(t *testing.T) { 23 | n := GlobalNotifierPushbullet{} 24 | assert.Equal(t, nil, n.Close()) 25 | } 26 | 27 | func TestInitPipe(t *testing.T) { 28 | n := GlobalNotifierPushbullet{} 29 | bufInput := bytes.NewBuffer([]byte("hello world")) 30 | bufOutput := &bytes.Buffer{} 31 | require.NoError(t, n.InitPipe(bufOutput, bufInput)) 32 | } 33 | -------------------------------------------------------------------------------- /modules/global/notifier/rabbitmq/config.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | // Config for GlobalNotifierRabbitmq 4 | type Config struct { 5 | RabbitMQURL string 6 | QueueName string 7 | QueueDurable bool 8 | QueueAutoDelete bool 9 | QueueExclusive bool 10 | QueueNoWait bool 11 | PublishExchange string 12 | PublishKey string 13 | PublishMandatory bool 14 | PublishImmediate bool 15 | MsgContentType string 16 | MsgBody string 17 | } 18 | -------------------------------------------------------------------------------- /modules/global/notifier/rabbitmq/rabbitmq.go: -------------------------------------------------------------------------------- 1 | package rabbitmq 2 | 3 | import ( 4 | "context" 5 | "io" 6 | 7 | "github.com/copybird/copybird/core" 8 | "github.com/streadway/amqp" 9 | ) 10 | 11 | const ( 12 | GROUP_NAME = "global" 13 | TYPE_NAME = "notifier" 14 | MODULE_NAME = "rabbitmq" 15 | ) 16 | 17 | type GlobalNotifierRabbitmq struct { 18 | core.Module 19 | config *Config 20 | conn *amqp.Connection 21 | channel *amqp.Channel 22 | queue *amqp.Queue 23 | publ amqp.Publishing 24 | reader io.Reader 25 | writer io.Writer 26 | } 27 | 28 | func (m *GlobalNotifierRabbitmq) GetGroup() core.ModuleGroup { 29 | return GROUP_NAME 30 | } 31 | 32 | func (m *GlobalNotifierRabbitmq) GetType() core.ModuleType { 33 | return TYPE_NAME 34 | } 35 | 36 | func (m *GlobalNotifierRabbitmq) GetName() string { 37 | return MODULE_NAME 38 | } 39 | 40 | func (m *GlobalNotifierRabbitmq) GetConfig() interface{} { 41 | return &Config{} 42 | } 43 | 44 | func (m *GlobalNotifierRabbitmq) InitPipe(w io.Writer, r io.Reader) error { 45 | m.reader = r 46 | m.writer = w 47 | return nil 48 | } 49 | 50 | func (m *GlobalNotifierRabbitmq) InitModule(_cfg interface{}) error { 51 | m.config = _cfg.(*Config) 52 | 53 | // connect to server 54 | conn, err := amqp.Dial(m.config.RabbitMQURL) 55 | if err != nil { 56 | return err 57 | } 58 | m.conn = conn 59 | 60 | // init channel 61 | ch, err := m.conn.Channel() 62 | if err != nil { 63 | return err 64 | } 65 | m.channel = ch 66 | 67 | // declare queue 68 | q, err := m.channel.QueueDeclare( 69 | m.config.QueueName, // name 70 | m.config.QueueDurable, // durable 71 | m.config.QueueAutoDelete, // delete when unused 72 | m.config.QueueExclusive, // exclusive 73 | m.config.QueueNoWait, // no-wait 74 | nil, // arguments 75 | ) 76 | if err != nil { 77 | return err 78 | } 79 | m.queue = &q 80 | 81 | // init message to be published 82 | p := amqp.Publishing{ 83 | ContentType: m.config.MsgContentType, 84 | Body: []byte(m.config.MsgBody), 85 | } 86 | m.publ = p 87 | 88 | return nil 89 | } 90 | 91 | func (m *GlobalNotifierRabbitmq) Run(ctx context.Context) error { 92 | err := m.channel.Publish( 93 | m.config.PublishExchange, // exchange 94 | m.config.PublishKey, // routing key 95 | m.config.PublishMandatory, // mandatory 96 | m.config.PublishImmediate, // immediate 97 | m.publ, 98 | ) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | return nil 104 | } 105 | 106 | func (m *GlobalNotifierRabbitmq) Close() error { 107 | m.channel.Close() 108 | m.conn.Close() 109 | return nil 110 | } 111 | -------------------------------------------------------------------------------- /modules/global/notifier/rabbitmq/rabbitmq_test.go: -------------------------------------------------------------------------------- 1 | // +build disabled 2 | 3 | package rabbitmq 4 | 5 | import ( 6 | "context" 7 | "testing" 8 | 9 | "gotest.tools/assert" 10 | ) 11 | 12 | func TestRabbitMQ_InvalidConn(t *testing.T) { 13 | conf := &Config{ 14 | QueueName: "test.queue", 15 | PublishKey: "test.queue", 16 | RabbitMQURL: "amqp://guest:guest@localhost:5679/", 17 | MsgContentType: "text/plain", 18 | MsgBody: "Hello", 19 | } 20 | 21 | rmq := &GlobalNotifierRabbitmq{} 22 | assert.Assert(t, rmq.GetConfig() != nil) 23 | 24 | err := rmq.InitModule(conf) 25 | assert.Error(t, err, "dial tcp [::1]:5679: connect: connection refused") 26 | } 27 | 28 | func TestRabbitMQ_ValidConn(t *testing.T) { 29 | conf := &Config{ 30 | QueueName: "test.queue", 31 | MsgContentType: "text/plain", 32 | MsgBody: "Hello", 33 | PublishKey: "test.queue", 34 | RabbitMQURL: "amqp://guest:guest@localhost:5672/", 35 | } 36 | 37 | rmq := &GlobalNotifierRabbitmq{} 38 | assert.Assert(t, rmq.GetConfig() != nil) 39 | 40 | err := rmq.InitModule(conf) 41 | if err != nil { 42 | t.Errorf("TestRabbitMQ: %v", err) 43 | } 44 | 45 | err = rmq.Run(context.TODO()) 46 | if err != nil { 47 | t.Errorf("TestRabbitMQ: %v", err) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /modules/global/notifier/slack/config.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | type Config struct { 4 | Hook string 5 | MessageSuccess string 6 | MessageFail string 7 | Success bool 8 | } 9 | -------------------------------------------------------------------------------- /modules/global/notifier/slack/slack.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "encoding/json" 7 | "errors" 8 | "fmt" 9 | "io" 10 | "net/http" 11 | 12 | "github.com/copybird/copybird/core" 13 | ) 14 | 15 | const ( 16 | GROUP_NAME = "global" 17 | TYPE_NAME = "notifier" 18 | MODULE_NAME = "slack" 19 | SlackHookSite = "https://hooks.slack.com/services" 20 | HeaderContentType = "Content-Type" 21 | MIMEApplicationJSON = "application/json" 22 | ) 23 | 24 | type GlobalNotifierSlack struct { 25 | core.Module 26 | config *Config 27 | reader io.Reader 28 | writer io.Writer 29 | } 30 | 31 | type SlackMessage struct { 32 | Text string `json:"text"` 33 | } 34 | 35 | func (m *GlobalNotifierSlack) GetGroup() core.ModuleGroup { 36 | return GROUP_NAME 37 | } 38 | 39 | func (m *GlobalNotifierSlack) GetType() core.ModuleType { 40 | return TYPE_NAME 41 | } 42 | 43 | func (m *GlobalNotifierSlack) GetName() string { 44 | return MODULE_NAME 45 | } 46 | 47 | func (m *GlobalNotifierSlack) InitPipe(w io.Writer, r io.Reader) error { 48 | m.reader = r 49 | m.writer = w 50 | return nil 51 | } 52 | 53 | func (m *GlobalNotifierSlack) InitModule(_cfg interface{}) error { 54 | m.config = _cfg.(*Config) 55 | return nil 56 | } 57 | 58 | func (m *GlobalNotifierSlack) Run(ctx context.Context) error { 59 | if err := m.config.NotifySlackChannel(); err != nil { 60 | return err 61 | } 62 | 63 | return nil 64 | } 65 | func (m *GlobalNotifierSlack) GetConfig() interface{} { 66 | return &Config{} 67 | } 68 | 69 | func (m *GlobalNotifierSlack) Close() error { 70 | return nil 71 | } 72 | 73 | func (c *Config) NotifySlackChannel() error { 74 | 75 | urls := fmt.Sprintf("%s/%s", SlackHookSite, c.Hook) 76 | 77 | var slackMessage []byte 78 | var err error 79 | 80 | client := &http.Client{} 81 | if c.Success { 82 | slackMessage, err = json.Marshal(SlackMessage{Text: " " + c.MessageSuccess}) 83 | } else { 84 | slackMessage, err = json.Marshal(SlackMessage{Text: " " + c.MessageFail}) 85 | } 86 | 87 | if err != nil { 88 | return err 89 | } 90 | 91 | req, err := http.NewRequest(http.MethodPost, urls, bytes.NewBuffer(slackMessage)) 92 | 93 | if err != nil { 94 | return err 95 | } 96 | 97 | req.Header.Set(HeaderContentType, MIMEApplicationJSON) 98 | 99 | resp, err := client.Do(req) 100 | 101 | if err != nil { 102 | return err 103 | } 104 | 105 | if resp.StatusCode != http.StatusOK { 106 | statusCode := fmt.Sprintf("%v", resp.StatusCode) 107 | return errors.New("StatusCode: " + statusCode) 108 | } 109 | 110 | return nil 111 | } 112 | -------------------------------------------------------------------------------- /modules/global/notifier/slack/slack_test.go: -------------------------------------------------------------------------------- 1 | package slack 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "errors" 7 | "fmt" 8 | "testing" 9 | 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | 13 | "github.com/jarcoal/httpmock" 14 | ) 15 | 16 | func TestGetName(t *testing.T) { 17 | n := &GlobalNotifierSlack{} 18 | require.Equal(t, MODULE_NAME, n.GetName()) 19 | } 20 | 21 | func TestGetConfig(t *testing.T) { 22 | n := &GlobalNotifierSlack{} 23 | require.Equal(t, &Config{}, n.GetConfig()) 24 | } 25 | func TestClose(t *testing.T) { 26 | n := &GlobalNotifierSlack{} 27 | assert.Equal(t, nil, n.Close()) 28 | } 29 | 30 | func TestInitPipe(t *testing.T) { 31 | n := &GlobalNotifierSlack{} 32 | bufInput := bytes.NewBuffer([]byte("hello world")) 33 | bufOutput := &bytes.Buffer{} 34 | require.NoError(t, n.InitPipe(bufOutput, bufInput)) 35 | } 36 | 37 | func TestNotifySlackChannel(t *testing.T) { 38 | 39 | n := &GlobalNotifierSlack{} 40 | httpmock.Activate() 41 | defer httpmock.DeactivateAndReset() 42 | 43 | testCase := []struct { 44 | Responder httpmock.Responder 45 | Hook string 46 | Message string 47 | Success bool 48 | Error error 49 | }{ 50 | {httpmock.NewStringResponder(200, "{}"), "TKBM/BKFLY1L/tL3RAwn9EYWMaMX", "Hello", true, nil}, 51 | {httpmock.NewStringResponder(400, "{}"), "TKBM/YWMaMX", "Hello", false, errors.New("StatusCode: 400")}, 52 | } 53 | 54 | for _, tt := range testCase { 55 | urls := fmt.Sprintf("%s/%s", SlackHookSite, tt.Hook) 56 | httpmock.RegisterResponder("POST", urls, tt.Responder) 57 | assert.NoError(t, n.InitModule(&Config{Hook: tt.Hook, MessageSuccess: tt.Message, Success: tt.Success})) 58 | err := n.Run(context.TODO()) 59 | assert.Equal(t, tt.Error, err) 60 | } 61 | } 62 | 63 | func TestRun(t *testing.T) { 64 | 65 | httpmock.Activate() 66 | defer httpmock.DeactivateAndReset() 67 | 68 | testCase := []struct { 69 | Responder httpmock.Responder 70 | Hook string 71 | Message string 72 | Success bool 73 | Error error 74 | }{ 75 | {httpmock.NewStringResponder(200, "{}"), "TKBM/BKFLY1L/tL3RAwn9EYWMaMX", "Hello", true, nil}, 76 | {httpmock.NewStringResponder(400, "{}"), "TKBM/YWMaMX", "Hello", false, errors.New("StatusCode: 400")}, 77 | } 78 | 79 | for _, tt := range testCase { 80 | n := &GlobalNotifierSlack{} 81 | urls := fmt.Sprintf("%s/%s", SlackHookSite, tt.Hook) 82 | httpmock.RegisterResponder("POST", urls, tt.Responder) 83 | assert.NoError(t, n.InitModule(&Config{Hook: tt.Hook, MessageSuccess: tt.Message, Success: tt.Success})) 84 | err := n.Run(context.TODO()) 85 | assert.Equal(t, tt.Error, err) 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /modules/global/notifier/telegram/config.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | type Config struct { 4 | Token string 5 | Message string 6 | ChannelID int64 7 | } 8 | -------------------------------------------------------------------------------- /modules/global/notifier/telegram/telegram.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "context" 5 | "io" 6 | "log" 7 | 8 | tgbotapi "github.com/go-telegram-bot-api/telegram-bot-api" 9 | 10 | "github.com/copybird/copybird/core" 11 | ) 12 | 13 | const GROUP_NAME = "global" 14 | const TYPE_NAME = "notifier" 15 | const MODULE_NAME = "telegram" 16 | 17 | type GlobalNotifierTelegram struct { 18 | core.Module 19 | config *Config 20 | reader io.Reader 21 | writer io.Writer 22 | } 23 | 24 | func (m *GlobalNotifierTelegram) GetGroup() core.ModuleGroup { 25 | return GROUP_NAME 26 | } 27 | 28 | func (m *GlobalNotifierTelegram) GetType() core.ModuleType { 29 | return TYPE_NAME 30 | } 31 | 32 | func (m *GlobalNotifierTelegram) GetName() string { 33 | return MODULE_NAME 34 | } 35 | 36 | func (m *GlobalNotifierTelegram) InitPipe(w io.Writer, r io.Reader) error { 37 | m.reader = r 38 | m.writer = w 39 | return nil 40 | } 41 | 42 | func (m *GlobalNotifierTelegram) InitModule(_cfg interface{}) error { 43 | m.config = _cfg.(*Config) 44 | return nil 45 | } 46 | 47 | func (m *GlobalNotifierTelegram) Run(ctx context.Context) error { 48 | if err := m.config.NotifyTelegramChannel(); err != nil { 49 | return err 50 | } 51 | 52 | return nil 53 | } 54 | func (m *GlobalNotifierTelegram) GetConfig() interface{} { 55 | return &Config{} 56 | } 57 | 58 | func (m *GlobalNotifierTelegram) Close() error { 59 | return nil 60 | } 61 | 62 | func (conf *Config) NotifyTelegramChannel() error { 63 | 64 | bot, err := tgbotapi.NewBotAPI(conf.Token) 65 | if err != nil { 66 | return err 67 | } 68 | 69 | bot.Debug = true 70 | 71 | log.Printf("Authorized on account %s", bot.Self.UserName) 72 | 73 | u := tgbotapi.NewUpdate(0) 74 | u.Timeout = 60 75 | 76 | msg := tgbotapi.NewMessage(conf.ChannelID, conf.Message) 77 | bot.Send(msg) 78 | 79 | return nil 80 | } 81 | -------------------------------------------------------------------------------- /modules/global/notifier/telegram/telegram_test.go: -------------------------------------------------------------------------------- 1 | package telegram 2 | 3 | import ( 4 | "bytes" 5 | "testing" 6 | 7 | "github.com/stretchr/testify/assert" 8 | "github.com/stretchr/testify/require" 9 | ) 10 | 11 | func TestGetName(t *testing.T) { 12 | n := &GlobalNotifierTelegram{} 13 | require.Equal(t, MODULE_NAME, n.GetName()) 14 | } 15 | 16 | func TestGetConfig(t *testing.T) { 17 | n := &GlobalNotifierTelegram{} 18 | require.Equal(t, &Config{}, n.GetConfig()) 19 | } 20 | func TestClose(t *testing.T) { 21 | n := &GlobalNotifierTelegram{} 22 | assert.Equal(t, nil, n.Close()) 23 | } 24 | 25 | func TestInitPipe(t *testing.T) { 26 | n := &GlobalNotifierTelegram{} 27 | bufInput := bytes.NewBuffer([]byte("hello world")) 28 | bufOutput := &bytes.Buffer{} 29 | require.NoError(t, n.InitPipe(bufOutput, bufInput)) 30 | } 31 | -------------------------------------------------------------------------------- /modules/global/notifier/twilio/config.go: -------------------------------------------------------------------------------- 1 | package twillio 2 | 3 | type Config struct { 4 | AccountSid string 5 | AuthToken string 6 | From string 7 | To string 8 | } 9 | -------------------------------------------------------------------------------- /modules/global/notifier/twilio/twilio.go: -------------------------------------------------------------------------------- 1 | package twillio 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | 7 | "github.com/copybird/copybird/core" 8 | 9 | "github.com/sfreiberg/gotwilio" 10 | ) 11 | 12 | const GROUP_NAME = "global" 13 | const TYPE_NAME = "notifier" 14 | const MODULE_NAME = "twillio" 15 | 16 | type GlobalNotifierTwilio struct { 17 | core.Module 18 | config *Config 19 | client *gotwilio.Twilio 20 | } 21 | 22 | func (m *GlobalNotifierTwilio) GetGroup() core.ModuleGroup { 23 | return GROUP_NAME 24 | } 25 | 26 | func (m *GlobalNotifierTwilio) GetType() core.ModuleType { 27 | return TYPE_NAME 28 | } 29 | 30 | func (t *GlobalNotifierTwilio) GetName() string { 31 | return MODULE_NAME 32 | } 33 | 34 | func (t *GlobalNotifierTwilio) GetConfig() interface{} { 35 | return &Config{} 36 | } 37 | 38 | func (t *GlobalNotifierTwilio) InitModule(_conf interface{}) error { 39 | conf := _conf.(*Config) 40 | t.client = gotwilio.NewTwilioClient(conf.AccountSid, conf.AuthToken) 41 | return nil 42 | } 43 | 44 | func (t *GlobalNotifierTwilio) Run(ctx context.Context) error { 45 | 46 | _, exception, err := t.client.SendSMS(t.config.From, t.config.To, "Dump created successfully", "", "") 47 | if err != nil { 48 | return err 49 | } 50 | if exception != nil { 51 | return errors.New(exception.Message) 52 | } 53 | return nil 54 | } 55 | 56 | func (t *GlobalNotifierTwilio) Close() error { 57 | return nil 58 | } 59 | -------------------------------------------------------------------------------- /modules/global/notifier/twilio/twilio_test.go: -------------------------------------------------------------------------------- 1 | package twillio 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/stretchr/testify/require" 7 | ) 8 | 9 | func TestGetName(t *testing.T) { 10 | n := &GlobalNotifierTwilio{} 11 | require.Equal(t, "twillio", n.GetName()) 12 | } 13 | 14 | func TestGetConfig(t *testing.T) { 15 | n := &GlobalNotifierTwilio{} 16 | require.Equal(t, &Config{}, n.GetConfig()) 17 | } 18 | 19 | func TestInitModule(t *testing.T) { 20 | n := &GlobalNotifierTwilio{} 21 | require.NoError(t, n.InitModule(&Config{}), "should not be any error here") 22 | } 23 | -------------------------------------------------------------------------------- /modules/global/notifier/webcallback/config.go: -------------------------------------------------------------------------------- 1 | package webcallback 2 | 3 | type Config struct { 4 | TargetUrl string 5 | SuccessMsg string 6 | FailMsg string 7 | } 8 | -------------------------------------------------------------------------------- /modules/global/notifier/webcallback/webcallback.go: -------------------------------------------------------------------------------- 1 | package webcallback 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "net/http" 8 | "net/url" 9 | "strings" 10 | "time" 11 | 12 | "github.com/copybird/copybird/core" 13 | ) 14 | 15 | const GROUP_NAME = "global" 16 | const TYPE_NAME = "notifier" 17 | const MODULE_NAME = "webcallback" 18 | 19 | type GlobalNotifierWebcallback struct { 20 | core.Module 21 | config *Config 22 | } 23 | 24 | func (m *GlobalNotifierWebcallback) GetGroup() core.ModuleGroup { 25 | return GROUP_NAME 26 | } 27 | 28 | func (m *GlobalNotifierWebcallback) GetType() core.ModuleType { 29 | return TYPE_NAME 30 | } 31 | 32 | func (m *GlobalNotifierWebcallback) GetName() string { 33 | return MODULE_NAME 34 | } 35 | 36 | func (m *GlobalNotifierWebcallback) GetConfig() interface{} { 37 | return &Config{} 38 | } 39 | 40 | func (m *GlobalNotifierWebcallback) InitModule(_cfg interface{}) error { 41 | m.config = _cfg.(*Config) 42 | return nil 43 | } 44 | 45 | func (m *GlobalNotifierWebcallback) Run(ctx context.Context) error { 46 | if err := m.SendNotification(); err != nil { 47 | return err 48 | } 49 | 50 | return nil 51 | } 52 | 53 | func (m *GlobalNotifierWebcallback) Close() error { 54 | return nil 55 | } 56 | 57 | func (m *GlobalNotifierWebcallback) SendNotification() error { 58 | 59 | //Set request body params 60 | data := url.Values{} 61 | data.Set("success", "true") 62 | 63 | req, err := http.NewRequest("GET", m.config.TargetUrl, strings.NewReader(data.Encode())) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | // Set headers 69 | req.Header.Add("Content-Type", "application/x-www-form-urlencoded") 70 | 71 | // Set client timeout 72 | client := &http.Client{Timeout: time.Second * 10} 73 | 74 | // Send request 75 | resp, err := client.Do(req) 76 | if err != nil { 77 | return err 78 | } 79 | 80 | if resp.StatusCode != http.StatusOK { 81 | statusCode := fmt.Sprintf("%v", resp.StatusCode) 82 | return errors.New("StatusCode: " + statusCode) 83 | } 84 | 85 | defer resp.Body.Close() 86 | 87 | return nil 88 | } 89 | -------------------------------------------------------------------------------- /modules/global/notifier/webcallback/webcallback_test.go: -------------------------------------------------------------------------------- 1 | package webcallback 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | 10 | "github.com/jarcoal/httpmock" 11 | ) 12 | 13 | func TestSendNotification(t *testing.T) { 14 | 15 | httpmock.Activate() 16 | defer httpmock.DeactivateAndReset() 17 | 18 | testCase := []struct { 19 | Responder httpmock.Responder 20 | TargetUrl string 21 | SuccessMsg string 22 | FailMsg string 23 | Success bool 24 | Error error 25 | }{ 26 | {httpmock.NewStringResponder(200, "{}"), "google.com", "Succes", "Fail", true, nil}, 27 | {httpmock.NewStringResponder(400, "{}"), "google.com", "Succces", "Fail", false, errors.New("StatusCode: 400")}, 28 | } 29 | 30 | for _, tc := range testCase { 31 | urls := fmt.Sprintf("%s", tc.TargetUrl) 32 | httpmock.RegisterResponder("GET", urls, tc.Responder) 33 | n := &GlobalNotifierWebcallback{} 34 | assert.NoError(t, n.InitModule(&Config{TargetUrl: tc.TargetUrl, SuccessMsg: tc.SuccessMsg, FailMsg: tc.FailMsg})) 35 | err := n.SendNotification() 36 | assert.Equal(t, tc.Error, err) 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /modules/restore/decompress/gzip/gzip.go: -------------------------------------------------------------------------------- 1 | package gzip 2 | 3 | import ( 4 | "compress/gzip" 5 | "context" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/copybird/copybird/core" 10 | ) 11 | 12 | const GROUP_NAME = "restore" 13 | const TYPE_NAME = "decompress" 14 | const MODULE_NAME = "gzip" 15 | 16 | type RestoreDecompressGzip struct { 17 | core.Module 18 | reader io.Reader 19 | writer io.Writer 20 | } 21 | 22 | func (m *RestoreDecompressGzip) GetGroup() core.ModuleGroup { 23 | return GROUP_NAME 24 | } 25 | 26 | func (m *RestoreDecompressGzip) GetType() core.ModuleType { 27 | return TYPE_NAME 28 | } 29 | 30 | func (m *RestoreDecompressGzip) GetName() string { 31 | return MODULE_NAME 32 | } 33 | 34 | func (m *RestoreDecompressGzip) GetConfig() interface{} { 35 | return nil 36 | } 37 | 38 | func (m *RestoreDecompressGzip) InitPipe(w io.Writer, r io.Reader) error { 39 | m.reader = r 40 | m.writer = w 41 | return nil 42 | } 43 | 44 | func (m *RestoreDecompressGzip) InitModule(_cfg interface{}) error { 45 | return nil 46 | } 47 | 48 | func (m *RestoreDecompressGzip) Run(ctx context.Context) error { 49 | gr, err := gzip.NewReader(m.reader) 50 | if err != nil { 51 | return fmt.Errorf("cant start gzip reader with error: %s", err) 52 | } 53 | defer gr.Close() 54 | _, err = io.Copy(m.writer, gr) 55 | if err != nil { 56 | return fmt.Errorf("copy error: %s", err) 57 | } 58 | return nil 59 | } 60 | 61 | func (m *RestoreDecompressGzip) Close() error { 62 | return nil 63 | } 64 | -------------------------------------------------------------------------------- /modules/restore/decompress/gzip/gzip_test.go: -------------------------------------------------------------------------------- 1 | package gzip 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "testing" 7 | 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | var decompressor RestoreDecompressGzip 12 | 13 | func TestCompress_InitCompress_Default_Compress(t *testing.T) { 14 | err := decompressor.InitModule(nil) 15 | assert.Equal(t, err, nil) 16 | } 17 | 18 | func TestCompress_Unzip_Success_Unzip(t *testing.T) { 19 | var wr bytes.Buffer 20 | rd := bytes.NewReader([]byte{ 21 | 0x1f, 0x8b, 0x08, 0x08, 0xc8, 0x58, 0x13, 0x4a, 22 | 0x00, 0x03, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x2e, 23 | 0x74, 0x78, 0x74, 0x00, 0xcb, 0x48, 0xcd, 0xc9, 24 | 0xc9, 0x57, 0x28, 0xcf, 0x2f, 0xca, 0x49, 0xe1, 25 | 0x02, 0x00, 0x2d, 0x3b, 0x08, 0xaf, 0x0c, 0x00, 26 | 0x00, 0x00, 27 | }) 28 | 29 | _ = decompressor.InitModule(nil) 30 | _ = decompressor.InitPipe(&wr, rd) 31 | err := decompressor.Run(context.TODO()) 32 | assert.Equal(t, err, nil) 33 | assert.Equal(t, wr.String(), "hello world\n") 34 | } 35 | 36 | func TestCompress_Unzip_Unsuccess_Unzip(t *testing.T) { 37 | var wr bytes.Buffer 38 | rd := bytes.NewReader([]byte{}) 39 | 40 | _ = decompressor.InitModule(nil) 41 | _ = decompressor.InitPipe(&wr, rd) 42 | err := decompressor.Run(context.TODO()) 43 | assert.NotEqual(t, err, nil) 44 | assert.Equal(t, wr.String(), "") 45 | } 46 | -------------------------------------------------------------------------------- /modules/restore/decompress/lz4/config.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | type Config struct{} 4 | -------------------------------------------------------------------------------- /modules/restore/decompress/lz4/lz4.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | import ( 4 | "context" 5 | "errors" 6 | "fmt" 7 | "io" 8 | 9 | "github.com/copybird/copybird/core" 10 | "github.com/pierrec/lz4" 11 | ) 12 | 13 | var ( 14 | errCompLevel = errors.New("compression level must be between -1 and 9") 15 | errNotCompressible = errors.New("is not compressible") 16 | ) 17 | 18 | const GROUP_NAME = "restore" 19 | const TYPE_NAME = "decompress" 20 | const MODULE_NAME = "lz4" 21 | 22 | // RestoreDecompressLz4 represents ... 23 | type RestoreDecompressLz4 struct { 24 | core.Module 25 | reader io.Reader 26 | writer io.Writer 27 | } 28 | 29 | func (m *RestoreDecompressLz4) GetGroup() core.ModuleGroup { 30 | return GROUP_NAME 31 | } 32 | 33 | func (m *RestoreDecompressLz4) GetType() core.ModuleType { 34 | return TYPE_NAME 35 | } 36 | 37 | func (m *RestoreDecompressLz4) GetName() string { 38 | return MODULE_NAME 39 | } 40 | 41 | func (m *RestoreDecompressLz4) GetConfig() interface{} { 42 | return &Config{} 43 | } 44 | 45 | func (m *RestoreDecompressLz4) InitPipe(w io.Writer, r io.Reader) error { 46 | m.reader = r 47 | m.writer = w 48 | return nil 49 | } 50 | 51 | func (m *RestoreDecompressLz4) InitModule(_cfg interface{}) error { 52 | return nil 53 | } 54 | 55 | func (m *RestoreDecompressLz4) Run(ctx context.Context) error { 56 | // make a buffer to keep chunks that are read 57 | lr := lz4.NewReader(m.reader) 58 | 59 | _, err := io.Copy(m.writer, lr) 60 | if err != nil { 61 | return fmt.Errorf("copy error: %s", err) 62 | } 63 | 64 | return nil 65 | } 66 | 67 | // Close closes compressor 68 | func (m *RestoreDecompressLz4) Close() error { 69 | return nil 70 | } 71 | -------------------------------------------------------------------------------- /modules/restore/decompress/lz4/lz4_test.go: -------------------------------------------------------------------------------- 1 | package lz4 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "io" 7 | "testing" 8 | 9 | "github.com/pierrec/lz4" 10 | "github.com/stretchr/testify/assert" 11 | ) 12 | 13 | func TestDecompress(t *testing.T) { 14 | wr := new(bytes.Buffer) 15 | var decompressor RestoreDecompressLz4 16 | 17 | s := `I bomb atomically, Socrates' philosophies 18 | And hypotheses can't define how I be droppin' these 19 | Mockeries, lyrically perform armed robbery 20 | Flee with the lottery, possibly they spotted me 21 | Battle-scarred shogun, explosion when my pen hits 22 | Tremendous, ultra-violet shine blind forensics 23 | I inspect view through the future see millennium 24 | Killa Beez sold fifty gold sixty platinum 25 | Shackling the masses with drastic rap tactics 26 | Graphic displays melt the steel like blacksmiths 27 | Black Wu jackets Queen Beez ease the guns in 28 | Rumblin' patrolmen tear gas laced the function 29 | Heads by the score take flight incite a war 30 | Chicks hit the floor, die hard fans demand more 31 | Behold the bold soldier, control the globe slowly 32 | Proceeds to blow swingin' swords like Shinobi 33 | Stomp grounds I pound footprints in solid rock 34 | Wu got it locked, performin' live on your hottest block` 35 | hw, err := compress(s) 36 | assert.NoError(t, err) 37 | rd := bytes.NewReader(hw) 38 | assert.NoError(t, decompressor.InitModule(nil)) 39 | assert.NoError(t, decompressor.InitPipe(wr, rd)) 40 | assert.NoError(t, decompressor.Run(context.TODO())) 41 | 42 | assert.Equal(t, s, wr.String()) 43 | } 44 | 45 | func compress(s string) ([]byte, error) { 46 | r := bytes.NewReader([]byte(s)) 47 | w := &bytes.Buffer{} 48 | zw := lz4.NewWriter(w) 49 | _, err := io.Copy(zw, r) 50 | if err != nil { 51 | return nil, err 52 | } 53 | if err := zw.Close(); err != nil { 54 | return nil, err 55 | } 56 | return w.Bytes(), nil 57 | 58 | } 59 | -------------------------------------------------------------------------------- /modules/restore/decompress/lz4/test.txt: -------------------------------------------------------------------------------- 1 | ssdmnfbsnmdfbmsdnfbsmdnfbdsf -------------------------------------------------------------------------------- /modules/restore/decrypt/aesgcm/aesgcm.go: -------------------------------------------------------------------------------- 1 | package aesgcm 2 | 3 | import ( 4 | "context" 5 | "crypto/aes" 6 | "crypto/cipher" 7 | "crypto/rand" 8 | "encoding/hex" 9 | "errors" 10 | "fmt" 11 | "io" 12 | 13 | "github.com/copybird/copybird/core" 14 | ) 15 | 16 | const GROUP_NAME = "restore" 17 | const TYPE_NAME = "decrypt" 18 | const MODULE_NAME = "aesgcm" 19 | 20 | type RestoreDecryptAesgcm struct { 21 | core.Module 22 | reader io.Reader 23 | writer io.Writer 24 | gcm cipher.AEAD 25 | nonce []byte 26 | } 27 | 28 | func (m *RestoreDecryptAesgcm) GetGroup() core.ModuleGroup { 29 | return GROUP_NAME 30 | } 31 | 32 | func (m *RestoreDecryptAesgcm) GetType() core.ModuleType { 33 | return TYPE_NAME 34 | } 35 | 36 | func (m *RestoreDecryptAesgcm) GetName() string { 37 | return MODULE_NAME 38 | } 39 | 40 | func (m *RestoreDecryptAesgcm) GetConfig() interface{} { 41 | return &Config{} 42 | } 43 | 44 | func (m *RestoreDecryptAesgcm) InitPipe(w io.Writer, r io.Reader) error { 45 | m.reader = r 46 | m.writer = w 47 | return nil 48 | } 49 | 50 | func (m *RestoreDecryptAesgcm) InitModule(_cfg interface{}) error { 51 | cfg := _cfg.(*Config) 52 | 53 | if cfg.Key == "" { 54 | return errors.New("need key") 55 | } 56 | key, err := hex.DecodeString(cfg.Key) 57 | if err != nil { 58 | return fmt.Errorf("cipher key hex decode err: %s", err) 59 | } 60 | block, err := aes.NewCipher(key) 61 | if err != nil { 62 | return fmt.Errorf("cipher init err: %s", err) 63 | } 64 | m.gcm, err = cipher.NewGCM(block) 65 | if err != nil { 66 | return fmt.Errorf("aes gcm init err: %s", err) 67 | } 68 | m.nonce = make([]byte, m.gcm.NonceSize()) 69 | if _, err = io.ReadFull(rand.Reader, m.nonce); err != nil { 70 | return fmt.Errorf("nonce generate err: %s", err) 71 | } 72 | return nil 73 | } 74 | 75 | func (m *RestoreDecryptAesgcm) Run(ctx context.Context) error { 76 | var err error 77 | buf := make([]byte, 16) 78 | bufOut := make([]byte, 16) 79 | for { 80 | _, err = m.reader.Read(buf) 81 | if err != nil { 82 | if err == io.EOF { 83 | break 84 | } 85 | return fmt.Errorf("read err: %s", err) 86 | } 87 | _, err = m.gcm.Open(bufOut, m.nonce, buf, nil) 88 | if err != nil { 89 | return fmt.Errorf("decrypt err: %s", err) 90 | } 91 | _, err = m.writer.Write(bufOut) 92 | if err != nil { 93 | return fmt.Errorf("write err: %s", err) 94 | } 95 | } 96 | return nil 97 | } 98 | 99 | func (m *RestoreDecryptAesgcm) Close() error { 100 | return nil 101 | } 102 | -------------------------------------------------------------------------------- /modules/restore/decrypt/aesgcm/aesgcm_test.go: -------------------------------------------------------------------------------- 1 | // +build disabled 2 | 3 | package aesgcm 4 | 5 | import ( 6 | "bytes" 7 | "context" 8 | "testing" 9 | 10 | "gotest.tools/assert" 11 | ) 12 | 13 | func TestEncryptionAESGCM(t *testing.T) { 14 | key := "666f6f6261726b657973616d706c655f" 15 | enc := &RestoreDecryptAesgcm{} 16 | 17 | // TODO insert valid encrypted string 18 | bufInput := bytes.NewBuffer([]byte("invalid encrypted string")) 19 | bufOutput := &bytes.Buffer{} 20 | assert.Assert(t, enc.GetConfig() != nil) 21 | assert.NilError(t, enc.InitPipe(bufOutput, bufInput)) 22 | assert.NilError(t, enc.InitModule(&Config{Key: key})) 23 | assert.NilError(t, enc.Run(context.TODO())) 24 | assert.NilError(t, enc.Close()) 25 | } 26 | -------------------------------------------------------------------------------- /modules/restore/decrypt/aesgcm/config.go: -------------------------------------------------------------------------------- 1 | package aesgcm 2 | 3 | type Config struct { 4 | Key string 5 | } 6 | -------------------------------------------------------------------------------- /modules/restore/output/mysql/config.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | // MySQLConfig stores configuration for MySQL backups 4 | type MySQLConfig struct { 5 | DSN string 6 | } 7 | -------------------------------------------------------------------------------- /modules/restore/output/mysql/mysql.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "io" 7 | 8 | "github.com/copybird/copybird/core" 9 | "github.com/davecgh/go-spew/spew" 10 | "github.com/xwb1989/sqlparser" 11 | 12 | _ "github.com/go-sql-driver/mysql" 13 | ) 14 | 15 | // Module Constants 16 | const ( 17 | GROUP_NAME = "restore" 18 | TYPE_NAME = "output" 19 | MODULE_NAME = "mysql" 20 | ) 21 | 22 | type ( 23 | // RestoreOutputMysql is struct storing inner properties for mysql backups 24 | RestoreOutputMysql struct { 25 | core.Module 26 | conn *sql.DB 27 | config *MySQLConfig 28 | reader io.Reader 29 | writer io.Writer 30 | } 31 | ) 32 | 33 | // GetGroup returns group 34 | func (m *RestoreOutputMysql) GetGroup() core.ModuleGroup { 35 | return GROUP_NAME 36 | } 37 | 38 | // GetType returns type 39 | func (m *RestoreOutputMysql) GetType() core.ModuleType { 40 | return TYPE_NAME 41 | } 42 | 43 | // GetName returns name of module 44 | func (m *RestoreOutputMysql) GetName() string { 45 | return MODULE_NAME 46 | } 47 | 48 | // GetConfig returns config of module 49 | func (m *RestoreOutputMysql) GetConfig() interface{} { 50 | return &MySQLConfig{ 51 | DSN: "root:root@tcp(localhost:3306)/test", 52 | } 53 | } 54 | 55 | // InitPipe initializes pipe 56 | func (m *RestoreOutputMysql) InitPipe(w io.Writer, r io.Reader) error { 57 | m.reader = r 58 | m.writer = w 59 | return nil 60 | } 61 | 62 | // InitModule initializes module 63 | func (m *RestoreOutputMysql) InitModule(cfg interface{}) error { 64 | m.config = cfg.(*MySQLConfig) 65 | conn, err := sql.Open("mysql", m.config.DSN) 66 | if err != nil { 67 | return err 68 | } 69 | if err := conn.Ping(); err != nil { 70 | return err 71 | } 72 | m.conn = conn 73 | 74 | return nil 75 | } 76 | 77 | // Run dumps database 78 | func (m *RestoreOutputMysql) Run(ctx context.Context) error { 79 | return m.RestoreDatabase() 80 | } 81 | 82 | // RestoreDatabase restores db 83 | func (m *RestoreOutputMysql) RestoreDatabase() error { 84 | tokenizer := sqlparser.NewTokenizer(m.reader) 85 | 86 | for { 87 | stmt, err := sqlparser.ParseNext(tokenizer) 88 | if err != nil { 89 | spew.Dump(err) 90 | } 91 | if err == io.EOF { 92 | break 93 | } 94 | switch stmt.(type) { 95 | case *sqlparser.Select, *sqlparser.Insert, *sqlparser.DBDDL, *sqlparser.DDL: 96 | if _, err := m.conn.Exec(sqlparser.String(stmt)); err != nil { 97 | return err 98 | } 99 | default: 100 | continue 101 | } 102 | 103 | } 104 | return nil 105 | } 106 | 107 | // Close connection to DB. 108 | func (m *RestoreOutputMysql) Close() error { 109 | return m.conn.Close() 110 | } 111 | 112 | func (m *RestoreOutputMysql) execute(line string) error { 113 | // Start transaction 114 | tx, err := m.conn.Begin() 115 | if err != nil { 116 | return err 117 | } 118 | 119 | // Execute transaction 120 | _, err = tx.Exec(line) 121 | if err != nil { 122 | return err 123 | } 124 | 125 | // Commit transaction 126 | err = tx.Commit() 127 | if err != nil { 128 | return err 129 | } 130 | 131 | return nil 132 | } 133 | -------------------------------------------------------------------------------- /modules/restore/output/mysql/mysql_test.go: -------------------------------------------------------------------------------- 1 | package mysql 2 | 3 | import ( 4 | "context" 5 | "os" 6 | "testing" 7 | 8 | "gotest.tools/assert" 9 | ) 10 | 11 | var rs RestoreOutputMysql 12 | var conf = MySQLConfig{ 13 | DSN: "root:root@tcp(localhost:3306)/test", 14 | } 15 | 16 | func TestRestoreOutputMysql_Run(t *testing.T) { 17 | err := rs.InitModule(&conf) 18 | assert.Equal(t, err, nil) 19 | 20 | // TODO: Need parse file, but after implement sql formatter 21 | f, _ := os.Open("../../../../samples/mysql.sql") 22 | rs.InitPipe(nil, f) 23 | 24 | err = rs.Run(context.TODO()) 25 | if err != nil { 26 | print(err.Error()) 27 | } 28 | assert.Equal(t, err, nil) 29 | } 30 | -------------------------------------------------------------------------------- /modules/restore/output/postgresql/config.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | // Config PostgreSQL 4 | type ( 5 | Config struct { 6 | DSN string 7 | } 8 | ) 9 | -------------------------------------------------------------------------------- /modules/restore/output/postgresql/postgresql.go: -------------------------------------------------------------------------------- 1 | package postgres 2 | 3 | import ( 4 | "context" 5 | "database/sql" 6 | "io" 7 | 8 | "github.com/copybird/copybird/core" 9 | "github.com/davecgh/go-spew/spew" 10 | _ "github.com/lib/pq" 11 | "github.com/xwb1989/sqlparser" 12 | ) 13 | 14 | // Module Constants 15 | const GROUP_NAME = "restore" 16 | const TYPE_NAME = "output" 17 | const MODULE_NAME = "postgresql" 18 | 19 | type ( 20 | // BackupInputPostgresql is struct storing inner properties for mysql backups 21 | RestoreOutputPostgresql struct { 22 | core.Module 23 | reader io.Reader 24 | writer io.Writer 25 | conn *sql.DB 26 | config *Config 27 | } 28 | ) 29 | 30 | // GetGroup returns module group 31 | func (r *RestoreOutputPostgresql) GetGroup() core.ModuleGroup { 32 | return GROUP_NAME 33 | } 34 | 35 | // GetType returns module type 36 | func (r *RestoreOutputPostgresql) GetType() core.ModuleType { 37 | return TYPE_NAME 38 | } 39 | 40 | // GetName returns name of module 41 | func (r *RestoreOutputPostgresql) GetName() string { 42 | return MODULE_NAME 43 | } 44 | 45 | // GetConfig returns Config of module 46 | func (r *RestoreOutputPostgresql) GetConfig() interface{} { 47 | return &Config{} 48 | } 49 | 50 | // InitPipe initialize reader and writer 51 | func (r *RestoreOutputPostgresql) InitPipe(w io.Writer, rd io.Reader) error { 52 | r.reader = rd 53 | r.writer = w 54 | return nil 55 | } 56 | 57 | // InitModule initialize connection to DB 58 | func (r *RestoreOutputPostgresql) InitModule(cfg interface{}) error { 59 | r.config = cfg.(*Config) 60 | conn, err := sql.Open("postgres", r.config.DSN) 61 | if err != nil { 62 | return err 63 | } 64 | if err := conn.Ping(); err != nil { 65 | return err 66 | } 67 | r.conn = conn 68 | return nil 69 | } 70 | 71 | // Run dumps database 72 | func (r *RestoreOutputPostgresql) Run(ctx context.Context) error { 73 | tokenizer := sqlparser.NewTokenizer(r.reader) 74 | 75 | for { 76 | stmt, err := sqlparser.ParseNext(tokenizer) 77 | if err != nil { 78 | spew.Dump(err) 79 | } 80 | if err == io.EOF { 81 | break 82 | } 83 | spew.Dump(stmt) 84 | spew.Dump(sqlparser.String(stmt)) 85 | switch stmt.(type) { 86 | case *sqlparser.Select, *sqlparser.Insert, *sqlparser.DBDDL, *sqlparser.DDL: 87 | if _, err := r.conn.Exec(sqlparser.String(stmt)); err != nil { 88 | return err 89 | } 90 | default: 91 | continue 92 | } 93 | 94 | } 95 | return nil 96 | } 97 | 98 | // Close connection to DB. 99 | func (r *RestoreOutputPostgresql) Close() error { 100 | return r.conn.Close() 101 | } 102 | 103 | func (r *RestoreOutputPostgresql) execute(line string) error { 104 | // Start transaction 105 | tx, err := r.conn.Begin() 106 | if err != nil { 107 | return err 108 | } 109 | 110 | // Execute transaction 111 | _, err = tx.Exec(line) 112 | if err != nil { 113 | return err 114 | } 115 | 116 | // Commit transaction 117 | err = tx.Commit() 118 | if err != nil { 119 | return err 120 | } 121 | return nil 122 | } 123 | -------------------------------------------------------------------------------- /modules/restore/output/postgresql/postgresql_test.go: -------------------------------------------------------------------------------- 1 | // +build disabled 2 | 3 | package postgres 4 | 5 | import ( 6 | "context" 7 | "os" 8 | "testing" 9 | 10 | "gotest.tools/assert" 11 | ) 12 | 13 | var rs RestoreOutputPostgresql 14 | var conf = Config{ 15 | DSN: "host=127.0.0.1 port=5432 user=postgres password=postgres dbname=test sslmode=disable", 16 | } 17 | 18 | func TestRestoreOutputPostgresql_Run(t *testing.T) { 19 | err := rs.InitModule(&conf) 20 | assert.Equal(t, err, nil) 21 | 22 | // TODO: Need parse file, but after implement sql formatter 23 | //f, err := os.Open("../../../../samples/postgres.sql") 24 | //assert.Equal(t, err, nil) 25 | //rs.reader = bufio.NewReader(f) 26 | 27 | f, _ := os.Open("../../../../samples/postgres.sql") 28 | rs.InitPipe(nil, f) 29 | err = rs.Run(context.TODO()) 30 | assert.Equal(t, err, nil) 31 | } 32 | -------------------------------------------------------------------------------- /samples/config_backup_example.yml: -------------------------------------------------------------------------------- 1 | input: 2 | type: mysql 3 | config: 4 | dsn: root:root@tcp(localhost:3306)/test 5 | compress: 6 | type: gzip 7 | config: 8 | level: 1 9 | output: 10 | - type: local 11 | config: 12 | file: dump.sql.gz 13 | notifier: 14 | - type: stdout -------------------------------------------------------------------------------- /samples/mysql.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 8.0.15-6, for Linux (x86_64) 2 | -- 3 | -- Host: 127.0.0.1 Database: test 4 | -- ------------------------------------------------------ 5 | -- Server version 5.7.26 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | SET NAMES utf8mb4 ; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | /*!50717 SELECT COUNT(*) INTO @rocksdb_has_p_s_session_variables FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = 'performance_schema' AND TABLE_NAME = 'session_variables' */; 18 | /*!50717 SET @rocksdb_get_is_supported = IF (@rocksdb_has_p_s_session_variables, 'SELECT COUNT(*) INTO @rocksdb_is_supported FROM performance_schema.session_variables WHERE VARIABLE_NAME=\'rocksdb_bulk_load\'', 'SELECT 0') */; 19 | /*!50717 PREPARE s FROM @rocksdb_get_is_supported */; 20 | /*!50717 EXECUTE s */; 21 | /*!50717 DEALLOCATE PREPARE s */; 22 | /*!50717 SET @rocksdb_enable_bulk_load = IF (@rocksdb_is_supported, 'SET SESSION rocksdb_bulk_load = 1', 'SET @rocksdb_dummy_bulk_load = 0') */; 23 | /*!50717 PREPARE s FROM @rocksdb_enable_bulk_load */; 24 | /*!50717 EXECUTE s */; 25 | /*!50717 DEALLOCATE PREPARE s */; 26 | 27 | -- 28 | -- Table structure for table `authors` 29 | -- 30 | 31 | DROP TABLE IF EXISTS `authors`; 32 | /*!40101 SET @saved_cs_client = @@character_set_client */; 33 | SET character_set_client = utf8mb4 ; 34 | CREATE TABLE `authors` ( 35 | `id` int(11) NOT NULL AUTO_INCREMENT, 36 | `first_name` varchar(50) COLLATE utf8_unicode_ci NOT NULL, 37 | `last_name` varchar(50) COLLATE utf8_unicode_ci NOT NULL, 38 | `email` varchar(100) COLLATE utf8_unicode_ci NOT NULL, 39 | `birthdate` date NOT NULL, 40 | `added` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP, 41 | PRIMARY KEY (`id`), 42 | UNIQUE KEY `email` (`email`) 43 | ) ENGINE=InnoDB AUTO_INCREMENT=101 DEFAULT CHARSET=utf8 COLLATE=utf8_unicode_ci; 44 | /*!40101 SET character_set_client = @saved_cs_client */; 45 | 46 | -- 47 | -- Dumping data for table `authors` 48 | -- 49 | 50 | LOCK TABLES `authors` WRITE; 51 | /*!40000 ALTER TABLE `authors` DISABLE KEYS */; 52 | INSERT INTO `authors` VALUES (1,'Bradly','McLaughlin','purdy.richard@example.net','2001-07-21','1978-06-04 08:02:42'),(2,'Camryn','Osinski','jayce.haag@example.org','2002-04-04','1974-12-07 10:31:11'),(3,'Daryl','Haag','vleffler@example.com','1992-10-26','1978-12-05 01:11:43'),(4,'Ozella','Grant','bradtke.chaya@example.com','2013-11-20','1993-02-13 16:10:24'),(5,'Daryl','Ritchie','nader.laura@example.com','1980-02-03','1972-10-18 21:30:41'),(6,'Ellis','Donnelly','sanford.gaylord@example.org','1986-04-21','1972-02-16 16:45:19'),(7,'Clinton','Rau','shemar56@example.org','2012-09-28','1977-05-03 11:06:45'),(8,'Alberta','Denesik','xrempel@example.net','2015-04-16','1986-11-13 00:53:44'),(9,'Johanna','Nolan','anabelle20@example.com','1998-03-28','1998-04-14 18:20:08'),(10,'Monique','Pollich','gorczany.gilbert@example.net','1972-06-10','2010-02-20 03:39:31'),(11,'Tess','Rodriguez','jaren.pacocha@example.com','2001-08-01','1989-06-26 02:35:32'),(12,'Jamir','Bartell','arnoldo.smith@example.org','1970-11-23','1978-05-12 10:04:37'),(13,'Yadira','Hermiston','qdibbert@example.org','1996-07-18','1991-02-09 23:08:26'),(14,'Seth','Harris','hermiston.meta@example.com','2016-11-19','1975-01-28 13:35:52'),(15,'Judge','Fahey','trevor83@example.net','2003-01-11','1983-03-07 10:04:10'),(16,'Jarrett','Stoltenberg','kenneth.zieme@example.net','2000-07-17','2008-04-13 13:03:41'),(17,'Summer','O\'Hara','niko85@example.com','1988-12-21','2018-03-05 18:07:26'),(18,'Erik','Green','hkulas@example.net','1979-08-06','1977-10-12 07:48:33'),(19,'Julian','Marquardt','paolo50@example.com','2017-11-23','2016-11-14 00:12:28'),(20,'Tiana','Jerde','haley.marjory@example.org','1984-09-27','2001-04-25 15:01:20'),(21,'Kaia','Connelly','dach.dorothea@example.com','1974-02-18','1997-02-25 11:30:57'),(22,'Yolanda','Lockman','julianne.johns@example.net','1998-07-02','1977-05-20 01:38:16'),(23,'Brian','Keeling','laila17@example.org','1997-03-02','1985-02-27 22:14:42'),(24,'Milford','Cartwright','bridie.pollich@example.com','2008-05-23','2013-02-02 19:54:54'),(25,'Santa','Jacobi','block.kitty@example.net','1984-10-17','2008-07-30 11:06:35'),(26,'Declan','Cartwright','rossie.hartmann@example.org','1977-12-10','2015-11-26 07:53:10'),(27,'Ressie','Gerlach','wilber.armstrong@example.org','2009-03-09','1972-04-28 16:42:28'),(28,'Rosamond','Nikolaus','bednar.price@example.com','2018-04-16','2000-11-03 00:50:30'),(29,'Tatum','Abbott','willis09@example.net','2003-02-06','2010-01-19 04:09:58'),(30,'Ismael','Skiles','marley03@example.net','2018-07-22','2009-11-24 14:08:01'),(31,'Loy','Glover','lyric10@example.com','1987-12-05','1991-05-28 19:22:25'),(32,'Margaretta','Gleichner','lelah.ziemann@example.org','1981-04-16','1974-11-29 14:16:41'),(33,'Toney','Senger','colten10@example.net','1984-12-04','1985-08-20 12:54:23'),(34,'Holden','Wilderman','thalia19@example.org','2009-08-26','1994-03-29 08:47:31'),(35,'Ottis','Muller','julie45@example.com','2006-10-19','2012-01-11 13:07:48'),(36,'Destinee','Abbott','xborer@example.org','2000-02-07','2005-01-26 18:48:05'),(37,'Blanca','Grimes','kelly.yundt@example.net','1997-11-13','1989-10-21 18:50:33'),(38,'Rachel','Herzog','kaylin53@example.com','2003-12-24','1971-09-19 10:47:01'),(39,'Kallie','Ebert','greg07@example.com','2002-02-15','1989-09-28 12:58:00'),(40,'Shaina','McCullough','tyrel57@example.net','1971-10-05','1984-04-24 14:50:15'),(41,'Madilyn','Greenholt','pierre99@example.net','1993-06-08','2013-08-06 11:51:39'),(42,'Augustine','Gutkowski','dustin.schuppe@example.org','1971-12-19','1978-08-05 12:57:02'),(43,'Magdalen','Adams','estevan.bruen@example.com','1970-08-15','2002-10-07 05:15:43'),(44,'Phoebe','Bashirian','mohammed.stokes@example.org','2001-06-14','1970-12-23 03:09:45'),(45,'Kenyon','Wilderman','cassandra12@example.com','2012-08-19','1998-10-23 01:43:03'),(46,'Salvador','O\'Conner','jamie61@example.net','2003-03-09','1978-05-12 06:13:39'),(47,'Electa','Hoppe','qledner@example.com','1988-04-29','2008-05-01 17:24:21'),(48,'Marley','Fay','horacio.ankunding@example.org','1998-02-03','1999-05-13 04:50:26'),(49,'Gerard','Weber','ona.mills@example.org','2017-01-26','2005-11-20 22:17:49'),(50,'Zelda','Effertz','zrolfson@example.net','1995-08-06','1986-08-21 18:44:46'),(51,'Nikita','Crona','dietrich.michele@example.net','2009-06-30','2006-02-06 01:10:51'),(52,'Sanford','Little','bbradtke@example.org','1978-06-16','1975-02-07 06:21:51'),(53,'Dulce','Parisian','iwill@example.org','1991-10-28','2019-05-15 10:04:40'),(54,'Merl','Braun','qerdman@example.com','1978-07-29','2004-02-04 01:04:21'),(55,'Melvina','Bednar','beier.toy@example.com','1996-01-24','1981-04-25 03:13:24'),(56,'Erica','Swift','roxanne.dibbert@example.org','2011-09-10','2001-12-20 06:48:28'),(57,'Aniyah','Wilderman','kris.lela@example.com','2002-06-19','2002-05-25 01:04:56'),(58,'Kaya','Casper','murazik.keeley@example.com','1977-12-22','2004-02-06 11:03:08'),(59,'Jayda','O\'Keefe','tillman.kade@example.com','1992-02-21','1976-09-03 12:36:30'),(60,'Shanna','Hammes','damon.abernathy@example.com','1970-06-22','1981-07-06 10:21:00'),(61,'Loy','Goodwin','smarquardt@example.org','2011-05-11','1983-12-02 08:57:46'),(62,'Paige','Stracke','jdaniel@example.com','1970-10-22','1986-09-15 16:09:08'),(63,'Thad','Prosacco','avis11@example.net','2018-12-17','1978-07-17 05:03:19'),(64,'Kyla','Schaden','tabitha.schinner@example.org','1985-04-27','1993-07-09 03:12:21'),(65,'Jamey','Rath','kaylin.lesch@example.org','2007-07-31','1980-01-14 19:50:45'),(66,'Jackie','McCullough','brianne74@example.net','1995-12-21','2004-01-15 10:43:08'),(67,'Norma','Dare','cecelia.mann@example.org','2003-06-27','2019-05-31 10:08:00'),(68,'Hunter','Witting','sasha71@example.net','1971-02-10','2007-10-04 06:34:12'),(69,'Izaiah','Hilpert','kunze.patricia@example.org','2015-01-19','1986-12-16 15:38:47'),(70,'Kelsie','Herman','pollich.jammie@example.org','2015-01-02','2010-03-15 20:48:55'),(71,'Maia','Haag','roxane.kohler@example.net','2011-07-28','2001-07-15 23:06:58'),(72,'Grayson','Grimes','leora.kozey@example.net','2002-10-26','2016-05-01 21:39:01'),(73,'Loma','Morissette','mariana31@example.org','1995-01-31','1972-09-30 08:02:09'),(74,'Logan','Von','russel.adriana@example.com','2015-05-29','1984-01-22 22:05:42'),(75,'Clinton','Beer','eusebio34@example.org','1975-04-08','1990-10-01 16:11:41'),(76,'Clotilde','Johnson','nquitzon@example.net','2009-12-20','2007-11-29 08:03:49'),(77,'Tevin','Friesen','marvin.rosamond@example.net','2014-04-03','1982-09-27 21:35:21'),(78,'Davon','Tromp','lia.botsford@example.net','1977-06-03','2008-01-28 21:52:30'),(79,'Robyn','Walsh','hernser@example.org','1994-08-15','1994-04-10 20:14:53'),(80,'Alyson','Yost','lreichel@example.org','1971-09-15','1995-07-29 21:06:36'),(81,'Chaz','Abbott','evan.moore@example.com','1997-05-13','1996-01-20 13:05:10'),(82,'Rosendo','Howe','gerlach.cicero@example.net','2001-03-19','2016-06-15 20:14:35'),(83,'Turner','Mertz','rippin.chloe@example.com','2000-12-31','1970-06-10 23:51:52'),(84,'Kitty','Witting','abigail09@example.net','1984-05-26','2008-09-04 15:08:06'),(85,'Rusty','Schroeder','ukiehn@example.org','2000-03-15','2010-08-11 16:34:45'),(86,'Lauren','Stamm','lowe.clementine@example.com','1995-09-15','1997-11-02 22:09:36'),(87,'Adolf','Rippin','joany.huel@example.org','1995-08-11','2012-11-21 20:08:59'),(88,'Robin','Block','lowe.maida@example.org','1985-03-27','1992-08-03 18:08:44'),(89,'Delta','Eichmann','tjohnson@example.org','2004-03-28','1982-05-13 22:12:14'),(90,'Lucy','Franecki','jgorczany@example.net','1974-10-09','2003-08-14 08:18:12'),(91,'Lucio','Sauer','helga22@example.com','2019-06-22','1997-10-30 08:43:29'),(92,'Jennifer','Moen','vfranecki@example.org','1982-12-28','2009-10-03 23:21:33'),(93,'Agnes','Stoltenberg','abshire.lura@example.com','1970-12-17','2003-04-08 01:28:05'),(94,'Adrian','Abernathy','iwaelchi@example.com','1986-08-07','2000-09-14 08:21:33'),(95,'Jalen','Wehner','hertha11@example.com','1999-01-17','1993-12-05 16:44:28'),(96,'Federico','DuBuque','ryan.freeda@example.net','2006-10-26','1990-08-03 20:44:58'),(97,'Kenna','Goodwin','jacobi.mervin@example.net','1984-10-18','1972-05-23 13:42:59'),(98,'Declan','Gislason','arch73@example.net','2019-05-31','2005-04-21 08:04:44'),(99,'Gaetano','Davis','bins.adele@example.net','1993-10-05','1984-01-13 06:24:51'),(100,'Tobin','Prohaska','kemard@example.net','2008-07-09','1993-01-09 08:10:38'); 53 | /*!40000 ALTER TABLE `authors` ENABLE KEYS */; 54 | UNLOCK TABLES; 55 | -------------------------------------------------------------------------------- /samples/postgres.sql: -------------------------------------------------------------------------------- 1 | SET statement_timeout = 0; 2 | SET lock_timeout = 0; 3 | SET idle_in_transaction_session_timeout = 0; 4 | SET client_encoding = 'UTF8'; 5 | SET standard_conforming_strings = on; 6 | SET check_function_bodies = false; 7 | SET client_min_messages = warning; 8 | SET row_security = off; 9 | 10 | 11 | 12 | \connect test 13 | 14 | SET statement_timeout = 0; 15 | SET lock_timeout = 0; 16 | SET idle_in_transaction_session_timeout = 0; 17 | SET client_encoding = 'UTF8'; 18 | SET standard_conforming_strings = on; 19 | SET check_function_bodies = false; 20 | SET client_min_messages = warning; 21 | SET row_security = off; 22 | 23 | SET default_tablespace = ''; 24 | 25 | SET default_with_oids = false; 26 | 27 | --- 28 | -- Name: authors; Type: TABLE; Schema: public; Owner: - 29 | -- 30 | 31 | CREATE TABLE public.authors ( 32 | id integer NOT NULL, 33 | first_name character varying(100) NOT NULL, 34 | last_name character varying(100) NOT NULL, 35 | email character varying(100), 36 | created timestamp without time zone DEFAULT now() NOT NULL 37 | ); 38 | 39 | 40 | -- 41 | -- Name: authors_id_seq; Type: SEQUENCE; Schema: public; Owner: - 42 | -- 43 | 44 | CREATE SEQUENCE public.authors_id_seq 45 | AS integer 46 | START WITH 1 47 | INCREMENT BY 1 48 | NO MINVALUE 49 | NO MAXVALUE 50 | CACHE 1; 51 | 52 | 53 | -- 54 | -- Name: authors_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - 55 | -- 56 | 57 | ALTER SEQUENCE public.authors_id_seq OWNED BY public.authors.id; 58 | 59 | 60 | -- 61 | -- Name: posts; Type: TABLE; Schema: public; Owner: - 62 | -- 63 | 64 | CREATE TABLE public.posts ( 65 | id integer NOT NULL, 66 | author_id integer NOT NULL, 67 | title character varying(500) NOT NULL, 68 | intro text, 69 | created timestamp without time zone DEFAULT now() NOT NULL 70 | ); 71 | 72 | 73 | -- 74 | -- Name: posts_id_seq; Type: SEQUENCE; Schema: public; Owner: - 75 | -- 76 | 77 | CREATE SEQUENCE public.posts_id_seq 78 | AS integer 79 | START WITH 1 80 | INCREMENT BY 1 81 | NO MINVALUE 82 | NO MAXVALUE 83 | CACHE 1; 84 | 85 | 86 | -- 87 | -- Name: posts_id_seq; Type: SEQUENCE OWNED BY; Schema: public; Owner: - 88 | -- 89 | 90 | ALTER SEQUENCE public.posts_id_seq OWNED BY public.posts.id; 91 | 92 | 93 | -- 94 | -- Name: authors id; Type: DEFAULT; Schema: public; Owner: - 95 | -- 96 | 97 | ALTER TABLE ONLY public.authors ALTER COLUMN id SET DEFAULT nextval('public.authors_id_seq'::regclass); 98 | 99 | 100 | -- 101 | -- Name: posts id; Type: DEFAULT; Schema: public; Owner: - 102 | -- 103 | 104 | ALTER TABLE ONLY public.posts ALTER COLUMN id SET DEFAULT nextval('public.posts_id_seq'::regclass); 105 | 106 | 107 | -- 108 | -- Data for Name: authors; Type: TABLE DATA; Schema: public; Owner: - 109 | -- 110 | 111 | COPY public.authors (id, first_name, last_name, email, created) FROM stdin; 112 | 1 test test te@asd.ru 2019-06-22 13:26:37.078767 113 | 2 vanya ivanov \N 2019-06-22 14:45:54.81458 114 | \. 115 | 116 | 117 | -- 118 | -- Data for Name: posts; Type: TABLE DATA; Schema: public; Owner: - 119 | -- 120 | 121 | COPY public.posts (id, author_id, title, intro, created) FROM stdin; 122 | \. 123 | 124 | 125 | -- 126 | -- Name: authors_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - 127 | -- 128 | 129 | SELECT pg_catalog.setval('public.authors_id_seq', 2, true); 130 | 131 | 132 | -- 133 | -- Name: posts_id_seq; Type: SEQUENCE SET; Schema: public; Owner: - 134 | -- 135 | 136 | SELECT pg_catalog.setval('public.posts_id_seq', 1, false); 137 | 138 | 139 | -- 140 | -- Name: authors authors_pk; Type: CONSTRAINT; Schema: public; Owner: - 141 | -- 142 | 143 | ALTER TABLE ONLY public.authors 144 | ADD CONSTRAINT authors_pk PRIMARY KEY (id); 145 | 146 | 147 | -- 148 | -- Name: posts posts_pk; Type: CONSTRAINT; Schema: public; Owner: - 149 | -- 150 | 151 | ALTER TABLE ONLY public.posts 152 | ADD CONSTRAINT posts_pk PRIMARY KEY (id); 153 | 154 | 155 | -- 156 | -- Name: authors_id_uindex; Type: INDEX; Schema: public; Owner: - 157 | -- 158 | 159 | CREATE UNIQUE INDEX authors_id_uindex ON public.authors USING btree (id); 160 | 161 | 162 | -- 163 | -- Name: posts_id_uindex; Type: INDEX; Schema: public; Owner: - 164 | -- 165 | 166 | CREATE UNIQUE INDEX posts_id_uindex ON public.posts USING btree (id); 167 | 168 | 169 | -- 170 | -- PostgreSQL database dump complete 171 | -- 172 | 173 | --------------------------------------------------------------------------------