├── .editorconfig ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── command.go ├── command_deploy.go ├── command_list.go ├── command_selfupdate.go ├── command_sync.go ├── command_version.go ├── examples ├── gosync.simple.yml └── gosync.yml ├── go.mod ├── go.sum ├── logger └── logger.go ├── main.go └── sync ├── config.go ├── configparser.go ├── database.go ├── database_mysql.go ├── database_mysql_local.go ├── database_mysql_remote.go ├── database_postgres.go ├── database_postgres_local.go ├── database_postgres_remote.go ├── execution.go ├── filesystem.go ├── filter.go ├── global.go ├── helper.go ├── server.go ├── server_deploy.go ├── server_deploy_database.go ├── server_deploy_filesystem.go ├── server_exec.go ├── server_sync.go ├── server_sync_database.go ├── server_sync_filesystem.go ├── server_sync_filesystem_stubs.go ├── yaml_commandbuilder_argument.go ├── yaml_commandbuilder_connection.go └── yaml_string_array.go /.editorconfig: -------------------------------------------------------------------------------- 1 | # EditorConfig is awesome: http://EditorConfig.org 2 | 3 | # top-most EditorConfig file 4 | root = true 5 | charset = utf-8 6 | trim_trailing_whitespace = true 7 | 8 | [*] 9 | end_of_line = lf 10 | insert_final_newline = true 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [Makefile] 15 | indent_style = tab 16 | 17 | [*.yml] 18 | indent_size = 2 19 | 20 | [*.conf] 21 | indent_size = 2 22 | 23 | [*.t] 24 | indent_size = 2 25 | trim_trailing_whitespace = false 26 | 27 | [*.go] 28 | indent_style = tab 29 | indent_size = 4 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /gosync 2 | /build/ 3 | /.idea 4 | /gosync.yml 5 | /vendor 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015-2016 WebDevOps 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SOURCE = $(wildcard *.go) 2 | TAG ?= $(shell git describe --tags) 3 | GOBUILD = go build -ldflags '-w' 4 | 5 | ALL = \ 6 | $(foreach arch,x64 x32,\ 7 | $(foreach suffix,linux osx windows,\ 8 | build/gosync-$(suffix)-$(arch))) \ 9 | $(foreach arch,arm arm64,\ 10 | build/gosync-linux-$(arch)) 11 | 12 | all: test build 13 | 14 | build: clean module test $(ALL) 15 | 16 | # cram is a python app, so 'easy_install/pip install cram' to run tests 17 | test: 18 | echo "No tests" 19 | #cram tests/*.test 20 | 21 | clean: 22 | rm -f $(ALL) 23 | 24 | module: 25 | go mod vendor 26 | 27 | # os is determined as thus: if variable of suffix exists, it's taken, if not, then 28 | # suffix itself is taken 29 | osx = darwin 30 | build/gosync-%-x64: $(SOURCE) 31 | @mkdir -p $(@D) 32 | CGO_ENABLED=0 GOOS=$(firstword $($*) $*) GOARCH=amd64 $(GOBUILD) -o $@ 33 | 34 | build/gosync-%-x32: $(SOURCE) 35 | @mkdir -p $(@D) 36 | CGO_ENABLED=0 GOOS=$(firstword $($*) $*) GOARCH=386 $(GOBUILD) -o $@ 37 | 38 | build/gosync-linux-arm: $(SOURCE) 39 | @mkdir -p $(@D) 40 | CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 $(GOBUILD) -o $@ 41 | 42 | build/gosync-linux-arm64: $(SOURCE) 43 | @mkdir -p $(@D) 44 | CGO_ENABLED=0 GOOS=linux GOARCH=arm64 $(GOBUILD) -o $@ 45 | 46 | release: build 47 | github-release release -u webdevops -r go-sync -t "$(TAG)" -n "$(TAG)" --description "$(TAG)" 48 | @for x in $(ALL); do \ 49 | echo "Uploading $$x" && \ 50 | github-release upload -u webdevops \ 51 | -r go-sync \ 52 | -t $(TAG) \ 53 | -f "$$x" \ 54 | -n "$$(basename $$x)"; \ 55 | done 56 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # go-sync utility 2 | 3 | [![GitHub release](https://img.shields.io/github/release/webdevops/go-sync.svg)](https://github.com/webdevops/go-sync/releases) 4 | [![license](https://img.shields.io/github/license/webdevops/go-sync.svg)](https://github.com/webdevops/go-sync/blob/master/LICENSE) 5 | [![Build Status](https://travis-ci.org/webdevops/go-sync.svg?branch=master)](https://travis-ci.org/webdevops/go-sync) 6 | [![Github All Releases](https://img.shields.io/github/downloads/webdevops/go-sync/total.svg)]() 7 | [![Github Releases](https://img.shields.io/github/downloads/webdevops/go-sync/latest/total.svg)]() 8 | 9 | Easy project file and database synchronization for developers 10 | 11 | Successor for [CliTools Sync](https://github.com/webdevops/clitools) written on Golang 12 | 13 | Features 14 | ======== 15 | 16 | General: 17 | - Yaml based configuration files (`gosync.yml` or `.gosync.yml`) 18 | - PostgreSQL and MySQL support 19 | - SSH and Docker support (with docker-compose support) 20 | 21 | Sync: 22 | - Filesync (rsync) from remote servers using SSH 23 | - Create file stubs instead of fetching files from remote (with real images, see ``options.generate-stubs = true``) 24 | - Dump databases from remote servers using SSH, Docker and SSH+Docker 25 | - Restore databases to local database servers or Docker/Docker-Compose containers 26 | - Filtering databases tables with regexp 27 | - Rsync filters 28 | - Custom exec scripts (startup/finish) on local or remote machine (using SSH) 29 | 30 | Deployment: 31 | - Filesync (rsync) from local to remote servers using SSH 32 | - Dump databases from local database servers or Docker/Docker-Compose containers 33 | - Filtering databases tables with regexp 34 | - Rsync filters 35 | - Custom exec scripts (startup/finish) on local or remote machine (using SSH) 36 | 37 | Install 38 | ======= 39 | 40 | The binary file can be found in the [project releases](https://github.com/webdevops/go-sync/releases). 41 | 42 | ``` 43 | DOWNLOAD_VERSION=0.5.5 44 | DOWNLOAD_OS=linux 45 | DOWNLOAD_ARCH=x64 46 | 47 | wget -O/usr/local/bin/gosync "https://github.com/webdevops/go-sync/releases/download/${DOWNLOAD_VERSION}/gosync-${DOWNLOAD_OS}-${DOWNLOAD_ARCH}" 48 | chmod +x /usr/local/bin/gosync 49 | ``` 50 | 51 | Help 52 | ==== 53 | 54 | ``` 55 | Usage: 56 | gosync [OPTIONS] 57 | 58 | Application Options: 59 | -v, --verbose verbose mode 60 | 61 | Help Options: 62 | -h, --help Show this help message 63 | 64 | Available commands: 65 | deploy Deploy to server 66 | list List server configurations 67 | self-update Self update 68 | sync Sync from server 69 | version Show version 70 | 71 | ``` 72 | 73 | Configuration (gosync.yml) 74 | ========================== 75 | 76 | Gosync is controlled by `gosync.yml` (or `.gosync.yml`) which will be 77 | searched in current and parent directories. 78 | 79 | * [Simple gosync.yml](examples/gosync.simple.yml) 80 | * [Full gosync.yml](examples/gosync.yml) 81 | 82 | Example 83 | ======= 84 | 85 | ``` 86 | > gosync sync production 87 | 88 | :: Initialisation 89 | -> found configuration file /Users/xxxxxx/Projects/examples/gosync.yml 90 | -> using production server 91 | -> using connection Exec[Type:ssh SSH:ssh-user@example.com] 92 | :: Starting exec mode "startup" 93 | -> executing >> Exec[Type:local Command:date +%s] 94 | -> executing >> Exec[Type:local Command:date +%s] 95 | -> executing >> Exec[Type:local Workdir:/ Command:date] 96 | -> executing >> Exec[Type:local Workdir:/ Command:date] 97 | -> executing >> Exec[Type:remote Workdir:/ Command:date] 98 | :: Starting sync of Filesystem[Path:/home/xxxxxx/application1/ -> Local:./application1/] 99 | :: Starting sync of Filesystem[Path:/home/xxxxxx/application2/ -> Local:./application2/] 100 | :: Starting sync of Database[Schema:application1 User:mysql-user Passwd:***** -> Schema:test-local] 101 | -> dropping local database "test-application1" 102 | -> creating local database "test-application1" 103 | -> syncing database structure 104 | -> get list of mysql tables for table filter 105 | -> syncing database data 106 | :: Starting sync of Database[Schema:application2 User:mysql-user Passwd:***** -> Schema:test] 107 | -> dropping local database "test-application2" 108 | -> creating local database "test-application2" 109 | -> syncing database structure 110 | -> get list of mysql tables for table filter 111 | -> syncing database data 112 | :: Starting exec mode "finish" 113 | -> executing >> Exec[Type:remote Workdir:/ Command:date] 114 | -> finished 115 | ``` 116 | 117 | ## Docker support 118 | 119 | Docker support 120 | ============== 121 | 122 | Using the configuration ``connection.docker=configuration`` this command can be 123 | execued with docker containers. If the container id is passed the 124 | container is used without lookup using eg. `docker-compose`. 125 | 126 | **docker-compose:** 127 | 128 | *CONTAINER* is the name of the docker-compose container. 129 | 130 | | DSN style configuration | Description | 131 | |:--------------------------------------------------------------------|:------------------------------------------------------------------------------------------------| 132 | | ``compose:CONTAINER`` | Use container with docker-compose in current directory | 133 | | ``compose:CONTAINER;path=/path/to/project`` | Use container with docker-compose in `/path/to/project` directory | 134 | | ``compose:CONTAINER;path=/path/to/project;file=custom-compose.yml`` | Use container with docker-compose in `/path/to/project` directory and `custom-compose.yml` file | 135 | | ``compose:CONTAINER;project-name=foobar`` | Use container with docker-compose in current directory with project name `foobar` | 136 | | ``compose:CONTAINER;host=example.com`` | Use container with docker-compose in current directory with docker host `example.com` | 137 | | ``compose:CONTAINER;env[FOOBAR]=BARFOO`` | Use container with docker-compose in current directory with env var `FOOBAR` set to `BARFOO` | 138 | 139 | | Query style configuration | Description | 140 | |:----------------------------------------------------------------------|:------------------------------------------------------------------------------------------------| 141 | | ``compose://CONTAINER`` | Use container with docker-compose in current directory | 142 | | ``compose://CONTAINER?path=/path/to/project`` | Use container with docker-compose in `/path/to/project` directory | 143 | | ``compose://CONTAINER?path=/path/to/project&file=custom-compose.yml`` | Use container with docker-compose in `/path/to/project` directory and `custom-compose.yml` file | 144 | | ``compose://CONTAINER?project-name=foobar`` | Use container with docker-compose in current directory with project name `foobar` | 145 | | ``compose://CONTAINER?host=example.com`` | Use container with docker-compose in current directory with docker host `example.com` | 146 | | ``compose://CONTAINER?env[FOOBAR]=BARFOO`` | Use container with docker-compose in current directory with env var `FOOBAR` set to `BARFOO` | 147 | -------------------------------------------------------------------------------- /command.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "strings" 5 | "os" 6 | "fmt" 7 | "path" 8 | "github.com/webdevops/go-sync/sync" 9 | "gopkg.in/AlecAivazis/survey.v1" 10 | ) 11 | 12 | type AbstractCommand struct { 13 | 14 | } 15 | 16 | func (command *AbstractCommand) GetConfig() *sync.SyncConfig { 17 | Logger.Main("Initialisation") 18 | configFile := command.findConfigFile() 19 | if configFile == "" { 20 | Logger.FatalExit(2, "Unable to find configuration file (searched %s)", strings.Join(validConfigFiles, " ")) 21 | } 22 | Logger.Step("found configuration file %s", configFile) 23 | 24 | sync.Logger = Logger 25 | config := sync.NewConfigParser(configFile) 26 | 27 | return config 28 | } 29 | 30 | func (command *AbstractCommand) findConfigFile() string { 31 | pwd, err := os.Getwd() 32 | if err != nil { 33 | Logger.FatalErrorExit(1, err) 34 | fmt.Println(err) 35 | } 36 | 37 | for true { 38 | for _, filename := range validConfigFiles { 39 | filepath := path.Join(pwd, filename) 40 | if sync.FileExists(filepath) { 41 | return filepath 42 | } 43 | } 44 | 45 | 46 | // already found root, we finished here 47 | if pwd == "/" { 48 | break 49 | } 50 | 51 | pwd = path.Dir(pwd) 52 | 53 | // oh, path seems to be empty.. stopping here now 54 | if pwd == "." || pwd == "" { 55 | break 56 | } 57 | } 58 | 59 | return "" 60 | } 61 | 62 | 63 | func (command *AbstractCommand) getServerSelectionFromUser(config *sync.SyncConfig, confType string, userSelection string) string { 64 | if userSelection == "" { 65 | prompt := &survey.Select{ 66 | Message: "Choose configuration:", 67 | Options: config.GetServerList(confType), 68 | } 69 | survey.AskOne(prompt, &userSelection, nil) 70 | } 71 | 72 | return userSelection 73 | } 74 | -------------------------------------------------------------------------------- /command_deploy.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/webdevops/go-sync/sync" 6 | ) 7 | 8 | type DeployCommand struct { 9 | AbstractCommand 10 | Positional struct { 11 | Server string `description:"server configuration key"` 12 | } `positional-args:"true"` 13 | Dump bool `long:"dump" description:"dump configuration as yaml"` 14 | OnlyFilesystem bool `long:"filesystem" description:"deploy only filesystem"` 15 | OnlyDatabase bool `long:"database" description:"deploy only database"` 16 | SkipExec bool `long:"skip-exec" description:"skip execution"` 17 | } 18 | 19 | // Run deployment command 20 | func (command *DeployCommand) Execute(args []string) error { 21 | config := command.GetConfig() 22 | server := command.getServerSelectionFromUser(config, "deploy", command.Positional.Server) 23 | confServer, err := config.GetDeployServer(server) 24 | if err != nil { 25 | Logger.FatalErrorExit(3, err) 26 | } 27 | Logger.Step("using Server[%s]", server) 28 | Logger.Step("using %s", confServer.Connection.GetInstance().String()) 29 | 30 | confServer.SetRunConfiguration(command.buildSyncRunConfig()) 31 | 32 | // --dump 33 | if command.Dump { 34 | fmt.Println() 35 | fmt.Println(confServer.AsYaml()) 36 | } else { 37 | confServer.Deploy() 38 | Logger.Println("-> finished") 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func (command *DeployCommand) buildSyncRunConfig() (conf sync.RunConfiguration) { 45 | // Init 46 | conf.Exec = true 47 | conf.Database = true 48 | conf.Filesystem = true 49 | 50 | if command.OnlyFilesystem { 51 | conf.Database = false 52 | conf.Filesystem = true 53 | } 54 | 55 | if command.OnlyDatabase { 56 | conf.Database = true 57 | conf.Filesystem = false 58 | } 59 | 60 | if command.SkipExec { 61 | conf.Exec = false 62 | } 63 | 64 | return 65 | } 66 | -------------------------------------------------------------------------------- /command_list.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | type ListCommand struct { 4 | AbstractCommand 5 | } 6 | 7 | // List all possible server configurations (for sync and deploy) 8 | func (command *ListCommand) Execute(args []string) error { 9 | config := command.GetConfig() 10 | config.ShowConfiguration() 11 | return nil 12 | } 13 | -------------------------------------------------------------------------------- /command_selfupdate.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "context" 6 | "log" 7 | "runtime" 8 | "strings" 9 | "github.com/google/go-github/github" 10 | "github.com/inconshreveable/go-update" 11 | "net/http" 12 | ) 13 | 14 | type SelfUpdateCommand struct { 15 | CurrentVersion string 16 | GithubOrganization string 17 | GithubRepository string 18 | GithubAssetTemplate string 19 | Force bool `long:"force" description:"force update"` 20 | } 21 | 22 | var ( 23 | selfUpdateOsTranslationMap = map[string]string{ 24 | "darwin": "osx", 25 | } 26 | selfUpdateArchTranslationMap = map[string]string{ 27 | "amd64": "x64", 28 | "386": "x32", 29 | } 30 | ) 31 | 32 | func (conf *SelfUpdateCommand) Execute(args []string) error { 33 | fmt.Println("Starting self update") 34 | 35 | client := github.NewClient(nil) 36 | release, _, err := client.Repositories.GetLatestRelease(context.Background(), conf.GithubOrganization, conf.GithubRepository) 37 | 38 | if _, ok := err.(*github.RateLimitError); ok { 39 | log.Println("GitHub rate limit, please try again later") 40 | } 41 | 42 | fmt.Println(fmt.Sprintf(" - latest version is %s", release.GetName())) 43 | 44 | // check if latest version is current version 45 | if !conf.Force && release.GetName() == conf.CurrentVersion { 46 | fmt.Println(" - already using the latest version") 47 | return nil 48 | } 49 | 50 | // translate OS names 51 | os := runtime.GOOS 52 | if val, ok := selfUpdateOsTranslationMap[os]; ok { 53 | os = val 54 | } 55 | 56 | // translate arch names 57 | arch := runtime.GOARCH 58 | if val, ok := selfUpdateArchTranslationMap[arch]; ok { 59 | arch = val 60 | } 61 | 62 | // build asset name 63 | assetName := conf.GithubAssetTemplate 64 | assetName = strings.Replace(assetName, "%OS%", os, -1) 65 | assetName = strings.Replace(assetName, "%ARCH%", arch, -1) 66 | 67 | // search assets in release for the desired filename 68 | fmt.Println(fmt.Sprintf(" - searching for asset \"%s\"", assetName)) 69 | for _, asset := range release.Assets { 70 | if asset.GetName() == assetName { 71 | downloadUrl := asset.GetBrowserDownloadURL() 72 | fmt.Println(fmt.Sprintf(" - found new update url \"%s\"", downloadUrl)) 73 | conf.runUpdate(downloadUrl) 74 | fmt.Println(fmt.Sprintf(" - finished update to version %s", release.GetName())) 75 | return nil 76 | } 77 | } 78 | 79 | fmt.Println(" - unable to find asset, please contact maintainer") 80 | return nil 81 | } 82 | 83 | func (conf *SelfUpdateCommand) runUpdate(url string) error { 84 | fmt.Println(" - downloading update") 85 | resp, err := http.Get(url) 86 | if err != nil { 87 | return err 88 | } 89 | defer resp.Body.Close() 90 | fmt.Println(" - applying update") 91 | err = update.Apply(resp.Body, update.Options{}) 92 | if err != nil { 93 | // error handling 94 | fmt.Println(fmt.Sprintf(" - updating application failed: %s", err)) 95 | } 96 | return err 97 | } 98 | -------------------------------------------------------------------------------- /command_sync.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/webdevops/go-sync/sync" 6 | ) 7 | 8 | type SyncCommand struct { 9 | AbstractCommand 10 | Positional struct { 11 | Server string `description:"server configuration key"` 12 | } `positional-args:"true"` 13 | Dump bool `long:"dump" description:"dump configuration as yaml"` 14 | OnlyFilesystem bool `long:"filesystem" description:"sync only filesystem"` 15 | OnlyDatabase bool `long:"database" description:"sync only database"` 16 | SkipExec bool `long:"skip-exec" description:"skip execution"` 17 | } 18 | 19 | // Run sync command 20 | func (command *SyncCommand) Execute(args []string) error { 21 | config := command.GetConfig() 22 | server := command.getServerSelectionFromUser(config, "sync", command.Positional.Server) 23 | confServer, err := config.GetSyncServer(server) 24 | if err != nil { 25 | Logger.FatalErrorExit(3, err) 26 | } 27 | Logger.Step("using Server[%s]", server) 28 | Logger.Step("using %s", confServer.Connection.GetInstance().String()) 29 | 30 | confServer.SetRunConfiguration(command.buildSyncRunConfig()) 31 | 32 | // --dump 33 | if command.Dump { 34 | fmt.Println() 35 | fmt.Println(confServer.AsYaml()) 36 | } else { 37 | confServer.Sync() 38 | Logger.Println("-> finished") 39 | } 40 | 41 | return nil 42 | } 43 | 44 | func (command *SyncCommand) buildSyncRunConfig() (conf sync.RunConfiguration) { 45 | // Init 46 | conf.Exec = true 47 | conf.Database = true 48 | conf.Filesystem = true 49 | 50 | if command.OnlyFilesystem { 51 | conf.Database = false 52 | conf.Filesystem = true 53 | } 54 | 55 | if command.OnlyDatabase { 56 | conf.Database = true 57 | conf.Filesystem = false 58 | } 59 | 60 | if command.SkipExec { 61 | conf.Exec = false 62 | } 63 | 64 | return 65 | } 66 | -------------------------------------------------------------------------------- /command_version.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | ) 6 | 7 | type VersionCommand struct { 8 | ShowOnlyVersion bool `long:"dump" description:"show only version number and exit"` 9 | Name string 10 | Version string 11 | Author string 12 | } 13 | 14 | // Show app version 15 | func (conf *VersionCommand) Execute(args []string) error { 16 | if conf.ShowOnlyVersion { 17 | fmt.Println(conf.Version) 18 | } else { 19 | fmt.Println(fmt.Sprintf("%s version %s", conf.Name, conf.Version)) 20 | fmt.Println(fmt.Sprintf("Copyright (C) 2017 %s", conf.Author)) 21 | } 22 | 23 | return nil 24 | } 25 | -------------------------------------------------------------------------------- /examples/gosync.simple.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | sync: 4 | production: 5 | path: "./local/path/" 6 | 7 | connection: 8 | ssh: user@example.com 9 | 10 | filesystem: 11 | - path: /path/to/project/on/example.com/ 12 | filter: 13 | exclude: 14 | - .git 15 | - node_modules/ 16 | options: 17 | generate-stubs: true 18 | 19 | database: 20 | - type: mysql 21 | database: app 22 | user: mysql-user 23 | password: mysql-password 24 | filter: 25 | exclude: 26 | include: 27 | local: 28 | database: app 29 | connection: 30 | docker: compose:mysql 31 | options: 32 | clear-database: true 33 | -------------------------------------------------------------------------------- /examples/gosync.yml: -------------------------------------------------------------------------------- 1 | --- 2 | 3 | ######################### 4 | ## Sync 5 | ######################### 6 | sync: 7 | production: &production 8 | 9 | ## Base path for sync 10 | path: "./test/" 11 | 12 | connection: 13 | ssh: user@example.com 14 | 15 | ######################### 16 | ## Filesystem rsync 17 | ######################### 18 | filesystem: 19 | ## Sync /tmp/remote/path to ./test/ (defined before) 20 | - path: /tmp/remote/path 21 | filter: 22 | exclude: 23 | - .git 24 | - node_modules/ 25 | 26 | ## Sync /tmp/remote/other/path to ./other/ 27 | - path: /tmp/remote/other/path 28 | local: ./other/ 29 | filter: 30 | exclude: 31 | - .git 32 | - node_modules/ 33 | 34 | ## Create file stubs for /tmp/remote/other/path-big-files to ./other-stubs/ 35 | - path: /tmp/remote/other/path-big-files 36 | local: ./other-stubs/ 37 | options: 38 | generate-stubs: true 39 | rsync: 40 | - "--iconv=UTF8-MAC,UTF8" 41 | 42 | ## Sync /tmp/remote/other/path-other-files to ./other2/ with custom rsync options 43 | - path: /tmp/remote/other/path-other-files 44 | local: ./other2/ 45 | options: 46 | rsync: 47 | - "--iconv=UTF8-MAC,UTF8" 48 | 49 | ######################### 50 | ## Database 51 | ######################### 52 | database: 53 | 54 | ## NORMAL MYSQL 55 | ## Sync database typo3 to local test-local (without cache tables) 56 | ## 57 | ## remote: connect via ssh and get dump from local installed mysql 58 | ## local: connect via mysql to a database server "mysql.example.com" 59 | - type: mysql 60 | database: application1 61 | user: mysql-user 62 | password: mysql-password 63 | filter: 64 | exclude: 65 | - "^cache_.*" 66 | include: 67 | local: 68 | ## use plain mysql connection 69 | database: test-application1 70 | user: root 71 | password: dev 72 | port: 13306 73 | hostname: mysql.example.com 74 | options: 75 | # custom mysqldump options (for local) 76 | mysqldump: 77 | # custom mysql options (for local) 78 | mysql: 79 | options: 80 | ## clear database before restore 81 | ## (delete all tables inside) 82 | clear-database: true 83 | # custom mysqldump options (for remote) 84 | mysqldump: 85 | # custom mysql options (for remote) 86 | mysql: 87 | 88 | ## DOCKER 89 | ## Sync database typo3 to local test-local into a docker container 90 | ## 91 | ## remote: connect via ssh and get dump from local installed mysql 92 | ## local: use docker container "2713f9093bed" for restore 93 | - type: mysql 94 | database: application2 95 | user: mysql-user 96 | password: mysql-password 97 | filter: 98 | exclude: 99 | include: 100 | local: 101 | database: test-application2 102 | user: root 103 | password: dev 104 | connection: 105 | docker: 2713f9093bed 106 | options: 107 | ## clear database before restore 108 | ## (delete all tables inside) 109 | clear-database: true 110 | 111 | ## DOCKER-COMPOSE 112 | ## Sync database typo3 to local test-local into a docker-compose container 113 | ## 114 | ## docker-compose.yml has to be in the same or parent directory! 115 | ## 116 | ## remote: connect via ssh and get dump from local installed mysql 117 | ## local: use docker-compose container "mysql" for restore 118 | - type: mysql 119 | database: application3 120 | user: mysql-user 121 | password: mysql-password 122 | filter: 123 | exclude: 124 | include: 125 | local: 126 | database: test-application3 127 | user: root 128 | password: dev 129 | connection: 130 | docker: compose:mysql 131 | options: 132 | ## clear database before restore 133 | ## (delete all tables inside) 134 | clear-database: true 135 | 136 | ######################### 137 | ## Execute 138 | ######################### 139 | exec-startup: 140 | ## Execute date on local system 141 | - type: local 142 | command: date +%s 143 | 144 | ## Execute date on local system (alternative style) 145 | - type: local 146 | command: 147 | - date 148 | - +%s 149 | 150 | ## Execute date with workdir 151 | - type: local 152 | command: 153 | - date 154 | workdir: / 155 | 156 | ## Execute remote date with workdir 157 | - type: remote 158 | command: date 159 | workdir: / 160 | 161 | 162 | ## Execute remote date with workdir and custom environment settings 163 | - type: remote 164 | command: date 165 | workdir: / 166 | environment: 167 | - name: FOOBAR 168 | value: barfoo 169 | 170 | exec-finish: 171 | ## Execute remote date with workdir 172 | - type: remote 173 | command: date 174 | workdir: / 175 | 176 | ######################### 177 | ## Deploy 178 | ######################### 179 | deploy: 180 | production: &foobar 181 | path: "./test/" 182 | 183 | connection: 184 | ssh: user@example.com 185 | 186 | filesystem: 187 | - path: /tmp/path 188 | filter: 189 | exclude: 190 | - .git 191 | - node_modules/ 192 | 193 | - path: /tmp/other/path 194 | local: ./other/ 195 | filter: 196 | exclude: 197 | - .git 198 | - node_modules/ 199 | 200 | database: 201 | - type: mysql 202 | database: foobar 203 | user: mysql-user 204 | password: mysql-password 205 | filter: 206 | exclude: 207 | include: 208 | local: 209 | database: test 210 | user: root 211 | password: dev 212 | connection: 213 | docker: compose:mysql 214 | ssh: foo@example.com 215 | options: 216 | clear-database: true 217 | 218 | production2: 219 | <<: *foobar 220 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/webdevops/go-sync 2 | 3 | go 1.13 4 | 5 | require ( 6 | github.com/google/go-github v17.0.0+incompatible 7 | github.com/google/go-querystring v1.0.0 // indirect 8 | github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf 9 | github.com/jessevdk/go-flags v1.4.0 10 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 11 | github.com/remeh/sizedwaitgroup v1.0.0 12 | github.com/webdevops/go-shell v0.0.0-20171102182145-aac7e8b74cc8 13 | github.com/webdevops/go-stubfilegenerator v0.0.0-20171011222655-fb60bac44836 14 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 // indirect 15 | gopkg.in/AlecAivazis/survey.v1 v1.8.7 16 | gopkg.in/yaml.v2 v2.2.5 17 | ) 18 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= 4 | github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= 5 | github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= 6 | github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= 7 | github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= 8 | github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf h1:WfD7VjIE6z8dIvMsI4/s+1qr5EL+zoIGev1BQj1eoJ8= 9 | github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h1:hyb9oH7vZsitZCiBt0ZvifOrB+qc8PS5IiilCIb87rg= 10 | github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= 11 | github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= 12 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= 13 | github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= 14 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 15 | github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= 16 | github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= 17 | github.com/mattn/go-isatty v0.0.3 h1:ns/ykhmWi7G9O+8a448SecJU3nSMBXJfqQkl0upE1jI= 18 | github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= 19 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= 20 | github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= 21 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 h1:RWengNIwukTxcDr9M+97sNutRR1RKhG96O6jWumTTnw= 22 | github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826/go.mod h1:TaXosZuwdSHYgviHp1DAtfrULt5eUgsSMsZf+YrPgl8= 23 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 24 | github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= 25 | github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= 26 | github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 27 | github.com/webdevops/go-shell v0.0.0-20171102182145-aac7e8b74cc8 h1:vDoEaub4wAh8ZJYi119SZc+J2FImfyBi6kZlRQochfc= 28 | github.com/webdevops/go-shell v0.0.0-20171102182145-aac7e8b74cc8/go.mod h1:5GqiDpJsTO6JVzRG5Yd6p+5J6FDHI0rJK/jAZhAbWok= 29 | github.com/webdevops/go-stubfilegenerator v0.0.0-20171011222655-fb60bac44836 h1:ifZL1/eiqZn6Apk+ZGdLvNM1trbh5WpwQbXmNA+81Go= 30 | github.com/webdevops/go-stubfilegenerator v0.0.0-20171011222655-fb60bac44836/go.mod h1:hyhUgME78Jq40CpbOd0oeHkG6UVyGxxNJIXLXX+CF/M= 31 | golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= 32 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8 h1:hVwzHzIUGRjiF7EcUjqNxk3NCfkPxbDKRdnNE1Rpg0U= 33 | golang.org/x/image v0.0.0-20191009234506-e7c1f5e7dbb8/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= 34 | golang.org/x/sys v0.0.0-20180606202747-9527bec2660b h1:5rOiLYVqtE+JehJPVJTXQJaP8aT3cpJC1Iy22+5WLFU= 35 | golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 36 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 37 | gopkg.in/AlecAivazis/survey.v1 v1.8.7 h1:oBJqtgsyBLg9K5FK9twNUbcPnbCPoh+R9a+7nag3qJM= 38 | gopkg.in/AlecAivazis/survey.v1 v1.8.7/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= 39 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 40 | gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= 41 | gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 42 | -------------------------------------------------------------------------------- /logger/logger.go: -------------------------------------------------------------------------------- 1 | package logger 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "fmt" 7 | "strings" 8 | ) 9 | 10 | const ( 11 | LogPrefix = "" 12 | prefixMain = ":: " 13 | prefixSub = " -> " 14 | prefixCmd = " $ " 15 | prefixErr = "[ERROR] " 16 | ) 17 | 18 | type SyncLogger struct { 19 | *log.Logger 20 | } 21 | 22 | var ( 23 | Logger *SyncLogger 24 | Verbose bool 25 | CommandName string 26 | ) 27 | 28 | func GetInstance(commandName string, flags int) *SyncLogger { 29 | CommandName = commandName 30 | 31 | if Logger == nil { 32 | Logger = &SyncLogger{log.New(os.Stdout, LogPrefix, flags)} 33 | } 34 | return Logger 35 | } 36 | 37 | func (SyncLogger SyncLogger) Verbose(message string, sprintf ...interface{}) { 38 | if Verbose { 39 | if len(sprintf) > 0 { 40 | message = fmt.Sprintf(message, sprintf...) 41 | } 42 | 43 | SyncLogger.Println(message) 44 | } 45 | } 46 | 47 | func (SyncLogger SyncLogger) Main(message string, sprintf ...interface{}) { 48 | if len(sprintf) > 0 { 49 | message = fmt.Sprintf(message, sprintf...) 50 | } 51 | 52 | SyncLogger.Println(prefixMain + message) 53 | } 54 | 55 | func (SyncLogger SyncLogger) Step(message string, sprintf ...interface{}) { 56 | if len(sprintf) > 0 { 57 | message = fmt.Sprintf(message, sprintf...) 58 | } 59 | 60 | SyncLogger.Println(prefixSub + message) 61 | } 62 | 63 | 64 | func (SyncLogger SyncLogger) Command(message string) { 65 | SyncLogger.Println(prefixCmd + message) 66 | } 67 | 68 | func (SyncLogger SyncLogger) FatalExit(exitCode int, message string, sprintf ...interface{}) { 69 | if len(sprintf) > 0 { 70 | message = fmt.Sprintf(message, sprintf...) 71 | } 72 | 73 | SyncLogger.Fatal(message) 74 | os.Exit(exitCode) 75 | } 76 | 77 | 78 | // Log error object as message 79 | func (SyncLogger SyncLogger) FatalErrorExit(exitCode int, err error) { 80 | 81 | if CommandName != "" { 82 | cmdline := fmt.Sprintf("%s %s", CommandName, strings.Join(os.Args[1:], " ")) 83 | fmt.Fprintln(os.Stderr, fmt.Sprintf("Command: %s", cmdline)) 84 | } 85 | 86 | fmt.Fprintln(os.Stderr, fmt.Sprintf("%s %s", prefixErr, err)) 87 | 88 | os.Exit(exitCode) 89 | } 90 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "log" 6 | "fmt" 7 | "runtime/debug" 8 | flags "github.com/jessevdk/go-flags" 9 | "github.com/webdevops/go-shell" 10 | "github.com/webdevops/go-sync/logger" 11 | ) 12 | 13 | const ( 14 | // application informations 15 | Name = "gosync" 16 | Author = "webdevops.io" 17 | Version = "0.6.1" 18 | 19 | // self update informations 20 | GithubOrganization = "webdevops" 21 | GithubRepository = "go-sync" 22 | GithubAssetTemplate = "gosync-%OS%-%ARCH%" 23 | ) 24 | 25 | var ( 26 | Logger *logger.SyncLogger 27 | argparser *flags.Parser 28 | args []string 29 | ) 30 | 31 | var opts struct { 32 | Verbose []bool `short:"v" long:"verbose" description:"verbose mode"` 33 | } 34 | 35 | var validConfigFiles = []string{ 36 | "gosync.yml", 37 | "gosync.yaml", 38 | ".gosync.yml", 39 | ".gosync.yaml", 40 | } 41 | 42 | func handleArgParser() { 43 | var err error 44 | argparser = flags.NewParser(&opts, flags.Default) 45 | argparser.CommandHandler = func(command flags.Commander, args []string) error { 46 | switch { 47 | case len(opts.Verbose) >= 2: 48 | shell.Trace = true 49 | shell.TracePrefix = "[CMD] " 50 | Logger = logger.GetInstance(argparser.Command.Name, log.Ldate|log.Ltime|log.Lshortfile) 51 | fallthrough 52 | case len(opts.Verbose) >= 1: 53 | logger.Verbose = true 54 | shell.VerboseFunc = func(c *shell.Command) { 55 | Logger.Command(c.ToString()) 56 | } 57 | fallthrough 58 | default: 59 | if Logger == nil { 60 | Logger = logger.GetInstance(argparser.Command.Name, 0) 61 | } 62 | } 63 | 64 | return command.Execute(args) 65 | } 66 | 67 | argparser.AddCommand("version", "Show version", fmt.Sprintf("Show %s version", Name), &VersionCommand{Name:Name, Version:Version, Author:Author}) 68 | argparser.AddCommand("self-update", "Self update", "Run self update of this application", &SelfUpdateCommand{GithubOrganization:GithubOrganization, GithubRepository:GithubRepository, GithubAssetTemplate:GithubAssetTemplate, CurrentVersion:Version}) 69 | 70 | argparser.AddCommand("list", "List server configurations", "List server configurations", &ListCommand{}) 71 | argparser.AddCommand("sync", "Sync from server", "Sync filesystem and databases from server", &SyncCommand{}) 72 | argparser.AddCommand("deploy", "Deploy to server", "Deploy filesystem and databases to server", &DeployCommand{}) 73 | 74 | args, err = argparser.Parse() 75 | 76 | // check if there is an parse error 77 | if err != nil { 78 | if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { 79 | os.Exit(0) 80 | } else { 81 | fmt.Println() 82 | argparser.WriteHelp(os.Stdout) 83 | os.Exit(1) 84 | } 85 | } 86 | } 87 | 88 | func main() { 89 | defer func() { 90 | if r := recover(); r != nil { 91 | fmt.Println() 92 | fmt.Println("PANIC CATCHED") 93 | 94 | message := fmt.Sprintf("%v", r) 95 | 96 | if obj, ok := r.(*shell.Process); ok { 97 | message = obj.Debug() 98 | } 99 | 100 | if len(opts.Verbose) >= 2 { 101 | fmt.Println(message) 102 | debug.PrintStack() 103 | } else { 104 | fmt.Println(message) 105 | } 106 | os.Exit(255) 107 | } 108 | }() 109 | 110 | shell.SetDefaultShell("bash") 111 | handleArgParser() 112 | 113 | os.Exit(0) 114 | } 115 | -------------------------------------------------------------------------------- /sync/config.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "regexp" 5 | "sync" 6 | ) 7 | 8 | var waitGroup sync.WaitGroup 9 | 10 | type SyncConfig struct { 11 | // Sync (remote -> local) configurations 12 | Sync map[string]Server `yaml:"sync"` 13 | // Deploy (local -> remote) configurations 14 | Deploy map[string]Server `yaml:"deploy"` 15 | } 16 | 17 | type Server struct { 18 | // General working path (for filesystem syncs) 19 | Path string `yaml:"path"` 20 | // General connection (default for all remote connections) 21 | Connection *YamlCommandBuilderConnection `yaml:"connection"` 22 | // Filesystem sync list 23 | Filesystem []Filesystem `yaml:"filesystem"` 24 | // Database sync list 25 | Database []Database `yaml:"database"` 26 | // Startup execution list (executed before sync) 27 | ExecStartup []Execution `yaml:"exec-startup"` 28 | // Finish execution list (executed after sync) 29 | ExecFinish []Execution `yaml:"exec-finish"` 30 | 31 | // Runtime configuration (not part of configuration) 32 | runConfiguration *RunConfiguration 33 | } 34 | 35 | type Filesystem struct { 36 | // Remove path 37 | Path string `yaml:"path"` 38 | // Local path (optional) 39 | Local string `yaml:"local"` 40 | // Filter 41 | Filter Filter `yaml:"filter"` 42 | // Connection for filesystem sync (optional, default is Server connection) 43 | Connection *YamlCommandBuilderConnection `yaml:"connection"` 44 | Options FilesystemOptions `yaml:"options"` 45 | } 46 | 47 | type Database struct { 48 | // Type of database (either mysql or postgres) 49 | Type string `yaml:"type"` 50 | // Database name on remote database server 51 | Db string `yaml:"database"` 52 | // Hostname of remote database server 53 | Hostname string `yaml:"hostname"` 54 | // Port of remote database server 55 | Port string `yaml:"port"` 56 | // Username of remote database server 57 | User string `yaml:"user"` 58 | // Password of remote database server 59 | Password string `yaml:"password"` 60 | 61 | // Table filter 62 | Filter Filter `yaml:"filter"` 63 | // Connection for database sync (optional, default is Server connection) 64 | Connection *YamlCommandBuilderConnection `yaml:"connection"` 65 | // Database options 66 | Options DatabaseOptions `yaml:"options"` 67 | 68 | Local struct { 69 | // Database name on local database server 70 | Db string `yaml:"database"` 71 | // Hostname of local database server 72 | Hostname string `yaml:"hostname"` 73 | // Port of local database server 74 | Port string `yaml:"port"` 75 | // Username of local database server 76 | User string `yaml:"user"` 77 | // Password of local database server 78 | Password string `yaml:"password"` 79 | 80 | // Connection for database sync (optional, default is empty) 81 | Connection *YamlCommandBuilderConnection `yaml:"connection"` 82 | 83 | // Database options 84 | Options DatabaseOptions `yaml:"options"` 85 | } `yaml:"local"` 86 | 87 | // local cache for remote table list 88 | cacheRemoteTableList []string 89 | // local cache for local table list 90 | cacheLocalTableList []string 91 | } 92 | 93 | type Execution struct { 94 | // Type of execution (remote or local) 95 | Type string `yaml:"type"` 96 | // Command as string or as elements 97 | Command YamlStringArray `yaml:"command"` 98 | // Workdir for execution 99 | Workdir string `yaml:"workdir"` 100 | 101 | // Environment variables 102 | Environment []EnvironmentVar `yaml:"environment"` 103 | 104 | // Execution options 105 | Options struct { 106 | } `yaml:"options"` 107 | } 108 | 109 | type Filter struct { 110 | // Exclude as strings (regexp) 111 | Exclude []string `yaml:"exclude"` 112 | // compiled regexp excludes 113 | excludeRegexp []*regexp.Regexp 114 | 115 | // Includes as strings (regexp) 116 | Include []string `yaml:"include"` 117 | // compiled regexp includes 118 | includeRegexp []*regexp.Regexp 119 | } 120 | 121 | type DatabaseOptions struct { 122 | // Clear database with DROP/CREATE before sync 123 | ClearDatabase bool `yaml:"clear-database"` 124 | // Arguments for mysqldump command 125 | Mysqldump *YamlStringArray `yaml:"mysqldump"` 126 | // Arguments for mysql command 127 | Mysql *YamlStringArray `yaml:"mysql"` 128 | // Arguments for pgdump command 129 | Pgdump *YamlStringArray `yaml:"pgdump"` 130 | // Arguments for psql command 131 | Psql *YamlStringArray `yaml:"psql"` 132 | } 133 | 134 | type FilesystemOptions struct { 135 | // Generate stubs (small example files) instead of fetching files from remote 136 | GenerateStubs bool `yaml:"generate-stubs"` 137 | // Arguments for psql command 138 | Rsync *YamlStringArray `yaml:"rsync"` 139 | } 140 | 141 | type EnvironmentVar struct { 142 | // Name of variable 143 | Name string `yaml:"name"` 144 | // Value of variable 145 | Value string `yaml:"value"` 146 | } 147 | 148 | type RunConfiguration struct { 149 | // Enable database sync 150 | Database bool 151 | // Enable filesystem sync 152 | Filesystem bool 153 | // Enable exec runner 154 | Exec bool 155 | } 156 | -------------------------------------------------------------------------------- /sync/configparser.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "io/ioutil" 5 | "gopkg.in/yaml.v2" 6 | "fmt" 7 | "errors" 8 | "strings" 9 | ) 10 | 11 | const globalYamlHeader = ` 12 | --- 13 | databaseExcludeTYPO3: 14 | - ^cachingframework_.* 15 | - ^cf_.*" 16 | - ^cache_.* 17 | - ^index_.* 18 | - ^sys_log$ 19 | - ^sys_history$ 20 | - ^sys_registry$ 21 | - ^tx_extbase_cache.* 22 | - ^tx_extensionmanager_domain_model_extension.* 23 | - ^zzz_deleted_.* 24 | --- 25 | ` 26 | 27 | func NewConfigParser(file string) (config *SyncConfig) { 28 | ymlData, err := ioutil.ReadFile(file) 29 | if err != nil { 30 | Logger.FatalErrorExit(1, err) 31 | } 32 | 33 | // ymlDataCombined := globalYamlHeader + fmt.Sprintf("%s", ymlData) 34 | ymlDataCombined := fmt.Sprintf("%s", ymlData) 35 | 36 | if err := yaml.Unmarshal([]byte(ymlDataCombined), &config); err != nil { 37 | Logger.FatalErrorExit(1, err) 38 | } 39 | 40 | return 41 | } 42 | 43 | func (config *SyncConfig) GetSyncServer(serverName string) (Server, error) { 44 | if val, ok := config.Sync[serverName]; ok { 45 | return val, nil 46 | } else { 47 | return Server{}, errors.New(fmt.Sprintf("Server name %s doesn't exists", serverName)) 48 | } 49 | } 50 | 51 | func (config *SyncConfig) GetDeployServer(serverName string) (Server, error) { 52 | if val, ok := config.Deploy[serverName]; ok { 53 | return val, nil 54 | } else { 55 | return Server{}, errors.New(fmt.Sprintf("Server name %s doesn't exists", serverName)) 56 | } 57 | } 58 | 59 | func (config *SyncConfig) GetServerList(confType string) (list []string) { 60 | switch confType { 61 | case "sync": 62 | for key := range config.Sync { 63 | list = append(list, key) 64 | } 65 | case "deploy": 66 | for key := range config.Deploy { 67 | list = append(list, key) 68 | } 69 | } 70 | 71 | return 72 | } 73 | 74 | // List all possible server configurations 75 | func (config *SyncConfig) ListServer() (list map[string][]string) { 76 | if len(config.Sync) > 0 { 77 | list["Sync"] = make([]string, len(config.Sync)-1) 78 | for key := range config.Sync { 79 | list["Sync"] = append(list["Sync"], key) 80 | } 81 | } 82 | 83 | if len(config.Deploy) > 0 { 84 | list["Deploy"] = make([]string, len(config.Deploy)-1) 85 | for key := range config.Deploy { 86 | list["Deploy"] = append(list["Deploy"], key) 87 | } 88 | } 89 | 90 | return 91 | } 92 | 93 | // Show all possible server configurations 94 | // in an human readable style 95 | func (config *SyncConfig) ShowConfiguration() { 96 | serverList := config.ListServer() 97 | 98 | for area, keyList := range serverList { 99 | fmt.Println() 100 | fmt.Println(area) 101 | fmt.Println(strings.Repeat("=", len(area))) 102 | 103 | for _, serverKey := range keyList { 104 | fmt.Println(fmt.Sprintf(" -> %s ", serverKey)) 105 | } 106 | } 107 | 108 | 109 | } 110 | -------------------------------------------------------------------------------- /sync/database.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | func (database *Database) ApplyDefaults(server *Server) { 9 | // set default connection if not set 10 | if database.Connection == nil { 11 | database.Connection = server.Connection.Clone() 12 | } 13 | 14 | // default local connection 15 | if database.Local.Connection == nil { 16 | database.Local.Connection = &YamlCommandBuilderConnection{} 17 | } 18 | } 19 | 20 | func (database *Database) GetType() (dbtype string) { 21 | switch database.Type { 22 | case "mysql": 23 | dbtype = "mysql" 24 | case "postgresql": 25 | fallthrough 26 | case "postgres": 27 | dbtype = "postgres" 28 | default: 29 | panic(fmt.Sprintf("Database type %s is not valid or supported", database.Type)) 30 | } 31 | 32 | return dbtype 33 | } 34 | 35 | func (database *Database) GetMysql() DatabaseMysql { 36 | mysql := DatabaseMysql{*database} 37 | mysql.init() 38 | return mysql 39 | } 40 | 41 | func (database *Database) GetPostgres() DatabasePostgres { 42 | postgres := DatabasePostgres{*database} 43 | postgres.init() 44 | return postgres 45 | } 46 | 47 | func (database *Database) String(direction string) string { 48 | var parts, remote, local []string 49 | 50 | connRemote := database.Connection.GetInstance() 51 | connLocal := database.Local.Connection.GetInstance() 52 | 53 | // general 54 | parts = append(parts, fmt.Sprintf("Type:%s", database.Type)) 55 | 56 | //------------------------------------------- 57 | // remote 58 | remote = append(remote, fmt.Sprintf("Database:%s", database.Db)) 59 | remote = append(remote, fmt.Sprintf("Connection:%s", connRemote.GetType())) 60 | 61 | if connRemote.IsSsh() { 62 | if connRemote.SshConnectionHostnameString() != "" { 63 | remote = append(remote, fmt.Sprintf("SSH:%s", connRemote.SshConnectionHostnameString())) 64 | } 65 | } 66 | 67 | if connRemote.IsDocker() { 68 | remote = append(remote, fmt.Sprintf("Docker:%s", connRemote.Docker.Hostname)) 69 | } else if database.Hostname != "" { 70 | hostname := database.Hostname 71 | 72 | if database.Port != "" { 73 | hostname += ":"+database.Port 74 | } 75 | remote = append(remote, fmt.Sprintf("Host:%s", hostname)) 76 | } 77 | 78 | if database.User != "" { 79 | remote = append(remote, fmt.Sprintf("User:%s", database.User)) 80 | } 81 | 82 | if database.Password != "" { 83 | remote = append(remote, fmt.Sprintf("Passwd:%s", "*****")) 84 | } 85 | 86 | //------------------------------------------- 87 | // local 88 | local = append(local, fmt.Sprintf("Database:%s", database.Local.Db)) 89 | local = append(local, fmt.Sprintf("Connection:%s", connLocal.GetType())) 90 | 91 | if connLocal.IsSsh() { 92 | if connLocal.SshConnectionHostnameString() != "" { 93 | local = append(local, fmt.Sprintf("SSH:%s", connLocal.SshConnectionHostnameString())) 94 | } 95 | } 96 | 97 | if connLocal.IsDocker() { 98 | local = append(local, fmt.Sprintf("Docker:%s", connLocal.Docker.Hostname)) 99 | } else if database.Local.Hostname != "" { 100 | hostname := database.Local.Hostname 101 | 102 | if database.Local.Port != "" { 103 | hostname += ":"+database.Local.Port 104 | } 105 | local = append(local, fmt.Sprintf("Host:%s", hostname)) 106 | } 107 | 108 | // build parts 109 | switch direction { 110 | case "sync": 111 | parts = append(parts, remote...) 112 | parts = append(parts, "->") 113 | parts = append(parts, local...) 114 | case "deploy": 115 | parts = append(parts, local...) 116 | parts = append(parts, "->") 117 | parts = append(parts, remote...) 118 | } 119 | 120 | return fmt.Sprintf("Database[%s]", strings.Join(parts[:]," ")) 121 | } 122 | -------------------------------------------------------------------------------- /sync/database_mysql.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "github.com/webdevops/go-shell" 7 | ) 8 | 9 | type DatabaseMysql struct { 10 | Database 11 | } 12 | 13 | func (database *DatabaseMysql) init() { 14 | connLocal := database.Local.Connection.GetInstance() 15 | connRemote := database.Connection.GetInstance() 16 | 17 | // LOCAL 18 | if connLocal.IsDocker() { 19 | if database.Local.User == "" || database.Local.Db == "" { 20 | containerEnv := connLocal.DockerGetEnvironment() 21 | 22 | // try to guess user/password 23 | if database.Local.User == "" { 24 | if val, ok := containerEnv["MYSQL_ROOT_PASSWORD"]; ok { 25 | // get root pass from env 26 | if database.Local.User == "" && database.Local.Password == "" { 27 | fmt.Println(" -> local: using mysql root account (from env:MYSQL_ROOT_PASSWORD)") 28 | database.Local.User = "root" 29 | database.Local.Password = val 30 | } 31 | } else if val, ok := containerEnv["MYSQL_ALLOW_EMPTY_PASSWORD"]; ok { 32 | // get root without password from env 33 | if val == "yes" && database.Local.User == "" { 34 | fmt.Println(" -> local: using mysql root account (from env:MYSQL_ALLOW_EMPTY_PASSWORD)") 35 | database.Local.User = "root" 36 | database.Local.Password = "" 37 | } 38 | } else if user, ok := containerEnv["MYSQL_USER"]; ok { 39 | if pass, ok := containerEnv["MYSQL_PASSWORD"]; ok { 40 | if database.Local.User == "" && database.Local.Password == "" { 41 | fmt.Println(fmt.Sprintf(" -> local: using mysql user account \"%s\" (from env:MYSQL_USER and env:MYSQL_PASSWORD)", user)) 42 | database.Local.User = user 43 | database.Local.Password = pass 44 | } 45 | } 46 | } 47 | } 48 | 49 | // get database from env 50 | if database.Local.Db == "" { 51 | if db, ok := containerEnv["MYSQL_DATABASE"]; ok { 52 | fmt.Println(fmt.Sprintf(" -> local: using mysql database \"%s\" (from env:MYSQL_DATABASE)", db)) 53 | database.Local.Db = db 54 | } 55 | } 56 | } 57 | } 58 | 59 | // Remote 60 | if connRemote.IsDocker() { 61 | if database.User == "" || database.Db == "" { 62 | containerEnv := connRemote.DockerGetEnvironment() 63 | 64 | // try to guess user/password 65 | if database.User == "" { 66 | if val, ok := containerEnv["MYSQL_ROOT_PASSWORD"]; ok { 67 | // get root pass from env 68 | if database.User == "" && database.Password == "" { 69 | fmt.Println(" -> remote: using mysql root account (from env:MYSQL_ROOT_PASSWORD)") 70 | database.User = "root" 71 | database.Password = val 72 | } 73 | } else if val, ok := containerEnv["MYSQL_ALLOW_EMPTY_PASSWORD"]; ok { 74 | // get root without password from env 75 | if val == "yes" && database.User == "" { 76 | fmt.Println(" -> remote: using mysql root account (from env:MYSQL_ALLOW_EMPTY_PASSWORD)") 77 | database.User = "root" 78 | database.Password = "" 79 | } 80 | } else if user, ok := containerEnv["MYSQL_USER"]; ok { 81 | if pass, ok := containerEnv["MYSQL_PASSWORD"]; ok { 82 | if database.User == "" && database.Password == "" { 83 | fmt.Println(fmt.Sprintf(" -> remote: using mysql user account \"%s\" (from env:MYSQL_USER and env:MYSQL_PASSWORD)", user)) 84 | database.User = user 85 | database.Password = pass 86 | } 87 | } 88 | } 89 | } 90 | 91 | // get database from env 92 | if database.Db == "" { 93 | if db, ok := containerEnv["MYSQL_DATABASE"]; ok { 94 | fmt.Println(fmt.Sprintf(" -> remote: using mysql database \"%s\" (from env:MYSQL_DATABASE)", db)) 95 | database.Db = db 96 | } 97 | } 98 | } 99 | } 100 | } 101 | 102 | func (database *DatabaseMysql) tableFilter(connectionType string) (exclude []string, include []string) { 103 | var tableList []string 104 | 105 | if connectionType == "local" { 106 | if len(database.cacheLocalTableList) == 0 { 107 | Logger.Step("get list of mysql tables for table filter") 108 | database.cacheLocalTableList = database.tableList(connectionType) 109 | } 110 | 111 | tableList = database.cacheLocalTableList 112 | } else { 113 | if len(database.cacheRemoteTableList) == 0 { 114 | Logger.Step("get list of mysql tables for table filter") 115 | database.cacheRemoteTableList = database.tableList(connectionType) 116 | } 117 | 118 | tableList = database.cacheRemoteTableList 119 | } 120 | 121 | 122 | // calc excludes 123 | excludeTableList := database.Filter.CalcExcludes(tableList) 124 | for _, table := range excludeTableList { 125 | exclude = append(exclude, shell.Quote(fmt.Sprintf("--ignore-table=%s.%s", database.Db, table))) 126 | } 127 | 128 | // calc includes 129 | includeTableList := database.Filter.CalcIncludes(tableList) 130 | for _, table := range includeTableList { 131 | include = append(include, table) 132 | } 133 | 134 | return exclude, include 135 | } 136 | 137 | func (database *DatabaseMysql) mysqlCommandBuilder(direction string, args ...string) []interface{} { 138 | if direction == "local" { 139 | return database.localMysqlCmdBuilder(args...) 140 | } else { 141 | return database.remoteMysqlCmdBuilder(args...) 142 | } 143 | } 144 | 145 | func (database *DatabaseMysql) tableList(connectionType string) []string { 146 | sqlStmt := "SHOW TABLES" 147 | 148 | cmd := shell.Cmd("echo", shell.Quote(sqlStmt)).Pipe(database.mysqlCommandBuilder(connectionType)...) 149 | output := cmd.Run().Stdout.String() 150 | 151 | outputString := strings.TrimSpace(string(output)) 152 | tmp := strings.Split(outputString, "\n") 153 | 154 | // trim spaces from tables 155 | ret := make([]string, len(tmp)) 156 | for _, table := range tmp { 157 | ret = append(ret, strings.TrimSpace(table)) 158 | } 159 | 160 | return ret 161 | } 162 | 163 | func (database *DatabaseMysql) quote(value string) string { 164 | return "'" + strings.Replace(value, "'", "\\'", -1) + "'" 165 | } 166 | 167 | func (database *DatabaseMysql) quoteIdentifier(value string) string { 168 | return "`" + strings.Replace(value, "`", "\\`", -1) + "`" 169 | } 170 | -------------------------------------------------------------------------------- /sync/database_mysql_local.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "github.com/webdevops/go-shell" 5 | ) 6 | 7 | func (database *DatabaseMysql) localMysqldumpCmdBuilder(additionalArgs []string, useFilter bool) []interface{} { 8 | var args []string 9 | 10 | connection := database.Local.Connection.GetInstance().Clone() 11 | 12 | // add custom options (raw) 13 | if database.Local.Options.Mysqldump != nil { 14 | args = append(args, database.Local.Options.Mysqldump.Array()...) 15 | } 16 | 17 | if database.Local.User != "" { 18 | args = append(args, shell.Quote("-u" + database.Local.User)) 19 | } 20 | 21 | if database.Local.Password != "" { 22 | connection.Environment.Set("MYSQL_PWD", database.Local.Password) 23 | } 24 | 25 | if database.Local.Hostname != "" { 26 | args = append(args, shell.Quote("-h" + database.Local.Hostname)) 27 | } 28 | 29 | if database.Local.Port != "" { 30 | args = append(args, shell.Quote("-P" + database.Local.Port)) 31 | } 32 | 33 | if len(args) > 0 { 34 | args = append(args, additionalArgs...) 35 | } 36 | 37 | // exclude 38 | excludeArgs, includeArgs := database.tableFilter("local"); 39 | if useFilter && len(excludeArgs) > 0 { 40 | args = append(args, excludeArgs...) 41 | } 42 | 43 | // database 44 | args = append(args, shell.Quote(database.Local.Db)) 45 | 46 | // include 47 | if useFilter && len(includeArgs) > 0 { 48 | args = append(args, includeArgs...) 49 | } 50 | 51 | return connection.RawCommandBuilder("mysqldump", args...) 52 | } 53 | 54 | func (database *DatabaseMysql) localMysqlCmdBuilder(additonalArgs ...string) []interface{} { 55 | var args []string 56 | 57 | connection := database.Local.Connection.GetInstance().Clone() 58 | 59 | // add custom options (raw) 60 | if database.Local.Options.Mysql != nil { 61 | args = append(args, database.Local.Options.Mysql.Array()...) 62 | } 63 | 64 | args = append(args, "-BN") 65 | 66 | if database.Local.User != "" { 67 | args = append(args, shell.Quote("-u" + database.Local.User)) 68 | } 69 | 70 | if database.Local.Password != "" { 71 | connection.Environment.Set("MYSQL_PWD", database.Local.Password) 72 | } 73 | 74 | if database.Local.Hostname != "" { 75 | args = append(args, shell.Quote("-h" + database.Local.Hostname)) 76 | } 77 | 78 | if database.Local.Port != "" { 79 | args = append(args, shell.Quote("-P" + database.Local.Port)) 80 | } 81 | 82 | if database.Local.Db != "" { 83 | args = append(args, shell.Quote(database.Local.Db)) 84 | } 85 | 86 | if len(additonalArgs) > 0 { 87 | args = append(args, additonalArgs...) 88 | } 89 | 90 | return connection.RawCommandBuilder("mysql", args...) 91 | } 92 | 93 | -------------------------------------------------------------------------------- /sync/database_mysql_remote.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "strings" 5 | "github.com/webdevops/go-shell" 6 | ) 7 | 8 | func (database *DatabaseMysql) remoteMysqldumpCmdBuilder(additionalArgs []string, useFilter bool) []interface{} { 9 | var args []string 10 | 11 | connection := database.Connection.GetInstance().Clone() 12 | 13 | if database.User != "" { 14 | args = append(args, shell.Quote("-u" + database.User)) 15 | } 16 | 17 | if database.Password != "" { 18 | connection.Environment.Set("MYSQL_PWD", database.Password) 19 | } 20 | 21 | if database.Hostname != "" { 22 | args = append(args, shell.Quote("-h" + database.Hostname)) 23 | } 24 | 25 | if database.Port != "" { 26 | args = append(args, shell.Quote("-P" + database.Port)) 27 | } 28 | 29 | if len(args) > 0 { 30 | args = append(args, additionalArgs...) 31 | } 32 | 33 | // exclude 34 | excludeArgs, includeArgs := database.tableFilter("remote"); 35 | if useFilter && len(excludeArgs) > 0 { 36 | args = append(args, excludeArgs...) 37 | } 38 | 39 | // database 40 | args = append(args, shell.Quote(database.Db)) 41 | 42 | // include 43 | if useFilter && len(includeArgs) > 0 { 44 | args = append(args, includeArgs...) 45 | } 46 | 47 | cmd := []string{"mysqldump"} 48 | 49 | // add custom options (raw) 50 | if database.Options.Mysqldump != nil { 51 | cmd = append(cmd, database.Options.Mysqldump.Array()...) 52 | } 53 | 54 | cmd = append(cmd, args...) 55 | cmd = append(cmd, "|", "gzip", "--stdout") 56 | 57 | return connection.RawShellCommandBuilder(cmd...) 58 | } 59 | 60 | func (database *DatabaseMysql) remoteMysqlCmdBuilder(additonalArgs ...string) []interface{} { 61 | var args []string 62 | 63 | connection := database.Connection.GetInstance().Clone() 64 | 65 | // append options in raw 66 | if database.Options.Mysql != nil { 67 | args = append(args, database.Options.Mysql.Array()...) 68 | } 69 | 70 | args = append(args, "-BN") 71 | 72 | if database.User != "" { 73 | args = append(args, shell.Quote("-u" + database.User)) 74 | } 75 | 76 | if database.Password != "" { 77 | connection.Environment.Set("MYSQL_PWD", database.Password) 78 | } 79 | 80 | if database.Hostname != "" { 81 | args = append(args, shell.Quote("-h" + database.Hostname)) 82 | } 83 | 84 | if database.Port != "" { 85 | args = append(args, shell.Quote("-P" + database.Port)) 86 | } 87 | 88 | if database.Db != "" { 89 | args = append(args, shell.Quote(database.Db)) 90 | } 91 | 92 | if len(additonalArgs) > 0 { 93 | args = append(args, additonalArgs...) 94 | } 95 | 96 | return connection.RawCommandBuilder("mysql", args...) 97 | } 98 | 99 | 100 | func (database *DatabaseMysql) remoteMysqlCmdBuilderUncompress(additonalArgs ...string) []interface{} { 101 | var args []string 102 | 103 | connection := database.Connection.GetInstance().Clone() 104 | 105 | // add custom options (raw) 106 | if database.Options.Mysql != nil { 107 | args = append(args, database.Options.Mysql.Array()...) 108 | } 109 | 110 | args = append(args, "-BN") 111 | 112 | if database.User != "" { 113 | args = append(args, shell.Quote("-u" + database.User)) 114 | } 115 | 116 | if database.Password != "" { 117 | connection.Environment.Set("MYSQL_PWD", database.Password) 118 | } 119 | 120 | if database.Hostname != "" { 121 | args = append(args, shell.Quote("-h" + database.Hostname)) 122 | } 123 | 124 | if database.Port != "" { 125 | args = append(args, shell.Quote("-P" + database.Port)) 126 | } 127 | 128 | if len(additonalArgs) > 0 { 129 | args = append(args, additonalArgs...) 130 | } 131 | 132 | if database.Db != "" { 133 | args = append(args, shell.Quote(database.Db)) 134 | } 135 | 136 | cmd := []string{"gunzip", "--stdout", "|", "mysql", strings.Join(args, " ")} 137 | 138 | return connection.RawShellCommandBuilder(cmd...) 139 | } 140 | -------------------------------------------------------------------------------- /sync/database_postgres.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | "github.com/webdevops/go-shell" 7 | ) 8 | 9 | type DatabasePostgres struct { 10 | Database 11 | } 12 | 13 | func (database *DatabasePostgres) init() { 14 | connLocal := database.Local.Connection.GetInstance() 15 | connRemote := database.Connection.GetInstance() 16 | 17 | // LOCAL 18 | if connLocal.IsDocker() { 19 | if database.Local.User == "" || database.Local.Db == "" { 20 | containerEnv := connLocal.DockerGetEnvironment() 21 | 22 | // try to guess user/password 23 | if database.Local.User == "" { 24 | // get superuser pass from env 25 | if pass, ok := containerEnv["POSTGRES_PASSWORD"]; ok { 26 | if user, ok := containerEnv["POSTGRES_USER"]; ok { 27 | fmt.Println(fmt.Sprintf(" -> local: using postgres superadmin account \"%s\" (from env:POSTGRES_USER and env:POSTGRES_PASSWORD)", user)) 28 | database.Local.User = user 29 | database.Local.Password = pass 30 | } else { 31 | fmt.Println(" -> local: using postgres superadmin account \"postgres\" (from env:POSTGRES_PASSWORD)") 32 | // only password available 33 | database.Local.User = "postgres" 34 | database.Local.Password = pass 35 | } 36 | } 37 | } 38 | 39 | // get database from env 40 | if database.Local.Db == "" { 41 | if db, ok := containerEnv["POSTGRES_DB"]; ok { 42 | fmt.Println(fmt.Sprintf(" -> remote: using postgres database \"%s\" (from env:POSTGRES_DB)", db)) 43 | database.Local.Db = db 44 | } 45 | } 46 | } 47 | } 48 | 49 | // Remote 50 | if connRemote.IsDocker() { 51 | if database.User == "" || database.Db == "" { 52 | containerEnv := connRemote.DockerGetEnvironment() 53 | 54 | // try to guess user/password 55 | if database.User == "" { 56 | // get superuser pass from env 57 | if pass, ok := containerEnv["POSTGRES_PASSWORD"]; ok { 58 | if user, ok := containerEnv["POSTGRES_USER"]; ok { 59 | fmt.Println(fmt.Sprintf(" -> remote: using postgres superadmin account \"%s\" (from env:POSTGRES_USER and env:POSTGRES_PASSWORD)", user)) 60 | database.User = user 61 | database.Password = pass 62 | } else { 63 | fmt.Println(" -> remote: using postgres superadmin account \"postgres\" (from env:POSTGRES_PASSWORD)") 64 | // only password available 65 | database.User = "postgres" 66 | database.Password = pass 67 | } 68 | } 69 | } 70 | 71 | // get database from env 72 | if database.Db == "" { 73 | if db, ok := containerEnv["POSTGRES_DB"]; ok { 74 | fmt.Println(fmt.Sprintf(" -> remote: using postgres database \"%s\" (from env:POSTGRES_DB)", db)) 75 | database.Db = db 76 | } 77 | } 78 | } 79 | } 80 | } 81 | 82 | func (database *DatabasePostgres) tableFilter(connectionType string) (exclude []string, include []string) { 83 | var tableList []string 84 | 85 | if connectionType == "local" { 86 | if len(database.cacheLocalTableList) == 0 { 87 | Logger.Step("get list of postgres tables for table filter") 88 | database.cacheLocalTableList = database.tableList(connectionType) 89 | } 90 | 91 | tableList = database.cacheLocalTableList 92 | } else { 93 | if len(database.cacheRemoteTableList) == 0 { 94 | Logger.Step("get list of postgres tables for table filter") 95 | database.cacheRemoteTableList = database.tableList(connectionType) 96 | } 97 | 98 | tableList = database.cacheRemoteTableList 99 | } 100 | 101 | // calc excludes 102 | excludeTableList := database.Filter.CalcExcludes(tableList) 103 | for _, table := range excludeTableList { 104 | exclude = append(exclude, shell.Quote(fmt.Sprintf("--exclude-table=%s", table))) 105 | } 106 | 107 | // calc includes 108 | includeTableList := database.Filter.CalcIncludes(tableList) 109 | for _, table := range includeTableList { 110 | include = append(include, shell.Quote(fmt.Sprintf("--table=%s", table))) 111 | } 112 | 113 | return 114 | } 115 | 116 | func (database *DatabasePostgres) psqlCommandBuilder(direction string, args ...string) []interface{} { 117 | if direction == "local" { 118 | return database.localPsqlCmdBuilder(args...) 119 | } else { 120 | return database.remotePsqlCmdBuilder(args...) 121 | } 122 | } 123 | 124 | func (database *DatabasePostgres) tableList(connectionType string) []string { 125 | sqlStmt := `SELECT table_name 126 | FROM information_schema.tables 127 | WHERE table_type = 'BASE TABLE' 128 | AND table_catalog = %s` 129 | sqlStmt = fmt.Sprintf(sqlStmt, database.quote(database.Db)) 130 | 131 | cmd := shell.Cmd("echo", shell.Quote(sqlStmt)).Pipe(database.psqlCommandBuilder(connectionType)...) 132 | output := cmd.Run().Stdout.String() 133 | 134 | outputString := strings.TrimSpace(string(output)) 135 | tmp := strings.Split(outputString, "\n") 136 | 137 | // trim spaces from tables 138 | ret := make([]string, len(tmp)) 139 | for _, table := range tmp { 140 | ret = append(ret, strings.TrimSpace(table)) 141 | } 142 | 143 | return ret 144 | } 145 | 146 | func (database *DatabasePostgres) quote(value string) string { 147 | return "'" + strings.Replace(value, "'", "\\'", -1) + "'" 148 | } 149 | 150 | func (database *DatabasePostgres) quoteIdentifier(value string) string { 151 | return "\"" + strings.Replace(value, "\"", "\\\"", -1) + "\"" 152 | } 153 | -------------------------------------------------------------------------------- /sync/database_postgres_local.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import "github.com/webdevops/go-shell" 4 | 5 | func (database *DatabasePostgres) localPgdumpCmdBuilder(additionalArgs []string, useFilter bool) []interface{} { 6 | var args []string 7 | 8 | connection := database.Local.Connection.GetInstance().Clone() 9 | 10 | // add custom options (raw) 11 | if database.Local.Options.Pgdump != nil { 12 | args = append(args, database.Local.Options.Pgdump.Array()...) 13 | } 14 | 15 | if database.Local.User != "" { 16 | args = append(args, "-U", shell.Quote(database.Local.User)) 17 | } 18 | 19 | if database.Local.Password != "" { 20 | connection.Environment.Set("PGPASSWORD", database.Local.Password) 21 | } 22 | 23 | if database.Local.Hostname != "" { 24 | args = append(args, "-h", shell.Quote(database.Local.Hostname)) 25 | } 26 | 27 | if database.Local.Port != "" { 28 | args = append(args, "-p", shell.Quote(database.Local.Port)) 29 | } 30 | 31 | if len(args) > 0 { 32 | args = append(args, additionalArgs...) 33 | } 34 | 35 | // exclude 36 | excludeArgs, includeArgs := database.tableFilter("local"); 37 | if useFilter && len(excludeArgs) > 0 { 38 | args = append(args, excludeArgs...) 39 | } 40 | 41 | // database 42 | args = append(args, shell.Quote(database.Local.Db)) 43 | 44 | // include 45 | if useFilter && len(includeArgs) > 0 { 46 | args = append(args, includeArgs...) 47 | } 48 | 49 | return connection.RawCommandBuilder("pg_dump", args...) 50 | } 51 | 52 | func (database *DatabasePostgres) localPsqlCmdBuilder(additonalArgs ...string) []interface{} { 53 | var args []string 54 | 55 | connection := database.Local.Connection.GetInstance().Clone() 56 | 57 | // add custom options (raw) 58 | if database.Local.Options.Psql != nil { 59 | args = append(args, database.Local.Options.Psql.Array()...) 60 | } 61 | 62 | args = append(args, "-t") 63 | 64 | if database.Local.User != "" { 65 | args = append(args, "-U", shell.Quote(database.Local.User)) 66 | } 67 | 68 | if database.Local.Password != "" { 69 | connection.Environment.Set("PGPASSWORD", database.Local.Password) 70 | } 71 | 72 | if database.Local.Hostname != "" { 73 | args = append(args, "-h", shell.Quote(database.Local.Hostname)) 74 | } 75 | 76 | if database.Local.Port != "" { 77 | args = append(args, "-p", shell.Quote(database.Local.Port)) 78 | } 79 | 80 | if len(additonalArgs) > 0 { 81 | args = append(args, additonalArgs...) 82 | } 83 | 84 | if database.Local.Db != "" { 85 | args = append(args, shell.Quote(database.Local.Db)) 86 | } 87 | 88 | return connection.RawCommandBuilder("psql", args...) 89 | } 90 | -------------------------------------------------------------------------------- /sync/database_postgres_remote.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "strings" 5 | "github.com/webdevops/go-shell" 6 | ) 7 | 8 | func (database *DatabasePostgres) remotePgdumpCmdBuilder(additionalArgs []string, useFilter bool) []interface{} { 9 | connection := database.Connection.GetInstance().Clone() 10 | var args []string 11 | 12 | if database.User != "" { 13 | args = append(args, "-U", shell.Quote(database.User)) 14 | } 15 | 16 | if database.Password != "" { 17 | connection.Environment.Set("PGPASSWORD", database.Password) 18 | } 19 | 20 | if database.Hostname != "" { 21 | args = append(args, "-h", shell.Quote(database.Hostname)) 22 | } 23 | 24 | if database.Port != "" { 25 | args = append(args, "-p", shell.Quote(database.Port)) 26 | } 27 | 28 | if len(args) > 0 { 29 | args = append(args, additionalArgs...) 30 | } 31 | 32 | // exclude 33 | excludeArgs, includeArgs := database.tableFilter("remote") 34 | if useFilter && len(excludeArgs) > 0 { 35 | args = append(args, excludeArgs...) 36 | } 37 | 38 | // database 39 | args = append(args, shell.Quote(database.Db)) 40 | 41 | // include 42 | if useFilter && len(includeArgs) > 0 { 43 | args = append(args, includeArgs...) 44 | } 45 | 46 | cmd := []string{"pg_dump"} 47 | 48 | // add custom options (raw) 49 | if database.Options.Pgdump != nil { 50 | cmd = append(cmd, database.Options.Pgdump.Array()...) 51 | } 52 | 53 | cmd = append(cmd, args...) 54 | cmd = append(cmd, "|", "gzip", "--stdout") 55 | 56 | return connection.RawShellCommandBuilder(cmd...) 57 | } 58 | 59 | func (database *DatabasePostgres) remotePsqlCmdBuilder(additonalArgs ...string) []interface{} { 60 | var args []string 61 | 62 | connection := database.Connection.GetInstance().Clone() 63 | 64 | // append options in raw 65 | if database.Options.Psql != nil { 66 | args = append(args, database.Options.Psql.Array()...) 67 | } 68 | 69 | args = append(args, "-t") 70 | 71 | if database.User != "" { 72 | args = append(args, "-U", shell.Quote(database.User)) 73 | } 74 | 75 | if database.Password != "" { 76 | connection.Environment.Set("PGPASSWORD", database.Password) 77 | } 78 | 79 | if database.Hostname != "" { 80 | args = append(args, "-h", shell.Quote(database.Hostname)) 81 | } 82 | 83 | if database.Port != "" { 84 | args = append(args, "-p", shell.Quote(database.Port)) 85 | } 86 | 87 | if database.Db != "" { 88 | args = append(args, shell.Quote(database.Db)) 89 | } 90 | 91 | if len(additonalArgs) > 0 { 92 | args = append(args, additonalArgs...) 93 | } 94 | 95 | return connection.RawCommandBuilder("psql", args...) 96 | } 97 | 98 | 99 | func (database *DatabasePostgres) remotePsqlCmdBuilderUncompress(args ...string) []interface{} { 100 | connection := database.Connection.GetInstance().Clone() 101 | args = append(args, "-t") 102 | 103 | if database.User != "" { 104 | args = append(args, "-U", shell.Quote(database.User)) 105 | } 106 | 107 | if database.Password != "" { 108 | connection.Environment.Set("MYSQL_PWD", database.Password) 109 | } 110 | 111 | if database.Hostname != "" { 112 | args = append(args, "-h", shell.Quote(database.Hostname)) 113 | } 114 | 115 | if database.Port != "" { 116 | args = append(args, "-p", shell.Quote(database.Port)) 117 | } 118 | 119 | // add custom options (raw) 120 | if database.Options.Psql != nil { 121 | args = append(args, database.Options.Psql.Array()...) 122 | } 123 | 124 | if database.Db != "" { 125 | args = append(args, shell.Quote(database.Db)) 126 | } 127 | 128 | cmd := []string{"gunzip", "--stdout", "|", "psql", strings.Join(args, " ")} 129 | 130 | return connection.RawShellCommandBuilder(cmd...) 131 | } 132 | -------------------------------------------------------------------------------- /sync/execution.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "strings" 5 | "github.com/webdevops/go-shell" 6 | "fmt" 7 | "github.com/webdevops/go-shell/commandbuilder" 8 | ) 9 | 10 | func (execution *Execution) String(server *Server) string { 11 | var parts []string 12 | 13 | parts = append(parts, fmt.Sprintf("Type:%s", execution.GetType())) 14 | 15 | 16 | if execution.Workdir != "" { 17 | parts = append(parts, fmt.Sprintf("Workdir:%s", execution.Workdir)) 18 | } 19 | parts = append(parts, fmt.Sprintf("Command:%s", execution.Command.ToString(" "))) 20 | 21 | return fmt.Sprintf("Exec[%s]", strings.Join(parts[:]," ")) 22 | } 23 | 24 | func (execution *Execution) Execute(server *Server) { 25 | cmd := execution.commandBuilder(server) 26 | run := shell.Cmd(cmd...).Run() 27 | 28 | Logger.Verbose(run.Stdout.String()) 29 | } 30 | 31 | // Create commandBuilder for execution 32 | func (execution *Execution) commandBuilder(server *Server) []interface{} { 33 | var connection commandbuilder.Connection 34 | 35 | switch execution.GetType() { 36 | case "local": 37 | connection = commandbuilder.Connection{Type:"local"} 38 | case "remote": 39 | connection = *(server.Connection.GetInstance().Clone()) 40 | } 41 | 42 | // set working directory 43 | if execution.Workdir != "" { 44 | connection.Workdir = execution.Workdir 45 | } 46 | 47 | // set environment 48 | connection.Environment.Clear() 49 | for _, val := range execution.Environment { 50 | connection.Environment.Set(val.Name, val.Value) 51 | } 52 | 53 | if len(execution.Command.Multi) >= 1 { 54 | // multi element command (use safer quoting) 55 | return connection.ShellCommandBuilder(execution.Command.Multi...) 56 | } else { 57 | // single string command (use as is) 58 | return connection.RawShellCommandBuilder(execution.Command.Single) 59 | } 60 | } 61 | 62 | // Get execution type (local or remote) 63 | func (execution *Execution) GetType() (execType string) { 64 | switch strings.ToLower(execution.Type) { 65 | case "": 66 | fallthrough 67 | case "local": 68 | execType = "local" 69 | case "remote": 70 | execType = "remote" 71 | default: 72 | panic(execution) 73 | } 74 | 75 | return 76 | } 77 | -------------------------------------------------------------------------------- /sync/filesystem.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "fmt" 5 | "strings" 6 | ) 7 | 8 | func (filesystem *Filesystem) ApplyDefaults(server *Server) { 9 | // set default connection if not set 10 | if filesystem.Connection == nil { 11 | filesystem.Connection = server.Connection.Clone() 12 | } 13 | 14 | // set default path 15 | if filesystem.Local == "" { 16 | filesystem.Local = server.GetLocalPath() 17 | } 18 | } 19 | 20 | func (filesystem *Filesystem) localPath() string { 21 | return filesystem.Local 22 | } 23 | 24 | func (filesystem *Filesystem) String(direction string) string { 25 | var parts []string 26 | 27 | switch direction { 28 | case "sync": 29 | parts = append(parts, fmt.Sprintf("Path:%s", filesystem.Path)) 30 | parts = append(parts, "->") 31 | parts = append(parts, fmt.Sprintf("Local:%s", filesystem.localPath())) 32 | case "deploy": 33 | parts = append(parts, fmt.Sprintf("Local:%s", filesystem.localPath())) 34 | parts = append(parts, "->") 35 | parts = append(parts, fmt.Sprintf("Path:%s", filesystem.Path)) 36 | } 37 | 38 | return fmt.Sprintf("Filesystem[%s]", strings.Join(parts[:]," ")) 39 | } 40 | -------------------------------------------------------------------------------- /sync/filter.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "regexp" 5 | ) 6 | 7 | // Compile filter regexp (and cache them) 8 | func (filter *Filter) compile() { 9 | if len(filter.excludeRegexp) == 0 { 10 | filter.excludeRegexp = make([]*regexp.Regexp, len(filter.Exclude)) 11 | for i, filterVal := range filter.Exclude { 12 | filter.excludeRegexp[i] = regexp.MustCompile(filterVal) 13 | } 14 | } 15 | 16 | if len(filter.includeRegexp) == 0 { 17 | filter.includeRegexp = make([]*regexp.Regexp, len(filter.Include)) 18 | for i, filterVal := range filter.Include { 19 | filter.includeRegexp[i] = regexp.MustCompile(filterVal) 20 | } 21 | } 22 | } 23 | 24 | // Apply filter (exclude/include) and get filtered list 25 | func (filter *Filter) ApplyFilter(lines []string) []string { 26 | filter.compile() 27 | 28 | if len(filter.Include) > 0 { 29 | lines = filter.calculateMatching(filter.includeRegexp, lines) 30 | } 31 | 32 | if len(filter.Exclude) > 0 { 33 | excludes := filter.calculateMatching(filter.excludeRegexp, lines) 34 | 35 | tmp := []string{} 36 | for _, line := range lines { 37 | if ! stringInSlice(line, excludes) { 38 | tmp = append(tmp, line) 39 | } 40 | } 41 | 42 | lines = tmp 43 | } 44 | 45 | return lines 46 | } 47 | 48 | // Apply exclude filter only and get filtered excludes 49 | func (filter *Filter) CalcExcludes(lines []string) []string { 50 | filter.compile() 51 | return filter.calculateMatching(filter.excludeRegexp, lines) 52 | } 53 | 54 | // Apply includes filter only and get filtered includes 55 | func (filter *Filter) CalcIncludes(lines []string) []string { 56 | filter.compile() 57 | return filter.calculateMatching(filter.includeRegexp, lines) 58 | } 59 | 60 | // Calculate matches using regexp array 61 | func (filter *Filter) calculateMatching(regexpList []*regexp.Regexp, lines []string) (matches []string) { 62 | for _, filterRegexp := range regexpList { 63 | for _, value := range lines { 64 | if filterRegexp.MatchString(value) == true { 65 | matches = append(matches, value) 66 | } 67 | } 68 | } 69 | 70 | return 71 | } 72 | 73 | // check if string exists in slice 74 | func stringInSlice(a string, list []string) (status bool) { 75 | status = false 76 | 77 | for _, b := range list { 78 | if b == a { 79 | status = true 80 | return 81 | } 82 | } 83 | return 84 | } 85 | -------------------------------------------------------------------------------- /sync/global.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import "github.com/webdevops/go-sync/logger" 4 | 5 | var ( 6 | Logger *logger.SyncLogger 7 | ) 8 | -------------------------------------------------------------------------------- /sync/helper.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "os" 5 | "io/ioutil" 6 | "log" 7 | "strings" 8 | "github.com/webdevops/go-shell" 9 | "fmt" 10 | "runtime/debug" 11 | ) 12 | 13 | func CreateTempfile() *os.File { 14 | tmpfile, err := ioutil.TempFile("", "gsync") 15 | if err != nil { 16 | log.Fatal(err) 17 | } 18 | 19 | return tmpfile 20 | } 21 | 22 | func CreateTempfileWithContent(content ...string) *os.File { 23 | tmpfile := CreateTempfile() 24 | 25 | if _, err := tmpfile.Write([]byte(strings.Join(content[:],"\n"))); err != nil { 26 | log.Fatal(err) 27 | } 28 | 29 | return tmpfile 30 | } 31 | 32 | func PathExists(name string) bool { 33 | if _, err := os.Stat(name); err != nil { 34 | if os.IsNotExist(err) { 35 | return false 36 | } 37 | } 38 | return true 39 | } 40 | 41 | func FileExists(name string) bool { 42 | f, err := os.Stat(name); 43 | 44 | if err != nil { 45 | if os.IsNotExist(err) { 46 | return false 47 | } 48 | } 49 | 50 | if f.IsDir() { 51 | return false 52 | } 53 | 54 | return true 55 | } 56 | 57 | func RsyncPath(name string) string { 58 | return strings.TrimRight(name, "/") + "/" 59 | } 60 | 61 | func ShellErrorHandler(recover interface{}) { 62 | if process, ok := recover.(*shell.Process); ok { 63 | p := process.ExitStatus 64 | p = 2 65 | if p != 0 { 66 | 67 | printMessage := func(header string, body string) { 68 | fmt.Println(header) 69 | fmt.Println(strings.Repeat("-", len(header))) 70 | fmt.Println(" " + strings.Replace(body, "\n", "\n ", -1)) 71 | fmt.Println() 72 | } 73 | 74 | fmt.Println("\n\n[!!!] Command execution failed") 75 | fmt.Println() 76 | 77 | printMessage("Command", process.Command.ToString()) 78 | printMessage("Stdout", process.Stdout.String()) 79 | printMessage("Stderr", process.Stderr.String()) 80 | printMessage("Exit code", fmt.Sprintf("%d", p)) 81 | 82 | os.Exit(2) 83 | } 84 | } else if recover != nil { 85 | fmt.Print("ERROR:") 86 | fmt.Println(recover) 87 | debug.PrintStack() 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /sync/server.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "gopkg.in/yaml.v2" 5 | ) 6 | 7 | func (server *Server) Init() { 8 | if server.runConfiguration == nil { 9 | server.runConfiguration = &RunConfiguration{ 10 | Database: true, 11 | Filesystem: true, 12 | } 13 | } 14 | } 15 | 16 | func (server *Server) GetLocalPath() string { 17 | if server.Path == "" { 18 | Logger.FatalExit(1, "server.Path is empty") 19 | } 20 | 21 | return server.Path 22 | } 23 | 24 | func (server *Server) SetRunConfiguration(conf RunConfiguration) { 25 | server.runConfiguration = &conf 26 | } 27 | 28 | func (server *Server) AsYaml() string { 29 | conf, _ := yaml.Marshal(server) 30 | return string(conf) 31 | } 32 | -------------------------------------------------------------------------------- /sync/server_deploy.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | func (server *Server) Deploy() { 4 | defer func() { 5 | recover := recover() 6 | ShellErrorHandler(recover) 7 | }() 8 | 9 | server.Init() 10 | 11 | if server.runConfiguration.Exec { 12 | server.RunExec("startup") 13 | } 14 | 15 | if server.runConfiguration.Filesystem { 16 | server.DeployFilesystem() 17 | } 18 | 19 | if server.runConfiguration.Database { 20 | server.DeployDatabases() 21 | } 22 | 23 | if server.runConfiguration.Exec { 24 | server.RunExec("finish") 25 | } 26 | 27 | waitGroup.Wait() 28 | } 29 | 30 | func (server *Server) DeployFilesystem() { 31 | // check for generate-stubs option (not allowed) 32 | for _, filesystem := range server.Filesystem { 33 | if filesystem.Options.GenerateStubs { 34 | Logger.FatalExit(2, "Generate Stubs is not allowed for deployment") 35 | } 36 | } 37 | 38 | for _, filesystem := range server.Filesystem { 39 | filesystem.ApplyDefaults(server) 40 | 41 | Logger.Main("Starting deploy of %s", filesystem.String( "deploy")) 42 | filesystem.Deploy() 43 | } 44 | } 45 | 46 | func (server *Server) DeployDatabases() { 47 | for _, database := range server.Database { 48 | database.ApplyDefaults(server) 49 | Logger.Main("Starting deploy of %s", database.String("deploy")) 50 | database.Deploy() 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /sync/server_deploy_database.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "github.com/webdevops/go-shell" 5 | "fmt" 6 | ) 7 | 8 | func (database *Database) Deploy() { 9 | switch database.GetType() { 10 | case "mysql": 11 | mysql := database.GetMysql() 12 | if mysql.Options.ClearDatabase { 13 | mysql.deployClearDatabase() 14 | } 15 | 16 | mysql.deployStructure() 17 | mysql.deployData() 18 | 19 | case "postgres": 20 | postgres := database.GetPostgres() 21 | fmt.Sprintf(postgres.String("deploy")) 22 | } 23 | } 24 | 25 | // Deploy database structure 26 | func (database *DatabaseMysql) deployClearDatabase() { 27 | 28 | // don't use database which we're trying to drop, instead use "mysql" 29 | db := database.Db 30 | database.Db = "" 31 | 32 | Logger.Step("dropping remote database \"%s\"", db) 33 | dropStmt := fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", db) 34 | dropCmd := shell.Cmd("echo", shell.Quote(dropStmt)).Pipe(database.remoteMysqlCmdBuilder()...) 35 | dropCmd.Run() 36 | 37 | Logger.Step("creating remote database \"%s\"", db) 38 | createStmt := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", db) 39 | createCmd := shell.Cmd("echo", shell.Quote(createStmt)).Pipe(database.remoteMysqlCmdBuilder()...) 40 | createCmd.Run() 41 | 42 | database.Db = db 43 | } 44 | 45 | // Deploy database structure 46 | func (database *DatabaseMysql) deployStructure() { 47 | Logger.Step("deploy database structure") 48 | 49 | // Deploy structure only 50 | dumpCmd := database.localMysqldumpCmdBuilder([]string{"--no-data"}, false) 51 | restoreCmd := database.remoteMysqlCmdBuilderUncompress() 52 | 53 | cmd := shell.Cmd(dumpCmd...).Pipe("gzip", "--stdout").Pipe(restoreCmd...) 54 | cmd.Run() 55 | } 56 | 57 | // Deploy database data 58 | func (database *DatabaseMysql) deployData() { 59 | Logger.Step("deploy database data") 60 | 61 | // Deploy data only 62 | dumpCmd := database.localMysqldumpCmdBuilder([]string{"--no-create-info"}, true) 63 | restoreCmd := database.remoteMysqlCmdBuilderUncompress() 64 | 65 | cmd := shell.Cmd(dumpCmd...).Pipe("gzip", "--stdout").Pipe(restoreCmd...) 66 | cmd.Run() 67 | } 68 | -------------------------------------------------------------------------------- /sync/server_deploy_filesystem.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "os" 5 | "fmt" 6 | "errors" 7 | "strings" 8 | "github.com/webdevops/go-shell" 9 | "github.com/webdevops/go-shell/commandbuilder" 10 | ) 11 | 12 | // General sync 13 | func (filesystem *Filesystem) Deploy() { 14 | switch filesystem.Connection.GetInstance().GetType() { 15 | case "local": 16 | fallthrough 17 | case "ssh": 18 | filesystem.deployRsync() 19 | case "docker": 20 | errors.New("Docker not supported") 21 | } 22 | } 23 | 24 | // Sync filesystem using rsync 25 | func (filesystem *Filesystem) deployRsync() { 26 | connection := filesystem.Connection.GetInstance() 27 | 28 | args := []string{"-rlptD", "--delete-after", "--progress", "--human-readable"} 29 | 30 | // add custom options 31 | if filesystem.Options.Rsync != nil { 32 | args = append(args, filesystem.Options.Rsync.Array()...) 33 | } 34 | 35 | if filesystem.Connection.GetInstance().IsSsh() { 36 | args = append(args, "-e", shell.Quote("ssh " + strings.Join(commandbuilder.ConnectionSshArguments, " "))) 37 | } 38 | 39 | // include filter 40 | if len(filesystem.Filter.Include) > 0 { 41 | includeTempFile := CreateTempfileWithContent(filesystem.Filter.Include...) 42 | args = append(args, fmt.Sprintf("--files-from=%s", includeTempFile.Name())) 43 | 44 | // remove file after run 45 | defer os.Remove(includeTempFile.Name()) 46 | } 47 | 48 | // exclude filter 49 | if len(filesystem.Filter.Exclude) > 0 { 50 | excludeTempFile := CreateTempfileWithContent(filesystem.Filter.Exclude...) 51 | args = append(args, fmt.Sprintf("--exclude-from=%s", excludeTempFile.Name())) 52 | 53 | // remove file after run 54 | defer os.Remove(excludeTempFile.Name()) 55 | } 56 | 57 | // build source and target paths 58 | sourcePath := filesystem.localPath() 59 | targetPath := "" 60 | switch connection.GetType() { 61 | case "ssh": 62 | targetPath = fmt.Sprintf("%s:%s", connection.SshConnectionHostnameString(), filesystem.Path) 63 | case "local": 64 | targetPath = filesystem.Path 65 | } 66 | 67 | // make sure source/target paths are using suffix slash 68 | args = append(args, RsyncPath(sourcePath), RsyncPath(targetPath)) 69 | 70 | cmd := shell.NewCmd("rsync", args...) 71 | cmd.Run() 72 | } 73 | -------------------------------------------------------------------------------- /sync/server_exec.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import "fmt" 4 | 5 | func (server *Server) RunExec(when string) { 6 | defer func() { 7 | recover := recover() 8 | ShellErrorHandler(recover) 9 | }() 10 | 11 | server.Init() 12 | 13 | execList := server.GetExecByWhen(when) 14 | 15 | if len(execList) >= 1 { 16 | Logger.Main("Starting exec mode \"%s\"", when) 17 | 18 | for _, exec := range execList { 19 | Logger.Step("executing >> %s", exec.String(server)) 20 | exec.Execute(server) 21 | } 22 | } 23 | } 24 | 25 | func (server *Server) GetExecByWhen(when string) []Execution { 26 | switch when { 27 | case "startup": 28 | return server.ExecStartup 29 | case "finish": 30 | return server.ExecFinish 31 | default: 32 | panic(fmt.Sprintf("execution list %s is not valid", when)) 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /sync/server_sync.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | func (server *Server) Sync() { 4 | defer func() { 5 | recover := recover() 6 | ShellErrorHandler(recover) 7 | }() 8 | 9 | server.Init() 10 | 11 | if server.runConfiguration.Exec { 12 | server.RunExec("startup") 13 | } 14 | 15 | if server.runConfiguration.Filesystem { 16 | server.SyncFilesystem() 17 | } 18 | 19 | if server.runConfiguration.Database { 20 | server.SyncDatabases() 21 | } 22 | 23 | if server.runConfiguration.Exec { 24 | server.RunExec("finish") 25 | } 26 | 27 | waitGroup.Wait() 28 | } 29 | 30 | func (server *Server) SyncFilesystem() { 31 | for _, filesystem := range server.Filesystem { 32 | filesystem.ApplyDefaults(server) 33 | 34 | if filesystem.Options.GenerateStubs { 35 | Logger.Main("Starting stub generator for %s", filesystem.String( "sync")) 36 | filesystem.SyncStubs() 37 | } else { 38 | Logger.Main("Starting sync of %s", filesystem.String("sync")) 39 | filesystem.Sync() 40 | } 41 | } 42 | } 43 | 44 | func (server *Server) SyncDatabases() { 45 | for _, database := range server.Database { 46 | database.ApplyDefaults(server) 47 | Logger.Main("Starting sync of %s", database.String("sync")) 48 | database.Sync() 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /sync/server_sync_database.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "github.com/webdevops/go-shell" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | ) 9 | 10 | func (database *Database) Sync() { 11 | switch database.GetType() { 12 | case "mysql": 13 | mysql := database.GetMysql() 14 | if mysql.Options.ClearDatabase { 15 | mysql.syncClearDatabase() 16 | } 17 | 18 | mysql.syncStructure() 19 | mysql.syncData() 20 | 21 | case "postgres": 22 | postgres := database.GetPostgres() 23 | 24 | if postgres.Options.ClearDatabase { 25 | postgres.syncClearDatabase() 26 | } 27 | 28 | postgres.syncStructure() 29 | postgres.syncData() 30 | } 31 | } 32 | 33 | //############################################################################# 34 | // Postgres 35 | //############################################################################# 36 | 37 | // Sync database structure 38 | func (database *DatabasePostgres) syncClearDatabase() { 39 | 40 | // don't use database which we're trying to drop, instead use "mysql" 41 | db := database.Local.Db 42 | database.Local.Db = "postgres" 43 | 44 | Logger.Step("dropping local database \"%s\"", db) 45 | dropStmt := fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", db) 46 | dropCmd := shell.Cmd("echo", shell.Quote(dropStmt)).Pipe(database.localPsqlCmdBuilder()...) 47 | dropCmd.Run() 48 | 49 | Logger.Step("creating local database \"%s\"", db) 50 | createStmt := fmt.Sprintf("CREATE DATABASE `%s`", db) 51 | createCmd := shell.Cmd("echo", shell.Quote(createStmt)).Pipe(database.localPsqlCmdBuilder()...) 52 | createCmd.Run() 53 | 54 | database.Local.Db = db 55 | } 56 | 57 | // Sync database structure 58 | func (database *DatabasePostgres) syncStructure() { 59 | Logger.Step("syncing database structure") 60 | 61 | tmpfile, err := ioutil.TempFile("", "dump") 62 | if err != nil { 63 | panic(err) 64 | } 65 | defer os.Remove(tmpfile.Name()) 66 | 67 | // Sync structure only 68 | dumpCmd := database.remotePgdumpCmdBuilder([]string{"--schema-only"}, false) 69 | shell.Cmd(dumpCmd...).Pipe("cat", ">", tmpfile.Name()).Run() 70 | 71 | // Restore structure only 72 | restoreCmd := database.localPsqlCmdBuilder() 73 | shell.Cmd("cat", tmpfile.Name()).Pipe("gunzip", "--stdout").Pipe(restoreCmd...).Run() 74 | } 75 | 76 | 77 | // Sync database data 78 | func (database *DatabasePostgres) syncData() { 79 | Logger.Step("syncing database data") 80 | 81 | tmpfile, err := ioutil.TempFile("", "dump") 82 | if err != nil { 83 | panic(err) 84 | } 85 | defer os.Remove(tmpfile.Name()) 86 | 87 | // Sync structure only 88 | dumpCmd := database.remotePgdumpCmdBuilder([]string{"--data-only"}, true) 89 | shell.Cmd(dumpCmd...).Pipe("cat", ">", tmpfile.Name()).Run() 90 | 91 | // Restore structure only 92 | restoreCmd := database.localPsqlCmdBuilder() 93 | shell.Cmd("cat", tmpfile.Name()).Pipe("gunzip", "--stdout").Pipe(restoreCmd...).Run() 94 | } 95 | 96 | //############################################################################# 97 | // MySQL 98 | //############################################################################# 99 | 100 | // Sync database structure 101 | func (database *DatabaseMysql) syncClearDatabase() { 102 | 103 | // don't use database which we're trying to drop, instead use "mysql" 104 | db := database.Local.Db 105 | database.Local.Db = "" 106 | 107 | Logger.Step("dropping local database \"%s\"", db) 108 | dropStmt := fmt.Sprintf("DROP DATABASE IF EXISTS `%s`", db) 109 | dropCmd := shell.Cmd("echo", shell.Quote(dropStmt)).Pipe(database.localMysqlCmdBuilder()...) 110 | dropCmd.Run() 111 | 112 | Logger.Step("creating local database \"%s\"", db) 113 | createStmt := fmt.Sprintf("CREATE DATABASE IF NOT EXISTS `%s`", db) 114 | createCmd := shell.Cmd("echo", shell.Quote(createStmt)).Pipe(database.localMysqlCmdBuilder()...) 115 | createCmd.Run() 116 | 117 | database.Local.Db = db 118 | } 119 | 120 | // Sync database structure 121 | func (database *DatabaseMysql) syncStructure() { 122 | Logger.Step("syncing database structure") 123 | 124 | tmpfile, err := ioutil.TempFile("", "dump") 125 | if err != nil { 126 | panic(err) 127 | } 128 | defer os.Remove(tmpfile.Name()) 129 | 130 | // Sync structure only 131 | dumpCmd := database.remoteMysqldumpCmdBuilder([]string{"--no-data"}, true) 132 | shell.Cmd(dumpCmd...).Pipe("cat", ">", tmpfile.Name()).Run() 133 | 134 | // Restore structure only 135 | restoreCmd := database.localMysqlCmdBuilder() 136 | shell.Cmd("cat", tmpfile.Name()).Pipe("gunzip", "--stdout").Pipe(restoreCmd...).Run() 137 | } 138 | 139 | // Sync database data 140 | func (database *DatabaseMysql) syncData() { 141 | Logger.Step("syncing database data") 142 | 143 | tmpfile, err := ioutil.TempFile("", "dump") 144 | if err != nil { 145 | panic(err) 146 | } 147 | defer os.Remove(tmpfile.Name()) 148 | 149 | // Sync data only 150 | dumpCmd := database.remoteMysqldumpCmdBuilder([]string{"--no-create-info"}, true) 151 | shell.Cmd(dumpCmd...).Pipe("cat", ">", tmpfile.Name()).Run() 152 | 153 | // Restore data only 154 | restoreCmd := database.localMysqlCmdBuilder() 155 | shell.Cmd("cat", tmpfile.Name()).Pipe("gunzip", "--stdout").Pipe(restoreCmd...).Run() 156 | } 157 | -------------------------------------------------------------------------------- /sync/server_sync_filesystem.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "os" 5 | "fmt" 6 | "errors" 7 | "strings" 8 | "github.com/webdevops/go-shell" 9 | "github.com/webdevops/go-shell/commandbuilder" 10 | ) 11 | 12 | // General sync 13 | func (filesystem *Filesystem) Sync() { 14 | switch filesystem.Connection.GetInstance().GetType() { 15 | case "local": 16 | fallthrough 17 | case "ssh": 18 | filesystem.syncRsync() 19 | case "docker": 20 | errors.New("Docker not supported") 21 | } 22 | } 23 | 24 | // Sync filesystem using rsync 25 | func (filesystem *Filesystem) syncRsync() { 26 | connection := filesystem.Connection.GetInstance() 27 | 28 | args := []string{"-rlptD", "--delete-after", "--progress", "--human-readable"} 29 | 30 | // add custom options 31 | if filesystem.Options.Rsync != nil { 32 | args = append(args, filesystem.Options.Rsync.Array()...) 33 | } 34 | 35 | if filesystem.Connection.GetInstance().IsSsh() { 36 | args = append(args, "-e", shell.Quote("ssh " + strings.Join(commandbuilder.ConnectionSshArguments, " "))) 37 | } 38 | 39 | // include filter 40 | if len(filesystem.Filter.Include) > 0 { 41 | includeTempFile := CreateTempfileWithContent(filesystem.Filter.Include...) 42 | args = append(args, fmt.Sprintf("--files-from=%s", includeTempFile.Name())) 43 | 44 | // remove file after run 45 | defer os.Remove(includeTempFile.Name()) 46 | } 47 | 48 | // exclude filter 49 | if len(filesystem.Filter.Exclude) > 0 { 50 | excludeTempFile := CreateTempfileWithContent(filesystem.Filter.Exclude...) 51 | args = append(args, fmt.Sprintf("--exclude-from=%s", excludeTempFile.Name())) 52 | 53 | // remove file after run 54 | defer os.Remove(excludeTempFile.Name()) 55 | } 56 | 57 | // build source and target paths 58 | sourcePath := "" 59 | switch connection.GetType() { 60 | case "ssh": 61 | sourcePath = fmt.Sprintf("%s:%s", connection.SshConnectionHostnameString(), filesystem.Path) 62 | case "local": 63 | sourcePath = filesystem.Path 64 | } 65 | 66 | targetPath := filesystem.localPath() 67 | 68 | // make sure source/target paths are using suffix slash 69 | args = append(args, RsyncPath(sourcePath), RsyncPath(targetPath)) 70 | 71 | cmd := shell.NewCmd("rsync", args...) 72 | cmd.Run() 73 | } 74 | -------------------------------------------------------------------------------- /sync/server_sync_filesystem_stubs.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "errors" 5 | "bufio" 6 | "strings" 7 | "runtime" 8 | "path" 9 | "path/filepath" 10 | "github.com/webdevops/go-shell" 11 | "github.com/webdevops/go-stubfilegenerator" 12 | "github.com/remeh/sizedwaitgroup" 13 | ) 14 | 15 | // General sync 16 | func (filesystem *Filesystem) SyncStubs() { 17 | switch filesystem.Connection.GetInstance().GetType() { 18 | case "local": 19 | fallthrough 20 | case "ssh": 21 | filesystem.generateStubs() 22 | case "docker": 23 | errors.New("Docker not supported") 24 | } 25 | } 26 | 27 | // Sync filesystem using rsync 28 | func (filesystem *Filesystem) generateStubs() { 29 | connection := filesystem.Connection.GetInstance() 30 | 31 | cmd := shell.Cmd(connection.CommandBuilder("find", filesystem.Path, "-type", "f")...) 32 | output := cmd.Run().Stdout.String() 33 | 34 | removeBasePath := filesystem.Path 35 | localBasePath := filesystem.localPath() 36 | 37 | // build list and filter it by user filter list 38 | fileList := []string{} 39 | scanner := bufio.NewScanner(strings.NewReader(output)) 40 | for scanner.Scan() { 41 | fileList = append(fileList, strings.TrimPrefix(scanner.Text(), removeBasePath)) 42 | } 43 | fileList = filesystem.Filter.ApplyFilter(fileList) 44 | 45 | // generate stubs 46 | swg := sizedwaitgroup.New(runtime.GOMAXPROCS(0) * 10) 47 | stubGen := stubfilegenerator.NewStubGenerator() 48 | for _, filePath := range fileList { 49 | swg.Add() 50 | go func(filePath string, stubGen stubfilegenerator.StubGenerator) { 51 | defer swg.Done() 52 | localPath := path.Join(localBasePath, filePath) 53 | localAbsPath, _ := filepath.Abs(localPath) 54 | 55 | stubGen.TemplateVariables["PATH"] = localPath 56 | stubGen.Generate(localAbsPath) 57 | } (filePath, stubGen.Clone()) 58 | swg.Wait() 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /sync/yaml_commandbuilder_argument.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "github.com/webdevops/go-shell/commandbuilder" 5 | ) 6 | 7 | type YamlCommandBuilderArgument struct { 8 | commandbuilder.Argument 9 | } 10 | 11 | func (yarg *YamlCommandBuilderArgument) UnmarshalYAML(unmarshal func(interface{}) error) error { 12 | var argument commandbuilder.Argument 13 | err := unmarshal(&argument) 14 | if err == nil { 15 | // valid argument 16 | yarg.Argument = argument 17 | } else { 18 | // try to parse as string 19 | var config string 20 | err := unmarshal(&config) 21 | if err != nil { 22 | return err 23 | } 24 | 25 | err = yarg.Set(config) 26 | if err != nil { 27 | return err 28 | } 29 | } 30 | return nil 31 | } 32 | 33 | func (ysa *YamlCommandBuilderArgument) String() string { 34 | return "" 35 | } 36 | -------------------------------------------------------------------------------- /sync/yaml_commandbuilder_connection.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "github.com/mohae/deepcopy" 5 | "github.com/webdevops/go-shell/commandbuilder" 6 | ) 7 | 8 | type YamlCommandBuilderConnection struct { 9 | Type string 10 | Ssh *YamlCommandBuilderArgument 11 | Docker *YamlCommandBuilderArgument 12 | 13 | Environment *map[string]string 14 | Workdir string 15 | 16 | connection *commandbuilder.Connection 17 | } 18 | 19 | // Get (or create) connection instance 20 | // will be cached one it's created 21 | func (yconn *YamlCommandBuilderConnection) GetInstance() *commandbuilder.Connection { 22 | if yconn.connection == nil { 23 | conn := commandbuilder.Connection{} 24 | conn.Type = yconn.Type 25 | 26 | if yconn.Ssh != nil { 27 | conn.Ssh = yconn.Ssh.Argument 28 | } 29 | 30 | if yconn.Docker != nil { 31 | conn.Docker = yconn.Docker.Argument 32 | } 33 | 34 | if yconn.Environment != nil { 35 | conn.Environment.SetMap(*yconn.Environment) 36 | } 37 | 38 | if yconn.Workdir != "" { 39 | conn.Workdir = yconn.Workdir 40 | } 41 | 42 | yconn.connection = &conn 43 | } 44 | 45 | return yconn.connection 46 | } 47 | 48 | // Checks if connection is empty 49 | func (yconn *YamlCommandBuilderConnection) IsEmpty() (status bool) { 50 | status = false 51 | if yconn.Type != "" { return } 52 | if yconn.Ssh != nil { return } 53 | if yconn.Docker != nil { return } 54 | if yconn.Environment != nil { return } 55 | 56 | return true 57 | } 58 | 59 | // Clone yaml connection (without shell connection instance) 60 | func (yconn *YamlCommandBuilderConnection) Clone() (conn *YamlCommandBuilderConnection) { 61 | conn = deepcopy.Copy(yconn).(*YamlCommandBuilderConnection) 62 | conn.connection = nil 63 | return conn 64 | } 65 | -------------------------------------------------------------------------------- /sync/yaml_string_array.go: -------------------------------------------------------------------------------- 1 | package sync 2 | 3 | import ( 4 | "strings" 5 | ) 6 | 7 | type YamlStringArray struct { 8 | Multi []string 9 | Single string 10 | } 11 | 12 | func (ysa *YamlStringArray) UnmarshalYAML(unmarshal func(interface{}) error) error { 13 | var multi []string 14 | err := unmarshal(&multi) 15 | if err == nil { 16 | ysa.Multi = multi 17 | } else { 18 | var single string 19 | err := unmarshal(&single) 20 | if err != nil { 21 | return err 22 | } 23 | ysa.Single = single 24 | } 25 | return nil 26 | } 27 | 28 | func (ysa *YamlStringArray) String() string { 29 | return ysa.ToString(";") 30 | } 31 | 32 | func (ysa *YamlStringArray) ToString(sep string) string { 33 | if len(ysa.Multi) >= 1 { 34 | return strings.Join(ysa.Multi, sep) 35 | } else { 36 | return ysa.Single 37 | } 38 | } 39 | 40 | func (ysa *YamlStringArray) Array() (command []string) { 41 | if len(ysa.Multi) >= 1 { 42 | command = ysa.Multi 43 | } else if ysa.Single != "" { 44 | command = []string{ysa.Single} 45 | } 46 | 47 | return 48 | } 49 | --------------------------------------------------------------------------------