├── CNAME ├── .gitignore ├── _config.yml ├── pflag ├── patch.diff ├── README.md └── string_slice.go ├── main.go ├── common ├── version.go ├── strutils_test.go ├── tprintf.go ├── strutils.go ├── version_test.go ├── checks.go └── fileutil.go ├── mkreadme ├── build_readme.sh ├── api_template.md └── make_readme.go ├── docs ├── env_variables.md ├── development_plan.md └── features.md ├── abbreviations ├── sample_abbreviations.txt └── abbreviations.go ├── cmd ├── multiple.go ├── versions.go ├── root.go ├── usage.go ├── deploy.go ├── unpack.go ├── replication.go ├── defaults.go ├── tree.go ├── global.go ├── sandboxes.go ├── admin.go ├── delete.go └── single.go ├── test ├── sort_versions.go ├── docker-test.sh ├── mock │ ├── set-mock.sh │ ├── defaults-change.sh │ └── port-clash.sh └── all_tests.sh ├── sandbox ├── group_templates.go ├── multi_templates.go ├── multiple.go ├── multi-source-replication.go └── group_replication.go ├── concurrent └── concurrent.go ├── defaults └── catalog.go ├── unpack └── unpack.go ├── Changelog └── LICENSE /CNAME: -------------------------------------------------------------------------------- 1 | www.dbdeployer.com -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .idea/ 3 | 4 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | theme: jekyll-theme-minimal -------------------------------------------------------------------------------- /pflag/patch.diff: -------------------------------------------------------------------------------- 1 | diff --git a/string_slice.go b/string_slice.go 2 | index 05eee75..05888e2 100644 3 | --- a/string_slice.go 4 | +++ b/string_slice.go 5 | @@ -25,12 +25,16 @@ func readAsCSV(val string) ([]string, error) { 6 | } 7 | stringReader := strings.NewReader(val) 8 | csvReader := csv.NewReader(stringReader) 9 | + // Change CSV reader field separator 10 | + csvReader.Comma = ';' 11 | return csvReader.Read() 12 | } 13 | 14 | func writeAsCSV(vals []string) (string, error) { 15 | b := &bytes.Buffer{} 16 | w := csv.NewWriter(b) 17 | + // Change CSV writer field separator 18 | + w.Comma=';' 19 | err := w.Write(vals) 20 | if err != nil { 21 | return "", err 22 | -------------------------------------------------------------------------------- /pflag/README.md: -------------------------------------------------------------------------------- 1 | # Compiling dbdeployer 2 | If you want to compile dbdeployer, you must also modify this file, which is a dependency of the cobra package. 3 | 4 | github.com/spf13/pflag/string_slice.go 5 | 6 | The reason is that encoder/csv uses a literal value to initialize the field delimiter in readers and writers, and the depending package does not offer an interface to modify such value (comma ','). 7 | 8 | The problem that this creates is related to multi-string flags in dbdeployer. By default, due to cobra using the encoding/csv package, if the value contains a comma, it is assumed to be a separator between values. This may not be the case with dbdeployer, where we can pass option-file directives (which sometimes contain commas) and SQL commands (which **often** contain commas.) 9 | 10 | Whith this change, the default field separator is a semicolon. 11 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "github.com/datacharmer/dbdeployer/abbreviations" 20 | "github.com/datacharmer/dbdeployer/cmd" 21 | ) 22 | 23 | func main() { 24 | abbreviations.LoadAbbreviations() 25 | cmd.Execute() 26 | } 27 | -------------------------------------------------------------------------------- /common/version.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | var VersionDef string = "1.3.0" // 2018-04-15 19 | 20 | // Compatible version is the version used to mark compatible archives (templates, configuration). 21 | // It is usually major.minor.0, except when we are at version 0.x, when 22 | // every revision may bring incompatibility 23 | var CompatibleVersion string = "1.3.0" // 2018-04-15 24 | -------------------------------------------------------------------------------- /mkreadme/build_readme.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd $(dirname $0) 4 | # Need to recompile before generating documentation 5 | # because we want the latest version to be included 6 | for tool_name in make_readme 7 | do 8 | go build $tool_name.go 9 | if [ "$?" != "0" ] 10 | then 11 | exit 12 | fi 13 | if [ ! -x $tool_name ] 14 | then 15 | echo "executable $tool_name not found" 16 | exit 1 17 | fi 18 | done 19 | 20 | if [ ! -f readme_template.md ] 21 | then 22 | echo "readme_template.md not found" 23 | exit 1 24 | fi 25 | if [ -f $HOME/.dbdeployer/config.json ] 26 | then 27 | dbdeployer defaults reset 28 | fi 29 | # Build README file 30 | ./make_readme < readme_template.md > README.md 31 | #./make_readme < api_template.md > API.md 32 | 33 | # Build API reference 34 | dbdeployer-docs tree --api | ./make_readme > API.md 35 | dbdeployer defaults reset 36 | 37 | # Build completion file 38 | completion_file=dbdeployer_completion.sh 39 | if [ -f $completion_file ] 40 | then 41 | rm -f $completion_file 42 | fi 43 | 44 | dbdeployer-docs tree --bash-completion 45 | if [ ! -f $completion_file ] 46 | then 47 | echo "# An error occurred: completion file '$completion_file' was not created" 48 | exit 1 49 | fi 50 | 51 | echo "# $PWD" 52 | ls -lhotr 53 | 54 | -------------------------------------------------------------------------------- /docs/env_variables.md: -------------------------------------------------------------------------------- 1 | # Environmental variables 2 | 3 | The following variables are used by dbdeployer. 4 | 5 | ## Abbreviations 6 | 7 | * ``DEBUG_ABBR`` Enables debug information for abbreviations engine. 8 | * ``SKIP_ABBR`` Disables the abbreviations engine. 9 | * ``SILENT_ABBR`` Disables the verbosity with abbreviations. 10 | * ``DBDEPLOYER_ABBR_FILE`` Changes the abbreviations file 11 | 12 | ## Concurrency 13 | 14 | * ``RUN_CONCURRENTLY`` Run operations concurrently (multiple sandboxes and deletions) 15 | * ``DEBUG_CONCURRENCY`` Enables debug information for concurrent operations 16 | * ``VERBOSE_CONCURRENCY`` Gives more info during concurrency. 17 | 18 | ## Catalog 19 | 20 | * ``SKIP_DBDEPLOYER_CATALOG`` Stops using the centralized JSON catalog. 21 | 22 | ## Ports management 23 | 24 | * ``SHOW_CHANGED_PORTS`` will show which ports were changed to avoid clashes. 25 | 26 | ## Sandbox deployment 27 | 28 | * ``HOME`` Used to initialize sandboxes components: ``SANDBOX_HOME`` and ``SANDBOX_BINARY`` depend on this one. 29 | * ``TMPDIR`` Used to define where the socket file will be located. 30 | * ``USER`` Used to define which user will be used to run the database server. 31 | * ``SANDBOX_BINARY`` Where the MySQL unpacked binaries are stored. 32 | * ``SANDBOX_HOME`` Where the sandboxes are deployed. 33 | * ``INIT_OPTIONS`` Options to be added to the initialization script command line. 34 | * ``MY_CNF_OPTIONS`` Options to be added to the sandbox configuration file. 35 | * ``MY_CNF_FILE`` Alternate file to be used as source for the sandbox my.cnf. 36 | -------------------------------------------------------------------------------- /abbreviations/sample_abbreviations.txt: -------------------------------------------------------------------------------- 1 | # dbdeployer looks for a file "abbreviations.txt" and treats every line 2 | # as an abbreviation followed by its replacement. 3 | # Then, it looks at the command line arguments. 4 | # If an argument matches an abbreviation, it will be replaced by the replacement items. 5 | # For example, the file contains this line: 6 | # sbs sandboxes 7 | # 8 | # when the user types "dbdeployer sbs", it will be replaced with "dbdeployer sandboxes" 9 | # 10 | # A more interesting example: 11 | # groupr replication --topology=group 12 | # 13 | # Here, a command "dbdeployer groupr 8.0.4" becomes "dbdeployer deploy replication --topology=group 8.0.4" 14 | # 15 | # It is also possible to set variables in the replacement. 16 | # sbdef --sandbox-directory={{.sb}} --port={{.port}} 17 | # 18 | # To use this abbreviation, we need to provide the values for 'sb' and 'port' 19 | # dbdeployer sbdef:port=9000,sb=mysandbox deploy single 8.0.4 20 | # it will become "dbdeployer --sandbox-directory=mysandbox --port=9000 deploy single 8.0.4 21 | # ---------------------------------------------------------------------------- 22 | 23 | # use as "dbdeployer group 5.7.21" 24 | groupr deploy replication --topology=group 25 | 26 | groupsp deploy replication --topology=group --single-primary 27 | 28 | tl defaults templates list 29 | 30 | # Use dbdeployer sbdef:port=XXX,sb=YYYYYY 31 | sbdef --sandbox-directory={{.sb}} --port={{.port}} 32 | 33 | # Use dbdeployer msbdef:port=XXX,sb=YYYYYY 34 | msbdef --sandbox-directory={{.sb}} --base-port={{.port}} 35 | 36 | sbs sandboxes 37 | 38 | -------------------------------------------------------------------------------- /mkreadme/api_template.md: -------------------------------------------------------------------------------- 1 | This is the list of commands and modifiers available for dbdeployer 2 | 3 | {{dbdeployer --version}} 4 | # main 5 | {{dbdeployer -h }} 6 | 7 | ## admin 8 | 9 | {{dbdeployer admin -h}} 10 | {{dbdeployer admin lock -h}} 11 | {{dbdeployer admin unlock -h}} 12 | 13 | ## defaults 14 | {{dbdeployer defaults -h}} 15 | {{dbdeployer defaults export -h}} 16 | {{dbdeployer defaults load -h}} 17 | {{dbdeployer defaults reset -h}} 18 | {{dbdeployer defaults show -h}} 19 | {{dbdeployer defaults store -h}} 20 | {{dbdeployer defaults update -h}} 21 | {{dbdeployer defaults templates -h}} 22 | 23 | ### defaults-templates 24 | {{dbdeployer defaults templates describe -h}} 25 | {{dbdeployer defaults templates export -h}} 26 | {{dbdeployer defaults templates import -h}} 27 | {{dbdeployer defaults templates list -h}} 28 | {{dbdeployer defaults templates reset -h}} 29 | {{dbdeployer defaults templates show -h}} 30 | 31 | ## delete 32 | {{dbdeployer delete -h}} 33 | 34 | ## deploy 35 | {{dbdeployer deploy -h}} 36 | {{dbdeployer deploy single -h}} 37 | {{dbdeployer deploy multiple -h}} 38 | {{dbdeployer deploy replication -h}} 39 | 40 | ## global 41 | {{dbdeployer global -h}} 42 | {{dbdeployer global restart -h}} 43 | {{dbdeployer global start -h}} 44 | {{dbdeployer global status -h}} 45 | {{dbdeployer global stop -h}} 46 | {{dbdeployer global test -h}} 47 | {{dbdeployer global test-replication -h}} 48 | {{dbdeployer global use -h}} 49 | 50 | ## sandboxes 51 | {{dbdeployer sandboxes -h}} 52 | 53 | ## unpack 54 | {{dbdeployer unpack -h}} 55 | 56 | ## usage 57 | {{dbdeployer usage -h}} 58 | 59 | ## versions 60 | {{dbdeployer versions -h}} 61 | 62 | -------------------------------------------------------------------------------- /cmd/multiple.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | //"fmt" 20 | 21 | "github.com/datacharmer/dbdeployer/common" 22 | "github.com/datacharmer/dbdeployer/sandbox" 23 | "github.com/spf13/cobra" 24 | ) 25 | 26 | func MultipleSandbox(cmd *cobra.Command, args []string) { 27 | var sd sandbox.SandboxDef 28 | common.CheckOrigin(args) 29 | flags := cmd.Flags() 30 | sd = FillSdef(cmd, args) 31 | nodes, _ := flags.GetInt("nodes") 32 | sd.SBType = "multiple" 33 | sandbox.CreateMultipleSandbox(sd, args[0], nodes) 34 | } 35 | 36 | var multipleCmd = &cobra.Command{ 37 | Use: "multiple MySQL-Version", 38 | Args: cobra.ExactArgs(1), 39 | Short: "create multiple sandbox", 40 | Long: `Creates several sandboxes of the same version, 41 | without any replication relationship. 42 | For this command to work, there must be a directory $HOME/opt/mysql/5.7.21, containing 43 | the binary files from mysql-5.7.21-$YOUR_OS-x86_64.tar.gz 44 | Use the "unpack" command to get the tarball into the right directory. 45 | `, 46 | Run: MultipleSandbox, 47 | Example: ` 48 | $ dbdeployer deploy multiple 5.7.21 49 | `, 50 | } 51 | 52 | func init() { 53 | deployCmd.AddCommand(multipleCmd) 54 | multipleCmd.PersistentFlags().IntP("nodes", "n", 3, "How many nodes will be installed") 55 | } 56 | -------------------------------------------------------------------------------- /test/sort_versions.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "bufio" 20 | "fmt" 21 | "github.com/datacharmer/dbdeployer/common" 22 | "os" 23 | "sort" 24 | ) 25 | 26 | /* 27 | This utility reads a list of versions (format x.x.xx) 28 | and sorts them in numerical order, taking into account 29 | all three fields, making sure that 5.7.9 comes before 5.7.10. 30 | */ 31 | func main() { 32 | 33 | scanner := bufio.NewScanner(os.Stdin) 34 | 35 | type version_list struct { 36 | text string 37 | mmr []int 38 | } 39 | var vlist []version_list 40 | for scanner.Scan() { 41 | line := scanner.Text() 42 | vl := common.VersionToList(line) 43 | rec := version_list{ 44 | text: line, 45 | mmr: vl, 46 | } 47 | if vl[0] > 0 { 48 | vlist = append(vlist, rec) 49 | } 50 | } 51 | 52 | if err := scanner.Err(); err != nil { 53 | fmt.Fprintln(os.Stderr, "error:", err) 54 | os.Exit(1) 55 | } 56 | sort.Slice(vlist, func(i, j int) bool { 57 | return vlist[i].mmr[0] < vlist[j].mmr[0] || 58 | (vlist[i].mmr[0] == vlist[j].mmr[0] && vlist[i].mmr[1] < vlist[j].mmr[1]) || 59 | (vlist[i].mmr[0] == vlist[j].mmr[0] && vlist[i].mmr[1] == vlist[j].mmr[1] && vlist[i].mmr[2] < vlist[j].mmr[2]) 60 | }) 61 | for _, v := range vlist { 62 | fmt.Printf("%s\n", v.text) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /docs/development_plan.md: -------------------------------------------------------------------------------- 1 | # DBdeployer 2 | 3 | DBdeployer is a tool that deploys MySQL database servers easily. 4 | 5 | # Overview 6 | 7 | ## Goals: 8 | * Replace [MySQL-Sandbox](https://github.com/datacharmer/mysql-sandbox) 9 | * Allow easy deployment of MySQL database instances for testing. 10 | * Use a simple and intuitive syntax 11 | 12 | ## Main features: 13 | * Deploy a single or composite instance of MySQL 14 | * Easily administer the instance with auto generated tools 15 | * Easy installation (Simple deployment of a binary file without dependencies) 16 | 17 | ## Installation sources: 18 | * an expanded tarball from $HOME/opt/mysql/x.xx.xx (default) 19 | * a tarball (a .tar.gz file: requires additional command) 20 | 21 | ## Common custom scripts for each deployment: 22 | * start [options] 23 | * restart [options] 24 | * stop 25 | * status 26 | * clear 27 | * send\_kill 28 | * use 29 | * test\_sb 30 | 31 | ## Common configuration features: 32 | * Pass some options on the command line 33 | * Allow using a my.cnf as template 34 | * Produce automatic or custom server-ids 35 | * Produce custom server UUIDs 36 | * keep an easily accessible configuration file to inspect the instance 37 | 38 | ## Common administrative tasks: 39 | * Update configuration files with new/different options 40 | * start or restart an instance with different options 41 | * test basic instance functionality 42 | * test replication 43 | * make a single instance ready for replication (as master) 44 | 45 | Future: 46 | * make an instance a slave of an existing instance 47 | * move, copy, or delete an instance 48 | 49 | # Implementation 50 | 51 | ## CLI Interface: 52 | * It will be a command suite (similar to git or docker, with main commands and contextual options) 53 | * The help is a drill-down (general commands first, and help on single commands) 54 | 55 | ## Implementation requirements: 56 | * Unix operating system (Linux or MacOS tested so far) 57 | * All features are accessible from a single binary file 58 | * All features are implemented as easily extendable 59 | * The only hard requirement is that the OS is ready to run a MySQL server 60 | 61 | See features.md for a detailed list of implemented features. 62 | -------------------------------------------------------------------------------- /common/strutils_test.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | import ( 19 | "os" 20 | "testing" 21 | ) 22 | 23 | type path_info struct { 24 | value string 25 | env_var string 26 | expected string 27 | } 28 | 29 | func TestReplaceLiteralHome(t *testing.T) { 30 | os.Setenv("HOME", "/home/Groucho") 31 | os.Setenv("PWD", "/var/lib/MarxBrothers") 32 | var paths = []path_info{ 33 | {"/home/Groucho/", "HOME", "$HOME/"}, 34 | {"/home/Groucho/path1/path2", "HOME", "$HOME/path1/path2"}, 35 | {"/home/Harpo/path1/path2", "HOME", "/home/Harpo/path1/path2"}, 36 | {"/var/lib/MarxBrothers/path1/path2", "PWD", "$PWD/path1/path2"}, 37 | {"/var/lib/MarxCousins/path1/path2", "PWD", "/var/lib/MarxCousins/path1/path2"}, 38 | } 39 | for _, p := range paths { 40 | value := p.value 41 | env_var := p.env_var 42 | expected := p.expected 43 | canary := ReplaceLiteralEnvVar(value, env_var) 44 | if expected == canary { 45 | t.Logf("ok %-35s %-10s =--> %-25s\n", value, "(" + env_var + ")", expected) 46 | } else { 47 | t.Logf("NOT OK %-35s %-10s =--> %-25s\n", value, "(" + env_var + ")", expected) 48 | t.Fail() 49 | } 50 | } 51 | for _, p := range paths { 52 | value := p.expected 53 | env_var := p.env_var 54 | expected := p.value 55 | canary := ReplaceEnvVar(value, env_var) 56 | if expected == canary { 57 | t.Logf("ok %-35s %-10s --=> %-25s\n", value, "(" + env_var + ")", expected) 58 | } else { 59 | t.Logf("NOT OK %-35s %-10s --=> %-25s\n", value, "(" + env_var + ")", expected) 60 | t.Fail() 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /common/tprintf.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | import ( 19 | "bytes" 20 | "regexp" 21 | "text/template" 22 | "time" 23 | ) 24 | 25 | // Smap defines the map of variable types, for brevity 26 | type Smap map[string]interface{} 27 | 28 | // Given a multi-line string, this function removes leading 29 | // spaces from every line. 30 | // It also removes the first line, if it is empty 31 | func TrimmedLines(s string) string { 32 | // matches the start of the text followed by an EOL 33 | re := regexp.MustCompile(`(?m)\A\s*$`) 34 | s = re.ReplaceAllString(s, "") 35 | 36 | re = regexp.MustCompile(`(?m)^\t\t`) 37 | s = re.ReplaceAllString(s, "") 38 | return s 39 | /* 40 | 41 | // matches the start of every line, followed by any spaces 42 | re = regexp.MustCompile(`(?m)^\s*`) 43 | return re.ReplaceAllString(s, "") 44 | */ 45 | } 46 | 47 | // Tprintf passed template string is formatted using its operands and returns the resulting string. 48 | // Spaces are added between operands when neither is a string. 49 | // Based on code from https://play.golang.org/p/COHKlB2RML 50 | func Tprintf(tmpl string, data Smap) string { 51 | 52 | // Adds timestamp and version info 53 | timestamp := time.Now() 54 | _, time_stamp_exists := data["DateTime"] 55 | _, version_exists := data["AppVersion"] 56 | if !time_stamp_exists { 57 | data["DateTime"] = timestamp.Format(time.UnixDate) 58 | } 59 | if !version_exists { 60 | data["AppVersion"] = VersionDef 61 | } 62 | // Creates a template 63 | t := template.Must(template.New("tmp").Parse(tmpl)) 64 | buf := &bytes.Buffer{} 65 | 66 | // If an error occurs, returns an empty string 67 | if err := t.Execute(buf, data); err != nil { 68 | return "" 69 | } 70 | 71 | // Returns the populated template 72 | return buf.String() 73 | } 74 | -------------------------------------------------------------------------------- /cmd/versions.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "log" 21 | //"os" 22 | "github.com/datacharmer/dbdeployer/common" 23 | "github.com/spf13/cobra" 24 | "io/ioutil" 25 | ) 26 | 27 | // Shows the MySQL versions available in $SANDBOX_BINARY 28 | // (default $HOME/opt/mysql) 29 | func ShowVersions(cmd *cobra.Command, args []string) { 30 | flags := cmd.Flags() 31 | Basedir, _ := flags.GetString("sandbox-binary") 32 | files, err := ioutil.ReadDir(Basedir) 33 | if err != nil { 34 | log.Fatal(err) 35 | } 36 | var dirs []string 37 | for _, f := range files { 38 | fname := f.Name() 39 | //fmt.Printf("%#v\n", f) 40 | //os.Exit(0) 41 | fmode := f.Mode() 42 | //fmt.Printf("%#v\n", fmode) 43 | if fmode.IsDir() { 44 | //fmt.Println(fname) 45 | mysqld := Basedir + "/" + fname + "/bin/mysqld" 46 | if common.FileExists(mysqld) { 47 | dirs = append(dirs, fname) 48 | } 49 | } 50 | } 51 | max_width := 80 52 | max_len := 0 53 | for _, dir := range dirs { 54 | if len(dir) > max_len { 55 | max_len = len(dir) 56 | } 57 | } 58 | fmt.Printf("Basedir: %s\n", Basedir) 59 | columns := int(max_width / (max_len + 2)) 60 | mask := fmt.Sprintf("%%-%ds", max_len+2) 61 | count := 0 62 | for _, dir := range dirs { 63 | fmt.Printf(mask, dir) 64 | count += 1 65 | if count > columns { 66 | count = 0 67 | fmt.Println("") 68 | } 69 | } 70 | fmt.Println("") 71 | } 72 | 73 | // versionsCmd represents the versions command 74 | var versionsCmd = &cobra.Command{ 75 | Use: "versions", 76 | Aliases: []string{"available"}, 77 | Short: "List available versions", 78 | Long: ``, 79 | Run: ShowVersions, 80 | } 81 | 82 | func init() { 83 | rootCmd.AddCommand(versionsCmd) 84 | 85 | // versionsCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") 86 | } 87 | -------------------------------------------------------------------------------- /test/docker-test.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # DBDeployer - The MySQL Sandbox 3 | # Copyright © 2006-2018 Giuseppe Maxia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # This script tests dbdeployer with all available versions inside a docker 18 | # container. 19 | # This setup is useful for testing also template and defaults export/import 20 | # which could be intrusive in a regular environment. 21 | 22 | version=$1 23 | test_command=$2 24 | 25 | if [ -z "$version" ] 26 | then 27 | echo "version required" 28 | exit 1 29 | fi 30 | 31 | executable="dbdeployer-${version}.linux" 32 | if [ ! -x $executable ] 33 | then 34 | echo "executable not found" 35 | exit 1 36 | fi 37 | 38 | if [ ! -d "./test" ] 39 | then 40 | echo "directory ./test not found" 41 | exit 1 42 | fi 43 | 44 | container_name=dbtest 45 | #if [ "$(uname)" == "Darwin" ] 46 | #then 47 | # # This name identifies the container as running in Docker for mac, 48 | # # and will allow the test script to tune operations accordingly. 49 | # if [ -z "$INCLUDE_56" ] 50 | # then 51 | # container_name=dbtestmac 52 | # fi 53 | #fi 54 | 55 | exists=$(docker ps -a | grep $container_name ) 56 | if [ -n "$exists" ] 57 | then 58 | docker rm -v -f $container_name 59 | fi 60 | 61 | [ -n "$INTERACTIVE" ] && DOCKER_OPTIONS="$DOCKER_OPTIONS -e INTERACTIVE=1" 62 | [ -n "$RUN_CONCURRENTLY" ] && DOCKER_OPTIONS="$DOCKER_OPTIONS -e RUN_CONCURRENTLY=1" 63 | [ -n "$VERBOSE_CONCURRENCY" ] && DOCKER_OPTIONS="$DOCKER_OPTIONS -e VERBOSE_CONCURRENCY=1" 64 | [ -n "$EXIT_ON_FAILURE" ] && DOCKER_OPTIONS="$DOCKER_OPTIONS -e EXIT_ON_FAILURE=1" 65 | 66 | [ -z "$test_command" ] && test_command="./test/functional-test.sh" 67 | 68 | if [ "$test_command" != "bash" ] 69 | then 70 | test_command="bash -c $test_command" 71 | fi 72 | 73 | 74 | (set -x 75 | docker run -ti \ 76 | -v $PWD/$executable:/usr/bin/dbdeployer \ 77 | -v $PWD/test:/home/msandbox/test \ 78 | --name $container_name \ 79 | --hostname $container_name $DOCKER_OPTIONS \ 80 | datacharmer/mysql-sb-full $test_command 81 | ) 82 | 83 | # datacharmer/mysql-sb-full bash -c "./test/functional-test.sh" 84 | # datacharmer/mysql-sb-full bash 85 | 86 | -------------------------------------------------------------------------------- /sandbox/group_templates.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package sandbox 17 | 18 | // Templates for group replication 19 | 20 | var ( 21 | init_nodes_template string = `#!/bin/sh 22 | {{.Copyright}} 23 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 24 | multi_sb={{.SandboxDir}} 25 | # workaround for Bug#89959 26 | {{range .Nodes}} 27 | {{.SandboxDir}}/{{.NodeLabel}}{{.Node}}/use -h {{.MasterIp}} -u {{.RplUser}} -p{{.RplPassword}} -e 'set @a=1' 28 | {{end}} 29 | [ -z "$SLEEP_TIME" ] && SLEEP_TIME=1 30 | {{range .Nodes}} 31 | user_cmd='reset master;' 32 | user_cmd="$user_cmd CHANGE MASTER TO MASTER_USER='rsandbox', MASTER_PASSWORD='rsandbox' {{.ChangeMasterExtra}} FOR CHANNEL 'group_replication_recovery';" 33 | echo "# Node {{.Node}} # $user_cmd" 34 | $multi_sb/{{.NodeLabel}}{{.Node}}/use -u root -e "$user_cmd" 35 | {{end}} 36 | echo "" 37 | 38 | BEFORE_START_CMD="SET GLOBAL group_replication_bootstrap_group=ON;" 39 | START_CMD="START GROUP_REPLICATION;" 40 | AFTER_START_CMD="SET GLOBAL group_replication_bootstrap_group=OFF;" 41 | echo "# Node 1 # $BEFORE_START_CMD" 42 | $multi_sb/n1 -e "$BEFORE_START_CMD" 43 | {{ range .Nodes}} 44 | echo "# Node {{.Node}} # $START_CMD" 45 | $multi_sb/n{{.Node}} -e "$START_CMD" 46 | sleep $SLEEP_TIME 47 | {{end}} 48 | echo "# Node 1 # $AFTER_START_CMD" 49 | $multi_sb/n1 -e "$AFTER_START_CMD" 50 | $multi_sb/check_nodes 51 | ` 52 | check_nodes_template string = `#!/bin/sh 53 | {{.Copyright}} 54 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 55 | multi_sb={{.SandboxDir}} 56 | [ -z "$SLEEP_TIME" ] && SLEEP_TIME=1 57 | 58 | CHECK_NODE="select * from performance_schema.replication_group_members" 59 | {{ range .Nodes}} 60 | echo "# Node {{.Node}} # $CHECK_NODE" 61 | $multi_sb/{{.NodeLabel}}{{.Node}}/use -t -e "$CHECK_NODE" 62 | sleep $SLEEP_TIME 63 | {{end}} 64 | ` 65 | GroupTemplates = TemplateCollection{ 66 | "init_nodes_template": TemplateDesc{ 67 | Description: "Initialize group replication after deployment", 68 | Notes: "", 69 | Contents: init_nodes_template, 70 | }, 71 | "check_nodes_template": TemplateDesc{ 72 | Description: "Checks the status of group replication", 73 | Notes: "", 74 | Contents: check_nodes_template, 75 | }, 76 | } 77 | ) 78 | -------------------------------------------------------------------------------- /common/strutils.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | "regexp" 22 | ) 23 | 24 | // Given a path starting at the HOME directory 25 | // returns a string where the literal value for $HOME 26 | // is replaced by the string "$HOME" 27 | func ReplaceLiteralHome( path string) string { 28 | // home := os.Getenv("HOME") 29 | // re := regexp.MustCompile(`^` + home) 30 | // return re.ReplaceAllString(path, "$$HOME") 31 | return ReplaceLiteralEnvVar(path, "HOME") 32 | } 33 | 34 | func ReplaceLiteralEnvVar(name string, env_var string) string { 35 | value := os.Getenv(env_var) 36 | re := regexp.MustCompile(value) 37 | return re.ReplaceAllString(name, "$$" + env_var) 38 | } 39 | 40 | func ReplaceEnvVar(name string, env_var string) string { 41 | value := os.Getenv(env_var) 42 | re := regexp.MustCompile(`\$` + env_var+ `\b`) 43 | return re.ReplaceAllString(name, value) 44 | } 45 | 46 | // Given a path with the variable "$HOME" at the start, 47 | // returns a string with the value of HOME expanded 48 | func ReplaceHomeVar(path string) string { 49 | // home := os.Getenv("HOME") 50 | // re := regexp.MustCompile(`^\$HOME\b`) 51 | //return re.ReplaceAllString(path, home) 52 | return ReplaceEnvVar(path, "HOME") 53 | } 54 | 55 | func MakeCustomizedUuid(port , node_num int ) string { 56 | re_digit := regexp.MustCompile(`\d`) 57 | group1 := fmt.Sprintf("%08d", port) 58 | group2 := fmt.Sprintf("%04d-%04d-%04d", node_num, node_num, node_num) 59 | group3 := fmt.Sprintf("%012d", port) 60 | // 12345678 1234 1234 1234 123456789012 61 | // new_uuid="00000000-0000-0000-0000-000000000000" 62 | switch { 63 | case node_num > 0 && node_num <= 9: 64 | group2 = re_digit.ReplaceAllString(group2, fmt.Sprintf("%d", node_num)) 65 | group3 = re_digit.ReplaceAllString(group3, fmt.Sprintf("%d", node_num)) 66 | // Number greater than 10 make little sense for this purpose. 67 | // But we keep the rule so that a valid UUID will be formatted in any case. 68 | case node_num >= 10000 && node_num <= 99999: 69 | group2 = fmt.Sprintf("%04d-%04d-%04d", 0, int(node_num / 10000), node_num - 10000 * int(node_num / 10000)) 70 | case node_num >= 100000: 71 | group2 = fmt.Sprintf("%04d-%04d-%04d", int(node_num / 10000), 0, 0) 72 | case node_num >= 1000000: 73 | fmt.Printf("Node num out of boundaries: %d\n",node_num) 74 | os.Exit(1) 75 | } 76 | return fmt.Sprintf("%s-%s-%s", group1, group2, group3) 77 | } 78 | 79 | func Includes(main_string, contained string) bool { 80 | re := regexp.MustCompile(contained) 81 | return re.MatchString(main_string) 82 | 83 | } 84 | -------------------------------------------------------------------------------- /cmd/root.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | 22 | "github.com/datacharmer/dbdeployer/common" 23 | "github.com/datacharmer/dbdeployer/defaults" 24 | "github.com/spf13/cobra" 25 | //"github.com/davecgh/go-spew/spew" 26 | ) 27 | 28 | // rootCmd represents the base command when called without any subcommands 29 | var rootCmd = &cobra.Command{ 30 | Use: "dbdeployer", 31 | Short: "Installs multiple MySQL servers on the same host", 32 | Long: `dbdeployer makes MySQL server installation an easy task. 33 | Runs single, multiple, and replicated sandboxes.`, 34 | // Uncomment the following line if your bare application 35 | // has an action associated with it: 36 | // Run: func(cmd *cobra.Command, args []string) { }, 37 | Version: common.VersionDef, 38 | } 39 | 40 | // Execute adds all child commands to the root command and sets flags appropriately. 41 | // This is called by main.main(). It only needs to happen once to the rootCmd. 42 | func Execute() { 43 | if err := rootCmd.Execute(); err != nil { 44 | fmt.Println(err) 45 | os.Exit(1) 46 | } 47 | } 48 | 49 | func set_pflag(cmd *cobra.Command, key string, abbr string, env_var string, default_var string, help_str string, is_slice bool) { 50 | var default_value string 51 | if env_var != "" { 52 | default_value = os.Getenv(env_var) 53 | } 54 | if default_value == "" { 55 | default_value = default_var 56 | } 57 | if is_slice { 58 | cmd.PersistentFlags().StringSliceP(key, abbr, []string{default_value}, help_str) 59 | } else { 60 | cmd.PersistentFlags().StringP(key, abbr, default_value, help_str) 61 | } 62 | } 63 | 64 | func checkDefaultsFile() { 65 | flags := rootCmd.Flags() 66 | defaults.CustomConfigurationFile, _ = flags.GetString("config") 67 | if defaults.CustomConfigurationFile != defaults.ConfigurationFile { 68 | if common.FileExists(defaults.CustomConfigurationFile) { 69 | defaults.ConfigurationFile = defaults.CustomConfigurationFile 70 | } else { 71 | fmt.Printf("*** File %s not found\n", defaults.CustomConfigurationFile) 72 | os.Exit(1) 73 | } 74 | } 75 | defaults.LoadConfiguration() 76 | LoadTemplates() 77 | } 78 | 79 | func init() { 80 | cobra.OnInitialize(checkDefaultsFile) 81 | // spew.Dump(rootCmd) 82 | rootCmd.PersistentFlags().StringVar(&defaults.CustomConfigurationFile, "config", defaults.ConfigurationFile, "configuration file") 83 | set_pflag(rootCmd,"sandbox-home", "", "SANDBOX_HOME", defaults.Defaults().SandboxHome, "Sandbox deployment directory", false) 84 | set_pflag(rootCmd,"sandbox-binary", "", "SANDBOX_BINARY", defaults.Defaults().SandboxBinary, "Binary repository", false) 85 | 86 | rootCmd.InitDefaultVersionFlag() 87 | } 88 | -------------------------------------------------------------------------------- /concurrent/concurrent.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package concurrent 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | "os/exec" 22 | "sync" 23 | ) 24 | 25 | type CommonChan chan *exec.Cmd 26 | 27 | type ExecCommand struct { 28 | Cmd string 29 | Args []string 30 | } 31 | 32 | type ExecCommands []ExecCommand 33 | 34 | type ExecutionList struct { 35 | Priority int 36 | Command ExecCommand 37 | } 38 | 39 | var DebugConcurrency bool 40 | var VerboseConcurrency bool 41 | 42 | 43 | func add_task (num int, wg *sync.WaitGroup, tasks CommonChan, cmd string, args []string) { 44 | wg.Add(1) 45 | go start_task(num, wg, tasks) 46 | tasks <- exec.Command(cmd, args...) 47 | } 48 | 49 | func start_task (num int, w *sync.WaitGroup, tasks CommonChan) { 50 | defer w.Done() 51 | var ( 52 | out []byte 53 | err error 54 | ) 55 | for cmd := range tasks { // this will exit the loop when the channel closes 56 | out, err = cmd.Output() 57 | if err != nil { 58 | fmt.Printf("Error executing goroutine %d : %s", num, err) 59 | //os.Exit(1) 60 | } 61 | if DebugConcurrency { 62 | fmt.Printf("goroutine %d command output: %s", num, string(out)) 63 | } else { 64 | if VerboseConcurrency { 65 | fmt.Printf("%s",string(out)) 66 | } 67 | } 68 | } 69 | } 70 | 71 | func RunParallelTasks( priority_level int, operations ExecCommands ) { 72 | tasks := make(CommonChan, 64) 73 | 74 | var wg sync.WaitGroup 75 | 76 | for N, ec := range operations { 77 | add_task(N, &wg, tasks, ec.Cmd, ec.Args) 78 | } 79 | close(tasks) 80 | wg.Wait() 81 | if VerboseConcurrency { 82 | fmt.Printf("#%d\n", priority_level) 83 | } 84 | } 85 | 86 | func RunParallelTasksByPriority ( exec_lists []ExecutionList) { 87 | maxPriority := 0 88 | if len(exec_lists) == 0 { 89 | return 90 | } 91 | if DebugConcurrency { 92 | fmt.Printf("RunParallelTasksByPriority exec_list %#v\n", exec_lists) 93 | } 94 | for _, list := range exec_lists { 95 | if list.Priority > maxPriority { 96 | maxPriority = list.Priority 97 | } 98 | } 99 | for N := 0; N <= maxPriority ; N++ { 100 | var operations ExecCommands 101 | for _, list := range exec_lists { 102 | if list.Priority == N { 103 | operations = append(operations, list.Command) 104 | } 105 | } 106 | if DebugConcurrency { 107 | fmt.Printf("%d %v\n", N, operations) 108 | } 109 | RunParallelTasks(N, operations) 110 | } 111 | } 112 | 113 | func init() { 114 | if os.Getenv("DEBUG_CONCURRENCY") != "" { 115 | DebugConcurrency = true 116 | } 117 | if os.Getenv("VERBOSE_CONCURRENCY") != "" { 118 | VerboseConcurrency = true 119 | } 120 | } 121 | -------------------------------------------------------------------------------- /mkreadme/make_readme.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package main 17 | 18 | import ( 19 | "bufio" 20 | "fmt" 21 | "io/ioutil" 22 | "os" 23 | "os/exec" 24 | "regexp" 25 | "strings" 26 | "github.com/datacharmer/dbdeployer/common" 27 | "time" 28 | ) 29 | 30 | func get_cmd_output(cmdText string) string { 31 | cmdList := strings.Split(cmdText, " ") 32 | command := cmdList[0] 33 | var args []string 34 | for n, arg := range cmdList { 35 | if n > 0 { 36 | args = append(args, arg) 37 | } 38 | } 39 | cmd := exec.Command(command, args...) 40 | stdout, err := cmd.StdoutPipe() 41 | if err = cmd.Start(); err != nil { 42 | fmt.Println(err) 43 | os.Exit(1) 44 | } 45 | slurp, _ := ioutil.ReadAll(stdout) 46 | stdout.Close() 47 | return fmt.Sprintf("%s", slurp) 48 | } 49 | 50 | /* 51 | Reads the README template and replaces the commands indicated 52 | as {{command argument}} with their output. 53 | This allows us to produce a file README.md with the output 54 | from the current release. 55 | 56 | Use as: 57 | ./mkreadme/make_readme < ./mkreadme/readme_template.md > README.md 58 | 59 | */ 60 | func main() { 61 | // Gets input from stdin 62 | scanner := bufio.NewScanner(os.Stdin) 63 | 64 | re_version := regexp.MustCompile(`{{\.Version}}`) 65 | re_date := regexp.MustCompile(`{{\.Date}}`) 66 | re_cmd := regexp.MustCompile(`{{([^}]+)}}`) 67 | re_flag := regexp.MustCompile(`(?sm)Global Flags:.*`) 68 | re_spaces := regexp.MustCompile(`(?m)^`) 69 | home := os.Getenv("HOME") 70 | re_home := regexp.MustCompile(home) 71 | time_format := "02-Jan-2006 15:04 MST" 72 | timestamp := time.Now().UTC().Format(time_format) 73 | if os.Getenv("DBDEPLOYER_TIMESTAMP") != "" { 74 | timestamp = os.Getenv("DBDEPLOYER_TIMESTAMP") 75 | } 76 | for scanner.Scan() { 77 | line := scanner.Text() 78 | line = re_version.ReplaceAllString(line, common.VersionDef) 79 | line = re_date.ReplaceAllString(line, timestamp) 80 | // Find a placeholder for a {{command}} 81 | findList := re_cmd.FindAllStringSubmatch(line, -1) 82 | if findList != nil { 83 | commandText := findList[0][1] 84 | // Run the command and gets its output 85 | out := get_cmd_output(commandText) 86 | // remove global flags from the output 87 | out = re_flag.ReplaceAllString(out, "") 88 | // Add 4 spaces to every line of the output 89 | out = re_spaces.ReplaceAllString(out, " ") 90 | // Replace the literal $HOME value with its variable name 91 | out = re_home.ReplaceAllString(out, `$$HOME`) 92 | 93 | fmt.Printf(" $ %s\n", commandText) 94 | fmt.Printf("%s\n", out) 95 | } else { 96 | fmt.Printf("%s\n", line) 97 | } 98 | } 99 | 100 | if err := scanner.Err(); err != nil { 101 | fmt.Fprintln(os.Stderr, "error:", err) 102 | os.Exit(1) 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /cmd/usage.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | func ShowUsage(cmd *cobra.Command, args []string) { 25 | const basic_usage string = ` 26 | USING A SANDBOX 27 | 28 | Change directory to the newly created one (default: $SANDBOX_HOME/msb_VERSION 29 | for single sandboxes) 30 | [ $SANDBOX_HOME = $HOME/sandboxes unless modified with flag --sandbox-home ] 31 | 32 | The sandbox directory of the instance you just created contains some handy 33 | scripts to manage your server easily and in isolation. 34 | 35 | "./start", "./status", "./restart", and "./stop" do what their name suggests. 36 | start and restart accept parameters that are eventually passed to the server. 37 | e.g.: 38 | 39 | ./start --server-id=1001 40 | 41 | ./restart --event-scheduler=disabled 42 | 43 | "./use" calls the command line client with the appropriate parameters, 44 | Example: 45 | 46 | ./use -BN -e "select @@server_id" 47 | ./use -u root 48 | 49 | "./clear" stops the server and removes everything from the data directory, 50 | letting you ready to start from scratch. (Warning! It's irreversible!) 51 | 52 | "./my" is a prefix script to invoke any command named "my*" from the 53 | MySQL /bin directory. It is important to use it rather than the 54 | corresponding globally installed tool, because this guarantees 55 | that you will be using the tool for the version you have deployed. 56 | Examples: 57 | 58 | ./my sqldump db_name 59 | ./my sqlbinlog somefile 60 | ` 61 | const multiple_usage string = ` USING MULTIPLE SERVER SANDBOX 62 | On a replication sandbox, you have the same commands (run "dbdeployer usage single"), 63 | with an "_all" suffix, meaning that you propagate the command to all the members. 64 | Then you have "./m" as a shortcut to use the master, "./s1" and "./s2" to access 65 | the slaves (and "s3", "s4" ... if you define more). 66 | 67 | In group sandboxes without a master slave relationship (group replication and 68 | multiple sandboxes) the nodes can be accessed by ./n1, ./n2, ./n3, and so on. 69 | 70 | start_all 71 | status_all 72 | restart_all 73 | stop_all 74 | use_all 75 | use_all_masters 76 | use_all_slaves 77 | clear_all 78 | m 79 | s1, s2, n1, n2 80 | 81 | The scripts "check_slaves" or "check_nodes" give the status of replication in the sandbox. 82 | ` 83 | request := "" 84 | if len(args) > 0 { 85 | request = args[0] 86 | } 87 | switch request { 88 | case "single": 89 | fmt.Println(basic_usage) 90 | case "multiple": 91 | fmt.Println(multiple_usage) 92 | default: 93 | fmt.Println(basic_usage) 94 | fmt.Println(multiple_usage) 95 | } 96 | } 97 | 98 | var usageCmd = &cobra.Command{ 99 | Use: "usage [single|multiple]", 100 | Short: "Shows usage of installed sandboxes", 101 | Long: `Shows syntax and examples of tools installed in database sandboxes.`, 102 | Run: ShowUsage, 103 | } 104 | 105 | func init() { 106 | rootCmd.AddCommand(usageCmd) 107 | 108 | // usageCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") 109 | } 110 | -------------------------------------------------------------------------------- /test/mock/set-mock.sh: -------------------------------------------------------------------------------- 1 | # DBDeployer - The MySQL Sandbox 2 | # Copyright © 2006-2018 Giuseppe Maxia 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | mock_dir=mock_dir 17 | if [ -d $mock_dir ] 18 | then 19 | echo "mock directory "$PWD/$mock_dir" already exists" 20 | exit 1 21 | fi 22 | 23 | mkdir $mock_dir 24 | cd $mock_dir 25 | mock_dir=$PWD 26 | export HOME=$mock_dir/home 27 | export CATALOG=$HOME/.dbdeployer/sandboxes.json 28 | export SANDBOX_HOME=$HOME/sandboxes 29 | export SANDBOX_BINARY=$HOME/opt/mysql 30 | export SANDBOX_TARBALL=$HOME/downloads 31 | export SLEEP_TIME=0 32 | 33 | pwd 34 | ls -l 35 | echo "HOME : $HOME" 36 | echo "SANDBOX_HOME : $SANDBOX_HOME" 37 | echo "SANDBOX_BINARY : $SANDBOX_BINARY" 38 | echo "SANDBOX_TARBALL: $SANDBOX_TARBALL" 39 | 40 | mkdir $HOME 41 | mkdir -p $SANDBOX_BINARY 42 | mkdir $SANDBOX_HOME 43 | mkdir $SANDBOX_TARBALL 44 | 45 | function make_dir { 46 | dir=$1 47 | if [ ! -d $dir ] 48 | then 49 | mkdir -p $dir 50 | fi 51 | } 52 | 53 | # A mock version is a collection of 54 | # fake MySQL executable that will create 55 | # empty sandboxes with no-op key executables. 56 | # Its purpose is testing the administrative part of 57 | # dbdeployer. 58 | function create_mock_version { 59 | version_label=$1 60 | if [ -z "$SANDBOX_BINARY" ] 61 | then 62 | echo "SANDBOX_BINARY not set" 63 | exit 1 64 | fi 65 | if [ ! -d "$SANDBOX_BINARY" ] 66 | then 67 | echo "$SANDBOX_BINARY not found" 68 | exit 1 69 | fi 70 | make_dir $SANDBOX_BINARY/$version_label 71 | make_dir $SANDBOX_BINARY/$version_label/bin 72 | make_dir $SANDBOX_BINARY/$version_label/scripts 73 | dbdeployer defaults templates show no_op_mock_template > $SANDBOX_BINARY/$version_label/bin/mysqld 74 | dbdeployer defaults templates show no_op_mock_template > $SANDBOX_BINARY/$version_label/bin/mysql 75 | dbdeployer defaults templates show mysqld_safe_mock_template > $SANDBOX_BINARY/$version_label/bin/mysqld_safe 76 | dbdeployer defaults templates show no_op_mock_template > $SANDBOX_BINARY/$version_label/scripts/mysql_install_db 77 | chmod +x $SANDBOX_BINARY/$version_label/bin/* 78 | chmod +x $SANDBOX_BINARY/$version_label/scripts/* 79 | } 80 | 81 | # a mock tarball is a tarball that contains mock MySQL executables 82 | # for the purpose of testing "dbdeployer unpack" 83 | function create_mock_tarball { 84 | version_label=$1 85 | tarball_dir=$2 86 | save_sandbox_binary=$SANDBOX_BINARY 87 | # Changes SANDBOX_BINARY so that create_mock_version 88 | # will create the mock directory in the tarball place. 89 | SANDBOX_BINARY=$tarball_dir 90 | create_mock_version $version_label 91 | cd $tarball_dir 92 | if [ ! -d $version_label ] 93 | then 94 | echo "$version_label not found in $PWD" 95 | exit 1 96 | fi 97 | # Change the name of the directory, so that it's different 98 | # from the ultimate destination 99 | mv $version_label mysql-${version_label} 100 | tar -c mysql-${version_label} | gzip -c > mysql-${version_label}.tar.gz 101 | rm -rf mysql-$version_label 102 | cd - 103 | export SANDBOX_BINARY=$save_sandbox_binary 104 | } 105 | 106 | -------------------------------------------------------------------------------- /test/all_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ ! -d "./test" ] 4 | then 5 | echo "directory ./test not found" 6 | exit 1 7 | fi 8 | 9 | version=$1 10 | if [ -z $version ] 11 | then 12 | echo "version needed" 13 | exit 1 14 | fi 15 | 16 | executable=dbdeployer-${version}.linux 17 | if [ ! -x $executable ] 18 | then 19 | echo "executable '$executable' not found" 20 | exit 1 21 | fi 22 | 23 | if [ ! -f ./test/common.sh ] 24 | then 25 | echo "script './test/common.sh' not found" 26 | exit 1 27 | fi 28 | 29 | source ./test/common.sh 30 | 31 | start_timer 32 | timestamp=$(date +%Y-%m-%d-%H.%M) 33 | 34 | if [ ! -d ./test/logs ] 35 | then 36 | mkdir ./test/logs 37 | fi 38 | 39 | mkdir ./test/logs/$timestamp 40 | log_summary=./test/logs/$timestamp/all_tests-summary.log 41 | 42 | function summary { 43 | exit_code=$1 44 | cat $log_summary 45 | stop_timer $log_summary 46 | #rm -f $log_summary 47 | echo "# Exit code: $exit_code" 48 | concurrency=no 49 | if [ -n "$RUN_CONCURRENTLY" ] 50 | then 51 | concurrency=yes 52 | fi 53 | echo "Runs concurrently: $concurrency" 54 | echo "Runs concurrently: $concurrency" >> $log_summary 55 | exit $exit_code 56 | } 57 | 58 | function run_test { 59 | test_name="$1" 60 | test_arg="$2" 61 | start_test=$(date +%s) 62 | test_base_name=$(basename $test_name .sh) 63 | test_log=./test/logs/$timestamp/${test_base_name}.log 64 | echo "# Running $test_name $test_arg" 65 | $test_name $test_arg > $test_log 66 | exit_code=$? 67 | end_test=$(date +%s) 68 | elapsed=$((end_test-start_test)) 69 | test_arg="[$test_arg]" 70 | printf "%-30s %-9s - time: %4ds (%10s) - exit code: %d\n" $test_base_name $test_arg ${elapsed} $(minutes_seconds $elapsed) $exit_code >> $log_summary 71 | if [ "$test_base_name" == "port-clash" ] 72 | then 73 | sandboxes=$(grep catalog $test_log | wc -l) 74 | ports=$(grep "Total ports installed" $test_log | awk '{print $NF}') 75 | changed=$(grep changed $test_log | wc -l) 76 | echo "# Deployed: $sandboxes sandboxes ($ports total ports) - Changed: $changed" >> $log_summary 77 | fi 78 | if [ "$exit_code" != "0" ] 79 | then 80 | echo $dash_line 81 | echo "# Error detected: $test_base_name " 82 | echo $dash_line 83 | tail -n 20 $test_log 84 | echo $dash_line 85 | summary $exit_code 86 | fi 87 | } 88 | 89 | function all_tests { 90 | run_test ./test/functional-test.sh 91 | run_test ./test/docker-test.sh $version 92 | run_test ./test/mock/defaults-change.sh 93 | if [ -n "$COMPLETE_PORT_TEST" ] 94 | then 95 | run_test ./test/mock/port-clash.sh 96 | else 97 | run_test ./test/mock/port-clash.sh sparse 98 | fi 99 | } 100 | 101 | all_tests 102 | 103 | echo $dash_line 104 | echo $dash_line >> $log_summary 105 | for logfile in ./test/logs/$timestamp/*.log 106 | do 107 | fname=$(basename $logfile .log) 108 | if [ "$fname" != "all_tests-summary" ] 109 | then 110 | lf_pass=$(grep '^ok' $logfile | wc -l | tr -d ' ') 111 | lf_fail=$(grep '^not ok' $logfile | wc -l | tr -d ' ') 112 | lf_tests=$((lf_pass+lf_fail)) 113 | printf "# %-20s - tests: %4d - pass: %4d - fail: %4d\n" $fname $lf_tests $lf_pass $lf_fail 114 | printf "# %-20s - tests: %4d - pass: %4d - fail: %4d\n" $fname $lf_tests $lf_pass $lf_fail >> $log_summary 115 | fi 116 | done 117 | echo $dash_line 118 | echo $dash_line >> $log_summary 119 | pass=$(grep '^ok' ./test/logs/$timestamp/*.log | wc -l | tr -d ' ') 120 | fail=$(grep -i '^not ok' ./test/logs/$timestamp/*.log | wc -l | tr -d ' ') 121 | tests=$((pass+fail)) 122 | echo $dash_line >> $log_summary 123 | echo "# Total tests: $tests" >> $log_summary 124 | echo "# pass : $pass" >> $log_summary 125 | echo "# fail : $fail" >> $log_summary 126 | echo $dash_line >> $log_summary 127 | summary 0 128 | -------------------------------------------------------------------------------- /cmd/deploy.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "github.com/spf13/cobra" 20 | ) 21 | 22 | var deployCmd = &cobra.Command{ 23 | Use: "deploy", 24 | Short: "deploy sandboxes", 25 | Long: `Deploys single, multiple, or replicated sandboxes`, 26 | } 27 | 28 | func init() { 29 | rootCmd.AddCommand(deployCmd) 30 | deployCmd.PersistentFlags().Int("port", 0, "Overrides default port") 31 | deployCmd.PersistentFlags().Int("base-port", 0, "Overrides default base-port (for multiple sandboxes)") 32 | deployCmd.PersistentFlags().Bool("gtid", false, "enables GTID") 33 | deployCmd.PersistentFlags().Bool("native-auth-plugin", false, "in 8.0.4+, uses the native password auth plugin") 34 | deployCmd.PersistentFlags().Bool("keep-server-uuid", false, "Does not change the server UUID") 35 | deployCmd.PersistentFlags().Bool("force", false, "If a destination sandbox already exists, it will be overwritten") 36 | deployCmd.PersistentFlags().Bool("skip-start", false, "Does not start the database server") 37 | deployCmd.PersistentFlags().Bool("disable-mysqlx", false, "Disable MySQLX plugin (8.0.11+)") 38 | deployCmd.PersistentFlags().Bool("skip-load-grants", false, "Does not load the grants") 39 | deployCmd.PersistentFlags().Bool("skip-report-host", false, "Does not include report host in my.sandbox.cnf") 40 | deployCmd.PersistentFlags().Bool("skip-report-port", false, "Does not include report port in my.sandbox.cnf") 41 | deployCmd.PersistentFlags().Bool("expose-dd-tables", false, "In MySQL 8.0+ shows data dictionary tables") 42 | deployCmd.PersistentFlags().Bool("concurrent", false, "Runs multiple sandbox deployments concurrently") 43 | 44 | set_pflag(deployCmd,"remote-access", "", "", "127.%", "defines the database access ", false) 45 | set_pflag(deployCmd,"bind-address", "", "", "127.0.0.1", "defines the database bind-address ", false) 46 | set_pflag(deployCmd,"custom-mysqld", "", "", "", "Uses an alternative mysqld (must be in the same directory as regular mysqld)", false) 47 | set_pflag(deployCmd,"defaults", "", "", "", "Change defaults on-the-fly (--defaults=label:value)", true) 48 | set_pflag(deployCmd,"init-options", "i", "INIT_OPTIONS", "", "mysqld options to run during initialization", true) 49 | set_pflag(deployCmd,"my-cnf-options", "c", "MY_CNF_OPTIONS", "", "mysqld options to add to my.sandbox.cnf", true) 50 | set_pflag(deployCmd,"pre-grants-sql-file", "", "", "", "SQL file to run before loading grants", false) 51 | set_pflag(deployCmd,"pre-grants-sql", "", "", "", "SQL queries to run before loading grants", true) 52 | set_pflag(deployCmd,"post-grants-sql", "", "", "", "SQL queries to run after loading grants", true) 53 | set_pflag(deployCmd,"post-grants-sql-file", "", "", "", "SQL file to run after loading grants", false) 54 | // This option will allow to merge the template with an external my.cnf 55 | // The options that are essential for the sandbox will be preserved 56 | set_pflag(deployCmd,"my-cnf-file", "", "MY_CNF_FILE", "", "Alternative source file for my.sandbox.cnf", false) 57 | set_pflag(deployCmd,"db-user", "u", "", "msandbox", "database user", false) 58 | set_pflag(deployCmd,"rpl-user", "", "", "rsandbox", "replication user", false) 59 | set_pflag(deployCmd,"db-password", "p", "", "msandbox", "database password", false) 60 | set_pflag(deployCmd,"rpl-password", "", "", "rsandbox", "replication password", false) 61 | set_pflag(deployCmd,"use-template", "", "", "", "[template_name:file_name] Replace existing template with one from file", true) 62 | set_pflag(deployCmd,"sandbox-directory", "", "", "", "Changes the default sandbox directory", false) 63 | } 64 | -------------------------------------------------------------------------------- /defaults/catalog.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package defaults 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | //"strings" 22 | "time" 23 | "encoding/json" 24 | "github.com/datacharmer/dbdeployer/common" 25 | ) 26 | 27 | type SandboxItem struct { 28 | Origin string `json:"origin"` 29 | SBType string `json:"type"` // single multi master-slave group all-masters fan-in 30 | Version string `json:"version"` 31 | Port []int `json:"port"` 32 | Nodes []string `json:"nodes"` 33 | Destination string `json:"destination"` 34 | } 35 | 36 | type SandboxCatalog map[string]SandboxItem 37 | const ( 38 | timeout = 5 39 | ) 40 | 41 | var enable_catalog_management bool = true 42 | 43 | func setLock(label string) bool { 44 | if !enable_catalog_management { 45 | return true 46 | } 47 | lock_file := SandboxRegistryLock 48 | if !common.DirExists(ConfigurationDir) { 49 | common.Mkdir(ConfigurationDir) 50 | } 51 | elapsed := 0 52 | for common.FileExists(lock_file) { 53 | elapsed += 1 54 | time.Sleep(1000 * time.Millisecond) 55 | if elapsed > timeout { 56 | return false 57 | } 58 | } 59 | common.WriteString(label, lock_file) 60 | return true 61 | } 62 | 63 | func releaseLock() { 64 | if !enable_catalog_management { 65 | return 66 | } 67 | lock_file := SandboxRegistryLock 68 | if common.FileExists(lock_file) { 69 | os.Remove(lock_file) 70 | } 71 | } 72 | 73 | func WriteCatalog(sc SandboxCatalog) { 74 | if !enable_catalog_management { 75 | return 76 | } 77 | b, err := json.MarshalIndent(sc, " ", "\t") 78 | if err != nil { 79 | fmt.Println("error encoding sandbox catalog: ", err) 80 | os.Exit(1) 81 | } 82 | json_string := fmt.Sprintf("%s", b) 83 | filename := SandboxRegistry 84 | common.WriteString(json_string, filename) 85 | } 86 | 87 | func ReadCatalog() (sc SandboxCatalog) { 88 | if !enable_catalog_management { 89 | return 90 | } 91 | filename := SandboxRegistry 92 | if !common.FileExists(filename) { 93 | return 94 | } 95 | sc_blob := common.SlurpAsBytes(filename) 96 | 97 | err := json.Unmarshal(sc_blob, &sc) 98 | if err != nil { 99 | fmt.Println("error decoding sandbox catalog: ", err) 100 | os.Exit(1) 101 | } 102 | return 103 | } 104 | 105 | func UpdateCatalog(sb_name string, details SandboxItem) { 106 | if !enable_catalog_management { 107 | return 108 | } 109 | // fmt.Printf("+%s\n",sb_name) 110 | if setLock(sb_name) { 111 | // fmt.Printf("+locked\n") 112 | current := ReadCatalog() 113 | if current == nil { 114 | current = make(SandboxCatalog) 115 | } 116 | current[sb_name] = details 117 | WriteCatalog(current) 118 | releaseLock() 119 | // fmt.Printf("+unlocked\n") 120 | } else { 121 | fmt.Printf("%s\n", HashLine) 122 | fmt.Printf("# Could not get lock on %s\n", SandboxRegistryLock) 123 | fmt.Printf("%s\n", HashLine) 124 | } 125 | } 126 | 127 | func DeleteFromCatalog(sb_name string) { 128 | if !enable_catalog_management { 129 | return 130 | } 131 | if setLock(sb_name) { 132 | current := ReadCatalog() 133 | defer releaseLock() 134 | if current == nil { 135 | return 136 | } 137 | //for name, _ := range current { 138 | // if strings.HasPrefix(name, sb_name) { 139 | // delete(current, name) 140 | // } 141 | //} 142 | delete(current, sb_name) 143 | WriteCatalog(current) 144 | releaseLock() 145 | } else { 146 | fmt.Printf("%s\n", HashLine) 147 | fmt.Printf("# Could not get lock on %s\n", SandboxRegistryLock) 148 | fmt.Printf("%s\n", HashLine) 149 | } 150 | } 151 | 152 | func init() { 153 | if os.Getenv("SKIP_DBDEPLOYER_CATALOG") != "" { 154 | enable_catalog_management = false 155 | } 156 | } 157 | -------------------------------------------------------------------------------- /common/version_test.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | import "testing" 19 | 20 | type version_port struct { 21 | version string 22 | port int 23 | } 24 | 25 | type version_pair struct { 26 | version string 27 | versionList []int 28 | expected bool 29 | } 30 | 31 | type UUID_component struct { 32 | port int 33 | node_num int 34 | expected string 35 | } 36 | 37 | func TestVersionToPort(t *testing.T) { 38 | //t.Parallel() 39 | var versions []version_port = []version_port{ 40 | {"", -1}, // FAIL: Empty string 41 | {"5.0.A", -1}, // FAIL: Hexadecimal number 42 | {"5.0.-9", -1}, // FAIL: Negative revision 43 | {"-5.0.9", -1}, // FAIL: Negative major version 44 | {"5.-1.9", -1}, // FAIL: Negative minor version 45 | {"5096", -1}, // FAIL: No separators 46 | {"50.96", -1}, // FAIL: Not enough separators 47 | {"dummy", -1}, // FAIL: Not numbers 48 | {"5.0.96.2", -1}, // FAIL: Too many components 49 | {"5.0.96", 5096}, // OK: 5.0 50 | {"5.1.72", 5172}, // OK: 5.1 51 | {"5.5.55", 5555}, // OK: 5.5 52 | {"ps5.7.20", 5720}, // OK: 5.7 with prefix 53 | {"5.7.21", 5721}, // OK: 5.7 54 | {"8.0.0", 8000}, // OK: 8.0 55 | {"8.0.4", 8004}, // OK: 8.0 56 | {"8.0.04", 8004}, // OK: 8.0 57 | {"ma10.2.1", 10201}, // OK: 10.2 with prefix 58 | } 59 | //t.Logf("Name: %s\n", t.Name()) 60 | for _, vp := range versions { 61 | version := vp.version 62 | expected := vp.port 63 | port := VersionToPort(version) 64 | //t.Logf("+%s\n", version) 65 | if expected == port { 66 | t.Logf("ok %-10s => %5d\n", version, port) 67 | } else { 68 | t.Logf("NOT OK %-10s => %5d\n", version, port) 69 | t.Fail() 70 | } 71 | } 72 | } 73 | 74 | func TestGreaterOrEqualVersion(t *testing.T) { 75 | 76 | var versions = []version_pair{ 77 | {"5.0.0", []int{5, 6, 0}, false}, 78 | {"8.0.0", []int{5, 6, 0}, true}, 79 | {"ps5.7.5", []int{5, 7, 0}, true}, 80 | {"10.0.1", []int{5, 6, 0}, false}, 81 | } 82 | for _, v := range versions { 83 | result := GreaterOrEqualVersion(v.version, v.versionList) 84 | if v.expected == result { 85 | t.Logf("ok %-10s => %v %v \n", v.version, v.versionList, result) 86 | } else { 87 | t.Logf("NOT OK %-10s => %v %v \n", v.version, v.versionList, result) 88 | t.Fail() 89 | } 90 | } 91 | } 92 | 93 | func TestCustomUuid(t *testing.T) { 94 | var uuid_samples = []UUID_component{ 95 | // 12345678 1234 1234 1234 123456789012 96 | // "00000000-0000-0000-0000-000000000000" 97 | UUID_component{ 5000, 0, "00005000-0000-0000-0000-000000005000"}, 98 | UUID_component{15000, 0, "00015000-0000-0000-0000-000000015000"}, 99 | UUID_component{15000, 1, "00015000-1111-1111-1111-111111111111"}, 100 | UUID_component{25000, 2, "00025000-2222-2222-2222-222222222222"}, 101 | UUID_component{12987, 7, "00012987-7777-7777-7777-777777777777"}, 102 | UUID_component{ 8004, 0, "00008004-0000-0000-0000-000000008004"}, 103 | UUID_component{ 8004, 11, "00008004-0011-0011-0011-000000008004"}, 104 | UUID_component{ 8004, 3452, "00008004-3452-3452-3452-000000008004"}, 105 | UUID_component{ 8004, 18976, "00008004-0000-0001-8976-000000008004"}, 106 | UUID_component{ 6000, 35281, "00006000-0000-0003-5281-000000006000"}, 107 | UUID_component{ 6000, 235281, "00006000-0023-0000-0000-000000006000"}, 108 | } 109 | for _, sample := range uuid_samples { 110 | new_uuid := MakeCustomizedUuid(sample.port, sample.node_num) 111 | if new_uuid == sample.expected { 112 | t.Logf("ok %5d %6d => %s \n", sample.port, sample.node_num, new_uuid) 113 | } else { 114 | t.Logf("NOT OK %5d %6d => <%#v> (expected: <%#v>) \n", sample.port, sample.node_num, new_uuid, sample.expected) 115 | t.Fail() 116 | } 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /pflag/string_slice.go: -------------------------------------------------------------------------------- 1 | package pflag 2 | 3 | import ( 4 | "bytes" 5 | "encoding/csv" 6 | "strings" 7 | ) 8 | 9 | // -- stringSlice Value 10 | type stringSliceValue struct { 11 | value *[]string 12 | changed bool 13 | } 14 | 15 | func newStringSliceValue(val []string, p *[]string) *stringSliceValue { 16 | ssv := new(stringSliceValue) 17 | ssv.value = p 18 | *ssv.value = val 19 | return ssv 20 | } 21 | 22 | func readAsCSV(val string) ([]string, error) { 23 | if val == "" { 24 | return []string{}, nil 25 | } 26 | stringReader := strings.NewReader(val) 27 | csvReader := csv.NewReader(stringReader) 28 | // Change CSV reader field separator 29 | csvReader.Comma = ';' 30 | return csvReader.Read() 31 | } 32 | 33 | func writeAsCSV(vals []string) (string, error) { 34 | b := &bytes.Buffer{} 35 | w := csv.NewWriter(b) 36 | // Change CSV writer field separator 37 | w.Comma = ';' 38 | err := w.Write(vals) 39 | if err != nil { 40 | return "", err 41 | } 42 | w.Flush() 43 | return strings.TrimSuffix(b.String(), "\n"), nil 44 | } 45 | 46 | func (s *stringSliceValue) Set(val string) error { 47 | v, err := readAsCSV(val) 48 | if err != nil { 49 | return err 50 | } 51 | if !s.changed { 52 | *s.value = v 53 | } else { 54 | *s.value = append(*s.value, v...) 55 | } 56 | s.changed = true 57 | return nil 58 | } 59 | 60 | func (s *stringSliceValue) Type() string { 61 | return "stringSlice" 62 | } 63 | 64 | func (s *stringSliceValue) String() string { 65 | str, _ := writeAsCSV(*s.value) 66 | return "[" + str + "]" 67 | } 68 | 69 | func stringSliceConv(sval string) (interface{}, error) { 70 | sval = sval[1 : len(sval)-1] 71 | // An empty string would cause a slice with one (empty) string 72 | if len(sval) == 0 { 73 | return []string{}, nil 74 | } 75 | return readAsCSV(sval) 76 | } 77 | 78 | // GetStringSlice return the []string value of a flag with the given name 79 | func (f *FlagSet) GetStringSlice(name string) ([]string, error) { 80 | val, err := f.getFlagType(name, "stringSlice", stringSliceConv) 81 | if err != nil { 82 | return []string{}, err 83 | } 84 | return val.([]string), nil 85 | } 86 | 87 | // StringSliceVar defines a string flag with specified name, default value, and usage string. 88 | // The argument p points to a []string variable in which to store the value of the flag. 89 | func (f *FlagSet) StringSliceVar(p *[]string, name string, value []string, usage string) { 90 | f.VarP(newStringSliceValue(value, p), name, "", usage) 91 | } 92 | 93 | // StringSliceVarP is like StringSliceVar, but accepts a shorthand letter that can be used after a single dash. 94 | func (f *FlagSet) StringSliceVarP(p *[]string, name, shorthand string, value []string, usage string) { 95 | f.VarP(newStringSliceValue(value, p), name, shorthand, usage) 96 | } 97 | 98 | // StringSliceVar defines a string flag with specified name, default value, and usage string. 99 | // The argument p points to a []string variable in which to store the value of the flag. 100 | func StringSliceVar(p *[]string, name string, value []string, usage string) { 101 | CommandLine.VarP(newStringSliceValue(value, p), name, "", usage) 102 | } 103 | 104 | // StringSliceVarP is like StringSliceVar, but accepts a shorthand letter that can be used after a single dash. 105 | func StringSliceVarP(p *[]string, name, shorthand string, value []string, usage string) { 106 | CommandLine.VarP(newStringSliceValue(value, p), name, shorthand, usage) 107 | } 108 | 109 | // StringSlice defines a string flag with specified name, default value, and usage string. 110 | // The return value is the address of a []string variable that stores the value of the flag. 111 | func (f *FlagSet) StringSlice(name string, value []string, usage string) *[]string { 112 | p := []string{} 113 | f.StringSliceVarP(&p, name, "", value, usage) 114 | return &p 115 | } 116 | 117 | // StringSliceP is like StringSlice, but accepts a shorthand letter that can be used after a single dash. 118 | func (f *FlagSet) StringSliceP(name, shorthand string, value []string, usage string) *[]string { 119 | p := []string{} 120 | f.StringSliceVarP(&p, name, shorthand, value, usage) 121 | return &p 122 | } 123 | 124 | // StringSlice defines a string flag with specified name, default value, and usage string. 125 | // The return value is the address of a []string variable that stores the value of the flag. 126 | func StringSlice(name string, value []string, usage string) *[]string { 127 | return CommandLine.StringSliceP(name, "", value, usage) 128 | } 129 | 130 | // StringSliceP is like StringSlice, but accepts a shorthand letter that can be used after a single dash. 131 | func StringSliceP(name, shorthand string, value []string, usage string) *[]string { 132 | return CommandLine.StringSliceP(name, shorthand, value, usage) 133 | } 134 | -------------------------------------------------------------------------------- /cmd/unpack.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | "path" 22 | "regexp" 23 | "strings" 24 | 25 | "github.com/datacharmer/dbdeployer/common" 26 | "github.com/datacharmer/dbdeployer/unpack" 27 | "github.com/spf13/cobra" 28 | ) 29 | 30 | func UnpackTarball(cmd *cobra.Command, args []string) { 31 | flags := cmd.Flags() 32 | Basedir, _ := flags.GetString("sandbox-binary") 33 | verbosity, _ := flags.GetInt("verbosity") 34 | if !common.DirExists(Basedir) { 35 | fmt.Printf("Directory %s does not exist.\n", Basedir) 36 | fmt.Println("You should create it or provide an alternate base directory using --sandbox-binary") 37 | os.Exit(1) 38 | } 39 | tarball := args[0] 40 | re_version := regexp.MustCompile(`(\d+\.\d+\.\d+)`) 41 | verList := re_version.FindAllStringSubmatch(tarball, -1) 42 | detected_version := verList[0][0] 43 | // fmt.Printf(">> %#v %s\n",verList, detected_version) 44 | 45 | Version, _ := flags.GetString("unpack-version") 46 | if Version == "" { 47 | Version = detected_version 48 | } 49 | if Version == "" { 50 | fmt.Println("unpack: No version was detected from tarball name. ") 51 | fmt.Println("Flag --unpack-version becomes mandatory") 52 | os.Exit(1) 53 | } 54 | // This call used to ensure that the port provided is in the right format 55 | common.VersionToPort(Version) 56 | Prefix, _ := flags.GetString("prefix") 57 | 58 | destination := Basedir + "/" + Prefix + Version 59 | if common.DirExists(destination) { 60 | fmt.Printf("Destination directory %s exists already\n", destination) 61 | os.Exit(1) 62 | } 63 | var extension string = ".tar.gz" 64 | extracted := path.Base(tarball) 65 | var barename string 66 | if strings.HasSuffix(tarball, extension) { 67 | barename = extracted[0 : len(extracted)-len(extension)] 68 | } else { 69 | fmt.Println("Tarball extension must be .tar.gz") 70 | os.Exit(1) 71 | } 72 | 73 | fmt.Printf("Unpacking tarball %s to %s\n", tarball, common.ReplaceLiteralHome(destination)) 74 | //verbosity_level := unpack.VERBOSE 75 | err := unpack.UnpackTar(tarball, Basedir, verbosity) 76 | if err != nil { 77 | fmt.Println(err) 78 | os.Exit(1) 79 | } 80 | final_name := Basedir + "/" + barename 81 | if final_name != destination { 82 | err = os.Rename(final_name, destination) 83 | if err != nil { 84 | fmt.Println(err) 85 | os.Exit(1) 86 | } 87 | } 88 | } 89 | 90 | // unpackCmd represents the unpack command 91 | var unpackCmd = &cobra.Command{ 92 | Use: "unpack MySQL-tarball", 93 | Args: cobra.ExactArgs(1), 94 | Aliases: []string{"extract", "untar", "unzip", "inflate", "expand"}, 95 | Short: "unpack a tarball into the binary directory", 96 | Long: `If you want to create a sandbox from a tarball, you first need to unpack it 97 | into the sandbox-binary directory. This command carries out that task, so that afterwards 98 | you can call 'single', 'multiple', and 'replication' commands with only the MySQL version 99 | for that tarball. 100 | If the version is not contained in the tarball name, it should be supplied using --unpack-version. 101 | If there is already an expanded tarball with the same version, a new one can be differentiated with --prefix. 102 | `, 103 | Run: UnpackTarball, 104 | Example: ` 105 | $ dbdeployer unpack mysql-8.0.4-rc-linux-glibc2.12-x86_64.tar.gz 106 | Unpacking tarball mysql-8.0.4-rc-linux-glibc2.12-x86_64.tar.gz to $HOME/opt/mysql/8.0.4 107 | 108 | $ dbdeployer unpack --prefix=ps Percona-Server-5.7.21-linux.tar.gz 109 | Unpacking tarball Percona-Server-5.7.21-linux.tar.gz to $HOME/opt/mysql/ps5.7.21 110 | 111 | $ dbdeployer unpack --unpack-version=8.0.18 --prefix=bld mysql-mybuild.tar.gz 112 | Unpacking tarball mysql-mybuild.tar.gz to $HOME/opt/mysql/bld8.0.18 113 | `, 114 | } 115 | 116 | func init() { 117 | rootCmd.AddCommand(unpackCmd) 118 | 119 | unpackCmd.PersistentFlags().Int("verbosity", 1, "Level of verbosity during unpack (0=none, 2=maximum)") 120 | unpackCmd.PersistentFlags().String("unpack-version", "", "which version is contained in the tarball") 121 | unpackCmd.PersistentFlags().String("prefix", "", "Prefix for the final expanded directory") 122 | } 123 | -------------------------------------------------------------------------------- /cmd/replication.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | 22 | "github.com/datacharmer/dbdeployer/common" 23 | "github.com/datacharmer/dbdeployer/sandbox" 24 | "github.com/spf13/cobra" 25 | ) 26 | 27 | func ReplicationSandbox(cmd *cobra.Command, args []string) { 28 | var sd sandbox.SandboxDef 29 | var semisync bool 30 | common.CheckOrigin(args) 31 | sd = FillSdef(cmd, args) 32 | sd.ReplOptions = sandbox.SingleTemplates["replication_options"].Contents 33 | flags := cmd.Flags() 34 | semisync, _ = flags.GetBool("semi-sync") 35 | nodes, _ := flags.GetInt("nodes") 36 | topology, _ := flags.GetString("topology") 37 | master_ip, _ := flags.GetString("master-ip") 38 | master_list, _ := flags.GetString("master-list") 39 | slave_list, _ := flags.GetString("slave-list") 40 | sd.SinglePrimary, _ = flags.GetBool("single-primary") 41 | if topology != "fan-in" && topology != "all-masters" { 42 | master_list = "" 43 | slave_list = "" 44 | } 45 | if semisync { 46 | if topology != "master-slave" { 47 | fmt.Println("--semi-sync is only available with master/slave topology") 48 | os.Exit(1) 49 | } 50 | if common.GreaterOrEqualVersion(sd.Version, []int{5, 5, 1}) { 51 | sd.SemiSyncOptions = sandbox.SingleTemplates["semisync_master_options"].Contents 52 | } else { 53 | fmt.Println("--semi-sync requires version 5.5.1+") 54 | os.Exit(1) 55 | } 56 | } 57 | if sd.SinglePrimary && topology != "group" { 58 | fmt.Println("Option 'single-primary' can only be used with 'group' topology ") 59 | os.Exit(1) 60 | } 61 | sandbox.CreateReplicationSandbox(sd, args[0], topology, nodes, master_ip, master_list, slave_list) 62 | } 63 | 64 | // replicationCmd represents the replication command 65 | var replicationCmd = &cobra.Command{ 66 | Use: "replication MySQL-Version", 67 | //Args: cobra.ExactArgs(1), 68 | Short: "create replication sandbox", 69 | Long: `The replication command allows you to deploy several nodes in replication. 70 | Allowed topologies are "master-slave" for all versions, and "group", "all-masters", "fan-in" 71 | for 5.7.17+. 72 | For this command to work, there must be a directory $HOME/opt/mysql/5.7.21, containing 73 | the binary files from mysql-5.7.21-$YOUR_OS-x86_64.tar.gz 74 | Use the "unpack" command to get the tarball into the right directory. 75 | `, 76 | //Allowed topologies are "master-slave", "group" (requires 5.7.17+), 77 | //"fan-in" and "all-msters" (require 5.7.9+) 78 | Run: ReplicationSandbox, 79 | Example: ` 80 | $ dbdeployer deploy replication 5.7.21 81 | # (implies topology = master-slave) 82 | 83 | $ dbdeployer deploy --topology=master-slave replication 5.7.21 84 | # (explicitly setting topology) 85 | 86 | $ dbdeployer deploy --topology=group replication 5.7.21 87 | $ dbdeployer deploy --topology=group replication 8.0.4 --single-primary 88 | $ dbdeployer deploy --topology=all-masters replication 5.7.21 89 | $ dbdeployer deploy --topology=fan-in replication 5.7.21 90 | `, 91 | } 92 | 93 | func init() { 94 | // rootCmd.AddCommand(replicationCmd) 95 | deployCmd.AddCommand(replicationCmd) 96 | //replicationCmd.PersistentFlags().StringSliceP("master-options", "", "", "Extra options for the master") 97 | //replicationCmd.PersistentFlags().StringSliceP("slave-options", "", "", "Extra options for the slaves") 98 | //replicationCmd.PersistentFlags().StringSliceP("node-options", "", "", "Extra options for all nodes") 99 | //replicationCmd.PersistentFlags().StringSliceP("one-node-options", "", "", "Extra options for one node (format #:option)") 100 | replicationCmd.PersistentFlags().StringP("master-list", "", "1,2", "Which nodes are masters in a multi-source deployment") 101 | replicationCmd.PersistentFlags().StringP("slave-list", "", "3", "Which nodes are slaves in a multi-source deployment") 102 | replicationCmd.PersistentFlags().StringP("master-ip", "", "127.0.0.1", "Which IP the slaves will connect to") 103 | replicationCmd.PersistentFlags().StringP("topology", "t", "master-slave", "Which topology will be installed") 104 | replicationCmd.PersistentFlags().IntP("nodes", "n", 3, "How many nodes will be installed") 105 | replicationCmd.PersistentFlags().BoolP("single-primary", "", false, "Using single primary for group replication") 106 | replicationCmd.PersistentFlags().BoolP("semi-sync", "", false, "Use semi-synchronous plugin") 107 | } 108 | -------------------------------------------------------------------------------- /docs/features.md: -------------------------------------------------------------------------------- 1 | # dbdeployer features 2 | 3 | This table compares features from [MySQL-Sandbox](https://github.com/datacharmer/mysql-sandbox) and [dbdeployer](https://github.com/datacharmer/dbdeployer). 4 | 5 | Feature | MySQL-Sandbox | dbdeployer | dbdeployer planned 6 | --------------------------- | :-------------- | :---------- | :----------------- 7 | Single sandbox deployment | yes | yes | 8 | unpack command | sort of [^1] | yes | 9 | multiple sandboxes | yes | yes | 10 | master-slave replication | yes | yes | 11 | "force" flag | yes | yes | 12 | pre-post grants SQL action | yes | yes | 13 | initialization options | yes | yes | 14 | my.cnf options | yes | yes | 15 | custom my.cnf | yes | yes | 16 | friendly UUID generation | yes | yes | 17 | global commands | yes | yes | 18 | test replication flow | yes | yes | 19 | delete command | yes [^2] | yes | 20 | show data dictionary tables | yes | yes | 21 | lock/unlock sandboxes | yes | yes | 22 | finding free ports | yes | yes | 23 | group replication SP | no | yes | 24 | group replication MP | no | yes | 25 | prevent port collision | no | yes [^3] | 26 | visible initialization | no | yes [^4] | 27 | visible script templates | no | yes [^5] | 28 | replaceable templates | no | yes [^6] | 29 | configurable defaults | no | yes [^7] | 30 | list of source binaries | no | yes [^8] | 31 | list of installed sandboxes | no | yes [^9] | 32 | test script per sandbox | no | yes [^10] | 33 | integrated usage help | no | yes [^11] | 34 | custom abbreviations | no | yes [^12] | 35 | version flag | no | yes [^13] | 36 | sandboxes global catalog | no | yes | 37 | concurrent deployment | no | yes | 38 | fan-in | no | yes [^14] | 39 | all-masters | no | yes [^15] | 40 | galera | no | no | yes [^16] 41 | mysql cluster | no | no | yes [^16] 42 | pre-post grants shell action| yes | no | maybe 43 | getting remote tarballs | yes | no | yes 44 | load plugins | yes | yes [^17] | 45 | circular replication | yes | no | no [^18] 46 | master-master (circular) | yes | no | no 47 | Windows support | no | no [^19] | 48 | 49 | [^1]: It's achieved using --export_binaries and then abandoning the operation. 50 | 51 | [^2]: Uses the sbtool command 52 | 53 | [^3]: dbdeployer sandboxes store their ports in a description JSON file, which allows the tool to get a list of used ports and act before a conflict happens. 54 | 55 | [^4]: The initialization happens with a script that is generated and stored in the sandbox itself. Users can inspect the *init_db* script and see what was executed. 56 | 57 | [^5]: All sandbox scripts are generated using templates, which can be examined and eventually changed and re-imported. 58 | 59 | [^6]: See also note 5. Using the flag --use-template you can replace an existing template on-the-fly. Group of templates can be exported and imported after editing. 60 | 61 | [^7]: Defaults can be exported to file, and eventually re-imported after editing. 62 | 63 | [^8]: This is little more than using an O.S. file listing, with the added awareness of the source directory. 64 | 65 | [^9]: Using the description files, this command lists the sandboxes with their topology and used ports. 66 | 67 | [^10]: It's a basic test that checks whether the sandbox is running and is using the expected port. 68 | 69 | [^11]: The "usage" command will show basic commands for single and multiple sandboxes. 70 | 71 | [^12]: the abbreviations file allows user to define custom shortcuts for frequently used commands. 72 | 73 | [^13]: Strangely enough, this simple feature was never implemented for MySQL-Sandbox, while it was one of the first additions to dbdeployer. 74 | 75 | [^14]: Uses the multi source technology introduced in MySQL 5.7. 76 | 77 | [^15]: Same as n. 13. 78 | 79 | [^16]: I may need some help on those. 80 | 81 | [^17]: Using pre-grants and post-grants options, all plugins can be loaded. 82 | 83 | [^18]: Circular replication should not be used anymore. There are enough good alternatives (multi-source, group replication) to avoid this old technology. 84 | 85 | [^19]: I don't do Windows. But you can fork the project if you do. 86 | -------------------------------------------------------------------------------- /cmd/defaults.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "github.com/datacharmer/dbdeployer/defaults" 22 | "github.com/spf13/cobra" 23 | "os" 24 | ) 25 | 26 | func ShowDefaults(cmd *cobra.Command, args []string) { 27 | defaults.ShowDefaults(defaults.Defaults()) 28 | } 29 | 30 | func WriteDefaults(cmd *cobra.Command, args []string) { 31 | defaults.WriteDefaultsFile(defaults.ConfigurationFile, defaults.Defaults()) 32 | fmt.Printf("# Default values exported to %s\n",defaults.ConfigurationFile) 33 | } 34 | 35 | func RemoveDefaults(cmd *cobra.Command, args []string) { 36 | defaults.RemoveDefaultsFile() 37 | } 38 | 39 | func LoadDefaults(cmd *cobra.Command, args []string) { 40 | if len(args) < 1 { 41 | fmt.Printf("'load' requires a file name\n") 42 | os.Exit(1) 43 | } 44 | filename := args[0] 45 | new_defaults := defaults.ReadDefaultsFile(filename) 46 | if defaults.ValidateDefaults(new_defaults) { 47 | defaults.WriteDefaultsFile(defaults.ConfigurationFile, new_defaults) 48 | } else { 49 | return 50 | } 51 | fmt.Printf("Defaults imported from %s into %s\n",filename, defaults.ConfigurationFile) 52 | } 53 | 54 | func ExportDefaults(cmd *cobra.Command, args []string) { 55 | if len(args) < 1 { 56 | fmt.Printf("'export' requires a file name\n") 57 | os.Exit(1) 58 | } 59 | filename := args[0] 60 | if common.FileExists(filename) { 61 | fmt.Printf("File %s already exists. Will not overwrite\n", filename) 62 | os.Exit(1) 63 | } 64 | defaults.WriteDefaultsFile(filename, defaults.Defaults()) 65 | fmt.Printf("# Defaults exported to file %s\n", filename) 66 | } 67 | 68 | func UpdateDefaults(cmd *cobra.Command, args []string) { 69 | if len(args) < 2 { 70 | fmt.Printf("'update' requires a label and a value\n") 71 | fmt.Printf("Example: dbdeployer defaults update master-slave-base-port 17500\n") 72 | os.Exit(1) 73 | } 74 | label := args[0] 75 | value := args[1] 76 | defaults.UpdateDefaults(label, value, true) 77 | defaults.ShowDefaults(defaults.Defaults()) 78 | } 79 | 80 | var ( 81 | defaultsCmd = &cobra.Command{ 82 | Use: "defaults", 83 | Short: "tasks related to dbdeployer defaults", 84 | Aliases: []string{"config"}, 85 | Long: `Runs commands related to the administration of dbdeployer, 86 | such as showing the defaults and saving new ones.`, 87 | } 88 | 89 | defaultsShowCmd = &cobra.Command{ 90 | Use: "show", 91 | Short: "shows defaults", 92 | Aliases: []string{"list"}, 93 | Long: `Shows currently defined defaults`, 94 | Run: ShowDefaults, 95 | } 96 | 97 | defaultsLoadCmd = &cobra.Command{ 98 | Use: "load file_name", 99 | Short: "Load defaults from file", 100 | Aliases: []string{"import"}, 101 | Long: fmt.Sprintf(`Reads defaults from file and saves them to dbdeployer configuration file (%s)`, defaults.ConfigurationFile), 102 | Run: LoadDefaults, 103 | } 104 | 105 | defaultsUpdateCmd = &cobra.Command{ 106 | Use: "update label value", 107 | Short: "Load defaults from file", 108 | Example: ` 109 | $ dbdeployer defaults update master-slave-base-port 17500 110 | `, 111 | Long: `Updates one field of the defaults. Stores the result in the dbdeployer configuration file. 112 | Use "dbdeployer defaults show" to see which values are available`, 113 | Run: UpdateDefaults, 114 | } 115 | 116 | defaultsExportCmd = &cobra.Command{ 117 | Use: "export filename", 118 | Short: "Export current defaults to a given file", 119 | Long: `Saves current defaults to a user-defined file`, 120 | Run: ExportDefaults, 121 | } 122 | 123 | defaultsStoreCmd = &cobra.Command{ 124 | Use: "store", 125 | Short: "Store current defaults", 126 | Long: fmt.Sprintf(`Saves current defaults to dbdeployer configuration file (%s)`, defaults.ConfigurationFile), 127 | Run: WriteDefaults, 128 | } 129 | 130 | defaultsRemoveCmd = &cobra.Command{ 131 | Use: "reset", 132 | Aliases: []string{"remove"}, 133 | Short: "Remove current defaults file", 134 | Long: fmt.Sprintf(`Removes current dbdeployer configuration file (%s)`, defaults.ConfigurationFile) + ` 135 | Afterwards, dbdeployer will use the internally stored defaults. 136 | `, 137 | Run: RemoveDefaults, 138 | } 139 | 140 | ) 141 | 142 | func init() { 143 | rootCmd.AddCommand(defaultsCmd) 144 | defaultsCmd.AddCommand(defaultsStoreCmd) 145 | defaultsCmd.AddCommand(defaultsShowCmd) 146 | defaultsCmd.AddCommand(defaultsRemoveCmd) 147 | defaultsCmd.AddCommand(defaultsLoadCmd) 148 | defaultsCmd.AddCommand(defaultsUpdateCmd) 149 | defaultsCmd.AddCommand(defaultsExportCmd) 150 | } 151 | -------------------------------------------------------------------------------- /cmd/tree.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | // +build docs 17 | 18 | package cmd 19 | 20 | import ( 21 | "fmt" 22 | "os" 23 | "strings" 24 | "github.com/spf13/cobra" 25 | "github.com/spf13/cobra/doc" 26 | "github.com/datacharmer/dbdeployer/common" 27 | ) 28 | 29 | func WriteApi(show_hidden bool) { 30 | fmt.Println("This is the list of commands and modifiers available for") 31 | fmt.Println("dbdeployer {{.Version}} as of {{.Date}}") 32 | fmt.Println("") 33 | fmt.Println("# main") 34 | fmt.Println("{{dbdeployer -h }}") 35 | fmt.Println("") 36 | fmt.Println("{{dbdeployer-docs tree }}") 37 | traverse(rootCmd, "", 0, true, show_hidden) 38 | } 39 | 40 | func WriteBashCompletion() { 41 | completion_file := "dbdeployer_completion.sh" 42 | rootCmd.GenBashCompletionFile(completion_file) 43 | fmt.Printf("Copy %s to the completion directory (/etc/bash_completion.d or /usr/local/etc/bash_completion.d)\n",completion_file) 44 | 45 | } 46 | 47 | func WriteManPages() { 48 | man_dir := "man_pages" 49 | if common.DirExists(man_dir) { 50 | fmt.Printf("manual pages directory '%s' exists already.\n",man_dir) 51 | os.Exit(1) 52 | } 53 | common.Mkdir(man_dir) 54 | header := &doc.GenManHeader{ 55 | Title: "dbdeployer", 56 | Section: "1", 57 | } 58 | err := doc.GenManTree(rootCmd, header, man_dir) 59 | if err != nil { 60 | fmt.Println(err) 61 | os.Exit(1) 62 | } 63 | fmt.Printf("Man pages generated in '%s'\n", man_dir) 64 | } 65 | 66 | func WriteMarkdownPages() { 67 | md_dir := "markdown_pages" 68 | if common.DirExists(md_dir) { 69 | fmt.Printf("Markdown pages directory '%s' exists already.\n",md_dir) 70 | os.Exit(1) 71 | } 72 | common.Mkdir(md_dir) 73 | err := doc.GenMarkdownTree(rootCmd, md_dir) 74 | if err != nil { 75 | fmt.Println(err) 76 | os.Exit(1) 77 | } 78 | err = doc.GenReSTTree(rootCmd, md_dir) 79 | fmt.Printf("Markdown pages generated in '%s'\n", md_dir) 80 | } 81 | 82 | func WriteRstPages() { 83 | rst_dir := "rst_pages" 84 | if common.DirExists(rst_dir) { 85 | fmt.Printf("Restructured Text pages directory '%s' exists already.\n",rst_dir) 86 | os.Exit(1) 87 | } 88 | common.Mkdir(rst_dir) 89 | err := doc.GenReSTTree(rootCmd, rst_dir) 90 | if err != nil { 91 | fmt.Println(err) 92 | os.Exit(1) 93 | } 94 | fmt.Printf("Restructured Text pages generated in '%s'\n", rst_dir) 95 | } 96 | 97 | func MakeDocumentation(cmd *cobra.Command, args []string) { 98 | flags := cmd.Flags() 99 | api, _ := flags.GetBool("api") 100 | show_hidden, _ := flags.GetBool("show-hidden") 101 | bash_completion, _ := flags.GetBool("bash-completion") 102 | man_pages, _ := flags.GetBool("man-pages") 103 | md_pages, _ := flags.GetBool("markdown-pages") 104 | rst_pages, _ := flags.GetBool("rst-pages") 105 | if (man_pages && api) || (api && bash_completion) || (api && md_pages) || (api && rst_pages) { 106 | fmt.Printf("Choose one option only\n") 107 | os.Exit(1) 108 | } 109 | if rst_pages { 110 | WriteRstPages() 111 | return 112 | } 113 | if man_pages { 114 | WriteManPages() 115 | return 116 | } 117 | if md_pages { 118 | WriteMarkdownPages() 119 | return 120 | } 121 | if bash_completion { 122 | WriteBashCompletion() 123 | return 124 | } 125 | if api { 126 | WriteApi(show_hidden) 127 | return 128 | } 129 | traverse(rootCmd, "", 0, api, show_hidden) 130 | } 131 | 132 | func traverse(cmd *cobra.Command, parent string, level int, api, show_hidden bool) { 133 | subcommands := cmd.Commands() 134 | indent := strings.Repeat(" ", level*4) + "-" 135 | for _, c := range subcommands { 136 | hidden_flag := "" 137 | if c.Hidden || c.Name() == "help" { 138 | if show_hidden { 139 | hidden_flag = " (HIDDEN) " 140 | } else { 141 | continue 142 | } 143 | } 144 | size := len(c.Commands()) 145 | if api { 146 | if size > 0 || level == 0 { 147 | fmt.Printf("\n##%s%s\n", parent + " " + c.Name(), hidden_flag) 148 | } 149 | fmt.Printf("{{dbdeployer%s %s -h}}\n", parent, c.Name()) 150 | } else { 151 | fmt.Printf("%s %-20s%s\n", indent, c.Name(), hidden_flag) 152 | } 153 | if size > 0 { 154 | traverse(c, parent + " " + c.Name(), level + 1, api, show_hidden) 155 | } 156 | } 157 | } 158 | 159 | var treeCmd = &cobra.Command{ 160 | Use: "tree", 161 | Short: "shows command tree and other docs", 162 | Aliases: []string{"docs"}, 163 | Long: `This command is only used to create API documentation. 164 | You can, however, use it to show the command structure at a glance.`, 165 | Hidden : true, 166 | Run: MakeDocumentation, 167 | } 168 | 169 | func init() { 170 | rootCmd.AddCommand(treeCmd) 171 | treeCmd.PersistentFlags().Bool("man-pages", false, "Writes man pages") 172 | treeCmd.PersistentFlags().Bool("markdown-pages", false, "Writes Markdown docs") 173 | treeCmd.PersistentFlags().Bool("rst-pages", false, "Writes Restructured Text docs") 174 | treeCmd.PersistentFlags().Bool("api", false, "Writes API template") 175 | treeCmd.PersistentFlags().Bool("bash-completion", false, "creates bash-completion file") 176 | treeCmd.PersistentFlags().Bool("show-hidden", false, "Shows also hidden commands") 177 | } 178 | -------------------------------------------------------------------------------- /sandbox/multi_templates.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package sandbox 17 | 18 | // Templates for multiple sandboxes 19 | 20 | var ( 21 | start_multi_template string = `#!/bin/sh 22 | {{.Copyright}} 23 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 24 | SBDIR={{.SandboxDir}} 25 | echo "# executing 'start' on $SBDIR" 26 | {{ range .Nodes }} 27 | echo 'executing "start" on {{.NodeLabel}} {{.Node}}' 28 | $SBDIR/{{.NodeLabel}}{{.Node}}/start "$@" 29 | {{end}} 30 | ` 31 | restart_multi_template string = `#!/bin/sh 32 | {{.Copyright}} 33 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 34 | SBDIR={{.SandboxDir}} 35 | $SBDIR/stop_all 36 | $SBDIR/start_all "$@" 37 | ` 38 | use_multi_template string = `#!/bin/sh 39 | {{.Copyright}} 40 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 41 | SBDIR={{.SandboxDir}} 42 | if [ "$1" = "" ] 43 | then 44 | echo "syntax: $0 command" 45 | exit 1 46 | fi 47 | 48 | {{range .Nodes}} 49 | echo "# server: {{.Node}} " 50 | echo "$@" | $SBDIR/{{.NodeLabel}}{{.Node}}/use $MYCLIENT_OPTIONS 51 | {{end}} 52 | ` 53 | stop_multi_template string = `#!/bin/sh 54 | {{.Copyright}} 55 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 56 | SBDIR={{.SandboxDir}} 57 | echo "# executing 'stop' on $SBDIR" 58 | {{ range .Nodes }} 59 | echo 'executing "stop" on {{.NodeLabel}} {{.Node}}' 60 | $SBDIR/{{.NodeLabel}}{{.Node}}/stop "$@" 61 | {{end}} 62 | ` 63 | send_kill_multi_template string = `#!/bin/sh 64 | {{.Copyright}} 65 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 66 | SBDIR={{.SandboxDir}} 67 | echo "# executing 'send_kill' on $SBDIR" 68 | {{ range .Nodes }} 69 | echo 'executing "send_kill" on {{.NodeLabel}} {{.Node}}' 70 | $SBDIR/{{.NodeLabel}}{{.Node}}/send_kill "$@" 71 | {{end}} 72 | ` 73 | clear_multi_template string = `#!/bin/sh 74 | {{.Copyright}} 75 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 76 | SBDIR={{.SandboxDir}} 77 | echo "# executing 'clear' on $SBDIR" 78 | {{range .Nodes}} 79 | echo 'executing "clear" on {{.NodeLabel}} {{.Node}}' 80 | $SBDIR/{{.NodeLabel}}{{.Node}}/clear "$@" 81 | {{end}} 82 | ` 83 | status_multi_template string = `#!/bin/sh 84 | {{.Copyright}} 85 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 86 | SBDIR={{.SandboxDir}} 87 | echo "MULTIPLE $SBDIR" 88 | {{ range .Nodes }} 89 | nstatus=$($SBDIR/{{.NodeLabel}}{{.Node}}/status ) 90 | if [ -f $SBDIR/{{.NodeLabel}}{{.Node}}/data/mysql_sandbox{{.NodePort}}.pid ] 91 | then 92 | nport=$($SBDIR/{{.NodeLabel}}{{.Node}}/use -BN -e "show variables like 'port'") 93 | fi 94 | echo "{{.NodeLabel}}{{.Node}} : $nstatus - $nport ({{.NodePort}})" 95 | {{end}} 96 | ` 97 | test_sb_multi_template string = `#!/bin/sh 98 | {{.Copyright}} 99 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 100 | SBDIR={{.SandboxDir}} 101 | echo "# executing 'test_sb' on $SBDIR" 102 | {{ range .Nodes }} 103 | echo 'executing "test_sb" on {{.NodeLabel}} {{.Node}}' 104 | $SBDIR/{{.NodeLabel}}{{.Node}}/test_sb "$@" 105 | exit_code=$? 106 | if [ "$exit_code" != "0" ] ; then exit $exit_code ; fi 107 | {{end}} 108 | ` 109 | 110 | node_template string = `#!/bin/sh 111 | {{.Copyright}} 112 | # Generated by dbdeployer {{.AppVersion}} using {{.TemplateName}} on {{.DateTime}} 113 | SBDIR={{.SandboxDir}} 114 | $SBDIR/{{.NodeLabel}}{{.Node}}/use "$@" 115 | ` 116 | 117 | MultipleTemplates = TemplateCollection{ 118 | "start_multi_template": TemplateDesc{ 119 | Description: "Starts all nodes (with optional mysqld arguments)", 120 | Notes: "", 121 | Contents: start_multi_template, 122 | }, 123 | "restart_multi_template": TemplateDesc{ 124 | Description: "Restarts all nodes (with optional mysqld arguments)", 125 | Notes: "", 126 | Contents: restart_multi_template, 127 | }, 128 | "use_multi_template": TemplateDesc{ 129 | Description: "Runs the same SQL query in all nodes", 130 | Notes: "", 131 | Contents: use_multi_template, 132 | }, 133 | "stop_multi_template": TemplateDesc{ 134 | Description: "Stops all nodes", 135 | Notes: "", 136 | Contents: stop_multi_template, 137 | }, 138 | "send_kill_multi_template": TemplateDesc{ 139 | Description: "Sends kill signal to all nodes", 140 | Notes: "", 141 | Contents: send_kill_multi_template, 142 | }, 143 | "clear_multi_template": TemplateDesc{ 144 | Description: "Removes data from all nodes", 145 | Notes: "", 146 | Contents: clear_multi_template, 147 | }, 148 | "status_multi_template": TemplateDesc{ 149 | Description: "Shows status for all nodes", 150 | Notes: "", 151 | Contents: status_multi_template, 152 | }, 153 | "test_sb_multi_template": TemplateDesc{ 154 | Description: "Run sb test on all nodes", 155 | Notes: "", 156 | Contents: test_sb_multi_template, 157 | }, 158 | "node_template": TemplateDesc{ 159 | Description: "Runs the MySQL client for a given node", 160 | Notes: "", 161 | Contents: node_template, 162 | }, 163 | } 164 | ) 165 | -------------------------------------------------------------------------------- /abbreviations/abbreviations.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package abbreviations 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "os" 22 | "regexp" 23 | "strings" 24 | ) 25 | 26 | /* 27 | This package implements custom abbreviations. 28 | It looks for a file "abbreviations.txt" and treats every line 29 | as an abbreviation followed by its replacement. 30 | Then, it looks at the command line arguments. 31 | If an argument matches an abbreviation, it will be replaced by the replacement items. 32 | For example, the file contains this line: 33 | sbs sandboxes 34 | 35 | when the user types "dbdeployer sbs", it will be replaced with "dbdeployer sandboxes" 36 | 37 | A more interesting example: 38 | groupr replication --topology=group 39 | 40 | Here, a command "dbdeployer groupr 8.0.4" becomes "dbdeployer deploy replication --topology=group 8.0.4" 41 | 42 | It is also possible to set variables in the replacement. 43 | sbdef --sandbox-directory={{.sb}} --port={{.port}} 44 | 45 | To use this abbreviation, we need to provide the values for 'sb' and 'port' 46 | dbdeployer deploy sbdef:port=9000,sb=mysandbox single 8.0.4 47 | it will become "dbdeployer deploy --sandbox-directory=mysandbox --port=9000 single 8.0.4 48 | */ 49 | 50 | type argList []string 51 | type AliasList map[string]argList 52 | 53 | var DebugAbbr bool = false 54 | 55 | func show_args( args argList) { 56 | for N, arg := range args { 57 | if DebugAbbr { 58 | fmt.Printf("%d <<%s>>\n", N, arg) 59 | } 60 | } 61 | } 62 | 63 | func debug_print(descr string, v interface{}) { 64 | if DebugAbbr { 65 | fmt.Printf("%s : %v\n", descr, v) 66 | } 67 | } 68 | 69 | func LoadAbbreviations() { 70 | if os.Getenv("SKIP_ABBR") != "" { 71 | fmt.Printf("# Abbreviations suppressed by env variable SKIP_ABBR\n") 72 | return 73 | } 74 | var abbrev_file string = "abbreviations.txt" 75 | var new_args []string 76 | var abbreviations = make(AliasList) 77 | var variables = make(common.Smap) 78 | var verbose_abbr bool = true 79 | var replacements_used bool = false 80 | if os.Getenv("SILENT_ABBR") != "" { 81 | verbose_abbr = false 82 | } 83 | user_defined_file := os.Getenv("DBDEPLOYER_ABBR_FILE") 84 | if user_defined_file != "" { 85 | abbrev_file = user_defined_file 86 | } 87 | if !common.FileExists(abbrev_file) { 88 | if DebugAbbr { 89 | fmt.Printf("# File %s not found\n", abbrev_file) 90 | } 91 | return 92 | } 93 | abbr_lines := common.SlurpAsLines(abbrev_file) 94 | // Loads abbreviations from file 95 | for _, abbreviation := range abbr_lines { 96 | abbreviation = strings.TrimSpace(abbreviation) 97 | list := strings.Split(abbreviation, " ") 98 | abbr := list[0] 99 | is_comment, _ := regexp.MatchString(`^\s*#`, abbr) 100 | is_empty, _ := regexp.MatchString(`^\s*#\$`, abbr) 101 | if is_comment || is_empty { 102 | continue 103 | } 104 | var new_list argList 105 | for N, repl := range list { 106 | // Skips the first item, which is the abbreviation 107 | if N > 0 { 108 | new_list = append(new_list, repl) 109 | } 110 | } 111 | abbreviations[abbr] = new_list 112 | } 113 | // Loop through original arguments. 114 | // Replaces every occurrence of the abbreviation with its components 115 | debug_print("os.Args", os.Args) 116 | show_args( os.Args) 117 | for _, arg := range os.Args { 118 | // An abbreviation may set variables 119 | // for example 120 | // myabbr:varname=var_value 121 | // myabbr:varname=var_value,other_var=other_value 122 | re := regexp.MustCompile(`(\w+)[-:](\S+)`) 123 | re_flag := regexp.MustCompile(`^-`) 124 | vars := re.FindStringSubmatch(arg) 125 | if re_flag.MatchString(arg) { 126 | new_args = append(new_args, arg) 127 | continue 128 | } 129 | if len(vars) > 0 { 130 | arg = vars[1] 131 | all_vars := vars[2] 132 | 133 | // Keys and values are separated by an equals (=) sign 134 | re = regexp.MustCompile(`(\w+)=(\w+)`) 135 | uvars := re.FindAllStringSubmatch(all_vars, -1) 136 | for _, vgroup := range uvars { 137 | variables[string(vgroup[1])] = string(vgroup[2]) 138 | } 139 | } 140 | // If exists an abbreviation for the current argument 141 | if abbreviations[arg] != nil { 142 | replacement := "" 143 | for _, item := range abbreviations[arg] { 144 | if item != "" { 145 | // Replaces possible vars with their value 146 | item = common.Tprintf(item, variables) 147 | // adds the replacement items to the new argument list 148 | replacement += " " + item 149 | new_args = append(new_args, item) 150 | replacements_used = true 151 | } 152 | } 153 | if verbose_abbr { 154 | fmt.Printf("# %s => %s\n", arg, replacement) 155 | } 156 | } else { 157 | // If there is no abbreviation for the current argument 158 | // it is added as it is. 159 | new_args = append(new_args, arg) 160 | } 161 | } 162 | debug_print("new_args", new_args) 163 | // Arguments replaced! 164 | if replacements_used { 165 | if DebugAbbr { 166 | fmt.Printf("# Using file %s\n", abbrev_file) 167 | } 168 | os.Args = new_args 169 | if verbose_abbr { 170 | fmt.Printf("# %s\n", os.Args) 171 | } 172 | } 173 | } 174 | 175 | func init() { 176 | if os.Getenv("DEBUG_ABBR") != "" { 177 | DebugAbbr = true 178 | } 179 | } 180 | -------------------------------------------------------------------------------- /cmd/global.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "github.com/spf13/cobra" 22 | "os" 23 | ) 24 | 25 | func GlobalRunCommand(cmd *cobra.Command, executable string, args []string, require_args bool, skip_missing bool) { 26 | flags := cmd.Flags() 27 | sandbox_dir, _ := flags.GetString("sandbox-home") 28 | run_list := common.SandboxInfoToFileNames(common.GetInstalledSandboxes(sandbox_dir)) 29 | if len(run_list) == 0 { 30 | fmt.Printf("No sandboxes found in %s\n", sandbox_dir) 31 | os.Exit(1) 32 | } 33 | if require_args && len(args) < 1 { 34 | fmt.Printf("Arguments required for command %s\n", executable) 35 | os.Exit(1) 36 | } 37 | for _, sb := range run_list { 38 | single_use := true 39 | full_dir_path := sandbox_dir + "/" + sb 40 | cmd_file := full_dir_path + "/" + executable 41 | real_executable := executable 42 | if !common.ExecExists(cmd_file) { 43 | cmd_file = full_dir_path + "/" + executable + "_all" 44 | real_executable = executable + "_all" 45 | single_use = false 46 | } 47 | if !common.ExecExists(cmd_file) { 48 | if skip_missing { 49 | fmt.Printf("# Sandbox %s: executable %s not found\n",full_dir_path, executable) 50 | continue 51 | } 52 | fmt.Printf("No %s or %s found in %s\n", executable, executable+"_all", full_dir_path) 53 | os.Exit(1) 54 | } 55 | var cmd_args []string 56 | 57 | if single_use && executable == "use" { 58 | cmd_args = append(cmd_args, "-e") 59 | } 60 | for _, arg := range args { 61 | cmd_args = append(cmd_args, arg) 62 | } 63 | var err error 64 | fmt.Printf("# Running \"%s\" on %s\n", real_executable, sb) 65 | if len(cmd_args) > 0 { 66 | err, _ = common.Run_cmd_with_args(cmd_file, cmd_args) 67 | } else { 68 | err, _ = common.Run_cmd(cmd_file) 69 | } 70 | if err != nil { 71 | fmt.Printf("Error while running %s\n", cmd_file) 72 | os.Exit(1) 73 | } 74 | fmt.Println("") 75 | } 76 | } 77 | 78 | func StartAllSandboxes(cmd *cobra.Command, args []string) { 79 | GlobalRunCommand(cmd, "start", args, false, false) 80 | } 81 | 82 | func RestartAllSandboxes(cmd *cobra.Command, args []string) { 83 | GlobalRunCommand(cmd, "restart", args, false, false) 84 | } 85 | 86 | func StopAllSandboxes(cmd *cobra.Command, args []string) { 87 | GlobalRunCommand(cmd, "stop", args, false, false) 88 | } 89 | 90 | func StatusAllSandboxes(cmd *cobra.Command, args []string) { 91 | GlobalRunCommand(cmd, "status", args, false, false) 92 | } 93 | 94 | func TestAllSandboxes(cmd *cobra.Command, args []string) { 95 | GlobalRunCommand(cmd, "test_sb", args, false, false) 96 | } 97 | 98 | func TestReplicationAllSandboxes(cmd *cobra.Command, args []string) { 99 | GlobalRunCommand(cmd, "test_replication", args, false, true) 100 | } 101 | 102 | func UseAllSandboxes(cmd *cobra.Command, args []string) { 103 | GlobalRunCommand(cmd, "use", args, true, false) 104 | } 105 | 106 | var ( 107 | globalCmd = &cobra.Command{ 108 | Use: "global", 109 | Short: "Runs a given command in every sandbox", 110 | Long: `This command can propagate the given action through all sandboxes.`, 111 | Example: ` 112 | $ dbdeployer global use "select version()" 113 | $ dbdeployer global status 114 | $ dbdeployer global stop 115 | `, 116 | } 117 | 118 | globalStartCmd = &cobra.Command{ 119 | Use: "start [options]", 120 | Short: "Starts all sandboxes", 121 | Long: ``, 122 | Run: StartAllSandboxes, 123 | } 124 | 125 | globalRestartCmd = &cobra.Command{ 126 | Use: "restart [options]", 127 | Short: "Restarts all sandboxes", 128 | Long: ``, 129 | Run: RestartAllSandboxes, 130 | } 131 | 132 | globalStopCmd = &cobra.Command{ 133 | Use: "stop", 134 | Short: "Stops all sandboxes", 135 | Long: ``, 136 | Run: StopAllSandboxes, 137 | } 138 | globalStatusCmd = &cobra.Command{ 139 | Use: "status", 140 | Short: "Shows the status in all sandboxes", 141 | Long: ``, 142 | Run: StatusAllSandboxes, 143 | } 144 | 145 | globalTestCmd = &cobra.Command{ 146 | Use: "test", 147 | Aliases: []string{"test_sb", "test-sb"}, 148 | Short: "Tests all sandboxes", 149 | Long: ``, 150 | Run: TestAllSandboxes, 151 | } 152 | 153 | globalTestReplicationCmd = &cobra.Command{ 154 | Use: "test-replication", 155 | Aliases: []string{"test_replication"}, 156 | Short: "Tests replication in all sandboxes", 157 | Long: ``, 158 | Run: TestReplicationAllSandboxes, 159 | } 160 | 161 | globalUseCmd = &cobra.Command{ 162 | Use: "use {query}", 163 | Short: "Runs a query in all sandboxes", 164 | Long: `Runs a query in all sandboxes. 165 | It does not check if the query is compatible with every version deployed. 166 | For example, a query using @@port won't run in MySQL 5.0.x`, 167 | Example: ` 168 | $ dbdeployer global use "select @@server_id, @@port"`, 169 | Run: UseAllSandboxes, 170 | } 171 | ) 172 | 173 | func init() { 174 | rootCmd.AddCommand(globalCmd) 175 | globalCmd.AddCommand(globalStartCmd) 176 | globalCmd.AddCommand(globalRestartCmd) 177 | globalCmd.AddCommand(globalStopCmd) 178 | globalCmd.AddCommand(globalStatusCmd) 179 | globalCmd.AddCommand(globalTestCmd) 180 | globalCmd.AddCommand(globalTestReplicationCmd) 181 | globalCmd.AddCommand(globalUseCmd) 182 | 183 | } 184 | -------------------------------------------------------------------------------- /sandbox/multiple.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package sandbox 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "github.com/datacharmer/dbdeployer/concurrent" 22 | "github.com/datacharmer/dbdeployer/defaults" 23 | "os" 24 | "time" 25 | ) 26 | 27 | type Node struct { 28 | Node int 29 | Port int 30 | ServerId int 31 | Name string 32 | } 33 | 34 | func CreateMultipleSandbox(sdef SandboxDef, origin string, nodes int) common.Smap { 35 | 36 | var exec_lists []concurrent.ExecutionList 37 | sb_type := sdef.SBType 38 | if sb_type == "" { 39 | sb_type = "multiple" 40 | } 41 | Basedir := sdef.Basedir + "/" + sdef.Version 42 | if !common.DirExists(Basedir) { 43 | fmt.Printf("Base directory %s does not exist\n", Basedir) 44 | os.Exit(1) 45 | } 46 | if sdef.DirName == "" { 47 | sdef.SandboxDir += "/" + defaults.Defaults().MultiplePrefix + common.VersionToName(origin) 48 | } else { 49 | sdef.SandboxDir += "/" + sdef.DirName 50 | } 51 | if common.DirExists(sdef.SandboxDir) { 52 | sdef = CheckDirectory(sdef) 53 | } 54 | 55 | vList := common.VersionToList(sdef.Version) 56 | rev := vList[2] 57 | // base_port := sdef.Port + defaults.Defaults().MultipleBasePort + (rev * 100) 58 | base_port := sdef.Port + defaults.Defaults().MultipleBasePort + rev 59 | if sdef.BasePort > 0 { 60 | base_port = sdef.BasePort 61 | } 62 | base_mysqlx_port := get_base_mysqlx_port(base_port, sdef, nodes) 63 | base_port = FindFreePort(base_port, sdef.InstalledPorts, nodes) 64 | for check_port := base_port + 1; check_port < base_port+nodes; check_port++ { 65 | CheckPort(sdef.SandboxDir, sdef.InstalledPorts, check_port) 66 | } 67 | common.Mkdir(sdef.SandboxDir) 68 | 69 | sdef.ReplOptions = SingleTemplates["replication_options"].Contents 70 | base_server_id := 0 71 | if nodes < 2 { 72 | fmt.Println("For single sandbox deployment, use the 'single' command") 73 | os.Exit(1) 74 | } 75 | timestamp := time.Now() 76 | var data common.Smap = common.Smap{ 77 | "Copyright": Copyright, 78 | "AppVersion": common.VersionDef, 79 | "DateTime": timestamp.Format(time.UnixDate), 80 | "SandboxDir": sdef.SandboxDir, 81 | "Nodes": []common.Smap{}, 82 | } 83 | 84 | sb_desc := common.SandboxDescription{ 85 | Basedir: Basedir, 86 | SBType: sdef.SBType, 87 | Version: sdef.Version, 88 | Port: []int{}, 89 | Nodes: nodes, 90 | NodeNum: 0, 91 | } 92 | 93 | sb_item := defaults.SandboxItem{ 94 | Origin : sb_desc.Basedir, 95 | SBType : sb_desc.SBType, 96 | Version: sdef.Version, 97 | Port: []int{}, 98 | Nodes: []string{}, 99 | Destination: sdef.SandboxDir, 100 | } 101 | 102 | node_label := defaults.Defaults().NodePrefix 103 | for i := 1; i <= nodes; i++ { 104 | sdef.Port = base_port + i 105 | data["Nodes"] = append(data["Nodes"].([]common.Smap), common.Smap{ 106 | "Copyright": Copyright, 107 | "AppVersion": common.VersionDef, 108 | "DateTime": timestamp.Format(time.UnixDate), 109 | "Node": i, 110 | "NodePort": sdef.Port, 111 | "NodeLabel": node_label, 112 | "SandboxDir": sdef.SandboxDir, 113 | }) 114 | sdef.LoadGrants = true 115 | sdef.DirName = fmt.Sprintf("%s%d", node_label, i) 116 | sdef.ServerId = (base_server_id + i) * 100 117 | sb_item.Nodes = append(sb_item.Nodes, sdef.DirName) 118 | sb_item.Port = append(sb_item.Port, sdef.Port) 119 | sb_desc.Port = append(sb_desc.Port, sdef.Port) 120 | if common.GreaterOrEqualVersion(sdef.Version, []int{8,0,11}) { 121 | sdef.MysqlXPort = base_mysqlx_port + i 122 | if !sdef.DisableMysqlX { 123 | sb_desc.Port = append(sb_desc.Port, base_mysqlx_port + i) 124 | sb_item.Port = append(sb_item.Port, base_mysqlx_port + i) 125 | } 126 | } 127 | sdef.Multi = true 128 | sdef.NodeNum = i 129 | sdef.Prompt = fmt.Sprintf("%s%d",node_label, i) 130 | sdef.SBType += "node" 131 | if ! sdef.RunConcurrently { 132 | fmt.Printf("Installing and starting %s %d\n", node_label, i) 133 | } 134 | exec_list := CreateSingleSandbox(sdef, origin) 135 | for _, list := range exec_list { 136 | exec_lists = append(exec_lists, list) 137 | } 138 | 139 | var data_node common.Smap = common.Smap{ 140 | "Node": i, 141 | "NodePort": sdef.Port, 142 | "NodeLabel": node_label, 143 | "SandboxDir": sdef.SandboxDir, 144 | "Copyright": Copyright, 145 | } 146 | write_script(MultipleTemplates, fmt.Sprintf("n%d", i), "node_template", sdef.SandboxDir, data_node, true) 147 | } 148 | common.WriteSandboxDescription(sdef.SandboxDir, sb_desc) 149 | defaults.UpdateCatalog(sdef.SandboxDir, sb_item) 150 | 151 | write_script(MultipleTemplates, "start_all", "start_multi_template", sdef.SandboxDir, data, true) 152 | write_script(MultipleTemplates, "restart_all", "restart_multi_template", sdef.SandboxDir, data, true) 153 | write_script(MultipleTemplates, "status_all", "status_multi_template", sdef.SandboxDir, data, true) 154 | write_script(MultipleTemplates, "test_sb_all", "test_sb_multi_template", sdef.SandboxDir, data, true) 155 | write_script(MultipleTemplates, "stop_all", "stop_multi_template", sdef.SandboxDir, data, true) 156 | write_script(MultipleTemplates, "clear_all", "clear_multi_template", sdef.SandboxDir, data, true) 157 | write_script(MultipleTemplates, "send_kill_all", "send_kill_multi_template", sdef.SandboxDir, data, true) 158 | write_script(MultipleTemplates, "use_all", "use_multi_template", sdef.SandboxDir, data, true) 159 | concurrent.RunParallelTasksByPriority(exec_lists) 160 | 161 | fmt.Printf("%s directory installed in %s\n", sb_type, common.ReplaceLiteralHome(sdef.SandboxDir)) 162 | fmt.Printf("run 'dbdeployer usage multiple' for basic instructions'\n") 163 | return data 164 | } 165 | -------------------------------------------------------------------------------- /cmd/sandboxes.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | 21 | "github.com/datacharmer/dbdeployer/common" 22 | "github.com/datacharmer/dbdeployer/defaults" 23 | "github.com/spf13/cobra" 24 | "strings" 25 | ) 26 | 27 | func ShowSandboxesFromCatalog(current_sandbox_home string, header bool) { 28 | sandbox_list := defaults.ReadCatalog() 29 | if len(sandbox_list) == 0 { 30 | return 31 | } 32 | template := "%-25s %-10s %-20s %5v %-25s %s \n" 33 | if header { 34 | fmt.Printf( template, "name", "version", "type", "nodes", "ports", "") 35 | fmt.Printf( template, "----", "-------", "-----", "-----", "-----", "") 36 | } 37 | for name, contents := range sandbox_list { 38 | ports := "[" 39 | for _, p := range contents.Port { 40 | ports += fmt.Sprintf("%d ", p) 41 | } 42 | ports += "]" 43 | extra := "" 44 | if ! strings.HasPrefix(contents.Destination, current_sandbox_home) { 45 | extra = "(" + common.DirName(contents.Destination) + ")" 46 | } 47 | fmt.Printf(template, common.BaseName(name), contents.Version, contents.SBType, len(contents.Nodes), ports, extra) 48 | } 49 | } 50 | 51 | 52 | // Shows installed sandboxes 53 | func ShowSandboxes(cmd *cobra.Command, args []string) { 54 | flags := cmd.Flags() 55 | SandboxHome, _ := flags.GetString("sandbox-home") 56 | read_catalog, _ := flags.GetBool("catalog") 57 | use_header, _ := flags.GetBool("header") 58 | if read_catalog { 59 | ShowSandboxesFromCatalog(SandboxHome, use_header) 60 | return 61 | } 62 | sandbox_list := common.GetInstalledSandboxes(SandboxHome) 63 | var dirs []string 64 | for _, sbinfo := range sandbox_list { 65 | //fname := f.Name() 66 | //fmode := f.Mode() 67 | //if fmode.IsDir() { 68 | fname := sbinfo.SandboxName 69 | description := "single" 70 | sbdesc := SandboxHome + "/" + fname + "/sbdescription.json" 71 | if common.FileExists(sbdesc) { 72 | sbd := common.ReadSandboxDescription(SandboxHome + "/" + fname) 73 | locked := "" 74 | if sbinfo.Locked { 75 | locked = "(LOCKED)" 76 | } 77 | if sbd.Nodes == 0 { 78 | port_text := "" 79 | for _, p := range sbd.Port { 80 | if port_text != "" { 81 | port_text += " " 82 | } 83 | port_text += fmt.Sprintf("%d", p) 84 | } 85 | description = fmt.Sprintf("%-20s %10s [%s]", sbd.SBType, sbd.Version, port_text) 86 | } else { 87 | var node_descr []common.SandboxDescription 88 | inner_files := common.SandboxInfoToFileNames(common.GetInstalledSandboxes(SandboxHome + "/" + fname)) 89 | for _, inner := range inner_files { 90 | inner_sbdesc := SandboxHome + "/" + fname + "/" + inner + "/sbdescription.json" 91 | if common.FileExists(inner_sbdesc) { 92 | sd_node := common.ReadSandboxDescription(fmt.Sprintf("%s/%s/%s", SandboxHome, fname, inner)) 93 | node_descr = append(node_descr, sd_node) 94 | } 95 | } 96 | ports := "" 97 | for _, nd := range node_descr { 98 | for _, p := range nd.Port { 99 | if ports != "" { 100 | ports += " " 101 | } 102 | ports += fmt.Sprintf("%d", p) 103 | } 104 | } 105 | //ports += " ]" 106 | description = fmt.Sprintf("%-20s %10s [%s]", sbd.SBType, sbd.Version, ports) 107 | } 108 | dirs = append(dirs, fmt.Sprintf("%-25s : %s %s", fname, description, locked)) 109 | } else { 110 | locked := "" 111 | no_clear := SandboxHome + "/" + fname + "/no_clear" 112 | no_clear_all := SandboxHome + "/" + fname + "/no_clear_all" 113 | start := SandboxHome + "/" + fname + "/start" 114 | start_all := SandboxHome + "/" + fname + "/start_all" 115 | initialize_slaves := SandboxHome + "/" + fname + "/initialize_slaves" 116 | initialize_nodes := SandboxHome + "/" + fname + "/initialize_nodes" 117 | if common.FileExists(no_clear) || common.FileExists(no_clear_all) { 118 | locked = "(LOCKED)" 119 | } 120 | if common.FileExists(start_all) { 121 | description = "multiple sandbox" 122 | } 123 | if common.FileExists(initialize_slaves) { 124 | description = "master-slave replication" 125 | } 126 | if common.FileExists(initialize_nodes) { 127 | description = "group replication" 128 | } 129 | if common.FileExists(start) || common.FileExists(start_all) { 130 | dirs = append(dirs, fmt.Sprintf("%-20s : *%s* %s ", fname, description, locked)) 131 | } 132 | } 133 | } 134 | if use_header { 135 | // 1 2 3 4 5 6 7 136 | // 12345678901234567890123456789012345678901234567890123456789012345678901234567890 137 | // fan_in_msb_5_7_21 : fan-in 5.7.21 [14001 14002 14003] 138 | template := "%-25s %-23s %-8s %s\n" 139 | fmt.Printf( template, "name", "type", "version", "ports") 140 | fmt.Printf( template, "----------------", "-------", "-------", "-----") 141 | } 142 | for _, dir := range dirs { 143 | fmt.Println(dir) 144 | } 145 | } 146 | 147 | var sandboxesCmd = &cobra.Command{ 148 | Use: "sandboxes", 149 | Short: "List installed sandboxes", 150 | Long: `Lists all sandboxes installed in $SANDBOX_HOME. 151 | If sandboxes are installed in a different location, use --sandbox-home to 152 | indicate where to look. 153 | Alternatively, using --catalog will list all sandboxes, regardless of where 154 | they were deployed. 155 | `, 156 | Aliases: []string{"installed", "deployed"}, 157 | Run: ShowSandboxes, 158 | } 159 | 160 | func init() { 161 | rootCmd.AddCommand(sandboxesCmd) 162 | 163 | sandboxesCmd.Flags().BoolP("catalog", "", false, "Use sandboxes catalog instead of scanning directory") 164 | sandboxesCmd.Flags().BoolP("header", "", false, "Shows header with catalog output") 165 | } 166 | -------------------------------------------------------------------------------- /cmd/admin.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "github.com/datacharmer/dbdeployer/sandbox" 22 | "github.com/spf13/cobra" 23 | "os" 24 | ) 25 | 26 | func UnpreserveSandbox(sandbox_dir, sandbox_name string) { 27 | full_path := sandbox_dir + "/" + sandbox_name 28 | if !common.DirExists(full_path) { 29 | fmt.Printf("Directory '%s' not found\n", full_path) 30 | os.Exit(1) 31 | } 32 | preserve := full_path + "/no_clear_all" 33 | if !common.ExecExists(preserve) { 34 | preserve = full_path + "/no_clear" 35 | } 36 | if !common.ExecExists(preserve) { 37 | fmt.Printf("Sandbox %s is not locked\n",sandbox_name) 38 | return 39 | } 40 | is_multiple := true 41 | clear := full_path + "/clear_all" 42 | if !common.ExecExists(clear) { 43 | clear = full_path + "/clear" 44 | is_multiple = false 45 | } 46 | if !common.ExecExists(clear) { 47 | fmt.Printf("Executable '%s' not found\n", clear) 48 | os.Exit(1) 49 | } 50 | no_clear := full_path + "/no_clear" 51 | if is_multiple { 52 | no_clear = full_path + "/no_clear_all" 53 | } 54 | err := os.Remove(clear) 55 | if err != nil { 56 | fmt.Printf("Error while removing %s \n%s\n",clear, err) 57 | os.Exit(1) 58 | } 59 | err = os.Rename(no_clear, clear) 60 | if err != nil { 61 | fmt.Printf("Error while renaming script\n%s\n", err) 62 | os.Exit(1) 63 | } 64 | fmt.Printf("Sandbox %s unlocked\n",sandbox_name) 65 | } 66 | 67 | 68 | 69 | func PreserveSandbox(sandbox_dir, sandbox_name string) { 70 | full_path := sandbox_dir + "/" + sandbox_name 71 | if !common.DirExists(full_path) { 72 | fmt.Printf("Directory '%s' not found\n", full_path) 73 | os.Exit(1) 74 | } 75 | preserve := full_path + "/no_clear_all" 76 | if !common.ExecExists(preserve) { 77 | preserve = full_path + "/no_clear" 78 | } 79 | if common.ExecExists(preserve) { 80 | fmt.Printf("Sandbox %s is already locked\n",sandbox_name) 81 | return 82 | } 83 | is_multiple := true 84 | clear := full_path + "/clear_all" 85 | if !common.ExecExists(clear) { 86 | clear = full_path + "/clear" 87 | is_multiple = false 88 | } 89 | if !common.ExecExists(clear) { 90 | fmt.Printf("Executable '%s' not found\n", clear) 91 | os.Exit(1) 92 | } 93 | no_clear := full_path + "/no_clear" 94 | clear_cmd := "clear" 95 | no_clear_cmd := "no_clear" 96 | if is_multiple { 97 | no_clear = full_path + "/no_clear_all" 98 | clear_cmd = "clear_all" 99 | no_clear_cmd = "no_clear_all" 100 | } 101 | err := os.Rename(clear, no_clear) 102 | if err != nil { 103 | fmt.Printf("Error while renaming script.\n%s\n",err) 104 | os.Exit(1) 105 | } 106 | template := sandbox.SingleTemplates["sb_locked_template"].Contents 107 | var data = common.Smap{ 108 | "TemplateName" : "sb_locked_template", 109 | "SandboxDir" : sandbox_name, 110 | "AppVersion" : common.VersionDef, 111 | "Copyright" : sandbox.Copyright, 112 | "ClearCmd" : clear_cmd, 113 | "NoClearCmd" : no_clear_cmd, 114 | } 115 | template = common.TrimmedLines(template) 116 | new_clear_message := common.Tprintf(template, data) 117 | common.WriteString(new_clear_message, clear) 118 | os.Chmod(clear, 0744) 119 | fmt.Printf("Sandbox %s locked\n",sandbox_name) 120 | } 121 | 122 | func LockSandbox(cmd *cobra.Command, args []string) { 123 | if len(args) < 1 { 124 | fmt.Printf("'lock' requires the name of a sandbox (or ALL)") 125 | fmt.Printf("Example: dbdeployer admin lock msb_5_7_21") 126 | os.Exit(1) 127 | } 128 | flags := cmd.Flags() 129 | sandbox := args[0] 130 | sandbox_dir, _ := flags.GetString("sandbox-home") 131 | lock_list := []string{sandbox} 132 | if sandbox == "ALL" || sandbox == "all" { 133 | lock_list = common.SandboxInfoToFileNames(common.GetInstalledSandboxes(sandbox_dir)) 134 | } 135 | if len(lock_list) == 0 { 136 | fmt.Printf("Nothing to lock in %s\n", sandbox_dir) 137 | return 138 | } 139 | for _, sb := range lock_list { 140 | PreserveSandbox(sandbox_dir, sb) 141 | } 142 | } 143 | 144 | func UnlockSandbox(cmd *cobra.Command, args []string) { 145 | if len(args) < 1 { 146 | fmt.Printf("'unlock' requires the name of a sandbox (or ALL)") 147 | fmt.Printf("Example: dbdeployer admin unlock msb_5_7_21") 148 | os.Exit(1) 149 | } 150 | flags := cmd.Flags() 151 | sandbox := args[0] 152 | sandbox_dir, _ := flags.GetString("sandbox-home") 153 | lock_list := []string{sandbox} 154 | if sandbox == "ALL" || sandbox == "all" { 155 | lock_list = common.SandboxInfoToFileNames(common.GetInstalledSandboxes(sandbox_dir)) 156 | } 157 | if len(lock_list) == 0 { 158 | fmt.Printf("Nothing to lock in %s\n", sandbox_dir) 159 | return 160 | } 161 | for _, sb := range lock_list { 162 | UnpreserveSandbox(sandbox_dir, sb) 163 | } 164 | } 165 | 166 | 167 | var ( 168 | adminCmd = &cobra.Command{ 169 | Use: "admin", 170 | Short: "sandbox management tasks", 171 | Aliases: []string{"manage"}, 172 | Long: `Runs commands related to the administration of sandboxes.`, 173 | } 174 | 175 | adminLockCmd = &cobra.Command{ 176 | Use: "lock sandbox_name", 177 | Aliases: []string{"preserve"}, 178 | Short: "Locks a sandbox, preventing deletion", 179 | Long: `Prevents deletion for a given sandbox. 180 | Note that the deletion being prevented is only the one occurring through dbdeployer. 181 | Users can still delete locked sandboxes manually.`, 182 | Run: LockSandbox, 183 | } 184 | 185 | adminUnlockCmd = &cobra.Command{ 186 | Use: "unlock sandbox_name", 187 | Aliases: []string{"unpreserve"}, 188 | Short: "Unlocks a sandbox", 189 | Long: `Removes lock, allowing deletion of a given sandbox`, 190 | Run: UnlockSandbox, 191 | } 192 | ) 193 | 194 | func init() { 195 | rootCmd.AddCommand(adminCmd) 196 | adminCmd.AddCommand(adminLockCmd) 197 | adminCmd.AddCommand(adminUnlockCmd) 198 | } 199 | -------------------------------------------------------------------------------- /cmd/delete.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "bufio" 20 | "fmt" 21 | "github.com/datacharmer/dbdeployer/common" 22 | "github.com/datacharmer/dbdeployer/concurrent" 23 | "github.com/datacharmer/dbdeployer/defaults" 24 | "github.com/spf13/cobra" 25 | "os" 26 | ) 27 | 28 | func RemoveSandbox(sandbox_dir, sandbox string, run_concurrently bool) (exec_list []concurrent.ExecutionList) { 29 | full_path := sandbox_dir + "/" + sandbox 30 | if !common.DirExists(full_path) { 31 | fmt.Printf("Directory '%s' not found\n", full_path) 32 | os.Exit(1) 33 | } 34 | preserve := full_path + "/no_clear_all" 35 | if !common.ExecExists(preserve) { 36 | preserve = full_path + "/no_clear" 37 | } 38 | if common.ExecExists(preserve) { 39 | fmt.Printf("The sandbox %s is locked\n",sandbox) 40 | fmt.Printf("You need to unlock it with \"dbdeployer admin unlock\"\n",) 41 | return 42 | } 43 | stop := full_path + "/stop_all" 44 | if !common.ExecExists(stop) { 45 | stop = full_path + "/stop" 46 | } 47 | if !common.ExecExists(stop) { 48 | fmt.Printf("Executable '%s' not found\n", stop) 49 | os.Exit(1) 50 | } 51 | 52 | if run_concurrently { 53 | var eCommand1 = concurrent.ExecCommand{ 54 | Cmd : stop, 55 | Args : []string{}, 56 | } 57 | exec_list = append(exec_list, concurrent.ExecutionList{0, eCommand1}) 58 | } else { 59 | fmt.Printf("Running %s\n", stop) 60 | err, _ := common.Run_cmd(stop) 61 | if err != nil { 62 | fmt.Printf("Error while stopping sandbox %s\n", full_path) 63 | os.Exit(1) 64 | } 65 | } 66 | 67 | cmd_str := "rm" 68 | rm_args := []string{"-rf", full_path} 69 | if run_concurrently { 70 | var eCommand2 = concurrent.ExecCommand{ 71 | Cmd : cmd_str, 72 | Args : rm_args, 73 | } 74 | exec_list = append(exec_list, concurrent.ExecutionList{1, eCommand2}) 75 | } else { 76 | for _, item := range rm_args { 77 | cmd_str += " " + item 78 | } 79 | fmt.Printf("Running %s\n", cmd_str) 80 | err, _ := common.Run_cmd_with_args("rm", rm_args) 81 | if err != nil { 82 | fmt.Printf("Error while deleting sandbox %s\n", full_path) 83 | os.Exit(1) 84 | } 85 | fmt.Printf("Sandbox %s deleted\n", full_path) 86 | } 87 | // fmt.Printf("%#v\n",exec_list) 88 | return 89 | } 90 | 91 | func DeleteSandbox(cmd *cobra.Command, args []string) { 92 | var exec_lists []concurrent.ExecutionList 93 | if len(args) < 1 { 94 | fmt.Println("Sandbox name (or \"ALL\") required.") 95 | fmt.Println("You can run 'dbdeployer sandboxes for a list of available deployments'") 96 | os.Exit(1) 97 | } 98 | flags := cmd.Flags() 99 | sandbox := args[0] 100 | confirm, _ := flags.GetBool("confirm") 101 | run_concurrently, _ := flags.GetBool("concurrent") 102 | if os.Getenv("RUN_CONCURRENTLY") != "" { 103 | run_concurrently = true 104 | } 105 | skip_confirm, _ := flags.GetBool("skip-confirm") 106 | sandbox_dir, _ := flags.GetString("sandbox-home") 107 | deletion_list := []common.SandboxInfo{common.SandboxInfo{sandbox, false}} 108 | if sandbox == "ALL" || sandbox == "all" { 109 | confirm = true 110 | if skip_confirm { 111 | confirm = false 112 | } 113 | deletion_list = common.GetInstalledSandboxes(sandbox_dir) 114 | } 115 | if len(deletion_list) == 0 { 116 | fmt.Printf("Nothing to delete in %s\n", sandbox_dir) 117 | return 118 | } 119 | if len(deletion_list) > 60 && run_concurrently { 120 | fmt.Println("# Concurrency disabled. Can't run more than 60 concurrent operations") 121 | run_concurrently = false 122 | } 123 | fmt.Printf("List of deployed sandboxes:\n") 124 | unlocked_found := false 125 | for _, sb := range deletion_list { 126 | locked := "" 127 | if sb.Locked { 128 | locked = "(*LOCKED*)" 129 | } else { 130 | unlocked_found = true 131 | } 132 | fmt.Printf("%s/%s %s\n", sandbox_dir, sb.SandboxName, locked) 133 | } 134 | if !unlocked_found { 135 | fmt.Printf("No unlocked sandboxes found.\n") 136 | return 137 | } 138 | if confirm { 139 | fmt.Printf("Do you confirm? y/[N] ") 140 | 141 | bio := bufio.NewReader(os.Stdin) 142 | line, _, err := bio.ReadLine() 143 | if err != nil { 144 | fmt.Println(err) 145 | } else { 146 | answer := string(line) 147 | if answer == "y" || answer == "Y" { 148 | fmt.Println("Proceeding with deletion") 149 | } else { 150 | fmt.Println("Execution interrupted by user") 151 | os.Exit(0) 152 | } 153 | } 154 | } 155 | for _, sb := range deletion_list { 156 | if sb.Locked { 157 | fmt.Printf("Sandbox %s is locked\n",sb.SandboxName) 158 | } else { 159 | exec_list := RemoveSandbox(sandbox_dir, sb.SandboxName, run_concurrently) 160 | for _, list := range exec_list { 161 | exec_lists = append(exec_lists, list) 162 | } 163 | } 164 | } 165 | concurrent.RunParallelTasksByPriority(exec_lists) 166 | for _, sb := range deletion_list { 167 | full_path := sandbox_dir + "/" + sb.SandboxName 168 | if !sb.Locked { 169 | defaults.DeleteFromCatalog(full_path) 170 | } 171 | } 172 | } 173 | 174 | // deleteCmd represents the delete command 175 | var deleteCmd = &cobra.Command{ 176 | Use: "delete sandbox_name (or \"ALL\")", 177 | Short: "delete an installed sandbox", 178 | Aliases: []string{"remove", "destroy"}, 179 | Example: ` 180 | $ dbdeployer delete msb_8_0_4 181 | $ dbdeployer delete rsandbox_5_7_21`, 182 | Long: `Stops the sandbox (and its depending sandboxes, if any), and removes it. 183 | Warning: this command is irreversible!`, 184 | Run: DeleteSandbox, 185 | } 186 | 187 | func init() { 188 | rootCmd.AddCommand(deleteCmd) 189 | 190 | deleteCmd.Flags().BoolP("skip-confirm", "", false, "Skips confirmation with multiple deletions.") 191 | deleteCmd.Flags().BoolP("confirm", "", false, "Requires confirmation.") 192 | deleteCmd.Flags().BoolP("concurrent", "", false, "Runs multiple deletion tasks concurrently.") 193 | } 194 | -------------------------------------------------------------------------------- /test/mock/defaults-change.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # DBDeployer - The MySQL Sandbox 3 | # Copyright © 2006-2018 Giuseppe Maxia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | test_dir=$(dirname $0) 19 | cd $test_dir 20 | test_dir=$PWD 21 | exit_code=0 22 | 23 | if [ ! -f set-mock.sh ] 24 | then 25 | echo "set-mock.sh not found in $PWD" 26 | exit 1 27 | fi 28 | 29 | if [ ! -f ../common.sh ] 30 | then 31 | echo "../common.sh not found" 32 | exit 1 33 | fi 34 | 35 | source ../common.sh 36 | #export results_log=$PWD/defaults-change.log 37 | source set-mock.sh 38 | export SHOW_CHANGED_PORTS=1 39 | start_timer 40 | 41 | tests=0 42 | fail=0 43 | pass=0 44 | 45 | 46 | function check_deployment { 47 | sandbox_dir=$1 48 | node_dir=$2 49 | master_name=$3 50 | master_abbr=$4 51 | slave_abbr=$5 52 | slave_name=$6 53 | ok_dir_exists $sandbox_dir 54 | ok_dir_exists $sandbox_dir/${master_name} 55 | ok_dir_exists $sandbox_dir/${node_dir}1 56 | ok_dir_exists $sandbox_dir/${node_dir}2 57 | ok_executable_exists $sandbox_dir/$master_abbr 58 | ok_executable_exists $sandbox_dir/${slave_abbr}1 59 | ok_executable_exists $sandbox_dir/${slave_abbr}2 60 | ok_executable_exists $sandbox_dir/check_${slave_name}s 61 | ok_executable_exists $sandbox_dir/initialize_${slave_name}s 62 | for sbdir in $master_name ${node_dir}1 ${node_dir}2 63 | do 64 | auto_cnf=$sandbox_dir/$sbdir/data/auto.cnf 65 | if [ -f $auto_cnf ] 66 | then 67 | tail -n 1 $auto_cnf | sed -e 's/server-uuid=//' 68 | #echo $auto_cnf 69 | #cat $auto_cnf 70 | fi 71 | done 72 | } 73 | 74 | function check_deployment_message { 75 | version=$1 76 | shift 77 | args="$@" 78 | path_version=$(echo $version | tr '.' '_') 79 | output_file=deployment$$.txt 80 | dbdeployer deploy $args $version > $output_file 2>&1 81 | grep "installed.*\$HOME.*$path_version" $output_file 82 | installed=$(grep "installed.*\$HOME.*$path_version" $output_file) 83 | ok "deploy $args $version" "$installed" 84 | rm $output_file 85 | } 86 | 87 | create_mock_version 5.5.66 88 | create_mock_version 5.6.66 89 | create_mock_version 5.7.66 90 | create_mock_version 8.0.66 91 | create_mock_version 8.0.67 92 | 93 | # Changing all defaults statically 94 | run dbdeployer defaults show 95 | run dbdeployer defaults update master-slave-prefix ms_replication_ 96 | run dbdeployer defaults update master-name primary 97 | run dbdeployer defaults update master-abbr p 98 | run dbdeployer defaults update slave-prefix replica 99 | run dbdeployer defaults update slave-abbr r 100 | run dbdeployer defaults update node-prefix branch 101 | run dbdeployer defaults show 102 | 103 | run dbdeployer deploy replication 5.6.66 104 | 105 | sandbox_dir=$SANDBOX_HOME/ms_replication_5_6_66 106 | check_deployment $sandbox_dir branch primary p r replica 107 | 108 | # Keeping the changes, we deploy a new replication cluster 109 | # with the defaults changing dynamically. 110 | 111 | run dbdeployer deploy replication 5.7.66 \ 112 | --defaults=master-slave-prefix:masterslave_ \ 113 | --defaults=master-name:batman \ 114 | --defaults=master-abbr:b \ 115 | --defaults=slave-prefix:robin \ 116 | --defaults=slave-abbr:rob \ 117 | --defaults=node-prefix:bat 118 | 119 | sandbox_dir=$SANDBOX_HOME/masterslave_5_7_66 120 | check_deployment $sandbox_dir bat batman b rob robin 121 | 122 | # We make sure that the defaults stay the same, and they 123 | # were not affected by the dynamic changes 124 | run dbdeployer deploy replication 5.5.66 125 | sandbox_dir=$SANDBOX_HOME/ms_replication_5_5_66 126 | check_deployment $sandbox_dir branch primary p r replica 127 | 128 | # Restore the original defaults 129 | run dbdeployer defaults reset 130 | run dbdeployer deploy replication 8.0.66 131 | 132 | sandbox_dir=$SANDBOX_HOME/rsandbox_8_0_66 133 | check_deployment $sandbox_dir node master m s slave 134 | 135 | echo "#Total sandboxes: $(count_catalog)" 136 | #echo "#Total sandboxes: $(count_catalog)" >> $results_log 137 | if [ "$fail" != "0" ] 138 | then 139 | exit 1 140 | fi 141 | 142 | temp_template=t$$.dat 143 | timestamp=$(date +%Y-%m-%d.%H:%M:%S) 144 | echo "#!/bin/bash" > $temp_template 145 | echo "echo 'I AM A CUSTOM_TEMPLATE CREATED ON $timestamp'" >> $temp_template 146 | run dbdeployer deploy --use-template=clear_template:$temp_template single 8.0.67 147 | sandbox_dir=$SANDBOX_HOME/msb_8_0_67 148 | message=$($sandbox_dir/clear) 149 | ok_contains "custom template" "$message" "CUSTOM_TEMPLATE" 150 | ok_contains "custom template" "$message" $timestamp 151 | 152 | run dbdeployer delete ALL --skip-confirm 153 | 154 | results "After deletion" 155 | 156 | run dbdeployer defaults templates export single $mock_dir/templates clear_template 157 | cp $temp_template $mock_dir/templates/single/clear_template 158 | rm -f $temp_template 159 | run dbdeployer defaults templates import single $mock_dir/templates 160 | installed=$(dbdeployer defaults templates list | grep "clear_template" | grep '{F}') 161 | echo "# installed template: <$installed>" 162 | ok "template was installed" "$installed" 163 | run dbdeployer deploy single 8.0.67 164 | 165 | sandbox_dir=$SANDBOX_HOME/msb_8_0_67 166 | message=$($sandbox_dir/clear) 167 | ok_contains "installed custom template" "$message" "CUSTOM_TEMPLATE" 168 | ok_contains "installed custom template" "$message" $timestamp 169 | 170 | run dbdeployer delete ALL --skip-confirm 171 | 172 | echo "Test installed messages" 173 | check_deployment_message 8.0.67 single 174 | check_deployment_message 8.0.67 multiple 175 | check_deployment_message 8.0.67 replication 176 | check_deployment_message 8.0.67 replication --topology=group 177 | check_deployment_message 8.0.67 replication --topology=group --single-primary 178 | check_deployment_message 8.0.67 replication --topology=fan-in 179 | check_deployment_message 8.0.67 replication --topology=all-masters 180 | 181 | 182 | run dbdeployer delete ALL --skip-confirm 183 | 184 | cd $test_dir 185 | 186 | run du -sh $mock_dir 187 | run rm -rf $mock_dir 188 | stop_timer 189 | tests=$((pass+fail)) 190 | echo "Tests: $tests" 191 | echo "Pass : $pass" 192 | echo "Fail : $fail" 193 | 194 | -------------------------------------------------------------------------------- /test/mock/port-clash.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # DBDeployer - The MySQL Sandbox 3 | # Copyright © 2006-2018 Giuseppe Maxia 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | test_dir=$(dirname $0) 18 | cd $test_dir 19 | test_dir=$PWD 20 | exit_code=0 21 | 22 | if [ ! -f set-mock.sh ] 23 | then 24 | echo "set-mock.sh not found in $PWD" 25 | exit 1 26 | fi 27 | 28 | if [ ! -f ../common.sh ] 29 | then 30 | echo "../common.sh not found" 31 | exit 1 32 | fi 33 | 34 | source ../common.sh 35 | #export results_log=$PWD/port-clash.log 36 | source set-mock.sh 37 | export SHOW_CHANGED_PORTS=1 38 | start_timer 39 | 40 | run dbdeployer defaults store 41 | run dbdeployer defaults show 42 | 43 | versions=(5.0 5.1 5.5 5.6 5.7 8.0) 44 | versions_mysqld_initialize=(5.7 8.0) 45 | versions_mysql_install_db=(5.0 5.1 5.5 5.6) 46 | 47 | rev_sparse="18 27 36 45 54 63 72 81 99" 48 | rev_all="$(seq 1 99)" 49 | rev_list=$rev_all 50 | if [ "$1" == "sparse" ] 51 | then 52 | rev_list=$rev_sparse 53 | fi 54 | 55 | function test_number_of_ports { 56 | version=$1 57 | dir_name=$2 58 | mode=$3 59 | nodes=$4 60 | expected_ports=$5 61 | version_name=$(echo $version | tr '.' '_') 62 | deploy_command=$mode 63 | case $mode in 64 | groupr) 65 | deploy_command="replication --topology=group" 66 | ;; 67 | groupsp) 68 | deploy_command="replication --topology=group --single-primary" 69 | ;; 70 | allmasters) 71 | deploy_command="replication --topology=all-masters" 72 | ;; 73 | fanin) 74 | deploy_command="replication --topology=fan-in" 75 | ;; 76 | esac 77 | run dbdeployer deploy $deploy_command $version --disable-mysqlx 78 | how_many_ports=$(sandbox_num_ports $version $dir_name) 79 | ok_equal "ports in $dir_name $version (without mysqlx)" $how_many_ports $expected_ports 80 | run dbdeployer delete $dir_name$version_name 81 | expected_ports=$((expected_ports+nodes)) 82 | run dbdeployer deploy $deploy_command $version 83 | how_many_ports=$(sandbox_num_ports $version $dir_name) 84 | ok_equal "ports in $dir_name $version (with mysqlx)" $how_many_ports $expected_ports 85 | run dbdeployer delete $dir_name$version_name 86 | } 87 | 88 | for rev in $rev_list 89 | do 90 | for vers in ${versions[*]} 91 | do 92 | version=${vers}.${rev} 93 | create_mock_tarball $version $SANDBOX_TARBALL 94 | #create_mock_version $version 95 | run dbdeployer unpack $SANDBOX_TARBALL/mysql-${version}.tar.gz 96 | # --unpack-version $version 97 | done 98 | 99 | run dbdeployer available 100 | for vers in ${versions_mysql_install_db[*]} 101 | do 102 | version=${vers}.${rev} 103 | version_name=$(echo $version | tr '.' '_') 104 | run dbdeployer deploy single $version 105 | run dbdeployer deploy multiple $version 106 | run dbdeployer deploy replication $version 107 | results "$version" 108 | right_installer=$(grep mysql_install_db $SANDBOX_HOME/msb_${version_name}/init_db) 109 | if [ -n "$right_installer" ] 110 | then 111 | echo "ok installer" 112 | else 113 | echo "NOT OK installer" 114 | exit_code=1 115 | fi 116 | done 117 | 118 | for vers in ${versions_mysqld_initialize[*]} 119 | do 120 | version=${vers}.${rev} 121 | if [[ $vers == "8.0" && $rev -ge 11 ]] 122 | then 123 | test_number_of_ports $version msb_ single 1 1 124 | test_number_of_ports $version rsandbox_ replication 3 3 125 | test_number_of_ports $version multi_msb_ multiple 3 3 126 | test_number_of_ports $version group_msb_ groupr 3 6 127 | test_number_of_ports $version group_sp_msb_ groupsp 3 6 128 | test_number_of_ports $version all_masters_msb_ allmasters 3 3 129 | test_number_of_ports $version fan_in_msb_ fanin 3 3 130 | fi 131 | version_name=$(echo $version | tr '.' '_') 132 | run dbdeployer deploy single $version 133 | run dbdeployer deploy multiple $version 134 | run dbdeployer deploy replication $version 135 | if [[ "$vers" == "5.7" && $rev -lt 18 ]] 136 | then 137 | echo "skipping group replication for version $version" 138 | else 139 | run dbdeployer deploy replication $version --topology=group 140 | run dbdeployer deploy replication $version --topology=group --single-primary 141 | run dbdeployer deploy replication $version --topology=all-masters 142 | run dbdeployer deploy replication $version --topology=fan-in 143 | run dbdeployer deploy replication $version --topology=fan-in \ 144 | --sandbox-directory=fan_in_msb2_$version_name \ 145 | --base-port=24000 \ 146 | --nodes=5 \ 147 | --master-list='1,2' \ 148 | --slave-list='3:4:5' 149 | run dbdeployer deploy replication $version --topology=fan-in \ 150 | --sandbox-directory=fan_in_msb3_$version_name \ 151 | --base-port=25000 \ 152 | --nodes=5 \ 153 | --master-list='1.2.3' \ 154 | --slave-list='4,5' 155 | fi 156 | results "$version" 157 | right_installer1=$(grep mysqld $SANDBOX_HOME/msb_${version_name}/init_db ) 158 | right_installer2=$(grep initialize-insecure $SANDBOX_HOME/msb_${version_name}/init_db ) 159 | if [ -n "$right_installer1" -a -n "$right_installer2" ] 160 | then 161 | echo "ok installer" 162 | else 163 | echo "NOT OK installer" 164 | exit_code=1 165 | fi 166 | done 167 | 168 | if [ "$exit_code" != "0" ] 169 | then 170 | exit $exit_code 171 | fi 172 | done 173 | 174 | echo "#Total sandboxes: $(count_catalog)" 175 | #echo "#Total sandboxes: $(count_catalog)" >> $results_log 176 | num_ports=$(grep -A10 port $CATALOG | grep '^\s*[0-9]\+' | wc -l) 177 | echo "# Total ports installed: $num_ports" 178 | #echo "# Total ports installed: $num_ports" >> $results_log 179 | run dbdeployer delete ALL --skip-confirm 180 | 181 | results "After deletion" 182 | cd $test_dir 183 | 184 | run du -sh $mock_dir 185 | run rm -rf $mock_dir 186 | stop_timer 187 | 188 | -------------------------------------------------------------------------------- /unpack/unpack.go: -------------------------------------------------------------------------------- 1 | // Copyright © 2011-12 Qtrac Ltd. 2 | // 3 | // This program or package and any associated files are licensed under the 4 | // Apache License, Version 2.0 (the "License"); you may not use these files 5 | // except in compliance with the License. You can get a copy of the License 6 | // at: http://www.apache.org/licenses/LICENSE-2.0. 7 | // 8 | // Unless required by applicable law or agreed to in writing, software 9 | // distributed under the License is distributed on an "AS IS" BASIS, 10 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 11 | // See the License for the specific language governing permissions and 12 | // limitations under the License. 13 | // 14 | /* 15 | Code adapted and enhanced from examples to the book: 16 | Programming in Go by Mark Summerfield 17 | http://www.qtrac.eu/gobook.html 18 | 19 | Original author: Mark Summerfield 20 | Converted to package by Giuseppe Maxia in 2018 21 | 22 | The original code was a stand-alone program, and it 23 | had a few bugs: 24 | * when extracting from a tar file: when there 25 | isn't a separate item for each directory, the 26 | extraction fails. 27 | * The attributes of the files were not reproduced 28 | in the extracted files. 29 | This code fixes those problems and introduces a 30 | destination directory and verbosity 31 | levels for the extraction 32 | 33 | */ 34 | 35 | package unpack 36 | 37 | import ( 38 | "archive/tar" 39 | "compress/gzip" 40 | "fmt" 41 | "io" 42 | "os" 43 | "path" 44 | "strconv" 45 | "strings" 46 | ) 47 | 48 | const ( 49 | SILENT = iota // No output 50 | VERBOSE // Minimal feedback about extraction operations 51 | CHATTY // Full details of what is being extracted 52 | ) 53 | 54 | var Verbose int 55 | 56 | func cond_print(s string, nl bool, level int) { 57 | if Verbose >= level { 58 | if nl { 59 | fmt.Println(s) 60 | } else { 61 | fmt.Printf(s) 62 | } 63 | } 64 | } 65 | 66 | func validSuffix(filename string) bool { 67 | for _, suffix := range []string{".tgz", ".tar", ".tar.gz"} { 68 | if strings.HasSuffix(filename, suffix) { 69 | return true 70 | } 71 | } 72 | return false 73 | } 74 | 75 | func UnpackTar(filename string, destination string, verbosity_level int) (err error) { 76 | Verbose = verbosity_level 77 | f, err := os.Stat(destination) 78 | if os.IsNotExist(err) { 79 | return fmt.Errorf("Destination directory '%s' does not exist", destination) 80 | } 81 | filemode := f.Mode() 82 | if filemode.IsDir() == false { 83 | return fmt.Errorf("Destination '%s' is not a directory", destination) 84 | } 85 | if !validSuffix(filename) { 86 | return fmt.Errorf("unrecognized archive suffix") 87 | } 88 | var file *os.File 89 | if file, err = os.Open(filename); err != nil { 90 | return err 91 | } 92 | defer file.Close() 93 | os.Chdir(destination) 94 | var fileReader io.Reader = file 95 | var decompressor *gzip.Reader 96 | if strings.HasSuffix(filename, ".gz") { 97 | if decompressor, err = gzip.NewReader(file); err != nil { 98 | return err 99 | } 100 | defer decompressor.Close() 101 | } 102 | var reader *tar.Reader 103 | if decompressor != nil { 104 | reader = tar.NewReader(decompressor) 105 | } else { 106 | reader = tar.NewReader(fileReader) 107 | } 108 | return unpackTarFiles(reader) 109 | } 110 | 111 | func unpackTarFiles(reader *tar.Reader) (err error) { 112 | var header *tar.Header 113 | var count int = 0 114 | 115 | for { 116 | if header, err = reader.Next(); err != nil { 117 | if err == io.EOF { 118 | cond_print("Files ", false, CHATTY) 119 | cond_print(strconv.Itoa(count), true, 1) 120 | return nil // OK 121 | } 122 | return err 123 | } 124 | // cond_print(fmt.Sprintf("%#v\n", header), true, CHATTY) 125 | /* 126 | tar.Header{ 127 | Typeflag:0x30, 128 | Name:"mysql-8.0.11-macos10.13-x86_64/docs/INFO_SRC", 129 | Linkname:"", 130 | Size:185, 131 | Mode:420, 132 | Uid:7161, 133 | Gid:10, 134 | Uname:"pb2user", 135 | Gname:"owner", 136 | ModTime:time.Time{wall:0x0, ext:63658769207, loc:(*time.Location)(0x13730e0)}, 137 | AccessTime:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}, 138 | ChangeTime:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}, 139 | Devmajor:0, Devminor:0, 140 | Xattrs:map[string]string(nil), 141 | PAXRecords:map[string]string(nil), 142 | Format:0} 143 | tar.Header{ 144 | Typeflag:0x32, 145 | Name:"mysql-8.0.11-macos10.13-x86_64/lib/libssl.dylib", 146 | Linkname:"libssl.1.0.0.dylib", 147 | Size:0, 148 | Mode:493, 149 | Uid:7161, 150 | Gid:10, 151 | Uname:"pb2user", 152 | Gname:"owner", 153 | ModTime:time.Time{wall:0x0, ext:63658772525, loc:(*time.Location)(0x13730e0)}, 154 | AccessTime:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}, 155 | ChangeTime:time.Time{wall:0x0, ext:0, loc:(*time.Location)(nil)}, 156 | Devmajor:0, 157 | Devminor:0, 158 | Xattrs:map[string]string(nil), 159 | PAXRecords:map[string]string(nil), 160 | Format:0} 161 | */ 162 | filemode := os.FileMode(header.Mode) 163 | filename := sanitizedName(header.Name) 164 | switch header.Typeflag { 165 | case tar.TypeDir: 166 | if err = os.MkdirAll(filename, 0755); err != nil { 167 | return err 168 | } 169 | case tar.TypeReg: 170 | fileDir := path.Dir(filename) 171 | if _, err := os.Stat(fileDir); os.IsNotExist(err) { 172 | if err = os.MkdirAll(fileDir, 0755); err != nil { 173 | return err 174 | } 175 | cond_print(" + "+fileDir+" ", true, CHATTY) 176 | } 177 | if err = unpackTarFile(filename, header.Name, reader); err != nil { 178 | return err 179 | } 180 | os.Chmod(filename, filemode) 181 | count++ 182 | cond_print(filename, true, CHATTY) 183 | if count%10 == 0 { 184 | mark := "." 185 | if count%100 == 0 { 186 | mark = strconv.Itoa(count) 187 | } 188 | if Verbose < CHATTY { 189 | cond_print(mark, false, 1) 190 | } 191 | } 192 | case tar.TypeSymlink: 193 | if header.Linkname != "" { 194 | cond_print(fmt.Sprintf ("%s -> %s",filename, header.Linkname), true, CHATTY) 195 | err := os.Symlink( header.Linkname, filename) 196 | if err != nil { 197 | fmt.Printf("%#v\n",header) 198 | fmt.Printf("# ERROR: %s\n",err) 199 | os.Exit(1) 200 | } 201 | } else { 202 | fmt.Printf("File %s is a symlonk, but no link information was provided\n", filename) 203 | os.Exit(1) 204 | } 205 | } 206 | } 207 | return nil 208 | } 209 | 210 | func unpackTarFile(filename, tarFilename string, 211 | reader *tar.Reader) (err error) { 212 | var writer *os.File 213 | if writer, err = os.Create(filename); err != nil { 214 | return err 215 | } 216 | defer writer.Close() 217 | if _, err = io.Copy(writer, reader); err != nil { 218 | return err 219 | } 220 | return nil 221 | } 222 | 223 | func sanitizedName(filename string) string { 224 | if len(filename) > 1 && filename[1] == ':' { 225 | filename = filename[2:] 226 | } 227 | filename = strings.TrimLeft(filename, "\\/.") 228 | filename = strings.Replace(filename, "../", "", -1) 229 | return strings.Replace(filename, "..\\", "", -1) 230 | } 231 | -------------------------------------------------------------------------------- /common/checks.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | import ( 19 | "fmt" 20 | "io/ioutil" 21 | "os" 22 | "regexp" 23 | "strconv" 24 | "strings" 25 | ) 26 | 27 | type SandboxInfo struct { 28 | SandboxName string 29 | Locked bool 30 | } 31 | 32 | func SandboxInfoToFileNames(sb_list []SandboxInfo) (file_names []string) { 33 | for _, sbinfo := range sb_list { 34 | file_names = append(file_names, sbinfo.SandboxName) 35 | } 36 | return 37 | } 38 | 39 | func GetInstalledSandboxes(sandbox_home string) (installed_sandboxes []SandboxInfo) { 40 | if !DirExists(sandbox_home) { 41 | return 42 | } 43 | files, err := ioutil.ReadDir(sandbox_home) 44 | if err != nil { 45 | fmt.Printf("%s", err) 46 | os.Exit(1) 47 | } 48 | for _, f := range files { 49 | fname := f.Name() 50 | fmode := f.Mode() 51 | if fmode.IsDir() { 52 | sbdesc := sandbox_home + "/" + fname + "/sbdescription.json" 53 | start := sandbox_home + "/" + fname + "/start" 54 | start_all := sandbox_home + "/" + fname + "/start_all" 55 | no_clear := sandbox_home + "/" + fname + "/no_clear" 56 | no_clear_all := sandbox_home + "/" + fname + "/no_clear_all" 57 | if FileExists(sbdesc) || FileExists(start) || FileExists(start_all) { 58 | if FileExists(no_clear_all) || FileExists(no_clear) { 59 | installed_sandboxes = append(installed_sandboxes, SandboxInfo{ fname, true}) 60 | } else { 61 | installed_sandboxes = append(installed_sandboxes, SandboxInfo{fname, false}) 62 | } 63 | } 64 | } 65 | } 66 | return 67 | } 68 | 69 | func GetInstalledPorts(sandbox_home string) []int { 70 | files := SandboxInfoToFileNames(GetInstalledSandboxes(sandbox_home)) 71 | // If there is a file sbdescription.json in the top directory 72 | // it will be included in the reporting 73 | files = append(files, "") 74 | var port_collection []int 75 | var seen_ports = make(map[int]bool) 76 | for _, fname := range files { 77 | sbdesc := sandbox_home + "/" + fname + "/sbdescription.json" 78 | if FileExists(sbdesc) { 79 | sbd := ReadSandboxDescription(sandbox_home + "/" + fname) 80 | if sbd.Nodes == 0 { 81 | for _, p := range sbd.Port { 82 | if !seen_ports[p] { 83 | port_collection = append(port_collection, p) 84 | seen_ports[p] = true 85 | } 86 | } 87 | } else { 88 | var node_descr []SandboxDescription 89 | inner_files := SandboxInfoToFileNames(GetInstalledSandboxes(sandbox_home + "/" + fname)) 90 | for _, inner := range inner_files { 91 | inner_sbdesc := sandbox_home + "/" + fname + "/" + inner + "/sbdescription.json" 92 | if FileExists(inner_sbdesc) { 93 | sd_node := ReadSandboxDescription(fmt.Sprintf("%s/%s/%s", sandbox_home, fname, inner)) 94 | node_descr = append(node_descr, sd_node) 95 | } 96 | } 97 | for _, nd := range node_descr { 98 | for _, p := range nd.Port { 99 | if !seen_ports[p] { 100 | port_collection = append(port_collection, p) 101 | seen_ports[p] = true 102 | } 103 | } 104 | } 105 | } 106 | } 107 | } 108 | // fmt.Printf("%v\n",port_collection) 109 | return port_collection 110 | } 111 | 112 | func CheckOrigin(args []string) { 113 | if len(args) < 1 { 114 | fmt.Println("This command requires the MySQL version (x.xx.xx) as argument ") 115 | os.Exit(1) 116 | } 117 | if len(args) > 1 { 118 | fmt.Println("Extra argument detected. This command requires only the MySQL version (x.xx.xx) as argument ") 119 | os.Exit(1) 120 | } 121 | origin := args[0] 122 | if FileExists(origin) && strings.HasSuffix(origin, ".tar.gz") { 123 | fmt.Println("Tarball detected. - If you want to use a tarball to create a sandbox,") 124 | fmt.Println("you should first use the 'unpack' command") 125 | os.Exit(1) 126 | } 127 | 128 | } 129 | 130 | func CheckSandboxDir(sandbox_home string) { 131 | if !DirExists(sandbox_home) { 132 | fmt.Printf("Creating directory %s\n", sandbox_home) 133 | err := os.Mkdir(sandbox_home, 0755) 134 | if err != nil { 135 | fmt.Println(err) 136 | os.Exit(1) 137 | } 138 | } 139 | 140 | } 141 | 142 | // Gets three integers for a version string 143 | // Converts "1.2.3" into []int{1, 2, 3} 144 | func VersionToList(version string) []int { 145 | // A valid version must be made of 3 integers 146 | re1 := regexp.MustCompile(`^(\d+)\.(\d+)\.(\d+)$`) 147 | // Also valid version is 3 numbers with a prefix 148 | re2 := regexp.MustCompile(`^[^.0-9-]+(\d+)\.(\d+)\.(\d+)$`) 149 | verList1 := re1.FindAllStringSubmatch(version, -1) 150 | verList2 := re2.FindAllStringSubmatch(version, -1) 151 | verList := verList1 152 | //fmt.Printf("%#v\n", verList) 153 | if verList == nil { 154 | verList = verList2 155 | } 156 | if verList == nil { 157 | fmt.Printf("Required version format: x.x.xx - Got '%s'\n", version) 158 | return []int{-1} 159 | //os.Exit(1) 160 | } 161 | 162 | major, err1 := strconv.Atoi(verList[0][1]) 163 | minor, err2 := strconv.Atoi(verList[0][2]) 164 | rev, err3 := strconv.Atoi(verList[0][3]) 165 | if err1 != nil || err2 != nil || err3 != nil { 166 | return []int{-1} 167 | } 168 | return []int{major, minor, rev} 169 | } 170 | 171 | // Converts a version string into a name. 172 | // Replaces dots with underscores. "1.2.3" -> "1_2_3" 173 | func VersionToName(version string) string { 174 | re := regexp.MustCompile(`\.`) 175 | name := re.ReplaceAllString(version, "_") 176 | return name 177 | } 178 | 179 | // Converts a version string into a port number 180 | // e.g. "5.6.33" -> 5633 181 | func VersionToPort(version string) int { 182 | verList := VersionToList(version) 183 | major := verList[0] 184 | if major < 0 { 185 | return -1 186 | } 187 | minor := verList[1] 188 | rev := verList[2] 189 | //if major < 0 || minor < 0 || rev < 0 { 190 | // return -1 191 | //} 192 | completeVersion := fmt.Sprintf("%d%d%02d", major, minor, rev) 193 | // fmt.Println(completeVersion) 194 | i, err := strconv.Atoi(completeVersion) 195 | if err == nil { 196 | return i 197 | } 198 | return -1 199 | } 200 | 201 | // Checks if a version string is greater or equal a given numeric version 202 | // "5.6.33" >= []{5.7.0} = false 203 | // "5.7.21" >= []{5.7.0} = true 204 | // "10.1.21" >= []{5.7.0} = false (!) 205 | // Note: MariaDB versions are skipped. The function returns false for MariaDB 10+ 206 | // So far (2018-02-19) this comparison holds, because MariaDB behaves like 5.5+ for 207 | // the purposes of sandbox deployment 208 | func GreaterOrEqualVersion(version string, compared_to []int) bool { 209 | var cmajor, cminor, crev int = compared_to[0], compared_to[1], compared_to[2] 210 | verList := VersionToList(version) 211 | major := verList[0] 212 | if major < 0 { 213 | return false 214 | } 215 | minor := verList[1] 216 | rev := verList[2] 217 | 218 | if major == 10 { 219 | return false 220 | } 221 | sversion := fmt.Sprintf("%02d%02d%02d", major, minor, rev) 222 | scompare := fmt.Sprintf("%02d%02d%02d", cmajor, cminor, crev) 223 | // fmt.Printf("<%s><%s>\n", sversion, scompare) 224 | return sversion >= scompare 225 | } 226 | -------------------------------------------------------------------------------- /cmd/single.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package cmd 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "github.com/datacharmer/dbdeployer/sandbox" 22 | "github.com/datacharmer/dbdeployer/defaults" 23 | "github.com/spf13/cobra" 24 | "os" 25 | "regexp" 26 | "strings" 27 | ) 28 | 29 | func replace_template(template_name string, file_name string) { 30 | group, _, contents := FindTemplate(template_name) 31 | if !common.FileExists(file_name) { 32 | fmt.Printf("File %s not found\n", file_name) 33 | os.Exit(1) 34 | } 35 | fmt.Printf("Replacing template %s.%s [%d chars] with contents of file %s\n", group, template_name, len(contents), file_name) 36 | new_contents := common.SlurpAsString(file_name) 37 | if len(new_contents) == 0 { 38 | fmt.Printf("File %s is empty\n", file_name) 39 | os.Exit(1) 40 | } 41 | var new_rec sandbox.TemplateDesc = sandbox.TemplateDesc{ 42 | Description: sandbox.AllTemplates[group][template_name].Description, 43 | Notes: sandbox.AllTemplates[group][template_name].Notes, 44 | Contents: new_contents, 45 | } 46 | sandbox.AllTemplates[group][template_name] = new_rec 47 | } 48 | 49 | func check_template_change_request(request string) (template_name, file_name string) { 50 | re := regexp.MustCompile(`(\w+):(\S+)`) 51 | reqList := re.FindAllStringSubmatch(request, -1) 52 | if len(reqList) == 0 { 53 | //fmt.Printf("%v\n", reqList) 54 | fmt.Printf("request '%s' invalid. Required format is 'template_name:file_name'\n", request) 55 | os.Exit(1) 56 | } 57 | template_name = reqList[0][1] 58 | file_name = reqList[0][2] 59 | return 60 | } 61 | 62 | func process_defaults(new_defaults []string) { 63 | for _, nd := range new_defaults { 64 | list := strings.Split(nd, ":") 65 | if list != nil && len(list) == 2 { 66 | label := list[0] 67 | value := list[1] 68 | defaults.UpdateDefaults(label, value, false) 69 | } 70 | } 71 | } 72 | 73 | func FillSdef(cmd *cobra.Command, args []string) sandbox.SandboxDef { 74 | var sd sandbox.SandboxDef 75 | 76 | flags := cmd.Flags() 77 | template_requests, _ := flags.GetStringSlice("use-template") 78 | for _, request := range template_requests { 79 | tname, fname := check_template_change_request(request) 80 | replace_template(tname, fname) 81 | } 82 | sd.Port = common.VersionToPort(args[0]) 83 | 84 | sd.UserPort, _ = flags.GetInt("port") 85 | sd.BasePort, _ = flags.GetInt("base-port") 86 | sd.DirName, _ = flags.GetString("sandbox-directory") 87 | if sd.UserPort > 0 { 88 | sd.Port = sd.UserPort 89 | } 90 | 91 | sd.Version = args[0] 92 | sd.Basedir, _ = flags.GetString("sandbox-binary") 93 | sd.SandboxDir, _ = flags.GetString("sandbox-home") 94 | common.CheckSandboxDir(sd.SandboxDir) 95 | sd.InstalledPorts = common.GetInstalledPorts(sd.SandboxDir) 96 | sd.LoadGrants = true 97 | sd.SkipStart, _ = flags.GetBool("skip-start") 98 | skip_load_grants, _ := flags.GetBool("skip-load-grants") 99 | if skip_load_grants || sd.SkipStart { 100 | sd.LoadGrants = false 101 | } 102 | sd.SkipReportHost, _ = flags.GetBool("skip-report-host") 103 | sd.SkipReportPort, _ = flags.GetBool("skip-report-port") 104 | sd.DisableMysqlX, _ = flags.GetBool("disable-mysqlx") 105 | sd.DbUser, _ = flags.GetString("db-user") 106 | sd.DbPassword, _ = flags.GetString("db-password") 107 | sd.RplUser, _ = flags.GetString("rpl-user") 108 | sd.RplPassword, _ = flags.GetString("rpl-password") 109 | sd.RemoteAccess, _ = flags.GetString("remote-access") 110 | sd.BindAddress, _ = flags.GetString("bind-address") 111 | sd.CustomMysqld, _ = flags.GetString("custom-mysqld") 112 | sd.InitOptions, _ = flags.GetStringSlice("init-options") 113 | sd.MyCnfOptions, _ = flags.GetStringSlice("my-cnf-options") 114 | sd.PreGrantsSqlFile, _ = flags.GetString("pre-grants-sql-file") 115 | sd.PreGrantsSql, _ = flags.GetStringSlice("pre-grants-sql") 116 | sd.PostGrantsSql, _ = flags.GetStringSlice("post-grants-sql") 117 | sd.PostGrantsSqlFile, _ = flags.GetString("post-grants-sql-file") 118 | sd.MyCnfFile, _ = flags.GetString("my-cnf-file") 119 | sd.NativeAuthPlugin, _ = flags.GetBool("native-auth-plugin") 120 | sd.KeepUuid, _ = flags.GetBool("keep-server-uuid") 121 | sd.Force, _ = flags.GetBool("force") 122 | sd.ExposeDdTables, _ = flags.GetBool("expose-dd-tables") 123 | 124 | sd.RunConcurrently, _ = flags.GetBool("concurrent") 125 | if os.Getenv("RUN_CONCURRENTLY") != "" { 126 | sd.RunConcurrently = true 127 | } 128 | 129 | new_defaults, _ := flags.GetStringSlice("defaults") 130 | process_defaults(new_defaults) 131 | 132 | var gtid bool 133 | var master bool 134 | master, _ = flags.GetBool("master") 135 | gtid, _ = flags.GetBool("gtid") 136 | if master { 137 | sd.ReplOptions = sandbox.SingleTemplates["replication_options"].Contents 138 | sd.ServerId = sd.Port 139 | } 140 | if gtid { 141 | if common.GreaterOrEqualVersion(sd.Version, []int{5, 6, 9}) { 142 | sd.GtidOptions = sandbox.SingleTemplates["gtid_options"].Contents 143 | sd.ReplOptions = sandbox.SingleTemplates["replication_options"].Contents 144 | sd.ServerId = sd.Port 145 | } else { 146 | fmt.Println("--gtid requires version 5.6.9+") 147 | os.Exit(1) 148 | } 149 | } 150 | return sd 151 | } 152 | 153 | func SingleSandbox(cmd *cobra.Command, args []string) { 154 | var sd sandbox.SandboxDef 155 | common.CheckOrigin(args) 156 | sd = FillSdef(cmd, args) 157 | // When deploying a single sandbox, we disable concurrency 158 | sd.RunConcurrently = false 159 | sandbox.CreateSingleSandbox(sd, args[0]) 160 | } 161 | 162 | /* 163 | func ReplacedCmd(cmd *cobra.Command, args []string) { 164 | invoked := cmd.Use 165 | fmt.Printf("The command \"%s\" has been replaced.\n",invoked) 166 | fmt.Printf("Use \"dbdeployer deploy %s\" instead.\n",invoked) 167 | os.Exit(0) 168 | } 169 | */ 170 | 171 | var singleCmd = &cobra.Command{ 172 | Use: "single MySQL-Version", 173 | // Args: cobra.ExactArgs(1), 174 | Short: "deploys a single sandbox", 175 | Long: `single installs a sandbox and creates useful scripts for its use. 176 | MySQL-Version is in the format x.x.xx, and it refers to a directory named after the version 177 | containing an unpacked tarball. The place where these directories are found is defined by 178 | --sandbox-binary (default: $HOME/opt/mysql.) 179 | For example: 180 | dbdeployer deploy single 5.7.21 181 | 182 | For this command to work, there must be a directory $HOME/opt/mysql/5.7.21, containing 183 | the binary files from mysql-5.7.21-$YOUR_OS-x86_64.tar.gz 184 | Use the "unpack" command to get the tarball into the right directory. 185 | `, 186 | Run: SingleSandbox, 187 | } 188 | 189 | /* 190 | var ( 191 | hiddenSingleCmd = &cobra.Command{ 192 | Use: "single", 193 | Short: "REMOVED: use 'deploy single' instead", 194 | Hidden: true, 195 | Run: ReplacedCmd, 196 | } 197 | hiddenReplicationCmd = &cobra.Command{ 198 | Use: "replication", 199 | Short: "REMOVED: use 'deploy replication' instead", 200 | Hidden: true, 201 | Run: ReplacedCmd, 202 | } 203 | 204 | hiddenMultipleCmd = &cobra.Command{ 205 | Use: "multiple", 206 | Short: "REMOVED: use 'deploy multiple' instead", 207 | Hidden: true, 208 | Run: ReplacedCmd, 209 | } 210 | ) 211 | */ 212 | 213 | func init() { 214 | //rootCmd.AddCommand(hiddenSingleCmd) 215 | //rootCmd.AddCommand(hiddenReplicationCmd) 216 | //rootCmd.AddCommand(hiddenMultipleCmd) 217 | deployCmd.AddCommand(singleCmd) 218 | singleCmd.PersistentFlags().Bool("master", false, "Make the server replication ready") 219 | 220 | } 221 | -------------------------------------------------------------------------------- /sandbox/multi-source-replication.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package sandbox 17 | 18 | import ( 19 | "fmt" 20 | "os" 21 | "regexp" 22 | "strings" 23 | "strconv" 24 | "github.com/datacharmer/dbdeployer/common" 25 | "github.com/datacharmer/dbdeployer/defaults" 26 | ) 27 | 28 | func check_node_lists(nodes int, mlist, slist []int) { 29 | for _, N := range mlist { 30 | if N > nodes { 31 | fmt.Printf("Master num '%d' greater than number of nodes (%d)\n", N, nodes) 32 | os.Exit(1) 33 | } 34 | } 35 | for _, N := range slist { 36 | if N > nodes { 37 | fmt.Printf("Slave num '%d' greater than number of nodes (%d)\n", N, nodes) 38 | os.Exit(1) 39 | } 40 | } 41 | for _, M := range mlist { 42 | for _, S := range slist { 43 | if S == M { 44 | fmt.Printf("Overlapping values: %d is in both master and slave list\n",M) 45 | os.Exit(1) 46 | } 47 | } 48 | } 49 | total_nodes := len(mlist) + len(slist) 50 | if total_nodes != nodes { 51 | fmt.Printf("Mismatched values: masters (%d) + slaves (%d) = %d. Expected: %d \n",len(mlist), len(slist), total_nodes, nodes) 52 | os.Exit(1) 53 | } 54 | } 55 | 56 | func nodes_list_to_int_slice(nodes_list string, nodes int) (int_list []int) { 57 | separator := " " 58 | if common.Includes(nodes_list, ",") { 59 | separator = "," 60 | } else if common.Includes(nodes_list, ":") { 61 | separator = ":" 62 | } else if common.Includes(nodes_list, ";") { 63 | separator = ";" 64 | } else if common.Includes(nodes_list, `\.`) { 65 | separator = "." 66 | } else { 67 | separator = " " 68 | } 69 | list := strings.Split(nodes_list, separator) 70 | // fmt.Printf("# separator: <%s> %#v\n",separator, list) 71 | if len(list) == 0 { 72 | fmt.Printf("Empty nodes list given (%s)\n",nodes_list) 73 | os.Exit(1) 74 | } 75 | for _, s := range list { 76 | if s != "" { 77 | num, err := strconv.Atoi(s) 78 | if err != nil { 79 | fmt.Printf("Error converting node number '%s' to int\n",s) 80 | os.Exit(1) 81 | } 82 | int_list = append(int_list, num) 83 | } 84 | } 85 | if len(int_list) == 0 { 86 | fmt.Printf("List '%s' is empty\n", nodes_list) 87 | } 88 | if len(int_list) > nodes { 89 | fmt.Printf("List '%s' is greater than the expected number of nodes (%d)\n", nodes_list, nodes) 90 | } 91 | return 92 | } 93 | 94 | func make_nodes_list(nodes int) (nodes_list string) { 95 | for N := 1 ; N<= nodes; N++ { 96 | nodes_list += fmt.Sprintf("%d ", N) 97 | } 98 | return nodes_list 99 | } 100 | 101 | func CreateAllMastersReplication(sdef SandboxDef, origin string, nodes int, master_ip string) { 102 | sdef.SBType= "all-masters" 103 | sdef.GtidOptions = SingleTemplates["gtid_options"].Contents 104 | if sdef.DirName == "" { 105 | sdef.DirName += defaults.Defaults().AllMastersPrefix + common.VersionToName(origin) 106 | } 107 | sandbox_dir := sdef.SandboxDir 108 | sdef.SandboxDir = common.DirName(sdef.SandboxDir) 109 | sdef.BasePort = defaults.Defaults().AllMastersReplicationBasePort 110 | master_abbr := defaults.Defaults().MasterAbbr 111 | slave_abbr := defaults.Defaults().SlaveAbbr 112 | master_label := defaults.Defaults().MasterName 113 | slave_label := defaults.Defaults().SlavePrefix 114 | data := CreateMultipleSandbox(sdef, origin, nodes) 115 | master_list := make_nodes_list(nodes) 116 | slist := nodes_list_to_int_slice(master_list, nodes) 117 | data["MasterIp"] = master_ip 118 | data["MasterAbbr"] = master_abbr 119 | data["MasterLabel"] = master_label 120 | data["MasterList"] = normalize_node_list(master_list) 121 | data["SlaveAbbr"] = slave_abbr 122 | data["SlaveLabel"] = slave_label 123 | data["SlaveList"] = normalize_node_list(master_list) 124 | data["RplUser"] = sdef.RplUser 125 | data["RplPassword"] = sdef.RplPassword 126 | data["NodeLabel"] = defaults.Defaults().NodePrefix 127 | for _, node := range slist { 128 | data["Node"] = node 129 | write_script(ReplicationTemplates, fmt.Sprintf("s%d", node), "slave_template", sandbox_dir, data, true) 130 | write_script(ReplicationTemplates, fmt.Sprintf("m%d", node), "slave_template", sandbox_dir, data, true) 131 | } 132 | write_script(ReplicationTemplates, "test_replication", "multi_source_test_template", sandbox_dir, data, true) 133 | write_script(ReplicationTemplates, "use_all_slaves", "multi_source_use_slaves_template", sandbox_dir, data, true) 134 | write_script(ReplicationTemplates, "use_all_masters", "multi_source_use_masters_template", sandbox_dir, data, true) 135 | write_script(ReplicationTemplates, "check_ms_nodes", "check_multi_source_template", sandbox_dir, data, true) 136 | write_script(ReplicationTemplates, "initialize_ms_nodes", "multi_source_template", sandbox_dir, data, true) 137 | if !sdef.SkipStart { 138 | fmt.Println(common.ReplaceLiteralHome(sandbox_dir) + "/initialize_ms_nodes") 139 | common.Run_cmd(sandbox_dir + "/initialize_ms_nodes") 140 | } 141 | } 142 | 143 | func normalize_node_list(list string) string { 144 | re := regexp.MustCompile(`[,:\.]`) 145 | return re.ReplaceAllString(list, " ") 146 | } 147 | 148 | func CreateFanInReplication(sdef SandboxDef, origin string, nodes int, master_ip, master_list, slave_list string) { 149 | sdef.SBType= "fan-in" 150 | sdef.GtidOptions = SingleTemplates["gtid_options"].Contents 151 | if sdef.DirName == "" { 152 | sdef.DirName = defaults.Defaults().FanInPrefix + common.VersionToName(origin) 153 | } 154 | sdef.BasePort = defaults.Defaults().FanInReplicationBasePort 155 | sandbox_dir := sdef.SandboxDir 156 | sdef.SandboxDir = common.DirName(sdef.SandboxDir) 157 | mlist := nodes_list_to_int_slice(master_list, nodes) 158 | slist := nodes_list_to_int_slice(slave_list, nodes) 159 | check_node_lists(nodes, mlist, slist) 160 | data := CreateMultipleSandbox(sdef, origin, nodes) 161 | master_abbr := defaults.Defaults().MasterAbbr 162 | slave_abbr := defaults.Defaults().SlaveAbbr 163 | master_label := defaults.Defaults().MasterName 164 | slave_label := defaults.Defaults().SlavePrefix 165 | data["MasterList"] = normalize_node_list(master_list) 166 | data["SlaveList"] = normalize_node_list(slave_list) 167 | data["MasterAbbr"] = master_abbr 168 | data["MasterLabel"] = master_label 169 | data["SlaveAbbr"] = slave_abbr 170 | data["SlaveLabel"] = slave_label 171 | data["RplUser"] = sdef.RplUser 172 | data["RplPassword"] = sdef.RplPassword 173 | data["NodeLabel"] = defaults.Defaults().NodePrefix 174 | data["MasterIp"] = master_ip 175 | for _, slave := range slist { 176 | data["Node"] = slave 177 | write_script(ReplicationTemplates, fmt.Sprintf("s%d", slave), "slave_template", sandbox_dir, data, true) 178 | } 179 | for _, master := range mlist { 180 | data["Node"] = master 181 | write_script(ReplicationTemplates, fmt.Sprintf("m%d", master), "slave_template", sandbox_dir, data, true) 182 | } 183 | write_script(ReplicationTemplates, "test_replication", "multi_source_test_template", sandbox_dir, data, true) 184 | write_script(ReplicationTemplates, "check_ms_nodes", "check_multi_source_template", sandbox_dir, data, true) 185 | write_script(ReplicationTemplates, "use_all_slaves", "multi_source_use_slaves_template", sandbox_dir, data, true) 186 | write_script(ReplicationTemplates, "use_all_masters", "multi_source_use_masters_template", sandbox_dir, data, true) 187 | write_script(ReplicationTemplates, "initialize_ms_nodes", "multi_source_template", sandbox_dir, data, true) 188 | if !sdef.SkipStart { 189 | fmt.Println(common.ReplaceLiteralHome(sandbox_dir) + "/initialize_ms_nodes") 190 | common.Run_cmd(sandbox_dir + "/initialize_ms_nodes") 191 | } 192 | } 193 | 194 | 195 | -------------------------------------------------------------------------------- /common/fileutil.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package common 17 | 18 | import ( 19 | "bufio" 20 | //"bytes" 21 | "encoding/json" 22 | "fmt" 23 | "io" 24 | "io/ioutil" 25 | "log" 26 | "os" 27 | "os/exec" 28 | "path/filepath" 29 | "regexp" 30 | ) 31 | 32 | type SandboxUser struct { 33 | Description string `json:"description"` 34 | Username string `json:"username"` 35 | Password string `json:"password"` 36 | Privileges string `json:"privileges"` 37 | } 38 | 39 | type SandboxDescription struct { 40 | Basedir string `json:"basedir"` 41 | SBType string `json:"type"` // single multi master-slave group 42 | Version string `json:"version"` 43 | Port []int `json:"port"` 44 | Nodes int `json:"nodes"` 45 | NodeNum int `json:"node_num"` 46 | } 47 | 48 | type keyvalue struct { 49 | Key string 50 | Value string 51 | } 52 | 53 | type configOptions map[string][]keyvalue 54 | 55 | func ParseConfigFile(filename string) configOptions { 56 | config := make(configOptions) 57 | lines := SlurpAsLines(filename) 58 | re_comment := regexp.MustCompile(`^\s*#`) 59 | re_empty := regexp.MustCompile(`^\s*$`) 60 | re_header := regexp.MustCompile(`\[\s*(\w+)\s*\]`) 61 | re_k_v := regexp.MustCompile(`(\S+)\s*=\s*(.*)`) 62 | current_header := "" 63 | for _, line := range lines { 64 | if re_comment.MatchString(line) || re_empty.MatchString(line) { 65 | continue 66 | } 67 | headerList := re_header.FindAllStringSubmatch(line, -1) 68 | if headerList != nil { 69 | header := headerList[0][1] 70 | current_header = header 71 | } 72 | kvList := re_k_v.FindAllStringSubmatch(line, -1) 73 | if kvList != nil { 74 | kv := keyvalue{ 75 | Key: kvList[0][1], 76 | Value: kvList[0][2], 77 | } 78 | config[current_header] = append(config[current_header], kv) 79 | } 80 | } 81 | /*for header, kvList := range config { 82 | fmt.Printf("%s \n", header) 83 | for N, kv := range kvList { 84 | fmt.Printf("%d %s : %s \n", N, kv.key, kv.value) 85 | } 86 | fmt.Printf("\n") 87 | }*/ 88 | return config 89 | } 90 | 91 | func WriteSandboxDescription(destination string, sd SandboxDescription) { 92 | b, err := json.MarshalIndent(sd, " ", "\t") 93 | if err != nil { 94 | fmt.Println("error encoding sandbox description: ", err) 95 | os.Exit(1) 96 | } 97 | json_string := fmt.Sprintf("%s", b) 98 | filename := destination + "/sbdescription.json" 99 | WriteString(json_string, filename) 100 | } 101 | 102 | func ReadSandboxDescription(sandbox_directory string) (sd SandboxDescription) { 103 | filename := sandbox_directory + "/sbdescription.json" 104 | sb_blob := SlurpAsBytes(filename) 105 | 106 | err := json.Unmarshal(sb_blob, &sd) 107 | if err != nil { 108 | fmt.Println("error decoding sandbox description: ", err) 109 | os.Exit(1) 110 | } 111 | return 112 | } 113 | 114 | func SlurpAsLines(filename string) []string { 115 | f, err := os.Open(filename) 116 | if err != nil { 117 | panic(err) 118 | } 119 | defer f.Close() 120 | 121 | var lines []string 122 | scanner := bufio.NewScanner(f) 123 | for scanner.Scan() { 124 | lines = append(lines, scanner.Text()) 125 | } 126 | if err := scanner.Err(); err != nil { 127 | fmt.Fprintln(os.Stderr, err) 128 | os.Exit(1) 129 | } 130 | return lines 131 | } 132 | 133 | func SlurpAsString(filename string) string { 134 | b := SlurpAsBytes(filename) 135 | str := string(b) 136 | return str 137 | } 138 | 139 | func SlurpAsBytes(filename string) []byte { 140 | b, err := ioutil.ReadFile(filename) 141 | if err != nil { 142 | panic(err) 143 | } 144 | return b 145 | } 146 | 147 | func WriteStrings(lines []string, filename string, termination string) error { 148 | file, err := os.Create(filename) 149 | if err != nil { 150 | return err 151 | } 152 | defer file.Close() 153 | 154 | w := bufio.NewWriter(file) 155 | for _, line := range lines { 156 | fmt.Fprintln(w, line+termination) 157 | } 158 | return w.Flush() 159 | } 160 | 161 | func AppendStrings(lines []string, filename string, termination string) error { 162 | file, err := os.Open(filename) 163 | if err != nil { 164 | return err 165 | } 166 | defer file.Close() 167 | 168 | w := bufio.NewWriter(file) 169 | for _, line := range lines { 170 | fmt.Fprintln(w, line+termination) 171 | } 172 | return w.Flush() 173 | } 174 | 175 | func WriteString(line string, filename string) error { 176 | return WriteStrings([]string{line}, filename, "") 177 | } 178 | 179 | func FileExists(filename string) bool { 180 | _, err := os.Stat(filename) 181 | if os.IsNotExist(err) { 182 | return false 183 | } 184 | return true 185 | } 186 | 187 | func DirExists(filename string) bool { 188 | f, err := os.Stat(filename) 189 | if os.IsNotExist(err) { 190 | return false 191 | } 192 | filemode := f.Mode() 193 | return filemode.IsDir() 194 | } 195 | 196 | func Which(filename string) string { 197 | path, err := exec.LookPath(filename) 198 | if err == nil { 199 | return path 200 | } 201 | return "" 202 | } 203 | 204 | func ExecExists(filename string) bool { 205 | _, err := exec.LookPath(filename) 206 | return err == nil 207 | } 208 | 209 | func Run_cmd_with_args(c string, args []string) (error, string) { 210 | cmd := exec.Command(c, args...) 211 | //var out bytes.Buffer 212 | //var stderr bytes.Buffer 213 | //cmd.Stdout = &out 214 | //cmd.Stderr = &stderr 215 | var out []byte 216 | var err error 217 | out, err = cmd.Output() 218 | if err != nil { 219 | fmt.Printf("err: %s\n", err) 220 | fmt.Printf("cmd: %#v\n", cmd) 221 | fmt.Printf("stdout: %s\n", out) 222 | //fmt.Printf("stderr: %s\n", stderr.String()) 223 | // os.Exit(1) 224 | } else { 225 | //fmt.Printf("%s", out.String()) 226 | fmt.Printf("%s", out) 227 | } 228 | return err, string(out) 229 | } 230 | 231 | func Run_cmd_ctrl(c string, silent bool) (error, string) { 232 | //cmd := exec.Command(c, args...) 233 | cmd := exec.Command(c, "") 234 | //var out bytes.Buffer 235 | //var stderr bytes.Buffer 236 | //cmd.Stdout = &out 237 | //cmd.Stderr = &stderr 238 | 239 | //err := cmd.Run() 240 | var out []byte 241 | var err error 242 | out, err = cmd.Output() 243 | if err != nil { 244 | fmt.Printf("err: %s\n", err) 245 | fmt.Printf("cmd: %#v\n", cmd) 246 | fmt.Printf("stdout: %s\n", out) 247 | //fmt.Printf("stdout: %s\n", out.String()) 248 | //fmt.Printf("stderr: %s\n", stderr.String()) 249 | // os.Exit(1) 250 | } else { 251 | if !silent { 252 | //fmt.Printf("%s", out.String()) 253 | fmt.Printf("%s", out) 254 | } 255 | } 256 | return err, string(out) 257 | } 258 | 259 | func Run_cmd(c string) (error, string) { 260 | return Run_cmd_ctrl(c, false) 261 | } 262 | 263 | func CopyFile(source, destination string) { 264 | sfile, err := os.Stat(source) 265 | if err != nil { 266 | log.Fatal(err) 267 | } 268 | fmode := sfile.Mode() 269 | from, err := os.Open(source) 270 | if err != nil { 271 | log.Fatal(err) 272 | } 273 | defer from.Close() 274 | 275 | to, err := os.OpenFile(destination, os.O_RDWR|os.O_CREATE, fmode) // 0666) 276 | if err != nil { 277 | log.Fatal(err) 278 | } 279 | defer to.Close() 280 | 281 | _, err = io.Copy(to, from) 282 | if err != nil { 283 | log.Fatal(err) 284 | } 285 | } 286 | 287 | func BaseName(filename string) string { 288 | return filepath.Base(filename) 289 | } 290 | 291 | func DirName(filename string) string { 292 | return filepath.Dir(filename) 293 | } 294 | 295 | func Mkdir(dir_name string) { 296 | err := os.Mkdir(dir_name, 0755) 297 | if err != nil { 298 | fmt.Printf("Error creating directory %s\n%s\n", dir_name, err) 299 | } 300 | } 301 | -------------------------------------------------------------------------------- /sandbox/group_replication.go: -------------------------------------------------------------------------------- 1 | // DBDeployer - The MySQL Sandbox 2 | // Copyright © 2006-2018 Giuseppe Maxia 3 | // 4 | // Licensed under the Apache License, Version 2.0 (the "License"); 5 | // you may not use this file except in compliance with the License. 6 | // You may obtain a copy of the License at 7 | // 8 | // http://www.apache.org/licenses/LICENSE-2.0 9 | // 10 | // Unless required by applicable law or agreed to in writing, software 11 | // distributed under the License is distributed on an "AS IS" BASIS, 12 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | // See the License for the specific language governing permissions and 14 | // limitations under the License. 15 | 16 | package sandbox 17 | 18 | import ( 19 | "fmt" 20 | "github.com/datacharmer/dbdeployer/common" 21 | "github.com/datacharmer/dbdeployer/concurrent" 22 | "github.com/datacharmer/dbdeployer/defaults" 23 | "os" 24 | "regexp" 25 | "time" 26 | ) 27 | 28 | const ( 29 | GroupReplOptions string = ` 30 | binlog_checksum=NONE 31 | log_slave_updates=ON 32 | plugin-load=group_replication.so 33 | group_replication=FORCE_PLUS_PERMANENT 34 | group_replication_start_on_boot=OFF 35 | group_replication_bootstrap_group=OFF 36 | transaction_write_set_extraction=XXHASH64 37 | report-host=127.0.0.1 38 | loose-group_replication_group_name="aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee" 39 | ` 40 | GroupReplSinglePrimary string = ` 41 | loose-group-replication-single-primary-mode=on 42 | ` 43 | GroupReplMultiPrimary string = ` 44 | loose-group-replication-single-primary-mode=off 45 | ` 46 | ) 47 | 48 | func get_base_mysqlx_port(base_port int, sdef SandboxDef, nodes int) int { 49 | base_mysqlx_port := base_port + 10000 50 | if common.GreaterOrEqualVersion(sdef.Version, []int{8,0,11}) { 51 | base_mysqlx_port = FindFreePort(base_mysqlx_port, sdef.InstalledPorts, nodes) 52 | for check_port := base_mysqlx_port + 1; check_port < base_mysqlx_port+nodes+1; check_port++ { 53 | CheckPort(sdef.SandboxDir, sdef.InstalledPorts, check_port) 54 | } 55 | } 56 | return base_mysqlx_port 57 | } 58 | 59 | func CreateGroupReplication(sdef SandboxDef, origin string, nodes int, master_ip string) { 60 | var exec_lists []concurrent.ExecutionList 61 | vList := common.VersionToList(sdef.Version) 62 | rev := vList[2] 63 | // base_port := sdef.Port + defaults.Defaults().GroupReplicationBasePort + (rev * 100) 64 | base_port := sdef.Port + defaults.Defaults().GroupReplicationBasePort + rev 65 | if sdef.SinglePrimary { 66 | base_port = sdef.Port + defaults.Defaults().GroupReplicationSpBasePort + (rev * 100) 67 | } 68 | if sdef.BasePort > 0 { 69 | base_port = sdef.BasePort 70 | } 71 | 72 | base_server_id := 0 73 | if nodes < 3 { 74 | fmt.Println("Can't run group replication with less than 3 nodes") 75 | os.Exit(1) 76 | } 77 | if common.DirExists(sdef.SandboxDir) { 78 | sdef = CheckDirectory(sdef) 79 | } 80 | base_port = FindFreePort(base_port, sdef.InstalledPorts, nodes) 81 | base_mysqlx_port := get_base_mysqlx_port(base_port, sdef, nodes) 82 | base_group_port := base_port + defaults.Defaults().GroupPortDelta 83 | base_group_port = FindFreePort(base_group_port, sdef.InstalledPorts, nodes) 84 | for check_port := base_port + 1; check_port < base_port+nodes+1; check_port++ { 85 | CheckPort(sdef.SandboxDir, sdef.InstalledPorts, check_port) 86 | } 87 | for check_port := base_group_port + 1; check_port < base_group_port+nodes+1; check_port++ { 88 | CheckPort(sdef.SandboxDir, sdef.InstalledPorts, check_port) 89 | } 90 | common.Mkdir(sdef.SandboxDir) 91 | timestamp := time.Now() 92 | slave_label := defaults.Defaults().SlavePrefix 93 | slave_abbr := defaults.Defaults().SlaveAbbr 94 | master_abbr := defaults.Defaults().MasterAbbr 95 | master_label := defaults.Defaults().MasterName 96 | master_list := make_nodes_list(nodes) 97 | slave_list := master_list 98 | if sdef.SinglePrimary { 99 | master_list = "1" 100 | slave_list = "" 101 | for N := 2; N <= nodes ; N++ { 102 | if slave_list != "" { 103 | slave_list += " " 104 | } 105 | slave_list += fmt.Sprintf("%d", N) 106 | } 107 | mlist := nodes_list_to_int_slice(master_list, nodes) 108 | slist := nodes_list_to_int_slice(slave_list, nodes) 109 | check_node_lists(nodes, mlist, slist) 110 | } 111 | change_master_extra := "" 112 | node_label := defaults.Defaults().NodePrefix 113 | //if common.GreaterOrEqualVersion(sdef.Version, []int{8,0,4}) { 114 | // if !sdef.NativeAuthPlugin { 115 | // change_master_extra = ", GET_MASTER_PUBLIC_KEY=1" 116 | // } 117 | //} 118 | var data common.Smap = common.Smap{ 119 | "Copyright": Copyright, 120 | "AppVersion": common.VersionDef, 121 | "DateTime": timestamp.Format(time.UnixDate), 122 | "SandboxDir": sdef.SandboxDir, 123 | "MasterIp": master_ip, 124 | "MasterList": master_list, 125 | "NodeLabel": node_label, 126 | "SlaveList": slave_list, 127 | "RplUser": sdef.RplUser, 128 | "RplPassword": sdef.RplPassword, 129 | "SlaveLabel": slave_label, 130 | "SlaveAbbr": slave_abbr, 131 | "ChangeMasterExtra" : change_master_extra, 132 | "MasterLabel": master_label, 133 | "MasterAbbr": master_abbr, 134 | "Nodes": []common.Smap{}, 135 | } 136 | connection_string := "" 137 | for i := 0; i < nodes; i++ { 138 | group_port := base_group_port + i + 1 139 | if connection_string != "" { 140 | connection_string += "," 141 | } 142 | connection_string += fmt.Sprintf("127.0.0.1:%d", group_port) 143 | } 144 | 145 | sb_type := "group-multi-primary" 146 | single_multi_primary := GroupReplMultiPrimary 147 | if sdef.SinglePrimary { 148 | sb_type = "group-single-primary" 149 | single_multi_primary = GroupReplSinglePrimary 150 | } 151 | 152 | sb_desc := common.SandboxDescription{ 153 | Basedir: sdef.Basedir + "/" + sdef.Version, 154 | SBType: sb_type, 155 | Version: sdef.Version, 156 | Port: []int{}, 157 | Nodes: nodes, 158 | NodeNum: 0, 159 | } 160 | 161 | sb_item := defaults.SandboxItem{ 162 | Origin : sb_desc.Basedir, 163 | SBType : sb_desc.SBType, 164 | Version: sdef.Version, 165 | Port: []int{}, 166 | Nodes: []string{}, 167 | Destination: sdef.SandboxDir, 168 | } 169 | 170 | for i := 1; i <= nodes; i++ { 171 | group_port := base_group_port + i 172 | data["Nodes"] = append(data["Nodes"].([]common.Smap), common.Smap{ 173 | "Copyright": Copyright, 174 | "AppVersion": common.VersionDef, 175 | "DateTime": timestamp.Format(time.UnixDate), 176 | "Node": i, 177 | "MasterIp": master_ip, 178 | "NodeLabel": node_label, 179 | "SlaveLabel": slave_label, 180 | "SlaveAbbr": slave_abbr, 181 | "ChangeMasterExtra" : change_master_extra, 182 | "MasterLabel": master_label, 183 | "MasterAbbr": master_abbr, 184 | "SandboxDir": sdef.SandboxDir, 185 | "RplUser": sdef.RplUser, 186 | "RplPassword": sdef.RplPassword}) 187 | 188 | sdef.DirName = fmt.Sprintf("%s%d", node_label, i) 189 | sdef.Port = base_port + i 190 | sdef.MorePorts = []int{group_port} 191 | sdef.ServerId = (base_server_id + i) * 100 192 | sb_item.Nodes = append(sb_item.Nodes, sdef.DirName) 193 | sb_item.Port = append(sb_item.Port, sdef.Port) 194 | sb_desc.Port = append(sb_desc.Port, sdef.Port) 195 | sb_item.Port = append(sb_item.Port, sdef.Port + defaults.Defaults().GroupPortDelta ) 196 | sb_desc.Port = append(sb_desc.Port, sdef.Port + defaults.Defaults().GroupPortDelta ) 197 | 198 | if !sdef.RunConcurrently { 199 | installation_message := "Installing and starting %s %d\n" 200 | if sdef.SkipStart { 201 | installation_message="Installing %s %d\n" 202 | } 203 | fmt.Printf(installation_message, node_label, i) 204 | } 205 | sdef.ReplOptions = SingleTemplates["replication_options"].Contents + fmt.Sprintf("\n%s\n%s\n", GroupReplOptions, single_multi_primary) 206 | re_master_ip := regexp.MustCompile(`127\.0\.0\.1`) 207 | sdef.ReplOptions = re_master_ip.ReplaceAllString(sdef.ReplOptions, master_ip) 208 | sdef.ReplOptions += fmt.Sprintf("\n%s\n", SingleTemplates["gtid_options"].Contents) 209 | sdef.ReplOptions += fmt.Sprintf("\nloose-group-replication-local-address=%s:%d\n", master_ip, group_port) 210 | sdef.ReplOptions += fmt.Sprintf("\nloose-group-replication-group-seeds=%s\n", connection_string) 211 | if common.GreaterOrEqualVersion(sdef.Version, []int{8,0,11}) { 212 | sdef.MysqlXPort = base_mysqlx_port + i 213 | if !sdef.DisableMysqlX { 214 | sb_desc.Port = append(sb_desc.Port, base_mysqlx_port + i) 215 | sb_item.Port = append(sb_item.Port, base_mysqlx_port + i) 216 | } 217 | } 218 | sdef.Multi = true 219 | sdef.LoadGrants = true 220 | sdef.Prompt = fmt.Sprintf("%s%d", node_label, i) 221 | sdef.SBType = "group-node" 222 | sdef.NodeNum = i 223 | // fmt.Printf("%#v\n",sdef) 224 | exec_list := CreateSingleSandbox(sdef, origin) 225 | for _, list := range exec_list { 226 | exec_lists = append(exec_lists, list) 227 | } 228 | var data_node common.Smap = common.Smap{ 229 | "Copyright": Copyright, 230 | "AppVersion": common.VersionDef, 231 | "DateTime": timestamp.Format(time.UnixDate), 232 | "Node": i, 233 | "NodeLabel": node_label, 234 | "MasterLabel": master_label, 235 | "MasterAbbr": master_abbr, 236 | "ChangeMasterExtra" : change_master_extra, 237 | "SlaveLabel": slave_label, 238 | "SlaveAbbr": slave_abbr, 239 | "SandboxDir": sdef.SandboxDir, 240 | } 241 | write_script(MultipleTemplates, fmt.Sprintf("n%d", i), "node_template", sdef.SandboxDir, data_node, true) 242 | } 243 | common.WriteSandboxDescription(sdef.SandboxDir, sb_desc) 244 | defaults.UpdateCatalog(sdef.SandboxDir, sb_item) 245 | 246 | write_script(MultipleTemplates, "start_all", "start_multi_template", sdef.SandboxDir, data, true) 247 | write_script(MultipleTemplates, "restart_all", "restart_multi_template", sdef.SandboxDir, data, true) 248 | write_script(MultipleTemplates, "status_all", "status_multi_template", sdef.SandboxDir, data, true) 249 | write_script(MultipleTemplates, "test_sb_all", "test_sb_multi_template", sdef.SandboxDir, data, true) 250 | write_script(MultipleTemplates, "stop_all", "stop_multi_template", sdef.SandboxDir, data, true) 251 | write_script(MultipleTemplates, "clear_all", "clear_multi_template", sdef.SandboxDir, data, true) 252 | write_script(MultipleTemplates, "send_kill_all", "send_kill_multi_template", sdef.SandboxDir, data, true) 253 | write_script(MultipleTemplates, "use_all", "use_multi_template", sdef.SandboxDir, data, true) 254 | write_script(ReplicationTemplates, "use_all_slaves", "multi_source_use_slaves_template", sdef.SandboxDir, data, true) 255 | write_script(ReplicationTemplates, "use_all_masters", "multi_source_use_masters_template", sdef.SandboxDir, data, true) 256 | write_script(GroupTemplates, "initialize_nodes", "init_nodes_template", sdef.SandboxDir, data, true) 257 | write_script(GroupTemplates, "check_nodes", "check_nodes_template", sdef.SandboxDir, data, true) 258 | //write_script(ReplicationTemplates, "test_replication", "test_replication_template", sdef.SandboxDir, data, true) 259 | write_script(ReplicationTemplates, "test_replication", "multi_source_test_template", sdef.SandboxDir, data, true) 260 | 261 | concurrent.RunParallelTasksByPriority(exec_lists) 262 | if !sdef.SkipStart { 263 | fmt.Println(common.ReplaceLiteralHome(sdef.SandboxDir) + "/initialize_nodes") 264 | common.Run_cmd(sdef.SandboxDir + "/initialize_nodes") 265 | } 266 | fmt.Printf("Replication directory installed in %s\n", common.ReplaceLiteralHome(sdef.SandboxDir)) 267 | fmt.Printf("run 'dbdeployer usage multiple' for basic instructions'\n") 268 | } 269 | -------------------------------------------------------------------------------- /Changelog: -------------------------------------------------------------------------------- 1 | 1.3.0 21-Apr-2018 2 | ADJUSTMENTS: 3 | - Added support for mysqlx plugin being enabled by default (MySQL 8.0.11+) 4 | - Added flag "--disable-mysqlx" to disable mysqlx plugin (8.0.11+) 5 | NEW FEATURES: 6 | - Added scripts use_all_masters and use_all_slaves to all replication 7 | sandboxes. 8 | - Added option --verbosity={0,1,2} to *unpack* command. 9 | BUGS FIXED: 10 | - Fixed Issue#10 "dbdeployer unpack does not handle symlinks" 11 | - Fixed minor bug in documentation test builder. 12 | TESTING 13 | - Added tests for number of ports, log errors, use_all_masters, 14 | use_all_slaves, running processes. 15 | - Added options to stop tests after a given set of operations. 16 | - Removed restriction on running 5.6 tests in docker for Mac. 17 | 1.2.0 14-Apr-2018 18 | - Added option --skip-start 19 | - Added report-port and report-host automatically to my.sandbox.cnf 20 | - Added options --skip-report-host and --skip-report-port 21 | - Added documentation dbdeployer compiling. 22 | - Added documentation for --skip-start 23 | - Enhanced build.sh to handle dependencies. 24 | - Added tests for --skip-start and report-host/report-port behavior. 25 | 1.1.1 02-Apr-2018 26 | - Added more documentation 27 | - Added bash-completion script 28 | - Moved hidden command "tree" to conditional compiling. 29 | Now there is a separated build for docs-enabled dbdeployer 30 | - Added ability of producing more documentation using command "tree" 31 | 1.1.0 30-Mar-2018 32 | - Added ability of handling environment variables 33 | in configuration file. $HOME and $PWD are expanded 34 | to actual values. 35 | - Added hidden command "tree" that can generate the 36 | full dbdeployer API. Using this feature, from now on 37 | we can compare API changes automatically. 38 | - Fixed visualization of sandboxes from catalog 39 | - Fixed minor code issues. 40 | - Added tests for environment variable replacement 41 | 1.0.1 28-Mar-2018 42 | - Fixed Issue #5 "Single deployment doesn't show the location of the 43 | sandbox" 44 | - Added API definition (./docs/API-1.0.md) 45 | - Added test for Issue #5 46 | - Fixed typos and improved docs. 47 | 1.0.0 26-Mar-2018 48 | - General Availability. 49 | - Fixed bug with single deployment and --force. On the second deployment, 50 | the port was changed. 51 | - More tests added. The test suite now runs a total of 3,013 tests (MacOS) 52 | and 3,143 (Linux). A total of 6,156 tests that ran at least twice (once 53 | with concurrency and once without) 54 | 0.3.9 25-Mar-2018 55 | - Added version detection to *unpack* command. now --unpack-version 56 | becomes mandatory only if a version is not detected from the tarball 57 | name. 58 | - Added --header flag to *sandboxes* command. 59 | - More tests and improved tests. 60 | 0.3.8 24-Mar-2018 61 | - Fixed deployment bug in fan-in replication 62 | - Added tests for fan-in replication, sandbox completeness, start, 63 | restart, and add_options. 64 | 0.3.7 24-Mar-2018 65 | - Added --semi-sync option to replication 66 | - Added more tests 67 | 0.3.6 21-Mar-2018 68 | - Minor change to templates 69 | - Added test for export/import templates 70 | - Added more tests for pre/post grants SQL 71 | 0.3.5 21-Mar-2018 72 | - Added test for on-the-fly template replacement 73 | - Trivial changes to "sandboxes" output 74 | 0.3.4 20-Mar-2018 75 | - Changed test for group replication (now uses the 76 | same defined for multi-source replication) 77 | - Improved usability of tests. 78 | - Made tests easier to extend. 79 | - Added test for pre/post grants SQL. 80 | 0.3.3 16-Mar-2018 81 | - Added (mock) tests for unpack command 82 | - Improved test reporting 83 | - Added list of environment variables 84 | 0.3.2 15-Mar-2018 85 | - Minor bug fixes 86 | - Added more tests 87 | 0.3.1 12-Mar-2018 88 | - Added topologies "fan-in" and "all-masters" 89 | - Feature complete: This is release candidate for 1.0 90 | - Fixed bug on UUID generation. 91 | 0.3.0 11-Mar-2018 92 | - Implemented parallel deployment of multiple sandboxes 93 | - Flag --concurrent is available for *deploy* and *delete* 94 | - Improved tests 95 | 0.2.5 10-Mar-2018 96 | - Added --catalog to "sandboxes" command 97 | - Improved tests 98 | 0.2.4 08-Mar-2018 99 | - INCOMPATIBLE CHANGES: 100 | . MySQL 8.0.x now starts with caching_sha2_password by default. 101 | . flag "--keep-auth-plugin" was removed. Instead, we now have 102 | "--native-auth-plugin", false by default, which will use the old 103 | plugin if enabled. 104 | . The sandbox catalog is now enabled by default. It can be disabled 105 | using either the environment variable SKIP_DBDEPLOYER_CATALOG 106 | or using the configuration file. 107 | - Added workaround for bug#89959: replication with 8.0 and 108 | caching_sha2_password fails 109 | 110 | 0.2.3 07-Mar-2018 (not released) 111 | - Improved mock test speed by parametrizing sleep intervals: 112 | . 200 mock sandboxes tested in 73 seconds (previously, 15 minutes). 113 | . 2144 mock sandboxes tested in 23 minutes (previously, 145 minutes) 114 | 0.2.2 07-Mar-2018 115 | - Now dbdeployer finds unused ports automatically, to avoid conflicts. 116 | - Added ability of running faster tests with mock MySQL packages. 117 | 0.2.1 04-Mar-2018 118 | - Added --defaults flag 119 | - Removed hardcoded names for multiple sandbox directories and shortcuts. 120 | - Added directory names and shortcuts for multiple sandboxes to configuration data 121 | - Added ability of exporting/importing a single template. 122 | - Fixed visualization error with template export 123 | - Added interactivity to main test script. 124 | 0.2.0 27-Feb-2018 125 | - INCOMPATIBLE CHANGES: 126 | . "single", "multiple", and "replication" are now subcommands of "deploy". 127 | . Previous "admin" commands are now under "defaults" 128 | . "templates" is now a subcommand of "defaults" 129 | . New "admin" command only supports "lock" and "unlock" 130 | 131 | - EXPERIMENTAL FEATURE: 132 | There is a sandbox catalog being created and updated in 133 | $HOME/.dbdeployer/sandboxes.json 134 | The deployment and deletion commands handle the catalog 135 | transparently. Disabled by default. It can be enabled by 136 | setting the environment variable DBDEPLOYER_CATALOG 137 | 138 | 0.1.25 26-Feb-2018 139 | - Added commands "admin lock" and "admin unlock" to prevent/allow deletion 140 | of a sandbox. 141 | - Added placeholder fields for multi-source clustering in defaults 142 | 0.1.24 20-Feb-2018 143 | - Fixed bug with "sandboxes" command. It would not check if the 144 | sandbox_home directory existed. 145 | - Fixed bug in "sandboxes" command. It would not report sandboxes 146 | created by other applications (MySQL-Sandbox) 147 | - Added check for template version during export/import 148 | - Added tests for UUID generation 149 | - Improved docker test 150 | 0.1.23 19-Feb-2018 151 | - Added "test-replication" to "global" command 152 | - Added several aliases to "unpack" 153 | - Changed template init_db, to allow for easier customization 154 | - Added test for docker. The full test suite can run in a container. 155 | - Simplified test.sh by using "dbdeployer global" rather than hand made 156 | loops. 157 | 0.1.22 18-Feb-2018 158 | - All values used for sandbox deployment are now modifiable. 159 | - Added command "admin" to deal with default values: 160 | show, store, remove, load, update, export 161 | - Refactored global variables to become modifiable through the "admin" 162 | command 163 | - Added commands for template export and import. 164 | 0.1.21 16-Feb-2018 165 | - Added flag --expose-dd-tables to show data dictionary hidden tables 166 | - Added flag --custom-mysqld to use different server executable 167 | 0.1.20 14-Feb-2018 168 | - Added flags for pre and post grants SQL execution. 169 | - --pre-grants-sql-file 170 | - --pre-grants-sql 171 | - --post-grants-sql-file 172 | - --post-grants-sql 173 | - Fixed bug (in cobra+pflag package) that splits multiple commands by comma. 174 | 0.1.19 14-Feb-2018 175 | - MySQL 8.0+ sandboxes now use roles instead of direct privilege 176 | assignments. 177 | - Added global flag --force to overwrite an existing deployment 178 | - Added global flag --my-cnf-file to use a customized my.cnf 179 | - Added flag --master-ip to replication deployments 180 | - Fixed bug in abbreviations: flags were not considered correctly. 181 | 0.1.18 12-Feb-2018 182 | - The "delete" command now supports "ALL" as argument. It will delete all installed sandboxes. 183 | - Added flag "--skip-confirm" for the "delete" command, to delete without confirmation. 184 | - Fixed mistake during initialization: the version search was happening before the check 185 | for the sandbox home directory. 186 | - Added the "global" command to propagate a command to all sandboxes 187 | 0.1.17 11-Feb-2018 188 | - Added automated README generation 189 | - minor code changes 190 | 0.1.16 10-Feb-2018 191 | - Added automatic generation of human-readable server-UUID 192 | - Added flag --keep-server-uuid to prevent the above change 193 | 0.1.15 08-Feb-2018 194 | - Changed default port and sandbox directory for single-primary group 195 | replication. 196 | - Added custom abbreviations feature. 197 | 0.1.14 07-Feb-2018 198 | - Added script test_sb to every single sandbox 199 | - Added script test_sb_all to all multiple/group/replication sandbox 200 | - Added script test_replication to replication sandboxes 201 | - Added test/test.sh, which runs a comprehensive test of most dbdeployer features 202 | 0.1.13 06-Feb-2018 203 | - Added command "templates export" 204 | - Added flag --single-primary for group replication 205 | - Added flags --sandbox-directory, --port, and base-port 206 | to allow deploying several sandboxes of the same version. 207 | - Added a check for clash on installed ports 208 | - INCOMPATIBLE change: Changed format of sbdescription.json: 209 | now can list several ports per sandbox. 210 | 0.1.12 04-Feb-2018 211 | - Added a check for version before applying gtid. 212 | - Added commands templates list/show/describe 213 | - Added --use-template=template_name:file_name flag 214 | 0.1.11 31-Jan-2018 215 | - Improved check for tarball as an argument to single, replication, 216 | multiple. 217 | - Improved help for single, multiple, and replication 218 | - Added customized prompt for configuration file 219 | 0.1.10 30-Jan-2018 220 | - Changed initialization method to use tarball libraries 221 | - Fixed glitch in "unpack" when original tarball has clashing name 222 | 0.1.09 30-Jan-2018 223 | - Updated README.md 224 | - Changed formatting for "usage" command 225 | - Run detection of invalid group replication earlier. 226 | - Added version to sandbox description file 227 | 0.1.08 29-Jan-2018 228 | - Added sandbox description file 229 | - 'sandboxes' command uses above file for sandbox listing 230 | - Added 'delete' command 231 | 0.1.07 29-Jan-2018 232 | - improved documentation 233 | - Added "usage" command 234 | - Added description to "sandboxes" output 235 | - Added check for version format 236 | - Changed message for missing argument 237 | - Added check for sandbox-home existence 238 | 0.1.06 28-Jan-2018 239 | - Added group replication topology. 240 | 0.1.05 27-Jan-2018 241 | - Added option --master to 'single' command 242 | - Added new commands to each sandbox: add_option, show_binlog, 243 | show_relaylog, my. 244 | 0.1.04 26-Jan-2018 245 | - Added short names for some flags. 246 | - Improved commands usage text 247 | 0.1.03 26-Jan-2018 248 | - Modified --my-cnf-options and --init-options to be accepted multiple 249 | times 250 | 0.1.02 25-Jan-2018 251 | - Fixed bug in unpack when basedir was not created. 252 | 0.1.01 25-Jan-2018 253 | - Fixed inclusion of options in my.sandbox.cnf (--my-cnf-options) 254 | - Added command 'multiple' 255 | - Enhanced documentation 256 | 0.1.00 24-Jan-2018 257 | - Initial commit with basic features migrated from MySQL-Sandbox 258 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------