├── .gitignore ├── .travis.yml ├── CHANGES.md ├── Dockerfile ├── LICENSE ├── Makefile ├── README.md ├── command_addpathrule.go ├── command_addpathrule_test.go ├── command_namedport.go ├── command_template.go ├── command_template_test.go ├── commands.go ├── commands_force.go ├── commands_init.go ├── commands_test.go ├── config.go ├── config_test.go ├── examples ├── 010_create_static_ip_for_load_balancer.yaml ├── 015_create_kiya_bucket_for_team.yaml ├── 020_create_cloud_sql_database.yaml ├── 030_add_storage_viewer_role.yaml ├── 035_add_cloud_kms_cryptokey_decrypter_to_cloudbuilder_account.yaml ├── 036_create_kubernetes_cluster_v2.yaml └── 040_create_service_account.yaml ├── export_gs_iam.go ├── export_iam.go ├── export_iam_test.go ├── gcloud.go ├── gcloud_test.go ├── gcs.go ├── go.mod ├── go.sum ├── main.go ├── migration.go ├── migration_context.go ├── migration_test.go ├── state_provider.go ├── test ├── 010_one.yaml ├── 020_two.yaml ├── 030_three.yaml ├── 040_error.yaml ├── 050_conditional.yaml ├── 060_conditional_fail.yaml ├── demo │ ├── gmig.json │ └── nested │ │ └── gmig.yaml ├── gmig.json └── template_test.txt ├── utils.go └── utils_test.go /.gitignore: -------------------------------------------------------------------------------- 1 | /*.yaml 2 | .gmig-last-migration 3 | /gmig.json 4 | .gmig.state 5 | target 6 | gmig -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: go 2 | go: 3 | - 1.19.x 4 | script: 5 | - ls -lr test 6 | - env GO111MODULE=on go build 7 | - env GO111MODULE=on go test -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | # Changelist of gmig releases 2 | 3 | ## v1.18.3 [2025-03-25] 4 | 5 | - update dependencies 6 | 7 | ## v1.18.2 [2024-04-05] 8 | 9 | - update dependencies 10 | - upgrade to 1.18 11 | 12 | ## v1.18.0 [2022-10-15] 13 | 14 | - add `down-all` command do execute `undo` section of all applied migrations 15 | (thanks Vladimir Oganesov 112 | 113 | Prepares your setup for working with migrations by creating a `gmig.json` file in a target folder. 114 | 115 | gmig init my-gcp-production-project 116 | 117 | Then your filesystem will have: 118 | 119 | /my-gcp-production-project/ 120 | gmig.yaml 121 | 122 | You must change the file `gmig.yaml` to set the Project and Bucket name. 123 | 124 | # gmig configuration file 125 | # 126 | # Google Cloud Platform migrations tool for infrastructure-as-code. See https://github.com/emicklei/gmig. 127 | 128 | # [project] must be the Google Cloud Project ID where the infrastructure is created. 129 | # Its value is available as $PROJECT in your migrations. 130 | # 131 | # Required by gmig. 132 | project: my-project 133 | 134 | # [region] must be a valid GCP region. See https://cloud.google.com/compute/docs/regions-zones/ 135 | # A region is a specific geographical location where you can run your resources. 136 | # Its value is available as $REGION in your migrations. 137 | # 138 | # Not required by gmig but some gcloud and gsutil commands do require it. 139 | # region: europe-west1 140 | 141 | # [zone] must be a valid GCP zone. See https://cloud.google.com/compute/docs/regions-zones/ 142 | # Each region has one or more zones; most regions have three or more zones. 143 | # Its value is available as $ZONE in your migrations. 144 | # 145 | # Not required by gmig but some gcloud and gsutil commands do require it. 146 | # zone: europe-west1-b 147 | 148 | # [bucket] must be a valid GCP Storage bucket. 149 | # A Google Storage Bucket is used to store information (object) about the last applied migration. 150 | # Bucket can contain multiple objects from multiple applications. Make sure the [state] is different for each app. 151 | # 152 | # Required by gmig. 153 | bucket: my-bucket 154 | 155 | # [state] is the name of the object that hold information about the last applied migration. 156 | # Required by gmig. 157 | state: myapp-gmig-last-migration 158 | 159 | # [env] are additional environment values that are available to each section of a migration file. 160 | # This can be used to create migrations that are independent of the target project. 161 | # By convention, use capitalized words for keys. 162 | # In the example, "myapp-cluster" is available as $K8S_CLUSTER in your migrations. 163 | # 164 | # Not required by gmig. 165 | env: 166 | K8S_CLUSTER: myapp-cluster 167 | 168 | If you decide to store state files of different projects in one Bucket then set the state object name to reflect this, eg. `myproject-gmig-state`. 169 | If you want to apply the same migrations to different regions/zones then choose a target folder name to reflect this, eg. `my-gcp-production-project-us-east`. Values for `region` and `zone` are required if you want to create Compute Engine resources. The `env` map can be used to parameterize commands in your migrations. In the example, all commands will have access to the value of `$K8S_CLUSTER`. 170 | 171 | ### new \ 172 | 173 | Creates a new migration for you to describe a change to the current state of infrastructure. 174 | 175 | gmig new "add storage view role to cloudbuild account" 176 | 177 | Using a combination of the options `--do`, `--undo` and `--view`, you can set the commands directly for the new migration. 178 | 179 | ### status \<path> [--migrations folder] 180 | 181 | List all migrations with an indicator (applied,pending) whether is has been applied or not. 182 | 183 | gmig status my-gcp-production-project/ 184 | 185 | Run this command in the directory where all migrations are stored. Use `--migrations` for a different location. 186 | 187 | ### plan \<path> [stop] [--migrations folder] 188 | 189 | Log commands of the `do` section of all pending migrations in order, one after the other. 190 | If `stop` is given, then stop after that migration file. 191 | 192 | ### up \<path> [stop] [--migrations folder] 193 | 194 | Executes the `do` section of each pending migration compared to the last applied change to the infrastructure. 195 | If `stop` is given, then stop after that migration file. 196 | Upon each completed migration, the `gmig-last-migration` object is updated in the bucket. 197 | 198 | gmig up my-gcp-production-project 199 | 200 | ### down \<path> [--migrations folder] 201 | 202 | Executes one `undo` section of the last applied change to the infrastructure. 203 | If completed then update the `gmig-last-migration` object. 204 | 205 | gmig down my-gcp-production-project 206 | 207 | ### down-all \<path> [--migrations folder] 208 | 209 | Executes `undo` section of all applied change to the infrastructure. 210 | Updates the `gmig-last-migration` object after each successfull step. 211 | 212 | gmig down-all my-gcp-production-project 213 | 214 | ### view \<path> [migration file] [--migrations folder] 215 | 216 | Executes the `view` section of each applied migration to the infrastructure. 217 | If `migration file` is given then run that view only. 218 | 219 | gmig view my-gcp-production-project 220 | 221 | ### template [-w] source-file 222 | 223 | Processes the source-file as a Go template and write the result to stdout. 224 | If the `-w` is given then rewrite the source with the processed content. 225 | The following functions are available: 226 | 227 | #### env 228 | 229 | This function takes the first argument and does a lookup in the available OS environment values. 230 | Example of a configuration snippet that needs the environment dependent value for $PROJECT. 231 | 232 | project: {{ env "PROJECT" }} 233 | 234 | Example: 235 | 236 | gmig template some-config.template.yaml > some-config.yaml 237 | 238 | ## Export existing infrastructure 239 | 240 | Exporting migrations from existing infrastructure is useful when you start working with `gmig` but do not want to start from scratch. 241 | Several sub commands are (or will become) available to inspect a project and export migrations to reflect the current state. 242 | After marking the current state in `gmig` (using `force-state`), new migrations can be added that will bring your infrastructure to the next state. 243 | The generated migration can ofcourse also be used to just copy commands to your own migration. 244 | 245 | ### export project-iam-policy \<path> 246 | 247 | Generate a new migration by reading all the IAM policy bindings from the current infrastructure of the project. 248 | 249 | gmig -v export project-iam-policy my-project/ 250 | 251 | ### export storage-iam-policy \<path> 252 | 253 | Generate a new migration by reading all the IAM policy bindings, per Google Storage Bucket owned by the project. 254 | 255 | gmig -v export storage-iam-policy my-project/ 256 | 257 | ## Working around migrations 258 | 259 | Sometimes you need to fix things because you made a mistake or want to reorganise your work. Use the `force` and confirm your action. 260 | 261 | ### force state \<path> \<filename> 262 | 263 | Explicitly set the state for the target to the last applied filename. This command can be useful if you need to work from existing infrastructure. Effectively, this filename is written to the bucket object. 264 | Use this command with care!. 265 | 266 | gmig force state my-gcp-production-project 010_create_some_account.yaml 267 | 268 | ### force do \<path> \<filename> 269 | 270 | Explicitly run the commands in the `do` section of a given migration filename. 271 | The `gmig-last-migration` object is `not` updated in the bucket. 272 | Use this command with care!. 273 | 274 | gmig force do my-gcp-production-project 010_create_some_account.yaml 275 | 276 | ### force undo \<path> \<filename> 277 | 278 | Explicitly run the commands in the `undo` section of a given migration filename. 279 | The `gmig-last-migration` object is `not` updated in the bucket. 280 | Use this command with care!. 281 | 282 | gmig force undo my-gcp-production-project 010_create_some_account.yaml 283 | 284 | ## export-env \<path> 285 | 286 | Export all available environment variable from the configuration file and also export $PROJECT, $REGION and $ZONE 287 | Use this command with care!. 288 | 289 | eval $(gmig export-env my-gcp-production-project) 290 | 291 | ## GCP utilities 292 | 293 | ### util create-named-port \<instance-group> \<name:port> 294 | 295 | The Cloud SDK has a command to [set-named-ports](https://cloud.google.com/sdk/gcloud/reference/compute/instance-groups/set-named-ports) but not a command to add or delete a single name:port mapping. To simplify the migration command for creating a name:port mapping, this gmig util command is added. 296 | First it calls `get-named-ports` to retrieve all existing mappings. Then it will call `set-named-ports` with the new mapping unless it already exists. 297 | 298 | ### util delete-named-port \<instance-group> \<name:port> 299 | 300 | The Cloud SDK has a command to [set-named-ports](https://cloud.google.com/sdk/gcloud/reference/compute/instance-groups/set-named-ports) but not a command to add or delete a single name:port mapping. To simplify the migration command for deleting a name:port mapping, this `gmig` util command is added. 301 | First it calls `get-named-ports` to retrieve all existing mappings. Then it will call `set-named-ports` without the mapping. 302 | 303 | ### util add-path-rules-to-path-matcher [config folder] -url-map [url-map-name] -service [backend-service-name] -path-matcher [path-matcher-name] -paths "/v1/path/_, /v1/otherpath/_" 304 | 305 | The Cloud SDK has a command to [add a patch matcher](https://cloud.google.com/sdk/gcloud/reference/compute/url-maps/add-path-matcher) with a set of paths but not a command update the path rules of an existing path matcher in the url map. To write a migration that changes the set of paths (add,remove), this `gmig` util command is added. 306 | First is [exports](https://cloud.google.com/sdk/gcloud/reference/compute/url-maps/export) an URL map, updates the paths of the rules of a path-matcher, then imports the changed URL map. Because this migration is changing a regional resource which is typically shared by multiple services, the patching of the URL map is executed using a global lock (using the Bucket from the config). 307 | 308 | ## Examples 309 | 310 | This repository has a number of [examples](https://github.com/emicklei/gmig/tree/master/examples) of migrations. 311 | 312 | © 2022, ernestmicklei.com. MIT License. Contributions welcome. 313 | -------------------------------------------------------------------------------- /command_addpathrule.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "context" 5 | "fmt" 6 | "log" 7 | "os" 8 | "os/exec" 9 | "path/filepath" 10 | "strings" 11 | "time" 12 | 13 | "github.com/marcacohen/gcslock" 14 | "github.com/urfave/cli" 15 | "gopkg.in/yaml.v2" 16 | ) 17 | 18 | type ( 19 | loadbalancerURLMap struct { 20 | DefaultService string `yaml:"defaultService"` 21 | Description string `yaml:"description"` 22 | HostRules []struct { 23 | Hosts []string `yaml:"hosts"` 24 | PathMatcher string `yaml:"pathMatcher"` 25 | } `yaml:"hostRules"` 26 | Kind string `yaml:"kind"` 27 | Name string `yaml:"name"` 28 | PathMatchers []pathMatcher `yaml:"pathMatchers"` 29 | Region string `yaml:"region"` 30 | SelfLink string `yaml:"selfLink"` 31 | } 32 | pathMatcher struct { 33 | DefaultService string `yaml:"defaultService"` 34 | Description string `yaml:"description"` 35 | Name string `yaml:"name"` 36 | PathRules []pathsAndService `yaml:"pathRules"` 37 | } 38 | pathsAndService struct { 39 | Paths []string `yaml:"paths"` 40 | Service string `yaml:"service"` 41 | } 42 | ) 43 | 44 | // findIndices returns (matcherIndex, ruleIndex ) 45 | func (m *loadbalancerURLMap) findIndices(pathMatcherName, fqnService string, verbose bool) (int, int) { 46 | ruleIndex := -1 47 | matcherIndex := -1 48 | for m, each := range m.PathMatchers { 49 | if each.Name == pathMatcherName { 50 | if verbose { 51 | log.Println("found existing path matcher:", pathMatcherName) 52 | } 53 | matcherIndex = m 54 | } 55 | for i, other := range each.PathRules { 56 | if other.Service == fqnService { 57 | if verbose { 58 | log.Println("found existing path rule:", fqnService) 59 | } 60 | ruleIndex = i 61 | } 62 | } 63 | } 64 | return matcherIndex, ruleIndex 65 | } 66 | 67 | func (m *loadbalancerURLMap) patchPathsAndService(isRemove bool, pathMatcherName, fqnService string, paths []string, verbose bool) error { 68 | // check if exists based on service 69 | matcherIndex, ruleIndex := m.findIndices(pathMatcherName, fqnService, verbose) 70 | if matcherIndex == -1 { 71 | err := fmt.Errorf("no path-matcher found with name [%s]", pathMatcherName) 72 | printError(err.Error()) 73 | return errAbort 74 | } 75 | if isRemove { 76 | // Delete 77 | rules := m.PathMatchers[matcherIndex].PathRules 78 | copy(rules[ruleIndex:], rules[ruleIndex+1:]) 79 | rules[len(rules)-1] = pathsAndService{} 80 | rules = rules[:len(rules)-1] 81 | m.PathMatchers[matcherIndex].PathRules = rules 82 | } else { 83 | // Update 84 | toPatch := pathsAndService{Service: fqnService, Paths: paths} 85 | if ruleIndex == -1 { 86 | // add new path rule set 87 | rules := m.PathMatchers[matcherIndex].PathRules 88 | m.PathMatchers[matcherIndex].PathRules = append(rules, toPatch) 89 | } else { 90 | // replace existing path rule set 91 | m.PathMatchers[matcherIndex].PathRules[ruleIndex] = toPatch 92 | } 93 | } 94 | return nil 95 | } 96 | 97 | func cmdAddPathRulesToPathMatcher(c *cli.Context) error { 98 | return patchPathRulesForPathMatcher(c, false) 99 | } 100 | 101 | func cmdRemovePathRulesFromPathMatcher(c *cli.Context) error { 102 | return patchPathRulesForPathMatcher(c, true) 103 | } 104 | 105 | func patchPathRulesForPathMatcher(c *cli.Context, isRemove bool) error { 106 | // prepare 107 | mtx, err := getMigrationContext(c) 108 | if err != nil { 109 | printError(err.Error()) 110 | return errAbort 111 | } 112 | verbose := c.GlobalBool("v") 113 | urlMapName := c.String("url-map") 114 | scope := c.String("region") // optional 115 | if len(scope) == 0 { 116 | scope = "global" 117 | } 118 | // aquire lock 119 | lockObjectName := fmt.Sprintf("project-%s-region-%s-urlmap-%s-gmig-lock", mtx.config().Project, scope, urlMapName) 120 | urlMapMutex, err := gcslock.New(context.Background(), mtx.config().Bucket, lockObjectName) 121 | if err != nil { 122 | printError(err.Error()) 123 | return errAbort 124 | } 125 | if verbose { 126 | log.Println("acquire global lock on:", lockObjectName, " in bucket:", mtx.config().Bucket) 127 | } 128 | ctx, cancel := context.WithTimeout(context.Background(), 1*time.Minute) // wait for at most 1 minute 129 | defer cancel() 130 | if err := urlMapMutex.ContextLock(ctx); err != nil { 131 | printError(err.Error()) 132 | return errAbort 133 | } 134 | // release lock on return 135 | defer func() { 136 | if verbose { 137 | log.Println("release global lock on:", lockObjectName, " in bucket:", mtx.config().Bucket) 138 | } 139 | if err := urlMapMutex.ContextUnlock(ctx); err != nil { 140 | printError(err.Error()) 141 | } 142 | }() 143 | // export 144 | args := []string{"compute", "url-maps", "export", urlMapName} 145 | region := c.String("region") // optional 146 | isGlobal := len(region) == 0 147 | if !isGlobal { 148 | args = append(args, "--region", region) 149 | } else { 150 | args = append(args, "--global") 151 | } 152 | cmd := exec.Command("gcloud", args...) 153 | if verbose { 154 | log.Println(strings.Join(append([]string{"gcloud"}, args...), " ")) 155 | } 156 | data, err := runCommand(cmd) 157 | if err != nil { 158 | log.Println(string(data)) // stderr 159 | return fmt.Errorf("failed to run gcloud command: %v", err) 160 | } 161 | // unmarshal 162 | urlMap := new(loadbalancerURLMap) 163 | if err := yaml.Unmarshal(data, urlMap); err != nil { 164 | printError(err.Error()) 165 | return errAbort 166 | } 167 | serviceName := c.String("service") 168 | var fqnService string 169 | if isGlobal { 170 | fqnService = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/global/backendServices/%s", 171 | mtx.config().Project, 172 | serviceName) 173 | } else { 174 | fqnService = fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s/backendServices/%s", 175 | mtx.config().Project, 176 | region, 177 | serviceName) 178 | } 179 | if err := urlMap.patchPathsAndService( 180 | isRemove, 181 | c.String("path-matcher"), fqnService, strings.Split(strings.ReplaceAll(c.String("paths"), " ", ""), ","), 182 | verbose); err != nil { 183 | return err 184 | } 185 | // can only import from source file 186 | importdata, err := yaml.Marshal(urlMap) 187 | if err != nil { 188 | printError(err.Error()) 189 | return errAbort 190 | } 191 | source := filepath.Join(os.TempDir(), "patchPathRulesForPathMatcher.yaml") 192 | err = os.WriteFile(source, importdata, os.ModePerm) 193 | if err != nil { 194 | printError(err.Error()) 195 | return errAbort 196 | } 197 | defer os.Remove(source) 198 | if verbose { 199 | fmt.Println(string(importdata)) 200 | } 201 | // import 202 | { 203 | args := []string{"compute", "url-maps", "import", urlMapName, "--source", source, "--quiet"} 204 | if !isGlobal { 205 | args = append(args, "--region", region) 206 | } else { 207 | args = append(args, "--global") 208 | } 209 | cmd := exec.Command("gcloud", args...) 210 | if verbose { 211 | log.Println(strings.Join(append([]string{"gcloud"}, args...), " ")) 212 | } 213 | data, err := runCommand(cmd) 214 | if err != nil { 215 | log.Println(string(data)) // stderr 216 | return fmt.Errorf("failed to run gcloud command: %v", err) 217 | } 218 | } 219 | return nil 220 | } 221 | -------------------------------------------------------------------------------- /command_addpathrule_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func Test_loadbalancerURLMap_patchPathsAndService(t *testing.T) { 6 | m := loadbalancerURLMap{ 7 | PathMatchers: []pathMatcher{ 8 | { 9 | Name: "a", 10 | PathRules: []pathsAndService{{ 11 | Paths: []string{"/a"}, 12 | Service: "theA", 13 | }}, 14 | }, 15 | }, 16 | } 17 | err := m.patchPathsAndService(true, "a", "theA", []string{}, false) 18 | if err != nil { 19 | t.Fatal(err) 20 | } 21 | if got, want := len(m.PathMatchers[0].PathRules), 0; got != want { 22 | t.Errorf("got [%v] want [%v]", got, want) 23 | } 24 | { 25 | err := m.patchPathsAndService(false, "a", "theB", []string{"/b"}, false) 26 | if err != nil { 27 | t.Fatal(err) 28 | } 29 | if got, want := len(m.PathMatchers[0].PathRules), 1; got != want { 30 | t.Log(m) 31 | t.Errorf("got [%v] want [%v]", got, want) 32 | } 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /command_namedport.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "os/exec" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/urfave/cli" 13 | ) 14 | 15 | type namedPort struct { 16 | Name string `json:"name"` 17 | Port int `json:"port"` 18 | } 19 | 20 | const ( 21 | create = 1 22 | delete = 2 23 | ) 24 | 25 | func cmdCreateNamedPort(c *cli.Context) error { 26 | return cmdChangeNamedPort(c, create) 27 | } 28 | 29 | func cmdDeleteNamedPort(c *cli.Context) error { 30 | return cmdChangeNamedPort(c, delete) 31 | } 32 | 33 | func cmdChangeNamedPort(c *cli.Context, action int) error { 34 | if len(c.Args()) != 2 { 35 | return fmt.Errorf("missing command argument, expected [INSTANCE_GROUP] NAME:PORT") 36 | } 37 | verbose := c.GlobalBool("v") 38 | instanceGroup := c.Args()[0] 39 | if len(instanceGroup) == 0 { 40 | return fmt.Errorf("missing Compute Instance Group command argument") 41 | } 42 | parts := strings.Split(c.Args()[1], ":") 43 | if len(parts) == 0 { 44 | return fmt.Errorf("invalid NAME:PORT command argument") 45 | } 46 | port, err := strconv.Atoi(parts[1]) 47 | if err != nil { 48 | return fmt.Errorf("PORT must be a positive integer:%v", err) 49 | } 50 | name := parts[0] 51 | // get named ports 52 | args := []string{"compute", "instance-groups", "get-named-ports", instanceGroup, "--format", "json"} 53 | cmd := exec.Command("gcloud", args...) 54 | if verbose { 55 | log.Println(strings.Join(append([]string{"gcloud"}, args...), " ")) 56 | } 57 | data, err := runCommand(cmd) 58 | if err != nil { 59 | log.Println(string(data)) // stderr 60 | return fmt.Errorf("failed to run gcloud get-named-ports: %v", err) 61 | } 62 | // parse named ports 63 | var list []namedPort 64 | if err := json.NewDecoder(bytes.NewReader(data)).Decode(&list); err != nil { 65 | return fmt.Errorf("parsing JSON output failed:%v", err) 66 | } 67 | if verbose { 68 | log.Println("current list of named ports", list) 69 | } 70 | if create == action { 71 | log.Printf("ensure named port exists %s:%d\n", name, port) 72 | // only append if not exists, update otherwise or abort 73 | updated := false 74 | for _, each := range list { 75 | if each.Name == name { 76 | if each.Port == port { 77 | log.Printf("named-port [%s:%d] already exists\n", name, port) 78 | return nil 79 | } 80 | log.Printf("change named port from [%s:%d] to [%s:%d]\n", each.Name, each.Port, each.Name, port) 81 | each.Port = port 82 | updated = true 83 | break 84 | } 85 | } 86 | if !updated { 87 | list = append(list, namedPort{Name: name, Port: port}) 88 | } 89 | } 90 | if delete == action { 91 | log.Printf("ensure named port no longer exists %s:%d\n", name, port) 92 | // only delete if exists, update otherwise 93 | deleted := false 94 | copyWithout := []namedPort{} 95 | for _, each := range list { 96 | if each.Name == name { 97 | if verbose { 98 | log.Printf("deleted named port %s=%d\n", each.Name, each.Port) 99 | } 100 | deleted = true 101 | } else { 102 | copyWithout = append(copyWithout, each) 103 | } 104 | } 105 | if !deleted { 106 | log.Printf("named-port [%s:%d] did not exists\n", name, port) 107 | return nil 108 | } 109 | list = copyWithout 110 | } 111 | // composed set argument 112 | newList := new(strings.Builder) 113 | for i, each := range list { 114 | if i > 0 { 115 | newList.WriteString(",") 116 | } 117 | fmt.Fprintf(newList, "%s:%d", each.Name, each.Port) 118 | } 119 | // set named ports using new list 120 | args = []string{"compute", "instance-groups", "set-named-ports", instanceGroup, "--named-ports", newList.String()} 121 | cmd = exec.Command("gcloud", args...) 122 | log.Println(strings.Join(append([]string{"gcloud"}, args...), " ")) 123 | if data, err := runCommand(cmd); err != nil { 124 | log.Println(string(data)) // stderr 125 | return fmt.Errorf("failed to run gcloud set-named-ports: %v", err) 126 | } 127 | return nil 128 | } 129 | -------------------------------------------------------------------------------- /command_template.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "os" 8 | "text/template" 9 | 10 | "github.com/urfave/cli" 11 | ) 12 | 13 | func cmdTemplate(c *cli.Context) error { 14 | source := c.Args().First() 15 | isRewrite := c.Bool("w") 16 | data, err := os.ReadFile(source) 17 | if err != nil { 18 | printError(err.Error()) 19 | return errAbort 20 | } 21 | funcs := template.FuncMap{"env": os.Getenv} 22 | tmpl, err := template.New("tmpl").Funcs(funcs).Parse(string(data)) 23 | if err != nil { 24 | printError(err.Error()) 25 | return errAbort 26 | } 27 | output := new(bytes.Buffer) 28 | err = tmpl.Execute(output, "") 29 | if err != nil { 30 | printError(err.Error()) 31 | return errAbort 32 | } 33 | if isRewrite { 34 | return ioutil.WriteFile(source, output.Bytes(), os.FileMode(0644)) // -rw-r--r--, see http://permissions-calculator.org/ 35 | } 36 | fmt.Fprintln(os.Stdout, output) 37 | return nil 38 | } 39 | -------------------------------------------------------------------------------- /command_template_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import "testing" 4 | 5 | func TestCmdTemplate(t *testing.T) { 6 | if err := newApp().Run([]string{"gmig", "template", "./test/template_test.txt"}); err != nil { 7 | t.Fatal("unexpected error", err) 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /commands.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "fmt" 6 | "log" 7 | "os" 8 | "path/filepath" 9 | "strconv" 10 | "strings" 11 | 12 | "github.com/urfave/cli" 13 | ) 14 | 15 | // space is right after timestamp 16 | const ( 17 | statusSeparator = "~-------------- ------------------~" 18 | viewSeparatorTop = "~---------------------------------------------------------------~" 19 | viewSeparatorBottom = " --------------------------------------------------------------- " 20 | applied = "--- applied ---" 21 | pending = "... pending ..." 22 | execDo = "... do ..." 23 | execUndo = "... undo ..." 24 | execPlan = "... plan ..." 25 | stopped = "... stopped ..." 26 | skipped = "--- skipped ---" 27 | skipping = "... skipping .." 28 | conditionErrored = "--- if error --" 29 | conditionError = "... if error .." 30 | ) 31 | 32 | func cmdCreateMigration(c *cli.Context) error { 33 | 34 | desc := c.Args().First() 35 | if len(desc) == 0 { 36 | printError("missing migration title") 37 | return errAbort 38 | } 39 | filename := NewFilenameWithIndex(desc) 40 | defaultCommands := []string{"gcloud config list"} 41 | doSection, undoSection, viewSection := defaultCommands, defaultCommands, []string{} 42 | if doValue := c.String("do"); len(doValue) > 0 { 43 | doSection = strings.Split(doValue, "\n") 44 | } 45 | if undoValue := c.String("undo"); len(undoValue) > 0 { 46 | undoSection = strings.Split(undoValue, "\n") 47 | } 48 | if viewValue := c.String("view"); len(viewValue) > 0 { 49 | viewSection = strings.Split(viewValue, "\n") 50 | } 51 | m := Migration{ 52 | Description: desc, 53 | Filename: filename, 54 | DoSection: doSection, 55 | UndoSection: undoSection, 56 | ViewSection: viewSection, 57 | } 58 | yaml, err := m.ToYAML() 59 | if err != nil { 60 | printError("YAML creation failed") 61 | return errAbort 62 | } 63 | return os.WriteFile(filename, []byte(yaml), os.FileMode(0644)) // -rw-r--r--, see http://permissions-calculator.org/ 64 | } 65 | 66 | func cmdMigrationsUp(c *cli.Context) error { 67 | return runMigrations(c, !true) 68 | } 69 | 70 | func cmdMigrationsPlan(c *cli.Context) error { 71 | return runMigrations(c, true) 72 | } 73 | 74 | func runMigrations(c *cli.Context, isLogOnly bool) error { 75 | mtx, err := getMigrationContext(c) 76 | if err != nil { 77 | printError(err.Error()) 78 | return errAbort 79 | } 80 | stopAfter := c.Args().Get(1) // empty if not specified 81 | all, err := LoadMigrationsBetweenAnd(mtx.migrationsPath, mtx.lastApplied, stopAfter) 82 | if err != nil { 83 | printError(err.Error()) 84 | return errAbort 85 | } 86 | // if stopAfter is specified then it must be one of all 87 | found := false 88 | for _, each := range all { 89 | if stopAfter == each.Filename { 90 | found = true 91 | break 92 | } 93 | } 94 | envs := mtx.shellEnv() 95 | // if lastApplied is after stopAfter then it is also not found but then we don't care 96 | if !found && stopAfter > mtx.lastApplied { 97 | reportError(mtx.stateProvider.Config(), envs, "up until stop", errors.New("No such migration file: "+stopAfter)) 98 | return errAbort 99 | } 100 | prettyWidth := largestWidthOf(all) 101 | for _, each := range all { 102 | log.Println(statusSeparator) 103 | leadingTitle := execDo 104 | if isLogOnly { 105 | leadingTitle = execPlan 106 | } 107 | log.Printf("%s %-"+strconv.Itoa(prettyWidth)+"s (%s)\n", leadingTitle, pretty(each.Filename), each.Filename) 108 | if isLogOnly { 109 | log.Println("") 110 | if LogAll(each.IfExpression, each.DoSection, envs, true) != nil { 111 | reportError(mtx.stateProvider.Config(), envs, "plan do", err) 112 | return errAbort 113 | } 114 | } else { 115 | if err := ExecuteAll(each.IfExpression, each.DoSection, envs, c.GlobalBool("v")); err != nil { 116 | reportError(mtx.stateProvider.Config(), envs, "do", err) 117 | return errAbort 118 | } 119 | mtx.lastApplied = each.Filename 120 | // save after each succesful migration 121 | if err := mtx.stateProvider.SaveState(mtx.lastApplied); err != nil { 122 | reportError(mtx.stateProvider.Config(), envs, "save state", err) 123 | return errAbort 124 | } 125 | } 126 | // if not empty then stop after applying this migration 127 | if stopAfter == each.Filename { 128 | log.Println(stopped) 129 | log.Println(statusSeparator) 130 | break 131 | } 132 | } 133 | return nil 134 | } 135 | 136 | func cmdMigrationsDownAll(c *cli.Context) error { 137 | mtx, err := getMigrationContext(c) 138 | if err != nil { 139 | return err 140 | } 141 | if mtx.lastApplied == "" { 142 | printWarning("There are no migrations to undo") 143 | return errAbort 144 | } 145 | //get all applied migrations 146 | all, err := LoadMigrationsBetweenAnd(".", "", mtx.lastApplied) 147 | if err != nil { 148 | printError(err.Error()) 149 | return errAbort 150 | } 151 | for range all { 152 | err := cmdMigrationsDown(c) 153 | if err != nil { 154 | return err 155 | } 156 | } 157 | return nil 158 | } 159 | 160 | func cmdMigrationsDown(c *cli.Context) error { 161 | mtx, err := getMigrationContext(c) 162 | if err != nil { 163 | printError(err.Error()) 164 | return errAbort 165 | } 166 | if mtx.lastApplied == "" { 167 | printWarning("There are no migrations to undo") 168 | return errAbort 169 | } 170 | all, err := LoadMigrationsBetweenAnd(mtx.migrationsPath, "", mtx.lastApplied) 171 | if err != nil { 172 | printError(err.Error()) 173 | return errAbort 174 | } 175 | lastMigration := all[len(all)-1] 176 | log.Println(statusSeparator) 177 | log.Println(execUndo, pretty(mtx.lastApplied)) 178 | log.Println(statusSeparator) 179 | envs := mtx.shellEnv() 180 | if err := ExecuteAll(lastMigration.IfExpression, lastMigration.UndoSection, envs, c.GlobalBool("v")); err != nil { 181 | reportError(mtx.stateProvider.Config(), envs, "undo", err) 182 | return errAbort 183 | } 184 | // save after succesful migration 185 | previousFilename := "" 186 | if len(all) > 1 { 187 | previousFilename = all[len(all)-2].Filename 188 | } 189 | if err := mtx.stateProvider.SaveState(previousFilename); err != nil { 190 | reportError(mtx.stateProvider.Config(), envs, "save state", err) 191 | return errAbort 192 | } 193 | return nil 194 | } 195 | 196 | func largestWidthOf(list []Migration) int { 197 | prettyWidth := 0 198 | for _, each := range list { 199 | pf := pretty(each.Filename) 200 | if len(pf) > prettyWidth { 201 | prettyWidth = len(pf) 202 | } 203 | } 204 | return prettyWidth 205 | } 206 | 207 | func cmdMigrationsStatus(c *cli.Context) error { 208 | mtx, err := getMigrationContext(c) 209 | if err != nil { 210 | printError(err.Error()) 211 | return errAbort 212 | } 213 | all, err := LoadMigrationsBetweenAnd(mtx.migrationsPath, "", "") 214 | if err != nil { 215 | printError(err.Error()) 216 | return errAbort 217 | } 218 | log.Println(statusSeparator) 219 | envs := mtx.shellEnv() 220 | for i, each := range all { 221 | var status string 222 | // check skipped 223 | pass, err := evaluateCondition(each.IfExpression, envs) 224 | isPending := each.Filename > mtx.lastApplied 225 | if err != nil { 226 | if isPending { 227 | status = conditionError 228 | } else { 229 | status = conditionErrored 230 | } 231 | } else { 232 | // no error condition 233 | if pass { 234 | if isPending { 235 | status = pending 236 | } else { 237 | status = applied 238 | } 239 | } else { 240 | if isPending { 241 | status = skipping 242 | } else { 243 | status = skipped 244 | } 245 | } 246 | } 247 | if err != nil { 248 | printWarning("if: expression is invalid:", err) 249 | } 250 | if i > 0 && isPending { 251 | log.Println(statusSeparator) 252 | } 253 | log.Printf("%s %s\n", status, pretty(each.Filename)) 254 | } 255 | log.Println(statusSeparator) 256 | return nil 257 | } 258 | 259 | func cmdView(c *cli.Context) error { 260 | mtx, err := getMigrationContext(c) 261 | if err != nil { 262 | printError(err.Error()) 263 | return errAbort 264 | } 265 | var all []Migration 266 | if len(c.Args()) == 2 { 267 | localMigrationFilename := filepath.Base(c.Args().Get(1)) 268 | if len(localMigrationFilename) > 0 { 269 | one, err := LoadMigration(filepath.Join(mtx.migrationsPath, localMigrationFilename)) 270 | if err != nil { 271 | printError(err.Error()) 272 | return errAbort 273 | } 274 | all = append(all, one) 275 | } 276 | } else { 277 | all, err = LoadMigrationsBetweenAnd(mtx.migrationsPath, "", "") 278 | if err != nil { 279 | printError(err.Error()) 280 | return errAbort 281 | } 282 | } 283 | for _, each := range all { 284 | log.Println(viewSeparatorTop) 285 | log.Printf(" %s (%s)\n", pretty(each.Filename), each.Filename) 286 | log.Println(viewSeparatorBottom) 287 | if each.Filename > mtx.lastApplied { 288 | log.Println(" ** this migration is pending...") 289 | break 290 | } 291 | if len(each.ViewSection) == 0 { 292 | log.Println(" ** this migration has no commands to describe its change on infrastructure.") 293 | } 294 | if mtx.config().verbose { 295 | log.Printf("executing view section (%d commands)\n", len(each.ViewSection)) 296 | } 297 | if err := ExecuteAll(each.IfExpression, each.ViewSection, mtx.shellEnv(), c.GlobalBool("v")); err != nil { 298 | printError(err.Error()) 299 | return errAbort 300 | } 301 | } 302 | return nil 303 | } 304 | 305 | func cmdExportProjectIAMPolicy(c *cli.Context) error { 306 | mtx, err := getMigrationContext(c) 307 | if err != nil { 308 | printError(err.Error()) 309 | return errAbort 310 | } 311 | if err := ExportProjectsIAMPolicy(mtx.stateProvider.Config()); err != nil { 312 | printError(err.Error()) 313 | return errAbort 314 | } 315 | return nil 316 | } 317 | 318 | func cmdExportStorageIAMPolicy(c *cli.Context) error { 319 | mtx, err := getMigrationContext(c) 320 | if err != nil { 321 | printError(err.Error()) 322 | return errAbort 323 | } 324 | if err := ExportStorageIAMPolicy(mtx.stateProvider.Config()); err != nil { 325 | printError(err.Error()) 326 | return errAbort 327 | } 328 | return nil 329 | } 330 | 331 | func cmdExportEnv(c *cli.Context) error { 332 | pathToConfig := c.Args().First() 333 | config, err := TryToLoadConfig(pathToConfig) 334 | if err != nil { 335 | printError(err.Error()) 336 | return errAbort 337 | } 338 | 339 | tmpl := "export %s=%s\n" 340 | fmt.Printf(tmpl, "PROJECT", config.Project) 341 | fmt.Printf(tmpl, "REGION", config.Region) 342 | fmt.Printf(tmpl, "ZONE", config.Zone) 343 | for key, value := range config.EnvironmentVars { 344 | fmt.Printf(tmpl, key, value) 345 | } 346 | return nil 347 | } 348 | -------------------------------------------------------------------------------- /commands_force.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | 7 | "github.com/urfave/cli" 8 | ) 9 | 10 | func cmdMigrationsSetState(c *cli.Context) error { 11 | mtx, err := getMigrationContext(c) 12 | if err != nil { 13 | printWarning(err.Error()) 14 | return nil 15 | } 16 | filename := c.Args().Get(1) // 0=path, 1=relative filename 17 | if !c.GlobalBool("q") { // be quiet 18 | if !promptForYes(fmt.Sprintf("Are you sure to overwrite the current last applied migration [%s -> %s] (y/N)? ", mtx.lastApplied, filename)) { 19 | return errAbort 20 | } 21 | } 22 | if err := checkExists(filepath.Join(mtx.migrationsPath, filename)); err != nil { 23 | printError(err.Error()) 24 | return errAbort 25 | } 26 | if err := mtx.stateProvider.SaveState(filename); err != nil { 27 | printError(err.Error()) 28 | return errAbort 29 | } 30 | return nil 31 | } 32 | 33 | func cmdRundoOnly(c *cli.Context) error { 34 | return runSectionOnly(c, true) 35 | } 36 | 37 | func cmdRunUndoOnly(c *cli.Context) error { 38 | return runSectionOnly(c, false) 39 | } 40 | 41 | func runSectionOnly(c *cli.Context, isDo bool) error { 42 | mtx, err := getMigrationContext(c) 43 | if err != nil { 44 | printWarning(err.Error()) 45 | return nil 46 | } 47 | section := "do" 48 | if !isDo { 49 | section = "undo" 50 | } 51 | filename := c.Args().Get(1) // 0=path, 1=relative filename 52 | if !c.GlobalBool("q") { // be quiet 53 | if !promptForYes(fmt.Sprintf("Are you sure to run the [%s] section of migration [%s] for config [%s] (y/N)? ", section, filename, c.Args().Get(0))) { 54 | return errAbort 55 | } 56 | } 57 | full := filepath.Join(mtx.migrationsPath, filename) 58 | if err := checkExists(full); err != nil { 59 | printError(err.Error()) 60 | return errAbort 61 | } 62 | m, err := LoadMigration(full) 63 | if err != nil { 64 | printError(err.Error()) 65 | return errAbort 66 | } 67 | lines := m.DoSection 68 | if !isDo { 69 | lines = m.UndoSection 70 | } 71 | envs := mtx.shellEnv() 72 | if err := ExecuteAll(m.IfExpression, lines, envs, c.GlobalBool("v")); err != nil { 73 | reportError(mtx.stateProvider.Config(), envs, section, err) 74 | return errAbort 75 | } 76 | return nil 77 | } 78 | -------------------------------------------------------------------------------- /commands_init.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "path/filepath" 7 | 8 | "github.com/urfave/cli" 9 | ) 10 | 11 | var initialYAMLConfig = ` 12 | # gmig configuration file 13 | # 14 | # Google Cloud Platform migrations tool for infrastructure-as-code. See https://github.com/emicklei/gmig . 15 | 16 | # [project] must be the Google Cloud Project ID where the infrastructure is created. 17 | # Its value is available as $PROJECT in your migrations. 18 | # 19 | # Required by gmig. 20 | project: my-project 21 | 22 | # [region] must be a valid GCP region. See https://cloud.google.com/compute/docs/regions-zones/ 23 | # A region is a specific geographical location where you can run your resources. 24 | # Its value is available as $REGION in your migrations. 25 | # 26 | # Not required by gmig but some gcloud and gsutil commands do require it. 27 | # region: europe-west1 28 | 29 | # [zone] must be a valid GCP zone. See https://cloud.google.com/compute/docs/regions-zones/ 30 | # Each region has one or more zones; most regions have three or more zones. 31 | # Its value is available as $ZONE in your migrations. 32 | # 33 | # Not required by gmig but some gcloud and gsutil commands do require it. 34 | # zone: europe-west1-b 35 | 36 | # [bucket] must be a valid GPC bucket. 37 | # A Google Storage Bucket is used to store information (object) about the last applied migration. 38 | # Bucket can contain multiple objects from multiple applications. Make sure the [state] is different for each app. 39 | # 40 | # Required by gmig. 41 | bucket: my-bucket 42 | 43 | # [state] is the name of the object that hold information about the last applied migration. 44 | # 45 | # Required by gmig. 46 | state: myapp-gmig-last-migration 47 | 48 | # [env] are additional environment values that are available to each section of a migration file. 49 | # This can be used to create migrations that are independent of the target project. 50 | # By convention, use capitalized words for keys. 51 | # In the example, "myapp-cluster" is available as $K8S_CLUSTER in your migrations. 52 | # 53 | # Not required by gmig. 54 | #env: 55 | # K8S_CLUSTER: myapp-cluster 56 | ` 57 | 58 | func cmdInit(c *cli.Context) error { 59 | target := c.Args().First() 60 | if len(target) == 0 { 61 | printError("missing target name in command line") 62 | return errAbort 63 | } 64 | if err := os.MkdirAll(target, os.ModePerm|os.ModeDir); err != nil { 65 | printError(err.Error()) 66 | return errAbort 67 | } 68 | config, err := TryToLoadConfig(target) 69 | if config != nil && err == nil { 70 | log.Println("config file [", config.filename, "] already present.") 71 | // TODO move to Config 72 | log.Println("config [ bucket=", config.Bucket, ",state=", config.LastMigrationObjectName, ",verbose=", config.verbose, "]") 73 | return nil 74 | } else if config != nil && err != nil { 75 | printError(err.Error()) 76 | return errAbort 77 | } 78 | location := filepath.Join(target, YAMLConfigFilename) 79 | err = os.WriteFile(location, []byte(initialYAMLConfig), os.ModePerm) 80 | if err != nil { 81 | printError(err.Error()) 82 | return errAbort 83 | } 84 | return nil 85 | } 86 | -------------------------------------------------------------------------------- /commands_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "errors" 5 | "os" 6 | "path/filepath" 7 | "testing" 8 | ) 9 | 10 | func TestCmdInit(t *testing.T) { 11 | defer os.RemoveAll("test/new") 12 | if err := newApp().Run([]string{"gmig", "init", "test/new"}); err != nil { 13 | t.Fatal("unexpected error", err) 14 | } 15 | } 16 | 17 | func TestCmdInitMissingConfig(t *testing.T) { 18 | defer os.RemoveAll("test/missing") 19 | if err := newApp().Run([]string{"gmig", "init", "test/missing"}); err != nil { 20 | t.Fatal("unexpected error", err) 21 | } 22 | if _, err := os.Stat("test/missing"); err != nil { 23 | t.Fatal("unexpected error", err) 24 | } 25 | if _, err := os.Stat("test/missing/gmig.yaml"); err != nil { 26 | t.Fatal("unexpected error", err) 27 | } 28 | if err := newApp().Run([]string{"gmig", "init", "test/missing"}); err != nil { 29 | t.Fatal("unexpected error", err) 30 | } 31 | } 32 | 33 | func TestCmdStatusDemo(t *testing.T) { 34 | osTempDir = func() string { return "." } 35 | // simulate effect of GS download state 36 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 37 | t.Fatal("unable to write state", err) 38 | } 39 | defer os.Remove("state") 40 | 41 | // capture GC command 42 | cc := new(commandCapturer) 43 | runCommand = cc.runCommand 44 | if err := newApp().Run([]string{"gmig", "-v", "status", "test/demo"}); err != nil { 45 | t.Fatal("unexpected error", err) 46 | } 47 | if got, want := len(cc.args), 2; got != want { 48 | t.Logf("got [%v] want [%v]", got, want) 49 | } 50 | for i, each := range []string{"gcloud", "config", "set", "core/project", "demo"} { 51 | if got, want := cc.args[0][i], each; got != want { 52 | t.Logf("got [%v] want [%v]", got, want) 53 | } 54 | } 55 | for i, each := range []string{"gsutil", "-q", "cp", "gs://bucket/state", "state"} { 56 | if got, want := cc.args[1][i], each; got != want { 57 | t.Logf("got [%v] want [%v]", got, want) 58 | } 59 | } 60 | } 61 | 62 | func TestCmdStatusDemoWithMigrationsOption(t *testing.T) { 63 | osTempDir = func() string { return "." } 64 | // simulate effect of GS download state 65 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 66 | t.Fatal("unable to write state", err) 67 | } 68 | defer os.Remove("state") 69 | 70 | // capture GC command 71 | cc := new(commandCapturer) 72 | runCommand = cc.runCommand 73 | if err := newApp().Run([]string{"gmig", "-v", "status", "test/demo", "--migrations", "test"}); err != nil { 74 | t.Fatal("unexpected error", err) 75 | } 76 | } 77 | 78 | func TestCmdForceState(t *testing.T) { 79 | osTempDir = func() string { return "." } 80 | // simulate effect of GS download old state 81 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 82 | t.Fatal("unable to write state", err) 83 | } 84 | defer os.Remove("state") 85 | 86 | // capture GC command 87 | cc := new(commandCapturer) 88 | runCommand = cc.runCommand 89 | // do not remove state because we need to inspect it 90 | removeCount := 0 91 | osRemove = func(string) error { removeCount++; return nil } 92 | defer func() { osRemove = os.Remove }() 93 | 94 | newState := "020_two.yaml" 95 | if err := newApp().Run([]string{"gmig", "-q", "force", "state", "test/demo", newState}); err != nil { 96 | wd, _ := os.Getwd() 97 | t.Fatal("unexpected error", err, wd) 98 | } 99 | if got, want := removeCount, 2; got != want { 100 | t.Logf("got [%v] want [%v]", got, want) 101 | } 102 | data, err := os.ReadFile("state") 103 | if err != nil { 104 | abs, _ := filepath.Abs("state") 105 | t.Fatal("unreadable state", abs, err) 106 | } 107 | if got, want := string(data), newState; got != want { 108 | t.Logf("got [%v] want [%v]", got, want) 109 | } 110 | for i, each := range []string{"gsutil", "-q", "-h", "Content-Type:text/plain", "cp", "state", "gs://bucket/state"} { 111 | if got, want := cc.args[2][i], each; got != want { 112 | t.Logf("got [%v] want [%v]", got, want) 113 | } 114 | } 115 | } 116 | 117 | func TestCmdForceStateNested(t *testing.T) { 118 | osTempDir = func() string { return "." } 119 | // simulate effect of GS download old state 120 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 121 | t.Fatal("unable to write state", err) 122 | } 123 | defer os.Remove("state") 124 | 125 | // capture GC command 126 | cc := new(commandCapturer) 127 | runCommand = cc.runCommand 128 | // do not remove state because we need to inspect it 129 | removeCount := 0 130 | osRemove = func(string) error { removeCount++; return nil } 131 | defer func() { osRemove = os.Remove }() 132 | 133 | newState := "020_two.yaml" 134 | if err := newApp().Run([]string{"gmig", "-v", "-q", "force", "state", "--migrations", "test", "test/demo/nested", newState}); err != nil { 135 | wd, _ := os.Getwd() 136 | t.Fatal("unexpected error", err, wd) 137 | } 138 | if got, want := removeCount, 2; got != want { 139 | t.Logf("got [%v] want [%v]", got, want) 140 | } 141 | data, err := os.ReadFile("state") 142 | if err != nil { 143 | abs, _ := filepath.Abs("state") 144 | t.Fatal("unreadable state", abs, err) 145 | } 146 | if got, want := string(data), newState; got != want { 147 | t.Logf("got [%v] want [%v]", got, want) 148 | } 149 | for i, each := range []string{"gsutil", "-q", "-h", "Content-Type:text/plain", "cp", "state", "gs://bucket/state"} { 150 | if got, want := cc.args[2][i], each; got != want { 151 | t.Logf("got [%v] want [%v]", got, want) 152 | } 153 | } 154 | } 155 | 156 | func TestCmdUp(t *testing.T) { 157 | // simulate effect of GS download old state 158 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 159 | t.Fatal("unable to write state", err) 160 | } 161 | defer os.Remove("state") 162 | // capture GC command 163 | cc := new(commandCapturer) 164 | cc.err = errors.New("shell error") 165 | runCommand = cc.runCommand 166 | if err := newApp().Run([]string{"gmig", "up", "test/demo"}); err == nil { 167 | wd, _ := os.Getwd() 168 | t.Error("expected error", err, wd) 169 | } 170 | if got, want := len(cc.args), 4; got != want { // set config, load 1, save 2, save 3, did not succeed apply error 171 | t.Logf("got [%v] want [%v]", got, want) 172 | } 173 | } 174 | 175 | func TestCmdUpAndStop(t *testing.T) { 176 | keepState() 177 | // simulate effect of GS download old state 178 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 179 | t.Fatal("unable to write state", err) 180 | } 181 | defer os.Remove("state") 182 | // capture GC command 183 | cc := new(commandCapturer) 184 | cc.output = []byte("error") 185 | runCommand = cc.runCommand 186 | if err := newApp().Run([]string{"gmig", "up", "test/demo", "020_two.yaml"}); err != nil { 187 | wd, _ := os.Getwd() 188 | t.Fatal("unexpected error", err, wd) 189 | } 190 | if got, want := len(cc.args), 6; got != want { // set config, load 1, do, save 2, stop 191 | t.Errorf("got [%v] want [%v]", got, want) 192 | } 193 | } 194 | 195 | func TestCmdUpAndStopAfterLastApplied(t *testing.T) { 196 | // simulate effect of GS download old state 197 | if err := os.WriteFile("state", []byte("030_three.yaml"), os.ModePerm); err != nil { 198 | t.Fatal("unable to write state", err) 199 | } 200 | defer os.Remove("state") 201 | // capture GC command 202 | cc := new(commandCapturer) 203 | cc.err = errors.New("shell error") 204 | runCommand = cc.runCommand 205 | if err := newApp().Run([]string{"gmig", "up", "test/demo", "020_two.yaml"}); err == nil { 206 | wd, _ := os.Getwd() 207 | t.Fatal("expected error", err, wd) 208 | } 209 | } 210 | 211 | func TestCmdUpAndStopAfterUnexistingFilename(t *testing.T) { 212 | // simulate effect of GS download old state 213 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 214 | t.Fatal("unable to write state", err) 215 | } 216 | // capture GC command 217 | cc := new(commandCapturer) 218 | runCommand = cc.runCommand 219 | if err := newApp().Run([]string{"gmig", "up", "test/demo", "missing.yaml"}); err == nil { 220 | wd, _ := os.Getwd() 221 | t.Fatal("expected error", err, wd) 222 | } 223 | if got, want := len(cc.args), 2; got != want { // set config, load 1 224 | t.Errorf("got [%v] want [%v]", got, want) 225 | } 226 | } 227 | 228 | func keepState() { 229 | osTempDir = func() string { return "." } 230 | osRemove = func(s string) error { return nil } // do not remove because we run status after up 231 | } 232 | 233 | func TestCmdDown(t *testing.T) { 234 | keepState() 235 | // simulate effect of GS download old state 236 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 237 | t.Fatal("unable to write state", err) 238 | } 239 | // capture GC command 240 | cc := new(commandCapturer) 241 | runCommand = cc.runCommand 242 | if err := newApp().Run([]string{"gmig", "-v", "down", "test/demo"}); err != nil { 243 | wd, _ := os.Getwd() 244 | t.Fatal("unexpected error", err, wd) 245 | } 246 | if got, want := len(cc.args), 3; got != want { // set config, load state 2, save state 1 247 | t.Logf("got [%v] want [%v]", got, want) 248 | } 249 | } 250 | 251 | func TestCmdDownWhenNoLastMigration(t *testing.T) { 252 | keepState() 253 | // simulate effect of GS download old state 254 | if err := os.WriteFile("state", []byte(""), os.ModePerm); err != nil { 255 | t.Fatal("unable to write state", err) 256 | } 257 | // capture GC command 258 | cc := new(commandCapturer) 259 | runCommand = cc.runCommand 260 | if err := newApp().Run([]string{"gmig", "down", "test/demo"}); err != nil { 261 | expected := "gmig ABORTED" 262 | if err.Error() != expected { 263 | t.Errorf("got [%v] want [%v]", err, expected) 264 | } 265 | } 266 | } 267 | 268 | func TestCmdView(t *testing.T) { 269 | osTempDir = func() string { return "." } 270 | // simulate effect of GS download old state 271 | if err := os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm); err != nil { 272 | t.Fatal("unable to write state", err) 273 | } 274 | defer os.Remove("state") 275 | // capture GC command 276 | cc := new(commandCapturer) 277 | runCommand = cc.runCommand 278 | if err := newApp().Run([]string{"gmig", "view", "test/demo"}); err != nil { 279 | wd, _ := os.Getwd() 280 | t.Fatal("unexpected error", err, wd) 281 | } 282 | if got, want := len(cc.args), 3; got != want { // set config, load state, echo 3 283 | t.Errorf("got [%v] want [%v]", got, want) 284 | } 285 | } 286 | 287 | func TestCmdUpConditional(t *testing.T) { 288 | osTempDir = func() string { return "." } 289 | // simulate effect of GS download old state 290 | if err := os.WriteFile("state", []byte("040_error.yaml"), os.ModePerm); err != nil { 291 | t.Fatal("unable to write state", err) 292 | } 293 | defer os.Remove("state") 294 | // capture GC command 295 | cc := new(commandCapturer) 296 | runCommand = cc.runCommand 297 | if err := newApp().Run([]string{"gmig", "up", "test/demo", "050_conditional.yaml"}); err != nil { 298 | wd, _ := os.Getwd() 299 | t.Fatal("unexpected error", err, wd) 300 | } 301 | if got, want := len(cc.args), 6; got != want { 302 | t.Log(cc.args) 303 | t.Errorf("got [%v] want [%v]", got, want) 304 | } 305 | } 306 | 307 | func TestCmdUpConditionalFail(t *testing.T) { 308 | osTempDir = func() string { return "." } 309 | // simulate effect of GS download old state 310 | if err := os.WriteFile("state", []byte("050_conditional.yaml"), os.ModePerm); err != nil { 311 | t.Fatal("unable to write state", err) 312 | } 313 | defer os.Remove("state") 314 | // capture GC command 315 | cc := new(commandCapturer) 316 | runCommand = cc.runCommand 317 | if err := newApp().Run([]string{"gmig", "up", "test/demo"}); err == nil { 318 | wd, _ := os.Getwd() 319 | t.Fatal("expected error", err, wd) 320 | } 321 | if got, want := len(cc.args), 2; got != want { 322 | t.Errorf("got [%v] want [%v]", got, want) 323 | } 324 | } 325 | -------------------------------------------------------------------------------- /config.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "errors" 6 | "fmt" 7 | "os" 8 | "path/filepath" 9 | 10 | "gopkg.in/yaml.v2" 11 | ) 12 | 13 | // ConfigFilename is for reading bucket info 14 | const jsonConfigFilename = "gmig.json" 15 | const YAMLConfigFilename = "gmig.yaml" 16 | const ymlConfigFilename = "gmig.yml" 17 | 18 | // Config holds gmig program config 19 | type Config struct { 20 | // Project is a GCP project name. 21 | Project string `json:"project" yaml:"project"` 22 | 23 | // Region is a GCP region. Optional, use the default one if absent. 24 | Region string `json:"region,omitempty" yaml:"region,omitempty"` 25 | 26 | // Region is a GCP zone. Optional, use the default one if absent. 27 | Zone string `json:"zone,omitempty" yaml:"zone,omitempty"` 28 | 29 | // Bucket is the name of the Google Storage Bucket. 30 | Bucket string `json:"bucket" yaml:"bucket"` 31 | 32 | //LastMigrationObjectName is the name of the bucket object and the local (temporary) file. 33 | LastMigrationObjectName string `json:"state" yaml:"state"` 34 | 35 | // EnvironmentVars hold additional environment values 36 | // that can be accessed by each command line in the Do & Undo section. 37 | // Note that PROJECT,REGION and ZONE are already available. 38 | EnvironmentVars map[string]string `json:"env,omitempty" yaml:"env,omitempty"` 39 | 40 | // verbose if true then produce more logging. 41 | verbose bool 42 | 43 | // source filename 44 | filename string 45 | } 46 | 47 | func loadAndUnmarshalConfig(location string, unmarshaller func(in []byte, out interface{}) (err error)) (*Config, error) { 48 | data, err := os.ReadFile(location) 49 | 50 | if err != nil { 51 | return nil, err 52 | } 53 | 54 | c := &Config{ 55 | filename: location, 56 | } 57 | 58 | if err := unmarshaller(data, &c); err != nil { 59 | return nil, err 60 | } 61 | 62 | if err := c.Validate(); err != nil { 63 | return nil, err 64 | } 65 | 66 | return c, nil 67 | } 68 | 69 | func loadYAMLConfig(location string) (*Config, error) { 70 | return loadAndUnmarshalConfig(location, yaml.Unmarshal) 71 | } 72 | 73 | func loadJSONConfig(location string) (*Config, error) { 74 | return loadAndUnmarshalConfig(location, json.Unmarshal) 75 | } 76 | 77 | // TryToLoadConfig reads configuration from path first looking for gmig.yaml, 78 | // if not exists fallback to gmig.yml and gmig.json then validates it. 79 | func TryToLoadConfig(pathToConfig string) (*Config, error) { 80 | yamlLocation := filepath.Join(pathToConfig, YAMLConfigFilename) 81 | ymlLocation := filepath.Join(pathToConfig, ymlConfigFilename) 82 | jsonLocation := filepath.Join(pathToConfig, jsonConfigFilename) 83 | 84 | if checkExists(yamlLocation) == nil { 85 | return loadYAMLConfig(yamlLocation) 86 | } else if checkExists(ymlLocation) == nil { 87 | return loadYAMLConfig(ymlLocation) 88 | } else if checkExists(jsonLocation) == nil { 89 | config, err := loadJSONConfig(jsonLocation) 90 | printWarning("JSON configuration (gmig.json) is deprecated, your configuration (gmig.yaml)") 91 | return config, err 92 | } 93 | 94 | return nil, errors.New("can not find any configuration") 95 | } 96 | 97 | // ToJSON returns the JSON representation. 98 | func (c Config) ToJSON() string { 99 | data, _ := json.MarshalIndent(c, "", "\t") 100 | return string(data) 101 | } 102 | 103 | // ToYAML returns the YAML representation. 104 | func (c Config) ToYAML() string { 105 | data, _ := yaml.Marshal(c) 106 | return string(data) 107 | } 108 | 109 | // Validate checks required fields in the configuration. 110 | func (c Config) Validate() error { 111 | if len(c.Project) == 0 { 112 | return errors.New("missing project in configuration") 113 | } 114 | if len(c.Bucket) == 0 { 115 | return errors.New("missing bucket in configuration") 116 | } 117 | if len(c.LastMigrationObjectName) == 0 { 118 | return errors.New("missing state name in configuration") 119 | } 120 | return nil 121 | } 122 | 123 | func (c Config) shellEnv() (envs []string) { 124 | envs = append(envs, "PROJECT="+c.Project, "REGION="+c.Region, "ZONE="+c.Zone) 125 | // now (override) with any custom values ; do not check values 126 | for k, v := range c.EnvironmentVars { 127 | envs = append(envs, fmt.Sprintf("%s=%s", k, v)) 128 | } 129 | return 130 | } 131 | -------------------------------------------------------------------------------- /config_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "testing" 6 | ) 7 | 8 | var cfg = ` 9 | { 10 | "project":"demo", 11 | "zone" : "zone", 12 | "region" : "region", 13 | "bucket":"bucket", 14 | "state": "state", 15 | "env" : { 16 | "FOO" : "BAR" 17 | } 18 | } 19 | ` 20 | 21 | func TestConfig(t *testing.T) { 22 | var c Config 23 | if err := json.Unmarshal([]byte(cfg), &c); err != nil { 24 | t.Fatal(err) 25 | } 26 | if err := c.Validate(); err != nil { 27 | t.Fatal(err) 28 | } 29 | if got, want := c.EnvironmentVars["FOO"], "BAR"; got != want { 30 | t.Logf("got [%v] want [%v]", got, want) 31 | } 32 | if got, want := c.LastMigrationObjectName, "state"; got != want { 33 | t.Logf("got [%v] want [%v]", got, want) 34 | } 35 | if got, want := c.shellEnv()[3], "FOO=BAR"; got != want { 36 | t.Logf("got [%v] want [%v]", got, want) 37 | } 38 | } 39 | 40 | func TestTryToLoadConfig(t *testing.T) { 41 | c, err := TryToLoadConfig("bogus") 42 | if err == nil { 43 | t.Error("should return error about non existing bogus folder/file") 44 | } 45 | t.Log(c, err) 46 | } 47 | -------------------------------------------------------------------------------- /examples/010_create_static_ip_for_load_balancer.yaml: -------------------------------------------------------------------------------- 1 | # create static ip for load balancer 2 | 3 | do: 4 | - gcloud compute addresses create my-static-ip --global 5 | undo: 6 | - gcloud compute addresses delete my-static-ip --global --quiet 7 | view: 8 | - gcloud compute addresses list | grep my-static-ip 9 | -------------------------------------------------------------------------------- /examples/015_create_kiya_bucket_for_team.yaml: -------------------------------------------------------------------------------- 1 | # create kiya bucket for team 2 | 3 | do: 4 | - gsutil mb -c regional -l europe-west1 gs://my-kiya-secrets 5 | - gsutil iam ch group:team@hub.com:objectCreator gs://my-kiya-secrets 6 | undo: 7 | - gsutil iam ch -d group:team@hub.com:objectCreator gs://my-kiya-secrets 8 | - gsutil rm -r gs://my-kiya-secrets 9 | -------------------------------------------------------------------------------- /examples/020_create_cloud_sql_database.yaml: -------------------------------------------------------------------------------- 1 | # Create Cloud SQL Database 2 | 3 | do: 4 | # Standard tiers are not working for some reason using the CLI. It works using the UI 5 | # Note regarding the name. If already used, then cannot be used again for some time: https://github.com/hashicorp/terraform/issues/4557 6 | - gcloud beta sql instances create my-db --database-version=POSTGRES_9_6 --region=europe-west1 --gce-zone=europe-west1-b --availability-type=REGIONAL --cpu=1 --memory=4GB 7 | 8 | undo: 9 | - gcloud beta sql instances delete my-db -------------------------------------------------------------------------------- /examples/030_add_storage_viewer_role.yaml: -------------------------------------------------------------------------------- 1 | # Add Storage Viewer role 2 | 3 | # https://cloud.google.com/iam/docs/understanding-roles#predefined_roles 4 | 5 | do: 6 | - gcloud projects add-iam-policy-binding $PROJECT --member serviceAccount:loadrunner@$PROJECT.iam.gserviceaccount.com --role roles/storage.objectViewer 7 | 8 | undo: 9 | - gcloud projects remove-iam-policy-binding $PROJECT --member serviceAccount:loadrunner@$PROJECT.iam.gserviceaccount.com --role roles/storage.objectViewer -------------------------------------------------------------------------------- /examples/035_add_cloud_kms_cryptokey_decrypter_to_cloudbuilder_account.yaml: -------------------------------------------------------------------------------- 1 | # Add Cloud KMS CryptoKey Decrypter to cloudbuilder account 2 | 3 | # https://cloud.google.com/kms/docs/iam 4 | # https://cloud.google.com/kms/docs/reference/permissions-and-roles 5 | 6 | do: 7 | - gcloud kms keys add-iam-policy-binding CRYPTOKEY --location LOCATION --keyring KEYRING --member serviceAccount:00000000@cloudbuild.gserviceaccount.com --role roles/cloudkms.cryptoKeyDecrypter 8 | 9 | undo: 10 | - gcloud kms keys remove-iam-policy-binding CRYPTOKEY --location LOCATION --keyring KEYRING --member serviceAccount:00000000@cloudbuild.gserviceaccount.com --role roles/cloudkms.cryptoKeyDecrypter -------------------------------------------------------------------------------- /examples/036_create_kubernetes_cluster_v2.yaml: -------------------------------------------------------------------------------- 1 | 2 | # create kubernetes cluster v2 3 | # 4 | # file: 036_create_kubernetes_cluster_v2.yaml 5 | 6 | do: 7 | - gcloud beta container clusters create "valentio-cluster-v2" 8 | --verbosity $GCLOUD_VERBOSITY 9 | --cluster-version $GKE_VERSION 10 | --project $PROJECT 11 | --zone $ZONE 12 | --username "admin" 13 | --machine-type "n1-standard-2" 14 | --image-type "COS" 15 | --disk-type "pd-standard" 16 | --disk-size "100" 17 | --scopes "https://www.googleapis.com/auth/devstorage.read_only","https://www.googleapis.com/auth/logging.write","https://www.googleapis.com/auth/monitoring","https://www.googleapis.com/auth/servicecontrol","https://www.googleapis.com/auth/service.management.readonly","https://www.googleapis.com/auth/trace.append" 18 | --num-nodes "3" 19 | --enable-stackdriver-kubernetes 20 | --enable-ip-alias 21 | --network "projects/$PROJECT/global/networks/default" 22 | --subnetwork "projects/$PROJECT/regions/europe-west1/subnetworks/default" 23 | --enable-autoscaling --min-nodes "3" --max-nodes "10" 24 | --addons HorizontalPodAutoscaling 25 | --enable-autoupgrade 26 | --enable-autorepair 27 | 28 | undo: 29 | - gcloud beta container clusters delete "valentio-cluster-v2" --quiet 30 | 31 | view: 32 | - gcloud beta container clusters list 33 | - gcloud beta container clusters describe valentio-cluster-v2 34 | -------------------------------------------------------------------------------- /examples/040_create_service_account.yaml: -------------------------------------------------------------------------------- 1 | # create diy service account 2 | 3 | do: 4 | - gcloud iam service-accounts create diy-user --display-name "Doityourself" 5 | 6 | undo: 7 | - gcloud iam service-accounts delete diy-user@$PROJECT.iam.gserviceaccount.com 8 | 9 | view: 10 | - gcloud iam service-accounts list | grep diy-user 11 | -------------------------------------------------------------------------------- /export_gs_iam.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io/ioutil" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "strings" 11 | ) 12 | 13 | // ExportStorageIAMPolicy prints a migration that sets IAM for each bucket owned by the project. 14 | func ExportStorageIAMPolicy(cfg Config) error { 15 | // get all buckets 16 | cmdline := []string{"gsutil", "list"} 17 | if cfg.verbose { 18 | log.Println(strings.Join(cmdline, " ")) 19 | } 20 | out := new(bytes.Buffer) 21 | cmd := exec.Command(cmdline[0], cmdline[1:]...) 22 | cmd.Stdout = out 23 | cmd.Stderr = os.Stderr 24 | if err := cmd.Run(); err != nil { 25 | return err 26 | } 27 | buckets := strings.Split(strings.TrimSpace(out.String()), "\n") 28 | 29 | // build data structure for migration 30 | type memberRolesPerBucket struct { 31 | bucket string 32 | memberToRoles map[string][]string 33 | } 34 | list := []memberRolesPerBucket{} 35 | for _, each := range buckets { 36 | policy, err := fetchIAMPolicy([]string{"gsutil", "iam", "get", each}, cfg.verbose) 37 | if err != nil { 38 | return err 39 | } 40 | list = append(list, memberRolesPerBucket{ 41 | bucket: each, 42 | memberToRoles: policy.buildMemberToRoles(), 43 | }) 44 | } 45 | 46 | // begin writing migration 47 | content := new(bytes.Buffer) 48 | fmt.Fprintln(content, "# exported buckets iam policy") 49 | fmt.Fprint(content, "\ndo:") 50 | for _, each := range list { 51 | fmt.Fprintf(content, "\n # bucket = %s\n", each.bucket) 52 | if len(each.memberToRoles) == 3 { // projectViewer,Editor,Owner // skip defaults 53 | continue 54 | } 55 | for member, roles := range each.memberToRoles { 56 | if strings.HasPrefix(member, "project") { // skip defaults 57 | continue 58 | } 59 | fmt.Fprintf(content, "\n # member = %s\n", member) 60 | for _, role := range roles { 61 | shortRole := strings.TrimPrefix(role, "roles/storage.") 62 | cmd := fmt.Sprintf(" - gsutil iam ch %s:%s %s\n", member, shortRole, each.bucket) 63 | fmt.Fprintln(content, cmd) 64 | } 65 | } 66 | } 67 | 68 | // UNDO section 69 | fmt.Fprint(content, "\nundo:") 70 | for _, each := range list { 71 | fmt.Fprintf(content, "\n # bucket = %s\n", each.bucket) 72 | for member, roles := range each.memberToRoles { 73 | if strings.HasPrefix(member, "project") { // skip defaults 74 | continue 75 | } 76 | fmt.Fprintf(content, "\n # member = %s\n", member) 77 | for _, role := range roles { 78 | shortRole := strings.TrimPrefix(role, "roles/storage.") 79 | cmd := fmt.Sprintf(" - gsutil iam ch -d %s:%s %s\n", member, shortRole, each.bucket) 80 | fmt.Fprint(content, cmd) 81 | } 82 | } 83 | } 84 | 85 | // write the migration 86 | filename := NewFilenameWithIndex("exported buckets iam policy") 87 | if cfg.verbose { 88 | log.Println("writing", filename) 89 | } 90 | return ioutil.WriteFile(filename, content.Bytes(), os.ModePerm) 91 | } 92 | -------------------------------------------------------------------------------- /export_iam.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "encoding/json" 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "strings" 11 | ) 12 | 13 | // IAMPolicy is for capturing the project iam policy 14 | type IAMPolicy struct { 15 | Bindings []struct { 16 | Members []string 17 | Role string 18 | } 19 | } 20 | 21 | // buildMemberToRoles builds the reverse mapping 22 | func (i IAMPolicy) buildMemberToRoles() map[string][]string { 23 | memberToRoles := map[string][]string{} 24 | for _, each := range i.Bindings { 25 | role := each.Role 26 | for _, member := range each.Members { 27 | list, ok := memberToRoles[member] 28 | if !ok { 29 | list = []string{} 30 | } 31 | memberToRoles[member] = append(list, role) 32 | } 33 | } 34 | return memberToRoles 35 | } 36 | 37 | func fetchIAMPolicy(cmdline []string, verbose bool) (IAMPolicy, error) { 38 | var p IAMPolicy 39 | if verbose { 40 | log.Println(strings.Join(cmdline, " ")) 41 | } 42 | cmd := exec.Command(cmdline[0], cmdline[1:]...) 43 | combined, err := runCommand(cmd) 44 | if err != nil { 45 | return p, err 46 | } 47 | if err := json.Unmarshal(combined, &p); err != nil { 48 | return p, err 49 | } 50 | return p, nil 51 | } 52 | 53 | // ExportProjectsIAMPolicy reads the current IAM bindings on project level 54 | // and outputs the contents of a gmig migration file. 55 | // Return the filename of the migration. 56 | func ExportProjectsIAMPolicy(cfg Config) error { 57 | policy, err := fetchIAMPolicy([]string{"gcloud", "projects", "get-iam-policy", cfg.Project, "--format", "json"}, cfg.verbose) 58 | if err != nil { 59 | return err 60 | } 61 | memberToRoles := policy.buildMemberToRoles() 62 | content := new(bytes.Buffer) 63 | fmt.Fprintln(content, "# exported projects iam policy") 64 | fmt.Fprint(content, "\ndo:") 65 | for member, roles := range memberToRoles { 66 | fmt.Fprintf(content, "\n # member = %s\n", member) 67 | for _, role := range roles { 68 | cmd := fmt.Sprintf(" - gcloud projects add-iam-policy-binding $PROJECT --member %s --role %s\n", member, role) 69 | fmt.Fprint(content, cmd) 70 | } 71 | } 72 | fmt.Fprintf(content, "\nundo:") 73 | for member, roles := range memberToRoles { 74 | fmt.Fprintf(content, "\n # member = %s\n", member) 75 | for _, role := range roles { 76 | cmd := fmt.Sprintf(" - gcloud projects remove-iam-policy-binding $PROJECT --member %s --role %s\n", member, role) 77 | fmt.Fprint(content, cmd) 78 | } 79 | } 80 | filename := NewFilenameWithIndex("exported project iam policy") 81 | if cfg.verbose { 82 | log.Println("writing", filename) 83 | } 84 | return os.WriteFile(filename, content.Bytes(), os.ModePerm) 85 | } 86 | -------------------------------------------------------------------------------- /export_iam_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os" 5 | "testing" 6 | "time" 7 | ) 8 | 9 | func TestExportProjectsIAMPolicy(t *testing.T) { 10 | // simulate effect of GS download old state 11 | os.WriteFile("state", []byte("010_one.yaml"), os.ModePerm) 12 | defer os.Remove("state") 13 | 14 | // simulate now 15 | timeNow = func() time.Time { return time.Time{} } 16 | 17 | // cleanup generated migration 18 | generated := "010_exported_project_iam_policy.yaml" 19 | defer os.Remove(generated) 20 | 21 | // capture GC command 22 | cc := new(commandCapturer) 23 | cc.output = []byte(`{ 24 | "bindings": [{ 25 | "members": [ 26 | "member" 27 | ], 28 | "role": "role" 29 | }] 30 | }`) 31 | runCommand = cc.runCommand 32 | if err := newApp().Run([]string{"gmig", "export", "project-iam-policy", "test/demo"}); err != nil { 33 | wd, _ := os.Getwd() 34 | t.Fatal("unexpected error", err, wd) 35 | } 36 | 37 | if m, err := LoadMigration(generated); err != nil { 38 | t.Fatal("unable to load generated migration", err) 39 | } else { 40 | if got, want := len(m.DoSection), 1; got != want { 41 | t.Logf("got [%v] want [%v]", got, want) 42 | } 43 | if got, want := len(m.DoSection), 2; got != want { 44 | t.Logf("got [%v] want [%v]", got, want) 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /gcloud.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "os/exec" 7 | 8 | "github.com/emicklei/tre" 9 | ) 10 | 11 | func gcloudConfigList() { 12 | log.Println("checking gcloud config list ...") 13 | cmd := exec.Command("gcloud", "config", "list") 14 | out, _ := cmd.CombinedOutput() 15 | fmt.Println(string(out)) 16 | } 17 | 18 | func gcloudConfigSetProject(cfg Config) error { 19 | for _, each := range []struct { 20 | Key, Value string 21 | }{ 22 | {"core/project", cfg.Project}, 23 | {"compute/region", cfg.Region}, 24 | {"compute/zone", cfg.Zone}, 25 | } { 26 | k := each.Key 27 | v := each.Value 28 | if len(v) > 0 { // skip optional values 29 | if cfg.verbose { 30 | log.Printf("setting gcloud config [%s] to [%s]\n", k, v) 31 | } 32 | cmd := exec.Command("gcloud", "config", "set", k, v) 33 | data, err := runCommand(cmd) 34 | if cfg.verbose { 35 | log.Println(string(data)) 36 | } 37 | if err != nil { 38 | return tre.New(err, "error changing gcloud config", k, v) 39 | } 40 | } 41 | } 42 | return nil 43 | } 44 | -------------------------------------------------------------------------------- /gcloud_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestGcloudConfigSetProject(t *testing.T) { 9 | cc := new(commandCapturer) 10 | old := runCommand 11 | runCommand = cc.runCommand 12 | defer func() { runCommand = old }() 13 | cfg := Config{Project: "p", Region: "r", Zone: "z"} 14 | if err := gcloudConfigSetProject(cfg); err != nil { 15 | t.Fatal(err) 16 | } 17 | if got, want := len(cc.args), 3; got != want { 18 | t.Errorf("got [%v] want [%v]", got, want) 19 | } 20 | if got, want := fmt.Sprint(cc.args[0]), "[gcloud config set core/project p]"; got != want { 21 | t.Errorf("got [%v] want [%v]", got, want) 22 | } 23 | if got, want := fmt.Sprint(cc.args[1]), "[gcloud config set compute/region r]"; got != want { 24 | t.Errorf("got [%v] want [%v]", got, want) 25 | } 26 | if got, want := fmt.Sprint(cc.args[2]), "[gcloud config set compute/zone z]"; got != want { 27 | t.Errorf("got [%v] want [%v]", got, want) 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /gcs.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os/exec" 6 | "path/filepath" 7 | "strings" 8 | 9 | "github.com/emicklei/tre" 10 | ) 11 | 12 | // GCS = Google Cloud Storage 13 | type GCS struct { 14 | onDiskAccess FileStateProvider 15 | } 16 | 17 | // NewGCS returns a new GCS 18 | func NewGCS(cfg Config) GCS { 19 | return GCS{onDiskAccess: NewFileStateProvider(cfg)} 20 | } 21 | 22 | // LoadState implements StateProvider 23 | func (g GCS) LoadState() (string, error) { 24 | defer g.onDiskAccess.DeleteState() 25 | cmdline := []string{"gsutil", "-q", "cp", 26 | "gs://" + filepath.Join(g.Config().Bucket, g.Config().LastMigrationObjectName), 27 | g.onDiskAccess.stateFilename()} 28 | if err := g.gsutil(cmdline); err != nil { 29 | // see if there was no last applied state 30 | if strings.Contains(err.Error(), "No URLs matched") { // lame detection method TODO 31 | if g.Config().verbose { 32 | log.Println("no last applied migration found.") 33 | } 34 | return "", nil 35 | } 36 | return "", err 37 | } 38 | return g.onDiskAccess.LoadState() 39 | } 40 | 41 | // SaveState implements StateProvider 42 | func (g GCS) SaveState(filename string) error { 43 | defer g.onDiskAccess.DeleteState() 44 | if err := g.onDiskAccess.SaveState(filename); err != nil { 45 | return err 46 | } 47 | cmdline := []string{"gsutil", "-q", "-h", "Content-Type:text/plain", "cp", 48 | g.onDiskAccess.stateFilename(), 49 | "gs://" + filepath.Join(g.Config().Bucket, g.Config().LastMigrationObjectName)} 50 | return g.gsutil(cmdline) 51 | } 52 | 53 | // Config implements StateProvider 54 | func (g GCS) Config() Config { 55 | return g.onDiskAccess.Config() 56 | } 57 | 58 | func (g GCS) gsutil(cmdline []string) error { 59 | if g.Config().verbose { 60 | log.Println(strings.Join(cmdline, " ")) 61 | } 62 | cmd := exec.Command(cmdline[0], cmdline[1:]...) 63 | stdoutStderr, err := runCommand(cmd) 64 | if err != nil { 65 | return tre.New(err, "gsutil cp failed", "output:", string(stdoutStderr)) 66 | } 67 | return nil 68 | } 69 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/emicklei/gmig 2 | 3 | require ( 4 | github.com/emicklei/tre v1.7.0 5 | github.com/expr-lang/expr v1.17.2 6 | github.com/marcacohen/gcslock v0.0.0-20180212104141-5782a95db7e2 7 | github.com/urfave/cli v1.22.16 8 | gopkg.in/yaml.v2 v2.4.0 9 | ) 10 | 11 | require ( 12 | cloud.google.com/go/compute/metadata v0.6.0 // indirect 13 | github.com/cpuguy83/go-md2man/v2 v2.0.6 // indirect 14 | github.com/google/go-cmp v0.7.0 // indirect 15 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 16 | golang.org/x/net v0.37.0 // indirect 17 | golang.org/x/oauth2 v0.28.0 // indirect 18 | golang.org/x/sys v0.31.0 // indirect 19 | ) 20 | 21 | go 1.23.0 22 | 23 | toolchain go1.24.0 24 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= 2 | cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= 3 | github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= 4 | github.com/cpuguy83/go-md2man/v2 v2.0.5/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= 5 | github.com/cpuguy83/go-md2man/v2 v2.0.6 h1:XJtiaUW6dEEqVuZiMTn1ldk455QWwEIsMIJlo5vtkx0= 6 | github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= 7 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 8 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 9 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 10 | github.com/emicklei/tre v1.7.0 h1:bEZhzT3o+9lcJ4KQh+uf0mv7rmMe0LEtVDa/7F5iq/c= 11 | github.com/emicklei/tre v1.7.0/go.mod h1:3e+DTVKARET1BwLFwOIJwgYLUtkN+X9XZT0JDnb+OYY= 12 | github.com/expr-lang/expr v1.17.2 h1:o0A99O/Px+/DTjEnQiodAgOIK9PPxL8DtXhBRKC+Iso= 13 | github.com/expr-lang/expr v1.17.2/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= 14 | github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= 15 | github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= 16 | github.com/marcacohen/gcslock v0.0.0-20180212104141-5782a95db7e2 h1:8XwBm77EGpxFUrdPCWtlhEXIB1fMXtLhvtSu3Bj/SQA= 17 | github.com/marcacohen/gcslock v0.0.0-20180212104141-5782a95db7e2/go.mod h1:jK3NXVSlRXbuKNPsIDN4RtDNayIS6bDxh92ApnKRZ9w= 18 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 19 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 20 | github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= 21 | github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= 22 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 23 | github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= 24 | github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= 25 | github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= 26 | github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 27 | github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= 28 | github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= 29 | github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= 30 | github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 31 | github.com/urfave/cli v1.22.16 h1:MH0k6uJxdwdeWQTwhSO42Pwr4YLrNLwBtg1MRgTqPdQ= 32 | github.com/urfave/cli v1.22.16/go.mod h1:EeJR6BKodywf4zciqrdw6hpCPk68JO9z5LazXZMn5Po= 33 | golang.org/x/net v0.37.0 h1:1zLorHbz+LYj7MQlSf1+2tPIIgibq2eL5xkrGk6f+2c= 34 | golang.org/x/net v0.37.0/go.mod h1:ivrbrMbzFq5J41QOQh0siUuly180yBYtLp+CKbEaFx8= 35 | golang.org/x/oauth2 v0.28.0 h1:CrgCKl8PPAVtLnU3c+EDw6x11699EWlsDeWNWKdIOkc= 36 | golang.org/x/oauth2 v0.28.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= 37 | golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik= 38 | golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= 39 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= 40 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 41 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 42 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 43 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 44 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 45 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 46 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "log" 5 | "os" 6 | "sort" 7 | 8 | "github.com/urfave/cli" 9 | ) 10 | 11 | var Version string 12 | 13 | func main() { 14 | if err := newApp().Run(os.Args); err != nil { 15 | log.Fatalln(err) 16 | } 17 | } 18 | 19 | func newApp() *cli.App { 20 | app := cli.NewApp() 21 | app.Version = Version 22 | app.EnableBashCompletion = true 23 | app.Name = "gmig" 24 | app.Usage = "Google Cloud Platform infrastructure migration tool" 25 | 26 | // override -v 27 | cli.VersionFlag = cli.BoolFlag{ 28 | Name: "print-version, V", 29 | Usage: "print only the version", 30 | } 31 | app.Flags = []cli.Flag{ 32 | cli.BoolFlag{ 33 | Name: "v", 34 | Usage: "verbose logging", 35 | }, 36 | cli.BoolFlag{ 37 | Name: "q", 38 | Usage: "quiet mode, accept any prompt", 39 | }, 40 | } 41 | migrationsFlag := cli.StringFlag{ 42 | Name: "migrations", 43 | Usage: `folder containing the migrations to apply on the target project. 44 | If not specified then set it to the parent folder of the configuration file.`, 45 | } 46 | 47 | app.Commands = []cli.Command{ 48 | { 49 | Name: "init", 50 | Usage: "Create the initial configuration, if absent.", 51 | Action: func(c *cli.Context) error { 52 | defer started(c, "init")() 53 | return cmdInit(c) 54 | }, 55 | ArgsUsage: `<path> 56 | path - name of the folder that contains the configuration of the target project. The folder name may end with a path separator and can be relative or absolute.`, 57 | }, 58 | { 59 | Name: "new", 60 | Usage: "Create a new migration file from a template using an index prefix and a given title.", 61 | Action: func(c *cli.Context) error { 62 | defer started(c, "create migration")() 63 | return cmdCreateMigration(c) 64 | }, 65 | Flags: []cli.Flag{ 66 | cli.StringFlag{ 67 | Name: "do", 68 | Usage: "commands to run in the 'do' section of this migration. Multiple commands need to be separated by a newline.", 69 | }, 70 | cli.StringFlag{ 71 | Name: "undo", 72 | Usage: "commands to run in the 'undo' section of this migration. Multiple commands need to be separated by a newline.", 73 | }, 74 | cli.StringFlag{ 75 | Name: "view", 76 | Usage: "commands to run in the 'view' section of this migration. Multiple commands need to be separated by a newline.", 77 | }, 78 | }, 79 | ArgsUsage: `<title> 80 | title - what the effect of this migration is on infrastructure.`, 81 | }, 82 | { 83 | Name: "plan", 84 | Usage: "Log commands of the do section of all pending migrations in order, one after the other. If a migration file is specified then stop after applying that one.", 85 | Action: func(c *cli.Context) error { 86 | defer started(c, "plan = log commands of pending migrations")() 87 | return cmdMigrationsPlan(c) 88 | }, 89 | Flags: []cli.Flag{migrationsFlag}, 90 | ArgsUsage: `<path> [stop] 91 | path - name of the folder that contains the configuration of the target project. 92 | stop - (optional) the name of the migration file after which applying migrations will stop.`, 93 | }, 94 | { 95 | Name: "up", 96 | Usage: "Runs the do section of all pending migrations in order, one after the other. If a migration file is specified then stop after applying that one.", 97 | Action: func(c *cli.Context) error { 98 | defer started(c, "up = apply pending migrations")() 99 | if err := cmdMigrationsUp(c); err != nil { 100 | return err 101 | } 102 | return cmdMigrationsStatus(c) 103 | }, 104 | Flags: []cli.Flag{migrationsFlag}, 105 | ArgsUsage: `<path> [stop] 106 | path - name of the folder that contains the configuration of the target project. 107 | stop - (optional) the name of the migration file after which applying migrations will stop.`, 108 | }, 109 | { 110 | Name: "down", 111 | Usage: "Runs the undo section of only the last applied migration.", 112 | Action: func(c *cli.Context) error { 113 | defer started(c, "down = undo last applied migration")() 114 | if err := cmdMigrationsDown(c); err != nil { 115 | return err 116 | } 117 | return cmdMigrationsStatus(c) 118 | }, 119 | Flags: []cli.Flag{migrationsFlag}, 120 | ArgsUsage: `<path> 121 | path - name of the folder that contains the configuration of the target project.`, 122 | }, 123 | { 124 | Name: "down-all", 125 | Usage: "Runs the undo section of all applied migrations.", 126 | Action: func(c *cli.Context) error { 127 | defer started(c, "down-all = undo all applied migration")() 128 | if err := cmdMigrationsDownAll(c); err != nil { 129 | return err 130 | } 131 | return cmdMigrationsStatus(c) 132 | }, 133 | Flags: []cli.Flag{migrationsFlag}, 134 | ArgsUsage: `<path> 135 | path - name of the folder that contains the configuration of the target project.`, 136 | }, 137 | { 138 | Name: "status", 139 | Usage: "List all migrations with details compared to the current state.", 140 | Action: func(c *cli.Context) error { 141 | defer started(c, "show status of migrations")() 142 | return cmdMigrationsStatus(c) 143 | }, 144 | Flags: []cli.Flag{migrationsFlag}, 145 | ArgsUsage: `<path> 146 | path - name of the folder that contains the configuration of the target project.`, 147 | }, 148 | { 149 | Name: "view", 150 | Usage: "Show infrastructure information for the current state.", 151 | Action: func(c *cli.Context) error { 152 | defer started(c, "show status of infrastructure")() 153 | return cmdView(c) 154 | }, 155 | Flags: []cli.Flag{migrationsFlag}, 156 | ArgsUsage: `<path> 157 | path - name of the folder that contains the configuration of the target project.`, 158 | }, 159 | { 160 | Name: "export-env", 161 | Usage: "print export for each environment variable.", 162 | Action: func(c *cli.Context) error { 163 | defer started(c, "export environment variables")() 164 | return cmdExportEnv(c) 165 | }, 166 | Flags: []cli.Flag{migrationsFlag}, 167 | ArgsUsage: `<path> 168 | path - name of the folder that contains the configuration of the target project.`, 169 | }, 170 | { 171 | Name: "template", 172 | Usage: "Process a template file (Go syntax)", 173 | Action: func(c *cli.Context) error { 174 | defer started(c, "process a template file")() 175 | return cmdTemplate(c) 176 | }, 177 | Flags: []cli.Flag{cli.BoolFlag{ 178 | Name: "w", 179 | Usage: `write result back to the source file instead of stdout.`, 180 | }}, 181 | ArgsUsage: `<source> 182 | source - name of the template file to process.`, 183 | }, 184 | { 185 | Name: "util", 186 | Usage: "Handle named ports {create-named-port|delete-named-port}", 187 | Subcommands: []cli.Command{ 188 | { 189 | Name: "create-named-port", 190 | Usage: "add a new name->port mapping to a compute instance group. ignore if exists.", 191 | Action: func(c *cli.Context) error { 192 | defer started(c, "create-named-port")() 193 | return cmdCreateNamedPort(c) 194 | }, 195 | ArgsUsage: `<instance-group> <name:port> 196 | instance-group - identifier of the compute instance group 197 | name:port - mapping of a name to a port, e.g http-port:80`, 198 | }, 199 | { 200 | Name: "delete-named-port", 201 | Usage: "delete a name->port mapping from a compute instance group. ignore if not exists.", 202 | Action: func(c *cli.Context) error { 203 | defer started(c, "delete-named-port")() 204 | return cmdDeleteNamedPort(c) 205 | }, 206 | ArgsUsage: `<instance-group> <name:port> 207 | instance-group - identifier of the compute instance group 208 | name:port - mapping of a name to a port, e.g http-port:80`, 209 | }, 210 | { 211 | Name: "add-path-rules-to-path-matcher", 212 | Usage: "Add a set of path rules to an existing path-matcher in a loadbalancer url-map.", 213 | Action: func(c *cli.Context) error { 214 | defer started(c, "add-path-rules-to-path-matcher")() 215 | return cmdAddPathRulesToPathMatcher(c) 216 | }, 217 | Flags: []cli.Flag{cli.StringFlag{ 218 | Name: "url-map", 219 | Required: true, 220 | Usage: `name of the url-map in the current project/region.`, 221 | }, cli.StringFlag{ 222 | Name: "path-matcher", 223 | Required: true, 224 | Usage: `name of the path-matcher known to the url-map.`, 225 | }, cli.StringFlag{ 226 | Name: "service", 227 | Required: true, 228 | Usage: `name of the backend service that handles call on the paths.`, 229 | }, cli.StringFlag{ 230 | Name: "paths", 231 | Required: true, 232 | Usage: `comma separated list of paths of the set known to the service.`, 233 | }, cli.StringFlag{ 234 | Name: "region", 235 | Required: false, 236 | Usage: `if set then the url-map is regional, global otherwise`, 237 | }}, 238 | ArgsUsage: ``, 239 | }, 240 | { 241 | Name: "remove-path-rules-from-path-matcher", 242 | Usage: "Remove a set of path rules from an existing path-matcher in a loadbalancer url-map.", 243 | Action: func(c *cli.Context) error { 244 | defer started(c, "remove-path-rules-from-path-matcher")() 245 | return cmdRemovePathRulesFromPathMatcher(c) 246 | }, 247 | Flags: []cli.Flag{cli.StringFlag{ 248 | Name: "url-map", 249 | Required: true, 250 | Usage: `name of the url-map in the current project/region.`, 251 | }, cli.StringFlag{ 252 | Name: "path-matcher", 253 | Required: true, 254 | Usage: `name of the path-matcher known to the url-map.`, 255 | }, cli.StringFlag{ 256 | Name: "service", 257 | Required: true, 258 | Usage: `name of the backend service that handles call on the paths.`, 259 | }, cli.StringFlag{ 260 | Name: "region", 261 | Required: false, 262 | Usage: `if set then the url-map is regional, global otherwise`, 263 | }}, 264 | ArgsUsage: ``, 265 | }, 266 | }, 267 | }, 268 | { 269 | Name: "force", 270 | Usage: "Force an action {state|do|undo}", 271 | Subcommands: []cli.Command{ 272 | { 273 | Name: "state", 274 | Usage: "Explicitly set the state to a specified migration filename.", 275 | Action: func(c *cli.Context) error { 276 | defer started(c, "force last applied migration (state)")() 277 | if err := cmdMigrationsSetState(c); err != nil { 278 | return err 279 | } 280 | return cmdMigrationsStatus(c) 281 | }, 282 | Flags: []cli.Flag{migrationsFlag}, 283 | ArgsUsage: `<path> 284 | path - name of the folder that contains the configuration of the target project.`, 285 | }, 286 | { 287 | Name: "do", 288 | Usage: "Force run the DO section of a migration. State will not be updated.", 289 | Action: func(c *cli.Context) error { 290 | defer started(c, "execute DO section")() 291 | return cmdRundoOnly(c) 292 | }, 293 | Flags: []cli.Flag{migrationsFlag}, 294 | ArgsUsage: `<path> <filename> 295 | path - name of the folder that contains the configuration of the target project. 296 | filename - name of the migration that contains a do: section.`, 297 | }, 298 | { 299 | Name: "undo", 300 | Usage: "Force run the UNDO section of a migration. State will not be updated.", 301 | Action: func(c *cli.Context) error { 302 | defer started(c, "execute UNDO section")() 303 | return cmdRunUndoOnly(c) 304 | }, 305 | Flags: []cli.Flag{migrationsFlag}, 306 | ArgsUsage: `<path> <filename> 307 | path - name of the folder that contains the configuration of the target project. 308 | filename - name of the migration that contains a undo: section.`, 309 | }, 310 | }, 311 | }, 312 | { 313 | Name: "export", 314 | Usage: "Export existing infrastructure {project-iam-policy|storage-iam-policy}", 315 | Subcommands: []cli.Command{ 316 | { 317 | Name: "project-iam-policy", 318 | Usage: "Print a migration that describes the current IAM policy binding on project level.", 319 | Action: func(c *cli.Context) error { 320 | defer started(c, "export project IAM policy")() 321 | return cmdExportProjectIAMPolicy(c) 322 | }, 323 | ArgsUsage: `<path> 324 | path - name of the folder that contains the configuration of the target project.`, 325 | }, 326 | { 327 | Name: "storage-iam-policy", 328 | Usage: "Print a migration that describes the current IAM policy bindings for Google Storage.", 329 | Action: func(c *cli.Context) error { 330 | defer started(c, "export storage IAM policy")() 331 | return cmdExportStorageIAMPolicy(c) 332 | }, 333 | ArgsUsage: `<path> 334 | path - name of the folder that contains the configuration of the target project.`, 335 | }, 336 | }, 337 | }, 338 | } 339 | sort.Sort(cli.FlagsByName(app.Flags)) 340 | return app 341 | } 342 | -------------------------------------------------------------------------------- /migration.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "errors" 6 | "fmt" 7 | "io/ioutil" 8 | "log" 9 | "os" 10 | "os/exec" 11 | "path" 12 | "path/filepath" 13 | "sort" 14 | "strconv" 15 | "strings" 16 | "time" 17 | 18 | "text/template" 19 | 20 | "github.com/expr-lang/expr" 21 | "gopkg.in/yaml.v2" 22 | ) 23 | 24 | // Migration holds shell commands for applying or reverting a change. 25 | type Migration struct { 26 | Filename string `yaml:"-"` 27 | Description string `yaml:"-"` 28 | IfExpression string `yaml:"if"` 29 | DoSection []string `yaml:"do"` 30 | UndoSection []string `yaml:"undo"` 31 | ViewSection []string `yaml:"view"` 32 | } 33 | 34 | // evaluateCondition evaluates the expression to a bool ; report error otherwise. 35 | func evaluateCondition(ifExpression string, envs []string) (bool, error) { 36 | if len(ifExpression) == 0 { 37 | return true, nil 38 | } 39 | envMap := map[string]string{} 40 | for _, each := range envs { 41 | kv := strings.Split(each, "=") 42 | if len(kv) != 2 { 43 | continue 44 | } 45 | k := strings.TrimSpace(kv[0]) 46 | v := strings.TrimSpace(kv[1]) 47 | envMap[k] = v 48 | } 49 | program, err := expr.Compile(ifExpression, expr.Env(envMap)) 50 | if err != nil { 51 | return false, err 52 | } 53 | output, err := expr.Run(program, envMap) 54 | if err != nil { 55 | return false, err 56 | } 57 | if b, ok := output.(bool); ok { 58 | return b, nil 59 | } 60 | return false, errors.New("expression does not evaluate to a boolean") 61 | } 62 | 63 | // for testing 64 | var timeNow = time.Now 65 | 66 | // NewFilenameWithIndex generates a filename using an index for storing 67 | // a new migration. 68 | func NewFilenameWithIndex(desc string) string { 69 | all, err := LoadMigrationsBetweenAnd(".", "", "") 70 | if err != nil { 71 | printError(err.Error()) 72 | return "" 73 | } 74 | sanitized := strings.Replace(strings.ToLower(desc), " ", "_", -1) 75 | if len(all) == 0 { 76 | return fmt.Sprintf("010_%s.yaml", sanitized) 77 | } 78 | lastFilename := all[len(all)-1].Filename 79 | hasTimestamp := regexpTimestamp.MatchString(lastFilename) 80 | hasIndex := regexpIndex.MatchString(lastFilename) 81 | if hasIndex { 82 | i, err := strconv.Atoi(lastFilename[:3]) 83 | if err != nil { 84 | fmt.Printf("%T, %v", i, i) 85 | } 86 | return fmt.Sprintf("%03d_%s.yaml", i+5, sanitized) 87 | } 88 | if hasTimestamp { 89 | return fmt.Sprintf("300_%s.yaml", sanitized) 90 | } 91 | return fmt.Sprintf("010_%s.yaml", sanitized) 92 | } 93 | 94 | // LoadMigration reads and parses a migration from a named file. 95 | func LoadMigration(absFilename string) (m Migration, err error) { 96 | data, err := os.ReadFile(absFilename) 97 | if err != nil { 98 | wd, _ := os.Getwd() 99 | return m, fmt.Errorf("in %s, %s reading failed: %v", wd, absFilename, err) 100 | } 101 | m.Filename = filepath.Base(absFilename) 102 | err = yaml.Unmarshal(data, &m) 103 | if err != nil { 104 | err = fmt.Errorf("%s parsing failed: %v", absFilename, err) 105 | } 106 | return 107 | } 108 | 109 | // ToYAML returns the contents of a YAML encoded fixture. 110 | func (m Migration) ToYAML() ([]byte, error) { 111 | out := new(bytes.Buffer) 112 | err := migrationTemplate.Execute(out, m) 113 | return out.Bytes(), err 114 | } 115 | 116 | // ExecuteAll the commands for this migration unless the condition evaluates to false 117 | // We create a temporary executable file with all commands. 118 | // This allows for using shell variables in multiple commands. 119 | func ExecuteAll(ifExpression string, commands []string, envs []string, verbose bool) error { 120 | // check condition 121 | pass, err := evaluateCondition(ifExpression, envs) 122 | if err != nil { 123 | log.Printf("unable to evaluate condition [%s] because:%v\n", ifExpression, err) 124 | return errAbort 125 | } 126 | if !pass { 127 | log.Printf(".. skipping ... (%d) commands because %s is false.\n", len(commands), ifExpression) 128 | return nil 129 | } 130 | if len(commands) == 0 { 131 | return nil 132 | } 133 | tempScript := path.Join(os.TempDir(), "gmig.sh") 134 | content := new(bytes.Buffer) 135 | fmt.Fprintln(content, setupShellScript(verbose)) 136 | 137 | for _, each := range commands { 138 | fmt.Fprintln(content, each) 139 | } 140 | if err := ioutil.WriteFile(tempScript, content.Bytes(), os.ModePerm); err != nil { 141 | return fmt.Errorf("failed to write temporary migration section:%v", err) 142 | } 143 | if verbose { 144 | log.Println("--- BEGIN gmig.sh:\n", content.String(), "--- END gmig.sh") 145 | } 146 | defer func() { 147 | if err := os.Remove(tempScript); err != nil { 148 | log.Printf("warning: failed to remove temporary migration execution script:%s\n", tempScript) 149 | } 150 | }() 151 | cmd := exec.Command("sh", "-c", tempScript) 152 | cmd.Env = append(os.Environ(), envs...) // extend, not replace 153 | if out, err := runCommand(cmd); err != nil { 154 | return fmt.Errorf("failed to run migration section:\n%s\nerror:%v", string(out), err) 155 | } else { 156 | fmt.Println(string(out)) 157 | } 158 | return nil 159 | } 160 | 161 | // LogAll logs expanded commands using the environment variables of both the config and the OS. 162 | func LogAll(ifExpression string, commands []string, envs []string, verbose bool) error { 163 | // check condition 164 | pass, err := evaluateCondition(ifExpression, envs) 165 | if err != nil { 166 | log.Printf("unable to evaluate condition [%s] because:%v\n", ifExpression, err) 167 | return errAbort 168 | } 169 | if !pass { 170 | log.Printf(".. skipping ... (%d) commands because %s is false.\n", len(commands), ifExpression) 171 | return nil 172 | } 173 | if len(commands) == 0 { 174 | return nil 175 | } 176 | allEnv := append(os.Environ(), envs...) 177 | envMap := map[string]string{} 178 | for _, each := range allEnv { 179 | kv := strings.Split(each, "=") 180 | envMap[kv[0]] = kv[1] 181 | } 182 | for _, each := range commands { 183 | log.Println(expandVarsIn(envMap, each)) 184 | } 185 | return nil 186 | } 187 | 188 | // expandVarsIn returns a command with all occurrences of environment variables replaced by known values. 189 | func expandVarsIn(envs map[string]string, command string) string { 190 | // assume no recurse expand 191 | expanded := command 192 | for k, v := range envs { 193 | // if the value itself is a known variable then skip it 194 | if strings.HasPrefix(v, "$") { 195 | if _, ok := envs[v]; ok { 196 | log.Printf("Warning, skipping non-expandable environment var %s=%v\n", k, v) 197 | continue 198 | } 199 | } 200 | varName := "$" + k 201 | expanded = strings.Replace(expanded, varName, v, -1) 202 | } 203 | return expanded 204 | } 205 | 206 | func setupShellScript(verbose bool) string { 207 | flag := "-v" 208 | if verbose { 209 | flag = "-x" 210 | } 211 | return fmt.Sprintf(`#!/bin/bash 212 | # temporary gmig execution script 213 | set -e %s`, flag) 214 | } 215 | 216 | // LoadMigrationsBetweenAnd returns a list of pending Migration <firstFilename..lastFilename] 217 | func LoadMigrationsBetweenAnd(migrationsPath, firstFilename, lastFilename string) (list []Migration, err error) { 218 | // collect all filenames 219 | filenames := []string{} 220 | // firstFilename and lastFilename are relative to workdir. 221 | here, _ := os.Getwd() 222 | // change and restore finally 223 | if err = os.Chdir(migrationsPath); err != nil { 224 | return 225 | } 226 | defer os.Chdir(here) 227 | files, err := os.ReadDir(".") 228 | if err != nil { 229 | log.Println("unable to read migrations from folder", err) 230 | return 231 | } 232 | for _, each := range files { 233 | if each.IsDir() || !isYamlFile(each.Name()) { 234 | continue 235 | } 236 | filenames = append(filenames, each.Name()) 237 | } 238 | // old -> new 239 | sort.StringSlice(filenames).Sort() 240 | // load only pending migrations 241 | for _, each := range filenames { 242 | // do not include firstFilename 243 | if each <= firstFilename { 244 | continue 245 | } 246 | var m Migration 247 | m, err = LoadMigration(filepath.Join(migrationsPath, each)) 248 | if err != nil { 249 | return 250 | } 251 | list = append(list, m) 252 | // include lastFilename 253 | if len(lastFilename) == 0 { 254 | continue 255 | } 256 | if each == lastFilename { 257 | return 258 | } 259 | } 260 | return 261 | } 262 | 263 | var migrationTemplate = template.Must(template.New("gen").Parse(` 264 | # {{.Description}} 265 | # 266 | # file: {{.Filename}} 267 | 268 | do:{{range .DoSection}} 269 | - {{.}}{{end}} 270 | 271 | undo:{{range .UndoSection}} 272 | - {{.}}{{end}} 273 | 274 | view:{{range .ViewSection}} 275 | - {{.}}{{end}} 276 | `)) 277 | -------------------------------------------------------------------------------- /migration_context.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | "path/filepath" 7 | 8 | "github.com/urfave/cli" 9 | ) 10 | 11 | type migrationContext struct { 12 | // lastApplied is the filename of last migration, relative to migrationsPath 13 | lastApplied string 14 | stateProvider StateProvider 15 | // folder that contains migrations files 16 | migrationsPath string 17 | // actual absolute location of the configration file (gmig.yaml) 18 | configurationPath string 19 | } 20 | 21 | func getMigrationContext(c *cli.Context) (ctx migrationContext, err error) { 22 | pathToConfig := c.Args().First() 23 | if len(pathToConfig) == 0 { 24 | err = fmt.Errorf("missing path containing gmig.yaml in command line") 25 | return 26 | } 27 | stateProvider, err := getStateProvider(c) 28 | if err != nil { 29 | return 30 | } 31 | err = gcloudConfigSetProject(stateProvider.Config()) 32 | if err != nil { 33 | return 34 | } 35 | lastApplied, err := stateProvider.LoadState() 36 | if err != nil { 37 | return 38 | } 39 | ctx.stateProvider = stateProvider 40 | fullPathToConfig, err := filepath.Abs(pathToConfig) 41 | if err != nil { 42 | return 43 | } 44 | ctx.configurationPath = fullPathToConfig 45 | ctx.migrationsPath = filepath.Dir(fullPathToConfig) 46 | // see if flag overrides this 47 | if migrationsHolder := c.String("migrations"); len(migrationsHolder) > 0 { 48 | newPath, perr := filepath.Abs(migrationsHolder) 49 | if ctx.config().verbose { 50 | log.Printf("override migrations path with [%s] from [%s] to [%s] err:[%v]\n", migrationsHolder, ctx.migrationsPath, newPath, perr) 51 | } 52 | if perr != nil { 53 | return 54 | } 55 | ctx.migrationsPath = newPath 56 | } 57 | if ctx.config().verbose { 58 | log.Println("reading migrations from", ctx.migrationsPath) 59 | } 60 | ctx.lastApplied = lastApplied 61 | if len(lastApplied) > 0 { 62 | e := checkExists(filepath.Join(ctx.migrationsPath, lastApplied)) 63 | if e != nil { 64 | err = e 65 | return 66 | } 67 | } 68 | return 69 | } 70 | 71 | func (m migrationContext) config() Config { 72 | return m.stateProvider.Config() 73 | } 74 | 75 | func (m migrationContext) shellEnv() (envs []string) { 76 | envs = m.config().shellEnv() 77 | envs = append(envs, fmt.Sprintf("%s=%s", "GMIG_CONFIG_DIR", m.configurationPath)) 78 | return 79 | } 80 | -------------------------------------------------------------------------------- /migration_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | "testing" 9 | 10 | "github.com/urfave/cli" 11 | "gopkg.in/yaml.v2" 12 | ) 13 | 14 | var one = ` 15 | do: 16 | - going up 17 | # comment for down 18 | undo: 19 | - going down 20 | ` 21 | 22 | func TestParseMigration(t *testing.T) { 23 | var m Migration 24 | if err := yaml.Unmarshal([]byte(one), &m); err != nil { 25 | t.Error(err) 26 | } 27 | if got, want := m.DoSection[0], "going up"; got != want { 28 | t.Errorf("got [%v] want [%v]", got, want) 29 | } 30 | if got, want := m.UndoSection[0], "going down"; got != want { 31 | t.Errorf("got [%v] want [%v]", got, want) 32 | } 33 | } 34 | 35 | func TestSetupShellScriptNotVerbose(t *testing.T) { 36 | 37 | want := `#!/bin/bash 38 | # temporary gmig execution script 39 | set -e -v` 40 | 41 | if got := setupShellScript(false); got != want { 42 | t.Errorf("got [%v] want [%v]", got, want) 43 | } 44 | } 45 | 46 | func TestSetupShellScriptVerbose(t *testing.T) { 47 | 48 | want := `#!/bin/bash 49 | # temporary gmig execution script 50 | set -e -x` 51 | 52 | if got := setupShellScript(true); got != want { 53 | t.Errorf("got [%v] want [%v]", got, want) 54 | } 55 | } 56 | 57 | func TestNewFilenameWithIndex(t *testing.T) { 58 | wd, _ := os.Getwd() 59 | dir, err := ioutil.TempDir("", "testing") 60 | if err != nil { 61 | log.Fatal(err) 62 | } 63 | defer os.RemoveAll(dir) // clean up 64 | // change and restore finally 65 | if err := os.Chdir(dir); err != nil { 66 | return 67 | } 68 | defer os.Chdir(wd) 69 | desc := "first migration" 70 | want := "010_first_migration.yaml" 71 | if got := NewFilenameWithIndex(desc); got != want { 72 | t.Errorf("NewFilenameWithIndex(%s) = %v, want %v", desc, got, want) 73 | } 74 | tmpfn := filepath.Join(dir, "20181026t183700_starts_with_timestamp.yaml") 75 | if err := ioutil.WriteFile(tmpfn, []byte(""), 0444); err != nil { 76 | log.Fatal(err) 77 | } 78 | desc = "first after timestamp" 79 | want = "300_first_after_timestamp.yaml" 80 | if got := NewFilenameWithIndex(desc); got != want { 81 | t.Errorf("NewFilenameWithIndex(%s) = %v, want %v", desc, got, want) 82 | } 83 | tmpfn = filepath.Join(dir, "400_starts_with_high_index.yaml") 84 | if err := ioutil.WriteFile(tmpfn, []byte(""), 0444); err != nil { 85 | log.Fatal(err) 86 | } 87 | desc = "first after high index" 88 | want = "405_first_after_high_index.yaml" 89 | if got := NewFilenameWithIndex(desc); got != want { 90 | t.Errorf("NewFilenameWithIndex(%s) = %v, want %v", desc, got, want) 91 | } 92 | tmpfn = filepath.Join(dir, "unexpected_yaml_in_directory.yaml") 93 | if err := ioutil.WriteFile(tmpfn, []byte(""), 0444); err != nil { 94 | log.Fatal(err) 95 | } 96 | desc = "potentially unexpected naming" 97 | want = "010_potentially_unexpected_naming.yaml" 98 | if got := NewFilenameWithIndex(desc); got != want { 99 | t.Errorf("NewFilenameWithIndex(%s) = %v, want %v", desc, got, want) 100 | } 101 | } 102 | 103 | func TestEvaluateCondition(t *testing.T) { 104 | envs := []string{"ZONE=A", "PROJECT=B"} 105 | ok, err := evaluateCondition(`PROJECT == "B"`, envs) 106 | if err != nil { 107 | log.Fatal(err) 108 | } 109 | if got, want := ok, true; got != want { 110 | t.Errorf("got [%v] want [%v]", got, want) 111 | } 112 | } 113 | 114 | func TestMigrationContextWithMigrationsOverride(t *testing.T) { 115 | t.Skip() 116 | c := new(cli.Context) 117 | c.Set("migrations", "/tmp/here") 118 | ctx, err := getMigrationContext(c) 119 | if err != nil { 120 | t.Fatal(err) 121 | } 122 | t.Log(ctx) 123 | } 124 | -------------------------------------------------------------------------------- /state_provider.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "io/ioutil" 5 | "log" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/emicklei/tre" 10 | "github.com/urfave/cli" 11 | ) 12 | 13 | // StateProvider knowns how to load state. 14 | type StateProvider interface { 15 | // LoadState returns the last applied migration 16 | LoadState() (string, error) 17 | SaveState(filename string) error 18 | Config() Config 19 | } 20 | 21 | // FileStateProvider use a local file to store state (last migration applied). 22 | type FileStateProvider struct { 23 | Configuration Config 24 | tempDir string 25 | } 26 | 27 | // for testing 28 | var osTempDir = os.TempDir 29 | 30 | func NewFileStateProvider(c Config) FileStateProvider { 31 | return FileStateProvider{ 32 | Configuration: c, 33 | tempDir: osTempDir(), 34 | } 35 | } 36 | 37 | func (l FileStateProvider) stateFilename() string { 38 | return filepath.Join(l.tempDir, l.Configuration.LastMigrationObjectName) 39 | } 40 | 41 | // LoadState implements StateProvider 42 | func (l FileStateProvider) LoadState() (string, error) { 43 | if l.Configuration.verbose { 44 | d, _ := os.Getwd() 45 | log.Println("reading local copy", l.stateFilename(), ",cwd=", d) 46 | } 47 | data, err := os.ReadFile(l.stateFilename()) 48 | return string(data), tre.New(err, "error reading state", "tempDir", l.tempDir, "lastMigration", l.Configuration.LastMigrationObjectName) 49 | } 50 | 51 | // SaveState implements StateProvider 52 | func (l FileStateProvider) SaveState(filename string) error { 53 | if l.Configuration.verbose { 54 | d, _ := os.Getwd() 55 | log.Println("writing local copy", l.stateFilename(), ",cwd=", d) 56 | } 57 | return ioutil.WriteFile(l.stateFilename(), []byte(filename), os.ModePerm) 58 | } 59 | 60 | // Config implements StateProvider 61 | func (l FileStateProvider) Config() Config { 62 | return l.Configuration 63 | } 64 | 65 | // for testing 66 | var osRemove = os.Remove 67 | 68 | // DeleteState implements StateProvider 69 | func (l FileStateProvider) DeleteState() { 70 | if l.Configuration.verbose { 71 | d, _ := os.Getwd() 72 | log.Println("deleting local copy", l.stateFilename(), ",cwd=", d) 73 | } 74 | osRemove(l.stateFilename()) 75 | } 76 | 77 | // read it once 78 | var currentStateProvider StateProvider 79 | 80 | func getStateProvider(c *cli.Context) (StateProvider, error) { 81 | if currentStateProvider != nil { 82 | return currentStateProvider, nil 83 | } 84 | verbose := c.GlobalBool("v") 85 | pathToConfig := c.Args().First() 86 | cfg, err := TryToLoadConfig(pathToConfig) 87 | if verbose && err == nil { 88 | abs, _ := filepath.Abs(cfg.filename) 89 | log.Println("loading configuration from", abs) 90 | } 91 | if err != nil { 92 | workdir, _ := os.Getwd() 93 | abs := "?" 94 | if cfg != nil { 95 | abs, _ = filepath.Abs(cfg.filename) 96 | } 97 | return currentStateProvider, tre.New(err, "error loading configuration (did you init?)", "path", pathToConfig, "workdir", workdir, "location", abs) 98 | } 99 | cfg.verbose = cfg.verbose || verbose 100 | currentStateProvider = NewGCS(*cfg) 101 | return currentStateProvider, nil 102 | } 103 | -------------------------------------------------------------------------------- /test/010_one.yaml: -------------------------------------------------------------------------------- 1 | # one 2 | 3 | do: 4 | - echo "one up" 5 | undo: 6 | - echo "one down" 7 | view: 8 | - echo "one view" 9 | -------------------------------------------------------------------------------- /test/020_two.yaml: -------------------------------------------------------------------------------- 1 | # two 2 | 3 | do: 4 | - echo "two up" 5 | undo: 6 | - echo "two down" 7 | -------------------------------------------------------------------------------- /test/030_three.yaml: -------------------------------------------------------------------------------- 1 | # three 2 | 3 | do: 4 | - echo "three up" 5 | undo: 6 | - echo "three down" 7 | -------------------------------------------------------------------------------- /test/040_error.yaml: -------------------------------------------------------------------------------- 1 | # error 2 | 3 | do: 4 | - abcde -------------------------------------------------------------------------------- /test/050_conditional.yaml: -------------------------------------------------------------------------------- 1 | # conditional 2 | 3 | if: ENV == "A" 4 | do: 5 | - echo -------------------------------------------------------------------------------- /test/060_conditional_fail.yaml: -------------------------------------------------------------------------------- 1 | # conditional 2 | 3 | if: A == "ENV" 4 | do: 5 | - echo -------------------------------------------------------------------------------- /test/demo/gmig.json: -------------------------------------------------------------------------------- 1 | { 2 | "project":"demo", 3 | "bucket":"bucket", 4 | "state": "state", 5 | "env": { 6 | "ENV": "A" 7 | } 8 | } -------------------------------------------------------------------------------- /test/demo/nested/gmig.yaml: -------------------------------------------------------------------------------- 1 | project: demo 2 | bucket: bucket 3 | state: state 4 | env: 5 | ENV: A -------------------------------------------------------------------------------- /test/gmig.json: -------------------------------------------------------------------------------- 1 | { 2 | "bucket": "gmig-demo-test", 3 | "state": ".gmig-last-migration", 4 | "verbose": false 5 | } -------------------------------------------------------------------------------- /test/template_test.txt: -------------------------------------------------------------------------------- 1 | {{ env "PATH" }} -------------------------------------------------------------------------------- /utils.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bufio" 5 | "errors" 6 | "fmt" 7 | "log" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "regexp" 12 | "strings" 13 | "time" 14 | 15 | "github.com/urfave/cli" 16 | ) 17 | 18 | func printError(args ...interface{}) { 19 | log.Println(append([]interface{}{"\033[1;31mERROR:\033[0m"}, args...)...) 20 | } 21 | 22 | func printWarning(args ...interface{}) { 23 | log.Println(append([]interface{}{"\033[1;31mWARNING:\033[0m"}, args...)...) 24 | } 25 | 26 | var errAbort = errors.New("gmig ABORTED") 27 | 28 | func checkExists(filename string) error { 29 | _, err := os.Stat(filename) 30 | if err == nil { 31 | return nil 32 | } 33 | abs, _ := filepath.Abs(filename) 34 | return fmt.Errorf("no such migration (wrong project?, git pull?) [%s]", abs) 35 | } 36 | 37 | // runCommand is wrapper for CombinedOutput to make this package easy testable. 38 | var runCommand = func(c *exec.Cmd) ([]byte, error) { 39 | return c.CombinedOutput() 40 | } 41 | 42 | func reportError(cfg Config, envs []string, action string, err error) error { 43 | log.Printf("executing [%s] failed, error: [%v]\n", action, err) 44 | 45 | fmt.Println() 46 | log.Println("reporting environment variables ...") 47 | for _, each := range envs { 48 | fmt.Println(each) 49 | } 50 | 51 | fmt.Println() 52 | log.Println("checking gmig config ...") 53 | fmt.Println(cfg.ToJSON()) 54 | 55 | fmt.Println() 56 | gcloudConfigList() 57 | return err 58 | } 59 | 60 | func started(c *cli.Context, action string) func() { 61 | v := c.GlobalBool("v") 62 | if !v { 63 | return func() {} 64 | } 65 | log.Println("gmig version", Version) 66 | log.Println("BEGIN", action) 67 | start := time.Now() 68 | return func() { log.Println("END", action, "completed in", time.Since(start)) } 69 | } 70 | 71 | func promptForYes(message string) bool { 72 | reader := bufio.NewReader(os.Stdin) 73 | fmt.Print(message) 74 | yn, _ := reader.ReadString('\n') 75 | return strings.HasPrefix(yn, "Y") || strings.HasPrefix(yn, "y") 76 | } 77 | 78 | // 20180227t140600_permit_infra_manager_to_deploy_to_gateway_cluster.yaml 79 | // 2018-02-27 14:06:00 permit infra manager to deploy to gateway cluster 80 | func pretty(filename string) string { 81 | 82 | fn := strings.Replace(strings.TrimSuffix(filename, filepath.Ext(filename)), "_", " ", -1) 83 | 84 | if len(fn) < 16 { 85 | return fn 86 | } 87 | 88 | // 20060102t150405 is used as a sample format, see https://golang.org/pkg/time/#Parse 89 | _, err := time.Parse("20060102t150405", fn[0:15]) 90 | if err != nil { 91 | return fn 92 | } 93 | 94 | return fmt.Sprintf("%s-%s-%s %s:%s:%s %s", 95 | filename[0:4], 96 | filename[4:6], 97 | filename[6:8], 98 | filename[9:11], 99 | filename[11:13], 100 | filename[13:15], 101 | fn[16:]) 102 | } 103 | 104 | func isYamlFile(filename string) bool { 105 | ext := filepath.Ext(filename) 106 | return ext == ".yaml" || ext == ".yml" 107 | } 108 | 109 | var regexpIndex, _ = regexp.Compile("^[0-9]{3}_") 110 | var regexpTimestamp, _ = regexp.Compile("^[0-9]{8}t[0-9]{6}_") 111 | -------------------------------------------------------------------------------- /utils_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "os/exec" 5 | "testing" 6 | ) 7 | 8 | type commandCapturer struct { 9 | args [][]string 10 | output []byte 11 | err error 12 | } 13 | 14 | func (c *commandCapturer) runCommand(cmd *exec.Cmd) ([]byte, error) { 15 | c.args = append(c.args, cmd.Args) 16 | return c.output, c.err 17 | } 18 | 19 | var dateTests = []struct { 20 | in string 21 | out string 22 | }{ 23 | {"20180227t140600_permit_infra_manager_to_deploy_to_gateway_cluster.yaml", "2018-02-27 14:06:00 permit infra manager to deploy to gateway cluster"}, 24 | {"20161230t235959_permit_infra_manager_to_deploy_to_gateway_cluster.yaml", "2016-12-30 23:59:59 permit infra manager to deploy to gateway cluster"}, 25 | {"20170604t031224_permit_infra_manager_to_deploy_to_gateway_cluster.yaml", "2017-06-04 03:12:24 permit infra manager to deploy to gateway cluster"}, 26 | {"20180101t000000_permit_infra_manager_to_deploy_to_gateway_cluster.yaml", "2018-01-01 00:00:00 permit infra manager to deploy to gateway cluster"}, 27 | {"20011122t090620_permit_infra_manager_to_deploy_to_gateway_cluster.yaml", "2001-11-22 09:06:20 permit infra manager to deploy to gateway cluster"}, 28 | {"permit_infra_manager_to_deploy_to_gateway_cluster.yaml", "permit infra manager to deploy to gateway cluster"}, 29 | {"permit.yaml", "permit"}, 30 | } 31 | 32 | func TestPrettyPrint(t *testing.T) { 33 | for _, tt := range dateTests { 34 | actual := pretty(tt.in) 35 | if actual != tt.out { 36 | t.Errorf("pretty(%v): expected %v, actual %v", tt.in, tt.out, actual) 37 | } 38 | } 39 | } 40 | --------------------------------------------------------------------------------