├── .github ├── build ├── main.workflow ├── run-tests.sh └── workflows │ └── go.yml ├── .gitignore ├── .pre-commit-config.yaml ├── HomebrewFormula └── tfvars-annotations.rb ├── LICENSE ├── README.md ├── examples ├── project1-terragrunt │ └── eu-west-1 │ │ ├── app │ │ ├── full.tfvars │ │ ├── main.tf │ │ └── terraform.tfvars │ │ └── core │ │ ├── main.tf │ │ └── terraform.tfvars └── project2-terraform │ ├── app │ └── terraform.tfvars │ └── vpc │ ├── public_subnets │ └── vpc_id ├── go.mod ├── go.sum ├── hcl_ast.go ├── main.go ├── main_test.go └── util └── util.go /.github/build: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # The basename of our binary 4 | BASE="tfvars-annotations" 5 | 6 | # Save our directory, since we build in child-directories later. 7 | D=$(pwd) 8 | 9 | # 10 | # We build on multiple platforms/archs 11 | # 12 | BUILD_PLATFORMS="linux windows darwin freebsd" 13 | BUILD_ARCHS="amd64 386" 14 | 15 | # For each platform 16 | for OS in ${BUILD_PLATFORMS[@]}; do 17 | 18 | # For each arch 19 | for ARCH in ${BUILD_ARCHS[@]}; do 20 | 21 | cd ${D} 22 | 23 | # Setup a suffix for the binary 24 | SUFFIX="${OS}" 25 | 26 | # i386 is better than 386 27 | if [ "$ARCH" = "386" ]; then 28 | SUFFIX="${SUFFIX}-i386" 29 | else 30 | SUFFIX="${SUFFIX}-${ARCH}" 31 | fi 32 | 33 | # Windows binaries should end in .EXE 34 | if [ "$OS" = "windows" ]; then 35 | SUFFIX="${SUFFIX}.exe" 36 | fi 37 | 38 | echo "Building for ${OS} [${ARCH}] -> ${BASE}-${SUFFIX}" 39 | 40 | # Run the build 41 | export GOARCH=${ARCH} 42 | export GOOS=${OS} 43 | export CGO_ENABLED=0 44 | 45 | # Build the main-binary 46 | go build -ldflags "-X main.buildVersion=$(git describe --tags 2>/dev/null || echo 'master')" -o "${BASE}-${SUFFIX}" 47 | 48 | done 49 | done -------------------------------------------------------------------------------- /.github/main.workflow: -------------------------------------------------------------------------------- 1 | # pushes trigger the testsuite 2 | workflow "Push Event" { 3 | on = "push" 4 | resolves = ["Test"] 5 | } 6 | 7 | # pull-requests trigger the testsuite 8 | workflow "Pull Request" { 9 | on = "pull_request" 10 | resolves = ["Test"] 11 | } 12 | 13 | # releases trigger new binary artifacts 14 | workflow "Handle Release" { 15 | on = "release" 16 | resolves = ["Upload"] 17 | } 18 | 19 | ## 20 | ## The actions 21 | ## 22 | 23 | ## 24 | ## Run the test-cases, via .github/run-tests.sh 25 | ## 26 | action "Test" { 27 | uses = "skx/github-action-tester@master" 28 | } 29 | 30 | ## 31 | ## Build the binaries, via .github/build, then upload them. 32 | ## 33 | action "Upload" { 34 | uses = "skx/github-action-publish-binaries@master" 35 | args = "*-*" 36 | secrets = ["GITHUB_TOKEN"] 37 | } 38 | -------------------------------------------------------------------------------- /.github/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | # Install the lint-tool, and the shadow-tool 4 | go get -u golang.org/x/lint/golint 5 | go get -u golang.org/x/tools/go/analysis/passes/shadow/cmd/shadow 6 | 7 | # At this point failures cause aborts 8 | set -e 9 | 10 | # Run the linter 11 | echo "Launching linter .." 12 | golint -set_exit_status ./... 13 | echo "Completed linter .." 14 | 15 | # Run the shadow-checker 16 | echo "Launching shadowed-variable check .." 17 | go vet -vettool=$(which shadow) ./... 18 | echo "Completed shadowed-variable check .." 19 | 20 | # Run golang tests 21 | go test ./... -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | on: [push] 3 | jobs: 4 | 5 | build: 6 | name: Build 7 | runs-on: ubuntu-latest 8 | steps: 9 | 10 | - name: Set up Go 1.12 11 | uses: actions/setup-go@v1 12 | with: 13 | go-version: 1.12 14 | id: go 15 | 16 | - name: Check out code into the Go module directory 17 | uses: actions/checkout@v1 18 | 19 | - name: Get dependencies 20 | run: | 21 | go get -v -t -d ./... 22 | if [ -f Gopkg.toml ]; then 23 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 24 | dep ensure 25 | fi 26 | 27 | - name: Build 28 | run: go build -v . 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | 3 | .terragrunt-cache 4 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: git://github.com/troian/pre-commit-golang 3 | rev: master 4 | hooks: 5 | - id: go-imports 6 | - repo: git://github.com/pre-commit/pre-commit-hooks 7 | rev: v2.1.0 8 | hooks: 9 | - id: check-merge-conflict 10 | -------------------------------------------------------------------------------- /HomebrewFormula/tfvars-annotations.rb: -------------------------------------------------------------------------------- 1 | class TfvarsAnnotations < Formula 2 | desc "Update values in terraform.tfvars using annotations" 3 | homepage "https://github.com/antonbabenko/tfvars-annotations" 4 | 5 | # Update these when a new version is released 6 | url "https://github.com/antonbabenko/tfvars-annotations/archive/v0.0.3.tar.gz" 7 | sha256 "d2f2e6afbba4b4901cbb6bb04713fc17769df9f07eb7675449ea011e4ac8cf3e" 8 | 9 | head "https://github.com/antonbabenko/tfvars-annotations.git" 10 | 11 | depends_on "go" => :build 12 | 13 | def install 14 | ENV["GOPATH"] = buildpath 15 | 16 | # Move the contents of the repo (which are currently in the buildpath) into 17 | # a go-style subdir, so we can build it without spewing deps everywhere. 18 | app_path = buildpath/"src/github.com/antonbabenko/tfvars-annotations" 19 | app_path.install Dir["*"] 20 | 21 | # Fetch the deps (into our temporary gopath) and build 22 | cd "src/github.com/antonbabenko/tfvars-annotations" do 23 | system "go", "get" 24 | system "go", "build", "-ldflags", "-X main.buildVersion='#{version}'" 25 | end 26 | 27 | # Install the resulting binary 28 | bin.install "bin/tfvars-annotations" 29 | end 30 | 31 | test do 32 | system "#{bin}/tfvars-annotations", "-version" 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Anton Babenko 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Update values in terraform.tfvars using annotations 2 | 3 | ## This project has become redundant (yay!) 4 | 5 | ## The same functionality is available natively in Terragrunt since [version 0.19.20](https://github.com/gruntwork-io/terragrunt/releases/tag/v0.19.20) (released 15th of August 2019). 6 | 7 | ### See [PR #3](https://github.com/antonbabenko/tfvars-annotations/pull/3) for the explanation and some extra unreleased code if you want to continue developing this for Terraform. 8 | 9 | --- 10 | 11 | [Terraform](https://www.terraform.io/) is awesome! 12 | 13 | As of today, Terraform 0.11 and 0.12 support only static (known, fixed, already computed) values in `tfvars` files. There is no way to use Terraform interpolation functions, or data-sources inside `tfvars` files in Terraform to update values. 14 | 15 | While working on [modules.tf](https://github.com/antonbabenko/modules.tf-lambda) (a tool which converts visual diagrams created with [Cloudcraft.co](https://cloudcraft.co/) into Terraform configurations), I had a need to generate code which would chain invocations of [Terraform AWS modules](https://github.com/terraform-aws-modules) and pass arguments between them without requiring any extra Terraform code as a glue. [Terragrunt](https://github.com/gruntwork-io/terragrunt) is a great fit for this, it allows to reduce amount of Terraform configurations by reusing Terraform modules and providing arguments as values in `tfvars` files. 16 | 17 | Some languages I know have concepts like annotations and decorators, so at first I made a [shell script](https://github.com/antonbabenko/modules.tf-lambda/blob/v1.2.0/templates/terragrunt-common-layer/common/scripts/update_dynamic_values_in_tfvars.sh) which replaced values in `tfvars` based on annotations and was called by Terragrunt hooks. `tfvars-annotations` shares the same goal and it has no external dependencies (except `terraform` or `terragrunt`). 18 | 19 | 20 | ## Use cases 21 | 22 | 1. modules.tf and Terragrunt (recommended) 23 | 1. Terraform (not implemented yet) 24 | 1. General example - AMI ID, AWS region 25 | 26 | ## Features 27 | 28 | 1. Supported annotations: 29 | - [x] terragrunt_output: 30 | - `@tfvars:terragrunt_output.vpc.vpc_id` 31 | - `@tfvars:terragrunt_output.security-group.this_security_group_id` 32 | - [ ] terraform_output 33 | - [ ] data-sources generic 34 | 1. Type wrapping: 35 | - `to_list`: Wrap original value with `[]` to make it it as a list 36 | 37 | ## How to use 38 | 39 | Run `tfvars-annotations` before `terraform plan, apply, refresh`. 40 | 41 | It will process tfvars file in the current directory and set updated values. 42 | 43 | E.g.: 44 | 45 | $ tfvars-annotations examples/project1-terragrunt/eu-west-1/app 46 | $ terraform plan 47 | 48 | ## How to disable processing entirely 49 | 50 | Put `@tfvars:disable_annotations` anywhere in the `terraform.tfvars` to not process the file. 51 | 52 | ## Examples 53 | 54 | See `examples` for some basics. 55 | 56 | ## To-do 57 | 58 | 1. Get values from other sources: 59 | - data sources generic 60 | - aws_account_id or aws_region data sources 61 | 2. terragrunt_outputs from stacks: 62 | - in any folder 63 | - in current region 64 | 3. cache values unless stack is changed/updated 65 | 4. functions (limit(2), to_list) 66 | 5. rewrite in go (invoke like this => update_dynamic_values_in_tfvars ${get_parent_tfvars_dir()}/${path_relative_to_include()}) 67 | 6. make it much faster, less verbose 68 | 7. add dry-run flag 69 | 8. Proposed syntax: 70 | 71 | - `@tfvars:terragrunt_output.security-group_5.this_security_group_id.to_list` 72 | 73 | - `@tfvars:terragrunt_output.["eu-west-1/security-group_5"].this_security_group_id.to_list` 74 | 75 | - `@tfvars:terragrunt_output.["global/route53-zones"].zone_id` 76 | 77 | - `@tfvars:terragrunt_data.aws_region.zone_id` 78 | 79 | - `@tfvars:terragrunt_data.aws_region[{current=true}].zone_id` 80 | 81 | ## Bugs 82 | 83 | 1. Add support for `maps` (and lists of maps). Strange bugs with rendering comments in wrong places. 84 | 85 | ## Installation 86 | 87 | On OSX, install it with Homebrew (not enough github stars to get it to the official repo): 88 | 89 | ``` 90 | brew install -s HomebrewFormula/tfvars-annotations.rb 91 | ``` 92 | 93 | Alternatively, you can download a [release](https://github.com/antonbabenko/tfvars-annotations/releases) suitable for your platform and unzip it. Make sure the `tfvars-annotations` binary is executable, and you're ready to go. 94 | 95 | You can also install it like this: 96 | 97 | ``` 98 | go get github.com/antonbabenko/tfvars-annotations 99 | ``` 100 | 101 | Or run it from source: 102 | 103 | ``` 104 | go run . -debug examples/project1-terragrunt/eu-west-1/app 105 | go run . examples/project1-terragrunt/eu-west-1/app 106 | ``` 107 | 108 | ## Release 109 | 110 | 1. Make GitHub Release: `hub release create v0.0.3`. Then Github Actions will build binaries and attach them to Github release. 111 | 2. Update Homebrew version in `HomebrewFormula/tfvars-annotations.rb`. Install locally - `brew install -s HomebrewFormula/tfvars-annotations.rb` 112 | 113 | ## Authors 114 | 115 | This project is created and maintained by [Anton Babenko](https://github.com/antonbabenko) with the help from [different contributors](https://github.com/antonbabenko/tfvars-annotations/graphs/contributors). 116 | 117 | [![@antonbabenko](https://img.shields.io/twitter/follow/antonbabenko.svg?style=social&label=Follow%20@antonbabenko%20on%20Twitter)](https://twitter.com/antonbabenko) 118 | 119 | 120 | ## License 121 | 122 | This work is licensed under MIT License. See LICENSE for full details. 123 | 124 | Copyright (c) 2019 Anton Babenko 125 | -------------------------------------------------------------------------------- /examples/project1-terragrunt/eu-west-1/app/full.tfvars: -------------------------------------------------------------------------------- 1 | terragrunt = { 2 | terraform = { 3 | source = "." 4 | } 5 | } 6 | 7 | ################ 8 | # Static values 9 | ################ 10 | 11 | title = "This value is not going to be changed by tfvars-annotations" 12 | 13 | ################# 14 | # Dynamic values 15 | ################# 16 | 17 | name = "Anton Babenko" # @tfvars:terragrunt_output.core.name 18 | 19 | score = "37" # @tfvars:terragrunt_output.core.score 20 | 21 | name_as_list = ["Anton Babenko"] # @tfvars:terragrunt_output.core.name.to_list 22 | 23 | love_sailing = "true" # @tfvars:terragrunt_output.core.love_sailing 24 | 25 | understand_how_to_use_twitter = "false" # @tfvars:terragrunt_output.core.understand_how_to_use_twitter 26 | 27 | languages = [ 28 | "ukrainian", 29 | "russian", 30 | "english", 31 | "norwegian", 32 | "spanish", 33 | ] # @tfvars:terragrunt_output.core.languages 34 | 35 | ############### 36 | # Compositions 37 | ############### 38 | 39 | custom_map = { 40 | Score = "37" # @tfvars:terragrunt_output.core.score 41 | Name = "Anton Babenko" # @tfvars:terragrunt_output.core.name 42 | MixedValue = "" # @ tfvars:terragrunt_output.core.mixed_value <-- same reason as below. Maps are tricky. 43 | } 44 | 45 | ###### 46 | # These don't work yet because there are `maps` inside of them. 47 | ###### 48 | list_of_properties = "" # @ tfvars:terragrunt_output.core.list_of_properties 49 | 50 | map_of_properties = "" # @ tfvars:terragrunt_output.core.map_of_properties 51 | 52 | mixed_value = "" # @ tfvars:terragrunt_output.core.mixed_value 53 | -------------------------------------------------------------------------------- /examples/project1-terragrunt/eu-west-1/app/main.tf: -------------------------------------------------------------------------------- 1 | # Content of Terraform module 2 | -------------------------------------------------------------------------------- /examples/project1-terragrunt/eu-west-1/app/terraform.tfvars: -------------------------------------------------------------------------------- 1 | terragrunt = { 2 | terraform = { 3 | source = "." 4 | } 5 | } 6 | 7 | ################ 8 | # Static values 9 | ################ 10 | 11 | title = "This value is not going to be changed by tfvars-annotations" 12 | 13 | ################# 14 | # Dynamic values 15 | ################# 16 | 17 | name = "Anton Babenko" # @tfvars:terragrunt_output.core.name 18 | 19 | score = "37" # @tfvars:terragrunt_output.core.score 20 | 21 | name_as_list = ["Anton Babenko"] # @tfvars:terragrunt_output.core.name.to_list 22 | 23 | love_sailing = "true" # @tfvars:terragrunt_output.core.love_sailing 24 | 25 | understand_how_to_use_twitter = "false" # @tfvars:terragrunt_output.core.understand_how_to_use_twitter 26 | 27 | languages = [ 28 | "ukrainian", 29 | "russian", 30 | "english", 31 | "norwegian", 32 | "spanish", 33 | ] # @tfvars:terragrunt_output.core.languages 34 | 35 | ############### 36 | # Compositions 37 | ############### 38 | 39 | custom_map = { 40 | Score = "37" # @tfvars:terragrunt_output.core.score 41 | Name = "Anton Babenko" # @tfvars:terragrunt_output.core.name 42 | MixedValue = "" # @ tfvars:terragrunt_output.core.mixed_value <-- same reason as below. Maps are tricky. 43 | } 44 | 45 | ###### 46 | # These don't work yet because there are `maps` inside of them. 47 | ###### 48 | list_of_properties = "" # @ tfvars:terragrunt_output.core.list_of_properties 49 | 50 | map_of_properties = "" # @ tfvars:terragrunt_output.core.map_of_properties 51 | 52 | mixed_value = "" # @ tfvars:terragrunt_output.core.mixed_value 53 | -------------------------------------------------------------------------------- /examples/project1-terragrunt/eu-west-1/core/main.tf: -------------------------------------------------------------------------------- 1 | output "name" { 2 | value = "Anton Babenko" 3 | } 4 | 5 | output "score" { 6 | value = "37" 7 | } 8 | 9 | output "love_sailing" { 10 | value = "true" 11 | } 12 | 13 | output "understand_how_to_use_twitter" { 14 | value = "false" 15 | } 16 | 17 | output "languages" { 18 | value = ["ukrainian", "russian", "english", "norwegian", "spanish"] 19 | } 20 | 21 | output "map_of_properties" { 22 | value = { 23 | Name = "Anton Babenko" 24 | Age = 34 25 | LoveSailing = true 26 | } 27 | } 28 | 29 | output "list_of_properties" { 30 | value = [ 31 | { 32 | Name = "Anton Babenko" 33 | Age = 34 34 | LoveSailing = true 35 | UnderstandHowToUseTwitter = false 36 | }, 37 | { 38 | Name = "Kapitoshka" 39 | Age = 123 40 | LoveSailing = false 41 | }, 42 | ] 43 | } 44 | 45 | output "mixed_value" { 46 | value = [ 47 | "This is just a string", 48 | [ 49 | { 50 | Name = "Anton Babenko" 51 | Age = 34 52 | LoveSailing = true 53 | UnderstandHowToUseTwitter = false 54 | }, 55 | ], 56 | { 57 | Github = "antonbabenko" 58 | Twitter = "antonbabenko" 59 | }, 60 | ] 61 | } 62 | 63 | // Failing values ("=" inside values): 64 | output "failing_values_list" { 65 | value = ["ukrainian = 100", "english", "unknown"] 66 | } 67 | 68 | output "failing_values_map" { 69 | value = { 70 | Name = "Anton Babenko" 71 | Age = 34 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /examples/project1-terragrunt/eu-west-1/core/terraform.tfvars: -------------------------------------------------------------------------------- 1 | terragrunt = { 2 | terraform = { 3 | source = "." 4 | } 5 | } 6 | 7 | name = "test" 8 | -------------------------------------------------------------------------------- /examples/project2-terraform/app/terraform.tfvars: -------------------------------------------------------------------------------- 1 | # @ modulestf:disable_values_updates 2 | 3 | vpc_id = "vpc-443a8116aae25c7e9" # @modulestf:terraform_output.vpc.vpc_id 4 | 5 | public_subnets = ["subnet-297e8b509d8aeebfe","subnet-3f831847e5802071c","subnet-6c6f959a9063d32b2"] # @modulestf:terraform_output.vpc.public_subnets 6 | 7 | something = "" # @modulestf:terraform_output.something.id 8 | 9 | vpc4_id = "vpc-443a8116aae25c7e9" # @modulestf:terraform_output.vpc.vpc_id 10 | 11 | the end! -------------------------------------------------------------------------------- /examples/project2-terraform/vpc/public_subnets: -------------------------------------------------------------------------------- 1 | { 2 | "sensitive": false, 3 | "type": "list", 4 | "value": [ 5 | "subnet-297e8b509d8aeebfe", 6 | "subnet-3f831847e5802071c", 7 | "subnet-6c6f959a9063d32b2" 8 | ] 9 | } -------------------------------------------------------------------------------- /examples/project2-terraform/vpc/vpc_id: -------------------------------------------------------------------------------- 1 | { 2 | "sensitive": false, 3 | "type": "string", 4 | "value": "vpc-443a8116aae25c7e9" 5 | } -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/antonbabenko/tfvars-annotations 2 | 3 | go 1.12 4 | 5 | require ( 6 | github.com/davecgh/go-spew v1.1.1 7 | github.com/hashicorp/hcl v1.0.0 8 | github.com/pkg/errors v0.8.1 9 | github.com/rodaine/hclencoder v0.0.0-20190213202847-fb9757bb536e 10 | github.com/sirupsen/logrus v1.4.1 11 | golang.org/x/sys v0.0.0-20190425145619-16072639606e // indirect 12 | ) 13 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 2 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= 4 | github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= 5 | github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= 6 | github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= 7 | github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= 8 | github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 9 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 10 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 11 | github.com/rodaine/hclencoder v0.0.0-20190213202847-fb9757bb536e h1:H9k4BAinsVIlHvkUQxmLq194u5Av7VofhxtPGBOAhAo= 12 | github.com/rodaine/hclencoder v0.0.0-20190213202847-fb9757bb536e/go.mod h1:hkWgI+PWPCjkVbx8rdk7GhVkpbc/zCLrY/9yF5xGSzI= 13 | github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= 14 | github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= 15 | github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 16 | github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= 17 | github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= 18 | golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 19 | golang.org/x/sys v0.0.0-20190425145619-16072639606e h1:4ktJgTV34+N3qOZUc5fAaG3Pb11qzMm3PkAoTAgUZ2I= 20 | golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 21 | -------------------------------------------------------------------------------- /hcl_ast.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "flag" 6 | "fmt" 7 | 8 | "github.com/antonbabenko/tfvars-annotations/util" 9 | "github.com/hashicorp/hcl" 10 | "github.com/hashicorp/hcl/hcl/ast" 11 | "github.com/hashicorp/hcl/hcl/printer" 12 | "github.com/sirupsen/logrus" 13 | 14 | //"github.com/hashicorp/hcl/hcl/token" 15 | "io/ioutil" 16 | "reflect" 17 | "regexp" 18 | "sort" 19 | "strings" 20 | 21 | "github.com/rodaine/hclencoder" 22 | 23 | "github.com/davecgh/go-spew/spew" 24 | ) 25 | 26 | var ( 27 | // String marker 28 | tfvarsDisableAnnotations = `@tfvars:disable_annotations` 29 | 30 | // Regexp 31 | tfvarsTerragruntOutputRegexp = regexp.MustCompile(`@tfvars:terragrunt_output\.[^ \n]+`) 32 | 33 | _ = spew.Config 34 | _ = fmt.Sprint() 35 | ) 36 | 37 | func parseContent(hclString *string) (*ast.File, error) { 38 | astf, err := hcl.Parse(*hclString) 39 | if err != nil { 40 | return nil, err 41 | } 42 | 43 | return astf, nil 44 | } 45 | 46 | func scanComments(astf *ast.File) (bool, []string) { 47 | comments := astf.Comments 48 | 49 | isDisabled := false 50 | keysFound := []string{} 51 | 52 | for _, commentList := range comments { 53 | for _, comment := range commentList.List { 54 | 55 | if strings.Contains(comment.Text, tfvarsDisableAnnotations) { 56 | isDisabled = true 57 | } 58 | 59 | allKeys := tfvarsTerragruntOutputRegexp.FindAllString(comment.Text, -1) 60 | 61 | keysFound = append(keysFound, allKeys...) 62 | } 63 | } 64 | 65 | sort.Strings(keysFound) 66 | 67 | keysFound = util.UniqueNonEmptyElementsOf(keysFound) 68 | 69 | log.Debugf("Found keys: %s", keysFound) 70 | 71 | return isDisabled, keysFound 72 | } 73 | 74 | func updateValuesInTfvarsFile(astf *ast.File, allKeyValues map[string]interface{}) (ast.File, []string) { 75 | 76 | var errors []string 77 | var hclContent string 78 | 79 | flag.Parse() 80 | 81 | if *debug == true { 82 | log.Level = logrus.TraceLevel 83 | } else { 84 | log.Level = logrus.InfoLevel 85 | } 86 | 87 | ast.Walk(astf.Node, func(n ast.Node) (ast.Node, bool) { 88 | if n == nil { 89 | return n, false 90 | } 91 | 92 | typeName := reflect.TypeOf(n).String() 93 | 94 | if typeName == "*ast.ObjectItem" { 95 | //log.Traceln("* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *") 96 | //log.Traceln("Node type=", typeName) 97 | 98 | //log.Traceln("Node=") 99 | //log.Traceln(spew.Sdump(n)) 100 | 101 | leadCommentText := "" 102 | lineCommentText := "" 103 | 104 | //spew.Dump(n.(*ast.ObjectItem).Keys[0]) 105 | 106 | leadComment := n.(*ast.ObjectItem).LeadComment 107 | if leadComment != nil { 108 | leadCommentText = leadComment.List[0].Text 109 | } 110 | 111 | lineComment := n.(*ast.ObjectItem).LineComment 112 | if lineComment != nil { 113 | lineCommentText = lineComment.List[0].Text 114 | log.Traceln("Found line comment:", lineCommentText) 115 | 116 | for key, value := range allKeyValues { 117 | 118 | if strings.Contains(lineCommentText, key) { 119 | 120 | //lineComment.List[0].Text = lineCommentText // + "!!!" 121 | 122 | currentVal := n.(*ast.ObjectItem).Val 123 | 124 | currentValPos := currentVal.Pos().Line 125 | //log.Traceln("Current line number of the value: ", currentValPos) 126 | 127 | valueType := reflect.TypeOf(value).String() // real value type 128 | //desiredValueType := reflect.TypeOf(value).String() // desired value type (to_list will set this to `list`) 129 | 130 | log.Tracef("Found line comment value to replace using value from key %s", key) 131 | 132 | log.Traceln(spew.Sdump(value)) 133 | log.Traceln(spew.Sdump(valueType)) 134 | 135 | hclBytes, err := hclencoder.Encode(value) 136 | 137 | if err != nil { 138 | log.Warnln("Error during hclencoder: ", err) 139 | } 140 | 141 | // Create HCL string with properly encoded values and with real number of newlines prefixed to be able to replace in current value 142 | prefixNewlines := strings.Repeat("\n", currentValPos-1) 143 | //hclString := strings.TrimSuffix(string(hclBytes), "\n") 144 | hclString := string(hclBytes) 145 | 146 | // @todo: Maps are not supported yet, because comments are placed in strange places 147 | if valueType == "map[string]interface {}" { 148 | //hclContent = prefixNewlines + `key = {` + hclString + `}` 149 | continue 150 | } else { 151 | 152 | split := strings.Split(key, ".") 153 | 154 | convertToType := "" 155 | 156 | if len(split) > 3 { 157 | convertToType = split[3] 158 | } 159 | 160 | if convertToType == "to_list" { 161 | hclContent = prefixNewlines + `key = [` + hclString + `]` 162 | } else { 163 | hclContent = prefixNewlines + `key = ` + hclString 164 | } 165 | 166 | } 167 | 168 | //log.Traceln("HCL content created from the new value to parse to AST: ", hclContent) 169 | astfNew, err2 := hcl.Parse(hclContent) 170 | 171 | if err2 != nil { 172 | log.Warnln(err2) 173 | continue 174 | } 175 | 176 | // Value from the first element item created earlier (new value) 177 | newVal := astfNew.Node.(*ast.ObjectList).Items[0].Val 178 | 179 | //log.Traceln("+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +") 180 | //log.Traceln("astfNew.Node=", spew.Sdump(astfNew.Node)) 181 | //log.Traceln("NEW VALUE:", spew.Sdump(newVal)) 182 | //log.Traceln("NEW VALUE OFFSET:", spew.Sdump(newVal.Pos().Offset)) 183 | //log.Traceln("OLD VALUE:", spew.Sdump(currentVal)) 184 | //log.Traceln("OLD VALUE OFFSET:", spew.Sdump(currentVal.Pos().Offset)) 185 | 186 | // Replacing old value 187 | n.(*ast.ObjectItem).Val = newVal 188 | } 189 | } 190 | 191 | } 192 | 193 | //spew.Dump(leadComment) 194 | //spew.Dump(lineComment) 195 | _ = leadCommentText 196 | _ = lineCommentText 197 | } 198 | 199 | return n, true 200 | }) 201 | 202 | if false { 203 | log.Traceln("= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =") 204 | 205 | spew.Dump(astf.Comments) 206 | 207 | for i, commentGroup := range astf.Comments { 208 | for _, comment := range commentGroup.List { 209 | 210 | log.Traceln("ORIGINAL i=", i, "; offset=", commentGroup.Pos().Offset, "; commentOffset=", comment.Pos().Offset, spew.Sdump(comment)) 211 | 212 | //lineNumber := comment.Pos().Line 213 | //var lineNumber, columnNumber, Offset int 214 | lineNumber := comment.Pos().Line 215 | columnNumber := commentGroup.Pos().Column 216 | Offset := commentGroup.Pos().Offset 217 | 218 | if i == 2 { 219 | lineNumber += 5 220 | //Offset += 10 // number of chars to shift? 221 | } 222 | 223 | if i == 4 { 224 | lineNumber += 5 225 | //Offset += 10 // number of chars to shift? 226 | } 227 | 228 | // @todo: Adjust Line to it bigger on the length of the previous block 229 | /*comment.Start = token.Pos{ 230 | Filename: "", 231 | Offset: Offset, 232 | Line: lineNumber, 233 | Column: columnNumber, 234 | }*/ 235 | 236 | fmt.Println(lineNumber, columnNumber, Offset) 237 | 238 | log.Traceln("NEW i=", i, "; offset=", comment.Pos().Offset, spew.Sdump(comment)) 239 | } 240 | } 241 | 242 | log.Traceln("= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =") 243 | spew.Dump(astf.Comments) 244 | 245 | } 246 | 247 | //log.Traceln("= = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = = =") 248 | //log.Traceln("Complete astf after update:", spew.Sdump(astf)) 249 | 250 | return *astf, errors 251 | } 252 | 253 | func fprintToFile(astf *ast.File, filename string) ([]byte, error) { 254 | 255 | var buf bytes.Buffer 256 | 257 | if err := printer.Fprint(&buf, astf); err != nil { 258 | return buf.Bytes(), err 259 | } 260 | 261 | // Add trailing newline to result to prevent from reformatting every time 262 | buf.WriteString("\n") 263 | if filename != "" { 264 | if err := ioutil.WriteFile(filename, buf.Bytes(), 0644); err != nil { 265 | return buf.Bytes(), err 266 | } 267 | } 268 | 269 | return buf.Bytes(), nil 270 | } 271 | -------------------------------------------------------------------------------- /main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "encoding/json" 5 | "flag" 6 | "fmt" 7 | "io/ioutil" 8 | "os" 9 | "os/exec" 10 | "path/filepath" 11 | "regexp" 12 | "strings" 13 | 14 | "github.com/antonbabenko/tfvars-annotations/util" 15 | 16 | "github.com/davecgh/go-spew/spew" 17 | "github.com/pkg/errors" 18 | 19 | "github.com/sirupsen/logrus" 20 | ) 21 | 22 | var ( 23 | version = flag.Bool("version", false, "print version information and exit") 24 | debug = flag.Bool("debug", false, "enable debug logging") 25 | 26 | // Main filename to work with 27 | tfvarsFile = "terraform.tfvars" 28 | 29 | // Dir where terragrunt cache lives 30 | terragruntCacheDir = ".terragrunt-cache" 31 | 32 | // Create a new instance of the logger. You can have any number of instances. 33 | log = logrus.New() 34 | 35 | // Deliberately uninitialized. See below. 36 | buildVersion string 37 | 38 | _ = spew.Config 39 | 40 | err error 41 | ) 42 | 43 | // versionInfo returns a string containing the version information of the 44 | // current build. It's empty by default, but can be included as part of the 45 | // build process by setting the main.buildVersion variable. 46 | func versionInfo() string { 47 | if buildVersion != "" { 48 | return buildVersion 49 | } 50 | 51 | return "unknown" 52 | } 53 | 54 | func main() { 55 | flag.Parse() 56 | 57 | if *debug == true { 58 | log.Level = logrus.DebugLevel 59 | } else { 60 | log.Level = logrus.InfoLevel 61 | } 62 | 63 | if *version == true { 64 | fmt.Printf("%s version %s\n", os.Args[0], versionInfo()) 65 | os.Exit(0) 66 | } 67 | 68 | // Relative path to original tfvars file 69 | tfvarsDir := flag.Arg(0) 70 | 71 | if tfvarsDir == "" { 72 | log.Errorf("Specify tfvars directory where %s is located", tfvarsFile) 73 | os.Exit(1) 74 | } 75 | 76 | if _, err = os.Stat(tfvarsDir); err != nil { 77 | log.Error(err) 78 | os.Exit(1) 79 | } 80 | 81 | // Full relative path to original tfvars file 82 | tfvarsFullpath := filepath.Join(tfvarsDir, tfvarsFile) 83 | 84 | // Relative path to destination ".terraform" working directory 85 | terraformWorkingDir := findWorkingDir(tfvarsDir) 86 | log.Infoln("Working dir: ", terraformWorkingDir) 87 | 88 | // Full relative path to destination tfvars file (inside .terragrunt-cache/.../.../.terraform) 89 | var terraformWorkingDirTfvarsFullPath = filepath.Join(terraformWorkingDir, tfvarsFile) 90 | 91 | // Map of all keys and values to replace in tfvars file 92 | allKeyValues := make(map[string]interface{}) 93 | 94 | log.Infof("Processing file: %s", tfvarsFullpath) 95 | log.Println() 96 | 97 | tfvarsContent, err := readTfvarsFile(tfvarsFullpath) 98 | if err != nil { 99 | log.Fatalf("Can't read file: %s", err) 100 | } 101 | 102 | astf, err := parseContent(&tfvarsContent) 103 | if err != nil { 104 | log.Fatalln("Can't parse content as HCL", err) 105 | } 106 | 107 | isDisabled, keysToReplace := scanComments(astf) 108 | if isDisabled { 109 | log.Fatalf("Dynamic update has been disabled in %s. Nothing to do.", tfvarsFile) 110 | 111 | } 112 | 113 | if len(keysToReplace) == 0 { 114 | log.Infoln("There are no keys to replace") 115 | } 116 | 117 | for _, key := range keysToReplace { 118 | log.Infof("Key: %s", key) 119 | 120 | split := strings.Split(key, ".") 121 | 122 | dirName := "" 123 | outputName := "" 124 | convertToType := "" 125 | 126 | if len(split) == 0 { 127 | continue 128 | } 129 | 130 | if len(split) > 1 { 131 | dirName = split[1] 132 | } 133 | 134 | if len(split) > 2 { 135 | outputName = split[2] 136 | } 137 | 138 | //if len(split) > 3 { 139 | // convertToType = split[3] 140 | //} 141 | 142 | workDir := filepath.Join(tfvarsDir, "../", dirName) 143 | //fmt.Println(workDir) 144 | 145 | resultValue, resultType, errResult := getResultFromTerragruntOutput(workDir, outputName) 146 | 147 | if errResult != nil { 148 | log.Warnf("Can't update value of %s in %s because key \"%s\"", key, tfvarsFullpath, outputName) 149 | log.Warnf("Error from terragrunt:", errResult) 150 | log.Println() 151 | } 152 | 153 | _ = resultType 154 | _ = convertToType 155 | 156 | // @todo: add support for to_list 157 | //if convertToType == "to_list" { 158 | // resultValue = fmt.Sprintf("[%s]", resultValue) 159 | //} 160 | 161 | allKeyValues[key] = resultValue 162 | 163 | log.Infof("Value: %s", spew.Sdump(resultValue)) 164 | //log.Infof("Value: %s", formattedResultValue) 165 | log.Infoln() 166 | log.Infoln() 167 | 168 | } 169 | 170 | log.Debugln("All key values:") 171 | log.Debugln(spew.Sdump(allKeyValues)) 172 | 173 | astfUpdated, err2 := updateValuesInTfvarsFile(astf, allKeyValues) 174 | 175 | if err2 != nil { 176 | log.Fatalf("%s: Can't replace all keys in %s", err2, tfvarsFullpath) 177 | } 178 | 179 | //spew.Dump(astfUpdated) 180 | 181 | tfvarsFullpathTmp := "" 182 | 183 | if !*debug { 184 | tfvarsFullpathTmp = tfvarsFullpath 185 | } 186 | 187 | hclFormatted, err := fprintToFile(&astfUpdated, tfvarsFullpathTmp) 188 | if err != nil { 189 | log.Fatalf("Can't fprint AST to file %s, Error: %s", tfvarsFullpathTmp, err) 190 | } 191 | 192 | log.Infoln("FINAL HCL:") 193 | log.Infoln(string(hclFormatted)) 194 | _ = hclFormatted 195 | 196 | log.Infoln() 197 | log.Infof("Copying updated %s into %s", tfvarsFullpath, terraformWorkingDirTfvarsFullPath) 198 | log.Infoln() 199 | 200 | _, err = util.CopyFile(tfvarsFullpath, terraformWorkingDirTfvarsFullPath) 201 | 202 | if err != nil { 203 | log.Fatalf("%s: Can't copy file to %s", err, terraformWorkingDirTfvarsFullPath) 204 | } 205 | 206 | log.Infoln("Done!") 207 | 208 | os.Exit(0) 209 | } 210 | 211 | func findWorkingDir(tfvarsDir string) string { 212 | 213 | var workingDir string 214 | 215 | _ = filepath.Walk(tfvarsDir, func(path string, info os.FileInfo, err error) error { 216 | if info.IsDir() && strings.Contains(path, terragruntCacheDir) && len(workingDir) == 0 { 217 | 218 | // eg: examples/project1-terragrunt/eu-west-1/app/.terragrunt-cache/F0pCE6ytQ7SNCsEA3BS4Wg57FJs/w9zgoLbGjuT9Afe34Zp8rkEMzXI 219 | if matched, _ := regexp.MatchString(terragruntCacheDir+`/[^/]+/[^/]+$`, path); matched { 220 | workingDir = path 221 | } 222 | } 223 | return nil 224 | }) 225 | 226 | return workingDir 227 | } 228 | 229 | func readTfvarsFile(tfvarsFullpath string) (string, error) { 230 | bytes, err := ioutil.ReadFile(tfvarsFullpath) 231 | if err != nil { 232 | return "", err 233 | } 234 | 235 | return string(bytes), nil 236 | } 237 | 238 | func getResultFromTerragruntOutput(dirName string, outputName string) (interface{}, string, error) { 239 | 240 | lsCmd := exec.Command("terragrunt", "output", "-json", outputName) 241 | //lsCmd := exec.Command("cat", outputName) 242 | lsCmd.Dir = dirName 243 | lsOut, err := lsCmd.Output() 244 | 245 | if err != nil { 246 | log.Debugln(spew.Sdump(lsCmd)) 247 | 248 | return "", "", errors.Wrapf(err, "running terragrunt output -json %s", outputName) 249 | } 250 | 251 | //fmt.Println("terragrunt value = ", string(lsOut)) 252 | 253 | // Unmarshal output into JSON 254 | var TerragruntOutput map[string]interface{} 255 | 256 | if err := json.Unmarshal(lsOut, &TerragruntOutput); err != nil { 257 | panic(err) 258 | } 259 | 260 | return TerragruntOutput["value"], TerragruntOutput["type"].(string), nil 261 | } 262 | -------------------------------------------------------------------------------- /main_test.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "os" 7 | "path" 8 | "runtime" 9 | "strings" 10 | "testing" 11 | ) 12 | 13 | //const TFVARS_BASE_PATH = "examples/terraform.tfvars" 14 | 15 | func assert(o bool) { 16 | if !o { 17 | fmt.Printf("\n%c[35m%s%c[0m\n\n", 27, _GetRecentLine(), 27) 18 | os.Exit(1) 19 | } 20 | } 21 | 22 | func _GetRecentLine() string { 23 | _, file, line, _ := runtime.Caller(2) 24 | buf, _ := ioutil.ReadFile(file) 25 | code := strings.TrimSpace(strings.Split(string(buf), "\n")[line-1]) 26 | return fmt.Sprintf("%v:%d\n%s", path.Base(file), line, code) 27 | } 28 | 29 | func TestMain(m *testing.M) { 30 | 31 | assert(true) 32 | } 33 | 34 | func TestVersionInfo(t *testing.T) { 35 | assert("unknown" == versionInfo()) 36 | 37 | buildVersion = "vXYZ" 38 | assert("vXYZ" == versionInfo()) 39 | } 40 | -------------------------------------------------------------------------------- /util/util.go: -------------------------------------------------------------------------------- 1 | package util 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | ) 8 | 9 | // UniqueNonEmptyElementsOf fetched from https://gist.github.com/johnwesonga/6301924 10 | func UniqueNonEmptyElementsOf(s []string) []string { 11 | unique := make(map[string]bool, len(s)) 12 | us := make([]string, len(unique)) 13 | for _, elem := range s { 14 | if len(elem) != 0 { 15 | if !unique[elem] { 16 | us = append(us, elem) 17 | unique[elem] = true 18 | } 19 | } 20 | } 21 | 22 | return us 23 | } 24 | 25 | // CopyFile fetched from https://opensource.com/article/18/6/copying-files-go 26 | func CopyFile(src string, dst string) (int64, error) { 27 | sourceFileStat, err := os.Stat(src) 28 | if err != nil { 29 | return 0, err 30 | } 31 | 32 | if !sourceFileStat.Mode().IsRegular() { 33 | return 0, fmt.Errorf("%s is not a regular file", src) 34 | } 35 | 36 | source, err := os.Open(src) 37 | if err != nil { 38 | return 0, err 39 | } 40 | defer source.Close() 41 | 42 | destination, err := os.Create(dst) 43 | if err != nil { 44 | return 0, err 45 | } 46 | defer destination.Close() 47 | nBytes, err := io.Copy(destination, source) 48 | return nBytes, err 49 | } 50 | --------------------------------------------------------------------------------