├── img ├── main.jpg ├── vars.jpg └── locals.jpg ├── docs └── images │ ├── main.jpg │ ├── vars.jpg │ ├── locals.jpg │ ├── logo │ ├── terra-crust.png │ └── terra-crust-single.png │ └── diagram │ └── terra-crust-levels.png ├── .github ├── dependabot.yml └── workflows │ ├── license.yml │ ├── release.yaml │ └── golang-build-test-coverage.yml ├── .gitignore ├── internal ├── services │ ├── README.md │ ├── templates │ │ ├── variables_file.tmpl │ │ ├── main_file.tmpl │ │ ├── variables_modules_tf.go │ │ ├── locals_modules_tf.go │ │ └── locals_file.tmpl │ ├── drivers │ │ ├── template_reader │ │ │ ├── test.tmpl │ │ │ ├── template_remote_module_test.go │ │ │ └── template_remote_module.go │ │ ├── parser.go │ │ ├── parser │ │ │ ├── terraform_parser_test.go │ │ │ └── terraform_parser.go │ │ └── version_control │ │ │ ├── git_test.go │ │ │ └── git.go │ ├── module_parser.go │ ├── template_api.go │ ├── template_handler.go │ └── terraform.go └── types │ ├── resource.go │ ├── output.go │ ├── variable.go │ └── module.go ├── Dockerfile ├── mock └── modules │ ├── consul_sync │ ├── locals.tf │ ├── main.tf │ └── variables.tf │ └── zookeeper │ ├── main.tf │ ├── locals.tf │ └── variables.tf ├── cmd ├── types │ └── terraform_variable.go ├── main.go └── app │ ├── initializer.go │ ├── root.go │ ├── generate_terraform_file.go │ ├── generate_main.go │ └── generate_all.go ├── examples ├── templates │ └── main.tf.tmpl └── example.md ├── Makefile ├── go.mod ├── README.md ├── LICENSE └── go.sum /img/main.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/img/main.jpg -------------------------------------------------------------------------------- /img/vars.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/img/vars.jpg -------------------------------------------------------------------------------- /img/locals.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/img/locals.jpg -------------------------------------------------------------------------------- /docs/images/main.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/docs/images/main.jpg -------------------------------------------------------------------------------- /docs/images/vars.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/docs/images/vars.jpg -------------------------------------------------------------------------------- /docs/images/locals.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/docs/images/locals.jpg -------------------------------------------------------------------------------- /docs/images/logo/terra-crust.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/docs/images/logo/terra-crust.png -------------------------------------------------------------------------------- /docs/images/logo/terra-crust-single.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/docs/images/logo/terra-crust-single.png -------------------------------------------------------------------------------- /docs/images/diagram/terra-crust-levels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/AppsFlyer/terra-crust/HEAD/docs/images/diagram/terra-crust-levels.png -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "gomod" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | main 2 | main.tmpl 3 | config 4 | bin/* 5 | .idea 6 | *.DS_Store 7 | .DS_Store 8 | internal/services/drivers/version_control/ext-temp-git-test/* 9 | internal/services/drivers/version_control/int-temp-git-test/* 10 | -------------------------------------------------------------------------------- /internal/services/README.md: -------------------------------------------------------------------------------- 1 | # Services 2 | Services contains all the buisness logic of the tool 3 | 4 | ## Template Api 5 | Template api contains all of the api for the external and internal usage of the templates, Example of ```GetDefaults``` is shown on `examples/templates`. 6 | 7 | 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | ARG GO_VERSION=1.16 2 | FROM golang:${GO_VERSION} AS builder 3 | COPY . /terra-crust 4 | WORKDIR /terra-crust 5 | RUN make compile 6 | 7 | 8 | FROM golang:${GO_VERSION} 9 | ARG ARCH=amd64 10 | WORKDIR /opt/ 11 | COPY --from=builder /terra-crust/bin/terra-crust-linux-${ARCH} ./ 12 | RUN mv terra-crust-linux-${ARCH} terra-crust 13 | ENTRYPOINT ["./terra-crust"] 14 | -------------------------------------------------------------------------------- /mock/modules/consul_sync/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | affinity = merge( 3 | tomap({ 4 | kafka_node_key = "dedicated" 5 | kafka_node_value = "Kafka" 6 | zookeeper_node_key = "dedicated" 7 | zookeeper_node_value = "Zookeeper" 8 | }), 9 | var.affinity, 10 | ) 11 | 12 | resources = merge( 13 | tomap({ 14 | requested_memory = "512Mi" 15 | requested_cpu = "0.5" 16 | limit_cpu = "2" 17 | limit_memory = "2Gi" 18 | }), 19 | var.resources, 20 | ) 21 | } -------------------------------------------------------------------------------- /internal/services/templates/variables_file.tmpl: -------------------------------------------------------------------------------- 1 | # ▼▼▼ Automatically generated by Terra-Crust, PLEASE DON'T EDIT MANUALLY, Please edit {main template/ sub-module variables} if changes are needed :slight_smile: ▼▼▼ 2 | {{range $moduleName,$moduleData := .}} 3 | variable {{ $moduleData.ModuleName }} { 4 | description = {{$moduleData.Description}} 5 | type = object({ {{range $objName,$objType := $moduleData.ObjectTypeMapping}} 6 | {{$objName}}=optional({{$objType}}) {{end}} 7 | }) 8 | default = { {{range $defaultKey,$defaultVal := $moduleData.DefaultValues}} 9 | {{$defaultKey}}={{$defaultVal}} {{end}} 10 | } 11 | } {{end}} -------------------------------------------------------------------------------- /internal/services/templates/main_file.tmpl: -------------------------------------------------------------------------------- 1 | {{$rootFolderPath := .RootPath}} 2 | {{range $moduleName,$moduleData := .Module}} 3 | module {{$moduleName}} { 4 | source = "{{$rootFolderPath}}/{{$moduleName}}" 5 | 6 | # Dependency Injection - Required Fields {{range $varName,$varValue := $moduleData.RequiredFields}} 7 | # {{$varName}} = module. TODO: Add Required Field {{end}} 8 | 9 | # Configurable Variables - Optinal Fields {{range $mapName,$mapProperties := $moduleData.ModuleData.MapLocals}} 10 | {{$mapName}} = local.{{$moduleName}}.{{$mapName}} {{end}} {{range $mapName,$mapProperties := $moduleData.ModuleData.SimpleLocals}} 11 | {{$mapName}} = local.{{$moduleName}}.{{$mapName}} {{end}} 12 | }{{end}} -------------------------------------------------------------------------------- /internal/services/drivers/template_reader/test.tmpl: -------------------------------------------------------------------------------- 1 | module "terra-crust" { 2 | source = "git::https://github.com/AppsFlyer/terra-crust" 3 | } 4 | 5 | module "naming" { 6 | source = "git::https://github.domain.com/test/terraform/modules/naming.git//modules/naming?ref=0.2.1" 7 | } 8 | 9 | module "otel_collector" { 10 | source = "git::https://github.com/streamnative/terraform-helm-charts.git//modules/otel-collector?ref=v0.2.1" 11 | } 12 | 13 | module "iam-account" { 14 | source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git//modules/iam-account" 15 | } 16 | 17 | module "zones" { 18 | source = "git::https://github.com/terraform-aws-modules/terraform-aws-route53.git/modules/zones" 19 | } -------------------------------------------------------------------------------- /internal/types/resource.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | type Resource struct { 18 | Type string `hcl:"type,label"` 19 | Name string `hcl:"name,label"` 20 | } 21 | -------------------------------------------------------------------------------- /internal/services/drivers/parser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package drivers 16 | 17 | import "github.com/AppsFlyer/terra-crust/internal/types" 18 | 19 | type Parser interface { 20 | Parse(path string) (map[string]*types.Module, error) 21 | } 22 | -------------------------------------------------------------------------------- /cmd/types/terraform_variable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | type TFGenerateFlags struct { 18 | SourcePath string 19 | DestinationPath string 20 | MainTemplateFilePath string 21 | FetchRemote bool 22 | ExternalGit bool 23 | } 24 | -------------------------------------------------------------------------------- /internal/types/output.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import "github.com/hashicorp/hcl/v2/hclwrite" 18 | 19 | type Output struct { 20 | Name string `hcl:"name,label"` 21 | Value hclwrite.Tokens `hcl:"value,attr"` 22 | Description string `hcl:"description,attr"` 23 | } 24 | -------------------------------------------------------------------------------- /internal/services/templates/variables_modules_tf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package templates 16 | 17 | type VariablesModuleList map[string]*VariablesModulesTF 18 | 19 | type VariablesModulesTF struct { 20 | ModuleName string 21 | Description string 22 | ObjectTypeMapping map[string]string 23 | DefaultValues map[string]string 24 | } 25 | -------------------------------------------------------------------------------- /internal/types/variable.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import "github.com/hashicorp/hcl/v2/hclwrite" 18 | 19 | type Variable struct { 20 | Name string `hcl:"name,label"` 21 | Type hclwrite.Tokens `hcl:"type,attr"` 22 | Description string `hcl:"description,attr"` 23 | Default hclwrite.Tokens `hcl:"default,attr"` 24 | } 25 | -------------------------------------------------------------------------------- /cmd/main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package main 16 | 17 | import ( 18 | "context" 19 | "github.com/AppsFlyer/terra-crust/cmd/app" 20 | "os" 21 | 22 | logger "github.com/AppsFlyer/go-logger" 23 | ) 24 | 25 | func main() { 26 | log := logger.NewSimple() 27 | if err := app.NewRootCommand(log).Execute(); err != nil { 28 | if err == context.Canceled { 29 | os.Exit(0) 30 | } 31 | 32 | log.Error("Error executing command. Exiting.", err.Error()) 33 | os.Exit(1) 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.github/workflows/license.yml: -------------------------------------------------------------------------------- 1 | name: License 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | jobs: 10 | License: 11 | runs-on: ubuntu-latest 12 | steps: 13 | - name: checkout repo 14 | uses: actions/checkout@v2 15 | - name: set up go 16 | uses: actions/setup-go@v2 17 | with: 18 | go-version: 1.17.2 19 | - name: Get addlicense 20 | run: go get -d github.com/google/addlicense 21 | - name: Build addlicense 22 | run: go build -o addlicense github.com/google/addlicense 23 | - name: License check 24 | id: check 25 | run: | 26 | set +e 27 | ./addlicense --check -c AppsFlyer -l apache ${PWD}/**/**/*.go 28 | echo ::set-output name=exit_status::$? 29 | # continue-on-error: true 30 | - name: Message 31 | if: steps.check.outputs.exit_status != 0 32 | run: | 33 | echo -e "License headers missing. \nIn order to fix run: 'docker run -it -v \${PWD}:/src -w /src ghcr.io/google/addlicense:v1.0.0 -c AppsFlyer -l apache ./**/**/*.go' " 34 | exit 1 -------------------------------------------------------------------------------- /internal/services/templates/locals_modules_tf.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package templates 16 | 17 | type LocalsModulesTF struct { 18 | Module map[string]*ModuleData 19 | } 20 | 21 | type MainModuleTF struct { 22 | Module map[string]*MainModuleData 23 | RootPath string 24 | } 25 | 26 | type ComplexVariableData map[string]string 27 | 28 | type ModuleData struct { 29 | SimpleLocals map[string]string 30 | MapLocals map[string]ComplexVariableData 31 | } 32 | 33 | type MainModuleData struct { 34 | *ModuleData 35 | RequiredFields map[string]string 36 | } 37 | -------------------------------------------------------------------------------- /mock/modules/zookeeper/main.tf: -------------------------------------------------------------------------------- 1 | output "template" { 2 | value = templatefile("${path.module}/zk-config-template.yaml.tftpl", { 3 | # General 4 | zookeeper_replicas = var.replicas 5 | zookeeper_root_log_level = var.root_log_level 6 | 7 | # Resources 8 | limit_cpu = local.resources.limit_cpu 9 | limit_memory = local.resources.limit_memory 10 | request_cpu = local.resources.requested_cpu 11 | request_memory = local.resources.requested_memory 12 | 13 | # JVM 14 | jvm_xmx = local.jvm.xmx 15 | jvm_xms = local.jvm.xms 16 | 17 | # Annotations 18 | pod_annotations = var.pod_annotations 19 | 20 | #Node Affinity 21 | node_affinity_key = local.affinity.node_affinity_key 22 | node_affinity_value = local.affinity.node_affinity_value 23 | 24 | #Service Annotations 25 | service_annotations = local.service_annotations 26 | 27 | #Storage 28 | storage_class = local.storage.class 29 | storage_size = local.storage.size 30 | storage_delete_claim = local.storage.delete_claim 31 | 32 | # ConfigMap 33 | configmap_name = var.configmap_name 34 | configmap_key = var.configmap_key 35 | }) 36 | } 37 | 38 | 39 | -------------------------------------------------------------------------------- /mock/modules/zookeeper/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | 3 | resources = merge( 4 | tomap({ 5 | requested_memory = "5Gi" 6 | requested_cpu = "1" 7 | limit_cpu = "2" 8 | limit_memory = "6Gi" 9 | }), 10 | var.resources, 11 | ) 12 | 13 | jvm = merge( 14 | tomap({ 15 | xms = "4096m" 16 | xmx = "4096m" 17 | }), 18 | var.jvm, 19 | ) 20 | 21 | pod_annotations = merge( 22 | tomap({ 23 | "ad.datadoghq.com/zookeeper.check_names" : "['zk']" 24 | "ad.datadoghq.com/zookeeper.init_configs" : "[{}]" 25 | "ad.datadoghq.com/zookeeper.instances" : "[{ 'host' : '%%host%%', 'port' : '2181' }]" 26 | "ad.datadoghq.com/zookeeper.logs" : "[{ 'source' : 'zookeeper', 'service' : 'zookeeper' }]" 27 | }), 28 | var.pod_annotations) 29 | 30 | affinity = merge( 31 | tomap({ 32 | node_affinity_key = "dedicated" 33 | node_affinity_value = "Zookeeper" 34 | }), 35 | var.affinity, 36 | ) 37 | 38 | service_annotations = merge( 39 | tomap({ 40 | "consul.hashicorp.com/service-sync" : "true" 41 | "consul.hashicorp.com/service-name" : "strimzi-zookeeper-${var.cluster_name}" 42 | }), 43 | var.service_annotations, 44 | ) 45 | 46 | storage = merge( 47 | tomap({ 48 | size = "69Gi" 49 | class = "nvme-ssd" 50 | delete_claim = "true" 51 | }), 52 | var.storage, 53 | ) 54 | 55 | } 56 | -------------------------------------------------------------------------------- /cmd/app/initializer.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | logger "github.com/AppsFlyer/go-logger" 19 | "github.com/AppsFlyer/terra-crust/internal/services" 20 | "github.com/AppsFlyer/terra-crust/internal/services/drivers/parser" 21 | ) 22 | 23 | const ( 24 | LocalsTemplatePath = "templates/locals_file.tmpl" 25 | VariableTemplatePath = "templates/variables_file.tmpl" 26 | MainTemplatePath = "templates/main_file.tmpl" 27 | ) 28 | 29 | func InitTerraformGeneratorService(log logger.Logger) *services.Terraform { 30 | parserDriver := parser.NewTerraformParser(log) 31 | parserSvc := services.NewParser(log, parserDriver) 32 | templateHandler := services.NewTemplateHandler(log) 33 | 34 | tfSvc := services.NewTerraform(log, parserSvc, templateHandler, LocalsTemplatePath, VariableTemplatePath, MainTemplatePath) 35 | 36 | return tfSvc 37 | } 38 | -------------------------------------------------------------------------------- /internal/services/module_parser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package services 16 | 17 | import ( 18 | logger "github.com/AppsFlyer/go-logger" 19 | "github.com/AppsFlyer/terra-crust/internal/services/drivers" 20 | "github.com/AppsFlyer/terra-crust/internal/types" 21 | ) 22 | 23 | type ModuleParser struct { 24 | parsingDriver drivers.Parser 25 | logger logger.Logger 26 | } 27 | 28 | func NewParser(log logger.Logger, driver drivers.Parser) *ModuleParser { 29 | return &ModuleParser{ 30 | parsingDriver: driver, 31 | logger: log, 32 | } 33 | } 34 | 35 | func (p *ModuleParser) GetModulesList(rootFolder string) (map[string]*types.Module, error) { 36 | modulesList, err := p.parsingDriver.Parse(rootFolder) 37 | if err != nil { 38 | p.logger.Error("Failed parsing root folder", err.Error()) 39 | 40 | return nil, err 41 | } 42 | 43 | return modulesList, nil 44 | } 45 | -------------------------------------------------------------------------------- /internal/services/templates/locals_file.tmpl: -------------------------------------------------------------------------------- 1 | # ▼▼▼ Automatically generated by Terra-Crust, PLEASE DON'T EDIT MANUALLY, Please edit {main template/ sub-module variables} if changes are needed :slight_smile: ▼▼▼ 2 | locals{ {{range $moduleName,$moduleData := .Module}} 3 | {{$moduleName}} = { {{template "simple_local_var" (SimpleWrap $moduleName $moduleData.SimpleLocals)}} 4 | {{template "typeMapLocalBlock" (ModuleDataWrapper $moduleName $moduleData)}} 5 | }{{end}} 6 | } 7 | 8 | {{define "simple_local_var"}} {{$moduleName := .VariableName}} {{range $localName, $localValue := .ModuleData}} 9 | {{if $localValue}}{{$localName}} = coalesce(var.{{$moduleName}}.{{$localName}}, {{$localValue}}){{else}} 10 | {{$localName}} = var.{{$moduleName}}.{{$localName}}{{end}}{{end}}{{end}} 11 | 12 | {{define "typeMapLocalBlock"}} {{$moduleData := .ModuleData}} {{$moduleName := .ModuleName}}{{range $mapKey,$mapValue := $moduleData.MapLocals}} 13 | {{$tempLen := len $mapValue }}{{if eq $tempLen 0}} {{$mapKey}} = var.{{$moduleName}}.{{$mapKey}} != null ? var.{{$moduleName}}.{{$mapKey}}:{} {{else}} 14 | {{$mapKey}} = var.{{$moduleName}}.{{$mapKey}} != null ? merge(var.{{$moduleName}}.{{$mapKey}}, 15 | tomap({ {{range $propertyName, $propertyValue := $mapValue}} 16 | {{$propertyName}} = contains(keys(var.{{$moduleName}}.{{$mapKey}}),{{$propertyName}}) != false ? var.{{$moduleName}}.{{$mapKey}}[{{$propertyName}}] : {{$propertyValue}}{{end}}} 17 | )):{ 18 | {{range $name, $value := $mapValue}} {{$name}} = {{$value}} 19 | {{end}}} {{end}}{{end}} 20 | {{end}} 21 | -------------------------------------------------------------------------------- /examples/templates/main.tf.tmpl: -------------------------------------------------------------------------------- 1 | module "consul_sync" { 2 | source = "../../modules/consul_sync" 3 | # Dependency Injection - Required Fields 4 | 5 | affinity = module.kafka.affinity 6 | eks_cluster = module.zookeeper.eks_cluster 7 | 8 | # Configurable Variables - Optinal Fields 9 | {{(GetDefaults "consul_sync" .)}} 10 | } 11 | 12 | module "zookeeper" { 13 | source = "../../modules/consul_sync" 14 | # Dependency Injection - Required Fields 15 | 16 | affinity = module.kafka.affinity 17 | eks_cluster = module.zookeeper.eks_cluster 18 | 19 | # Configurable Variables - Optinal Fields 20 | {{(GetDefaults "consul_sync" .)}} 21 | } 22 | 23 | module "zones" { 24 | source = "git::https://github.com/terraform-aws-modules/terraform-aws-route53.git/modules/zones" 25 | 26 | # Dependency Injection - Required Fields 27 | {{(GetRequired "zones" .)}} 28 | 29 | # Configurable Variables - Optinal Fields 30 | {{(GetDefaults "zones" .)}} 31 | } 32 | 33 | module "terraform-aws-resource-naming" { 34 | source = "git::https://github.com/traveloka/terraform-aws-resource-naming?ref=v0.23.1" 35 | 36 | # Dependency Injection - Required Fields 37 | {{(GetRequired "terraform-aws-resource-naming" .)}} 38 | 39 | # Configurable Variables - Optinal Fields 40 | {{(GetDefaults "terraform-aws-resource-naming" .)}} 41 | } 42 | 43 | module "iam-account" { 44 | source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account" 45 | 46 | {{(GetRequired "iam-account" .)}} 47 | 48 | {{(GetDefaults "iam-account" .)}} 49 | } -------------------------------------------------------------------------------- /mock/modules/consul_sync/main.tf: -------------------------------------------------------------------------------- 1 | resource "helm_release" "consul" { 2 | count = var.is_enabled ? 1 : 0 3 | chart = var.chart 4 | name = var.name 5 | repository = var.repository 6 | namespace = var.namespace 7 | version = var.chart_version 8 | create_namespace = var.create_namespace 9 | atomic = var.atomic 10 | wait = var.wait 11 | wait_for_jobs = var.wait_for_jobs 12 | timeout = var.timeout 13 | values = [ 14 | templatefile("${path.module}/helm-consule-values-template.tftpl", { 15 | # Consul Configuration 16 | consul_image = var.consul_image 17 | consul_datacenter = var.consul_datacenter 18 | consul_servers_address = var.consul_servers_address 19 | chart_version = var.chart_version 20 | 21 | # EKS cluster name 22 | eks_cluster = var.eks_cluster 23 | 24 | # Affinity 25 | kafka_node_affinity_key = local.affinity.kafka_node_key 26 | kafka_node_affinity_value = local.affinity.kafka_node_value 27 | zookeeper_key_affinity_key = local.affinity.zookeeper_node_key 28 | zookeeper_node_affinity_value = local.affinity.zookeeper_node_value 29 | 30 | # Resources 31 | limit_cpu = local.resources.limit_cpu 32 | limit_memory = local.resources.limit_memory 33 | request_cpu = local.resources.requested_cpu 34 | request_memory = local.resources.requested_memory 35 | }) 36 | ] 37 | } 38 | 39 | # Need to add aws security groups for TCP and UDP - port 8301 for consul service 40 | 41 | -------------------------------------------------------------------------------- /cmd/app/root.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | logger "github.com/AppsFlyer/go-logger" 19 | "github.com/spf13/cobra" 20 | ) 21 | 22 | type RootCommand struct { 23 | *cobra.Command 24 | log logger.Logger 25 | DryRun bool 26 | Verbose bool 27 | } 28 | 29 | func NewRootCommand(log logger.Logger) *RootCommand { 30 | root := &RootCommand{} 31 | root.Command = &cobra.Command{ 32 | Use: "generate [command]", 33 | Short: "terraform tool", 34 | SilenceErrors: true, 35 | SilenceUsage: true, 36 | } 37 | root.Command.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { 38 | root.log = log 39 | 40 | return nil 41 | } 42 | 43 | svc := InitTerraformGeneratorService(root.log) 44 | 45 | root.Command.AddCommand( 46 | generateTerraformFile(root, svc.GenerateModuleDefaultLocals, "terraform-locals"), 47 | generateTerraformFile(root, svc.GenerateModuleVariableObject, "terraform-variable"), 48 | generateMain(root), 49 | generateAllFiles(root), 50 | ) 51 | 52 | return root 53 | } 54 | -------------------------------------------------------------------------------- /internal/types/module.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package types 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | ) 21 | 22 | const TerraformMetaDirectoryName = ".terraform" 23 | 24 | type Module struct { 25 | Name string `hcl:"name,label"` 26 | Variables []*Variable `hcl:"variable,block"` 27 | Outputs []*Output `hcl:"output,block"` 28 | Resources []*Resource `hcl:"resource,block"` 29 | Source string `hcl:"source,attr"` 30 | } 31 | 32 | func (m *Module) String() string { 33 | var s string 34 | s = fmt.Sprintf("Name: %s\nSource: %s\nVariables: \n", m.Name, m.Source) 35 | for _, v := range m.Variables { 36 | s += fmt.Sprintf(" %s", v.Name) 37 | s += fmt.Sprintf(" %s", strings.ReplaceAll(string(v.Type.Bytes()), " ", "")) 38 | s += fmt.Sprintf(" %s", v.Description) 39 | s += fmt.Sprintf(" %s\n", strings.ReplaceAll(string(v.Default.Bytes()), " ", "")) 40 | } 41 | s += "Outputs: \n" 42 | for _, v := range m.Outputs { 43 | s += fmt.Sprintf(" %s", v.Name) 44 | s += fmt.Sprintf(" %s", strings.ReplaceAll(string(v.Value.Bytes()), " ", "")) 45 | s += fmt.Sprintf(" %s\n", v.Description) 46 | } 47 | 48 | return s 49 | } 50 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | SHELL := /bin/bash 2 | ROOT_DIR:=$(shell dirname $(realpath $(lastword $(MAKEFILE_LIST)))) 3 | 4 | export GO_VERSION ?= $(shell grep -E "^go " go.mod | awk -F' ' '{print $$2}' ) 5 | export GOLANGCI_LINT_VERSION=v1.50.1 6 | export MAIN_GO=cmd/main.go 7 | 8 | .PHONY: help 9 | help: ## Show this help message. 10 | 11 | @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' 12 | 13 | 14 | get-go-dependancies: ## Get dependancies as devlared in the project 15 | $(info) 16 | $(info ========[ $@ ]========) 17 | go get -v -t -d ./... 18 | 19 | get-linter: ## Get golangci-lint 20 | $(info) 21 | $(info ========[ $@ ]========) 22 | @if ! golangci-lint --version 2>/dev/null; then \ 23 | curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${GOLANGCI_LINT_VERSION};\ 24 | fi;\ 25 | 26 | lint: get-linter ## Run linter 27 | $(info) 28 | $(info ========[ $@ ]========) 29 | golangci-lint run 30 | 31 | test: ## Run go tests 32 | $(info) 33 | $(info ========[ $@ ]========) 34 | go test ./... 35 | 36 | build: get-go-dependancies lint ## Build 37 | $(info) 38 | $(info ========[ $@ ]========) 39 | go build -v ${MAIN_GO} 40 | 41 | 42 | compile: build ## Compile binaries for multiple OSs and architectures 43 | $(info) 44 | $(info ========[ $@ ]========) 45 | @for os in "linux" "darwin"; do \ 46 | for arch in "amd64" "arm64"; do \ 47 | echo "$${os} - $${arch}"; \ 48 | GOOS=$${os} GOARCH=$${arch} go build -o bin/terra-crust-$${os}-$${arch} ${MAIN_GO}; \ 49 | done \ 50 | done 51 | 52 | build-docker: ## Build a docker image 53 | $(info) 54 | $(info ========[ $@ ]========) 55 | docker build -t appsflyer/terra-crust:$$(git rev-parse --short HEAD) . -------------------------------------------------------------------------------- /.github/workflows/release.yaml: -------------------------------------------------------------------------------- 1 | name: Release 2 | on: 3 | release: 4 | types: [created] 5 | # workflow_run: 6 | # workflows: ["golang-build-test-coverage"] 7 | # branches: [main] 8 | # types: 9 | # - completed 10 | # Allows you to run this workflow manually from the Actions tab 11 | # workflow_dispatch: 12 | 13 | permissions: 14 | contents: read 15 | 16 | jobs: 17 | # release-blocker: 18 | # name: Release blocked 19 | # runs-on: ubuntu-latest 20 | # if: ${{ github.event.workflow_run.conclusion == 'failure' }} 21 | # steps: 22 | # - run: | 23 | # echo '::echo::on' 24 | # echo "Release flow is blocked by 'golang-build-test-coverage' workflow failure" 25 | releases-matrix: 26 | name: Release Go Binary 27 | runs-on: ubuntu-latest 28 | # if: ${{ github.event.workflow_run.conclusion == 'success' }} 29 | permissions: 30 | contents: write # for wangyoucao577/go-release-action to upload release assets 31 | strategy: 32 | matrix: 33 | goos: [linux, darwin] 34 | goarch: [amd64, arm64] 35 | steps: 36 | - uses: actions/checkout@v3 37 | 38 | - name: Get Go version 39 | id: get_go_version 40 | run: | 41 | echo '::echo::on' 42 | go mod edit -json 2> error 1> output 43 | echo "::set-output name=errors::$(cat error)" 44 | jq -r .Go output > version 45 | echo "::set-output name=version::$(cat version)" 46 | 47 | - uses: wangyoucao577/go-release-action@v1.40 48 | with: 49 | github_token: ${{ secrets.GITHUB_TOKEN }} 50 | goos: ${{ matrix.goos }} 51 | goarch: ${{ matrix.goarch }} 52 | goversion: ${{ steps.get_go_version.outputs.version }} 53 | project_path: "cmd/" 54 | binary_name: "terra-crust" 55 | # pre_command: "" 56 | # build_command: "" 57 | # build_flags: "" 58 | # ldflags: "" 59 | -------------------------------------------------------------------------------- /cmd/app/generate_terraform_file.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "fmt" 19 | "github.com/AppsFlyer/terra-crust/cmd/types" 20 | 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | func generateTerraformFile(root *RootCommand, f func(modulesFilePath string, destinationPath string) error, short string) *cobra.Command { 25 | flags := types.TFGenerateFlags{} 26 | cmd := &cobra.Command{ 27 | Use: short, 28 | Short: fmt.Sprintf("create general object %s file", short), 29 | Example: "", 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | log := root.log 32 | 33 | if err := f(flags.SourcePath, flags.DestinationPath); err != nil { 34 | log.Error("Failed generating the terraform locals file", err.Error()) 35 | 36 | return err 37 | } 38 | 39 | return nil 40 | }, 41 | } 42 | 43 | cmd.Flags().StringVar(&flags.DestinationPath, "destination-path", "", "Required: Destination path to write the new terraform file") 44 | cmd.Flags().StringVar(&flags.SourcePath, "source-path", "", "Required: General module folder path that contains all the sub modules flattened") 45 | if err := cmd.MarkFlagRequired("source-path"); err != nil { 46 | root.log.Error("failed to set required flag on source-path", err.Error()) 47 | } 48 | if err := cmd.MarkFlagRequired("destination-path"); err != nil { 49 | root.log.Error("failed to set required flag on destination-path", err.Error()) 50 | } 51 | 52 | return cmd 53 | } 54 | -------------------------------------------------------------------------------- /internal/services/drivers/template_reader/template_remote_module_test.go: -------------------------------------------------------------------------------- 1 | package template_reader_test 2 | 3 | import ( 4 | version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" 5 | "github.com/go-test/deep" 6 | "testing" 7 | 8 | logger "github.com/AppsFlyer/go-logger" 9 | tmplReader "github.com/AppsFlyer/terra-crust/internal/services/drivers/template_reader" 10 | ) 11 | 12 | var result = map[string]*version_control.RemoteModule{ 13 | "terra-crust": {Name: "terra-crust", Url: "https://github.com/AppsFlyer/terra-crust", Version: "", Path: ""}, 14 | "naming": {Name: "naming", Url: "https://github.domain.com/test/terraform/modules/naming.git", Version: "0.2.1", Path: "modules/naming"}, 15 | "otel-collector": {Name: "otel-collector", Url: "https://github.com/streamnative/terraform-helm-charts.git", Version: "v0.2.1", Path: "modules/otel-collector"}, 16 | "iam-account": {Name: "iam-account", Path: "modules/iam-account", Url: "https://github.com/terraform-aws-modules/terraform-aws-iam.git", Version: ""}, 17 | "zones": {Name: "zones", Path: "modules/zones", Url: "https://github.com/terraform-aws-modules/terraform-aws-route53.git", Version: ""}, 18 | } 19 | 20 | func TestGetRemoteModulesFromTemplate(t *testing.T) { 21 | t.Parallel() 22 | log := logger.NewSimple() 23 | templateReader := tmplReader.InitTemplateRemoteModule(log) 24 | 25 | modules, err := templateReader.GetRemoteModulesFromTemplate("./test.tmpl") 26 | if err != nil { 27 | t.Errorf("failed to extract sources from template %s", err.Error()) 28 | } 29 | 30 | if len(modules) != 5 { 31 | t.Errorf("Expected 5 modules found : %d", len(modules)) 32 | } 33 | 34 | if diff := deep.Equal(modules, result); diff != nil { 35 | t.Errorf("expected result to be equal, result is different : %s", diff) 36 | } 37 | } 38 | 39 | func TestBadPath(t *testing.T) { 40 | t.Parallel() 41 | log := logger.NewSimple() 42 | templateReader := tmplReader.InitTemplateRemoteModule(log) 43 | 44 | _, err := templateReader.GetRemoteModulesFromTemplate("./main.tmpl") 45 | if err == nil { 46 | t.Errorf("expected to have error for no file found") 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/AppsFlyer/terra-crust 2 | 3 | go 1.22 4 | 5 | require ( 6 | github.com/AppsFlyer/go-logger v1.1.1 7 | github.com/go-git/go-git/v5 v5.12.0 8 | github.com/go-test/deep v1.1.0 9 | github.com/hashicorp/go-version v1.6.0 10 | github.com/hashicorp/hc-install v0.6.4 11 | github.com/hashicorp/hcl/v2 v2.20.1 12 | github.com/hashicorp/terraform-exec v0.20.0 13 | github.com/otiai10/copy v1.14.0 14 | github.com/pkg/errors v0.9.1 15 | github.com/spf13/cobra v1.8.0 16 | github.com/zclconf/go-cty v1.14.4 17 | ) 18 | 19 | require ( 20 | dario.cat/mergo v1.0.0 // indirect 21 | github.com/Microsoft/go-winio v0.6.1 // indirect 22 | github.com/ProtonMail/go-crypto v1.1.0-alpha.2 // indirect 23 | github.com/agext/levenshtein v1.2.1 // indirect 24 | github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect 25 | github.com/cloudflare/circl v1.3.7 // indirect 26 | github.com/cyphar/filepath-securejoin v0.2.4 // indirect 27 | github.com/emirpasic/gods v1.18.1 // indirect 28 | github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect 29 | github.com/go-git/go-billy/v5 v5.5.0 // indirect 30 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 31 | github.com/google/go-cmp v0.6.0 // indirect 32 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 33 | github.com/hashicorp/terraform-json v0.19.0 // indirect 34 | github.com/inconshreveable/mousetrap v1.1.0 // indirect 35 | github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect 36 | github.com/kevinburke/ssh_config v1.2.0 // indirect 37 | github.com/mitchellh/go-wordwrap v0.0.0-20150314170334-ad45545899c7 // indirect 38 | github.com/pjbgf/sha1cd v0.3.0 // indirect 39 | github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect 40 | github.com/skeema/knownhosts v1.2.2 // indirect 41 | github.com/spf13/pflag v1.0.5 // indirect 42 | github.com/xanzy/ssh-agent v0.3.3 // indirect 43 | golang.org/x/crypto v0.21.0 // indirect 44 | golang.org/x/mod v0.16.0 // indirect 45 | golang.org/x/net v0.23.0 // indirect 46 | golang.org/x/sync v0.3.0 // indirect 47 | golang.org/x/sys v0.18.0 // indirect 48 | golang.org/x/text v0.14.0 // indirect 49 | golang.org/x/tools v0.13.0 // indirect 50 | gopkg.in/warnings.v0 v0.1.2 // indirect 51 | ) 52 | -------------------------------------------------------------------------------- /internal/services/drivers/parser/terraform_parser_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package parser_test 16 | 17 | import ( 18 | "github.com/AppsFlyer/terra-crust/internal/services/drivers/parser" 19 | "testing" 20 | 21 | logger "github.com/AppsFlyer/go-logger" 22 | ) 23 | 24 | func TestParse(t *testing.T) { 25 | t.Parallel() 26 | log := logger.NewSimple() 27 | parserDriver := parser.NewTerraformParser(log) 28 | 29 | m, err := parserDriver.Parse("../../../../mock/modules") 30 | if err != nil { 31 | t.Errorf("failed to parse , reason: %s", err.Error()) 32 | } 33 | 34 | if len(m) != 2 { 35 | t.Errorf("Failed to parse, expected 2 modules received : %d", len(m)) 36 | } 37 | 38 | if _, ok := m["consul_sync"]; !ok { 39 | t.Errorf("Expected consul_sync module to exist on the parsing , and not existing") 40 | } 41 | 42 | if _, ok := m["zookeeper"]; !ok { 43 | t.Errorf("Expected zookeeper module to exist on the parsing , and not existing") 44 | } 45 | } 46 | 47 | func TestParseBadPath(t *testing.T) { 48 | t.Parallel() 49 | log := logger.NewSimple() 50 | parserDriver := parser.NewTerraformParser(log) 51 | 52 | m, err := parserDriver.Parse("../../../../internal") 53 | if err != nil { 54 | t.Errorf("failed to parse , reason: %s", err.Error()) 55 | } 56 | 57 | if len(m) != 0 { 58 | t.Errorf("Failed to parse, expected 0 modules received : %d", len(m)) 59 | } 60 | } 61 | 62 | func TestParseNotExistingPath(t *testing.T) { 63 | t.Parallel() 64 | log := logger.NewSimple() 65 | parserDriver := parser.NewTerraformParser(log) 66 | 67 | _, err := parserDriver.Parse("../../../internal") 68 | if err == nil { 69 | t.Errorf("Expected error for bad route") 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /internal/services/template_api.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package services 16 | 17 | import ( 18 | "fmt" 19 | "strings" 20 | "text/template" 21 | 22 | "github.com/AppsFlyer/terra-crust/internal/services/templates" 23 | ) 24 | 25 | type TemplateAPI struct { 26 | APIFuncMap *template.FuncMap 27 | } 28 | 29 | func NewTemplateAPI() *TemplateAPI { 30 | return &TemplateAPI{ 31 | APIFuncMap: &template.FuncMap{ 32 | "SimpleWrap": SimpleWrap, 33 | "ModuleDataWrapper": ModuleDataWrapper, 34 | "GetDefaults": GetDefaults, 35 | "GetRequired": GetRequired, 36 | }, 37 | } 38 | } 39 | 40 | func ModuleDataWrapper(moduleName string, moduleData templates.ModuleData) map[string]interface{} { 41 | return map[string]interface{}{ 42 | "ModuleName": moduleName, 43 | "ModuleData": moduleData, 44 | } 45 | } 46 | 47 | func SimpleWrap(moduleName string, moduleData map[string]string) map[string]interface{} { 48 | return map[string]interface{}{ 49 | "ModuleData": moduleData, 50 | "VariableName": moduleName, 51 | } 52 | } 53 | 54 | func GetDefaults(moduleName string, modulesMap *templates.MainModuleTF) string { 55 | var sb strings.Builder 56 | for k := range modulesMap.Module[moduleName].SimpleLocals { 57 | sb.WriteString(fmt.Sprintf(mainDefaultVarRowTemplate, k, moduleName, k)) 58 | } 59 | 60 | for k := range modulesMap.Module[moduleName].MapLocals { 61 | sb.WriteString(fmt.Sprintf(mainDefaultVarRowTemplate, k, moduleName, k)) 62 | } 63 | 64 | return sb.String() 65 | } 66 | 67 | func GetRequired(moduleName string, modulesMap *templates.MainModuleTF) string { 68 | var sb strings.Builder 69 | for k := range modulesMap.Module[moduleName].RequiredFields { 70 | sb.WriteString(fmt.Sprintf(mainRequiredVarRowTemplate, k)) 71 | } 72 | 73 | return sb.String() 74 | } 75 | -------------------------------------------------------------------------------- /mock/modules/zookeeper/variables.tf: -------------------------------------------------------------------------------- 1 | variable "cluster_name" { 2 | type = string 3 | description = "(Required) The name that will be given to the kafka cluster and it's namespace" 4 | } 5 | variable "replicas" { 6 | type = number 7 | description = "(Optional) Number of zookeepers brokers to be created." 8 | default = 3 9 | } 10 | 11 | variable "root_log_level" { 12 | type = string 13 | description = <"` 68 | it will be generated from the cluster-name and will be auto implemented. 69 | in order to override that value: 70 | ``` 71 | service_annotations = 72 | { 73 | consul.hashicorp.com/service-name: "" 74 | } 75 | EOT 76 | default = { 77 | "consul.hashicorp.com/service-sync" : "true" 78 | } 79 | } 80 | 81 | variable "storage" { 82 | type = map(string) 83 | description = "(Required) Zookeeper Brokers persistence storage configuration." 84 | } 85 | 86 | variable "configmap_name" { 87 | description = "(Required) ConfigMap deployment name that holds Cruise Control Metrics." 88 | type = string 89 | } 90 | 91 | variable "configmap_key" { 92 | description = "(Required) ConfigMap deployment key that holds Cruise Control Metrics." 93 | type = string 94 | } 95 | -------------------------------------------------------------------------------- /cmd/app/generate_main.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "github.com/AppsFlyer/terra-crust/cmd/types" 19 | template_reader "github.com/AppsFlyer/terra-crust/internal/services/drivers/template_reader" 20 | version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" 21 | "github.com/spf13/cobra" 22 | ) 23 | 24 | func generateMain(root *RootCommand) *cobra.Command { 25 | flags := types.TFGenerateFlags{} 26 | cmd := &cobra.Command{ 27 | Use: "terraform-main", 28 | Short: "create general object terraform main file", 29 | Example: "", 30 | RunE: func(cmd *cobra.Command, args []string) error { 31 | log := root.log 32 | 33 | terraformSvc := InitTerraformGeneratorService(log) 34 | templateReader := template_reader.InitTemplateRemoteModule(log) 35 | gitDriver := version_control.InitGitProvider(log) 36 | 37 | if (flags.FetchRemote || flags.ExternalGit) && flags.MainTemplateFilePath != "" { 38 | log.Infof("Searching for remote modules") 39 | remoteModulesMap, err := templateReader.GetRemoteModulesFromTemplate(flags.MainTemplateFilePath) 40 | if err != nil { 41 | log.Error("Failed parsing remote modules from custom template", err.Error()) 42 | 43 | return err 44 | } 45 | 46 | log.Infof("found remote modules: ", remoteModulesMap) 47 | 48 | if err = gitDriver.CloneModules(remoteModulesMap, flags.SourcePath, flags.ExternalGit); err != nil { 49 | log.Error("Failed cloning remote modules ", err.Error()) 50 | 51 | return err 52 | } 53 | 54 | defer func() { 55 | if err = gitDriver.CleanModulesFolders(remoteModulesMap, flags.SourcePath); err != nil { 56 | log.Errorf("Failed to clean up some of the remote resources please make sure to clean it manually and check the error , %s", err.Error()) 57 | } 58 | }() 59 | } 60 | 61 | log.Infof("remote not found ") 62 | 63 | if err := terraformSvc.GenerateMain(flags.SourcePath, flags.DestinationPath, flags.MainTemplateFilePath); err != nil { 64 | log.Error("Failed generating the terraform main file", err.Error()) 65 | 66 | return err 67 | } 68 | 69 | return nil 70 | }, 71 | } 72 | 73 | cmd.Flags().StringVar(&flags.SourcePath, "source-path", "", "Required: General module folder path that contains all the sub modules flattened") 74 | cmd.Flags().StringVar(&flags.DestinationPath, "destination-path", "", "Required: Destination path to write the new terraform file") 75 | cmd.Flags().StringVar(&flags.MainTemplateFilePath, "main-template-path", "", "Optional: Custom main template path for generated module, will take default if not provided") 76 | cmd.Flags().BoolVar(&flags.FetchRemote, "fetch-remote", false, "Optional: Enable fetching of remote modules and exporting their variables in root module") 77 | cmd.Flags().BoolVar(&flags.ExternalGit, "ext-git", false, "Optional: Enable external git for remote modules") 78 | if err := cmd.MarkFlagRequired("source-path"); err != nil { 79 | root.log.Error("failed to set required flag on source-path", err.Error()) 80 | } 81 | if err := cmd.MarkFlagRequired("destination-path"); err != nil { 82 | root.log.Error("failed to set required flag on destination-path", err.Error()) 83 | } 84 | 85 | return cmd 86 | } 87 | -------------------------------------------------------------------------------- /internal/services/template_handler.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package services 16 | 17 | import ( 18 | "bytes" 19 | "context" 20 | "embed" 21 | "errors" 22 | "fmt" 23 | "os" 24 | "strings" 25 | "text/template" 26 | 27 | logger "github.com/AppsFlyer/go-logger" 28 | "github.com/hashicorp/go-version" 29 | "github.com/hashicorp/hc-install/product" 30 | "github.com/hashicorp/hc-install/releases" 31 | "github.com/hashicorp/terraform-exec/tfexec" 32 | ) 33 | 34 | var ( 35 | //go:embed templates/* 36 | assets embed.FS //nolint: gochecknoglobals // no other possibility 37 | ) 38 | 39 | type TemplateHandler struct { 40 | logger logger.Logger 41 | } 42 | 43 | func NewTemplateHandler(log logger.Logger) *TemplateHandler { 44 | return &TemplateHandler{ 45 | logger: log, 46 | } 47 | } 48 | 49 | func (th *TemplateHandler) runTerraformFmt(path string) error { 50 | installer := &releases.ExactVersion{ 51 | Product: product.Terraform, 52 | Version: version.Must(version.NewVersion("1.0.6")), 53 | } 54 | 55 | execPath, err := installer.Install(context.Background()) 56 | if err != nil { 57 | th.logger.Error("Failed installing terraform", err.Error()) 58 | 59 | return err 60 | } 61 | 62 | workingDir := path 63 | tf, err := tfexec.NewTerraform(workingDir, execPath) 64 | if err != nil { 65 | th.logger.Error("failed running NewTerraform", err.Error()) 66 | 67 | return err 68 | } 69 | 70 | if err := tf.FormatWrite(context.Background()); err != nil { 71 | th.logger.Error("failed running Show", err.Error()) 72 | 73 | return err 74 | } 75 | 76 | return nil 77 | } 78 | 79 | func (th *TemplateHandler) WriteTemplateToFile(fileName, templatePath, destinationPath string, out interface{}, isDefaultTemplate bool) error { 80 | tmpl, err := th.GetTemplate(templatePath, isDefaultTemplate) 81 | if err != nil { 82 | return err 83 | } 84 | 85 | buf := new(bytes.Buffer) 86 | if err = tmpl.Execute(buf, out); err != nil { 87 | th.logger.Error("Failed executing template", err.Error()) 88 | return err 89 | } 90 | 91 | filePath := fmt.Sprintf("%s/%s", destinationPath, fileName) 92 | if err = os.Remove(filePath); (err != nil) && (!errors.Is(err, os.ErrNotExist)) { 93 | th.logger.Error("Failed removing file", err.Error()) 94 | return err 95 | } 96 | 97 | file, err := os.OpenFile(filePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) 98 | if err != nil { 99 | th.logger.Error("Failed opening file", err.Error()) 100 | return err 101 | } 102 | 103 | defer file.Close() 104 | 105 | _, err = file.WriteString(buf.String()) 106 | if err != nil { 107 | return err 108 | } 109 | 110 | if err := th.runTerraformFmt(destinationPath); err != nil { 111 | th.logger.Errorf("Failed running terraform FMT please make sure directory is correct, or external template if provided is correct") 112 | } 113 | 114 | return nil 115 | } 116 | 117 | func (th *TemplateHandler) GetTemplate(templatePath string, isDefaultTemplate bool) (*template.Template, error) { 118 | splittedPath := strings.Split(templatePath, "/") 119 | templateName := splittedPath[len(splittedPath)-1] 120 | 121 | apiFunc := NewTemplateAPI() 122 | if isDefaultTemplate { 123 | langs, _ := assets.ReadFile(fmt.Sprintf("templates/%s", templateName)) 124 | 125 | return template.New(templateName).Funcs(*apiFunc.APIFuncMap).Parse(string(langs)) 126 | } 127 | 128 | return template.New(templateName).Funcs(*apiFunc.APIFuncMap).ParseFiles(templatePath) 129 | } 130 | -------------------------------------------------------------------------------- /mock/modules/consul_sync/variables.tf: -------------------------------------------------------------------------------- 1 | variable "is_enabled" { 2 | type = bool 3 | description = "(Optional) Create module if variable set to true, default = true" 4 | default = true 5 | } 6 | 7 | variable "name" { 8 | type = string 9 | description = "(Optional) The helm chart Release name." 10 | default = "consul" 11 | } 12 | 13 | variable "chart" { 14 | type = string 15 | description = "(Optional) Chart name to be installed. The chart name can be local path, a URL to a chart, or the name of the chart if repository is specified. It is also possible to use the / format here if you are running Terraform on a system that the repository has been added to with helm repo add but this is not recommended." 16 | default = "consul" 17 | } 18 | 19 | variable "repository" { 20 | type = string 21 | description = "(Optional) Repository URL where to locate the requested chart." 22 | default = "https://helm.releases.hashicorp.com" 23 | } 24 | 25 | variable "chart_version" { 26 | type = string 27 | description = "(Optional) Helm chart version." 28 | default = "0.39.0" 29 | } 30 | 31 | variable "namespace" { 32 | type = string 33 | description = "(Optional) The namespace to install the release into." 34 | default = "consul" 35 | } 36 | 37 | variable "create_namespace" { 38 | type = bool 39 | description = "(Optional) Create the namespace if it does not yet exist" 40 | default = true 41 | } 42 | 43 | variable "atomic" { 44 | type = bool 45 | description = "(Optional) If set, installation process purges chart on fail. The wait flag will be set automatically if atomic is used." 46 | default = true 47 | } 48 | 49 | variable "wait" { 50 | type = bool 51 | description = "(Optional) Will wait until all resources are in a ready state before marking the release as successful. It will wait for as long as timeout." 52 | default = true 53 | } 54 | 55 | variable "wait_for_jobs" { 56 | type = bool 57 | description = "(Optional) will wait until all Jobs have been completed before marking the release as successful." 58 | default = true 59 | } 60 | 61 | variable "timeout" { 62 | type = number 63 | description = "(Optional) Time in seconds to wait for any individual kubernetes operation (like Jobs for hooks)." 64 | default = 300 65 | } 66 | 67 | variable "cleanup_on_fail" { 68 | type = bool 69 | description = "(Optional) Allow deletion of new resources created in this upgrade when upgrade fails." 70 | default = true 71 | } 72 | 73 | variable "watch_any_namespace" { 74 | type = bool 75 | description = "(Optional) Tells strimzi to watch all namespaces , if used 1 strimzi operator on eks cluster then should be set to true." 76 | default = true 77 | } 78 | 79 | variable "consul_image" { 80 | type = string 81 | description = "(Optional) The consul image:tag name" 82 | default = "consul:1.11.0" 83 | } 84 | 85 | variable "consul_datacenter" { 86 | type = string 87 | description = "(Optional) The consul datacenter name." 88 | default = "dev-euw1-general" 89 | } 90 | 91 | variable "consul_servers_address" { 92 | type = string 93 | description = "(Optional) The consul server addresses to which the consul clients will connect to." 94 | default = "dev-euw1-general.consul.appsflyer.platform" 95 | } 96 | 97 | variable "eks_cluster" { 98 | type = string 99 | description = "(Required) The name of the EKS Cluster. Needed for the consul clients nodes names to be attached to the consul server." 100 | } 101 | 102 | variable "affinity" { 103 | type = map(string) 104 | description = "(Required) Labels and Taints that are used on nodes with nvme storage." 105 | } 106 | 107 | variable "resources" { 108 | type = map(string) 109 | description = <= 1 { 121 | version = verMatch[1] 122 | } 123 | 124 | modulePathReg := regexp.MustCompile(modulePathRegex) 125 | modulePathMatch := modulePathReg.FindStringSubmatch(line) 126 | modulePath := "" 127 | if len(modulePathMatch) >= 1 { 128 | modulePath = modulePathMatch[1] 129 | } 130 | 131 | if len(urlMatch) >= 1 { 132 | gitUrl := strings.TrimSuffix(urlMatch[1], `"`) 133 | 134 | return &version_control.RemoteModule{Name: moduleName, Url: gitUrl, Version: version, Path: modulePath}, nil 135 | } 136 | 137 | gitUrl := strings.TrimSuffix(urlVersionMatch[1], `"`) 138 | 139 | return &version_control.RemoteModule{Name: moduleName, Url: gitUrl, Version: version, Path: modulePath}, nil 140 | } 141 | -------------------------------------------------------------------------------- /cmd/app/generate_all.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package app 16 | 17 | import ( 18 | "github.com/AppsFlyer/terra-crust/cmd/types" 19 | "github.com/spf13/cobra" 20 | 21 | template_reader "github.com/AppsFlyer/terra-crust/internal/services/drivers/template_reader" 22 | version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" 23 | ) 24 | 25 | func generateAllFiles(root *RootCommand) *cobra.Command { 26 | flags := types.TFGenerateFlags{} 27 | cmd := &cobra.Command{ 28 | Use: "terraform-all", 29 | Short: "generate terraform locals , main and variables", 30 | Example: "", 31 | RunE: func(cmd *cobra.Command, args []string) error { 32 | log := root.log 33 | templateReader := template_reader.InitTemplateRemoteModule(log) 34 | gitDriver := version_control.InitGitProvider(log) 35 | 36 | if (flags.FetchRemote || flags.ExternalGit) && flags.MainTemplateFilePath != "" { 37 | remoteModulesMap, err := templateReader.GetRemoteModulesFromTemplate(flags.MainTemplateFilePath) 38 | if err != nil { 39 | log.Error("Failed parsing remote modules from custom template", err.Error()) 40 | 41 | return err 42 | } 43 | 44 | if err = gitDriver.CloneModules(remoteModulesMap, flags.SourcePath, flags.ExternalGit); err != nil { 45 | log.Error("Failed cloning remote modules ", err.Error()) 46 | 47 | return err 48 | } 49 | defer func() { 50 | if err = gitDriver. // The `CleanModulesFolders` function is used to clean up the folders of remote 51 | // modules that were cloned during the execution of the program. It takes the 52 | // `remoteModulesMap` and `flags.SourcePath` as parameters. It is called as a 53 | // deferred function at the end of the `generateAllFiles` command. 54 | CleanModulesFolders(remoteModulesMap, flags.SourcePath); err != nil { 55 | log.Errorf("Failed to clean up some of the remote resources please make sure to clean it manually and check the error , %s", err.Error()) 56 | } 57 | }() 58 | } 59 | 60 | terraformSvc := InitTerraformGeneratorService(log) 61 | 62 | if err := terraformSvc.GenerateMain(flags.SourcePath, flags.DestinationPath, flags.MainTemplateFilePath); err != nil { 63 | log.Error("Failed generating the terraform main file", err.Error()) 64 | 65 | return err 66 | } 67 | 68 | if err := terraformSvc.GenerateModuleDefaultLocals(flags.SourcePath, flags.DestinationPath); err != nil { 69 | log.Error("Failed generating the terraform locals file", err.Error()) 70 | 71 | return err 72 | } 73 | 74 | if err := terraformSvc.GenerateModuleVariableObject(flags.SourcePath, flags.DestinationPath); err != nil { 75 | log.Error("Failed generating the terraform variable file", err.Error()) 76 | 77 | return err 78 | } 79 | 80 | return nil 81 | }, 82 | } 83 | 84 | cmd.Flags().StringVar(&flags.SourcePath, "source-path", "", "Required: General module folder path that contains all the sub modules flattened") 85 | cmd.Flags().StringVar(&flags.DestinationPath, "destination-path", "", "Required: Destination path to write the new terraform file") 86 | cmd.Flags().StringVar(&flags.MainTemplateFilePath, "main-template-path", "", "Optional: Custom main template path for generated module, will take default if not provided") 87 | cmd.Flags().BoolVar(&flags.FetchRemote, "fetch-remote", false, "Optional: Enable fetching of remote modules and exporting their variables in root module") 88 | cmd.Flags().BoolVar(&flags.ExternalGit, "ext-git", false, "Optional: Enable external git for remote modules") 89 | if err := cmd.MarkFlagRequired("source-path"); err != nil { 90 | root.log.Error("failed to set required flag on source-path", err.Error()) 91 | } 92 | if err := cmd.MarkFlagRequired("destination-path"); err != nil { 93 | root.log.Error("failed to set required flag on destination-path", err.Error()) 94 | } 95 | 96 | return cmd 97 | } 98 | -------------------------------------------------------------------------------- /.github/workflows/golang-build-test-coverage.yml: -------------------------------------------------------------------------------- 1 | name: golang-build-test-coverage 2 | on: 3 | push: 4 | branches-ignore: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | release: 10 | 11 | jobs: 12 | conventional_commits: 13 | name: Conventional Commits 14 | runs-on: ubuntu-latest 15 | steps: 16 | - uses: actions/checkout@v2 17 | - uses: webiny/action-conventional-commits@v1.0.5 18 | 19 | build: 20 | name: Build 21 | runs-on: ubuntu-latest 22 | steps: 23 | - name: Checkout 24 | uses: actions/checkout@v3 25 | - name: Setup go 26 | uses: actions/setup-go@v3 27 | with: 28 | go-version-file: './go.mod' 29 | - name: Get dependencies 30 | run: | 31 | go get -v -t -d ./... 32 | if [ -f Gopkg.toml ]; then 33 | curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh 34 | dep ensure 35 | fi 36 | - name: Build 37 | run: go build -v cmd/main.go 38 | 39 | lint: 40 | name: Lint 41 | runs-on: ubuntu-latest 42 | steps: 43 | - name: Checkout 44 | uses: actions/checkout@v3 45 | - name: Setup go 46 | uses: actions/setup-go@v3 47 | with: 48 | go-version-file: './go.mod' 49 | - name: Lint 50 | uses: golangci/golangci-lint-action@v3 51 | # continue-on-error: true 52 | with: 53 | # Optional: version of golangci-lint to use in form of v1.2 or v1.2.3 or `latest` to use the latest version 54 | version: v1.56.0 55 | 56 | # Optional: working directory, useful for monorepos 57 | # TODO: remove before release 58 | working-directory: internal 59 | 60 | # Optional: golangci-lint command line arguments. 61 | args: 62 | 63 | # Optional: show only new issues if it's a pull request. The default value is `false`. 64 | # only-new-issues: true 65 | 66 | # Optional: if set to true then the all caching functionality will be complete disabled, 67 | # takes precedence over all other caching options. 68 | # skip-cache: true 69 | 70 | # Optional: if set to true then the action don't cache or restore ~/go/pkg. 71 | # skip-pkg-cache: true 72 | 73 | # Optional: if set to true then the action don't cache or restore ~/.cache/go-build. 74 | # skip-build-cache: true 75 | 76 | vet: 77 | name: Vet 78 | runs-on: ubuntu-latest 79 | steps: 80 | - name: Checkout 81 | uses: actions/checkout@v3 82 | - name: Setup go 83 | uses: actions/setup-go@v3 84 | with: 85 | go-version-file: './go.mod' 86 | - name: Vet 87 | run: | 88 | go vet ./... 89 | 90 | test: 91 | name: Unit tests 92 | runs-on: ubuntu-latest 93 | steps: 94 | - name: Checkout 95 | uses: actions/checkout@v3 96 | - name: Setup credentials 97 | uses: fusion-engineering/setup-git-credentials@v2 98 | with: 99 | credentials: https://x-access-token:${{ secrets.GITHUB_TOKEN }}@github.com 100 | - name: Setup go 101 | uses: actions/setup-go@v3 102 | with: 103 | go-version-file: './go.mod' 104 | - name: Run unit tests 105 | run: | 106 | go test ./... -coverprofile coverage.out -covermode atomic 107 | go tool cover -func coverage.out 108 | - uses: actions/upload-artifact@v2 109 | with: 110 | name: coverage_report 111 | path: coverage.out 112 | 113 | coverage: 114 | name: Coverage report 115 | runs-on: ubuntu-latest 116 | needs: test 117 | steps: 118 | - name: Checkout 119 | uses: actions/checkout@v3 120 | - name: Setup go 121 | uses: actions/setup-go@v3 122 | with: 123 | go-version-file: './go.mod' 124 | - uses: actions/download-artifact@v2 125 | with: 126 | name: coverage_report 127 | - name: Upload coverage to Codecov 128 | uses: codecov/codecov-action@v2 129 | with: 130 | files: coverage.out 131 | # dry_run: true 132 | fail_ci_if_error: true 133 | flags: unittests 134 | - name: Quality Gate - Test coverage shall be above threshold 135 | env: 136 | TESTCOVERAGE_THRESHOLD: 80 137 | run: | 138 | echo "Quality Gate: checking test coverage is above threshold ..." 139 | echo "Threshold : $TESTCOVERAGE_THRESHOLD %" 140 | totalCoverage=`go tool cover -func=coverage.out | grep total | grep -Eo '[0-9]+\.[0-9]+'` 141 | echo "Current test coverage : $totalCoverage %" 142 | if (( $(echo "$totalCoverage $TESTCOVERAGE_THRESHOLD" | awk '{print ($1 > $2)}') )); then 143 | echo "OK" 144 | else 145 | echo "Current test coverage is below threshold. Please add more unit tests or adjust threshold to a lower value." 146 | echo "Failed" 147 | exit 1 148 | fi -------------------------------------------------------------------------------- /internal/services/drivers/parser/terraform_parser.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package parser 16 | 17 | import ( 18 | "errors" 19 | "os" 20 | "path/filepath" 21 | "strings" 22 | 23 | "github.com/AppsFlyer/terra-crust/internal/services/drivers" 24 | 25 | logger "github.com/AppsFlyer/go-logger" 26 | 27 | "github.com/AppsFlyer/terra-crust/internal/types" 28 | 29 | "github.com/hashicorp/hcl/v2" 30 | "github.com/hashicorp/hcl/v2/hclwrite" 31 | "github.com/zclconf/go-cty/cty" 32 | ) 33 | 34 | const ( 35 | HclBlockDescriptionField = "description" 36 | HclBlockValueField = "value" 37 | ) 38 | 39 | type TerraformParser struct { 40 | logger logger.Logger 41 | } 42 | 43 | func NewTerraformParser(log logger.Logger) drivers.Parser { 44 | return &TerraformParser{ 45 | logger: log, 46 | } 47 | } 48 | 49 | // Parse parses a local terraform module and returns module structs 50 | func (p *TerraformParser) Parse(path string) (map[string]*types.Module, error) { 51 | modulesMap := make(map[string]*types.Module) 52 | err := filepath.Walk(path, 53 | func(path string, info os.FileInfo, err error) error { 54 | if err != nil { 55 | p.logger.Error("failed scraping folder on filepath.walk %s", err.Error()) 56 | 57 | return err 58 | } 59 | 60 | if info.IsDir() && info.Name() == types.TerraformMetaDirectoryName { 61 | return filepath.SkipDir 62 | } 63 | 64 | if info.IsDir() || !strings.HasSuffix(info.Name(), ".tf") { 65 | return nil 66 | } 67 | 68 | src, err := os.ReadFile(path) 69 | if err != nil { 70 | p.logger.Error("Failed reading file under path, make sure files exist %s", err.Error()) 71 | 72 | return err 73 | } 74 | 75 | file, diags := hclwrite.ParseConfig(src, path, hcl.InitialPos) 76 | if diags.HasErrors() { 77 | return errors.New(diags.Error()) 78 | } 79 | 80 | body := file.Body() 81 | splittedPath := strings.Split(path, "/") 82 | // Extracting module name based on folder file location 83 | moduleName := splittedPath[len(splittedPath)-2] 84 | 85 | if _, ok := modulesMap[moduleName]; !ok { 86 | modulesMap[moduleName] = &types.Module{ 87 | Variables: make([]*types.Variable, 0), 88 | Outputs: make([]*types.Output, 0), 89 | Resources: make([]*types.Resource, 0), 90 | } 91 | } 92 | 93 | for _, block := range body.Blocks() { 94 | switch block.Type() { 95 | case "variable": 96 | modulesMap[moduleName].Variables = append(modulesMap[moduleName].Variables, p.parseVariable(block)) 97 | case "output": 98 | modulesMap[moduleName].Outputs = append(modulesMap[moduleName].Outputs, p.parseOutput(block)) 99 | case "resource": 100 | modulesMap[moduleName].Resources = append(modulesMap[moduleName].Resources, p.parseResource(block)) 101 | } 102 | } 103 | 104 | return nil 105 | }) 106 | if err != nil { 107 | p.logger.Error("failed on scraping root folder please make sure path is correct, %s", err.Error()) 108 | 109 | return nil, err 110 | } 111 | 112 | return modulesMap, nil 113 | } 114 | 115 | func (p *TerraformParser) parseVariable(block *hclwrite.Block) *types.Variable { 116 | variable := types.Variable{ 117 | Name: block.Labels()[0], 118 | Default: hclwrite.TokensForValue(cty.StringVal("")), 119 | } 120 | body := block.Body() 121 | for k, v := range body.Attributes() { 122 | switch k { 123 | case "type": 124 | variable.Type = v.Expr().BuildTokens(nil) 125 | case "default": 126 | variable.Default = v.Expr().BuildTokens(nil) 127 | case HclBlockDescriptionField: 128 | description := string(v.Expr().BuildTokens(nil).Bytes()) 129 | variable.Description = description[2 : len(description)-1] 130 | } 131 | } 132 | 133 | return &variable 134 | } 135 | 136 | func (p *TerraformParser) parseOutput(block *hclwrite.Block) *types.Output { 137 | output := types.Output{ 138 | Name: block.Labels()[0], 139 | Description: "", 140 | } 141 | body := block.Body() 142 | for k, v := range body.Attributes() { 143 | switch k { 144 | case HclBlockValueField: 145 | var typeTokens hclwrite.Tokens 146 | typeTokens = append(typeTokens, v.Expr().BuildTokens(nil)...) 147 | output.Value = typeTokens 148 | case HclBlockDescriptionField: 149 | description := string(v.Expr().BuildTokens(nil).Bytes()) 150 | output.Description = description[2 : len(description)-1] 151 | } 152 | } 153 | 154 | return &output 155 | } 156 | 157 | func (p *TerraformParser) parseResource(block *hclwrite.Block) *types.Resource { 158 | resource := types.Resource{ 159 | Type: block.Labels()[0], 160 | Name: block.Labels()[1], 161 | } 162 | 163 | return &resource 164 | } 165 | -------------------------------------------------------------------------------- /internal/services/drivers/version_control/git_test.go: -------------------------------------------------------------------------------- 1 | package version_control_test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | logger "github.com/AppsFlyer/go-logger" 9 | version_control "github.com/AppsFlyer/terra-crust/internal/services/drivers/version_control" 10 | ) 11 | 12 | const ( 13 | TerraCrustModuleName = "terracrust" 14 | NamingModuleName = "terraform-aws-resource-naming" 15 | ZonesModuleName = "zones" 16 | TerraCrustURL = "https://github.com/AppsFlyer/terra-crust.git" 17 | ) 18 | 19 | var ModulesTestPath = "./temp-git-test" 20 | 21 | var mockBadUrl = map[string]*version_control.RemoteModule{ 22 | TerraCrustModuleName: { 23 | Url: "https://github.com/appsflyer/terra-crust/test/bad", 24 | }, 25 | } 26 | 27 | var mockBadVersion = map[string]*version_control.RemoteModule{ 28 | NamingModuleName: { 29 | Url: "https://github.com/fajrinazis/terraform-aws-resource-naming.git", 30 | Version: "bad-tag", 31 | }, 32 | } 33 | 34 | var mockBadRemote = map[string]*version_control.RemoteModule{ 35 | NamingModuleName: { 36 | Url: "https://gitbad.com/fajrinazis/terraform-aws-resource-naming.git", 37 | Version: "bad-tag", 38 | }, 39 | } 40 | 41 | func TestCloneAndCleanupInternalGit(t *testing.T) { 42 | CloneAndCleanup(t, false) 43 | } 44 | 45 | func TestCloneAndCleanupExternalGit(t *testing.T) { 46 | CloneAndCleanup(t, true) 47 | } 48 | 49 | func CloneAndCleanupModules(modules map[string]*version_control.RemoteModule, externalGit bool) error { 50 | log := logger.NewSimple() 51 | 52 | gitDriver := version_control.InitGitProvider(log) 53 | 54 | err := gitDriver.CloneModules(modules, ModulesTestPath, externalGit) 55 | cErr := gitDriver.CleanModulesFolders(modules, ModulesTestPath) 56 | if err != nil { 57 | if cErr != nil { 58 | return fmt.Errorf("failed to clone and cleanup module %v %v", err, cErr) 59 | } 60 | return err 61 | } 62 | return nil 63 | } 64 | func CloneAndCleanup(t *testing.T, externalGit bool) { 65 | var mockModules = map[string]*version_control.RemoteModule{ 66 | TerraCrustModuleName: { 67 | Url: TerraCrustURL, 68 | }, 69 | NamingModuleName: { 70 | Url: "https://github.com/fajrinazis/terraform-aws-resource-naming.git", 71 | Version: "v0.3.0", 72 | }, 73 | ZonesModuleName: { 74 | Url: "https://github.com/terraform-aws-modules/terraform-aws-route53.git", 75 | Path: "modules/zones", 76 | }, 77 | } 78 | log := logger.NewSimple() 79 | 80 | gitDriver := version_control.InitGitProvider(log) 81 | 82 | err := gitDriver.CloneModules(mockModules, ModulesTestPath, externalGit) 83 | if err != nil { 84 | t.Errorf(err.Error()) 85 | } 86 | 87 | folders, err := getFolderAsMap(ModulesTestPath) 88 | if err != nil { 89 | t.Errorf(err.Error()) 90 | } 91 | 92 | if _, exist := folders[TerraCrustModuleName]; !exist { 93 | t.Errorf("Clone failed , did not find terra-crust") 94 | } 95 | 96 | if _, exist := folders[ZonesModuleName]; !exist { 97 | t.Errorf("Clone failed , did not find zones") 98 | } 99 | 100 | if _, exist := folders[NamingModuleName]; !exist { 101 | t.Errorf("Clone failed , did not find naming") 102 | } 103 | 104 | if len(folders) != 3 { 105 | t.Errorf("Expected 3 folder count received %d", len(folders)) 106 | } 107 | 108 | if err = gitDriver.CleanModulesFolders(mockModules, ModulesTestPath); err != nil { 109 | t.Errorf("failed to clean up the downloaded modules, %s", err.Error()) 110 | } 111 | 112 | folders, err = getFolderAsMap(ModulesTestPath) 113 | if err != nil { 114 | t.Errorf(err.Error()) 115 | } 116 | 117 | if _, exist := folders[TerraCrustModuleName]; exist { 118 | t.Errorf("cleanup-failed terracrust folder still exists") 119 | } 120 | 121 | if _, exist := folders[ZonesModuleName]; exist { 122 | t.Errorf("cleanup-failed zones folder still exists") 123 | } 124 | 125 | if _, exist := folders[NamingModuleName]; exist { 126 | t.Errorf("cleanup-failed naming folder still exists") 127 | } 128 | 129 | if len(folders) != 0 { 130 | t.Errorf("Expected 0 folder count received %d", len(folders)) 131 | } 132 | 133 | } 134 | func TestFailBadUrlInternalGit(t *testing.T) { 135 | FailBadUrl(t, false) 136 | } 137 | 138 | func TestFailBadUrlExternalGit(t *testing.T) { 139 | FailBadUrl(t, true) 140 | } 141 | 142 | func TestFailBadVersionInternalGit(t *testing.T) { 143 | FailBadVersion(t, false) 144 | } 145 | 146 | func TestFailBadVersionExternalGit(t *testing.T) { 147 | FailBadVersion(t, true) 148 | } 149 | 150 | func TestFailBadRemoteInternalGit(t *testing.T) { 151 | FailGetGitCredentials(t, false) 152 | } 153 | 154 | func TestFailBadRemoteExternalGit(t *testing.T) { 155 | FailGetGitCredentials(t, true) 156 | } 157 | 158 | func FailBadUrl(t *testing.T, externalGit bool) { 159 | t.Parallel() 160 | 161 | err := CloneAndCleanupModules(mockBadUrl, externalGit) 162 | if err == nil { 163 | t.Errorf("expected error received error nil") 164 | } 165 | } 166 | 167 | func FailBadVersion(t *testing.T, externalGit bool) { 168 | t.Parallel() 169 | 170 | err := CloneAndCleanupModules(mockBadVersion, externalGit) 171 | if err == nil { 172 | t.Errorf("expected error received error nil") 173 | } 174 | } 175 | 176 | func FailGetGitCredentials(t *testing.T, externalGit bool) { 177 | log := logger.NewSimple() 178 | gitDriver := version_control.InitGitProvider(log) 179 | err := gitDriver.CloneModules(mockBadRemote, ModulesTestPath, externalGit) 180 | if err == nil { 181 | t.Errorf("expected error received error nil") 182 | } 183 | } 184 | 185 | func getFolderAsMap(path string) (map[string]struct{}, error) { 186 | folders := make(map[string]struct{}) 187 | 188 | files, err := os.ReadDir(path) 189 | if err != nil { 190 | return nil, err 191 | } 192 | 193 | for _, file := range files { 194 | if file.IsDir() { 195 | folders[file.Name()] = struct{}{} 196 | } 197 | } 198 | 199 | return folders, err 200 | } 201 | -------------------------------------------------------------------------------- /examples/example.md: -------------------------------------------------------------------------------- 1 | ## Example Usage 2 | 3 | Let's look at following terraform project structure: 4 | 5 | ``` 6 | . 7 | ├── main.tf 8 | └── modules 9 | ├── moduleA 10 | ├── moduleB 11 | └── ... 12 | ``` 13 | Terracrust can automatically extract all used varialbes and locals block from the modules folder, to generate a new layer that combines all modules to a single, simplified, root module. 14 | ``` 15 | . 16 | ├── developers_main.tf 17 | └── root_module 18 | ├── module_variables.tf 19 | ├── main.tf.tmpl 20 | ├── module_locals.tf 21 | ├── module_main.tf 22 | └── modules 23 | ├── moduleA 24 | ├── moduleB 25 | └── ... 26 | ``` 27 | 28 | ### TerraCrust Usage 29 | let's start from the modules layer. Let's say, both modules A and B have the same following variables in them: 30 | ```hcl 31 | variable "foo" { 32 | type = string 33 | description = "Optional variable." 34 | default = "bar" 35 | } 36 | 37 | variable "required" { 38 | type = string 39 | description = "Required variable." 40 | } 41 | ``` 42 | 43 | The main layer above will look samiliar to the following: 44 | ```hcl 45 | module "moduleA" { 46 | source = "./modules/moduleA" 47 | required = var.moduleA_requried_value 48 | } 49 | 50 | module "moduleB" { 51 | source = "./modules/moduleB" 52 | required = var.moduleB_requried_value 53 | } 54 | ``` 55 | 56 | When you wish to extract this structure to another layer, meaning, have a root module that contains both module A and B respectfully. \ 57 | You will need to extract their varialbes to the same main layer, including optional values configured at the module layer, so that their logical connection will not break. 58 | 59 | When looking at this example, it's pretty simple to do this manually, but what about cases when you have over 10 modules to manage, and each module contains 10 different/similiar variables. This can lead to managing over 100 varialbes in total, copying and pasting them with the same logical structure. 60 | 61 | This is where TerraCrust comes in place, by providing a path to the modules folder, TerraCrust will automatically extract and organize a `module_variables.tf`, a `module_locals.tf` as well as a `module_main.tf`, with all the modules default variables in them. 62 | 63 | Each module's variables is looked as an object, containing all it's optional variables and their optional values respectfully under `module_varialbes.tf`: 64 | ```hcl 65 | variable "moduleA" { 66 | type = object({ 67 | foo = optional(string) # The variable name and type 68 | }) 69 | default = { 70 | foo = "bar" # The default found in the module folder 71 | } 72 | } 73 | 74 | variable "moduleB" { 75 | type = object({ 76 | foo = optional(string) 77 | }) 78 | default = { 79 | foo = "bar" 80 | } 81 | } 82 | ``` 83 | 84 | `module_locals.tf`, each variable in every module object has a coalesce block that either uses the default value provided at the module level, or if a user entered a different default value, the locals will take it to account: 85 | ```hcl 86 | locals { 87 | moduleA = { 88 | foo = coalesce(var.drain_cleaner.foo, "bar") 89 | } 90 | moduleB = { 91 | foo = coalesce(var.drain_cleaner.foo, "bar") 92 | } 93 | } 94 | ``` 95 | `module_main.tf`, which contains all the connection to the default values, as well the required variables needed to be filled before apply will also be generated: 96 | ```hcl 97 | module "moduleA" { 98 | source = "./modules/moduleA" 99 | required = #TO DO: fill requried variable 100 | 101 | foo = local.moduleA.foo 102 | } 103 | 104 | module "moduleB" { 105 | source = "./modules/moduleB" 106 | required = #TO DO: fill requried variable 107 | 108 | foo = local.moduleB.foo 109 | } 110 | ``` 111 | 112 | Leaving the users to use a single ready module, combining everything in a simple way using the following structure like `developers_main.tf`: 113 | ```hcl 114 | module "root_module" { 115 | source = "./root_module" 116 | moduleA { 117 | requried = "user_input" 118 | } 119 | moduleB { 120 | requried = "user_input" 121 | foo = "foobar" # example of changing default value. 122 | } 123 | } 124 | ``` 125 | Saving time for platform teams extracting every module variable manually, and providing developers a simple, singular module to use without the need to deep dive on how each module works. 126 | 127 | ### Map type values usage 128 | When using map type variables. Use the following structure. 129 | under `variables.tf` in the module level: 130 | ```hcl 131 | variable "map_varialbe" { 132 | type = map(string) 133 | default = { 134 | "keyA" = "valueA", 135 | "keyB" = "ValueB", 136 | "KeyC" = "ValueC" 137 | } 138 | } 139 | ``` 140 | Create a map type variables with it's default values. 141 | 142 | under `locals.tf`: 143 | ```hcl 144 | locals { 145 | map_varialbe = merge( 146 | tomap({ 147 | "keyA" = "valueA", 148 | "keyB" = "ValueB", 149 | "KeyC" = "ValueC" 150 | }), 151 | var.map_varialbe, 152 | ) 153 | } 154 | ``` 155 | Create a tomap function, so when a user changes a value at the top level, it will not earase all other keys in the map. 156 | 157 | 158 | ### Working with main template usage 159 | 160 | TerraCrust also give the ability to use a ready main.tf template, that contains all module required variables dependencies, without changing them for every run. 161 | 162 | Simply take your ready made main.tf, with it's logical dependencies, and change the end of the file to `main.tmpl`. Add the following lines in the place you wish to add the default values to: 163 | 164 | ```hcl 165 | module "moduleA" { 166 | source = "./modules/moduleA" 167 | required = var.moduleA_User_Input 168 | 169 | {{(GetDefaults "moduleA" .)}} 170 | } 171 | 172 | module "moduleB" { 173 | source = "./modules/moduleB" 174 | required = module.moduleA.output 175 | 176 | {{(GetDefaults "moduleB" .)}} 177 | } 178 | ``` 179 | The function GetDefaults, will add the optional values to each module block, without chainging the required values. It uses the name of the module folder give under `source`. 180 | 181 | Example, this `module_main.tf` will get generated: 182 | ```hcl 183 | module "moduleA" { 184 | source = "./modules/moduleA" 185 | required = var.moduleA_User_Input # Remains the same 186 | 187 | foo = local.moduleA.foo # Gets added automatically 188 | } 189 | 190 | module "moduleB" { 191 | source = "./modules/moduleB" 192 | required = module.moduleA.output # Remains the same 193 | 194 | foo = local.moduleB.foo # Gets added automatically 195 | } 196 | ``` -------------------------------------------------------------------------------- /internal/services/drivers/version_control/git.go: -------------------------------------------------------------------------------- 1 | package version_control 2 | 3 | import ( 4 | "bytes" 5 | "context" 6 | "fmt" 7 | "os" 8 | "os/exec" 9 | "strings" 10 | "time" 11 | 12 | log "github.com/AppsFlyer/go-logger" 13 | "github.com/go-git/go-git/v5" /// with go modules disabled 14 | "github.com/go-git/go-git/v5/plumbing" 15 | "github.com/go-git/go-git/v5/plumbing/transport/http" 16 | cp "github.com/otiai10/copy" 17 | "github.com/pkg/errors" 18 | ) 19 | 20 | const ( 21 | FolderPathFormat = "%s/%s" 22 | TempFolderPath = "%s/temp_clone_path/%s" 23 | remoteName = "origin" 24 | GitlabTokenENV = "GITLAB_TOKEN" 25 | GitRefTag = "refs/tags/%s" 26 | GitRefBranch = "refs/remotes/%s/%s" 27 | GitCredentialUrl = "url=%s" 28 | GitCredentialDeadLineMs = 500 29 | GitCredentialUserNamePrefix = "username=" 30 | GitCredentialPasswordPrefix = "password=" 31 | GitlabUserENV = "GITLAB_USER" 32 | GithubTokenENV = "GITHUB_TOKEN" 33 | GithubUserENV = "GITHUB_USER" 34 | ) 35 | 36 | type RemoteModule struct { 37 | Name string 38 | Url string 39 | Version string 40 | Path string 41 | } 42 | 43 | type Git struct { 44 | log log.Logger 45 | } 46 | 47 | func InitGitProvider(log log.Logger) *Git { 48 | return &Git{ 49 | log: log, 50 | } 51 | } 52 | 53 | func (g *Git) CloneModules(modules map[string]*RemoteModule, modulesSource string, externalGit bool) error { 54 | for moduleName, moduleData := range modules { 55 | clonePath := fmt.Sprintf(FolderPathFormat, modulesSource, moduleName) 56 | if moduleData.Path != "" { 57 | clonePath = fmt.Sprintf(TempFolderPath, modulesSource, moduleName) 58 | } 59 | 60 | if err := g.clone(moduleData, clonePath, externalGit); err != nil { 61 | return errors.Wrapf(err, "failed to fetch module , url: %s", moduleData.Url) 62 | } 63 | 64 | g.log.Infof("Copy folder from : %s/%s to : %s", clonePath, moduleData.Path, modulesSource) 65 | 66 | if moduleData.Path != "" { 67 | modulePath := fmt.Sprintf(FolderPathFormat, modulesSource, moduleName) 68 | 69 | err := cp.Copy(fmt.Sprintf(FolderPathFormat, clonePath, moduleData.Path), modulePath) 70 | if err != nil { 71 | g.log.Errorf("failed to copy desired terraform module module path :%s, module name: %s", clonePath, moduleData.Name) 72 | } 73 | 74 | g.log.Infof("Changing module path from : %s to : %s", modules[moduleName].Path, modulePath) 75 | modules[moduleName].Path = moduleName 76 | } 77 | } 78 | 79 | if err := g.cleanTemp(modulesSource); err != nil { 80 | return errors.Wrap(err, "Failed to delete temp folder") 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func (g *Git) clone(moduleData *RemoteModule, directoryPath string, externalGit bool) error { 87 | 88 | userName, token, err := g.getGitCredentials(moduleData.Url, externalGit) 89 | if err != nil { 90 | return err 91 | } 92 | 93 | repo, err := git.PlainClone(directoryPath, false, &git.CloneOptions{ 94 | URL: moduleData.Url, 95 | Auth: &http.BasicAuth{Password: token, Username: userName}, 96 | RemoteName: remoteName, 97 | Depth: 1, 98 | }) 99 | if err != nil { 100 | return err 101 | } 102 | if moduleData.Version != "" { 103 | workTree, err := repo.Worktree() 104 | if err != nil { 105 | return err 106 | } 107 | tagRef := fmt.Sprintf(GitRefTag, moduleData.Version) 108 | g.log.Debugf("searching %s in %s", moduleData.Version, tagRef) 109 | err = workTree.Checkout(&git.CheckoutOptions{ 110 | Branch: plumbing.ReferenceName(tagRef), 111 | }) 112 | if err != nil { 113 | branchRef := fmt.Sprintf(GitRefBranch, remoteName, moduleData.Version) 114 | g.log.Debugf("version not found in tags ref, searching %s in %s", moduleData.Version, branchRef) 115 | bErr := workTree.Checkout(&git.CheckoutOptions{ 116 | Branch: plumbing.ReferenceName(branchRef), 117 | }) 118 | return bErr 119 | } 120 | } 121 | return nil 122 | } 123 | 124 | func (g *Git) CleanModulesFolders(modules map[string]*RemoteModule, modulesSource string) error { 125 | var returnedErr error = nil 126 | for moduleName := range modules { 127 | modulePath := fmt.Sprintf(FolderPathFormat, modulesSource, moduleName) 128 | 129 | err := os.RemoveAll(modulePath) 130 | if err != nil { 131 | g.log.Errorf("Failed to clear up a module %s at path: %s , error:%s", moduleName, modulePath, err.Error()) 132 | if returnedErr == nil { 133 | returnedErr = err 134 | } 135 | 136 | returnedErr = errors.Wrap(returnedErr, err.Error()) 137 | } 138 | } 139 | 140 | return returnedErr 141 | } 142 | 143 | func (g *Git) cleanTemp(modulesSourcePath string) error { 144 | tempPath := fmt.Sprintf("%s/temp_clone_path", modulesSourcePath) 145 | 146 | g.log.Infof("Deleting temp folder containing all clones") 147 | 148 | err := os.RemoveAll(tempPath) 149 | if err != nil { 150 | g.log.Errorf("Failed to clear up temp folder that been used for clone, please clean it manually , failed on temp folder, path : %s , err: %s", tempPath, err.Error()) 151 | 152 | return err 153 | } 154 | 155 | return nil 156 | } 157 | 158 | func (g *Git) getGitCredentials(url string, externalGit bool) (userName string, password string, err error) { 159 | if !externalGit { 160 | userName, password = g.getGitUserNameAndToken(url) 161 | if userName != "" && password != "" { 162 | return userName, password, nil 163 | } 164 | g.log.Warnf("credentials not found from env variables. falling back to git credentials") 165 | } 166 | // Required until https://github.com/go-git/go-git/issues/490 addressed 167 | ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(GitCredentialDeadLineMs*time.Millisecond)) 168 | cmd := exec.CommandContext(ctx, "git", "credential", "fill") 169 | defer cancel() 170 | cmd.Stdin = strings.NewReader(fmt.Sprintf(GitCredentialUrl, url)) 171 | var out bytes.Buffer 172 | cmd.Stdout = &out 173 | err = cmd.Run() 174 | if err != nil { 175 | return userName, password, err 176 | } 177 | lines := strings.Split(out.String(), "\n") 178 | for _, line := range lines { 179 | if strings.HasPrefix(line, GitCredentialUserNamePrefix) { 180 | userName = strings.TrimPrefix(line, GitCredentialUserNamePrefix) 181 | } 182 | if strings.HasPrefix(line, GitCredentialPasswordPrefix) { 183 | password = strings.TrimPrefix(line, GitCredentialPasswordPrefix) 184 | } 185 | } 186 | return userName, password, nil 187 | } 188 | func (g *Git) getGitUserNameAndToken(url string) (string, string) { 189 | if strings.Contains(url, "gitlab") { 190 | return os.Getenv(GitlabUserENV), os.Getenv(GitlabTokenENV) 191 | } 192 | 193 | if strings.Contains(url, "github") { 194 | return os.Getenv(GithubUserENV), os.Getenv(GithubTokenENV) 195 | } 196 | 197 | return "", "" 198 | } 199 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![Build](https://github.com/danielvrog/terra-crust/actions/workflows/golang-build-test-coverage.yml/badge.svg?branch=main) [![Go Report Card](https://goreportcard.com/badge/github.com/AppsFlyer/terra-crust)](https://goreportcard.com/report/github.com/AppsFlyer/terra-crust) [![codecov](https://codecov.io/gh/AppsFlyer/terra-crust/branch/main/graph/badge.svg)](https://codecov.io/gh/AppsFlyer/terra-crust) 2 | 3 | 4 | 5 | ---- 6 | Terra crust was created to allow platform teams to expose terraform as the main API communication with the developers, by giving a simple interface in Terraform. 7 | Terraform was adopted as our primary language because it is intuitive, declarative, and community driven. 8 | We wanted to find a way to have an excellent user experience while increasing the platform productivity, moreover to keep it accessible to use and decrease management overhead. 9 | Our goal in developing terra crust was to hide the system's complexity and provide a simple interface to the client based on Facade Pattern and KISS principles. 10 | 11 | 12 | 13 | ## HashiConf Europe 14 | 15 | Listen to a 12 minutes talk about Terra Crust at HashiConf conference Amsterdam June 2022 16 | 17 | [![Watch the video](https://img.youtube.com/vi/_LhbL0ZRz_c/maxresdefault.jpg)](https://www.youtube.com/watch?v=_LhbL0ZRz_c) 18 | 19 | ## AWS reinvent 2022 20 | 21 | Check the brakeout session **"Running high-throughput, real-time ad platforms in the cloud (ADM302)"** at AWS reinvent 2022 presenting Terra crust and the way we empower our developers. 22 | 23 | [![Watch the video](https://img.youtube.com/vi/iHNgS1YKtH8/maxresdefault.jpg)](https://www.youtube.com/watch?v=iHNgS1YKtH8&t=1348s) 24 | 25 | ## How it works 26 | 27 | Terra crust creates a **root module** from existing modules folders inside your terraform project, and exposes all the `variables`,`locals` and `main` used in each module automatically. \ 28 | It takes the following components and exports it through 3 files, `local.tf` , `variables.tf`, `main.tf`. 29 | * **local** - module_local.tf exports the locals of each sub module. 30 | * **variables** - module_variables.tf exports the variables of each sub module. 31 | * **main** - module_main.tf , exports the required variables as commented values in order to fill in by their logical connection. Optional values are genereated automatically by default, also supports an external main template for development. 32 | 33 | Terra crust is going over the module folders and extracting from the `Variables.tf` files the defaults values for each variable. \ 34 | Every variable that is `Optional` must have a default value in it's variable block, even if there is a merge on the locals that already contains the default. \ 35 | Terra crust will release 3 files that wraps all of the existing modules folder under a general module. \ 36 | At the end of the flow it will run Terraform FMT to format the files. 37 | 38 | ## Output Examples: 39 | 40 | ### Variables: 41 | ![](/docs/images/vars.jpg) 42 | 43 | ### Locals: 44 | ![](/docs/images/locals.jpg) 45 | 46 | ### Main: 47 | ![](/docs/images/main.jpg) 48 | 49 | ## Commands: 50 | ### Create Main: 51 | ``` 52 | terra-crust terraform-main --destination-path="." --source-path=".../modules" 53 | ``` 54 | * terra-crust has additional flag: ``main-template-path`` to support external main templates like in examples/templates/main.tf.tmpl 55 | ### Create Variables: 56 | ``` 57 | terra-crust terraform-variables --destination-path="." --source-path=".../modules" 58 | ``` 59 | ### Create Locals: 60 | ``` 61 | terra-crust terraform-locals --destination-path="." --source-path=".../modules" 62 | ``` 63 | ### Create All: 64 | ``` 65 | terra-crust terraform-all --destination-path="." --source-path=".../modules" 66 | ``` 67 | * same as Main has additional flag: ``main-template-path`` to support external main templates like in examples/templates/main.tf.tmpl 68 | * Upon failing on create one of the files , It wont fail the entire flow(Will keep on to the next files). 69 | 70 | ## Remote TerraCrust: 71 | 72 | ### Description 73 | From version v0.1.0 TerraCrust will support fetching remote modules 74 | and composite them into the root module that is being created. 75 | From version v0.1.5 TerraCrust will support fetching remote modules 76 | with git cli and not only with internal go git client. 77 | so save credentials can be used, instead of generating tokens. 78 | ### How to use 79 | in order to activate this feature all you have to do is to set `fetch-remote` to true like so: 80 | ``` bash 81 | terra-crust terraform-all --main-template-path=./terraform/main.tmpl --destination-path="." --source-path=".../modules" --fetch-remote=true 82 | ``` 83 | or with an external git credential with the flag `--ext-git` to true like so: 84 | ``` bash 85 | terra-crust terraform-all --main-template-path=./terraform/main.tmpl --destination-path="." --source-path=".../modules" --ext-git=true 86 | ``` 87 | 88 | When activating `FetchRemote` you must use Custom main template, terracrust will look for all sources 89 | that are from git it will look for this pattern: 90 | 1. `"git::REPOSITORY.git/PATH/MODULE"` 91 | 2. `"git::REPOSITORY"` 92 | it will start with git:: then the repository has to end with .git if it has 93 | more than 1 module inside it, if the repo is just the root module then no need to add .git to the end. 94 | it will also support versioning/ branch reference so feel free to add them , 95 | more examples could be found under `examples/templates`. 96 | 97 | Output Example for this module: 98 | `"git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account"` 99 | 100 | ```terraform 101 | module "iam-account" { 102 | source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account" 103 | 104 | # account_alias = module. TODO: Add Required Field 105 | 106 | 107 | hard_expiry = local.iam-account.hard_expiry 108 | require_numbers = local.iam-account.require_numbers 109 | require_symbols = local.iam-account.require_symbols 110 | create_account_password_policy = local.iam-account.create_account_password_policy 111 | allow_users_to_change_password = local.iam-account.allow_users_to_change_password 112 | minimum_password_length = local.iam-account.minimum_password_length 113 | password_reuse_prevention = local.iam-account.password_reuse_prevention 114 | require_lowercase_characters = local.iam-account.require_lowercase_characters 115 | require_uppercase_characters = local.iam-account.require_uppercase_characters 116 | get_caller_identity = local.iam-account.get_caller_identity 117 | max_password_age = local.iam-account.max_password_age 118 | 119 | } 120 | ``` 121 | 122 | this will propagate all the default and required variables using the Templates Api 123 | 124 | ## Template API: 125 | The main.tmpl exposing Template API that includes for now 2 functions: 126 | 1. GetRequired - Will expose the require variables with option for you to fill 127 | 2. GetDefaults - Will expose the optional variables - without needing to fill them up 128 | 129 | For example: 130 | ```terraform 131 | module "iam-account" { 132 | source = "git::https://github.com/terraform-aws-modules/terraform-aws-iam.git/modules/iam-account" 133 | 134 | {{(GetRequired "iam-account" .)}} 135 | 136 | {{(GetDefaults "iam-account" .)}} 137 | } 138 | ``` 139 | 140 | Results can be seen at the section above, basically after you fill the required variables, 141 | you want to get rid of `{{(GetRequired "iam-account" .)}}` because it will keep overwriting your changes 142 | so once you filled the required variables you can drop it instead and put the required variables you filled in the template. 143 | 144 | ## Example Usage 145 | Further guidelines as well as examples, can be shown [here](./examples/example.md). 146 | 147 | ## Contribution 148 | In order to contribute, please make sure to test and validate that everything is working, including lint ci, open a merge request and wait for approval. 149 | -------------------------------------------------------------------------------- /internal/services/terraform.go: -------------------------------------------------------------------------------- 1 | // Copyright 2022 AppsFlyer 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package services 16 | 17 | import ( 18 | "fmt" 19 | "sort" 20 | "strings" 21 | 22 | logger "github.com/AppsFlyer/go-logger" 23 | 24 | "github.com/AppsFlyer/terra-crust/internal/services/templates" 25 | "github.com/AppsFlyer/terra-crust/internal/types" 26 | ) 27 | 28 | const ( 29 | moduleDescription = `<