├── examples ├── pex │ ├── sample_python_script │ │ ├── __init__.py │ │ └── main.py │ ├── pex-env │ │ ├── build_scripts │ │ │ ├── .python-version │ │ │ └── build.sh │ │ ├── requirements.txt │ │ └── bin │ │ │ └── sample_python_script_py3_env.pex │ ├── outputs.tf │ ├── variables.tf │ ├── README.md │ └── main.tf ├── join-path │ ├── outputs.tf │ ├── README.md │ └── main.tf ├── request-quota-increase │ ├── outputs.tf │ ├── variables.tf │ ├── README.md │ └── main.tf ├── operating-system │ ├── outputs.tf │ ├── README.md │ └── main.tf ├── list-remove │ ├── outputs.tf │ ├── variables.tf │ ├── README.md │ └── main.tf ├── executable-dependency │ ├── outputs.tf │ ├── README.md │ ├── main.tf │ └── variables.tf ├── instance-type │ ├── variables.tf │ ├── README.md │ ├── outputs.tf │ └── main.tf └── require-executable │ ├── README.md │ ├── main.tf │ └── variables.tf ├── codegen └── quotas │ ├── requirements.txt │ ├── README.md │ ├── templates.py │ └── generate_quotas.py ├── modules ├── list-remove │ ├── outputs.tf │ ├── variables.tf │ ├── main.tf │ └── README.md ├── join-path │ ├── outputs.tf │ ├── main.tf │ ├── vars.tf │ └── README.md ├── request-quota-increase │ ├── outputs.tf │ ├── CHANGELOG.md │ └── README.md ├── patcher-test │ ├── variables.tf │ ├── CHANGELOG.md │ ├── outputs.tf │ ├── main.tf │ └── README.md ├── operating-system │ ├── outputs.tf │ ├── main.tf │ └── README.md ├── run-pex-as-data-source │ ├── outputs.tf │ ├── variables.tf │ ├── main.tf │ └── README.md ├── require-executable │ ├── outputs.tf │ ├── main.tf │ ├── variables.tf │ ├── README.md │ └── require_executable.py ├── executable-dependency │ ├── outputs.tf │ ├── main.tf │ ├── variables.tf │ ├── README.md │ └── download-dependency-if-necessary.py ├── run-pex-as-resource │ ├── outputs.tf │ ├── main.tf │ ├── README.md │ └── variables.tf ├── instance-type │ ├── variables.tf │ ├── main.tf │ ├── outputs.tf │ └── README.md └── prepare-pex-environment │ ├── outputs.tf │ ├── variables.tf │ ├── dependencies.tf │ ├── main.tf │ ├── entrypoint.py │ ├── determine_python_path.py │ └── README.md ├── NOTICE ├── .pre-commit-config.yaml ├── CODEOWNERS ├── setup.cfg ├── .patcher ├── config.yaml └── patches │ ├── sample-breaking-change │ └── patch.yaml │ └── sample-breaking-change2 │ └── patch.yaml ├── test ├── join_path_test.go ├── operating_system_test.go ├── instance_type_test.go ├── validation │ └── validate_all_modules_and_examples_test.go ├── test_helpers.go ├── request_quota_increase_test.go ├── README.md ├── upgrades │ └── upgrade_test.go ├── pex_test.go ├── require_executable_test.go ├── list_remove_test.go ├── executable_dependency_test.go └── go.mod ├── .circleci ├── post-upgrade-test-results.sh ├── set-upgrade-test-vars.sh └── config.yml ├── .gitignore ├── .github └── pull_request_template.md ├── terraform-cloud-enterprise-private-module-registry-placeholder.tf ├── CONTRIBUTING.md ├── README.md └── LICENSE.txt /examples/pex/sample_python_script/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /codegen/quotas/requirements.txt: -------------------------------------------------------------------------------- 1 | boto3>=1.20.0,<2.0 2 | -------------------------------------------------------------------------------- /examples/join-path/outputs.tf: -------------------------------------------------------------------------------- 1 | output "path" { 2 | value = module.path.path 3 | } 4 | -------------------------------------------------------------------------------- /examples/pex/pex-env/build_scripts/.python-version: -------------------------------------------------------------------------------- 1 | 3.8.0 2 | 3.9.0 3 | 3.10.0 4 | 3.11.0 5 | -------------------------------------------------------------------------------- /examples/pex/pex-env/requirements.txt: -------------------------------------------------------------------------------- 1 | click==7.0 2 | six==1.12.0 3 | jsonschema==2.6.0 4 | -------------------------------------------------------------------------------- /modules/list-remove/outputs.tf: -------------------------------------------------------------------------------- 1 | output "output_list" { 2 | value = local.list_without_items 3 | } 4 | -------------------------------------------------------------------------------- /modules/join-path/outputs.tf: -------------------------------------------------------------------------------- 1 | output "path" { 2 | value = join(module.os.path_separator, var.path_parts) 3 | } 4 | -------------------------------------------------------------------------------- /examples/request-quota-increase/outputs.tf: -------------------------------------------------------------------------------- 1 | output "new_quotas" { 2 | value = module.quota_increase.new_quotas 3 | } 4 | -------------------------------------------------------------------------------- /modules/request-quota-increase/outputs.tf: -------------------------------------------------------------------------------- 1 | output "new_quotas" { 2 | value = aws_servicequotas_service_quota.increase_quotas 3 | } 4 | -------------------------------------------------------------------------------- /modules/patcher-test/variables.tf: -------------------------------------------------------------------------------- 1 | variable "sampleinput" { 2 | type = string 3 | description = "Sample input for the module" 4 | } 5 | -------------------------------------------------------------------------------- /modules/join-path/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | module "os" { 6 | source = "../operating-system" 7 | } 8 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | terraform-aws-utilities 2 | Copyright 2019 Gruntwork, Inc. 3 | 4 | This product includes software developed at Gruntwork (https://www.gruntwork.io/). 5 | -------------------------------------------------------------------------------- /examples/operating-system/outputs.tf: -------------------------------------------------------------------------------- 1 | output "os_name" { 2 | value = module.os.name 3 | } 4 | 5 | output "path_separator" { 6 | value = module.os.path_separator 7 | } 8 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | repos: 2 | - repo: https://github.com/gruntwork-io/pre-commit 3 | rev: v0.1.10 4 | hooks: 5 | - id: terraform-fmt 6 | - id: goimports 7 | -------------------------------------------------------------------------------- /examples/pex/pex-env/bin/sample_python_script_py3_env.pex: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/gruntwork-io/terraform-aws-utilities/HEAD/examples/pex/pex-env/bin/sample_python_script_py3_env.pex -------------------------------------------------------------------------------- /examples/request-quota-increase/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_region" { 2 | description = "The AWS region to run this code in" 3 | type = string 4 | default = "us-east-1" 5 | } 6 | -------------------------------------------------------------------------------- /modules/operating-system/outputs.tf: -------------------------------------------------------------------------------- 1 | output "name" { 2 | value = data.external.os.result.platform 3 | } 4 | 5 | output "path_separator" { 6 | value = data.external.os.result.path_separator 7 | } 8 | -------------------------------------------------------------------------------- /modules/run-pex-as-data-source/outputs.tf: -------------------------------------------------------------------------------- 1 | output "result" { 2 | description = "Data source result of executing the PEX binary." 3 | value = concat(data.external.pex.*.result, [null])[0] 4 | } 5 | -------------------------------------------------------------------------------- /examples/list-remove/outputs.tf: -------------------------------------------------------------------------------- 1 | output "output_list" { 2 | value = module.list_remove.output_list 3 | } 4 | 5 | output "output_list_as_csv" { 6 | value = join(",", module.list_remove.output_list) 7 | } 8 | -------------------------------------------------------------------------------- /examples/executable-dependency/outputs.tf: -------------------------------------------------------------------------------- 1 | output "output" { 2 | value = data.external.output.result.output 3 | } 4 | 5 | output "downloaded_executable" { 6 | value = module.executable.executable_path 7 | } 8 | -------------------------------------------------------------------------------- /modules/require-executable/outputs.tf: -------------------------------------------------------------------------------- 1 | output "executables" { 2 | description = "A map of the executables to the resolved path where they reside." 3 | value = data.external.required_executable.result 4 | } 5 | -------------------------------------------------------------------------------- /modules/patcher-test/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog patcher-test 2 | 3 | ## [v0.10.4](https://github.com/gruntwork-io/terraform-aws-utilities/releases/tag/v0.10.4) - 2024-08-12 4 | 5 | ### Changed 6 | - Sample change in the patcher-test module to add a new input. -------------------------------------------------------------------------------- /modules/patcher-test/outputs.tf: -------------------------------------------------------------------------------- 1 | output "name" { 2 | value = data.external.os.result.platform 3 | } 4 | 5 | output "path_separator" { 6 | value = data.external.os.result.path_separator 7 | } 8 | 9 | output "sampleinput" { 10 | value = var.sampleinput 11 | } -------------------------------------------------------------------------------- /modules/executable-dependency/outputs.tf: -------------------------------------------------------------------------------- 1 | output "executable_path" { 2 | value = var.enabled ? data.external.executable.0.result.path : var.executable 3 | description = "The path to use to run the executable. Will either be the path of the executable on the system PATH or a path in var.install_dir." 4 | } -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # GRUNTWORK CODEOWNERS FILE 2 | # Gruntwork sets the codeowner for its repositories to the machine user `gruntwork-codeowner`. 3 | # We use an internal process for triaging all incoming issues and pull requests and that process will end up setting an assignee for any new issues. 4 | 5 | * @gruntwork-codeowner -------------------------------------------------------------------------------- /examples/list-remove/variables.tf: -------------------------------------------------------------------------------- 1 | variable "input_list" { 2 | description = "The list of items from which you wish to remove items." 3 | type = list(any) 4 | } 5 | 6 | variable "items_to_remove" { 7 | description = "The list of items you wish to remove from the input_list." 8 | type = list(any) 9 | } 10 | -------------------------------------------------------------------------------- /modules/run-pex-as-resource/outputs.tf: -------------------------------------------------------------------------------- 1 | output "pex_done" { 2 | description = "This output is populated when the pex script successfully runs to completion. As such, it can be used to register hooks for terraform resources to depend on the pex execution." 3 | value = var.enabled ? null_resource.run_pex[0].id : null 4 | } 5 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [flake8] 2 | max-line-length = 120 3 | 4 | [yapf] 5 | based_on_style = google 6 | align_closing_bracket_with_visual_indent = true 7 | column_limit = 120 8 | blank_line_before_nested_class_or_def = true 9 | coalesce_brackets = false 10 | dedent_closing_brackets = true 11 | split_before_dot = true 12 | split_complex_comprehension = true 13 | -------------------------------------------------------------------------------- /modules/instance-type/variables.tf: -------------------------------------------------------------------------------- 1 | variable "instance_types" { 2 | description = "A list of instance types to look up in the current AWS region. We recommend putting them in order of preference, as the recommended_instance_type output variable will contain the first instance type from this list that is available in all AZs." 3 | type = list(string) 4 | } -------------------------------------------------------------------------------- /examples/list-remove/README.md: -------------------------------------------------------------------------------- 1 | # List Remove example 2 | 3 | This folder shows examples of how to use the [list-remove module](/modules/list-remove) to remove items from a list 4 | based on a lookup list. 5 | 6 | 7 | ## How do you run these examples? 8 | 9 | 1. Install [Terraform](https://www.terraform.io/). 10 | 1. `terraform init`. 11 | 1. `terraform apply`. 12 | -------------------------------------------------------------------------------- /examples/instance-type/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_region" { 2 | description = "The AWS region to run this code in" 3 | type = string 4 | default = "eu-west-1" 5 | } 6 | 7 | variable "instance_types" { 8 | description = "A list of instance types to look up in the current AWS region." 9 | type = list(string) 10 | default = ["t3.micro", "t2.micro"] 11 | } -------------------------------------------------------------------------------- /modules/operating-system/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | data "external" "os" { 6 | program = ["python3", "-c", <<-EOF 7 | import json 8 | import os 9 | import platform 10 | 11 | print(json.dumps({ 12 | "platform": platform.system(), 13 | "path_separator": os.sep, 14 | })) 15 | EOF 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /modules/patcher-test/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | data "external" "os" { 6 | program = ["python3", "-c", <<-EOF 7 | import json 8 | import os 9 | import platform 10 | 11 | print(json.dumps({ 12 | "platform": platform.system(), 13 | "path_separator": os.sep, 14 | })) 15 | EOF 16 | ] 17 | } 18 | -------------------------------------------------------------------------------- /examples/request-quota-increase/README.md: -------------------------------------------------------------------------------- 1 | # Request AWS Quota Increase 2 | 3 | This folder shows examples of how to use the [request-quota-increase module](/modules/request-quota-increase) to request a bigger quota for an AWS resource. 4 | 5 | 6 | ## How do you run these examples? 7 | 8 | 1. Install [Terraform](https://www.terraform.io/). 9 | 1. `terraform init`. 10 | 1. `terraform apply`. 11 | 12 | -------------------------------------------------------------------------------- /examples/require-executable/README.md: -------------------------------------------------------------------------------- 1 | # Require Executable example 2 | 3 | This folder shows examples of how to use the [require-executable module](/modules/require-executable) to validate the 4 | provided executables exist on the OS PATH. 5 | 6 | 7 | 8 | 9 | ## How do you run these examples? 10 | 11 | 1. Install [Terraform](https://www.terraform.io/). 12 | 1. `terraform init`. 13 | 1. `terraform apply`. 14 | -------------------------------------------------------------------------------- /examples/join-path/README.md: -------------------------------------------------------------------------------- 1 | # Join path example 2 | 3 | This folder shows examples of how to use the [join-path module](/modules/join-path) to create a file path with the 4 | proper path separator for the current operating system. 5 | 6 | 7 | 8 | 9 | ## How do you run these examples? 10 | 11 | 1. Install [Terraform](https://www.terraform.io/). 12 | 1. `terraform init`. 13 | 1. `terraform apply`. 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /examples/operating-system/README.md: -------------------------------------------------------------------------------- 1 | # Operating system example 2 | 3 | This folder shows examples of how to use the [operating-system module](/modules/operating-system) to access information 4 | about the current operating system in Terraform. 5 | 6 | 7 | 8 | 9 | ## How do you run these examples? 10 | 11 | 1. Install [Terraform](https://www.terraform.io/). 12 | 1. `terraform init`. 13 | 1. `terraform apply`. 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /examples/instance-type/README.md: -------------------------------------------------------------------------------- 1 | # Instance types example 2 | 3 | This folder shows examples of how to use the [instance-type module](/modules/instance-type) to pick an instance type 4 | that's available in all Availability Zones in the current AWS region. 5 | 6 | 7 | 8 | 9 | ## How do you run these examples? 10 | 11 | 1. Install [Terraform](https://www.terraform.io/). 12 | 1. `terraform init`. 13 | 1. `terraform apply`. 14 | 15 | 16 | 17 | -------------------------------------------------------------------------------- /.patcher/config.yaml: -------------------------------------------------------------------------------- 1 | versions: 2 | - tag: v0.10.2 3 | patches: 4 | - slug: "sample-breaking-change" 5 | modules_affected: 6 | - patcher-test 7 | - tag: v0.10.3 8 | patches: 9 | - slug: "sample-breaking-change" 10 | modules_affected: 11 | - patcher-test 12 | - tag: v0.10.4 13 | patches: 14 | - slug: "sample-breaking-change2" 15 | modules_affected: 16 | - patcher-test -------------------------------------------------------------------------------- /examples/pex/outputs.tf: -------------------------------------------------------------------------------- 1 | output "command_echo" { 2 | description = "For the pex data source, if successful, this will contain the echo string." 3 | value = module.pex_data.result != null ? module.pex_data.result["echo"] : null 4 | } 5 | 6 | output "command_python_version" { 7 | description = "Read out the python version that was used to run the PEX" 8 | value = module.pex_data.result != null ? module.pex_data.result["python_version_info"] : null 9 | } 10 | -------------------------------------------------------------------------------- /examples/operating-system/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | module "os" { 6 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 7 | # to a specific version of the modules, such as the following example: 8 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/operating-system?ref=v1.0.8" 9 | source = "../../modules/operating-system" 10 | } 11 | -------------------------------------------------------------------------------- /test/join_path_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | ) 8 | 9 | func TestJoinPath(t *testing.T) { 10 | t.Parallel() 11 | 12 | terratestOptions := createBaseTerratestOptions(t, "../examples/join-path") 13 | defer terraform.Destroy(t, terratestOptions) 14 | 15 | terraform.InitAndApply(t, terratestOptions) 16 | 17 | assertOutputEquals(t, "path", "foo/bar/baz.txt", terratestOptions) 18 | } 19 | -------------------------------------------------------------------------------- /examples/join-path/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | module "path" { 6 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 7 | # to a specific version of the modules, such as the following example: 8 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/join-path?ref=v1.0.8" 9 | source = "../../modules/join-path" 10 | 11 | path_parts = ["foo", "bar", "baz.txt"] 12 | } 13 | -------------------------------------------------------------------------------- /.circleci/post-upgrade-test-results.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Post the results of the upgrade test so that they're easy to understand. 3 | 4 | # Comment the upgrade test results on GitHub if there is a PR number. 5 | CIRCLE_PR_NUMBER="${CIRCLE_PR_NUMBER:-${CIRCLE_PULL_REQUEST##*/}}" 6 | if [[ -n "$CIRCLE_PR_NUMBER" ]]; then 7 | echo -e "Upgrade test results for build [$CIRCLE_BUILD_NUM]($CIRCLE_BUILD_URL)\n\n$(cat $UPGRADE_TEST_LOG_FOLDER/results.log)\n" \ 8 | | gh pr comment $CIRCLE_PR_NUMBER --body-file=- 9 | fi 10 | -------------------------------------------------------------------------------- /modules/executable-dependency/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | data "external" "executable" { 6 | count = var.enabled ? 1 : 0 7 | 8 | program = concat( 9 | [ 10 | "python3", 11 | "${path.module}/download-dependency-if-necessary.py", 12 | "--executable", 13 | var.executable, 14 | "--download-url", 15 | var.download_url 16 | ], 17 | var.install_dir != null ? ["--install-dir", var.install_dir] : [], 18 | var.append_os_arch ? ["--append-os-arch"] : [] 19 | ) 20 | } 21 | -------------------------------------------------------------------------------- /examples/list-remove/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | module "list_remove" { 6 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 7 | # to a specific version of the modules, such as the following example: 8 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/list-remove?ref=v0.0.8" 9 | source = "../../modules/list-remove" 10 | 11 | original_list = var.input_list 12 | items_to_remove = var.items_to_remove 13 | } 14 | -------------------------------------------------------------------------------- /examples/request-quota-increase/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | provider "aws" { 6 | region = var.aws_region 7 | } 8 | 9 | module "quota_increase" { 10 | source = "../../modules/request-quota-increase" 11 | 12 | # In this example, to avoid opening a new request every time we run an automated test, we are setting the quotas 13 | # to their default values. In the real world, you'd want to set these quotes to higher values. 14 | vpc_rules_per_network_acl = 20 15 | vpc_nat_gateways_per_availability_zone = 5 16 | } 17 | -------------------------------------------------------------------------------- /test/operating_system_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "runtime" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/terratest/modules/terraform" 9 | ) 10 | 11 | func TestOperatingSystem(t *testing.T) { 12 | t.Parallel() 13 | 14 | terratestOptions := createBaseTerratestOptions(t, "../examples/operating-system") 15 | defer terraform.Destroy(t, terratestOptions) 16 | 17 | terraform.InitAndApply(t, terratestOptions) 18 | 19 | assertOutputEquals(t, "os_name", strings.Title(runtime.GOOS), terratestOptions) 20 | assertOutputEquals(t, "path_separator", "/", terratestOptions) 21 | } 22 | -------------------------------------------------------------------------------- /examples/instance-type/outputs.tf: -------------------------------------------------------------------------------- 1 | output "instance_type_map" { 2 | description = "A map where the keys are the instance types in var.instance_types and the values are true or false, depending on whether every AZ in the current region contains this instance type." 3 | value = module.instance_types.instance_type_map 4 | } 5 | 6 | output "recommended_instance_type" { 7 | description = "The recommended instance type to use in this AWS region. This will be the first instance type in var.instance_types which is available in all AZs in this region." 8 | value = module.instance_types.recommended_instance_type 9 | } -------------------------------------------------------------------------------- /modules/prepare-pex-environment/outputs.tf: -------------------------------------------------------------------------------- 1 | output "pex_path" { 2 | description = "Path to PEX file that should be run." 3 | value = module.python_pex_path.path 4 | } 5 | 6 | output "python_path" { 7 | description = "The python path that should be used for running PEX file. This should be set as the PYTHONPATH environment variable." 8 | value = data.external.determine_python_path.result["python_path"] 9 | } 10 | 11 | output "entrypoint_path" { 12 | description = "The path to the entrypoint script that should be used to call the module code." 13 | value = "${path.module}${module.os.path_separator}entrypoint.py" 14 | } 15 | -------------------------------------------------------------------------------- /examples/pex/variables.tf: -------------------------------------------------------------------------------- 1 | variable "echo_string" { 2 | description = "This string will be echo'd back from the pex data source." 3 | type = string 4 | default = "Hello world!" 5 | } 6 | 7 | variable "triggers" { 8 | description = "Triggers for the pex null resource to rerun." 9 | type = map(string) 10 | default = null 11 | } 12 | 13 | # These variables are only used for testing purposes and should not be touched in normal operations, unless you know 14 | # what you are doing. 15 | 16 | variable "enabled" { 17 | description = "Whether or not to run the PEX scripts." 18 | type = bool 19 | default = true 20 | } 21 | -------------------------------------------------------------------------------- /modules/join-path/vars.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # REQUIRED MODULE PARAMETERS 3 | # These variables must be passed in by the operator. 4 | # --------------------------------------------------------------------------------------------------------------------- 5 | 6 | variable "path_parts" { 7 | description = "A list of folder and file names to combine into a path, using the proper path separator for the current OS." 8 | type = list(string) 9 | 10 | # Example: 11 | # path_parts = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 12 | } 13 | -------------------------------------------------------------------------------- /modules/prepare-pex-environment/variables.tf: -------------------------------------------------------------------------------- 1 | variable "python_pex_path_parts" { 2 | description = "Parts of the path (folders and files names) to the PEX executable for python as a list of strings." 3 | type = list(string) 4 | # Example: 5 | # python_pex_path_parts = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 6 | } 7 | 8 | variable "pex_module_path_parts" { 9 | description = "Parts of the path (folders and file names) to the python package directory housing the pex file." 10 | type = list(string) 11 | # Example: 12 | # pex_module_path_parts = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 13 | } 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Terraform files 2 | .terraform 3 | terraform.tfstate 4 | terraform.tfvars 5 | *.tfstate* 6 | 7 | # OS X files 8 | .history 9 | .DS_Store 10 | 11 | # IntelliJ files 12 | .idea_modules 13 | *.iml 14 | *.iws 15 | *.ipr 16 | .idea/ 17 | build/ 18 | */build/ 19 | out/ 20 | 21 | # Python bytecode files 22 | __pycache__ 23 | *.pyc 24 | 25 | # Python virtual environment 26 | .venv 27 | 28 | # Go best practices dictate that libraries should not include the vendor directory 29 | vendor 30 | 31 | # Ignore Terraform lock files, as we want to test the Terraform code in these repos with the latest provider 32 | # versions. 33 | .terraform.lock.hcl 34 | test/.go-version 35 | -------------------------------------------------------------------------------- /modules/list-remove/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # REQUIRED MODULE PARAMETERS 3 | # These variables must be passed in by the operator. 4 | # --------------------------------------------------------------------------------------------------------------------- 5 | 6 | variable "original_list" { 7 | description = "The list of items where you want to remove items from." 8 | type = list(any) 9 | } 10 | 11 | variable "items_to_remove" { 12 | description = "The list of items that you want to remove from the original list." 13 | type = list(any) 14 | } 15 | -------------------------------------------------------------------------------- /modules/instance-type/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | required_providers { 4 | aws = { 5 | source = "hashicorp/aws" 6 | version = ">= 3.75.1, < 7.0.0" 7 | } 8 | } 9 | } 10 | 11 | data "aws_ec2_instance_type_offerings" "offerings" { 12 | for_each = toset(data.aws_availability_zones.available.names) 13 | 14 | filter { 15 | name = "instance-type" 16 | values = var.instance_types 17 | } 18 | 19 | filter { 20 | name = "location" 21 | values = [each.key] 22 | } 23 | 24 | location_type = "availability-zone" 25 | } 26 | 27 | data "aws_availability_zones" "available" { 28 | state = "available" 29 | } 30 | -------------------------------------------------------------------------------- /modules/require-executable/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | 4 | required_providers { 5 | external = { 6 | source = "hashicorp/external" 7 | version = ">= 2.3.5" 8 | } 9 | } 10 | } 11 | 12 | data "external" "required_executable" { 13 | program = ["python3", "${path.module}/require_executable.py"] 14 | 15 | # Currently the external data source provider does not support list values in the query, so we convert the input list 16 | # to be a comma separated string. 17 | # See https://github.com/terraform-providers/terraform-provider-external/issues/2 18 | query = { 19 | required_executables = join(",", var.required_executables) 20 | error_message = var.error_message 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /.patcher/patches/sample-breaking-change/patch.yaml: -------------------------------------------------------------------------------- 1 | name: "Sample Breaking Change" 2 | description: A sample breaking change that renames a resource 3 | author: Gruntwork 4 | 5 | # Optional list of dependencies that the patch requires. 6 | dependencies: 7 | - name: terrapatch 8 | version: "0.1.0" 9 | 10 | # List of steps that this patch should execute. 11 | # Each step has a name field (string) and a run field, which can denote either an OS command, or an external script to be run. 12 | # If there are any external scripts, then make sure you include these in the same directory where the patch.yaml file is. 13 | steps: 14 | - name: 15 | run: terrapatch move-resource --path $PATCHER_MODULE_PATH terraform $PATCHER_MODULE_ADDRESS module.null_resource.test2 -------------------------------------------------------------------------------- /.patcher/patches/sample-breaking-change2/patch.yaml: -------------------------------------------------------------------------------- 1 | name: "Sample Breaking Change" 2 | description: A sample breaking change that adds a new argument 3 | author: Gruntwork 4 | 5 | # Optional list of dependencies that the patch requires. 6 | dependencies: 7 | - name: terrapatch 8 | version: "0.1.0" 9 | 10 | # List of steps that this patch should execute. 11 | # Each step has a name field (string) and a run field, which can denote either an OS command, or an external script to be run. 12 | # If there are any external scripts, then make sure you include these in the same directory where the patch.yaml file is. 13 | steps: 14 | - name: Add sampleinput argument to the module 15 | run: terrapatch add-module-argument $PATCHER_MODULE_ADDRESS sampleinput "\"samplevalue\"" -------------------------------------------------------------------------------- /test/instance_type_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/aws" 7 | "github.com/gruntwork-io/terratest/modules/terraform" 8 | ) 9 | 10 | func TestInstanceType(t *testing.T) { 11 | t.Parallel() 12 | 13 | awsRegion := aws.GetRandomRegion(t, nil, nil) 14 | 15 | terratestOptions := &terraform.Options{ 16 | TerraformDir: "../examples/instance-type", 17 | Vars: map[string]interface{}{ 18 | "aws_region": awsRegion, 19 | }, 20 | } 21 | defer terraform.Destroy(t, terratestOptions) 22 | 23 | // We only need to run 'apply' for this test. If the instance launches successfully, it's because the code picked 24 | // the right instance type to use for the current region (note: we pick the region at random). 25 | terraform.InitAndApply(t, terratestOptions) 26 | } 27 | -------------------------------------------------------------------------------- /modules/list-remove/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | # Remove the items in items_to_remove from original_list. This works because: 6 | # a) concat will create a list in the order of the arguments it's given 7 | # b) distinct will discard duplicate items in the order they are found 8 | # => If you slice the new list from the point of the length of the first list in the concat, then you will get the 9 | # second list of concat with all the elements from the first list removed. 10 | # Inspired by https://github.com/hashicorp/terraform/issues/16044#issuecomment-392269246 11 | locals { 12 | combined_list = distinct(concat(var.items_to_remove, var.original_list)) 13 | list_without_items = slice( 14 | local.combined_list, 15 | length(var.items_to_remove), 16 | length(local.combined_list), 17 | ) 18 | } 19 | -------------------------------------------------------------------------------- /test/validation/validate_all_modules_and_examples_test.go: -------------------------------------------------------------------------------- 1 | package testvalidate 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 9 | 10 | "github.com/stretchr/testify/require" 11 | ) 12 | 13 | // TestValidateAllTerraformModulesAndExamples recursively finds all modules and examples (by default) subdirectories in 14 | // the repo and runs Terraform InitAndValidate on them to flush out missing variables, typos, unused vars, etc 15 | func TestValidateAllTerraformModulesAndExamples(t *testing.T) { 16 | t.Parallel() 17 | 18 | cwd, err := os.Getwd() 19 | require.NoError(t, err) 20 | 21 | opts, optsErr := test_structure.NewValidationOptions(filepath.Join(cwd, "../.."), []string{}, []string{}) 22 | require.NoError(t, optsErr) 23 | 24 | test_structure.ValidateAllTerraformModules(t, opts) 25 | } 26 | -------------------------------------------------------------------------------- /test/test_helpers.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | "github.com/stretchr/testify/assert" 8 | ) 9 | 10 | func checkOutputs(t *testing.T, expectedFoo string, terratestOptions *terraform.Options) { 11 | assertOutputEquals(t, "map_example", expectedFoo, terratestOptions) 12 | assertOutputEquals(t, "list_example", expectedFoo, terratestOptions) 13 | } 14 | 15 | func assertOutputEquals(t *testing.T, outputName string, expectedValue string, terratestOptions *terraform.Options) { 16 | output := terraform.Output(t, terratestOptions, outputName) 17 | assert.Equal(t, output, expectedValue) 18 | } 19 | 20 | func createBaseTerratestOptions( 21 | t *testing.T, 22 | templatePath string, 23 | ) *terraform.Options { 24 | terratestOptions := terraform.Options{ 25 | TerraformDir: templatePath, 26 | } 27 | return &terratestOptions 28 | } 29 | -------------------------------------------------------------------------------- /modules/prepare-pex-environment/dependencies.tf: -------------------------------------------------------------------------------- 1 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 | # DEPENDENCIES FOR PEX PREPARATION 3 | # Determine and calculate various intermediate variables that will help with setting up the PEX execution environment. 4 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 5 | 6 | terraform { 7 | required_version = ">= 1.0.0" 8 | } 9 | 10 | # Figure out the OS environment 11 | module "os" { 12 | source = "../operating-system" 13 | } 14 | 15 | # Combine the path parts for each variable into an actual path. 16 | 17 | module "python_pex_path" { 18 | source = "../join-path" 19 | path_parts = var.python_pex_path_parts 20 | } 21 | 22 | module "pex_module_path" { 23 | source = "../join-path" 24 | path_parts = var.pex_module_path_parts 25 | } 26 | -------------------------------------------------------------------------------- /modules/prepare-pex-environment/main.tf: -------------------------------------------------------------------------------- 1 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 | # PREPARE NECESSARY VARIABLES FOR EXECUTING PEX 3 | # This terraform module provides the information necessary to call a PEX binary in a platform independent manner. 4 | # Specifically, this will return two outputs: 5 | # - The PEX to use based on python version 6 | # - A value for PYTHONPATH that is setup to import the packages embedded in the PEX binary 7 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 8 | 9 | terraform { 10 | required_version = ">= 1.0.0" 11 | } 12 | 13 | data "external" "determine_python_path" { 14 | program = [ 15 | "python3", 16 | "${path.module}${module.os.path_separator}determine_python_path.py", 17 | "--module-path", 18 | module.pex_module_path.path, 19 | ] 20 | } 21 | -------------------------------------------------------------------------------- /examples/executable-dependency/README.md: -------------------------------------------------------------------------------- 1 | # Executable dependency example 2 | 3 | This folder shows examples of how to use the [executable-dependency module](/modules/executable-dependency) to check if 4 | an executable is already installed, and if it's not, download it from a URL. This example will then run this executable 5 | to prove that it was installed correctly. 6 | 7 | The executable to run, the args to pass to it, and the download URL are all configurable in `variables.tf`. The default 8 | values show an example of ensuring the [kubergrunt](https://github.com/gruntwork-io/kubergrunt) executable is installed 9 | before running `kubergrunt --version`. 10 | 11 | **NOTE**: This module requires that Python is installed on your system. It should work with Python 2 or 3. 12 | 13 | 14 | 15 | 16 | ## How do you run these examples? 17 | 18 | 1. Install [Terraform](https://www.terraform.io/). 19 | 1. `terraform init`. 20 | 1. `terraform apply`. 21 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /modules/instance-type/outputs.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | instance_type_map = { for instance_type in var.instance_types : instance_type => ( 3 | !contains([for az, offerings in data.aws_ec2_instance_type_offerings.offerings : contains(offerings.instance_types, instance_type)], false)) 4 | } 5 | 6 | recommended_instance_type = [for instance_type in var.instance_types : instance_type if local.instance_type_map[instance_type]][0] 7 | } 8 | 9 | output "instance_type_map" { 10 | description = "A map where the keys are the instance types in var.instance_types and the values are true or false, depending on whether every AZ in the current region contains this instance type." 11 | value = local.instance_type_map 12 | } 13 | 14 | output "recommended_instance_type" { 15 | description = "The recommended instance type to use in this AWS region. This will be the first instance type in var.instance_types which is available in all AZs in this region." 16 | value = local.recommended_instance_type 17 | } -------------------------------------------------------------------------------- /examples/executable-dependency/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | module "executable" { 6 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 7 | # to a specific version of the modules, such as the following example: 8 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/executable-dependency?ref=v1.0.8" 9 | source = "../../modules/executable-dependency" 10 | 11 | executable = var.executable 12 | download_url = var.download_url 13 | append_os_arch = var.append_os_arch 14 | enabled = var.enabled 15 | } 16 | 17 | # We run the executable here, with the specified args, and write the output to stdout in the form of JSON, as that's 18 | # what the Terraform external data source requires to be able to read and parse that output. 19 | data "external" "output" { 20 | program = [ 21 | "bash", 22 | "-c", 23 | "echo \"{\\\"output\\\": \\\"$(${module.executable.executable_path} ${var.executable_args})\\\"}\"", 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /.circleci/set-upgrade-test-vars.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # Set some variables that are used by the upgrade test runner (run-go-tests), passing in 3 | # these variables as overrides with --extra-flags and -ldflags. 4 | 5 | DEFAULT_BRANCH="origin/HEAD" 6 | 7 | # Call gh to get the base ref that we should run our upgrade tests against. 8 | baseRef="" 9 | if [[ "$CIRCLE_BRANCH" == "$DEFAULT_BRANCH" ]]; then 10 | # On main, the commit to compare to should be the last release tag. 11 | baseRef="$(gh release list --exclude-drafts --limit 1 | awk '{print $3}')" 12 | echo "export UPGRADE_TEST_BASE_REF=$baseRef" >> $BASH_ENV 13 | else 14 | # On a PR branch, the commit to compare to should be the default branch. 15 | echo "export UPGRADE_TEST_BASE_REF=$DEFAULT_BRANCH" >> $BASH_ENV 16 | fi 17 | 18 | # Set these variables so that the CI server pre-installs these versions before the upgrade tests run. 19 | tfBaseVersion="1.2" 20 | tfTargetVersion="1.3" 21 | echo "export UPGRADE_TEST_TF_BASE_VERSION=$tfBaseVersion" >> $BASH_ENV 22 | echo "export UPGRADE_TEST_TF_TARGET_VERSION=$tfTargetVersion" >> $BASH_ENV 23 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | 2 | 3 | ## Description 4 | 5 | Fixes #000. 6 | 7 | 8 | 9 | ## TODOs 10 | 11 | Read the [Gruntwork contribution guidelines](https://gruntwork.notion.site/Gruntwork-Coding-Methodology-02fdcd6e4b004e818553684760bf691e). 12 | 13 | - [ ] Update the docs. 14 | - [ ] Run the relevant tests successfully, including pre-commit checks. 15 | - [ ] Ensure any 3rd party code adheres with our [license policy](https://www.notion.so/gruntwork/Gruntwork-licenses-and-open-source-usage-policy-f7dece1f780341c7b69c1763f22b1378) or delete this line if its not applicable. 16 | - [ ] Include release notes. If this PR is backward incompatible, include a migration guide. 17 | 18 | ## Release Notes (draft) 19 | 20 | 21 | Added / Removed / Updated [X]. 22 | 23 | ### Migration Guide 24 | 25 | 26 | -------------------------------------------------------------------------------- /examples/pex/sample_python_script/main.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import os 3 | import click 4 | import json 5 | import sys 6 | 7 | 8 | @click.command() 9 | @click.option("--is-data/--no-is-data", default=False, help="Whether or not this is run as a data source") 10 | @click.option("--triggers-json", help="JSON encoded triggers for the resource") 11 | def main(is_data, triggers_json): 12 | if is_data: 13 | query = json.loads(sys.stdin.read()) 14 | print( 15 | json.dumps( 16 | { 17 | "echo": query.get("echo", ""), 18 | "out": "This was successfully run as data source.", 19 | "python_version_info": str(sys.version_info), 20 | "triggers": triggers_json, 21 | } 22 | ) 23 | ) 24 | else: 25 | print("python version: {}".format(sys.version_info)) 26 | print("This was successfully run as a local-exec provisioner") 27 | print("Environment variable: {}".format(os.environ.get("RUN_PEX_TEST_ENV", None))) 28 | print("Triggers: {}".format(triggers_json)) 29 | 30 | 31 | if __name__ == "__main__": 32 | main() 33 | -------------------------------------------------------------------------------- /examples/require-executable/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | module "require_executables" { 6 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 7 | # to a specific version of the modules, such as the following example: 8 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/require-executable?ref=v1.0.8" 9 | source = "../../modules/require-executable" 10 | 11 | required_executables = var.required_executables 12 | error_message = var.error_message 13 | } 14 | 15 | # Conditional checking example 16 | module "conditional_require_executables" { 17 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 18 | # to a specific version of the modules, such as the following example: 19 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/require-executable?ref=v1.0.8" 20 | source = "../../modules/require-executable" 21 | 22 | required_executables = [var.validate_bad_executable ? "this-executable-should-not-exist" : ""] 23 | error_message = var.bad_executable_error_message 24 | } 25 | -------------------------------------------------------------------------------- /examples/require-executable/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # REQUIRED MODULE PARAMETERS 3 | # These variables must be passed in by the operator. 4 | # --------------------------------------------------------------------------------------------------------------------- 5 | 6 | variable "required_executables" { 7 | description = "A list of named executables that should exist on the OS PATH." 8 | type = list(string) 9 | } 10 | 11 | variable "error_message" { 12 | description = "Error message to show if the required executable is not found. This is printed for each executable that was not found. The module will make the following substitutions in the string: `__EXECUTABLE_NAME__` will become the name of the executable that was not found." 13 | type = string 14 | } 15 | 16 | variable "validate_bad_executable" { 17 | description = "Whether or not to validate the existence of a bad executable." 18 | type = bool 19 | default = false 20 | } 21 | 22 | variable "bad_executable_error_message" { 23 | description = "Error message to show for bad_executable check." 24 | type = string 25 | default = "" 26 | } 27 | -------------------------------------------------------------------------------- /modules/list-remove/README.md: -------------------------------------------------------------------------------- 1 | # List Remove Module 2 | 3 | This is a module that can be used to remove items in a given list from another list. This functionality is not yet 4 | available as an interpolation function. 5 | 6 | For example, suppose you have a list of availability zones (`["us-east-1a", "us-east-1b", "us-east-1c", "us-east-1d", 7 | "us-east-1e"]`) and you want to remove specific zones that don't support the features you need (`["us-east-1b", 8 | "us-east-1c"]`). You can use this module: 9 | 10 | ```hcl 11 | module "list_remove" { 12 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/list-remove?ref=v0.0.8" 13 | 14 | original_list = ["us-east-1a", "us-east-1b", "us-east-1c", "us-east-1d", "us-east-1e"] 15 | items_to_remove = ["us-east-1b", "us-east-1c"] 16 | } 17 | 18 | output "output_list" { 19 | value = "${module.list_remove.output_list}" 20 | } 21 | ``` 22 | 23 | The output `new_list` should be the list `["us-east-1a", "us-east-1d", "us-east-1e"]`. 24 | 25 | 26 | **NOTE**: This will dedup the input list due to the way it is implemented. This module will not work if you are expecting duplicate items to remain. 27 | 28 | 29 | ## Example code 30 | 31 | See the [list-remove example](/examples/list-remove) for working sample code. 32 | -------------------------------------------------------------------------------- /modules/join-path/README.md: -------------------------------------------------------------------------------- 1 | # Join Path Module 2 | 3 | This is a module that can be used to join a list of given path parts (that is, file and folder names) into a single 4 | path with the appropriate path separator (backslash or forward slash) for the current operating system. This is useful 5 | for ensuring the paths you build will work properly on Windows, Linux, and OS X. 6 | 7 | This module uses Python under the hood so, the Python must be installed on the OS. 8 | 9 | 10 | 11 | 12 | ## Example code 13 | 14 | See the [join-path example](/examples/join-path) for working sample code. 15 | 16 | 17 | 18 | 19 | ## Usage 20 | 21 | Simply use the module in your Terraform code, replacing `` with the latest version from the [releases 22 | page](https://github.com/gruntwork-io/terraform-aws-utilities/releases), and specifying the path parts using the 23 | `path_parts` input: 24 | 25 | ```hcl 26 | module "path" { 27 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/join-path?ref=" 28 | 29 | path_parts = ["foo", "bar", "baz.txt"] 30 | } 31 | ``` 32 | 33 | You can now get the joined path using the `path` output: 34 | 35 | ```hcl 36 | # Will be set to "foo/bar/baz.txt" on Linux and OS X, "foo\bar\baz.txt" on Windows 37 | joined_path = "${module.path.path}" 38 | ``` 39 | -------------------------------------------------------------------------------- /modules/require-executable/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # REQUIRED MODULE PARAMETERS 3 | # These variables must be passed in by the operator. 4 | # --------------------------------------------------------------------------------------------------------------------- 5 | 6 | variable "required_executables" { 7 | description = "A list of named executables that should exist on the OS PATH." 8 | type = list(string) 9 | } 10 | 11 | # --------------------------------------------------------------------------------------------------------------------- 12 | # OPTIONAL MODULE PARAMETERS 13 | # These variables have a sane default that can be used, and thus are not necessary to be set to run the module. 14 | # --------------------------------------------------------------------------------------------------------------------- 15 | 16 | variable "error_message" { 17 | description = "Error message to show if the required executable is not found. This is printed for each executable that was not found. The module will make the following substitutions in the string: `__EXECUTABLE_NAME__` will become the name of the executable that was not found." 18 | type = string 19 | default = "Not found: __EXECUTABLE_NAME__" 20 | } 21 | -------------------------------------------------------------------------------- /modules/run-pex-as-resource/main.tf: -------------------------------------------------------------------------------- 1 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 | # RUN PEX BINARY AS A local-exec PROVISIONER 3 | # This terraform module runs the provided pex binary in the context of a local-exec provisioner on a null_resource. 4 | # This utilizes the `prepare-pex-environment` module to ensure the execution of the binary is done in a portable manner. 5 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | terraform { 8 | required_version = ">= 1.0.0" 9 | } 10 | 11 | module "pex_env" { 12 | source = "../prepare-pex-environment" 13 | python_pex_path_parts = var.python_pex_path_parts 14 | pex_module_path_parts = var.pex_module_path_parts 15 | } 16 | 17 | resource "null_resource" "run_pex" { 18 | count = var.enabled ? 1 : 0 19 | 20 | triggers = var.triggers 21 | 22 | provisioner "local-exec" { 23 | command = "${local.python_call} ${var.command_args}" 24 | environment = merge( 25 | { 26 | PYTHONPATH = module.pex_env.python_path 27 | }, 28 | var.env, 29 | ) 30 | } 31 | } 32 | 33 | locals { 34 | python_call = "python3 ${module.pex_env.pex_path} ${module.pex_env.entrypoint_path} ${var.script_main_function}" 35 | } 36 | -------------------------------------------------------------------------------- /modules/patcher-test/README.md: -------------------------------------------------------------------------------- 1 | # Operating System Module 2 | 3 | This is a module that can be used to figure out what operating system is being used to run Terraform. This may be used 4 | to modify Terraform's behavior depending on the OS, such as modifying the way you format file paths on Linux vs 5 | Windows (see also the [join-path module](/modules/join-path)). 6 | 7 | This module uses Python under the hood so, the Python must be installed on the OS. 8 | 9 | 10 | 11 | 12 | ## Example code 13 | 14 | See the [operating-system example](/examples/operating-system) for working sample code. 15 | 16 | 17 | 18 | 19 | ## Usage 20 | 21 | Simply use the module in your Terraform code, replacing `` with the latest version from the [releases 22 | page](https://github.com/gruntwork-io/terraform-aws-utilities/releases): 23 | 24 | ```hcl 25 | module "os" { 26 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/operating-system?ref=" 27 | } 28 | ``` 29 | 30 | * You can now get the name of the operating system from the `name` output, which will be set to either `Linux`, 31 | `Darwin`, or `Windows` 32 | 33 | * You can also get the path separator for the current OS—backslash for Windows, forward slash everywhere else—from the 34 | `path_separator` output. 35 | 36 | ```hcl 37 | operating_system_name = "${module.os.name}" 38 | path_separator = "${module.os.path_separator}" 39 | ``` 40 | -------------------------------------------------------------------------------- /modules/operating-system/README.md: -------------------------------------------------------------------------------- 1 | # Operating System Module 2 | 3 | This is a module that can be used to figure out what operating system is being used to run Terraform. This may be used 4 | to modify Terraform's behavior depending on the OS, such as modifying the way you format file paths on Linux vs 5 | Windows (see also the [join-path module](/modules/join-path)). 6 | 7 | This module uses Python under the hood so, the Python must be installed on the OS. 8 | 9 | 10 | 11 | 12 | ## Example code 13 | 14 | See the [operating-system example](/examples/operating-system) for working sample code. 15 | 16 | 17 | 18 | 19 | ## Usage 20 | 21 | Simply use the module in your Terraform code, replacing `` with the latest version from the [releases 22 | page](https://github.com/gruntwork-io/terraform-aws-utilities/releases): 23 | 24 | ```hcl 25 | module "os" { 26 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/operating-system?ref=" 27 | } 28 | ``` 29 | 30 | * You can now get the name of the operating system from the `name` output, which will be set to either `Linux`, 31 | `Darwin`, or `Windows` 32 | 33 | * You can also get the path separator for the current OS—backslash for Windows, forward slash everywhere else—from the 34 | `path_separator` output. 35 | 36 | ```hcl 37 | operating_system_name = "${module.os.name}" 38 | path_separator = "${module.os.path_separator}" 39 | ``` 40 | -------------------------------------------------------------------------------- /examples/executable-dependency/variables.tf: -------------------------------------------------------------------------------- 1 | variable "executable" { 2 | type = string 3 | description = "The executable to look for on the system PATH and in var.install_dir. If not found, this executable will be downloaded from var.download_url." 4 | default = "kubergrunt" 5 | } 6 | 7 | variable "download_url" { 8 | type = string 9 | description = "The URL to download the executable from if var.executable is not found on the system PATH or in var.install_dir." 10 | default = "https://github.com/gruntwork-io/kubergrunt/releases/download/v0.5.12/kubergrunt" 11 | } 12 | 13 | variable "executable_args" { 14 | type = string 15 | description = "The CLI args to pass when running the executable." 16 | default = "--version" 17 | } 18 | 19 | variable "append_os_arch" { 20 | type = bool 21 | description = "If set to true, append the operating system and architecture to the URL. E.g., Append linux_amd64 if this code is being run on a 64 bit Linux OS." 22 | default = true 23 | } 24 | 25 | variable "enabled" { 26 | description = "Set to false to have disable this module, so it does not try to download the executable, and always returns its path unchanged. This weird parameter exists solely because Terraform does not support conditional modules. Therefore, this is a hack to allow you to conditionally decide if this module should run or not." 27 | type = bool 28 | default = true 29 | } 30 | -------------------------------------------------------------------------------- /examples/pex/pex-env/build_scripts/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | # Script to generate a single script with all requirements packed in that is compatible with multiple python 4 | # versions and multiple platforms. 5 | # 6 | 7 | set -e 8 | 9 | readonly FILEDIR="$(dirname "$0")" 10 | 11 | build() { 12 | echo "Building execution environment for sample_python_script" 13 | 14 | # Build python3 15 | pex --python-shebang='/usr/bin/env python' \ 16 | -r ../requirements.txt \ 17 | --python=python3.8 \ 18 | --python=python3.9 \ 19 | --python=python3.10 \ 20 | --python=python3.11 \ 21 | --platform macosx_10.12-x86_64-cp-38-m \ 22 | --platform macosx_10.12-x86_64-cp-39-m \ 23 | --platform macosx_10.12-x86_64-cp-310-m \ 24 | --platform macosx_10.12-x86_64-cp-311-m \ 25 | --platform macosx_10.13-x86_64-cp-38-m \ 26 | --platform macosx_10.13-x86_64-cp-39-m \ 27 | --platform macosx_10.13-x86_64-cp-310-m \ 28 | --platform macosx_10.13-x86_64-cp-311-m \ 29 | --platform macosx_10.14-x86_64-cp-38-m \ 30 | --platform macosx_10.14-x86_64-cp-39-m \ 31 | --platform macosx_10.14-x86_64-cp-310-m \ 32 | --platform macosx_10.14-x86_64-cp-311-m \ 33 | --platform linux-x86_64-cp-38-m \ 34 | --platform linux-x86_64-cp-39-m \ 35 | --platform linux-x86_64-cp-310-m \ 36 | --platform linux-x86_64-cp-311-m \ 37 | --disable-cache \ 38 | -o ../bin/sample_python_script_py3_env.pex 39 | } 40 | 41 | (cd "${FILEDIR}" && build) 42 | -------------------------------------------------------------------------------- /modules/require-executable/README.md: -------------------------------------------------------------------------------- 1 | # Require Executable Module 2 | 3 | This is a module that can be used to ensure particular executables is available in the `PATH`. This module will search 4 | the OS `PATH` for the provided named executables and validate that it exists, as well as making sure the OS user running 5 | terraform has permissions to run the named executable. 6 | 7 | This module will exit with an error if any executable in the list does not exist, printing an error message indicating 8 | which executables were missing. 9 | 10 | This module uses Python under the hood, so Python must be installed and available on the OS. 11 | 12 | 13 | ## Example code 14 | 15 | See the [require-executable example](/examples/require-executable) for working sample code. 16 | 17 | 18 | ## Conditional check 19 | 20 | Sometimes you might want to guard the check for a required executable on a condition (e.g only check if an executable 21 | exists if a particular input flag is set). However, Terraform currently [does not support conditional module 22 | blocks](https://github.com/hashicorp/terraform/issues/953). 23 | 24 | For this reason, this module will accept and noop on empty strings as a workaround. For example, suppose you want to 25 | check if `go` is installed based on the condition `validate_go`. You can achieve this with the following terraform code: 26 | 27 | ```hcl 28 | module "require_executables" { 29 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/require-executable?ref=v1.0.8" 30 | required_executables = ["${var.validate_go ? "go" : ""}"] 31 | } 32 | ``` 33 | -------------------------------------------------------------------------------- /modules/request-quota-increase/CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog request-quota-increase 2 | 3 | ## [v0.9.3](https://github.com/gruntwork-io/terraform-aws-utilities/releases/tag/v0.9.3) - 2023-08-18 4 | 5 | ### Changed 6 | - No changes, safe to bump 7 | 8 | ## [v0.9.2](https://github.com/gruntwork-io/terraform-aws-utilities/releases/tag/v0.9.2) - 2023-06-01 9 | 10 | ### Changed 11 | - No breaking changes, safe to bump 12 | 13 | ### Description 14 | - Restrict AWS provider version `>= 3.75.1, < 6.0.0` 15 | 16 | ### Related Links 17 | - https://github.com/gruntwork-io/terraform-aws-utilities/pull/80 18 | 19 | ## [v0.9.1](https://github.com/gruntwork-io/terraform-aws-utilities/releases/tag/v0.9.1) - 2023-03-06 20 | 21 | ### Changed 22 | - No changes, safe to bump 23 | 24 | ## [v0.9.0](https://github.com/gruntwork-io/terraform-aws-utilities/releases/tag/v0.9.0) - 2022-06-20 25 | 26 | ### Changed 27 | - Breaking changes 28 | - No patch available 29 | 30 | ### Description 31 | - Unlock AWS provider v4. Require minimum 3.75.1 32 | 33 | ### Migration Guide 34 | 35 | The AWS Provider v4 unlock is a functionally backward compatible update. Modules no longer have the AWS Provider v4 lock. Upgrade tests were run to give reasonable confidence that upgrading to this version of the modules from the last tagged release is backward compatible, requiring no further modifications from you. However, the AWS Provider version must be `3.75.1` or newer, so you may need to run `terraform init` with the `-upgrade` flag, which will allow terraform to pull the latest version of the AWS provider, as in `terraform init -upgrade`. 36 | 37 | ### Related Links 38 | - https://github.com/gruntwork-io/terraform-aws-utilities/pull/58 39 | -------------------------------------------------------------------------------- /codegen/quotas/README.md: -------------------------------------------------------------------------------- 1 | # AWS Service Quotas Generator 2 | 3 | This Python script is used to generate Terraform files for managing AWS service quota requests. It interacts with the AWS Service Quotas API and fetches information about the quotas for different services. The script then generates Terraform code based on this information and writes it to (`main.tf` and `variables.tf`) files. 4 | 5 | ## Gotchas 6 | 7 | - Generating the quotas could be time consuming as the script honors the API limits for the used AWS APIs. 8 | - Certain services have duplicate quotas - same description but different code. Those are handled by appending the quota code to the input variable name. 9 | 10 | ## Requirements 11 | - Python 3.6+ 12 | - Boto3 13 | - AWS CLI (optional, for configuring AWS credentials) 14 | 15 | ## Usage 16 | 17 | Ensure you have valid AWS credentials to access the service quotas service. 18 | 19 | ### Install Dependencies 20 | Install the required Python packages by running: 21 | 22 | ``` 23 | pip install -r requirements.txt 24 | ``` 25 | 26 | ### Command Line Arguments 27 | The script accepts the following command line arguments: 28 | 29 | - `--region` (optional): Specify the AWS region to query service quotas for. Defaults to `us-east-1`. 30 | - `--outdir` (optional): Output directory for the resulting terraform files. Defaults to `../../modules/request-quota-increase`. 31 | 32 | ### Running the Script 33 | To run the script with default settings (region `us-east-1` and output `../../modules/request-quota-increase`): 34 | 35 | ``` 36 | python generate_quotas.py 37 | ``` 38 | 39 | To specify a different region and output file: 40 | 41 | ``` 42 | python generate_quotas.py --region us-west-2 --outdir "./path/to/your/dir" 43 | 44 | ``` 45 | -------------------------------------------------------------------------------- /modules/run-pex-as-data-source/variables.tf: -------------------------------------------------------------------------------- 1 | variable "python_pex_path_parts" { 2 | description = "Parts of the path (folders and files names) to the PEX executable for python as a list of strings." 3 | type = list(string) 4 | # Example: 5 | # python_pex_path_parts = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 6 | } 7 | 8 | variable "pex_module_path_parts" { 9 | description = "Parts of the path (folders and file names) to the python package directory housing the pex file." 10 | type = list(string) 11 | # Example: 12 | # pex_module_path_parts = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 13 | } 14 | 15 | variable "script_main_function" { 16 | description = "Main function of the script, encoded as SCRIPT_MODULE:FUNCTION. So for example, if the main function of the script is in a file named `entrypoint.py` which houses the function `main`, then this should be `entrypoint:main`." 17 | type = string 18 | } 19 | 20 | variable "command_args" { 21 | description = "The arguments to pass to the command as a string" 22 | type = string 23 | 24 | # We don't use null here because this is interpolated into the python script. 25 | default = "" 26 | } 27 | 28 | variable "command_query" { 29 | description = "The query for the command run as a data source." 30 | type = map(string) 31 | default = {} 32 | } 33 | 34 | variable "enabled" { 35 | description = "If you set this variable to false, this module will not run the PEX script. This is used as a workaround because Terraform does not allow you to use the 'count' parameter on modules. By using this parameter, you can optionally enable the data source within this module. Note that when false, the 'result' output will be null." 36 | type = bool 37 | default = true 38 | } 39 | -------------------------------------------------------------------------------- /test/request_quota_increase_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/aws" 7 | "github.com/gruntwork-io/terratest/modules/terraform" 8 | "github.com/stretchr/testify/assert" 9 | ) 10 | 11 | type ( 12 | QuotaAndServiceName struct { 13 | QuotaName string `json:"quota_name"` 14 | QuotaCode string `json:"quota_code"` 15 | ServiceName string `json:"service_name"` 16 | ServiceCode string `json:"service_code"` 17 | } 18 | QuotaIncreaseOutput struct { 19 | NatGateway QuotaAndServiceName `json:"vpc_nat_gateways_per_availability_zone"` 20 | NaclRules QuotaAndServiceName `json:"vpc_rules_per_network_acl"` 21 | } 22 | ) 23 | 24 | func TestRequestQuotaIncrease(t *testing.T) { 25 | t.Parallel() 26 | 27 | awsRegion := aws.GetRandomRegion(t, nil, nil) 28 | 29 | terraformOptions := &terraform.Options{ 30 | TerraformDir: "../examples/request-quota-increase", 31 | Vars: map[string]interface{}{ 32 | "aws_region": awsRegion, 33 | }, 34 | } 35 | 36 | defer terraform.Destroy(t, terraformOptions) 37 | 38 | terraform.InitAndApply(t, terraformOptions) 39 | 40 | output := QuotaIncreaseOutput{} 41 | terraform.OutputStruct(t, terraformOptions, "new_quotas", &output) 42 | 43 | assert.Equal(t, output.NatGateway.QuotaName, "NAT gateways per Availability Zone") 44 | assert.Equal(t, output.NatGateway.ServiceName, "Amazon Virtual Private Cloud (Amazon VPC)") 45 | assert.Equal(t, output.NatGateway.ServiceCode, "vpc") 46 | assert.Equal(t, output.NatGateway.QuotaCode, "L-FE5A380F") 47 | 48 | assert.Equal(t, output.NaclRules.QuotaName, "Rules per network ACL") 49 | assert.Equal(t, output.NaclRules.ServiceName, "Amazon Virtual Private Cloud (Amazon VPC)") 50 | assert.Equal(t, output.NaclRules.ServiceCode, "vpc") 51 | assert.Equal(t, output.NaclRules.QuotaCode, "L-2AEEBF1A") 52 | } 53 | -------------------------------------------------------------------------------- /examples/instance-type/main.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.0.0" 3 | } 4 | 5 | provider "aws" { 6 | region = var.aws_region 7 | } 8 | 9 | # ---------------------------------------------------------------------------------------------------------------------- 10 | # FIGURE OUT WHAT INSTANCE TYPE IS AVAILABLE IN ALL AZS IN THE CURRENT AWS REGION 11 | # ---------------------------------------------------------------------------------------------------------------------- 12 | 13 | module "instance_types" { 14 | source = "../../modules/instance-type" 15 | 16 | instance_types = var.instance_types 17 | } 18 | 19 | # ---------------------------------------------------------------------------------------------------------------------- 20 | # USE THAT INSTANCE TYPE TO LAUNCH AN EC2 INSTANCE 21 | # ---------------------------------------------------------------------------------------------------------------------- 22 | 23 | resource "aws_instance" "example" { 24 | ami = data.aws_ami.ubuntu.id 25 | instance_type = module.instance_types.recommended_instance_type 26 | 27 | tags = { 28 | Name = "instance-type-example" 29 | } 30 | } 31 | 32 | # ---------------------------------------------------------------------------------------------------------------------- 33 | # FETCH THE ID OF AN UBUNTU AMI IN THE CURRENT REGION 34 | # ---------------------------------------------------------------------------------------------------------------------- 35 | 36 | data "aws_ami" "ubuntu" { 37 | most_recent = true 38 | owners = ["099720109477"] # Canonical 39 | 40 | filter { 41 | name = "virtualization-type" 42 | values = ["hvm"] 43 | } 44 | 45 | filter { 46 | name = "architecture" 47 | values = ["x86_64"] 48 | } 49 | 50 | filter { 51 | name = "image-type" 52 | values = ["machine"] 53 | } 54 | 55 | filter { 56 | name = "name" 57 | values = ["ubuntu/images/hvm-ssd/ubuntu-focal-20.04-amd64-server-*"] 58 | } 59 | } -------------------------------------------------------------------------------- /test/README.md: -------------------------------------------------------------------------------- 1 | # Tests 2 | 3 | This folder contains the tests for the modules in this repo. 4 | 5 | 6 | 7 | 8 | ## Running the tests locally 9 | 10 | **Note #1**: Many of these tests create real resources in an AWS account. That means they cost money to run, especially 11 | if you don't clean up after yourself. Please be considerate of the resources you create and take extra care to clean 12 | everything up when you're done! 13 | 14 | **Note #2**: Never hit `CTRL + C` or cancel a build once tests are running or the cleanup tasks won't run! 15 | 16 | **Note #3**: We set `-timeout 45m` on all tests not because they necessarily take 45 minutes, but because Go has a 17 | default test timeout of 10 minutes, after which it does a `SIGQUIT`, preventing the tests from properly cleaning up 18 | after themselves. Therefore, we set a timeout of 45 minutes to make sure all tests have enough time to finish and 19 | cleanup. 20 | 21 | 22 | 23 | 24 | ### Prerequisites 25 | 26 | - Install the latest version of [Go](https://golang.org/). 27 | - Install [glide](https://glide.sh/) for Go dependency management. On OSX, the simplest way to install is 28 | `brew update; brew install glide`. 29 | - Install [Terraform](https://www.terraform.io/downloads.html). 30 | - Add your AWS credentials as environment variables: `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` 31 | - For some of the tests, you also need to set the `GITHUB_OAUTH_TOKEN` environment variable to a valid GitHub 32 | auth token with "repo" access. You can generate one here: https://github.com/settings/tokens 33 | 34 | 35 | ### Setup 36 | 37 | Download Go dependencies using glide: 38 | 39 | ``` 40 | cd test 41 | glide update 42 | ``` 43 | 44 | 45 | ### Run all the tests 46 | 47 | ```bash 48 | cd test 49 | go test -v -timeout 45m -parallel 128 50 | ``` 51 | 52 | 53 | ### Run a specific test 54 | 55 | To run a specific test called `TestFoo`: 56 | 57 | ```bash 58 | cd test 59 | go test -v -timeout 45m -parallel 128 -run TestFoo 60 | ``` 61 | -------------------------------------------------------------------------------- /modules/instance-type/README.md: -------------------------------------------------------------------------------- 1 | # Instance Type 2 | 3 | This is a module that can be used to look up a list of EC2 instance types and determine which of them is available in 4 | all Availability Zones (AZs) in the current AWS Region. This is useful because certain instance types, such as 5 | `t2.micro` are not available in some of the newer AZs, while `t3.micro` is not available in some of the older AZs, and 6 | if you have code that needs to run on a "small" instance across all AZs in many different regions, you can use this 7 | module to automatically figure out which instance type you should use. 8 | 9 | 10 | 11 | 12 | 13 | ## Example code 14 | 15 | See the [instance-type example](/examples/instance-type) for working sample code. 16 | 17 | 18 | 19 | 20 | ## Usage 21 | 22 | Use the module in your Terraform code, replacing `` with the latest version from the [releases 23 | page](https://github.com/gruntwork-io/terraform-aws-utilities/releases): 24 | 25 | ```hcl 26 | module "path" { 27 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/instance-type?ref=" 28 | 29 | instance_types = ["t2.micro", "t3.micro"] 30 | } 31 | ``` 32 | 33 | The arguments to pass are: 34 | 35 | * `instance_types`: A list of instance types to look up in the current AWS region. We recommend putting them in order 36 | of preference, as the recommended_instance_type output variable will contain the first instance type from this list 37 | that is available in all AZs. 38 | 39 | When you run `apply`, the `recommended_instance_type` output variable will contain the recommended instance type to 40 | use. This will be the first instance type from your `instance_types` input that is available in all AZs in the current 41 | region. If no instance type is available in all AZs, you'll get an error. 42 | 43 | For example, as of July, 2020, if you run `apply` on the code above in `eu-west-1`, the `recommended_instance_type` 44 | will be `t2.micro`, as that's available in all AZs in `eu-west-1`. However, if you run the same code in 45 | `ap-northeast-2`, the `recommended_instance_type` will be `t3.micro`, as `t2.micro` is only available in 2 of the 4 AZs. -------------------------------------------------------------------------------- /modules/run-pex-as-data-source/main.tf: -------------------------------------------------------------------------------- 1 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 | # RUN PEX BINARY AS AN EXTERNAL DATA SOURCE 3 | # This terraform module runs the provided pex binary in the context of an external data source, and pipes the output 4 | # back as an output to this module. 5 | # This utilizes the `prepare-pex-environment` module to ensure the execution of the binary is done in a portable manner. 6 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 7 | 8 | terraform { 9 | required_version = ">= 1.0.0" 10 | } 11 | 12 | module "pex_env" { 13 | source = "../prepare-pex-environment" 14 | python_pex_path_parts = var.python_pex_path_parts 15 | pex_module_path_parts = var.pex_module_path_parts 16 | } 17 | 18 | data "external" "pex" { 19 | count = var.enabled ? 1 : 0 20 | 21 | program = [ 22 | "python3", 23 | "-c", 24 | 25 | # Since an external data source is not run through a shell, we can't set the PYTHONPATH through terraform, so we 26 | # indirectly run the pex script through python. 27 | # Note: The interpolated strings are embedded as python raw strings to avoid escape char issues with Windows. 28 | # In Windows, the path separator is `\` which means that depending on the terraform module hash string, it can be 29 | # interpretted as an escape character. 30 | # E.g suppose the module where the PEX resides in hashes to 0555f. Then the path to the pex binary becomes 31 | # `path\to\terraform\code\0555f\bin\pex`. This is a problem because `\0` is the null character in a python string. 32 | <<-PROGRAM 33 | import os 34 | import shlex 35 | import subprocess 36 | env = os.environ.copy() 37 | env["PYTHONPATH"] = r"${module.pex_env.python_path}" 38 | subprocess.check_call( 39 | [ 40 | "python3", 41 | r"${module.pex_env.pex_path}", 42 | r"${module.pex_env.entrypoint_path}", 43 | r"${var.script_main_function}", 44 | ] + shlex.split(r"${var.command_args}"), 45 | env=env, 46 | ) 47 | PROGRAM 48 | ] 49 | 50 | query = var.command_query 51 | } 52 | -------------------------------------------------------------------------------- /modules/executable-dependency/variables.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # REQUIRED MODULE PARAMETERS 3 | # These variables must be passed in by the operator. 4 | # --------------------------------------------------------------------------------------------------------------------- 5 | 6 | variable "executable" { 7 | type = string 8 | description = "The executable to look for on the system PATH and in var.install_dir. If not found, this executable will be downloaded from var.download_url." 9 | } 10 | 11 | variable "download_url" { 12 | type = string 13 | description = "The URL to download the executable from if var.executable is not found on the system PATH or in var.install_dir." 14 | } 15 | 16 | # --------------------------------------------------------------------------------------------------------------------- 17 | # OPTIONAL MODULE PARAMETERS 18 | # These variables have reasonable defaults, but may be overridden if necessary. 19 | # --------------------------------------------------------------------------------------------------------------------- 20 | 21 | variable "append_os_arch" { 22 | type = bool 23 | description = "If set to true, append the operating system and architecture to the URL. E.g., Append linux_amd64 if this code is being run on a 64 bit Linux OS." 24 | default = true 25 | } 26 | 27 | variable "install_dir" { 28 | type = string 29 | description = "The folder to copy the executable to after downloading it from var.download_url. If set to null (the default), the executable will be copied to a folder in the system temp directory. The folder will be named based on an md5 hash of var.download_url, so for each var.download_url, the executable will only have to be downloaded once." 30 | default = null 31 | } 32 | 33 | variable "enabled" { 34 | description = "Set to false to have disable this module, so it does not try to download the executable, and always returns its path unchanged. This weird parameter exists solely because Terraform does not support conditional modules. Therefore, this is a hack to allow you to conditionally decide if this module should run or not." 35 | type = bool 36 | default = true 37 | } -------------------------------------------------------------------------------- /codegen/quotas/templates.py: -------------------------------------------------------------------------------- 1 | import re 2 | 3 | 4 | def get_variable_name(service_code, quota_name): 5 | variable_name = f"{service_code}_{quota_name}".lower() 6 | return re.sub(r'\W+', '_', variable_name) 7 | 8 | def terraform_variable_template(service_code, quota_name, quota_code): 9 | variable_name = get_variable_name(service_code, quota_name) 10 | return f'''variable "{variable_name}" {{ 11 | description = "Quota for [{service_code}]: {quota_name} ({quota_code})" 12 | type = number 13 | default = null 14 | }}\n\n''' 15 | 16 | def terraform_locals_template(service_code, quota_name, quota_code): 17 | variable_name = get_variable_name(service_code, quota_name) 18 | return f''' {variable_name} = {{ 19 | quota_code = "{quota_code}" 20 | service_code = "{service_code}" 21 | desired_quota = var.{variable_name} 22 | }},\n''' 23 | 24 | def terraform_main(all_quotas): 25 | return f'''# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 26 | # CONFIGURE SERVICE QUOTAS 27 | # NOTE: This module is autogenerated. Do not modify it manually. 28 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 29 | 30 | terraform {{ 31 | required_version = ">= 1.0.0" 32 | required_providers {{ 33 | aws = {{ 34 | source = "hashicorp/aws" 35 | version = ">= 3.75.1, < 7.0.0" 36 | }} 37 | }} 38 | }} 39 | 40 | locals {{ 41 | all_quotas = {{ 42 | {all_quotas} 43 | }} 44 | 45 | adjusted_quotas = {{ 46 | for k, v in local.all_quotas : k => v 47 | if v.desired_quota != null 48 | }} 49 | }} 50 | 51 | resource "aws_servicequotas_service_quota" "increase_quotas" {{ 52 | for_each = local.adjusted_quotas 53 | 54 | quota_code = each.value.quota_code 55 | service_code = each.value.service_code 56 | value = each.value.desired_quota 57 | }} 58 | 59 | ''' 60 | 61 | def terraform_vars(all_vars): 62 | return f'''# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 63 | # INPUT VARIABLES FOR SERVICE QUOTAS 64 | # NOTE: This module is autogenerated. Do not modify it manually. 65 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 66 | 67 | {all_vars} 68 | 69 | \n''' 70 | -------------------------------------------------------------------------------- /test/upgrades/upgrade_test.go: -------------------------------------------------------------------------------- 1 | package upgrades 2 | 3 | import ( 4 | "testing" 5 | 6 | upgrades "github.com/gruntwork-io/terraform-aws-ci/test/upgrade-tester" 7 | "github.com/gruntwork-io/terratest/modules/aws" 8 | "github.com/gruntwork-io/terratest/modules/terraform" 9 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 10 | ) 11 | 12 | // The following lists are to keep track of which of the examples we've added upgrade tests for, 13 | // and which modules we've tested upgrading. Commented ones are not covered in upgrade tests yet. 14 | var examplesToTest = upgrades.ExampleConfig{ 15 | //"executable-dependency", 16 | "instance-type": { 17 | SetupFn: setupForInstanceType, 18 | }, 19 | 20 | "join-path": { 21 | SetupFn: setupForJoinPath, 22 | }, 23 | //"list-remove", 24 | //"operating-system", 25 | //"pex", 26 | //"request-quota-increase", 27 | //"require-executable", 28 | } 29 | 30 | // Once all of the modules are uncommented, we can replace the modulesToUpgrade, passed into each test, 31 | // with this full list. Only the modules below that are found in the examples will get upgraded. 32 | var modulesToTest = []string{ 33 | //"executable-dependency", 34 | "instance-type", 35 | "join-path", 36 | //"list-remove", 37 | //"operating-system", 38 | //"prepare-pex-environment", 39 | //"request-quota-increase", 40 | //"require-executable", 41 | //"run-pex-as-data-source", 42 | //"run-pex-as-resource", 43 | } 44 | 45 | func TestUpgradeModules(t *testing.T) { 46 | config := upgrades.UpgradeModuleTestConfig{ 47 | RepoName: "terraform-aws-utilities", 48 | ModulesToTest: modulesToTest, 49 | ExampleConfig: examplesToTest, 50 | } 51 | 52 | upgrades.RunUpgradeModuleTests(t, config) 53 | } 54 | 55 | func setupForInstanceType(t *testing.T, workingDir string, uniqueID string) *terraform.Options { 56 | awsRegion := aws.GetRandomRegion(t, nil, nil) 57 | test_structure.SaveString(t, workingDir, "awsRegion", awsRegion) 58 | 59 | terraformOptions := &terraform.Options{ 60 | Vars: map[string]interface{}{ 61 | "aws_region": awsRegion, 62 | }, 63 | Upgrade: true, 64 | } 65 | 66 | return terraformOptions 67 | } 68 | 69 | func setupForJoinPath(t *testing.T, workingDir string, uniqueID string) *terraform.Options { 70 | terraformOptions := &terraform.Options{ 71 | Vars: map[string]interface{}{}, 72 | Upgrade: true, 73 | } 74 | 75 | return terraformOptions 76 | } 77 | -------------------------------------------------------------------------------- /modules/run-pex-as-resource/README.md: -------------------------------------------------------------------------------- 1 | # Run PEX as Resource 2 | 3 | This module runs the provided PEX binary in a portable manner that works with multiple platforms and python versions, in 4 | the context of a [local-exec provisioner](https://www.terraform.io/docs/provisioners/local-exec.html) in Terraform. 5 | 6 | This module uses [`prepare-pex-environment`](../prepare-pex-environment) under the hood. See [What is 7 | PEX?](../prepare-pex-environment/README.md#what-is-pex) for more details on what is a PEX file and how to construct one 8 | for use with this module. 9 | 10 | 11 | ## How do you use this module? 12 | 13 | * See the [pex example](/examples/pex) for example usage. 14 | * See [variables.tf](./variables.tf) for all the variables you can set on this module. 15 | * See [outputs.tf](./outputs.tf) for all the variables that are outputed by this module. 16 | 17 | 18 | ## Data Source vs Resource 19 | 20 | Terraform provides two escape hatches where a first-class Terraform provider is not more appropriate. The escape hatches 21 | allow you to call out to arbitrary binaries available on the operator machine. These are: 22 | 23 | - [External Data Source](https://www.terraform.io/docs/providers/external/data_source.html), where you can run the 24 | binary as a data source. 25 | - [local-exec Provisioners](https://www.terraform.io/docs/provisioners/local-exec.html), where you can run the binary to 26 | provision a resource. 27 | 28 | This module uses the Provisioner approach (you can see the [run-pex-as-data-source module](../run-pex-as-data-source) 29 | for running it as a data source). Which approach to use depends on your needs: 30 | 31 | - Data sources are calculated every time a terraform state needs to be refreshed. This includes all `plan` and `apply` 32 | calls, even if the data source isn't explicitly changed. 33 | - Data sources are useful if the logic can be used to determine if a resource needs to be changed. 34 | - Data sources can output values that can be used in other parts of the Terraform code. You cannot do this with the 35 | provisioner approach. 36 | - There are limitations with Data Sources and dependencies. See [this terraform issue 37 | comment](https://github.com/hashicorp/terraform/issues/10603#issuecomment-265777128) for example. 38 | - Provisioners with a `null_resource` implements the standard resource life cycle (create, destroy, etc). 39 | - Provisioners with a `null_resource` have explicit controls on when to trigger. 40 | -------------------------------------------------------------------------------- /modules/run-pex-as-data-source/README.md: -------------------------------------------------------------------------------- 1 | # Run PEX as Data Source 2 | 3 | This module runs the provided PEX binary in a portable manner that works with multiple platforms and python versions, to 4 | be used as an [external data source](https://www.terraform.io/docs/providers/external/data_source.html) in Terraform. 5 | 6 | This module uses [`prepare-pex-environment`](../prepare-pex-environment) under the hood. See [What is 7 | PEX?](../prepare-pex-environment/README.md#what-is-pex) for more details on what is a PEX file and how to construct one 8 | for use with this module. 9 | 10 | 11 | ## How do you use this module? 12 | 13 | * See the [pex example](/examples/pex) for example usage. 14 | * See [variables.tf](./variables.tf) for all the variables you can set on this module. 15 | * See [outputs.tf](./outputs.tf) for all the variables that are outputed by this module. 16 | 17 | 18 | ## Data Source vs Resource 19 | 20 | Terraform provides two escape hatches where a first-class Terraform provider is not more appropriate. The escape hatches 21 | allow you to call out to arbitrary binaries available on the operator machine. These are: 22 | 23 | - [External Data Source](https://www.terraform.io/docs/providers/external/data_source.html), where you can run the 24 | binary as a data source. 25 | - [local-exec Provisioners](https://www.terraform.io/docs/provisioners/local-exec.html), where you can run the binary to 26 | provision a resource. 27 | 28 | This module uses the data source approach (you can see the [run-pex-as-resource module](../run-pex-as-resource) for 29 | running it as a data source). Which approach to use depends on your needs: 30 | 31 | - Data sources are calculated every time a terraform state needs to be refreshed. This includes all `plan` and `apply` 32 | calls, even if the data source isn't explicitly changed. 33 | - Data sources are useful if the logic can be used to determine if a resource needs to be changed. 34 | - Data sources can output values that can be used in other parts of the Terraform code. You cannot do this with the 35 | provisioner approach. 36 | - There are limitations with Data Sources and dependencies. See [this terraform issue 37 | comment](https://github.com/hashicorp/terraform/issues/10603#issuecomment-265777128) for example. 38 | - Provisioners with a `null_resource` implements the standard resource life cycle (create, destroy, etc). 39 | - Provisioners with a `null_resource` have explicit controls on when to trigger. 40 | -------------------------------------------------------------------------------- /terraform-cloud-enterprise-private-module-registry-placeholder.tf: -------------------------------------------------------------------------------- 1 | # --------------------------------------------------------------------------------------------------------------------- 2 | # THIS IS A PLACEHOLDER MODULE SO YOU CAN ADD THIS REPO TO THE PRIVATE MODULE REGISTRY IN TERRAFORM CLOUD / ENTERPRISE 3 | # 4 | # All the real modules in this repo are in the /modules sub-folder, but Terraform Cloud and Terraform Enterprise 5 | # require that you have Terraform code in the root of the repo, so this placeholder .tf file is necessary to allow you 6 | # to add the modules in this repo to your Private Module Registry. The real modules will show up in the Private 7 | # Module Registry UI under the "sub-modules" drop-down and you can use a sub-module named by adding 8 | # //modules/ to the source URL: 9 | # 10 | # module "" { 11 | # source = "/////modules/" 12 | # version = "" 13 | # } 14 | # 15 | # For example, to use version v0.12.4 of the vpc-app module from the terraform-aws-vpc repo with Terraform Cloud and an 16 | # organization named acme: 17 | # 18 | # module "vpc" { 19 | # source = "app.terraform.io/acme/vpc/aws//modules/vpc-app" 20 | # version = "v0.12.4" 21 | # } 22 | # 23 | # Or, to use v0.15.2 of the eks-cluster module from the terraform-aws-service-catalog repo with a Terraform Enterprise 24 | # install at the URL terraform.acme.com and the organization name sre-team: 25 | # 26 | # module "vpc" { 27 | # source = "terraform.acme.com/sre-team/service-catalog/aws//modules/eks-cluster" 28 | # version = "v0.15.2" 29 | # } 30 | # 31 | # --------------------------------------------------------------------------------------------------------------------- 32 | 33 | # We add this variable here so that the instructions for using sub-modules are visible in the Private Module Registry 34 | # UI. 35 | variable "README" { 36 | description = "All the real modules in this repo are in the /modules sub-folder, but Terraform Cloud and Terraform Enterprise require that you have Terraform code in the root of the repo, so this placeholder .tf file is necessary to allow you to add the modules in this repo to your Private Module Registry. The real modules will show up in the Private Module Registry UI under the \"sub-modules\" drop-down and you can use a sub-module named by adding //modules/ to the source URL. For example, to use the vpc-app module from the terraform-aws-vpc repo with Terraform Cloud and an organization named acme, you'd set source = \"app.terraform.io/acme/vpc/aws//modules/vpc-app\"" 37 | } 38 | -------------------------------------------------------------------------------- /modules/run-pex-as-resource/variables.tf: -------------------------------------------------------------------------------- 1 | variable "python_pex_path_parts" { 2 | description = "Parts of the path (folders and files names) to the PEX executable for python as a list of strings." 3 | type = list(string) 4 | # Example: 5 | # default = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 6 | } 7 | 8 | variable "pex_module_path_parts" { 9 | description = "Parts of the path (folders and file names) to the python package directory housing the pex file." 10 | type = list(string) 11 | # Example: 12 | # default = ["foo", "bar", "baz.txt"] => outputs "foo/bar/baz.txt" on Linux 13 | } 14 | 15 | variable "script_main_function" { 16 | description = "Main function of the script, encoded as SCRIPT_MODULE:FUNCTION. So for example, if the main function of the script is in a file named `entrypoint.py` which houses the function `main`, then this should be `entrypoint:main`." 17 | type = string 18 | } 19 | 20 | variable "command_args" { 21 | description = "The arguments to pass to the command as a string" 22 | type = string 23 | 24 | # We don't use null here because this is interpolated into the python script. 25 | default = "" 26 | } 27 | 28 | variable "triggers" { 29 | description = "A map of arbitrary strings that, when changed, will force the null resource to be replaced, re-running any associated provisioners." 30 | type = map(string) 31 | default = null 32 | } 33 | 34 | variable "env" { 35 | description = "Additional environment variables to set for the command." 36 | type = map(string) 37 | default = {} 38 | } 39 | 40 | variable "enabled" { 41 | description = "If you set this variable to false, this module will not run the PEX script. This is used as a workaround because Terraform does not allow you to use the 'count' parameter on modules. By using this parameter, you can optionally enable the null_resource within this module." 42 | type = bool 43 | default = true 44 | } 45 | 46 | variable "pass_in_previous_triggers" { 47 | description = "If you set this variable to true, this module will pass in the json encoded triggers that were used when the resource was created. If the script expects option args, use var.previous_trigger_option to set which option to pass the triggers json as." 48 | type = bool 49 | default = false 50 | } 51 | 52 | variable "previous_trigger_option" { 53 | description = "Pass in the json encoded trigger with this string as the option to passing into the command. E.g, setting this to `--triggers` will pass in the option `--triggers TRIGGERS_JSON`." 54 | type = string 55 | default = "" 56 | } 57 | -------------------------------------------------------------------------------- /modules/executable-dependency/README.md: -------------------------------------------------------------------------------- 1 | # Executable Dependency 2 | 3 | This is a module that can be used to check if an executable is already installed, and if it's not, download it from a 4 | URL. This is useful if your Terraform code has an external dependency and you want that dependency to be auto installed 5 | if it's not installed already: e.g., [terraform-aws-eks](https://github.com/gruntwork-io/terraform-aws-eks) expects the 6 | [kubergrunt](https://github.com/gruntwork-io/kubergrunt) binary to be installed, and `executable-dependency` allows 7 | `terraform-aws-eks` to automatically download `kubergrunt` if it's not already available. 8 | 9 | **NOTE**: This module requires that Python 3 is installed on your system. 10 | 11 | 12 | 13 | 14 | ## Example code 15 | 16 | See the [executable-dependency example](/examples/executable-dependency) for working sample code. 17 | 18 | 19 | 20 | 21 | ## Usage 22 | 23 | Use the module in your Terraform code, replacing `` with the latest version from the [releases 24 | page](https://github.com/gruntwork-io/terraform-aws-utilities/releases): 25 | 26 | ```hcl 27 | module "path" { 28 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/executable-dependency?ref=" 29 | 30 | executable = "kubergrunt" 31 | download_url = "https://github.com/gruntwork-io/kubergrunt/releases/download/v0.5.13/kubergrunt" 32 | append_os_arch = true 33 | } 34 | ``` 35 | 36 | The arguments to pass are: 37 | 38 | * `executable`: The executable to look for on the system `PATH` and in `install_dir`. If not found, this executable 39 | will be downloaded from `download_url`. 40 | 41 | * `download_url`: The URL to download the executable from if `executable` is not found on the system `PATH` or 42 | `install_dir`. 43 | 44 | * `append_os_arch`: If set to true, append the operating system and architecture to the URL. E.g., Append `linux_amd64` 45 | if this code is being run on a 64 bit Linux OS. This is useful to download the proper binary (specifically, a binary 46 | using the Go naming conventions) for the current operating system and CPU. 47 | 48 | * `install_dir`: The folder to copy the executable to after downloading it from `download_url`. If set to `null` (the 49 | default), the executable will be copied to a folder in the system temp directory. The folder will be named based on 50 | an md5 hash of `download_url`, so for each `download_url`, the executable will only have to be downloaded once. 51 | 52 | This module has a single output, `executable_path`, which is the path you should use to run the executable. The value 53 | will either be the path of the executable on the system `PATH` or a path in `install_dir`. 54 | -------------------------------------------------------------------------------- /test/pex_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/terratest/modules/random" 9 | "github.com/gruntwork-io/terratest/modules/terraform" 10 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestRunPex(t *testing.T) { 15 | t.Parallel() 16 | 17 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 18 | terratestOptions := createBaseTerratestOptions(t, filepath.Join(testFolder, "pex")) 19 | defer terraform.Destroy(t, terratestOptions) 20 | 21 | expectedFoo := random.UniqueId() 22 | terratestOptions.Vars = map[string]interface{}{ 23 | "echo_string": expectedFoo, 24 | } 25 | 26 | output := terraform.InitAndApply(t, terratestOptions) 27 | 28 | assertOutputEquals(t, "command_echo", expectedFoo, terratestOptions) 29 | assert.Contains(t, output, fmt.Sprintf("Environment variable: %s", expectedFoo)) 30 | } 31 | 32 | func TestRunPexTriggers(t *testing.T) { 33 | t.Parallel() 34 | 35 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 36 | terratestOptions := createBaseTerratestOptions(t, filepath.Join(testFolder, "pex")) 37 | expectedFoo := random.UniqueId() 38 | 39 | defer terraform.Destroy(t, terratestOptions) 40 | 41 | terratestOptions.Vars = map[string]interface{}{ 42 | "echo_string": expectedFoo, 43 | "triggers": map[string]string{ 44 | "id": expectedFoo, 45 | }, 46 | } 47 | 48 | terraform.InitAndApply(t, terratestOptions) 49 | 50 | assertOutputEquals(t, "command_echo", expectedFoo, terratestOptions) 51 | 52 | // Assert that there is no diff 53 | exitCode := terraform.PlanExitCode(t, terratestOptions) 54 | assert.Equal(t, exitCode, 0) 55 | 56 | // Now modify the trigger and verify the null_resource is recreated 57 | terratestOptions.Vars["triggers"].(map[string]string)["id"] = random.UniqueId() 58 | exitCode = terraform.PlanExitCode(t, terratestOptions) 59 | assert.NotEqual(t, exitCode, 0) 60 | } 61 | 62 | func TestRunPexDisabled(t *testing.T) { 63 | t.Parallel() 64 | 65 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 66 | terratestOptions := createBaseTerratestOptions(t, filepath.Join(testFolder, "pex")) 67 | defer terraform.Destroy(t, terratestOptions) 68 | 69 | expectedFoo := random.UniqueId() 70 | terratestOptions.Vars = map[string]interface{}{ 71 | "echo_string": expectedFoo, 72 | "enabled": false, 73 | } 74 | 75 | output := terraform.InitAndApply(t, terratestOptions) 76 | assert.NotContains(t, output, fmt.Sprintf("Environment variable: %s", expectedFoo)) 77 | 78 | allOutputs := terraform.OutputAll(t, terratestOptions) 79 | _, hasOutput := allOutputs["command_echo"] 80 | assert.False(t, hasOutput) 81 | } 82 | -------------------------------------------------------------------------------- /test/require_executable_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "path/filepath" 5 | "strings" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/terratest/modules/random" 9 | "github.com/gruntwork-io/terratest/modules/terraform" 10 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 11 | "github.com/stretchr/testify/assert" 12 | ) 13 | 14 | func TestRequireExecutableWorksWithExistingExecutable(t *testing.T) { 15 | t.Parallel() 16 | 17 | randomString := random.UniqueId() 18 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 19 | terraformModulePath := filepath.Join(testFolder, "require-executable") 20 | terratestOptions := createBaseTerratestOptions(t, terraformModulePath) 21 | terratestOptions.Vars = map[string]interface{}{ 22 | "required_executables": []string{"go"}, 23 | "error_message": randomString, 24 | } 25 | defer terraform.Destroy(t, terratestOptions) 26 | 27 | out := terraform.InitAndApply(t, terratestOptions) 28 | assert.False(t, strings.Contains(out, randomString)) 29 | } 30 | 31 | func TestRequireExecutableFailsForMissingExecutable(t *testing.T) { 32 | t.Parallel() 33 | 34 | randomString := random.UniqueId() 35 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 36 | terraformModulePath := filepath.Join(testFolder, "require-executable") 37 | terratestOptions := createBaseTerratestOptions(t, terraformModulePath) 38 | terratestOptions.Vars = map[string]interface{}{ 39 | "required_executables": []string{"this-should-not-exist"}, 40 | "error_message": randomString, 41 | } 42 | // Swallow the error by using the E variant of Destroy, because the data source is expected to fail on both Apply 43 | // and Destroy 44 | defer terraform.DestroyE(t, terratestOptions) 45 | 46 | out, err := terraform.InitAndApplyE(t, terratestOptions) 47 | assert.Error(t, err) 48 | assert.True(t, strings.Contains(out, randomString)) 49 | } 50 | 51 | func TestConditionalRequireExecutable(t *testing.T) { 52 | t.Parallel() 53 | 54 | randomString := random.UniqueId() 55 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 56 | terraformModulePath := filepath.Join(testFolder, "require-executable") 57 | terratestOptions := createBaseTerratestOptions(t, terraformModulePath) 58 | terratestOptions.Vars = map[string]interface{}{ 59 | "required_executables": []string{}, 60 | "error_message": "", 61 | "validate_bad_executable": "1", 62 | "bad_executable_error_message": randomString, 63 | } 64 | defer terraform.Destroy(t, terratestOptions) 65 | 66 | out, err := terraform.InitAndApplyE(t, terratestOptions) 67 | assert.Error(t, err) 68 | assert.True(t, strings.Contains(out, randomString)) 69 | 70 | terratestOptions.Vars["validate_bad_executable"] = "0" 71 | out = terraform.InitAndApply(t, terratestOptions) 72 | assert.False(t, strings.Contains(out, randomString)) 73 | } 74 | -------------------------------------------------------------------------------- /examples/pex/README.md: -------------------------------------------------------------------------------- 1 | # PEX example 2 | 3 | This folder shows examples of how to use the [run-pex-as-data-source](/modules/run-pex-as-data-source) and [run-pex-as-resource](/modules/run-pex-as-resource) modules to run python scripts wrapped with a PEX environment. 4 | 5 | These modules use the [prepare-pex-environment module](/modules/prepare-pex-environment) under the hood to prepare the runtime environment for unpacking and running PEX binaries. You can learn more about PEX in [the module documentation](/modules/prepare-pex-environment/README.md). 6 | 7 | 8 | ## The python script 9 | 10 | This example will run the python script using the PEX environment provided in the [pex-env](./pex-env) folder. The PEX environment contains a virtualenv with all the requirements specified in [requirements.txt](./sample-python-script/requirements.txt) installed. The PEX environment only includes the virtualenv and does not include the script. If you run it directly, you will be dropped into a python shell with the PATH set such that you can import the requirements. If you would like to run the script directly, you must call it from the PEX: 11 | 12 | ```bash 13 | python ./sample-python-script/bin/sample_python_script_py3_env.pex ./sample-python-script/sample_python_script/main.py 14 | ``` 15 | 16 | Note that the provided py3 PEX will work with python versions 3.8-3.11. 17 | 18 | ### Building the PEX 19 | 20 | The PEX file is created using the `pex` utility. However, since it is versioned to be compatible with multiple versions of python, you must have each version of python installed. It is recommended to use [`pyenv`](https://github.com/pyenv/pyenv) to help setup an environment with multiple python interpreters. 21 | 22 | For convenience, a build script with a local pyenv version (a `.python-version` file) is provided. To use the build script, it is recommended to use [`pyenv`](https://github.com/pyenv/pyenv) to help setup multiple python interpreters. To install all necessary versions of python with pyenv: 23 | 24 | ```bash 25 | pyenv install 3.8.0 26 | pyenv install 3.9.0 27 | pyenv install 3.10.0 28 | pyenv install 3.11.0 29 | ``` 30 | 31 | After installing, you can use them to prepare for building: 32 | 33 | ```bash 34 | pyenv shell 3.8.0 3.9.0 3.10.0 3.11.0 35 | ``` 36 | 37 | Install PEX: 38 | 39 | ```bash 40 | python3 -m pip install pex==2.1.135 41 | ``` 42 | 43 | Once the python environment(s) are active, change your working directory to the `build_scripts` directory: 44 | 45 | ```bash 46 | # From the repo root 47 | cd ./examples/pex/pex-env/build_scripts 48 | ``` 49 | 50 | Finally, run the build script: 51 | 52 | ```bash 53 | ./build.sh 54 | ``` 55 | 56 | This will build the PEX binary files for each python version and put it in the `bin` directory. 57 | 58 | ## How do you run these examples? 59 | 60 | 1. Ensure python is installed in your OS. 61 | 1. Install [Terraform](https://www.terraform.io/). 62 | 1. `terraform init`. 63 | 1. `terraform apply`. 64 | -------------------------------------------------------------------------------- /modules/require-executable/require_executable.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to ensure the provided list of executables exist on the OS PATH. 3 | 4 | This script should: 5 | 6 | - Be platform independent (work with Windows, Linux, and Mac OS X). 7 | - Be python version independent (should support python 3.5+). 8 | - Have no external dependencies (should only use functions from stdlib). 9 | - Be compatible with Terraform external data source (should read in input from stdin, and read out output to stdout as a 10 | json). 11 | 12 | This script expects a json query from stdin that contains the key "required_executables", containing the list of 13 | names of required named executables as a comma separated value. 14 | """ 15 | 16 | from __future__ import print_function 17 | import json 18 | import logging 19 | import sys 20 | import shutil 21 | 22 | 23 | _ERROR_MESSAGE_EXECUTABLE_MARKER = "__EXECUTABLE_NAME__" 24 | _ERROR_MESSAGE_DEFAULT_TEMPLATE = "Not found: {}".format(_ERROR_MESSAGE_EXECUTABLE_MARKER) 25 | 26 | 27 | def configure_logger(): 28 | """ 29 | Configures the logging settings to log more information than default and set the appropriate log level. 30 | """ 31 | logger = logging.getLogger("require_executable") 32 | formatter = logging.Formatter( 33 | fmt="%(levelname)-8s %(asctime)s %(name)-28s %(message)s", 34 | datefmt="%Y-%m-%d %H:%M:%S", 35 | ) 36 | handler = logging.StreamHandler() 37 | handler.setFormatter(formatter) 38 | logger.addHandler(handler) 39 | logger.setLevel(logging.INFO) 40 | return logger 41 | 42 | 43 | def main(): 44 | logger = configure_logger() 45 | logger.info("Reading json input from stdin") 46 | query = json.loads(sys.stdin.read()) 47 | 48 | if "required_executables" not in query: 49 | logger.error("Input json is missing required key \"required_executables\".") 50 | sys.exit(1) 51 | 52 | required_executables = query["required_executables"].split(",") 53 | error_message_template = query.get("error_message", _ERROR_MESSAGE_DEFAULT_TEMPLATE) 54 | found = {} 55 | not_found = [] 56 | for executable in required_executables: 57 | # Ignore empty string 58 | if not executable.strip(): 59 | continue 60 | 61 | maybe_executable = shutil.which(executable) 62 | if not maybe_executable: 63 | not_found.append(executable) 64 | else: 65 | logger.info("{} resolved to {}".format(executable, maybe_executable)) 66 | found[executable] = maybe_executable 67 | 68 | if len(not_found) > 0: 69 | logger.error("Not all executables found:\n") 70 | for executable in not_found: 71 | print(error_message_template.replace(_ERROR_MESSAGE_EXECUTABLE_MARKER, executable), file=sys.stderr) 72 | sys.exit(1) 73 | 74 | # Output json to stdout so terraform can read it in 75 | print(json.dumps(found)) 76 | 77 | 78 | if __name__ == "__main__": 79 | main() 80 | -------------------------------------------------------------------------------- /examples/pex/main.tf: -------------------------------------------------------------------------------- 1 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 2 | # RUN PEX BINARY AS DATA SOURCE AND PROVISIONER 3 | # These templates show an example of how to run a pex binary as an external data source or local-exec provisioner in 4 | # terraform in a portable manner that can work with multiple platforms and python versions. 5 | # ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ 6 | 7 | terraform { 8 | required_version = ">= 1.0.0" 9 | } 10 | 11 | # Run the PEX binary as a local-exec provisioner on a null_resource. 12 | module "pex_resource" { 13 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 14 | # to a specific version of the modules, such as the following example: 15 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/run-pex-as-resource?ref=v1.0.8" 16 | source = "../../modules/run-pex-as-resource" 17 | 18 | triggers = var.triggers 19 | 20 | # Path components to each of the PEX binary 21 | python_pex_path_parts = [ 22 | path.module, 23 | "pex-env", 24 | "bin", 25 | "sample_python_script_py3_env.pex", 26 | ] 27 | 28 | # Path components to the folder that holds the python modules for sample_python_script 29 | pex_module_path_parts = [ 30 | path.module, 31 | ] 32 | 33 | # The entrypoint of the sample_python_script, encoded as MODULE:FUNCTION 34 | script_main_function = "sample_python_script.main:main" 35 | 36 | env = { 37 | RUN_PEX_TEST_ENV = var.echo_string 38 | } 39 | 40 | pass_in_previous_triggers = true 41 | previous_trigger_option = "--triggers-json" 42 | 43 | enabled = var.enabled 44 | } 45 | 46 | # Run the PEX binary as a data source. 47 | module "pex_data" { 48 | # When using these modules in your own templates, you will need to use a Git URL with a ref attribute that pins you 49 | # to a specific version of the modules, such as the following example: 50 | # source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/run-pex-as-data-source?ref=v1.0.8" 51 | source = "../../modules/run-pex-as-data-source" 52 | 53 | # Path components to each of the PEX binary 54 | python_pex_path_parts = [ 55 | path.module, 56 | "pex-env", 57 | "bin", 58 | "sample_python_script_py3_env.pex", 59 | ] 60 | 61 | # Path components to the folder that holds the python modules for sample_python_script 62 | pex_module_path_parts = [ 63 | path.module, 64 | ] 65 | 66 | # The entrypoint of the sample_python_script, encoded as MODULE:FUNCTION 67 | script_main_function = "sample_python_script.main:main" 68 | 69 | # Argument to be passed to the entrypoint of sample_python_script 70 | command_args = "--is-data" 71 | 72 | # Query parameter for the data source, that will be passed into the script in json format 73 | command_query = { 74 | "echo" = var.echo_string 75 | } 76 | 77 | enabled = var.enabled 78 | } 79 | -------------------------------------------------------------------------------- /modules/prepare-pex-environment/entrypoint.py: -------------------------------------------------------------------------------- 1 | """ 2 | entrypoint.py is the main entrypoint script to call out to the packaged module script. This script handles platform 3 | adaptation. Specifically, this module provides the following features: 4 | - On Windows environments, handle the args correctly as unicode 5 | 6 | This script is intended to be called as part of a PEX binary. The following packages should be included in the pex: 7 | - click 8 | - six 9 | 10 | Usage: pex_binary entrypoint.py MAIN_MODULE ... 11 | Where MAIN_MODULE is the import path to the main function, encoded as module_path:function_name, with the module_path 12 | encoded in dot notation (e.g package.module_file). 13 | """ 14 | from __future__ import print_function 15 | 16 | import click 17 | import importlib 18 | import os 19 | import platform 20 | import six 21 | import sys 22 | 23 | IS_WIN = platform.system() == "Windows" 24 | 25 | 26 | def __py2_win_argv(): 27 | # `click` includes a fancy argv parser to handle unicode arguments on windows when running python2 (since python2 28 | # does not default to unicode). However, this argv gets the arguments directly from the windows commandline instead 29 | # of via python, so it includes the PEX binary path when taking the PEX based approach. We handle this here by 30 | # detecting if this happened, and chopping it off the args. 31 | unicode_argv = click.get_os_args() 32 | if len(unicode_argv) > 0 and os.path.basename(unicode_argv[0]) == os.path.basename(__file__): 33 | unicode_argv = unicode_argv[1:] 34 | return unicode_argv 35 | 36 | 37 | def get_args(): 38 | if six.PY2 and IS_WIN: 39 | return __py2_win_argv() 40 | else: 41 | return sys.argv[1:] 42 | 43 | 44 | def main_module_func(module_arg): 45 | """ 46 | The main module should be encoded as MAIN_MODULE:ENTRY_FUNCTION. 47 | 48 | Args: 49 | module_arg (string): The first argument of the entrypoint script (as a string) to be validated for correctness. 50 | 51 | Returns: 52 | The entrypoint function or None if the arg does not conform to the expected format. 53 | """ 54 | args_split = module_arg.split(":") 55 | if len(args_split) != 2: 56 | print("MAIN_MODULE arg is not in a valid format", file=sys.stderr) 57 | return None 58 | 59 | module, func = args_split 60 | try: 61 | imported_mod = importlib.import_module(module) 62 | except ImportError: 63 | print("Could not import module {}".format(module), file=sys.stderr) 64 | return None 65 | 66 | imported_func = getattr(imported_mod, func, None) 67 | if imported_func is None: 68 | print("Could not import func {} from module {}".format(func, module), file=sys.stderr) 69 | return imported_func 70 | 71 | 72 | 73 | if __name__ == "__main__": 74 | args = get_args() 75 | if len(args) < 1 or main_module_func(args[0]) is None: 76 | # Missing required argument 77 | print("USAGE: [python] entrypoint.py MAIN_MODULE ...", file=sys.stderr) 78 | sys.exit(1) 79 | 80 | main_module_func(args[0])(args[1:]) 81 | -------------------------------------------------------------------------------- /modules/prepare-pex-environment/determine_python_path.py: -------------------------------------------------------------------------------- 1 | """ 2 | Script to return a platform compatible PYTHONPATH that can be used to resolve imports to a PEX binary, as well as 3 | additional python modules embedded in Terraform modules. 4 | 5 | Note that this should maximize platform portability, meaning that only the stdlib is available. 6 | """ 7 | from __future__ import print_function 8 | import argparse 9 | import json 10 | import os 11 | import platform 12 | import sys 13 | 14 | 15 | IS_WIN = platform.system() == "Windows" 16 | 17 | 18 | DESCRIPTION = """ 19 | Script to add module directory to the PYTHONPATH so that the included packages can be imported by the underlying 20 | command. 21 | 22 | This script takes care of Windows long paths by using the short path name. 23 | 24 | This script is intended to be run as a terraform data source. As such, the output is a json: 25 | 26 | \b 27 | { 28 | "python_path": "PYTHONPATH value" 29 | } 30 | """ 31 | 32 | 33 | def main(): 34 | args = parse_args() 35 | module_abspath = os.path.abspath(args.module_path) 36 | separator = ":" 37 | if IS_WIN: 38 | module_abspath = windows_short_path(module_abspath) 39 | separator = ";" 40 | 41 | python_path = [module_abspath] + sys.path 42 | out = {"python_path": separator.join(python_path)} 43 | # Terraform data source expects a json output to stdout 44 | print(json.dumps(out)) 45 | 46 | 47 | def parse_args(): 48 | """ Prepare the parser for the CLI. """ 49 | parser = argparse.ArgumentParser( 50 | description=DESCRIPTION 51 | ) 52 | parser.add_argument( 53 | "--module-path", 54 | required=True, 55 | help="Path to the module that contains the python package that the PYTHONPATH should be prepared for.", 56 | ) 57 | return parser.parse_args() 58 | 59 | 60 | def windows_short_path(path): 61 | # Windows has a max path length: 62 | # https://docs.microsoft.com/en-us/windows/desktop/FileIO/naming-a-file#maximum-path-length-limitation 63 | # We work around this by using the short path API that windows provides 64 | assert IS_WIN 65 | 66 | # We use the GetShortPathNameW kernel API from windows to get the short path form of a long path so that we can 67 | # access it without hitting the limit. 68 | # See https://stackoverflow.com/a/23598461 69 | import ctypes 70 | from ctypes import wintypes 71 | _GetShortPathNameW = ctypes.windll.kernel32.GetShortPathNameW 72 | _GetShortPathNameW.argtypes = [wintypes.LPCWSTR, wintypes.LPWSTR, wintypes.DWORD] 73 | _GetShortPathNameW.restype = wintypes.DWORD 74 | 75 | output_buf_size = 0 76 | # NOTE: this is a do while loop (exit condition checked at end of loop), because the condition needs to be checked 77 | # after the API call. 78 | while True: 79 | output_buf = ctypes.create_unicode_buffer(output_buf_size) 80 | needed = _GetShortPathNameW(path, output_buf, output_buf_size) 81 | if output_buf_size >= needed: 82 | return output_buf.value 83 | else: 84 | output_buf_size = needed 85 | 86 | 87 | if __name__ == "__main__": 88 | main() 89 | -------------------------------------------------------------------------------- /test/list_remove_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "os" 5 | "path/filepath" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/terratest/modules/logger" 9 | "github.com/gruntwork-io/terratest/modules/terraform" 10 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 11 | ) 12 | 13 | func TestListRemove(t *testing.T) { 14 | t.Parallel() 15 | 16 | testCases := []struct { 17 | name string 18 | inputList []string 19 | itemsToRemove []string 20 | expectedOut string 21 | }{ 22 | // Canonical use case: all elements in itemsToRemove exist in input 23 | { 24 | "canonical", 25 | []string{ 26 | "us-east-1a", 27 | "us-east-1b", 28 | "us-east-1c", 29 | "us-east-1d", 30 | "us-east-1e", 31 | }, 32 | []string{ 33 | "us-east-1b", 34 | "us-east-1c", 35 | }, 36 | "us-east-1a,us-east-1d,us-east-1e", 37 | }, 38 | // Empty lists case 39 | { 40 | "both_empty", 41 | []string{}, 42 | []string{}, 43 | "", 44 | }, 45 | // Empty items to remove case 46 | { 47 | "empty_remove", 48 | []string{ 49 | "us-east-1a", 50 | "us-east-1b", 51 | "us-east-1c", 52 | }, 53 | []string{}, 54 | "us-east-1a,us-east-1b,us-east-1c", 55 | }, 56 | // Empty input case 57 | { 58 | "empty_input", 59 | []string{}, 60 | []string{ 61 | "us-east-1a", 62 | "us-east-1b", 63 | "us-east-1c", 64 | }, 65 | "", 66 | }, 67 | // No items in items to remove actually exist in input 68 | { 69 | "no_overlap", 70 | []string{ 71 | "us-east-1a", 72 | "us-east-1b", 73 | "us-east-1c", 74 | }, 75 | []string{ 76 | "us-east-1d", 77 | "us-east-1e", 78 | }, 79 | "us-east-1a,us-east-1b,us-east-1c", 80 | }, 81 | // Removing duplicates 82 | { 83 | "remove_duplicates", 84 | []string{ 85 | "us-east-1a", 86 | "us-east-1a", 87 | "us-east-1a", 88 | "us-east-1b", 89 | "us-east-1c", 90 | "us-east-1c", 91 | }, 92 | []string{ 93 | "us-east-1a", 94 | "us-east-1c", 95 | }, 96 | "us-east-1b", 97 | }, 98 | // Remove all 99 | { 100 | "remove_all", 101 | []string{ 102 | "us-east-1a", 103 | "us-east-1b", 104 | "us-east-1c", 105 | }, 106 | []string{ 107 | "us-east-1a", 108 | "us-east-1b", 109 | "us-east-1c", 110 | }, 111 | "", 112 | }, 113 | } 114 | 115 | for _, testCase := range testCases { 116 | // Capture range variable to bring it in scope of this block, to avoid it being changed by for loop when test is 117 | // run in parallel. 118 | t.Run(testCase.name, func(t *testing.T) { 119 | t.Parallel() 120 | 121 | testFolder := test_structure.CopyTerraformFolderToTemp(t, "..", "examples") 122 | defer os.RemoveAll(testFolder) 123 | terraformModulePath := filepath.Join(testFolder, "list-remove") 124 | logger.Logf(t, "Test folder is %s", terraformModulePath) 125 | 126 | terratestOptions := createBaseTerratestOptions(t, terraformModulePath) 127 | terratestOptions.Vars = map[string]interface{}{ 128 | "input_list": testCase.inputList, 129 | "items_to_remove": testCase.itemsToRemove, 130 | } 131 | 132 | defer terraform.Destroy(t, terratestOptions) 133 | 134 | terraform.InitAndApply(t, terratestOptions) 135 | 136 | assertOutputEquals(t, "output_list_as_csv", testCase.expectedOut, terratestOptions) 137 | }) 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /test/executable_dependency_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "testing" 7 | 8 | "github.com/gruntwork-io/terratest/modules/terraform" 9 | test_structure "github.com/gruntwork-io/terratest/modules/test-structure" 10 | "github.com/stretchr/testify/assert" 11 | "github.com/stretchr/testify/require" 12 | ) 13 | 14 | // Make sure we can successfully download (if it's not installed already) and execute Kubergrunt 15 | func TestExecutableDependencyKubergrunt(t *testing.T) { 16 | t.Parallel() 17 | 18 | terraformDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples/executable-dependency") 19 | 20 | kubergrunt_version := "v0.5.13" 21 | 22 | terraformOptions := &terraform.Options{ 23 | TerraformDir: terraformDir, 24 | Vars: map[string]interface{}{ 25 | "executable": "kubergrunt", 26 | "download_url": fmt.Sprintf("https://github.com/gruntwork-io/kubergrunt/releases/download/%s/kubergrunt", kubergrunt_version), 27 | "executable_args": "--version", 28 | "append_os_arch": true, 29 | }, 30 | } 31 | 32 | defer terraform.Destroy(t, terraformOptions) 33 | 34 | terraform.Init(t, terraformOptions) 35 | 36 | // Run apply the first time and make sure we get the expected output 37 | terraform.Apply(t, terraformOptions) 38 | version := terraform.OutputRequired(t, terraformOptions, "output") 39 | require.Equal(t, fmt.Sprintf("kubergrunt version %s", kubergrunt_version), version) 40 | 41 | // Check the permissions on the downloaded executable 42 | downloadPath := terraform.OutputRequired(t, terraformOptions, "downloaded_executable") 43 | fileStat, err := os.Stat(downloadPath) 44 | require.NoError(t, err) 45 | expectedFileMode := os.FileMode(0744) 46 | actualFileMode := fileStat.Mode() 47 | assert.Equalf(t, actualFileMode, expectedFileMode, "The modes of the downloaded executable is not correct: Expected %v but is %v", expectedFileMode, actualFileMode) 48 | 49 | // Run apply once again to make sure the download code doesn't have issues with re-runs 50 | terraform.Apply(t, terraformOptions) 51 | version = terraform.OutputRequired(t, terraformOptions, "output") 52 | require.Equal(t, fmt.Sprintf("kubergrunt version %s", kubergrunt_version), version) 53 | 54 | } 55 | 56 | // Make sure we can successfully use an existing executable. In this case, we use Go, as you must have Go installed 57 | // already if you're running these tests! 58 | func TestExecutableDependencyGoLang(t *testing.T) { 59 | t.Parallel() 60 | 61 | terraformDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples/executable-dependency") 62 | 63 | terraformOptions := &terraform.Options{ 64 | TerraformDir: terraformDir, 65 | Vars: map[string]interface{}{ 66 | "executable": "go", 67 | "download_url": "this is an intentionally fake URL as we expect Go to already be installed, so the code should NOT try to download anything", 68 | "executable_args": "version", 69 | "append_os_arch": false, 70 | }, 71 | } 72 | 73 | defer terraform.Destroy(t, terraformOptions) 74 | 75 | terraform.InitAndApply(t, terraformOptions) 76 | 77 | version := terraform.OutputRequired(t, terraformOptions, "output") 78 | require.Contains(t, version, "go version") 79 | } 80 | 81 | // Make sure we can disable the module without errors 82 | func TestExecutableDependencyDisabled(t *testing.T) { 83 | t.Parallel() 84 | 85 | terraformDir := test_structure.CopyTerraformFolderToTemp(t, "../", "examples/executable-dependency") 86 | 87 | expectedOutput := "foo" 88 | 89 | terraformOptions := &terraform.Options{ 90 | TerraformDir: terraformDir, 91 | Vars: map[string]interface{}{ 92 | "executable": "echo", 93 | "download_url": "this is an intentionally fake URL as with the module disabled, the code should NOT try to download anything", 94 | "executable_args": expectedOutput, 95 | "append_os_arch": false, 96 | "enabled": false, 97 | }, 98 | } 99 | 100 | defer terraform.Destroy(t, terraformOptions) 101 | 102 | terraform.InitAndApply(t, terraformOptions) 103 | 104 | output := terraform.OutputRequired(t, terraformOptions, "output") 105 | require.Equal(t, expectedOutput, output) 106 | } 107 | -------------------------------------------------------------------------------- /modules/executable-dependency/download-dependency-if-necessary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # A script to check if an executable is installed, and if not, download it. In either case, returns the path to the 3 | # executable. This script is meant to be executed from a Terraform external data source. To be as portable as possible, 4 | # this script has no external dependencies (i.e., nothing that doesn't come with Python). 5 | 6 | 7 | import sys 8 | import json 9 | import tempfile 10 | import os 11 | import os.path 12 | import hashlib 13 | import platform 14 | import logging 15 | import errno 16 | import argparse 17 | import shutil 18 | 19 | from urllib.request import urlretrieve 20 | 21 | 22 | DEFAULT_INSTALL_DIR_NAME = 'download-dependency-if-necessary' 23 | 24 | 25 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') 26 | log = logging.getLogger(__name__) 27 | 28 | 29 | def main(): 30 | parser = argparse.ArgumentParser(description='This script, which is meant to be executed from a Terraform ' 31 | 'external data source, checks if an executable is installed in the ' 32 | 'system PATH or the given install dir, and if not, downloads the ' 33 | 'executable from the given download URL, and prints the executable ' 34 | 'path to stdout in JSON format.') 35 | parser.add_argument('--executable', help='The executable', required=True) 36 | parser.add_argument('--download-url', help='The URL to download the executable from', required=True) 37 | parser.add_argument('--install-dir', help='Download the executable into this folder') 38 | parser.add_argument('--append-os-arch', help='Append the OS and architecture to the download URL (e.g., linux_amd_64)', action='store_true') 39 | 40 | args = parser.parse_args() 41 | 42 | if not args.install_dir: 43 | args.install_dir = default_install_dir(args.download_url) 44 | executable_install_dir_path = os.path.join(args.install_dir, args.executable) 45 | 46 | # First, check if the executable is on the system PATH 47 | executable_path = shutil.which(args.executable) 48 | 49 | # If it's not on the system PATH, check if it's in the install dir passed in by the user 50 | if not executable_path and os.path.isfile(executable_install_dir_path): 51 | executable_path = executable_install_dir_path 52 | 53 | # If it's not there either, download the executable to the install dir 54 | if not executable_path: 55 | executable_path = download_executable(args.executable, args.download_url, args.install_dir, args.append_os_arch) 56 | 57 | # Print the executable path to stdout as JSON so the Terraform external data source can read it in 58 | result = {'path': executable_path} 59 | print(json.dumps(result)) 60 | 61 | 62 | def default_install_dir(download_url): 63 | # Use a URL hash so that if the URL changes (e.g., because the version of the executable changed), we get a 64 | # different local download folder, but if the URL stayed the same, we get the same folder, and don't have to 65 | # re-download it. 66 | url_hash = hashlib.md5(download_url.encode('utf-8')).hexdigest() 67 | return os.path.join(tempfile.gettempdir(), DEFAULT_INSTALL_DIR_NAME, url_hash) 68 | 69 | 70 | def download_executable(executable, download_url, install_dir, append_os_arch): 71 | if append_os_arch: 72 | # Use the old string formatting style so that it works with Python 2 73 | download_url = '{}_{}_{}'.format(download_url, get_os(), get_arch()) 74 | 75 | executable_path = os.path.join(install_dir, executable) 76 | 77 | log.info('Downloading from {} to {}'.format(download_url, executable_path)) 78 | 79 | # Make sure all the parent folders exist 80 | os.makedirs(install_dir) 81 | 82 | # Download the executable 83 | urlretrieve(download_url, executable_path) 84 | 85 | # Give the current user execute permissions 86 | os.chmod(executable_path, 0o744) 87 | 88 | return executable_path 89 | 90 | 91 | def get_os(): 92 | return platform.system().lower() 93 | 94 | 95 | def get_arch(): 96 | arch = platform.machine().lower() 97 | 98 | # Use the same architecture format as gox / go build, as that's what most Gruntwork binaries are built with 99 | if '64' in arch: 100 | return 'amd64' 101 | if '386' in arch: 102 | return '386' 103 | if 'arm' in arch: 104 | return 'arm' 105 | 106 | return arch 107 | 108 | 109 | if __name__ == '__main__': 110 | main() 111 | -------------------------------------------------------------------------------- /modules/request-quota-increase/README.md: -------------------------------------------------------------------------------- 1 | # Request AWS Quota Increase 2 | 3 | This module can be used to request a quota increase for AWS Resources. The module is [generated](../../codegen/quotas/) using [AWS Service Quotas API](https://docs.aws.amazon.com/servicequotas/2019-06-24/apireference/Welcome.html), and inputs for each adjustable quota for different services are added to the module. 4 | 5 | **NOTE:** The service quotas for certain services have duplicate items. Those duplicate quotas have been named differently in the [input variables](./variables.tf) by appending the service quota code at the end of the variable name, e.g. `networkmonitor_number_of_probes_per_monitor` and `networkmonitor_number_of_probes_per_monitor_l_f192a8d6`. 6 | 7 | ## Features 8 | 9 | - Request a quota increase for any AWS resource. 10 | 11 | ## Learn 12 | 13 | ### Core Concepts 14 | 15 | - [AWS Service Quotas Documentation](https://docs.aws.amazon.com/servicequotas/?id=docs_gateway) 16 | - [AWS Service Quotas Generator](../../codegen/quotas/) 17 | 18 | 19 | ### Example code 20 | 21 | See the [request-quota-increase example](/examples/request-quota-increase) for working sample code. 22 | 23 | 24 | 25 | ## Usage 26 | 27 | Use the module in your Terraform code, replacing `` with the latest version from the [releases 28 | page](https://github.com/gruntwork-io/terraform-aws-utilities/releases): 29 | 30 | ```hcl 31 | module "quota_increase" { 32 | source = "git::git@github.com:gruntwork-io/terraform-aws-utilities.git//modules/quota-increase?ref=" 33 | 34 | vpc_rules_per_network_acl = 30 35 | vpc_nat_gateways_per_availability_zone = 30 36 | } 37 | ``` 38 | 39 | The [input variables](../../modules/request-quota-increase/variables.tf) for the module have been automatically generated using the [AWS Service Quotas Generator](../../codegen/quotas/). All adjustable Service Quotas are as separate input variables. 40 | 41 | When you run `apply`, the `new_quotas` output variable will confirm to you that a quota request has been made! 42 | 43 | ```hcl 44 | new_quotas = { 45 | "vpc_nat_gateways_per_availability_zone" = { 46 | "adjustable" = true 47 | "arn" = "arn:aws:servicequotas:us-east-1::vpc/L-FE5A380F" 48 | "default_value" = 5 49 | "id" = "vpc/L-FE5A380F" 50 | "quota_code" = "L-FE5A380F" 51 | "quota_name" = "NAT gateways per Availability Zone" 52 | "request_id" = "" 53 | "request_status" = "PENDING" 54 | "service_code" = "vpc" 55 | "service_name" = "Amazon Virtual Private Cloud (Amazon VPC)" 56 | "value" = 30 57 | } 58 | } 59 | ``` 60 | 61 | ## Manage 62 | 63 | You can see a full history of quota request changes using the [AWS 64 | Console](https://console.aws.amazon.com/servicequotas/home#!/requests) or the AWS CLI: 65 | 66 | 67 | ``` 68 | aws service-quotas list-requested-service-quota-change-history --region 69 | ``` 70 | 71 | ### Finding out the Service Code and Quota Code 72 | 73 | You can check adjustable quotas in the [input variables](../../modules/request-quota-increase/variables.tf). 74 | 75 | 76 | Alternatively, you can check the available services with 77 | 78 | ``` 79 | aws service-quotas list-services --region --output table 80 | ``` 81 | 82 | And use the `ServiceCode` from the output to get the code for the resources 83 | 84 | ``` 85 | aws service-quotas list-service-quotas --service-code 86 | ``` 87 | 88 | 89 | ### Request a new quota smaller than the current one 90 | 91 | If the new value that you request is smaller than the current one, _nothing_ will happen. The 92 | `terraform apply` output will contain the current quota. For example, if the NAT Gateway current 93 | quota is 30 and you ask for a new quota of 25, this is the output: 94 | 95 | ```hcl 96 | new_quotas = { 97 | "vpc_nat_gateways_per_availability_zone" = { 98 | "adjustable" = true 99 | "arn" = "arn:aws:servicequotas:us-east-1::vpc/L-FE5A380F" 100 | "default_value" = 5 101 | "id" = "vpc/L-FE5A380F" 102 | "quota_code" = "L-FE5A380F" 103 | "quota_name" = "NAT gateways per Availability Zone" 104 | "service_code" = "vpc" 105 | "service_name" = "Amazon Virtual Private Cloud (Amazon VPC)" 106 | "value" = 30 <------ Returned the current quota, not the requested one. 107 | } 108 | } 109 | ``` 110 | 111 | 112 | ### What happens when you run `destroy` 113 | 114 | 115 | When you run `terraform destroy` on this module, it does not affect your current quotas or your 116 | existing quota requests. In other words, you don't have to worry about quotas being reset to old 117 | values; once they have been increased, they stay that way! 118 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guidelines 2 | 3 | Contributions to this Module are very welcome! We follow a fairly standard [pull request 4 | process](https://help.github.com/articles/about-pull-requests/) for contributions, subject to the following guidelines: 5 | 6 | 1. [File a GitHub issue](#file-a-github-issue) 7 | 1. [Update the documentation](#update-the-documentation) 8 | 1. [Update the tests](#update-the-tests) 9 | 1. [Update the code](#update-the-code) 10 | 1. [Create a pull request](#create-a-pull-request) 11 | 1. [Merge and release](#merge-and-release) 12 | 13 | ## File a GitHub issue 14 | 15 | Before starting any work, we recommend filing a GitHub issue in this repo. This is your chance to ask questions and 16 | get feedback from the maintainers and the community before you sink a lot of time into writing (possibly the wrong) 17 | code. If there is anything you're unsure about, just ask! 18 | 19 | ## Update the documentation 20 | 21 | We recommend updating the documentation *before* updating any code (see [Readme Driven 22 | Development](http://tom.preston-werner.com/2010/08/23/readme-driven-development.html)). This ensures the documentation 23 | stays up to date and allows you to think through the problem at a high level before you get lost in the weeds of 24 | coding. 25 | 26 | ## Update the tests 27 | 28 | We also recommend updating the automated tests *before* updating any code (see [Test Driven 29 | Development](https://en.wikipedia.org/wiki/Test-driven_development)). That means you add or update a test case, 30 | verify that it's failing with a clear error message, and *then* make the code changes to get that test to pass. This 31 | ensures the tests stay up to date and verify all the functionality in this Module, including whatever new 32 | functionality you're adding in your contribution. Check out the [tests](/test) folder for instructions on running the 33 | automated tests. 34 | 35 | ## Update the code 36 | 37 | At this point, make your code changes and use your new test case to verify that everything is working. As you work, 38 | keep in mind two things: 39 | 40 | 1. Backwards compatibility 41 | 1. Downtime 42 | 43 | ### Backwards compatibility 44 | 45 | Please make every effort to avoid unnecessary backwards incompatible changes. With Terraform code, this means: 46 | 47 | 1. Do not delete, rename, or change the type of input variables. 48 | 1. If you add an input variable, it should have a `default`. 49 | 1. Do not delete, rename, or change the type of output variables. 50 | 1. Do not delete or rename a module in the `modules` folder. 51 | 52 | If a backwards incompatible change cannot be avoided, please make sure to call that out when you submit a pull request, 53 | explaining why the change is absolutely necessary. 54 | 55 | ### Downtime 56 | 57 | Bear in mind that the Terraform code in this Module is used by real companies to run real infrastructure in 58 | production, and certain types of changes could cause downtime. For example, consider the following: 59 | 60 | 1. If you rename a resource (e.g. `aws_instance "foo"` -> `aws_instance "bar"`), Terraform will see that as deleting 61 | the old resource and creating a new one. 62 | 1. If you change certain attributes of a resource (e.g. the `name` of an `aws_elb`), the cloud provider (e.g. AWS) may 63 | treat that as an instruction to delete the old resource and a create a new one. 64 | 65 | Deleting certain types of resources (e.g. virtual servers, load balancers) can cause downtime, so when making code 66 | changes, think carefully about how to avoid that. For example, can you avoid downtime by using 67 | [create_before_destroy](https://www.terraform.io/docs/configuration/resources.html#create_before_destroy)? Or via 68 | the `terraform state` command? If so, make sure to note this in our pull request. If downtime cannot be avoided, 69 | please make sure to call that out when you submit a pull request. 70 | 71 | ## Create a pull request 72 | 73 | [Create a pull request](https://help.github.com/articles/creating-a-pull-request/) with your changes. Please make sure 74 | to include the following: 75 | 76 | 1. A description of the change, including a link to your GitHub issue. 77 | 1. The output of your automated test run, preferably in a [GitHub Gist](https://gist.github.com/). We cannot run 78 | automated tests for pull requests automatically due to [security 79 | concerns](https://circleci.com/docs/fork-pr-builds/#security-implications), so we need you to manually provide this 80 | test output so we can verify that everything is working. 81 | 1. Any notes on backwards incompatibility or downtime. 82 | 83 | ## Merge and release 84 | 85 | The maintainers for this repo will review your code and provide feedback. If everything looks good, they will merge the 86 | code and release a new version, which you'll be able to find in the [releases page](../../releases). 87 | -------------------------------------------------------------------------------- /modules/prepare-pex-environment/README.md: -------------------------------------------------------------------------------- 1 | # Prepare PEX Environment Module 2 | 3 | **NOTE**: This module should not be used directly. Use [run-pex-as-data-source](../run-pex-as-data-source) or 4 | [run-pex-as-resource](../run-pex-as-resource) instead. 5 | 6 | This module can be used to prepare an runtime environment that can call out to a PEX binary. Specifically, this module: 7 | 8 | - Selects the right PEX binary to use based on the installed python version. 9 | - Sets up a platform portable PYTHONPATH environment variable for the PEX, as well as the module. 10 | - Provides an entrypoint script that will parse args in a platform portable manner. 11 | 12 | This module uses Python under the hood so, the Python must be installed on the OS. 13 | 14 | 15 | ## What is PEX? 16 | 17 | PEX (or Python EXecutable) is an executable python environment in the spirit of [virtualenvs](virtualenv.org). It is 18 | generated using the [pex](https://github.com/pantsbuild/pex) library, and is an executable zip file containing: 19 | 20 | - An bootstrap script in python that unpacks the requirements and includes them in the `PYTHONPATH` (`sys.path`). 21 | - Dependencies packaged as wheels for each platform and python version the executable is intended to support. 22 | - (Optionally) An entrypoint script to run in the context of the unpacked environment. 23 | 24 | This provides a convenient way to package python dependencies in a portable manner, allowing execution of the script 25 | without the end user having to install all the necessary dependencies. 26 | 27 | 28 | ### How to construct a PEX binary 29 | 30 | The PEX binary is generated using the [pex tool](https://github.com/pantsbuild/pex). To package your script as a PEX for 31 | compatibility with this module, you need the following: 32 | 33 | - Python requirements defined as a `requirements.txt` file using [the Pip requirements file 34 | syntax](https://pip.pypa.io/en/stable/reference/pip_install/?highlight=requirements%20file#requirements-file-format). 35 | - An entrypoint script that should be run in the context of the environment. This should provide the actual logic 36 | you wish to provide in terraform (a terraform local-exec provisioner or data source script) 37 | 38 | Then, you call out to `pex` and provide the platform and python versions you wish to support: 39 | 40 | ``` 41 | pex --python-shebang='/usr/bin/env python' \ 42 | -r requirements.txt \ 43 | --python=python2.7 \ 44 | --platform macosx_10.12-x86_64 \ 45 | --platform macosx_10.13-x86_64 \ 46 | --platform macosx_10.14-x86_64 \ 47 | --platform linux-x86_64 \ 48 | --platform linux-x86_64-cp-27-mu \ 49 | --platform win32 \ 50 | --platform win_amd64 \ 51 | --disable-cache \ 52 | -o ../bin/sample_python_script_py27_env.pex 53 | ``` 54 | 55 | This will search [`pypi`](https://pypi.org/) for the python packages defined in `requirements.txt` that support the 56 | specified platform and python versions, download the wheel/package, inject the bootstrap script and produce an 57 | executable zip file. 58 | 59 | See the [`pex/sample-python-script` example](/examples/pex/sample-python-script) for an example implementation that you 60 | can use as a template. 61 | 62 | 63 | ### Known limitations of PEX 64 | 65 | - For compiled dependencies, PEX relies on pre-built wheel packages to avoid cross compilation. What this means is that 66 | the chosen `pypi` needs to hold a wheel for each compatible platform. For example, compare the available package files 67 | for [`ruamel.yaml`](https://pypi.org/project/ruamel.yaml/#files) with 68 | [pyyaml](https://pypi.org/project/PyYAML/#files). In `ruamel.yaml`, there is a wheel for each permutation of major 69 | platform and major python versions supported. However, for `pyyaml`, there is only wheels provided for Windows. 70 | This means that you need to setup a cross compiler for each compatible platform you wish to support. 71 | - To avoid this, prefer packages that only rely on python code, or that have prebuilt wheels for the platforms you 72 | wish to support. 73 | 74 | - When building a PEX for both python2 with python3, be aware that python2 packages tend to rely on backports of stdlib 75 | enhancements (e.g the [`futures`](https://pypi.org/project/futures/) package). These backports cannot be installed in 76 | a python3 environment, which creates complications in the PEX binary. Specifically, it will attempt to lookup a 77 | dependency that doesn't exist in the packaged zip. 78 | - You can mitigate this using [environment markers](https://www.python.org/dev/peps/pep-0508/#environment-markers). 79 | However, for the most robust solution, generate separate PEX binaries for python2 and python3. 80 | 81 | - The PEX binary is directly executable in Unix environments (Linux or Mac OS X). However, for all environments, it can 82 | be run using python (e.g `python my.pex`). 83 | 84 | - There is a known limitation in Windows environments with python 2 that prevents the usage of pex in directories with a 85 | long path name. The pex pathing for imports does not seem to support long path names on Windows, hitting the 255 86 | character limitation of path names. 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Maintained by Gruntwork.io](https://img.shields.io/badge/maintained%20by-gruntwork.io-%235849a6.svg)](https://gruntwork.io/?ref=repo_package_terraform_utilities) 2 | [![GitHub tag (latest SemVer)](https://img.shields.io/github/tag/gruntwork-io/terraform-aws-utilities.svg?label=latest)](https://github.com/gruntwork-io/terraform-aws-utilities/releases/latest) 3 | ![Terraform Version](https://img.shields.io/badge/tf-%3E%3D1.1.0-blue.svg) 4 | 5 | # Terraform Utility Modules 6 | 7 | This repo contains miscellaneous utility and helper modules for use with Terraform. 8 | 9 | ## What is in this repo 10 | 11 | This repo provides a Gruntwork IaC Package and has the following folder structure: 12 | 13 | * [modules](/modules): This folder contains the main implementation code for this repository, broken down into multiple 14 | standalone modules. 15 | * [examples](/examples): This folder contains examples of how to use the modules. 16 | * [test](/test): Automated tests for the modules and examples. 17 | 18 | The following modules are available: 19 | 20 | * [join-path](/modules/join-path): This module can be used to join a list of given path parts into a single path that is 21 | platform/operating system aware. **(This module requires Python)** 22 | * [operating-system](/modules/operating-system): This module can be used to figure out what operating system is being 23 | used to run Terraform. **(This module requires Python)** 24 | * [require-executable](/modules/require-executable): This is a module that can be used to ensure particular executables 25 | is available in the `PATH`. **(This module requires Python)** 26 | * [run-pex-as-data-source](/modules/run-pex-as-data-source): This module prepares a portable environment for running PEX 27 | files and runs them as an external data source. PEX files are python executables that contain all the requirements 28 | necessary to run the script. **(This module requires Python)** 29 | * [run-pex-as-resource](/modules/run-pex-as-resource): This module prepares a portable environment for running PEX files 30 | and runs them as an local-exec provisioner on a null_resource. PEX files are python executables that contain all the 31 | requirements necessary to run the script. **(This module requires Python)** 32 | 33 | The following modules were deprecated and removed: 34 | 35 | * `intermediate-variable`: This module has been superseded by [terraform local 36 | values](https://www.terraform.io/docs/configuration/locals.html). To upgrade, switch usage of `intermediate-variable` 37 | with `locals`. 38 | * `enabled-aws-regions`: This module has been superseded by [terraform aws_regions data 39 | source](https://www.terraform.io/docs/providers/aws/d/regions.html). To upgrade, switch the module block with: 40 | 41 | data "aws_regions" "enabled_regions" {} 42 | 43 | Then, you can get the list of enabled regions using `data.aws_regions.enabled_regions.names`. 44 | 45 | 46 | Click on each module above to see its documentation. Head over to the [examples](/examples) folder for example usage. 47 | 48 | 49 | 50 | 51 | ## What is a module? 52 | 53 | A Module is a canonical, reusable, best-practices definition for how to run a single piece of infrastructure, such as a 54 | database or server cluster. Each Module is written using a combination of Terraform and scripts (mostly bash) and 55 | include automated tests, documentation, and examples. It is maintained both by the open source community and companies 56 | that provide commercial support. 57 | 58 | Instead of figuring out the details of how to run a piece of infrastructure from scratch, you can reuse existing code 59 | that has been proven in production. And instead of maintaining all that infrastructure code yourself, you can leverage 60 | the work of the Module community to pick up infrastructure improvements through a version number bump. 61 | 62 | 63 | 64 | ## Who maintains this Module? 65 | 66 | This Module is maintained by [Gruntwork](http://www.gruntwork.io/). If you're looking for help or commercial 67 | support, send an email to [modules@gruntwork.io](mailto:modules@gruntwork.io?Subject=Terraform%20Utilities%20Module). 68 | Gruntwork can help with: 69 | 70 | * Setup, customization, and support for this Module. 71 | * Modules for other types of infrastructure, such as VPCs, Docker clusters, databases, and continuous integration. 72 | * Modules that meet compliance requirements, such as HIPAA. 73 | * Consulting & Training on AWS, Terraform, and DevOps. 74 | 75 | 76 | 77 | 78 | ## How is this Module versioned? 79 | 80 | This Module follows the principles of [Semantic Versioning](http://semver.org/). You can find each new release, 81 | along with the changelog, in the [Releases Page](../../releases). 82 | 83 | During initial development, the major version will be 0 (e.g., `0.x.y`), which indicates the code does not yet have a 84 | stable API. Once we hit `1.0.0`, we will make every effort to maintain a backwards compatible API and use the MAJOR, 85 | MINOR, and PATCH versions on each release to indicate any incompatibilities. 86 | 87 | 88 | 89 | 90 | 91 | ## License 92 | 93 | Please see [LICENSE.txt](/LICENSE.txt) and [NOTICE](/NOTICE) for details on how the code in this repo is licensed. 94 | -------------------------------------------------------------------------------- /codegen/quotas/generate_quotas.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | import os 3 | import subprocess 4 | import time 5 | 6 | import boto3 7 | from templates import ( 8 | get_variable_name, 9 | terraform_locals_template, 10 | terraform_main, 11 | terraform_variable_template, 12 | terraform_vars, 13 | ) 14 | 15 | # Parse command-line arguments 16 | parser = argparse.ArgumentParser( 17 | description="Generate a markdown document of all adjustable AWS service quotas." 18 | ) 19 | parser.add_argument( 20 | "--region", 21 | default="us-east-1", 22 | help="AWS region to query service quotas for. Defaults to us-east-1.", 23 | ) 24 | parser.add_argument( 25 | "--outdir", 26 | default="../../modules/request-quota-increase", 27 | help='Output directory for the resulting terraform files. Defaults to "../../modules/request-quota-increase".', 28 | ) 29 | args = parser.parse_args() 30 | 31 | # Initialize a boto3 client for Service Quotas in the specified region 32 | client = boto3.client("service-quotas", region_name=args.region) 33 | 34 | 35 | def list_all_services(): 36 | """List all AWS services that have quotas.""" 37 | services = [] 38 | response = client.list_services() 39 | services.extend(response["Services"]) 40 | while "NextToken" in response: 41 | time.sleep(0.3) # Delay to respect rate limits 42 | response = client.list_services(NextToken=response["NextToken"]) 43 | services.extend(response["Services"]) 44 | return services 45 | 46 | 47 | def list_quotas_for_service(service_code): 48 | """List the quotas for a given service by its service code.""" 49 | print(f"Fetching quotas for service {service_code}") 50 | quotas = [] 51 | response = client.list_aws_default_service_quotas(ServiceCode=service_code) 52 | quotas.extend(response["Quotas"]) 53 | while "NextToken" in response: 54 | time.sleep(0.3) # Delay to respect rate limits 55 | response = client.list_aws_default_service_quotas( 56 | ServiceCode=service_code, NextToken=response["NextToken"] 57 | ) 58 | quotas.extend(response["Quotas"]) 59 | return quotas 60 | 61 | 62 | def generate_terraform(services): 63 | """ 64 | Generate Terraform code for the given AWS services. 65 | 66 | This function iterates over the provided services, fetches the quotas for each service, 67 | and generates Terraform code for each adjustable quota. If a quota with the same variable name 68 | already exists, it appends the quota code to the quota name to make it unique, and stores the 69 | duplicate variable in a separate list. 70 | 71 | Parameters: 72 | services (list): A list of AWS services. Each service is a dictionary that contains the service details. 73 | 74 | Returns: 75 | tuple: A tuple containing two strings. The first string is the Terraform code for the main.tf file, 76 | and the second string is the Terraform code for the variables.tf file. 77 | 78 | Prints: 79 | For each duplicate variable, it prints a message in the format "Duplicate Variable: {variable_name}: {quota_code}". 80 | """ 81 | terraform_variables = "" 82 | terraform_maps = "" 83 | unique_variables = set() 84 | duplicate_variables = [] 85 | for service in services: 86 | # Adjust this based on your rate limit analysis and AWS documentation 87 | time.sleep(0.3) 88 | quotas = list_quotas_for_service(service["ServiceCode"]) 89 | for quota in quotas: 90 | if quota["Adjustable"]: 91 | variable_name = get_variable_name( 92 | service["ServiceCode"], quota["QuotaName"] 93 | ) 94 | if variable_name in unique_variables: 95 | duplicate_variables.append(f"{variable_name}: {quota['QuotaCode']}") 96 | quota["QuotaName"] = f"{quota['QuotaName']}_{quota['QuotaCode']}" 97 | else: 98 | unique_variables.add(variable_name) 99 | terraform_variables += terraform_variable_template( 100 | service["ServiceCode"], quota["QuotaName"], quota["QuotaCode"] 101 | ) 102 | terraform_maps += terraform_locals_template( 103 | service["ServiceCode"], quota["QuotaName"], quota["QuotaCode"] 104 | ) 105 | main_tf = terraform_main(terraform_maps) 106 | vars_tf = terraform_vars(terraform_variables) 107 | for variable in duplicate_variables: 108 | print(f"Duplicate Variable: {variable}") 109 | 110 | return main_tf, vars_tf 111 | 112 | 113 | # Fetch all services 114 | services = list_all_services() 115 | 116 | # Generate the Terraform code 117 | tf_main, tf_vars = generate_terraform(services) 118 | 119 | # Ensure the output directory exists 120 | output_dir = args.outdir 121 | if not os.path.exists(output_dir): 122 | os.makedirs(output_dir) 123 | 124 | # Write the main.tf to the specified output directory 125 | main_tf_path = os.path.join(output_dir, "main.tf") 126 | with open(main_tf_path, "w") as file: 127 | file.write(tf_main) 128 | 129 | # Write the variables.tf to the specified output directory 130 | variables_tf_path = os.path.join(output_dir, "variables.tf") 131 | with open(variables_tf_path, "w") as file: 132 | file.write(tf_vars) 133 | 134 | # Run terraform fmt on both files 135 | subprocess.run(["terraform", "fmt", main_tf_path], check=True) 136 | subprocess.run(["terraform", "fmt", variables_tf_path], check=True) 137 | 138 | # Print the success message 139 | print( 140 | f"Terraform files have been written to {output_dir} and formatted with terraform fmt" 141 | ) 142 | -------------------------------------------------------------------------------- /test/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/gruntwork.io/package-terraform-utilities/test 2 | 3 | go 1.21.1 4 | 5 | toolchain go1.21.3 6 | 7 | require ( 8 | github.com/gruntwork-io/terraform-aws-ci/test/upgrade-tester v0.0.0-20240417193241-8367ff1d958d 9 | github.com/gruntwork-io/terratest v0.44.1 10 | github.com/stretchr/testify v1.8.4 11 | ) 12 | 13 | require ( 14 | cloud.google.com/go v0.112.1 // indirect 15 | cloud.google.com/go/compute v1.24.0 // indirect 16 | cloud.google.com/go/compute/metadata v0.2.3 // indirect 17 | cloud.google.com/go/iam v1.1.7 // indirect 18 | cloud.google.com/go/storage v1.40.0 // indirect 19 | github.com/agext/levenshtein v1.2.3 // indirect 20 | github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect 21 | github.com/aws/aws-sdk-go v1.44.122 // indirect 22 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect 23 | github.com/bgentry/speakeasy v0.1.0 // indirect 24 | github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc // indirect 25 | github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect 26 | github.com/davecgh/go-spew v1.1.1 // indirect 27 | github.com/emicklei/go-restful/v3 v3.9.0 // indirect 28 | github.com/fatih/color v1.9.0 // indirect 29 | github.com/felixge/httpsnoop v1.0.4 // indirect 30 | github.com/go-errors/errors v1.0.2-0.20180813162953-d98b870cc4e0 // indirect 31 | github.com/go-logr/logr v1.4.1 // indirect 32 | github.com/go-logr/stdr v1.2.2 // indirect 33 | github.com/go-openapi/jsonpointer v0.19.6 // indirect 34 | github.com/go-openapi/jsonreference v0.20.1 // indirect 35 | github.com/go-openapi/swag v0.22.3 // indirect 36 | github.com/go-sql-driver/mysql v1.5.0 // indirect 37 | github.com/gogo/protobuf v1.3.2 // indirect 38 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 39 | github.com/golang/protobuf v1.5.4 // indirect 40 | github.com/google/gnostic v0.5.7-v3refs // indirect 41 | github.com/google/go-cmp v0.6.0 // indirect 42 | github.com/google/gofuzz v1.1.0 // indirect 43 | github.com/google/s2a-go v0.1.7 // indirect 44 | github.com/google/uuid v1.6.0 // indirect 45 | github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect 46 | github.com/googleapis/gax-go/v2 v2.12.3 // indirect 47 | github.com/gruntwork-io/go-commons v0.12.4 // indirect 48 | github.com/hashicorp/errwrap v1.1.0 // indirect 49 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 50 | github.com/hashicorp/go-getter v1.7.6 // indirect 51 | github.com/hashicorp/go-multierror v1.1.0 // indirect 52 | github.com/hashicorp/go-safetemp v1.0.0 // indirect 53 | github.com/hashicorp/go-version v1.6.0 // indirect 54 | github.com/hashicorp/hcl/v2 v2.9.1 // indirect 55 | github.com/hashicorp/terraform-json v0.13.0 // indirect 56 | github.com/imdario/mergo v0.3.11 // indirect 57 | github.com/jinzhu/copier v0.1.0 // indirect 58 | github.com/jmespath/go-jmespath v0.4.0 // indirect 59 | github.com/josharian/intern v1.0.0 // indirect 60 | github.com/json-iterator/go v1.1.12 // indirect 61 | github.com/klauspost/compress v1.15.11 // indirect 62 | github.com/mailru/easyjson v0.7.7 // indirect 63 | github.com/mattn/go-colorable v0.1.7 // indirect 64 | github.com/mattn/go-isatty v0.0.12 // indirect 65 | github.com/mattn/go-zglob v0.0.2-0.20190814121620-e3c945676326 // indirect 66 | github.com/mitchellh/go-homedir v1.1.0 // indirect 67 | github.com/mitchellh/go-testing-interface v1.14.1 // indirect 68 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 69 | github.com/moby/spdystream v0.2.0 // indirect 70 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 71 | github.com/modern-go/reflect2 v1.0.2 // indirect 72 | github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect 73 | github.com/pmezard/go-difflib v1.0.0 // indirect 74 | github.com/pquerna/otp v1.2.1-0.20191009055518-468c2dd2b58d // indirect 75 | github.com/russross/blackfriday/v2 v2.1.0 // indirect 76 | github.com/sirupsen/logrus v1.8.1 // indirect 77 | github.com/spf13/pflag v1.0.5 // indirect 78 | github.com/tmccombs/hcl2json v0.3.3 // indirect 79 | github.com/ulikunitz/xz v0.5.10 // indirect 80 | github.com/urfave/cli/v2 v2.3.0 // indirect 81 | github.com/zclconf/go-cty v1.9.1 // indirect 82 | go.opencensus.io v0.24.0 // indirect 83 | go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.49.0 // indirect 84 | go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect 85 | go.opentelemetry.io/otel v1.24.0 // indirect 86 | go.opentelemetry.io/otel/metric v1.24.0 // indirect 87 | go.opentelemetry.io/otel/trace v1.24.0 // indirect 88 | golang.org/x/crypto v0.32.0 // indirect 89 | golang.org/x/net v0.34.0 // indirect 90 | golang.org/x/oauth2 v0.18.0 // indirect 91 | golang.org/x/sync v0.10.0 // indirect 92 | golang.org/x/sys v0.29.0 // indirect 93 | golang.org/x/term v0.28.0 // indirect 94 | golang.org/x/text v0.21.0 // indirect 95 | golang.org/x/time v0.5.0 // indirect 96 | google.golang.org/api v0.170.0 // indirect 97 | google.golang.org/appengine v1.6.8 // indirect 98 | google.golang.org/genproto v0.0.0-20240213162025-012b6fc9bca9 // indirect 99 | google.golang.org/genproto/googleapis/api v0.0.0-20240314234333-6e1732d8331c // indirect 100 | google.golang.org/genproto/googleapis/rpc v0.0.0-20240311132316-a219d84964c2 // indirect 101 | google.golang.org/grpc v1.62.1 // indirect 102 | google.golang.org/protobuf v1.33.0 // indirect 103 | gopkg.in/inf.v0 v0.9.1 // indirect 104 | gopkg.in/yaml.v2 v2.4.0 // indirect 105 | gopkg.in/yaml.v3 v3.0.1 // indirect 106 | k8s.io/api v0.27.2 // indirect 107 | k8s.io/apimachinery v0.27.2 // indirect 108 | k8s.io/client-go v0.27.2 // indirect 109 | k8s.io/klog/v2 v2.90.1 // indirect 110 | k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect 111 | k8s.io/utils v0.0.0-20230209194617-a36077c30491 // indirect 112 | sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect 113 | sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect 114 | sigs.k8s.io/yaml v1.3.0 // indirect 115 | ) 116 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | env: &env 3 | environment: 4 | GO111MODULE: auto 5 | UPGRADE_TEST_LOG_FOLDER: /tmp/logs 6 | # The GOPRIVATE environment variable controls which modules the go command considers to be private 7 | # (not available publicly) and should therefore not use the proxy or checksum database. 8 | # Without this, tests fail with a "410 Gone" error 9 | GOPRIVATE: github.com/gruntwork-io 10 | defaults: &defaults 11 | docker: 12 | - image: 087285199408.dkr.ecr.us-east-1.amazonaws.com/circle-ci-test-image-base:go1.21.9-tf1.5-tg39.1-pck1.8-ci54.0 13 | <<: *env 14 | terrascan: &terrascan 15 | docker: 16 | - image: "tenable/terrascan:1.18.3" 17 | <<: *env 18 | run_precommit: &run_precommit 19 | # Fail the build if the pre-commit hooks don't pass. Note: if you run $ pre-commit install locally within this repo, these hooks will 20 | # execute automatically every time before you commit, ensuring the build never fails at this step! 21 | name: run pre-commit hooks 22 | command: | 23 | pre-commit install 24 | pre-commit run --all-files 25 | auth_gh_cli: &auth_gh_cli 26 | name: Authenticate gh CLI 27 | command: | 28 | echo 'export GITHUB_TOKEN="$GITHUB_OAUTH_TOKEN"' >> $BASH_ENV 29 | # --------------------------------------------------------------------------------------------------------------------- 30 | # REUSABLE STEPS 31 | # --------------------------------------------------------------------------------------------------------------------- 32 | commands: 33 | store_results: 34 | description: Store test results for easy viewing. 35 | steps: 36 | - run: 37 | command: terratest_log_parser --testlog /tmp/logs/all.log --outputdir /tmp/logs 38 | when: always 39 | - store_artifacts: 40 | path: /tmp/logs 41 | - store_test_results: 42 | path: /tmp/logs 43 | #---------------------------------------------------------------------------------------------------------------------- 44 | # BUILD JOBS 45 | #---------------------------------------------------------------------------------------------------------------------- 46 | jobs: 47 | precommit: 48 | <<: *defaults 49 | steps: 50 | - checkout 51 | # Fail the build if the pre-commit hooks don't pass. Note: if you run pre-commit install locally, these hooks will 52 | # execute automatically every time before you commit, ensuring the build never fails at this step! 53 | - run: 54 | <<: *run_precommit 55 | test: 56 | <<: *defaults 57 | steps: 58 | - checkout 59 | - run: 60 | # These tests include an Elasticsearch cluster which can be VERY slow to create/delete, so we massively increase 61 | # the test timeout to ensure cleanup jobs run correctly. 62 | # Also specify a CircleCI timeout of 5400 seconds (90m) 63 | name: run tests 64 | command: | 65 | mkdir -p /tmp/logs 66 | # Believe it or not, we've seen the tee command fail when we have too much logging all happening at once. 67 | # To ensure that tee failing doesn't cause the whole test suite to fail, we add an || true. 68 | run-go-tests --path ./test --timeout 2h --packages . | (tee /tmp/logs/all.log || true) 69 | no_output_timeout: 5400s 70 | - store_results 71 | terrascan: 72 | description: Run Terrascan 73 | <<: *terrascan 74 | steps: 75 | - checkout 76 | - run: 77 | name: Run terrascan 78 | command: | 79 | # We only want to fail on violations, so we need to ignore exit code 4 80 | # See https://runterrascan.io/docs/_print/#configuring-the-output-format-for-a-scan for information on terrascan exit codes. 81 | terrascan scan -d ./modules --output json || (ec=$?; if [[ $ec = 4 ]]; then exit 0; else exit $ec; fi;) 82 | no_output_timeout: 3600s 83 | upgrade_test: 84 | description: Run upgrades and post the results on the PR. 85 | <<: *defaults 86 | steps: 87 | - checkout 88 | - run: 89 | <<: *auth_gh_cli 90 | - run: 91 | name: Set environment variables for use in the upgrade test step. 92 | command: | 93 | # Set the UPGRADE_TEST_BASE_REF, UPGRADE_TEST_TF_BASE_VERSION, UPGRADE_TEST_TF_TARGET_VERSION env vars. 94 | ./.circleci/set-upgrade-test-vars.sh 95 | - run: 96 | name: Run upgrade tests 97 | command: | 98 | mkdir -p /tmp/logs 99 | # Pre-install terraform versions used in the test, before the tests run, to avoid installation racing. 100 | mise install terraform@$UPGRADE_TEST_TF_BASE_VERSION 101 | mise install terraform@$UPGRADE_TEST_TF_TARGET_VERSION 102 | # Believe it or not, we've seen the tee command fail when we have too much logging all happening at once. 103 | # To ensure that tee failing doesn't cause the whole test suite to fail, we add an || true. 104 | run-go-tests \ 105 | --path ./test/upgrades \ 106 | --timeout 1h \ 107 | --extra-flags "-ldflags '-X github.com/gruntwork-io/module-ci/test/upgrades.BaseRef=$UPGRADE_TEST_BASE_REF -X github.com/gruntwork-io/module-ci/test/upgrades.TFBaseVersion=$UPGRADE_TEST_TF_BASE_VERSION -X github.com/gruntwork-io/module-ci/test/upgrades.TFTargetVersion=$UPGRADE_TEST_TF_TARGET_VERSION'" \ 108 | | (tee /tmp/logs/all.log || true) 109 | no_output_timeout: 3600s 110 | - store_results 111 | - run: 112 | name: Post upgrade test results 113 | command: ./.circleci/post-upgrade-test-results.sh 114 | when: always 115 | #---------------------------------------------------------------------------------------------------------------------- 116 | # WORKFLOWS 117 | #---------------------------------------------------------------------------------------------------------------------- 118 | workflows: 119 | version: 2.1 120 | test: 121 | jobs: 122 | # By default CircleCI runs jobs on all branches but no tags. 123 | # We need a filter to ensure jobs run on all tags starting with v. 124 | - precommit: 125 | filters: 126 | tags: 127 | only: /^v.*/ 128 | context: 129 | - AWS__PHXDEVOPS__circle-ci-test 130 | - GITHUB__PAT__gruntwork-ci 131 | - test: 132 | requires: 133 | - precommit 134 | filters: 135 | tags: 136 | only: /^v.*/ 137 | context: 138 | - AWS__PHXDEVOPS__circle-ci-test 139 | - GITHUB__PAT__gruntwork-ci 140 | scan: 141 | jobs: 142 | - precommit: 143 | filters: 144 | tags: 145 | only: /^v.*/ 146 | context: 147 | - AWS__PHXDEVOPS__circle-ci-test 148 | - GITHUB__PAT__gruntwork-ci 149 | - terrascan: 150 | requires: 151 | - precommit 152 | filters: 153 | tags: 154 | only: /^v.*/ 155 | context: 156 | - AWS__PHXDEVOPS__circle-ci-test 157 | - GITHUB__PAT__gruntwork-ci 158 | upgrade-test: 159 | jobs: 160 | - precommit: 161 | filters: 162 | tags: 163 | only: /^v.*/ 164 | context: 165 | - AWS__PHXDEVOPS__circle-ci-test 166 | - GITHUB__PAT__gruntwork-ci 167 | - upgrade_test: 168 | requires: 169 | - precommit 170 | filters: 171 | tags: 172 | only: /^v.*/ 173 | context: 174 | - AWS__PHXDEVOPS__circle-ci-test 175 | - GITHUB__PAT__gruntwork-ci 176 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2019 Gruntwork, Inc 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | --------------------------------------------------------------------------------