├── VERSION ├── .github ├── CODEOWNERS ├── dependabot.yml ├── workflows │ ├── yaml.integration.yaml │ ├── github.repo.yaml │ ├── github.pr.yaml │ └── terraform.integration.yaml ├── auto-assign.yaml ├── labeler.yaml ├── labels.yaml └── labels.common.yaml ├── examples ├── s3-bucket-versioning │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── s3-bucket-lifecycle-rules │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── athena-workgroup │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── s3-bucket-full │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── s3-access-point-vpc │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── s3-access-point-internet │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── s3-bucket-access-logging │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── glue-data-catalog-full │ ├── versions.tf │ ├── outputs.tf │ └── main.tf ├── glue-data-catalog-simple │ ├── versions.tf │ ├── outputs.tf │ └── main.tf └── s3-bucket-encryption │ ├── outputs.tf │ ├── versions.tf │ └── main.tf ├── modules ├── athena-workgroup │ ├── migrations.tf │ ├── versions.tf │ ├── resource-group.tf │ ├── outputs.tf │ ├── main.tf │ ├── variables.tf │ └── README.md ├── glue-crawler │ ├── versions.tf │ ├── resource-group.tf │ ├── iam.tf │ ├── main.tf │ └── outputs.tf ├── glue-database │ ├── versions.tf │ ├── resource-group.tf │ ├── ram-shares.tf │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── README.md ├── glue-table │ ├── versions.tf │ ├── resource-group.tf │ ├── ram-shares.tf │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── README.md ├── s3-bucket │ ├── versions.tf │ ├── request-payment.tf │ ├── transfer-acceleration.tf │ ├── metrics.tf │ ├── resource-group.tf │ ├── logging.tf │ ├── policies.tf │ ├── iam.tf │ ├── main.tf │ ├── access-control.tf │ ├── outputs.tf │ └── lifecycle.tf ├── athena-data-catalog │ ├── versions.tf │ ├── resource-group.tf │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── README.md ├── glue-connection │ ├── versions.tf │ ├── resource-group.tf │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── README.md ├── glue-data-catalog │ ├── versions.tf │ ├── resource-group.tf │ ├── ram-shares.tf │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── README.md ├── s3-access-point │ ├── versions.tf │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── README.md └── lakeformation-data-catalog │ ├── versions.tf │ └── README.md ├── .editorconfig ├── .pre-commit-config.yaml ├── .yamllint.yaml ├── .gitignore ├── .tflint.hcl └── README.md /VERSION: -------------------------------------------------------------------------------- 1 | 0.7.0 2 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @posquit0 2 | -------------------------------------------------------------------------------- /examples/s3-bucket-versioning/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bucket" { 2 | value = module.bucket 3 | } 4 | -------------------------------------------------------------------------------- /examples/s3-bucket-lifecycle-rules/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bucket" { 2 | value = module.bucket 3 | } 4 | -------------------------------------------------------------------------------- /examples/athena-workgroup/outputs.tf: -------------------------------------------------------------------------------- 1 | output "workgroups" { 2 | value = { 3 | simple = module.simple 4 | full = module.full 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /examples/s3-bucket-full/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bucket" { 2 | value = module.bucket 3 | } 4 | 5 | output "log_bucket" { 6 | value = module.log_bucket 7 | } 8 | -------------------------------------------------------------------------------- /examples/s3-access-point-vpc/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bucket" { 2 | value = module.bucket 3 | } 4 | 5 | output "access_point" { 6 | value = module.access_point 7 | } 8 | -------------------------------------------------------------------------------- /examples/s3-access-point-internet/outputs.tf: -------------------------------------------------------------------------------- 1 | output "bucket" { 2 | value = module.bucket 3 | } 4 | 5 | output "access_point" { 6 | value = module.access_point 7 | } 8 | -------------------------------------------------------------------------------- /examples/s3-bucket-access-logging/outputs.tf: -------------------------------------------------------------------------------- 1 | output "buckets" { 2 | value = { 3 | source = module.source_bucket 4 | target = module.target_bucket 5 | } 6 | } 7 | -------------------------------------------------------------------------------- /modules/athena-workgroup/migrations.tf: -------------------------------------------------------------------------------- 1 | # 2023-01-13 2 | moved { 3 | from = aws_resourcegroups_group.this[0] 4 | to = module.resource_group[0].aws_resourcegroups_group.this 5 | } 6 | -------------------------------------------------------------------------------- /modules/glue-crawler/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.65" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/glue-database/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.66" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/glue-table/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.65" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/s3-bucket/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.12" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 6.12" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/athena-workgroup/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/athena-data-catalog/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.65" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/athena-workgroup/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.7" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/glue-connection/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.65" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/glue-data-catalog/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.65" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /modules/s3-access-point/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.49" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/glue-data-catalog-full/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/glue-data-catalog-simple/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /examples/s3-bucket-encryption/outputs.tf: -------------------------------------------------------------------------------- 1 | output "buckets" { 2 | value = { 3 | "AES256" = module.bucket 4 | "AWS_KMS" = module.bucket_kms 5 | } 6 | } 7 | 8 | output "kms_key" { 9 | value = module.kms_key 10 | } 11 | -------------------------------------------------------------------------------- /modules/lakeformation-data-catalog/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 4.65" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: github-actions 5 | directory: / 6 | schedule: 7 | interval: daily 8 | 9 | - package-ecosystem: terraform 10 | directories: 11 | - /modules/* 12 | schedule: 13 | interval: weekly 14 | -------------------------------------------------------------------------------- /examples/s3-bucket-full/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/s3-access-point-vpc/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/s3-bucket-encryption/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/s3-bucket-versioning/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/s3-access-point-internet/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/s3-bucket-access-logging/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /examples/s3-bucket-lifecycle-rules/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = "~> 1.5" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = "~> 4.0" 8 | } 9 | random = { 10 | source = "hashicorp/random" 11 | version = "~> 3.0" 12 | } 13 | } 14 | } 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # Top-most EditorConfig file 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | end_of_line = lf 7 | indent_style = space 8 | indent_size = 2 9 | trim_trailing_whitespace = true 10 | insert_final_newline = true 11 | 12 | [*.{tf,tfvars}] 13 | indent_size = 2 14 | indent_style = space 15 | 16 | [.gitignore] 17 | end_of_line = 18 | -------------------------------------------------------------------------------- /examples/glue-data-catalog-simple/outputs.tf: -------------------------------------------------------------------------------- 1 | output "data_catalog" { 2 | description = "The Glue Data Catalog." 3 | value = module.data_catalog 4 | } 5 | 6 | output "database" { 7 | description = "The Glue Catalog Database." 8 | value = module.database 9 | } 10 | 11 | output "table" { 12 | description = "The Glue Catalog Table." 13 | value = module.table 14 | } 15 | -------------------------------------------------------------------------------- /modules/s3-bucket/request-payment.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Request Payment for S3 Bucket 3 | ################################################### 4 | 5 | # INFO: Not supported attributes 6 | # - `expected_bucket_owner` 7 | resource "aws_s3_bucket_request_payment_configuration" "this" { 8 | region = var.region 9 | 10 | bucket = aws_s3_bucket.this.bucket 11 | payer = var.requester_payment_enabled ? "Requester" : "BucketOwner" 12 | } 13 | -------------------------------------------------------------------------------- /modules/s3-bucket/transfer-acceleration.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Transfer Acceleration for S3 Bucket 3 | ################################################### 4 | 5 | # INFO: Not supported attributes 6 | # - `expected_bucket_owner` 7 | resource "aws_s3_bucket_accelerate_configuration" "this" { 8 | region = var.region 9 | 10 | bucket = aws_s3_bucket.this.bucket 11 | status = var.transfer_acceleration_enabled ? "Enabled" : "Suspended" 12 | } 13 | -------------------------------------------------------------------------------- /examples/glue-data-catalog-full/outputs.tf: -------------------------------------------------------------------------------- 1 | output "kms_key" { 2 | description = "The KMS Key for encryption of Glue Data Catalog." 3 | value = module.kms_key 4 | } 5 | 6 | output "data_catalog" { 7 | description = "The Glue Data Catalog." 8 | value = module.data_catalog 9 | } 10 | 11 | output "databases" { 12 | description = "A set of databases of the Glue Catalog." 13 | value = module.database 14 | } 15 | 16 | output "tables" { 17 | description = "A set of tables of the Glue Catalog." 18 | value = module.table 19 | } 20 | -------------------------------------------------------------------------------- /.github/workflows/yaml.integration.yaml: -------------------------------------------------------------------------------- 1 | name: Integration - YAML 2 | 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - "**.yaml" 10 | - "**.yml" 11 | 12 | pull_request: 13 | paths: 14 | - "**.yaml" 15 | - "**.yml" 16 | 17 | workflow_dispatch: {} 18 | 19 | 20 | concurrency: 21 | group: integration-yaml-${{ github.ref }} 22 | cancel-in-progress: true 23 | 24 | 25 | jobs: 26 | lint: 27 | name: Lint (yamllint) 28 | uses: tedilabs/github-actions/.github/workflows/yaml.yamllint.yaml@main 29 | 30 | with: 31 | yamllint_version: latest 32 | yamllint_config_file: .yamllint.yaml 33 | yamllint_target_dir: ./ 34 | secrets: 35 | token: ${{ secrets.GITHUB_TOKEN }} 36 | -------------------------------------------------------------------------------- /modules/s3-bucket/metrics.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Request Metrics for S3 Bucket 3 | ################################################### 4 | 5 | # TODO: Support access point filter 6 | resource "aws_s3_bucket_metric" "this" { 7 | for_each = { 8 | for metric in var.request_metrics : 9 | metric.name => metric 10 | } 11 | 12 | region = var.region 13 | 14 | bucket = aws_s3_bucket.this.bucket 15 | name = each.key 16 | 17 | dynamic "filter" { 18 | for_each = each.value.filter != null ? [each.value.filter] : [] 19 | 20 | content { 21 | access_point = filter.value.access_point 22 | prefix = filter.value.prefix 23 | tags = filter.value.tags 24 | } 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /modules/glue-table/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/glue-connection/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/glue-crawler/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/glue-database/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/athena-data-catalog/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/athena-workgroup/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/glue-data-catalog/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | name = local.resource_group_name 20 | description = var.resource_group.description 21 | 22 | query = { 23 | resource_tags = local.module_tags 24 | } 25 | 26 | module_tags_enabled = false 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | ) 31 | } 32 | -------------------------------------------------------------------------------- /modules/s3-bucket/resource-group.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | resource_group_name = (var.resource_group.name != "" 3 | ? var.resource_group.name 4 | : join(".", [ 5 | local.metadata.package, 6 | local.metadata.module, 7 | replace(local.metadata.name, "/[^a-zA-Z0-9_\\.-]/", "-"), 8 | ]) 9 | ) 10 | } 11 | 12 | 13 | module "resource_group" { 14 | source = "tedilabs/misc/aws//modules/resource-group" 15 | version = "~> 0.12.0" 16 | 17 | count = (var.resource_group.enabled && var.module_tags_enabled) ? 1 : 0 18 | 19 | region = var.region 20 | 21 | name = local.resource_group_name 22 | description = var.resource_group.description 23 | 24 | query = { 25 | resource_tags = local.module_tags 26 | } 27 | 28 | module_tags_enabled = false 29 | tags = merge( 30 | local.module_tags, 31 | var.tags, 32 | ) 33 | } 34 | -------------------------------------------------------------------------------- /examples/s3-bucket-versioning/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # S3 Bucket 8 | ################################################### 9 | 10 | resource "random_string" "this" { 11 | length = 32 12 | special = false 13 | numeric = false 14 | upper = false 15 | } 16 | 17 | locals { 18 | bucket_name = random_string.this.id 19 | } 20 | 21 | module "bucket" { 22 | source = "../../modules/s3-bucket" 23 | # source = "tedilabs/data/aws//modules/s3-bucket" 24 | # version = "~> 0.2.0" 25 | 26 | name = local.bucket_name 27 | force_destroy = true 28 | 29 | versioning = { 30 | status = "ENABLED" 31 | mfa_deletion = { 32 | enabled = false 33 | device = null 34 | } 35 | } 36 | 37 | tags = { 38 | "project" = "terraform-aws-data-examples" 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /modules/glue-data-catalog/ram-shares.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Resource Sharing by RAM (Resource Access Manager) 3 | ################################################### 4 | 5 | module "share" { 6 | source = "tedilabs/account/aws//modules/ram-share" 7 | version = "~> 0.22.0" 8 | 9 | for_each = { 10 | for share in var.shares : 11 | share.name => share 12 | } 13 | 14 | name = "glue.data-catalog.${local.account_id}.${each.key}" 15 | 16 | resources = [ 17 | local.arn, 18 | ] 19 | permissions = each.value.permissions 20 | 21 | external_principals_allowed = each.value.external_principals_allowed 22 | principals = each.value.principals 23 | 24 | resource_group_enabled = false 25 | module_tags_enabled = false 26 | 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | each.value.tags, 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /modules/glue-table/ram-shares.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Resource Sharing by RAM (Resource Access Manager) 3 | ################################################### 4 | 5 | module "share" { 6 | source = "tedilabs/account/aws//modules/ram-share" 7 | version = "~> 0.22.0" 8 | 9 | for_each = { 10 | for share in var.shares : 11 | share.name => share 12 | } 13 | 14 | name = "glue.table.${var.name}.${each.key}" 15 | 16 | resources = [ 17 | aws_glue_catalog_table.this.arn, 18 | ] 19 | permissions = each.value.permissions 20 | 21 | external_principals_allowed = each.value.external_principals_allowed 22 | principals = each.value.principals 23 | 24 | resource_group_enabled = false 25 | module_tags_enabled = false 26 | 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | each.value.tags, 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /modules/glue-database/ram-shares.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # Resource Sharing by RAM (Resource Access Manager) 3 | ################################################### 4 | 5 | module "share" { 6 | source = "tedilabs/account/aws//modules/ram-share" 7 | version = "~> 0.22.0" 8 | 9 | for_each = { 10 | for share in var.shares : 11 | share.name => share 12 | } 13 | 14 | name = "glue.database.${var.name}.${each.key}" 15 | 16 | resources = [ 17 | aws_glue_catalog_database.this.arn, 18 | ] 19 | permissions = each.value.permissions 20 | 21 | external_principals_allowed = each.value.external_principals_allowed 22 | principals = each.value.principals 23 | 24 | resource_group_enabled = false 25 | module_tags_enabled = false 26 | 27 | tags = merge( 28 | local.module_tags, 29 | var.tags, 30 | each.value.tags, 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /.github/workflows/github.repo.yaml: -------------------------------------------------------------------------------- 1 | name: GitHub - Repository 2 | 3 | 4 | on: 5 | push: 6 | branches: 7 | - main 8 | paths: 9 | - .github/labels*.yaml 10 | - .github/workflows/github.repo.yaml 11 | 12 | pull_request: 13 | paths: 14 | - .github/labels*.yaml 15 | - .github/workflows/github.repo.yaml 16 | 17 | workflow_dispatch: {} 18 | 19 | 20 | jobs: 21 | sync-labels: 22 | name: Sync Labels 23 | strategy: 24 | matrix: 25 | config_file: 26 | - .github/labels.yaml 27 | - .github/labels.common.yaml 28 | uses: tedilabs/github-actions/.github/workflows/github.repo.sync-labels.yaml@main 29 | 30 | permissions: 31 | contents: read 32 | issues: write 33 | 34 | with: 35 | config_file: ${{ matrix.config_file }} 36 | skip_delete: true 37 | dry_run: ${{ github.event_name == 'pull_request' }} 38 | secrets: 39 | token: ${{ secrets.GITHUB_TOKEN }} 40 | -------------------------------------------------------------------------------- /.github/auto-assign.yaml: -------------------------------------------------------------------------------- 1 | runOnDraft: false 2 | # A list of keywords to be skipped the process that add reviewers if pull requests include it 3 | skipKeywords: 4 | - wip 5 | 6 | # Set to true to add reviewers to pull requests 7 | addReviewers: true 8 | # A number of reviewers added to the pull request 9 | # Set 0 to add all the reviewers (default: 0) 10 | numberOfReviewers: 1 11 | useReviewGroups: true 12 | reviewGroups: 13 | owners: 14 | - posquit0 15 | maintainers: 16 | - posquit0 17 | 18 | # true: Add assignees to pull requests 19 | # 'author': Set the PR creator as the assignee. 20 | addAssignees: 'author' 21 | # A number of assignees to add to the pull request 22 | # Set to 0 to add all of the assignees. 23 | # Uses numberOfReviewers if unset. 24 | # numberOfAssignees: 2 25 | useAssigneeGroups: false 26 | # assigneeGroups: 27 | # groupA: 28 | # - assigneeA 29 | # - assigneeB 30 | # - assigneeC 31 | # groupB: 32 | # - assigneeD 33 | # - assigneeE 34 | # - assigneeF 35 | -------------------------------------------------------------------------------- /modules/athena-data-catalog/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = var.name 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | 18 | resource "aws_athena_data_catalog" "this" { 19 | name = var.name 20 | description = var.description 21 | type = var.type 22 | 23 | parameters = var.parameters 24 | 25 | tags = merge( 26 | { 27 | "Name" = local.metadata.name 28 | }, 29 | local.module_tags, 30 | var.tags, 31 | ) 32 | } 33 | -------------------------------------------------------------------------------- /.github/labeler.yaml: -------------------------------------------------------------------------------- 1 | # Modules 2 | ":floppy_disk: athena-data-catalog": 3 | - changed-files: 4 | - any-glob-to-any-file: 5 | - modules/athena-data-catalog/**/* 6 | 7 | ":floppy_disk: athena-workgroup": 8 | - changed-files: 9 | - any-glob-to-any-file: 10 | - modules/athena-workgroup/**/* 11 | 12 | ":floppy_disk: glue-connection": 13 | - changed-files: 14 | - any-glob-to-any-file: 15 | - modules/glue-connection/**/* 16 | 17 | ":floppy_disk: glue-crawler": 18 | - changed-files: 19 | - any-glob-to-any-file: 20 | - modules/glue-crawler/**/* 21 | 22 | ":floppy_disk: glue-data-catalog": 23 | - changed-files: 24 | - any-glob-to-any-file: 25 | - modules/glue-data-catalog/**/* 26 | 27 | ":floppy_disk: glue-database": 28 | - changed-files: 29 | - any-glob-to-any-file: 30 | - modules/glue-database/**/* 31 | 32 | ":floppy_disk: glue-table": 33 | - changed-files: 34 | - any-glob-to-any-file: 35 | - modules/glue-table/**/* 36 | 37 | ":floppy_disk: s3-access-point": 38 | - changed-files: 39 | - any-glob-to-any-file: 40 | - modules/s3-access-point/**/* 41 | 42 | ":floppy_disk: s3-bucket": 43 | - changed-files: 44 | - any-glob-to-any-file: 45 | - modules/s3-bucket/**/* 46 | -------------------------------------------------------------------------------- /modules/glue-crawler/iam.tf: -------------------------------------------------------------------------------- 1 | data "aws_iam_role" "custom" { 2 | count = var.custom_iam_role != null ? 1 : 0 3 | 4 | name = (startswith(var.custom_iam_role, "arn:aws") 5 | ? split(":role/", var.custom_iam_role)[1] 6 | : var.custom_iam_role 7 | ) 8 | } 9 | 10 | 11 | ################################################### 12 | # IAM Role for Glue Crawler 13 | ################################################### 14 | 15 | module "role" { 16 | count = (var.custom_iam_role == null && var.iam_role.enabled) ? 1 : 0 17 | 18 | source = "tedilabs/account/aws//modules/iam-role" 19 | version = "~> 0.25.0" 20 | 21 | name = "aws-glue-crawler-${local.metadata.name}" 22 | path = "/" 23 | description = "Role for AWS Glue Crawler (${local.metadata.name})" 24 | 25 | force_detach_policies = true 26 | 27 | trusted_service_policies = [ 28 | { 29 | services = ["glue.amazonaws.com"] 30 | }, 31 | ] 32 | conditions = var.iam_role.conditions 33 | 34 | policies = var.iam_role.policies 35 | inline_policies = var.iam_role.inline_policies 36 | 37 | resource_group_enabled = false 38 | module_tags_enabled = false 39 | 40 | tags = merge( 41 | local.module_tags, 42 | var.tags, 43 | ) 44 | } 45 | 46 | -------------------------------------------------------------------------------- /modules/s3-bucket/logging.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | logging_partition_date_sources = { 3 | "PARTITIONED_BY_DELIVERY_TIME" = "DeliveryTime" 4 | "PARTITIONED_BY_EVENT_TIME" = "EventTime" 5 | } 6 | } 7 | 8 | 9 | ################################################### 10 | # Logging for S3 Bucket 11 | ################################################### 12 | 13 | # INFO: Not supported attributes 14 | # - `expected_bucket_owner` 15 | # - `target_grant` (Using ACL is not recommended) 16 | resource "aws_s3_bucket_logging" "this" { 17 | count = var.logging.enabled ? 1 : 0 18 | 19 | region = var.region 20 | 21 | bucket = aws_s3_bucket.this.bucket 22 | 23 | target_bucket = var.logging.s3_bucket 24 | target_prefix = var.logging.s3_key_prefix 25 | 26 | target_object_key_format { 27 | dynamic "partitioned_prefix" { 28 | for_each = contains(["SIMPLE", "PARTITIONED_BY_DELIVERY_TIME", "PARTITIONED_BY_EVENT_TIME"], var.logging.s3_key_format) ? ["go"] : [] 29 | 30 | content { 31 | partition_date_source = local.logging_partition_date_sources[var.logging.s3_key_format] 32 | } 33 | } 34 | 35 | dynamic "simple_prefix" { 36 | for_each = var.logging.s3_key_format == "SIMPLE" ? ["go"] : [] 37 | 38 | content {} 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | default_install_hook_types: 2 | - pre-commit 3 | - commit-msg 4 | 5 | repos: 6 | - repo: https://github.com/antonbabenko/pre-commit-terraform 7 | rev: v1.104.0 8 | hooks: 9 | - id: terraform_fmt 10 | name: (terraform) Format .tf files with `terraform fmt` 11 | args: 12 | - --args=-diff 13 | - id: terraform_validate 14 | name: (terraform) Check with `terraform validate` 15 | args: 16 | - --hook-config=--retry-once-with-cleanup=true 17 | - --tf-init-args=-upgrade 18 | - id: terraform_tflint 19 | name: (terraform) Check with `tflint` 20 | args: 21 | - --args=--config=__GIT_WORKING_DIR__/.tflint.hcl 22 | files: ^modules/ 23 | - id: terraform_docs 24 | name: (terraform) Generate docs with `terraform-docs` 25 | args: ["--args=--sort-by required"] 26 | 27 | - repo: https://github.com/pre-commit/pre-commit-hooks 28 | rev: v6.0.0 29 | hooks: 30 | - id: trailing-whitespace 31 | 32 | - repo: https://github.com/adrienverge/yamllint 33 | rev: v1.37.1 34 | hooks: 35 | - id: yamllint 36 | name: (yaml) Check with `yamllint` 37 | 38 | - repo: https://github.com/compilerla/conventional-pre-commit 39 | rev: v4.3.0 40 | hooks: 41 | - id: conventional-pre-commit 42 | name: (commit-message) Check conventional commit 43 | stages: [commit-msg] 44 | args: [] 45 | -------------------------------------------------------------------------------- /examples/s3-bucket-access-logging/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # S3 Bucket 8 | ################################################### 9 | 10 | resource "random_string" "this" { 11 | length = 32 12 | special = false 13 | numeric = false 14 | upper = false 15 | } 16 | 17 | locals { 18 | bucket_name = random_string.this.id 19 | } 20 | 21 | module "source_bucket" { 22 | source = "../../modules/s3-bucket" 23 | # source = "tedilabs/data/aws//modules/s3-bucket" 24 | # version = "~> 0.2.0" 25 | 26 | name = "${local.bucket_name}-source" 27 | force_destroy = true 28 | 29 | logging = { 30 | enabled = true 31 | s3_bucket = module.target_bucket.name 32 | s3_key_prefix = "" 33 | } 34 | 35 | tags = { 36 | "project" = "terraform-aws-data-examples" 37 | } 38 | } 39 | 40 | module "target_bucket" { 41 | source = "../../modules/s3-bucket" 42 | # source = "tedilabs/data/aws//modules/s3-bucket" 43 | # version = "~> 0.2.0" 44 | 45 | name = "${local.bucket_name}-target" 46 | force_destroy = true 47 | 48 | logging = { 49 | is_target_bucket = true 50 | allowed_source_buckets = ["${local.bucket_name}-*"] 51 | } 52 | 53 | tags = { 54 | "project" = "terraform-aws-data-examples" 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /.yamllint.yaml: -------------------------------------------------------------------------------- 1 | yaml-files: 2 | - '*.yaml' 3 | - '*.yml' 4 | 5 | rules: 6 | braces: 7 | min-spaces-inside: 0 8 | max-spaces-inside: 1 9 | min-spaces-inside-empty: 0 10 | max-spaces-inside-empty: 0 11 | brackets: 12 | min-spaces-inside: 0 13 | max-spaces-inside: 1 14 | min-spaces-inside-empty: 0 15 | max-spaces-inside-empty: 0 16 | colons: 17 | max-spaces-before: 0 18 | max-spaces-after: 1 19 | commas: 20 | max-spaces-before: 0 21 | comments: 22 | level: warning 23 | require-starting-space: true 24 | min-spaces-from-content: 1 25 | comments-indentation: disable 26 | document-end: disable 27 | document-start: disable 28 | empty-lines: 29 | level: warning 30 | max: 2 31 | max-start: 0 32 | max-end: 1 33 | empty-values: 34 | forbid-in-block-mappings: true 35 | forbid-in-flow-mappings: true 36 | hyphens: 37 | max-spaces-after: 1 38 | indentation: 39 | spaces: consistent 40 | indent-sequences: false 41 | key-duplicates: enable 42 | key-ordering: disable 43 | line-length: disable 44 | new-line-at-end-of-file: enable 45 | # Use UNIX new line characters `\n` instead of DOS new line characters `\r\n` 46 | new-lines: 47 | type: unix 48 | octal-values: disable 49 | quoted-strings: 50 | quote-type: any 51 | required: false 52 | trailing-spaces: enable 53 | truthy: disable 54 | -------------------------------------------------------------------------------- /modules/glue-database/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = var.name 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | 18 | ################################################### 19 | # Database for Glue Data Catalog 20 | ################################################### 21 | 22 | # - create_table_default_permission 23 | # - permissions 24 | # - principal 25 | # - data_lake_principal_identifier 26 | # - parameters 27 | # - target_database 28 | # - catalog_id 29 | # - database_name 30 | resource "aws_glue_catalog_database" "this" { 31 | catalog_id = var.catalog 32 | 33 | name = var.name 34 | description = var.description 35 | location_uri = var.location_uri 36 | 37 | # parameters = var.parameters 38 | 39 | tags = merge( 40 | { 41 | "Name" = local.metadata.name 42 | }, 43 | local.module_tags, 44 | var.tags, 45 | ) 46 | } 47 | -------------------------------------------------------------------------------- /modules/athena-data-catalog/outputs.tf: -------------------------------------------------------------------------------- 1 | output "arn" { 2 | description = "The Amazon Resource Name (ARN) of the data catalog." 3 | value = aws_athena_data_catalog.this.arn 4 | } 5 | 6 | output "id" { 7 | description = "The ID of the data catalog." 8 | value = aws_athena_data_catalog.this.id 9 | } 10 | 11 | output "name" { 12 | description = "The name of the data catalog." 13 | value = aws_athena_data_catalog.this.name 14 | } 15 | 16 | output "description" { 17 | description = "The description of the data catalog." 18 | value = aws_athena_data_catalog.this.description 19 | } 20 | 21 | output "type" { 22 | description = "The type of the data catalog." 23 | value = aws_athena_data_catalog.this.type 24 | } 25 | 26 | output "parameters" { 27 | description = "A set of key value pairs that specifies the Lambda function or functions to use for creating the data catalog." 28 | value = aws_athena_data_catalog.this.parameters 29 | } 30 | 31 | output "resource_group" { 32 | description = "The resource group created to manage resources in this module." 33 | value = merge( 34 | { 35 | enabled = var.resource_group.enabled && var.module_tags_enabled 36 | }, 37 | (var.resource_group.enabled && var.module_tags_enabled 38 | ? { 39 | arn = module.resource_group[0].arn 40 | name = module.resource_group[0].name 41 | } 42 | : {} 43 | ) 44 | ) 45 | } 46 | -------------------------------------------------------------------------------- /examples/s3-access-point-internet/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | data "aws_caller_identity" "this" {} 6 | data "aws_region" "this" {} 7 | 8 | resource "random_string" "this" { 9 | length = 32 10 | special = false 11 | numeric = false 12 | upper = false 13 | } 14 | 15 | locals { 16 | bucket_name = random_string.this.id 17 | access_point_name = "access-point-test-internet" 18 | 19 | account_id = data.aws_caller_identity.this.account_id 20 | region = data.aws_region.this.name 21 | } 22 | 23 | 24 | ################################################### 25 | # S3 Bucket 26 | ################################################### 27 | 28 | module "bucket" { 29 | source = "../../modules/s3-bucket" 30 | # source = "tedilabs/data/aws//modules/s3-bucket" 31 | # version = "~> 0.2.0" 32 | 33 | name = local.bucket_name 34 | force_destroy = true 35 | 36 | tags = { 37 | "project" = "terraform-aws-data-examples" 38 | } 39 | } 40 | 41 | 42 | ################################################### 43 | # S3 Access Point 44 | ################################################### 45 | 46 | module "access_point" { 47 | source = "../../modules/s3-access-point" 48 | # source = "tedilabs/data/aws//modules/s3-access-point" 49 | # version = "~> 0.2.0" 50 | 51 | name = local.access_point_name 52 | bucket = { 53 | name = module.bucket.name 54 | } 55 | 56 | policy = null 57 | block_public_access = { 58 | enabled = true 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /examples/glue-data-catalog-simple/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # Glue Data Catalog 8 | ################################################### 9 | 10 | module "data_catalog" { 11 | source = "../../modules/glue-data-catalog" 12 | # source = "tedilabs/data/aws//modules/glue-data-catalog" 13 | # version = "~> 0.2.0" 14 | 15 | encryption_for_connection_passwords = { 16 | enabled = false 17 | } 18 | encryption_at_rest = { 19 | enabled = false 20 | } 21 | 22 | tags = { 23 | "project" = "terraform-aws-data-examples" 24 | } 25 | } 26 | 27 | 28 | ################################################### 29 | # Glue Database 30 | ################################################### 31 | 32 | module "database" { 33 | source = "../../modules/glue-database" 34 | # source = "tedilabs/data/aws//modules/glue-database" 35 | # version = "~> 0.2.0" 36 | 37 | name = "example" 38 | 39 | tags = { 40 | "project" = "terraform-aws-data-examples" 41 | } 42 | } 43 | 44 | 45 | ################################################### 46 | # Glue Table 47 | ################################################### 48 | 49 | module "table" { 50 | source = "../../modules/glue-table" 51 | # source = "tedilabs/data/aws//modules/glue-table" 52 | # version = "~> 0.2.0" 53 | 54 | database = module.database.name 55 | name = "helloworld" 56 | 57 | tags = { 58 | "project" = "terraform-aws-data-examples" 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /.github/workflows/github.pr.yaml: -------------------------------------------------------------------------------- 1 | name: GitHub - Pull Request 2 | 3 | 4 | on: 5 | pull_request_target: {} 6 | 7 | 8 | jobs: 9 | auto-assign: 10 | name: Auto Assign 11 | uses: tedilabs/github-actions/.github/workflows/github.pr.auto-assign.yaml@main 12 | if: ${{ github.event_name == 'pull_request_target' }} 13 | 14 | permissions: 15 | contents: read 16 | pull-requests: write 17 | 18 | with: 19 | config_file: .github/auto-assign.yaml 20 | secrets: 21 | token: ${{ secrets.GITHUB_TOKEN }} 22 | 23 | auto-label: 24 | name: Auto Label 25 | uses: tedilabs/github-actions/.github/workflows/github.pr.auto-label.yaml@main 26 | if: ${{ github.event_name == 'pull_request_target' }} 27 | 28 | permissions: 29 | contents: read 30 | issues: write 31 | pull-requests: write 32 | 33 | with: 34 | labeler_config_file: .github/labeler.yaml 35 | labeler_dot_included: true 36 | labeler_sync_labels: true 37 | pr_size_xs_label: 'size/XS' 38 | pr_size_s_label: 'size/S' 39 | pr_size_m_label: 'size/M' 40 | pr_size_l_label: 'size/L' 41 | pr_size_xl_label: 'size/XL' 42 | secrets: 43 | token: ${{ secrets.GITHUB_TOKEN }} 44 | 45 | welcome: 46 | name: Welcome for First Pull Request 47 | uses: tedilabs/github-actions/.github/workflows/github.issue.welcome.yaml@main 48 | if: ${{ github.event_name == 'pull_request_target' && github.event.action == 'opened' }} 49 | 50 | secrets: 51 | token: ${{ secrets.GITHUB_TOKEN }} 52 | -------------------------------------------------------------------------------- /examples/s3-bucket-encryption/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # S3 Bucket 8 | ################################################### 9 | 10 | resource "random_string" "this" { 11 | length = 32 12 | special = false 13 | numeric = false 14 | upper = false 15 | } 16 | 17 | locals { 18 | bucket_name = random_string.this.id 19 | kms_key_name = random_string.this.id 20 | } 21 | 22 | module "bucket" { 23 | source = "../../modules/s3-bucket" 24 | # source = "tedilabs/data/aws//modules/s3-bucket" 25 | # version = "~> 0.2.0" 26 | 27 | name = "${local.bucket_name}-aes256" 28 | force_destroy = true 29 | 30 | encryption = { 31 | type = "AES256" 32 | bucket_key_enabled = true 33 | } 34 | 35 | tags = { 36 | "project" = "terraform-aws-data-examples" 37 | } 38 | } 39 | 40 | module "bucket_kms" { 41 | source = "../../modules/s3-bucket" 42 | # source = "tedilabs/data/aws//modules/s3-bucket" 43 | # version = "~> 0.2.0" 44 | 45 | name = "${local.bucket_name}-aws-kms" 46 | force_destroy = true 47 | 48 | encryption = { 49 | type = "AWS_KMS" 50 | kms_key = module.kms_key.id 51 | bucket_key_enabled = true 52 | } 53 | 54 | tags = { 55 | "project" = "terraform-aws-data-examples" 56 | } 57 | } 58 | 59 | module "kms_key" { 60 | source = "tedilabs/secret/aws//modules/kms-key" 61 | version = "~> 0.3.0" 62 | 63 | name = local.kms_key_name 64 | 65 | tags = { 66 | "project" = "terraform-aws-data-examples" 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /modules/s3-bucket/policies.tf: -------------------------------------------------------------------------------- 1 | data "aws_caller_identity" "this" {} 2 | 3 | locals { 4 | account_id = data.aws_caller_identity.this.account_id 5 | } 6 | 7 | ## Enforce TLS (HTTPS Only) 8 | data "aws_iam_policy_document" "tls_required" { 9 | statement { 10 | sid = "RequireTlsForRequest" 11 | 12 | effect = "Deny" 13 | actions = ["s3:*"] 14 | resources = [ 15 | "arn:aws:s3:::${var.name}", 16 | "arn:aws:s3:::${var.name}/*", 17 | ] 18 | 19 | principals { 20 | type = "AWS" 21 | identifiers = ["*"] 22 | } 23 | 24 | condition { 25 | test = "Bool" 26 | variable = "aws:SecureTransport" 27 | values = ["false"] 28 | } 29 | } 30 | } 31 | 32 | data "aws_iam_policy_document" "access_logging" { 33 | statement { 34 | sid = "S3ServerAccessLogsPolicy" 35 | 36 | effect = "Allow" 37 | actions = ["s3:PutObject"] 38 | resources = [ 39 | "arn:aws:s3:::${var.name}/*", 40 | ] 41 | 42 | principals { 43 | type = "Service" 44 | identifiers = ["logging.s3.amazonaws.com"] 45 | } 46 | 47 | dynamic "condition" { 48 | for_each = length(var.logging.allowed_source_buckets) > 0 ? ["go"] : [] 49 | 50 | content { 51 | test = "StringLike" 52 | variable = "aws:SourceArn" 53 | values = [ 54 | for bucket in var.logging.allowed_source_buckets : 55 | "arn:aws:s3:::${bucket}" 56 | ] 57 | } 58 | } 59 | condition { 60 | test = "StringEquals" 61 | variable = "aws:SourceAccount" 62 | values = [local.account_id] 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /modules/glue-table/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = var.name 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | 18 | ################################################### 19 | # Glue Table 20 | ################################################### 21 | 22 | 23 | # - owner 24 | # - parameters 25 | # - partition_index 26 | # - index_name 27 | # - keys 28 | # - partition_keys 29 | # - comment 30 | # - name 31 | # - type 32 | # - retention 33 | # - storage_descriptor 34 | # - bucket_columns 35 | # - columns 36 | # - compressed 37 | # - input_format 38 | # - location 39 | # - number_of_buckets 40 | # - output_format 41 | # - parameters 42 | # - schema_reference 43 | # - ser_de_info 44 | # - skewed_info 45 | # - sort_columns 46 | # - stored_as_sub_directories 47 | # - table_type 48 | # - target_table 49 | # - catalog_id 50 | # - database_name 51 | # - name 52 | # - view_expanded_text 53 | # - view_original_text 54 | resource "aws_glue_catalog_table" "this" { 55 | catalog_id = var.catalog 56 | database_name = var.database 57 | 58 | name = var.name 59 | description = var.description 60 | } 61 | -------------------------------------------------------------------------------- /modules/glue-connection/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = var.name 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | data "aws_subnet" "this" { 18 | count = var.vpc_association.enabled ? 1 : 0 19 | 20 | id = var.vpc_association.subnet 21 | } 22 | 23 | 24 | ################################################### 25 | # Glue Connection 26 | ################################################### 27 | 28 | resource "aws_glue_connection" "this" { 29 | catalog_id = var.catalog 30 | 31 | name = var.name 32 | description = var.description 33 | 34 | connection_type = var.type 35 | connection_properties = var.properties 36 | 37 | dynamic "physical_connection_requirements" { 38 | for_each = var.vpc_association.enabled ? [var.vpc_association] : [] 39 | iterator = association 40 | 41 | content { 42 | availability_zone = one(data.aws_subnet.this[*].availability_zone) 43 | subnet_id = association.value.subnet 44 | security_group_id_list = association.value.security_groups 45 | } 46 | } 47 | 48 | match_criteria = var.match_criteria 49 | 50 | tags = merge( 51 | { 52 | "Name" = local.metadata.name 53 | }, 54 | local.module_tags, 55 | var.tags, 56 | ) 57 | } 58 | -------------------------------------------------------------------------------- /examples/athena-workgroup/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # Athena Workgroup with simple configurations 8 | ################################################### 9 | 10 | module "simple" { 11 | source = "../../modules/athena-workgroup" 12 | # source = "tedilabs/data/aws//modules/athena-workgroup" 13 | # version = "~> 0.2.0" 14 | 15 | name = "simple" 16 | 17 | tags = { 18 | "project" = "terraform-aws-data-examples" 19 | } 20 | } 21 | 22 | 23 | ################################################### 24 | # Athena Workgroup with full configurations 25 | ################################################### 26 | 27 | module "full" { 28 | source = "../../modules/athena-workgroup" 29 | # source = "tedilabs/data/aws//modules/athena-workgroup" 30 | # version = "~> 0.2.0" 31 | 32 | name = "full" 33 | description = "This workgroup is created with full configurtaions." 34 | 35 | enabled = true 36 | force_destroy = true 37 | 38 | client_config_enabled = false 39 | cloudwatch_metrics_enabled = true 40 | query_on_s3_requester_pays_bucket_enabled = false 41 | 42 | per_query_data_usage_limit = 64 * 1024 * 1024 43 | 44 | query_result = { 45 | s3_bucket = "tedilabs-terraform-aws-data-examples-athena-workgroup" 46 | s3_key_prefix = "/athena-query-results" 47 | encryption_enabled = true 48 | encryption_mode = "SSE_S3" 49 | encryption_kms_key = null 50 | } 51 | 52 | named_queries = [ 53 | { 54 | name = "hello-world" 55 | database = "default" 56 | query = "SELECT 1" 57 | } 58 | ] 59 | 60 | tags = { 61 | "project" = "terraform-aws-data-examples" 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /.github/labels.yaml: -------------------------------------------------------------------------------- 1 | # Warning 2 | - color: "ee0701" 3 | description: "Categorize to note any change that requires extra attention." 4 | name: ":warning: breaking change" 5 | - color: "ee0701" 6 | description: "Categorize bug reports." 7 | name: ":warning: bug" 8 | - color: "ee0701" 9 | description: "Categorize vulnerability reports." 10 | name: ":warning: vulnerability" 11 | 12 | # Task Types 13 | - color: "1d76db" 14 | description: "Improvements or additions to documentation." 15 | name: ":pencil2: documentation" 16 | - color: "1d76db" 17 | description: "A new feature of the project." 18 | name: ":pencil2: feature" 19 | - color: "1d76db" 20 | description: "Any dependency updates, housekeeping, or maintenance work." 21 | name: ":pencil2: maintenance" 22 | 23 | # Highlight 24 | - color: "0e8a16" 25 | description: "Good for newcomers." 26 | name: ":fire: good first issue" 27 | - color: "0e8a16" 28 | description: "Extra attention is needed." 29 | name: ":fire: help wanted" 30 | 31 | # Cancel 32 | - color: "b60205" 33 | description: "This issue or pull request already exists." 34 | name: ":pray: duplicate" 35 | - color: "b60205" 36 | description: "This issue or pull request is invalid." 37 | name: ":pray: invalid" 38 | - color: "b60205" 39 | description: "This will not be worked on." 40 | name: ":pray: wontfix" 41 | 42 | # Size 43 | - color: "cfd3d7" 44 | description: "Extra Small size issue or PR." 45 | name: "size/XS" 46 | - color: "cfd3d7" 47 | description: "Small size issue or PR." 48 | name: "size/S" 49 | - color: "cfd3d7" 50 | description: "Medium size issue or PR." 51 | name: "size/M" 52 | - color: "cfd3d7" 53 | description: "Large size issue or PR." 54 | name: "size/L" 55 | - color: "cfd3d7" 56 | description: "Extra Large size issue or PR." 57 | name: "size/XL" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### OSX ### 2 | # General 3 | .DS_Store 4 | .AppleDouble 5 | .LSOverride 6 | 7 | # Icon must end with two \r 8 | Icon 9 | 10 | # Thumbnails 11 | ._* 12 | 13 | # Files that might appear in the root of a volume 14 | .DocumentRevisions-V100 15 | .fseventsd 16 | .Spotlight-V100 17 | .TemporaryItems 18 | .Trashes 19 | .VolumeIcon.icns 20 | .com.apple.timemachine.donotpresent 21 | 22 | # Directories potentially created on remote AFP share 23 | .AppleDB 24 | .AppleDesktop 25 | Network Trash Folder 26 | Temporary Items 27 | .apdisk 28 | 29 | 30 | ### Terraform ### 31 | # Lock file 32 | .terraform.lock.hcl 33 | 34 | # Local .terraform directories 35 | **/.terraform/* 36 | 37 | # .tfstate files 38 | *.tfstate 39 | *.tfstate.* 40 | 41 | # Crash log files 42 | crash.log 43 | 44 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 45 | # .tfvars files are managed as part of configuration and so should be included in 46 | # version control. 47 | # 48 | # example.tfvars 49 | 50 | # Ignore override files as they are usually used to override resources locally and so 51 | # are not checked in 52 | override.tf 53 | override.tf.json 54 | *_override.tf 55 | *_override.tf.json 56 | 57 | # Include override files you do wish to add to version control using negated pattern 58 | # !example_override.tf 59 | 60 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 61 | # example: *tfplan* 62 | 63 | 64 | ### Vim ### 65 | # Swap 66 | [._]*.s[a-v][a-z] 67 | !*.svg # comment out if you don't need vector files 68 | [._]*.sw[a-p] 69 | [._]s[a-rt-v][a-z] 70 | [._]ss[a-gi-z] 71 | [._]sw[a-p] 72 | 73 | # Session 74 | Session.vim 75 | Sessionx.vim 76 | 77 | # Temporary 78 | .netrwhist 79 | *~ 80 | # Auto-generated tag files 81 | tags 82 | # Persistent undo 83 | [._]*.un~ 84 | -------------------------------------------------------------------------------- /modules/s3-access-point/main.tf: -------------------------------------------------------------------------------- 1 | ################################################### 2 | # S3 Access Point 3 | ################################################### 4 | 5 | # INFO: Not supported attributes 6 | # - `account_id` 7 | resource "aws_s3_access_point" "this" { 8 | name = var.name 9 | 10 | bucket = var.bucket.name 11 | bucket_account_id = var.bucket.account_id 12 | 13 | dynamic "vpc_configuration" { 14 | for_each = var.network_origin == "VPC" ? [var.vpc_id] : [] 15 | 16 | content { 17 | vpc_id = vpc_configuration.value 18 | } 19 | } 20 | 21 | public_access_block_configuration { 22 | block_public_acls = (var.block_public_access.enabled || var.block_public_access.block_public_acls_enabled) 23 | ignore_public_acls = (var.block_public_access.enabled || var.block_public_access.ignore_public_acls_enabled) 24 | block_public_policy = (var.block_public_access.enabled || var.block_public_access.block_public_policy_enabled) 25 | restrict_public_buckets = (var.block_public_access.enabled || var.block_public_access.restrict_public_buckets_enabled) 26 | } 27 | 28 | lifecycle { 29 | precondition { 30 | condition = anytrue([ 31 | var.network_origin != "VPC", 32 | var.network_origin == "VPC" && var.vpc_id != null, 33 | ]) 34 | error_message = "`vpc_id` is required when the value of `network_origin` is `VPC`." 35 | } 36 | 37 | ignore_changes = [ 38 | policy, 39 | ] 40 | } 41 | } 42 | 43 | 44 | ################################################### 45 | # Policy for S3 Access Point 46 | ################################################### 47 | 48 | resource "aws_s3control_access_point_policy" "this" { 49 | count = var.policy != "" ? 1 : 0 50 | 51 | access_point_arn = aws_s3_access_point.this.arn 52 | policy = var.policy 53 | } 54 | -------------------------------------------------------------------------------- /examples/s3-access-point-vpc/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | data "aws_caller_identity" "this" {} 6 | data "aws_region" "this" {} 7 | 8 | data "aws_vpc" "default" { 9 | default = true 10 | } 11 | 12 | resource "random_string" "this" { 13 | length = 32 14 | special = false 15 | numeric = false 16 | upper = false 17 | } 18 | 19 | locals { 20 | bucket_name = random_string.this.id 21 | access_point_name = "access-point-test-vpc" 22 | 23 | account_id = data.aws_caller_identity.this.account_id 24 | region = data.aws_region.this.name 25 | } 26 | 27 | 28 | ################################################### 29 | # S3 Bucket 30 | ################################################### 31 | 32 | module "bucket" { 33 | source = "../../modules/s3-bucket" 34 | # source = "tedilabs/data/aws//modules/s3-bucket" 35 | # version = "~> 0.2.0" 36 | 37 | name = local.bucket_name 38 | force_destroy = true 39 | 40 | tags = { 41 | "project" = "terraform-aws-data-examples" 42 | } 43 | } 44 | 45 | 46 | ################################################### 47 | # S3 Access Point 48 | ################################################### 49 | 50 | module "access_point" { 51 | source = "../../modules/s3-access-point" 52 | # source = "tedilabs/data/aws//modules/s3-access-point" 53 | # version = "~> 0.2.0" 54 | 55 | name = local.access_point_name 56 | bucket = { 57 | name = module.bucket.name 58 | } 59 | 60 | network_origin = "VPC" 61 | vpc_id = data.aws_vpc.default.id 62 | 63 | policy = jsonencode({ 64 | Version = "2008-10-17" 65 | Statement = [{ 66 | Effect = "Allow" 67 | Action = "s3:GetObjectTagging" 68 | Principal = { 69 | AWS = "*" 70 | } 71 | Resource = "arn:aws:s3:${local.region}:${local.account_id}:accesspoint/${local.access_point_name}/object/*" 72 | }] 73 | }) 74 | block_public_access = { 75 | enabled = true 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /.github/labels.common.yaml: -------------------------------------------------------------------------------- 1 | # Warning 2 | - color: "ee0701" 3 | description: "Categorize to note any change that requires extra attention." 4 | name: ":warning: breaking change" 5 | - color: "ee0701" 6 | description: "Categorize bug reports." 7 | name: ":warning: bug" 8 | from_name: "bug" 9 | - color: "ee0701" 10 | description: "Categorize vulnerability reports." 11 | name: ":warning: vulnerability" 12 | 13 | # Task Types 14 | - color: "1d76db" 15 | description: "Improvements or additions to documentation." 16 | name: ":pencil2: documentation" 17 | from_name: "documentation" 18 | - color: "1d76db" 19 | description: "A new feature of the project." 20 | name: ":pencil2: feature" 21 | from_name: "enhancement" 22 | - color: "1d76db" 23 | description: "Any dependency updates, housekeeping, or maintenance work." 24 | name: ":pencil2: maintenance" 25 | 26 | # Highlight 27 | - color: "0e8a16" 28 | description: "Good for newcomers." 29 | name: ":fire: good first issue" 30 | from_name: "good first issue" 31 | - color: "0e8a16" 32 | description: "Extra attention is needed." 33 | name: ":fire: help wanted" 34 | from_name: "help wanted" 35 | 36 | # Cancel 37 | - color: "b60205" 38 | description: "This issue or pull request already exists." 39 | name: ":pray: duplicate" 40 | from_name: "duplicate" 41 | - color: "b60205" 42 | description: "This issue or pull request is invalid." 43 | name: ":pray: invalid" 44 | from_name: "invalid" 45 | - color: "b60205" 46 | description: "This will not be worked on." 47 | name: ":pray: wontfix" 48 | from_name: "wontfix" 49 | 50 | # Size 51 | - color: "cfd3d7" 52 | description: "Extra Small size issue or PR." 53 | name: "size/XS" 54 | - color: "cfd3d7" 55 | description: "Small size issue or PR." 56 | name: "size/S" 57 | - color: "cfd3d7" 58 | description: "Medium size issue or PR." 59 | name: "size/M" 60 | - color: "cfd3d7" 61 | description: "Large size issue or PR." 62 | name: "size/L" 63 | - color: "cfd3d7" 64 | description: "Extra Large size issue or PR." 65 | name: "size/XL" 66 | -------------------------------------------------------------------------------- /examples/s3-bucket-lifecycle-rules/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # S3 Bucket 8 | ################################################### 9 | 10 | resource "random_string" "this" { 11 | length = 32 12 | special = false 13 | numeric = false 14 | upper = false 15 | } 16 | 17 | locals { 18 | bucket_name = random_string.this.id 19 | } 20 | 21 | module "bucket" { 22 | source = "../../modules/s3-bucket" 23 | # source = "tedilabs/data/aws//modules/s3-bucket" 24 | # version = "~> 0.2.0" 25 | 26 | name = local.bucket_name 27 | force_destroy = true 28 | 29 | ## Lifecycle 30 | versioning = { 31 | status = "ENABLED" 32 | } 33 | lifecycle_rules = [ 34 | { 35 | id = "expiration-1" 36 | enabled = false 37 | prefix = "hello/" 38 | noncurrent_version_expiration = { 39 | days = 1 40 | count = 3 41 | } 42 | }, 43 | { 44 | id = "expiration-2" 45 | enabled = true 46 | min_object_size = 300 47 | max_object_size = 9000 48 | expiration = { 49 | days = 7 50 | } 51 | }, 52 | { 53 | id = "transitions-1" 54 | enabled = true 55 | tags = { 56 | "Product" = "User" 57 | } 58 | transitions = [ 59 | { 60 | days = 30 61 | storage_class = "STANDARD_IA" 62 | }, 63 | { 64 | days = 60 65 | storage_class = "INTELLIGENT_TIERING" 66 | }, 67 | ] 68 | }, 69 | { 70 | id = "transitions-2" 71 | enabled = true 72 | noncurrent_version_transitions = [ 73 | { 74 | days = 60 75 | storage_class = "STANDARD_IA" 76 | }, 77 | { 78 | days = 90 79 | storage_class = "INTELLIGENT_TIERING" 80 | }, 81 | ] 82 | }, 83 | { 84 | id = "abort-1" 85 | enabled = true 86 | abort_incomplete_multipart_upload = { 87 | days = 10 88 | } 89 | } 90 | ] 91 | 92 | 93 | tags = { 94 | "project" = "terraform-aws-data-examples" 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /modules/glue-table/outputs.tf: -------------------------------------------------------------------------------- 1 | output "arn" { 2 | description = "The Amazon Resource Name (ARN) of the Glue Catalog Table." 3 | value = aws_glue_catalog_table.this.arn 4 | } 5 | 6 | output "id" { 7 | description = "The ID of the database." 8 | value = aws_glue_catalog_table.this.id 9 | } 10 | 11 | output "catalog" { 12 | description = "The ID of the Glue Catalog of the table." 13 | value = aws_glue_catalog_table.this.catalog_id 14 | } 15 | 16 | output "database" { 17 | description = "The catalog database in which to create the new table." 18 | value = aws_glue_catalog_table.this.database_name 19 | } 20 | 21 | output "name" { 22 | description = "The name of the table." 23 | value = aws_glue_catalog_table.this.name 24 | } 25 | 26 | output "description" { 27 | description = "The description of the table." 28 | value = aws_glue_catalog_table.this.description 29 | } 30 | 31 | # output "z" { 32 | # value = { 33 | # for k, v in aws_glue_catalog_table.this : 34 | # k => v 35 | # if !contains(["name", "description", "catalog_id", "database_name", "arn", "id", "tags", "tags_all"], k) 36 | # } 37 | # } 38 | 39 | output "sharing" { 40 | description = < 0 ? "SHARED_BY_ME" : "NOT_SHARED" 47 | shares = module.share 48 | } 49 | } 50 | 51 | output "resource_group" { 52 | description = "The resource group created to manage resources in this module." 53 | value = merge( 54 | { 55 | enabled = var.resource_group.enabled && var.module_tags_enabled 56 | }, 57 | (var.resource_group.enabled && var.module_tags_enabled 58 | ? { 59 | arn = module.resource_group[0].arn 60 | name = module.resource_group[0].name 61 | } 62 | : {} 63 | ) 64 | ) 65 | } 66 | -------------------------------------------------------------------------------- /modules/glue-data-catalog/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = local.account_id 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | data "aws_partition" "this" {} 18 | data "aws_region" "this" {} 19 | data "aws_caller_identity" "this" {} 20 | 21 | locals { 22 | partition = data.aws_partition.this.partition 23 | region = data.aws_region.this.name 24 | account_id = data.aws_caller_identity.this.account_id 25 | arn = "arn:${local.partition}:glue:${local.region}:${local.account_id}:catalog" 26 | } 27 | 28 | 29 | ################################################### 30 | # Encryption options for Glue Data Catalog 31 | ################################################### 32 | 33 | resource "aws_glue_data_catalog_encryption_settings" "this" { 34 | catalog_id = var.catalog 35 | 36 | data_catalog_encryption_settings { 37 | connection_password_encryption { 38 | return_connection_password_encrypted = var.encryption_for_connection_passwords.enabled 39 | aws_kms_key_id = (var.encryption_for_connection_passwords.enabled 40 | ? var.encryption_for_connection_passwords.kms_key 41 | : null 42 | ) 43 | } 44 | 45 | encryption_at_rest { 46 | catalog_encryption_mode = var.encryption_at_rest.enabled ? "SSE-KMS" : "DISABLED" 47 | sse_aws_kms_key_id = (var.encryption_at_rest.enabled 48 | ? var.encryption_at_rest.kms_key 49 | : null 50 | ) 51 | } 52 | } 53 | } 54 | 55 | 56 | ################################################### 57 | # Resource Policy for Glue Data Catalog 58 | ################################################### 59 | 60 | resource "aws_glue_resource_policy" "this" { 61 | count = length(var.policy) > 16 ? 1 : 0 62 | 63 | policy = var.policy 64 | enable_hybrid = "TRUE" 65 | } 66 | -------------------------------------------------------------------------------- /modules/s3-bucket/iam.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | replication_iam_role = (var.default_replication_iam_role.enabled 3 | ? one(module.replication_iam_role[*].arn) 4 | : var.replication_iam_role 5 | ) 6 | } 7 | 8 | 9 | ################################################### 10 | # IAM Role for Replication 11 | ################################################### 12 | 13 | module "replication_iam_role" { 14 | count = length(var.replication_rules) > 0 && var.default_replication_iam_role.enabled ? 1 : 0 15 | 16 | source = "tedilabs/account/aws//modules/iam-role" 17 | version = "~> 0.32.0" 18 | 19 | name = coalesce( 20 | var.default_replication_iam_role.name, 21 | "s3-${local.metadata.name}-replication", 22 | ) 23 | path = var.default_replication_iam_role.path 24 | description = var.default_replication_iam_role.description 25 | 26 | trusted_service_policies = [ 27 | { 28 | services = ["s3.amazonaws.com"] 29 | } 30 | ] 31 | 32 | policies = var.default_replication_iam_role.policies 33 | inline_policies = merge({ 34 | "s3-replication" = jsonencode(yamldecode(< v 40 | # if !contains(["name", "description", "catalog_id", "location_uri", "arn", "id", "tags", "tags_all"], k) 41 | # } 42 | # } 43 | 44 | output "sharing" { 45 | description = < 0 ? "SHARED_BY_ME" : "NOT_SHARED" 52 | shares = module.share 53 | } 54 | } 55 | 56 | output "resource_group" { 57 | description = "The resource group created to manage resources in this module." 58 | value = merge( 59 | { 60 | enabled = var.resource_group.enabled && var.module_tags_enabled 61 | }, 62 | (var.resource_group.enabled && var.module_tags_enabled 63 | ? { 64 | arn = module.resource_group[0].arn 65 | name = module.resource_group[0].name 66 | } 67 | : {} 68 | ) 69 | ) 70 | } 71 | -------------------------------------------------------------------------------- /modules/glue-data-catalog/outputs.tf: -------------------------------------------------------------------------------- 1 | output "arn" { 2 | description = "The Amazon Resource Name (ARN) of the Glue Catalog." 3 | value = local.arn 4 | } 5 | 6 | output "catalog" { 7 | description = "The ID of the Glue Catalog." 8 | value = aws_glue_data_catalog_encryption_settings.this.catalog_id 9 | } 10 | 11 | output "policy" { 12 | description = "The policy to be applied to the aws glue data catalog for access control." 13 | value = one(aws_glue_resource_policy.this[*].policy) 14 | } 15 | 16 | output "encryption_for_connection_passwords" { 17 | description = "A configuration to encrypt connection passwords in the data catalog." 18 | value = { 19 | enabled = aws_glue_data_catalog_encryption_settings.this.data_catalog_encryption_settings[0].connection_password_encryption[0].return_connection_password_encrypted 20 | kms_key = aws_glue_data_catalog_encryption_settings.this.data_catalog_encryption_settings[0].connection_password_encryption[0].aws_kms_key_id 21 | } 22 | } 23 | 24 | output "encryption_at_rest" { 25 | description = "A configuration to encrypt at rest in the data catalog." 26 | value = { 27 | enabled = aws_glue_data_catalog_encryption_settings.this.data_catalog_encryption_settings[0].encryption_at_rest[0].catalog_encryption_mode == "SSE-KMS" 28 | kms_key = aws_glue_data_catalog_encryption_settings.this.data_catalog_encryption_settings[0].encryption_at_rest[0].sse_aws_kms_key_id 29 | } 30 | } 31 | 32 | output "sharing" { 33 | description = < 0 ? "SHARED_BY_ME" : "NOT_SHARED" 40 | shares = module.share 41 | } 42 | } 43 | 44 | output "resource_group" { 45 | description = "The resource group created to manage resources in this module." 46 | value = merge( 47 | { 48 | enabled = var.resource_group.enabled && var.module_tags_enabled 49 | }, 50 | (var.resource_group.enabled && var.module_tags_enabled 51 | ? { 52 | arn = module.resource_group[0].arn 53 | name = module.resource_group[0].name 54 | } 55 | : {} 56 | ) 57 | ) 58 | } 59 | -------------------------------------------------------------------------------- /modules/s3-access-point/outputs.tf: -------------------------------------------------------------------------------- 1 | output "name" { 2 | description = "The name of the S3 Access Point." 3 | value = aws_s3_access_point.this.name 4 | } 5 | 6 | output "id" { 7 | description = "The ID of the S3 Access Point." 8 | value = aws_s3_access_point.this.id 9 | } 10 | 11 | output "arn" { 12 | description = "The ARN of the S3 Access Point." 13 | value = aws_s3_access_point.this.arn 14 | } 15 | 16 | output "alias" { 17 | description = "The alias of the S3 Access Point." 18 | value = aws_s3_access_point.this.alias 19 | } 20 | 21 | output "bucket" { 22 | description = "The bucket assoicated to this Access Point." 23 | value = { 24 | name = aws_s3_access_point.this.bucket 25 | account_id = aws_s3_access_point.this.bucket_account_id 26 | } 27 | } 28 | 29 | output "network_origin" { 30 | description = "Indicates whether this access point allows access from the public Internet. Values are `VPC` (the access point doesn't allow access from the public Internet) and `INTERNET` (the access point allows access from the public Internet, subject to the access point and bucket access policies)." 31 | value = upper(aws_s3_access_point.this.network_origin) 32 | } 33 | 34 | output "vpc_id" { 35 | description = "The VPC ID is to be only allowed connections to this access point if `network_origin` is `VPC`." 36 | value = one(aws_s3_access_point.this.vpc_configuration[*].vpc_id) 37 | } 38 | 39 | output "domain_name" { 40 | description = "The DNS domain name of the S3 Access Point in the format `{name-account_id}.s3-accesspoint.{region}.amazonaws.com`. Note: S3 access points only support secure access by HTTPS. HTTP isn't supported." 41 | value = aws_s3_access_point.this.domain_name 42 | } 43 | 44 | output "endpoints" { 45 | description = "The VPC endpoints for the S3 Access Point." 46 | value = aws_s3_access_point.this.endpoints 47 | } 48 | 49 | output "block_public_access" { 50 | description = "The configuration for the S3 bucket access control." 51 | value = { 52 | block_public_acls_enabled = one(aws_s3_access_point.this.public_access_block_configuration[*]).block_public_acls 53 | ignore_public_acls_enabled = one(aws_s3_access_point.this.public_access_block_configuration[*]).ignore_public_acls 54 | block_public_policy_enabled = one(aws_s3_access_point.this.public_access_block_configuration[*]).block_public_policy 55 | restrict_public_buckets_enabled = one(aws_s3_access_point.this.public_access_block_configuration[*]).restrict_public_buckets 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /modules/s3-bucket/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = var.name 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | 18 | ################################################### 19 | # S3 Bucket 20 | ################################################### 21 | 22 | # TODO: website 23 | # TODO: aws_s3_bucket_analytics_configuration 24 | # TODO: aws_s3_bucket_intelligent_tiering_configuration 25 | # TODO: aws_s3_bucket_inventory 26 | # TODO: aws_s3_bucket_notification 27 | # TODO: aws_s3_object 28 | # TODO: aws_s3_bucket_metadata_configuration 29 | resource "aws_s3_bucket" "this" { 30 | region = var.region 31 | 32 | bucket = var.name 33 | force_destroy = var.force_destroy 34 | 35 | object_lock_enabled = var.object_lock.enabled && var.object_lock.token == "" 36 | 37 | tags = merge( 38 | { 39 | "Name" = local.metadata.name 40 | }, 41 | local.module_tags, 42 | var.tags, 43 | ) 44 | 45 | timeouts { 46 | create = var.timeouts.create 47 | read = var.timeouts.read 48 | update = var.timeouts.update 49 | delete = var.timeouts.delete 50 | } 51 | 52 | lifecycle { 53 | ignore_changes = [ 54 | lifecycle_rule, 55 | ] 56 | } 57 | } 58 | 59 | 60 | ################################################### 61 | # Server Side Encryption for S3 Bucket 62 | ################################################### 63 | 64 | locals { 65 | encryption_type = { 66 | "AES256" = "AES256" 67 | "AWS_KMS" = "aws:kms" 68 | "AWS_KMS_DSSE" = "aws:kms::dsse" 69 | } 70 | } 71 | 72 | # INFO: Not supported attributes 73 | # - `expected_bucket_owner` 74 | resource "aws_s3_bucket_server_side_encryption_configuration" "this" { 75 | region = var.region 76 | 77 | bucket = aws_s3_bucket.this.bucket 78 | 79 | rule { 80 | bucket_key_enabled = var.encryption.bucket_key_enabled 81 | 82 | apply_server_side_encryption_by_default { 83 | sse_algorithm = local.encryption_type[var.encryption.type] 84 | kms_master_key_id = var.encryption.kms_key 85 | } 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # terraform-aws-data 2 | 3 | ![GitHub release (latest SemVer)](https://img.shields.io/github/v/release/tedilabs/terraform-aws-data?color=blue&sort=semver&style=flat-square) 4 | ![GitHub](https://img.shields.io/github/license/tedilabs/terraform-aws-data?color=blue&style=flat-square) 5 | [![pre-commit](https://img.shields.io/badge/pre--commit-enabled-brightgreen?logo=pre-commit&logoColor=white&style=flat-square)](https://github.com/pre-commit/pre-commit) 6 | 7 | Terraform module which creates data related resources on AWS. 8 | 9 | - [athena-data-catalog](./modules/athena-data-catalog) 10 | - [athena-workgroup](./modules/athena-workgroup) 11 | - [glue-connection](./modules/glue-connection) 12 | - [glue-crawler](./modules/glue-crawler) 13 | - [glue-data-catalog](./modules/glue-data-catalog) 14 | - [glue-database](./modules/glue-database) 15 | - [glue-table](./modules/glue-table) 16 | - [s3-access-point](./modules/s3-access-point) 17 | - [s3-bucket](./modules/s3-bucket) 18 | 19 | 20 | ## Target AWS Services 21 | 22 | Terraform Modules from [this package](https://github.com/tedilabs/terraform-aws-data) were written to manage the following AWS Services with Terraform. 23 | 24 | - **AWS Athena** 25 | - Data Catalog 26 | - Workgroup 27 | - **AWS Glue** 28 | - Data Catalog 29 | - Connection 30 | - Crawler 31 | - Data Catalog 32 | - Database 33 | - Table 34 | - **AWS S3** 35 | - S3 Bucket 36 | - S3 Access Point 37 | 38 | 39 | ## Examples 40 | 41 | ### Athena 42 | 43 | - [Athena Workgroup](./examples/athena-workgroup) 44 | 45 | ### Glue 46 | 47 | - [Glue Data Catalog (Simple)](./examples/glue-data-catalog-simple) 48 | - [Glue Data Catalog (Full)](./examples/glue-data-catalog-full) 49 | 50 | ### S3 Bucket 51 | 52 | - [Full S3 Bucket](./examples/s3-bucket-full) 53 | - [S3 Bucket with Access Logging](./examples/s3-bucket-access-logging) 54 | - [S3 Bucket with Server-Side Encryption](./examples/s3-bucket-encryption) 55 | - [S3 Bucket with Lifecycle Rules](./examples/s3-bucket-lifecycle-rules) 56 | - [S3 Bucket with Versioning](./examples/s3-bucket-versioning) 57 | 58 | ### S3 Access Point 59 | 60 | - [S3 Access Point (Internet Access)](./examples/s3-access-point-internet) 61 | - [S3 Access Point (VPC Access)](./examples/s3-access-point-vpc) 62 | 63 | 64 | ## Self Promotion 65 | 66 | Like this project? Follow the repository on [GitHub](https://github.com/tedilabs/terraform-aws-data). And if you're feeling especially charitable, follow **[posquit0](https://github.com/posquit0)** on GitHub. 67 | 68 | 69 | ## License 70 | 71 | Provided under the terms of the [Apache License](LICENSE). 72 | 73 | Copyright © 2022-2025, [Byungjin Park](https://www.posquit0.com). -------------------------------------------------------------------------------- /modules/lakeformation-data-catalog/README.md: -------------------------------------------------------------------------------- 1 | # lakeformation-data-catalog 2 | 3 | This module creates following resources. 4 | 5 | - `aws_lakeformation_data_lake_settings` 6 | - `aws_lakeformation_lf_tag` (optional) 7 | 8 | 9 | ## Requirements 10 | 11 | | Name | Version | 12 | |------|---------| 13 | | [terraform](#requirement\_terraform) | >= 1.5 | 14 | | [aws](#requirement\_aws) | >= 4.65 | 15 | 16 | ## Providers 17 | 18 | | Name | Version | 19 | |------|---------| 20 | | [aws](#provider\_aws) | 5.19.0 | 21 | 22 | ## Modules 23 | 24 | No modules. 25 | 26 | ## Resources 27 | 28 | | Name | Type | 29 | |------|------| 30 | | [aws_lakeformation_data_lake_settings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lakeformation_data_lake_settings) | resource | 31 | | [aws_lakeformation_lf_tag.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/lakeformation_lf_tag) | resource | 32 | | [aws_caller_identity.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | 33 | 34 | ## Inputs 35 | 36 | | Name | Description | Type | Default | Required | 37 | |------|-------------|------|---------|:--------:| 38 | | [admins](#input\_admins) | (Optional) A set of ARNs of AWS Lake Formation principals. Administrators can view all metadata in the AWS Glue Data Catalog. They can also grant and revoke permissions on data resources to principals, including themselves. Valid values are following:
- IAM Users and Roles
- Active Directory
- Amazon QuickSight Users and Groups
- Federated Users | `set(string)` | `[]` | no | 39 | | [catalog](#input\_catalog) | (Optional) The ID of the Data Catalog to configure. If omitted, this defaults to the AWS Account ID. | `string` | `null` | no | 40 | | [lf\_tags](#input\_lf\_tags) | (Optional) LF-Tags have a key and one or more values that can be associated with data catalog resources. Tables automatically inherit from database LF-Tags, and columns inherit from table LF-Tags. Each key must have at least one value. The maximum number of values permitted is 15. | `map(set(string))` | `{}` | no | 41 | 42 | ## Outputs 43 | 44 | | Name | Description | 45 | |------|-------------| 46 | | [catalog](#output\_catalog) | The ID of the Data Catalog. | 47 | | [lf\_tags](#output\_lf\_tags) | LF-Tags have a key and one or more values that can be associated with data catalog resources. | 48 | 49 | -------------------------------------------------------------------------------- /modules/glue-table/variables.tf: -------------------------------------------------------------------------------- 1 | variable "catalog" { 2 | description = "(Optional) The ID of the Data Catalog in which to create the table. If omitted, this defaults to the AWS Account ID." 3 | type = string 4 | default = null 5 | } 6 | 7 | variable "database" { 8 | description = "(Required) The name of the metadata database where the table metadata resides. For Hive compatibility, this name must be all lowercase." 9 | type = string 10 | nullable = false 11 | } 12 | 13 | variable "name" { 14 | description = "(Required) The name of the table. For Hive compatibility, this must be entirely lowercase." 15 | type = string 16 | nullable = false 17 | } 18 | 19 | variable "description" { 20 | description = "(Optional) The description of the table." 21 | type = string 22 | default = "Managed by Terraform." 23 | nullable = false 24 | } 25 | 26 | variable "tags" { 27 | description = "(Optional) A map of tags to add to all resources." 28 | type = map(string) 29 | default = {} 30 | nullable = false 31 | } 32 | 33 | variable "module_tags_enabled" { 34 | description = "(Optional) Whether to create AWS Resource Tags for the module informations." 35 | type = bool 36 | default = true 37 | nullable = false 38 | } 39 | 40 | 41 | ################################################### 42 | # Resource Group 43 | ################################################### 44 | 45 | 46 | 47 | 48 | 49 | ################################################### 50 | # Resource Sharing by RAM (Resource Access Manager) 51 | ################################################### 52 | 53 | variable "shares" { 54 | description = "(Optional) A list of resource shares via RAM (Resource Access Manager)." 55 | type = list(object({ 56 | name = optional(string) 57 | 58 | # INFO 59 | # - `AWSRAMDefaultPermissionGlueTable` 60 | # - `AWSRAMPermissionGlueDatabaseReadWriteForTable` 61 | # - `AWSRAMPermissionGlueTableReadWrite` 62 | # - `AWSRAMPermissionLFTagGlueDatabaseReadWriteForTable` 63 | # - `AWSRAMPermissionLFTagGlueTableReadWrite` 64 | permissions = optional(set(string), ["AWSRAMDefaultPermissionGlueTable"]) 65 | 66 | external_principals_allowed = optional(bool, false) 67 | principals = optional(set(string), []) 68 | 69 | tags = optional(map(string), {}) 70 | })) 71 | default = [] 72 | nullable = false 73 | } 74 | 75 | variable "resource_group" { 76 | description = < { 72 | id = query.id 73 | name = query.name 74 | description = query.description 75 | database = query.database 76 | } 77 | } 78 | } 79 | 80 | output "resource_group" { 81 | description = "The resource group created to manage resources in this module." 82 | value = merge( 83 | { 84 | enabled = var.resource_group.enabled && var.module_tags_enabled 85 | }, 86 | (var.resource_group.enabled && var.module_tags_enabled 87 | ? { 88 | arn = module.resource_group[0].arn 89 | name = module.resource_group[0].name 90 | } 91 | : {} 92 | ) 93 | ) 94 | } 95 | -------------------------------------------------------------------------------- /modules/athena-workgroup/main.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | metadata = { 3 | package = "terraform-aws-data" 4 | version = trimspace(file("${path.module}/../../VERSION")) 5 | module = basename(path.module) 6 | name = var.name 7 | } 8 | module_tags = var.module_tags_enabled ? { 9 | "module.terraform.io/package" = local.metadata.package 10 | "module.terraform.io/version" = local.metadata.version 11 | "module.terraform.io/name" = local.metadata.module 12 | "module.terraform.io/full-name" = "${local.metadata.package}/${local.metadata.module}" 13 | "module.terraform.io/instance" = local.metadata.name 14 | } : {} 15 | } 16 | 17 | locals { 18 | query_result_s3_key_prefix = try(trim(var.query_result.s3_key_prefix, "/"), "") 19 | query_result_s3_path = (var.query_result != null 20 | ? "s3://${trim(var.query_result.s3_bucket, "/")}/${local.query_result_s3_key_prefix}" 21 | : null 22 | ) 23 | 24 | engine_versions = { 25 | "AUTO" = "AUTO" 26 | "ATHENA_V2" = "Athena engine version 2" 27 | "ATHENA_V3" = "Athena engine version 3" 28 | } 29 | } 30 | 31 | 32 | resource "aws_athena_workgroup" "this" { 33 | name = var.name 34 | description = var.description 35 | 36 | state = var.enabled ? "ENABLED" : "DISABLED" 37 | force_destroy = var.force_destroy 38 | 39 | configuration { 40 | enforce_workgroup_configuration = !var.client_config_enabled 41 | publish_cloudwatch_metrics_enabled = var.cloudwatch_metrics_enabled 42 | requester_pays_enabled = var.query_on_s3_requester_pays_bucket_enabled 43 | bytes_scanned_cutoff_per_query = var.per_query_data_usage_limit 44 | 45 | engine_version { 46 | selected_engine_version = "AUTO" 47 | } 48 | 49 | dynamic "result_configuration" { 50 | for_each = local.query_result_s3_path != null ? ["go"] : [] 51 | 52 | content { 53 | output_location = local.query_result_s3_path 54 | expected_bucket_owner = try(var.query_result.s3_bucket_expected_owner, null) 55 | 56 | dynamic "encryption_configuration" { 57 | for_each = try(var.query_result.encryption_enabled, false) ? ["go"] : [] 58 | 59 | content { 60 | encryption_option = try(var.query_result.encryption_mode, "SSE_S3") 61 | kms_key_arn = try(var.query_result.encryption_kms_key, null) 62 | } 63 | } 64 | 65 | dynamic "acl_configuration" { 66 | for_each = try(var.query_result.s3_bucket_owner_full_control_enabled, false) ? ["go"] : [] 67 | 68 | content { 69 | s3_acl_option = "BUCKET_OWNER_FULL_CONTROL" 70 | } 71 | } 72 | } 73 | } 74 | } 75 | 76 | tags = merge( 77 | { 78 | "Name" = local.metadata.name 79 | }, 80 | local.module_tags, 81 | var.tags, 82 | ) 83 | } 84 | 85 | 86 | ################################################### 87 | # Named Queries 88 | ################################################### 89 | 90 | resource "aws_athena_named_query" "this" { 91 | for_each = { 92 | for query in var.named_queries : 93 | "${query.database}:${query.name}" => query 94 | } 95 | 96 | workgroup = aws_athena_workgroup.this.id 97 | name = each.value.name 98 | description = try(each.value.description, "Managed by Terraform.") 99 | 100 | database = each.value.database 101 | query = each.value.query 102 | } 103 | -------------------------------------------------------------------------------- /modules/athena-data-catalog/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | description = "(Required) The name of the data catalog. The catalog name must be unique for the AWS account and can use a maximum of 128 alphanumeric, underscore, at sign, or hyphen characters." 3 | type = string 4 | nullable = false 5 | } 6 | 7 | variable "description" { 8 | description = "(Optional) The description of the data catalog." 9 | type = string 10 | default = "Managed by Terraform." 11 | nullable = false 12 | } 13 | 14 | variable "type" { 15 | description = <> $GITHUB_OUTPUT 56 | echo "modified=${{ steps.changed-directories.outputs.any_modified }}" >> $GITHUB_OUTPUT 57 | 58 | echo "changed_files=${{ steps.changed-files.outputs.all_changed_files }}" >> $GITHUB_OUTPUT 59 | echo "modified_files=${{ steps.changed-files.outputs.all_modified_files }}" >> $GITHUB_OUTPUT 60 | 61 | echo "changed_directories=${{ steps.changed-directories.outputs.all_changed_files }}" >> $GITHUB_OUTPUT 62 | echo "modified_directories=${{ steps.changed-directories.outputs.all_modified_files }}" >> $GITHUB_OUTPUT 63 | 64 | 65 | terraform: 66 | name: Lint (terraform) 67 | needs: 68 | - changed 69 | if: ${{ needs.changed.outputs.modified == 'true' }} 70 | uses: tedilabs/.github/.github/workflows/terraform.terraform.yaml@main 71 | 72 | strategy: 73 | matrix: 74 | path: ${{ fromJson(needs.changed.outputs.modified_directories) }} 75 | 76 | with: 77 | terraform_target_dir: ${{ matrix.path }} 78 | terraform_version: latest 79 | terraform_host: app.terraform.io 80 | secrets: 81 | gh_token: ${{ secrets.GITHUB_TOKEN }} 82 | token: ${{ secrets.GITHUB_TOKEN }} 83 | terraform_token: ${{ secrets.TERRAFORM_TOKEN }} 84 | 85 | 86 | tflint: 87 | name: Lint (tflint) 88 | needs: 89 | - changed 90 | if: ${{ needs.changed.outputs.modified == 'true' }} 91 | uses: tedilabs/.github/.github/workflows/terraform.tflint.yaml@main 92 | 93 | strategy: 94 | matrix: 95 | path: ${{ fromJson(needs.changed.outputs.modified_directories) }} 96 | 97 | with: 98 | tflint_version: latest 99 | tflint_config_file: .tflint.hcl 100 | tflint_target_dir: ${{ matrix.path }} 101 | tflint_recursive_enabled: false 102 | tflint_terraform_init_enabled: true 103 | terraform_version: latest 104 | terraform_host: app.terraform.io 105 | secrets: 106 | gh_token: ${{ secrets.GITHUB_TOKEN }} 107 | token: ${{ secrets.GITHUB_TOKEN }} 108 | terraform_token: ${{ secrets.TERRAFORM_TOKEN }} 109 | -------------------------------------------------------------------------------- /examples/s3-bucket-full/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | 6 | ################################################### 7 | # S3 Bucket 8 | ################################################### 9 | 10 | resource "random_string" "this" { 11 | length = 32 12 | special = false 13 | numeric = false 14 | upper = false 15 | } 16 | 17 | locals { 18 | bucket_name = random_string.this.id 19 | } 20 | 21 | module "bucket" { 22 | source = "../../modules/s3-bucket" 23 | # source = "tedilabs/data/aws//modules/s3-bucket" 24 | # version = "~> 0.2.0" 25 | 26 | name = local.bucket_name 27 | force_destroy = true 28 | 29 | 30 | ## Encryption 31 | encryption = { 32 | type = "AWS_KMS" 33 | kms_key = "aws/s3" 34 | bucket_key_enabled = true 35 | } 36 | 37 | ## Lifecycle 38 | versioning = { 39 | status = "ENABLED" 40 | mfa_deletion = { 41 | enabled = false 42 | device = null 43 | } 44 | } 45 | object_lock = { 46 | enabled = true 47 | default_retention = { 48 | mode = "GOVERNANCE" 49 | unit = "DAYS" 50 | value = 1 51 | } 52 | } 53 | lifecycle_rules = [ 54 | { 55 | id = "expiration-1" 56 | enabled = false 57 | prefix = "hello/" 58 | noncurrent_version_expiration = { 59 | days = 1 60 | count = 3 61 | } 62 | }, 63 | { 64 | id = "transitions-1" 65 | enabled = true 66 | tags = { 67 | "Product" = "User" 68 | } 69 | transitions = [ 70 | { 71 | days = 30 72 | storage_class = "STANDARD_IA" 73 | }, 74 | { 75 | days = 60 76 | storage_class = "INTELLIGENT_TIERING" 77 | }, 78 | ] 79 | } 80 | ] 81 | 82 | 83 | ## Access Control 84 | policy = null 85 | object_ownership = "BucketOwnerEnforced" 86 | block_public_access = { 87 | enabled = false 88 | block_public_acls_enabled = true 89 | ignore_public_acls_enabled = true 90 | block_public_policy_enabled = true 91 | restrict_public_buckets_enabled = true 92 | } 93 | cors_rules = [ 94 | { 95 | allowed_methods = ["GET", "POST"] 96 | allowed_origins = ["*"] 97 | max_age = 300 98 | } 99 | ] 100 | tls_required = true 101 | 102 | 103 | ## Logging & Metrics 104 | logging = { 105 | enabled = true 106 | s3_bucket = module.log_bucket.name 107 | s3_key_prefix = "" 108 | } 109 | 110 | request_metrics = [ 111 | { 112 | name = "all" 113 | }, 114 | { 115 | name = "john" 116 | filter = { 117 | prefix = "john/" 118 | } 119 | }, 120 | { 121 | name = "finance" 122 | filter = { 123 | tags = { 124 | "Team" = "Finance" 125 | } 126 | } 127 | }, 128 | { 129 | name = "finance-john" 130 | filter = { 131 | prefix = "john/" 132 | tags = { 133 | "Team" = "Finance" 134 | } 135 | } 136 | }, 137 | ] 138 | 139 | 140 | ## Misc 141 | requester_payment_enabled = true 142 | transfer_acceleration_enabled = true 143 | 144 | tags = { 145 | "project" = "terraform-aws-data-examples" 146 | } 147 | } 148 | 149 | module "log_bucket" { 150 | source = "../../modules/s3-bucket" 151 | # source = "tedilabs/data/aws//modules/s3-bucket" 152 | # version = "~> 0.2.0" 153 | 154 | name = "${local.bucket_name}-log" 155 | force_destroy = true 156 | 157 | logging = { 158 | is_target_bucket = true 159 | allowed_source_buckets = ["${local.bucket_name}-*"] 160 | } 161 | 162 | tags = { 163 | "project" = "terraform-aws-data-examples" 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /modules/glue-data-catalog/variables.tf: -------------------------------------------------------------------------------- 1 | variable "catalog" { 2 | description = "(Optional) The ID of the Data Catalog in which to set the security configuration for. If omitted, this defaults to the AWS Account ID." 3 | type = string 4 | default = null 5 | } 6 | 7 | variable "policy" { 8 | description = "(Optional) The policy to be applied to the aws glue data catalog for access control." 9 | type = string 10 | default = "" 11 | nullable = false 12 | } 13 | 14 | variable "encryption_for_connection_passwords" { 15 | description = < 8 | ## Requirements 9 | 10 | | Name | Version | 11 | |------|---------| 12 | | [terraform](#requirement\_terraform) | >= 1.5 | 13 | | [aws](#requirement\_aws) | >= 4.65 | 14 | 15 | ## Providers 16 | 17 | | Name | Version | 18 | |------|---------| 19 | | [aws](#provider\_aws) | 5.19.0 | 20 | 21 | ## Modules 22 | 23 | | Name | Source | Version | 24 | |------|--------|---------| 25 | | [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | 26 | 27 | ## Resources 28 | 29 | | Name | Type | 30 | |------|------| 31 | | [aws_athena_data_catalog.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_data_catalog) | resource | 32 | 33 | ## Inputs 34 | 35 | | Name | Description | Type | Default | Required | 36 | |------|-------------|------|---------|:--------:| 37 | | [name](#input\_name) | (Required) The name of the data catalog. The catalog name must be unique for the AWS account and can use a maximum of 128 alphanumeric, underscore, at sign, or hyphen characters. | `string` | n/a | yes | 38 | | [type](#input\_type) | (Required) A type of the data catalog. Valid values are `LAMBDA`, `GLUE`, `HIVE`.
- `LAMBDA` for a federated catalog.
- `GLUE` for an AWS Glue Data Catalog.
- `HIVE` for an external hive metastore. | `string` | n/a | yes | 39 | | [description](#input\_description) | (Optional) The description of the data catalog. | `string` | `"Managed by Terraform."` | no | 40 | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | 41 | | [parameters](#input\_parameters) | (Optional) A set of key value pairs that specifies the Lambda function or functions to use for creating the data catalog. The mapping used depends on the catalog type.

`LAMBDA`
Use one of the following sets of required parameters, but not both.
- If you have one Lambda function that processes metadata and another for reading the actual data.
(Required) `metadata-function` - The ARN of Lambda function to process metadata.
(Required) `record-function` - The ARN of Lambda function to read the actual data.
- If you have a composite Lambda function that processes both metadata and data.
(Required) `function` - The ARN of a composite Lambda function to process both metadata and data.

`GLUE`
(Required) `catalog_id` - The account ID of the AWS account to which the Glue Data Catalog belongs.

`HIVE`
(Required) `metadata-function` - The ARN of Lambda function to process metadata.
(Optional) `sdk-version` - Defaults to the currently supported version. | `map(string)` | `{}` | no | 42 | | [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | 43 | | [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | 44 | | [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | 45 | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | 46 | 47 | ## Outputs 48 | 49 | | Name | Description | 50 | |------|-------------| 51 | | [arn](#output\_arn) | The Amazon Resource Name (ARN) of the data catalog. | 52 | | [description](#output\_description) | The description of the data catalog. | 53 | | [id](#output\_id) | The ID of the data catalog. | 54 | | [name](#output\_name) | The name of the data catalog. | 55 | | [parameters](#output\_parameters) | A set of key value pairs that specifies the Lambda function or functions to use for creating the data catalog. | 56 | | [type](#output\_type) | The type of the data catalog. | 57 | 58 | -------------------------------------------------------------------------------- /modules/s3-bucket/access-control.tf: -------------------------------------------------------------------------------- 1 | data "aws_canonical_user_id" "this" {} 2 | 3 | locals { 4 | default_grants = [ 5 | { 6 | type = "CanonicalUser" 7 | grantee = data.aws_canonical_user_id.this.id 8 | permission = "FULL_CONTROL" 9 | } 10 | ] 11 | 12 | grants = concat( 13 | local.default_grants, 14 | var.grants 15 | ) 16 | } 17 | 18 | 19 | ################################################### 20 | # Policy for S3 Bucket 21 | ################################################### 22 | 23 | data "aws_iam_policy_document" "this" { 24 | source_policy_documents = concat( 25 | var.tls_required ? [data.aws_iam_policy_document.tls_required.json] : [], 26 | var.logging.is_target_bucket ? [data.aws_iam_policy_document.access_logging.json] : [], 27 | ) 28 | override_policy_documents = var.policy != null ? [var.policy] : null 29 | } 30 | 31 | resource "aws_s3_bucket_policy" "this" { 32 | region = var.region 33 | 34 | bucket = aws_s3_bucket.this.id 35 | policy = data.aws_iam_policy_document.this.json 36 | } 37 | 38 | 39 | ################################################### 40 | # Object Ownership for S3 Bucket 41 | ################################################### 42 | 43 | resource "aws_s3_bucket_ownership_controls" "this" { 44 | region = var.region 45 | 46 | bucket = aws_s3_bucket.this.bucket 47 | 48 | rule { 49 | object_ownership = var.object_ownership 50 | } 51 | } 52 | 53 | 54 | ################################################### 55 | # ACL for S3 Bucket 56 | ################################################### 57 | 58 | # INFO: Not supported attributes 59 | # - `expected_bucket_owner` 60 | # - `acl` 61 | # - `access_control_policy.owner.display_name` 62 | resource "aws_s3_bucket_acl" "this" { 63 | count = var.object_ownership != "BucketOwnerEnforced" ? 1 : 0 64 | 65 | region = var.region 66 | 67 | bucket = aws_s3_bucket.this.bucket 68 | 69 | access_control_policy { 70 | dynamic "grant" { 71 | for_each = local.grants 72 | 73 | content { 74 | grantee { 75 | type = grant.value.type 76 | id = grant.value.type == "CanonicalUser" ? grant.value.grantee : null 77 | uri = grant.value.type == "Group" ? grant.value.grantee : null 78 | email_address = grant.value.type == "AmazonCustomerByEmail" ? grant.value.grantee : null 79 | } 80 | permission = grant.value.permission 81 | } 82 | } 83 | 84 | owner { 85 | id = data.aws_canonical_user_id.this.id 86 | } 87 | } 88 | } 89 | 90 | 91 | ################################################### 92 | # Public Access Block for S3 Bucket 93 | ################################################### 94 | 95 | resource "aws_s3_bucket_public_access_block" "this" { 96 | region = var.region 97 | 98 | bucket = aws_s3_bucket.this.bucket 99 | 100 | # Block new public ACLs and uploading public objects 101 | block_public_acls = (var.block_public_access.enabled 102 | || var.block_public_access.block_public_acls_enabled) 103 | # Retroactively remove public access granted through public ACLs 104 | ignore_public_acls = (var.block_public_access.enabled 105 | || var.block_public_access.ignore_public_acls_enabled) 106 | # Block new public bucket policies 107 | block_public_policy = (var.block_public_access.enabled 108 | || var.block_public_access.block_public_policy_enabled) 109 | # Retroactivley block public and cross-account access if bucket has public policies 110 | restrict_public_buckets = (var.block_public_access.enabled 111 | || var.block_public_access.restrict_public_buckets_enabled) 112 | 113 | skip_destroy = true 114 | 115 | # To avoid OperationAborted: A conflicting conditional operation is currently in progress 116 | depends_on = [ 117 | aws_s3_bucket_policy.this, 118 | ] 119 | } 120 | 121 | 122 | ################################################### 123 | # CORS (Cross-Origin Resource Sharing) for S3 Bucket 124 | ################################################### 125 | 126 | # INFO: Not supported attributes 127 | # - `expected_bucket_owner` 128 | resource "aws_s3_bucket_cors_configuration" "this" { 129 | count = length(var.cors_rules) > 0 ? 1 : 0 130 | 131 | region = var.region 132 | 133 | bucket = aws_s3_bucket.this.bucket 134 | 135 | dynamic "cors_rule" { 136 | for_each = var.cors_rules 137 | 138 | content { 139 | id = coalesce(cors_rule.value.id, cors_rule.key) 140 | 141 | allowed_headers = cors_rule.value.allowed_headers 142 | allowed_methods = cors_rule.value.allowed_methods 143 | allowed_origins = cors_rule.value.allowed_origins 144 | expose_headers = cors_rule.value.expose_headers 145 | max_age_seconds = cors_rule.value.max_age 146 | } 147 | } 148 | } 149 | -------------------------------------------------------------------------------- /modules/glue-database/variables.tf: -------------------------------------------------------------------------------- 1 | variable "catalog" { 2 | description = "(Optional) The ID of the Data Catalog in which to create the database. If omitted, this defaults to the AWS Account ID." 3 | type = string 4 | default = null 5 | } 6 | 7 | variable "name" { 8 | description = "(Required) The name of the database. For Hive compatibility, this is folded to lowercase when it is stored. If you plan to access the database from Amazon Athena, then provide a name with only alphanumeric and underscore characters. The acceptable characters are lowercase letters, numbers, and the underscore character." 9 | type = string 10 | nullable = false 11 | } 12 | 13 | variable "description" { 14 | description = "(Optional) The description of the database." 15 | type = string 16 | default = "Managed by Terraform." 17 | nullable = false 18 | } 19 | 20 | variable "location_uri" { 21 | description = "(Optional) The location of the database. For example, an HDFS path." 22 | type = string 23 | default = "" 24 | nullable = false 25 | } 26 | 27 | variable "parameters" { 28 | description = < 11 | ## Requirements 12 | 13 | | Name | Version | 14 | |------|---------| 15 | | [terraform](#requirement\_terraform) | >= 1.5 | 16 | | [aws](#requirement\_aws) | >= 4.65 | 17 | 18 | ## Providers 19 | 20 | | Name | Version | 21 | |------|---------| 22 | | [aws](#provider\_aws) | 5.19.0 | 23 | 24 | ## Modules 25 | 26 | | Name | Source | Version | 27 | |------|--------|---------| 28 | | [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | 29 | | [share](#module\_share) | tedilabs/account/aws//modules/ram-share | ~> 0.22.0 | 30 | 31 | ## Resources 32 | 33 | | Name | Type | 34 | |------|------| 35 | | [aws_glue_catalog_table.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_catalog_table) | resource | 36 | 37 | ## Inputs 38 | 39 | | Name | Description | Type | Default | Required | 40 | |------|-------------|------|---------|:--------:| 41 | | [database](#input\_database) | (Required) The name of the metadata database where the table metadata resides. For Hive compatibility, this name must be all lowercase. | `string` | n/a | yes | 42 | | [name](#input\_name) | (Required) The name of the table. For Hive compatibility, this must be entirely lowercase. | `string` | n/a | yes | 43 | | [catalog](#input\_catalog) | (Optional) The ID of the Data Catalog in which to create the table. If omitted, this defaults to the AWS Account ID. | `string` | `null` | no | 44 | | [description](#input\_description) | (Optional) The description of the table. | `string` | `"Managed by Terraform."` | no | 45 | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | 46 | | [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | 47 | | [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | 48 | | [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | 49 | | [shares](#input\_shares) | (Optional) A list of resource shares via RAM (Resource Access Manager). | 
list(object({
    name = optional(string)

    # INFO
    # - `AWSRAMDefaultPermissionGlueTable`
    # - `AWSRAMPermissionGlueDatabaseReadWriteForTable`
    # - `AWSRAMPermissionGlueTableReadWrite`
    # - `AWSRAMPermissionLFTagGlueDatabaseReadWriteForTable`
    # - `AWSRAMPermissionLFTagGlueTableReadWrite`
    permissions = optional(set(string), ["AWSRAMDefaultPermissionGlueTable"])

    external_principals_allowed = optional(bool, false)
    principals                  = optional(set(string), [])

    tags = optional(map(string), {})
  }))
| `[]` | no | 50 | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | 51 | 52 | ## Outputs 53 | 54 | | Name | Description | 55 | |------|-------------| 56 | | [arn](#output\_arn) | The Amazon Resource Name (ARN) of the Glue Catalog Table. | 57 | | [catalog](#output\_catalog) | The ID of the Glue Catalog of the table. | 58 | | [database](#output\_database) | The catalog database in which to create the new table. | 59 | | [description](#output\_description) | The description of the table. | 60 | | [id](#output\_id) | The ID of the database. | 61 | | [name](#output\_name) | The name of the table. | 62 | | [sharing](#output\_sharing) | The configuration for sharing of the Glue Table.
`status` - An indication of whether the table is shared with other AWS accounts, or was shared with the current account by another AWS account. Sharing is configured through AWS Resource Access Manager (AWS RAM). Values are `NOT_SHARED`, `SHARED_BY_ME` or `SHARED_WITH_ME`.
`shares` - The list of resource shares via RAM (Resource Access Manager). | 63 | 64 | -------------------------------------------------------------------------------- /examples/glue-data-catalog-full/main.tf: -------------------------------------------------------------------------------- 1 | provider "aws" { 2 | region = "us-east-1" 3 | } 4 | 5 | data "aws_caller_identity" "this" {} 6 | data "aws_partition" "this" {} 7 | data "aws_region" "this" {} 8 | 9 | locals { 10 | partition = data.aws_partition.this.partition 11 | region = data.aws_region.this.name 12 | account_id = data.aws_caller_identity.this.account_id 13 | } 14 | 15 | data "aws_iam_policy_document" "this" { 16 | statement { 17 | actions = [ 18 | "glue:*", 19 | ] 20 | resources = [ 21 | "arn:${local.partition}:glue:${local.region}:${local.account_id}:catalog", 22 | "arn:${local.partition}:glue:${local.region}:${local.account_id}:database/*", 23 | "arn:${local.partition}:glue:${local.region}:${local.account_id}:table/*/*" 24 | ] 25 | principals { 26 | identifiers = ["*"] 27 | type = "AWS" 28 | } 29 | } 30 | 31 | statement { 32 | actions = [ 33 | "glue:ShareResource" 34 | ] 35 | resources = [ 36 | "arn:${local.partition}:glue:${local.region}:${local.account_id}:catalog", 37 | "arn:${local.partition}:glue:${local.region}:${local.account_id}:database/*", 38 | "arn:${local.partition}:glue:${local.region}:${local.account_id}:table/*/*" 39 | ] 40 | principals { 41 | identifiers = ["ram.amazonaws.com"] 42 | type = "Service" 43 | } 44 | } 45 | } 46 | 47 | locals { 48 | databases = [ 49 | { 50 | name = "foo" 51 | description = "Hello, foo" 52 | tables = [ 53 | { 54 | name = "foo1" 55 | descrition = "Hey, foo1" 56 | }, 57 | { 58 | name = "foo2" 59 | descrition = "Hey, foo2" 60 | }, 61 | ] 62 | }, 63 | { 64 | name = "bar" 65 | description = "Hello, bar" 66 | tables = [ 67 | { 68 | name = "bar1" 69 | descrition = "Hey, bar1" 70 | }, 71 | { 72 | name = "bar2" 73 | descrition = "Hey, bar2" 74 | }, 75 | ] 76 | } 77 | ] 78 | tables = flatten([ 79 | for database in try(local.databases, []) : [ 80 | for table in try(database.tables, []) : 81 | merge(table, { 82 | database = database.name 83 | }) 84 | ] 85 | ]) 86 | } 87 | 88 | ################################################### 89 | # KMS Keys for Glue 90 | ################################################### 91 | 92 | module "kms_key" { 93 | source = "tedilabs/secret/aws//modules/kms-key" 94 | version = "~> 0.3.0" 95 | 96 | name = "example-glue" 97 | aliases = ["alias/example-glue"] 98 | description = "Managed by Terraform." 99 | 100 | usage = "ENCRYPT_DECRYPT" 101 | spec = "SYMMETRIC_DEFAULT" 102 | 103 | policy = null 104 | 105 | deletion_window_in_days = 7 106 | 107 | tags = { 108 | "project" = "terraform-aws-data-examples" 109 | } 110 | } 111 | 112 | 113 | ################################################### 114 | # Glue Data Catalog 115 | ################################################### 116 | 117 | module "data_catalog" { 118 | source = "../../modules/glue-data-catalog" 119 | # source = "tedilabs/data/aws//modules/glue-data-catalog" 120 | # version = "~> 0.2.0" 121 | 122 | policy = data.aws_iam_policy_document.this.json 123 | 124 | encryption_for_connection_passwords = { 125 | enabled = true 126 | kms_key = module.kms_key.arn 127 | } 128 | encryption_at_rest = { 129 | enabled = true 130 | kms_key = module.kms_key.arn 131 | } 132 | 133 | tags = { 134 | "project" = "terraform-aws-data-examples" 135 | } 136 | } 137 | 138 | 139 | ################################################### 140 | # Glue Database 141 | ################################################### 142 | 143 | module "database" { 144 | source = "../../modules/glue-database" 145 | # source = "tedilabs/data/aws//modules/glue-database" 146 | # version = "~> 0.2.0" 147 | 148 | for_each = { 149 | for database in try(local.databases, []) : 150 | database.name => database 151 | } 152 | 153 | catalog = try(each.value.catalog, null) 154 | 155 | name = each.key 156 | description = try(each.value.description, "Managed by Terraform.") 157 | location_uri = try(each.value.location_uri, "") 158 | 159 | tags = { 160 | "project" = "terraform-aws-data-examples" 161 | } 162 | } 163 | 164 | 165 | ################################################### 166 | # Glue Table 167 | ################################################### 168 | 169 | module "table" { 170 | source = "../../modules/glue-table" 171 | # source = "tedilabs/data/aws//modules/glue-table" 172 | # version = "~> 0.2.0" 173 | 174 | for_each = { 175 | for table in try(local.tables, []) : 176 | "${table.database}/${table.name}" => table 177 | } 178 | 179 | catalog = try(each.value.catalog, null) 180 | database = module.database[each.value.database].name 181 | 182 | name = each.value.name 183 | description = try(each.value.description, "Managed by Terraform.") 184 | 185 | tags = { 186 | "project" = "terraform-aws-data-examples" 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /modules/athena-workgroup/variables.tf: -------------------------------------------------------------------------------- 1 | variable "name" { 2 | description = "(Required) The name of the workgroup." 3 | type = string 4 | } 5 | 6 | variable "description" { 7 | description = "(Optional) The description of the workgroup." 8 | type = string 9 | default = "Managed by Terraform." 10 | } 11 | 12 | variable "enabled" { 13 | description = "(Optional) Whether to enable the workgroup. Defaults to `true`." 14 | type = bool 15 | default = true 16 | } 17 | 18 | variable "force_destroy" { 19 | description = "(Optional) Whether to delete the workgroup and its contents even if the workgroup contains any named queries." 20 | type = bool 21 | default = false 22 | } 23 | 24 | variable "client_config_enabled" { 25 | description = "(Optional) Whether overriding workgroup configurations with client-side configurations is allowed. Defaults to `false`." 26 | type = bool 27 | default = false 28 | } 29 | 30 | variable "cloudwatch_metrics_enabled" { 31 | description = "(Optional) Whether Amazon CloudWatch metrics are enabled for the workgroup. Defaults to `true`." 32 | type = bool 33 | default = true 34 | } 35 | 36 | variable "query_on_s3_requester_pays_bucket_enabled" { 37 | description = "(Optional) Whether to allow members assigned to a workgroup to reference Amazon S3 Requester Pays buckets in queries. If set to false, workgroup members cannot query data from Requester Pays buckets, and queries that retrieve data from Requester Pays buckets cause an error. Defaults to `false`." 38 | type = bool 39 | default = false 40 | } 41 | 42 | variable "per_query_data_usage_limit" { 43 | description = "(Optional) Sets the limit in bytes for the maximum amount of data a query is allowed to scan. You can set only one per query limit for a workgroup. The limit applies to all queries in the workgroup and if query exceeds the limit, it will be cancelled. Minimum limit is 10 MB and maximum limit is 7EB per workgroup." 44 | type = number 45 | default = null 46 | } 47 | 48 | variable "query_result" { 49 | description = < 9 | ## Requirements 10 | 11 | | Name | Version | 12 | |------|---------| 13 | | [terraform](#requirement\_terraform) | >= 1.5 | 14 | | [aws](#requirement\_aws) | >= 4.49 | 15 | 16 | ## Providers 17 | 18 | | Name | Version | 19 | |------|---------| 20 | | [aws](#provider\_aws) | 5.19.0 | 21 | 22 | ## Modules 23 | 24 | No modules. 25 | 26 | ## Resources 27 | 28 | | Name | Type | 29 | |------|------| 30 | | [aws_s3_access_point.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_access_point) | resource | 31 | | [aws_s3control_access_point_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3control_access_point_policy) | resource | 32 | 33 | ## Inputs 34 | 35 | | Name | Description | Type | Default | Required | 36 | |------|-------------|------|---------|:--------:| 37 | | [bucket](#input\_bucket) | (Required) A configuration of S3 bucket to associate with this access point.
(Required) `name` - A name of an AWS Partition S3 Bucket or the Amazon Resource Name (ARN) of S3 on Outposts Bucket that you want to associate this access point with.
(Optional) `account_id` - The AWS account ID associated with the S3 bucket associated with this access point. Only used for creating an access point by using a bucket in a different AWS account. | 
object({
    name       = string
    account_id = optional(string)
  })
| n/a | yes | 38 | | [name](#input\_name) | (Required) A name you want to assign to this access point. Access point names must be unique within the account for this Region. | `string` | n/a | yes | 39 | | [block\_public\_access](#input\_block\_public\_access) | (Optional) A configurations of Block Public Access for the S3 access point. Public access is granted to buckets and objects through access control lists (ACLs), bucket policies, access point policies, or all. These settings apply only to this Access Point. Before applying these settings, ensure that your applications will work correctly without public access. These settings can’t be edited after the Access Point is created.
(Optional) `enabled` - Whether to block all public access to S3 access point. Defaults to `true`.
(Optional) `block_public_acls_enabled` - Block new public ACLs and uploading public objects if true. Always enabled if `block_public_access.enabled` is `true`.
(Optional) `ignore_public_acls_enabled` - Retroactively remove public access granted through public ACLs. Always enabled if `block_public_access.enabled` is `true`.
(Optional) `block_public_policy_enabled` - Block new public bucket policies. Always enabled if `block_public_access.enabled` is `true`.
(Optional) `restrict_public_buckets_enabled` - Retroactivley block public and cross-account access if bucket has public policies. Always enabled if `block_public_access.enabled` is `true`. | 
object({
    enabled                         = optional(bool, true)
    block_public_acls_enabled       = optional(bool, false)
    ignore_public_acls_enabled      = optional(bool, false)
    block_public_policy_enabled     = optional(bool, false)
    restrict_public_buckets_enabled = optional(bool, false)
  })
| `{}` | no | 40 | | [network\_origin](#input\_network\_origin) | (Optional) Indicates whether this access point allows access from the public Internet. Valid Values are `VPC` (the access point doesn't allow access from the public Internet) and `INTERNET` (the access point allows access from the public Internet, subject to the access point and bucket access policies). Defaults to `INTERNET`. | `string` | `"INTERNET"` | no | 41 | | [policy](#input\_policy) | (Optional) A valid policy JSON document that specifies the policy for this access point. | `string` | `""` | no | 42 | | [vpc\_id](#input\_vpc\_id) | (Optional) The VPC ID to be only allowed connections to this access point. Only required if `network_origin` is `VPC`. | `string` | `null` | no | 43 | 44 | ## Outputs 45 | 46 | | Name | Description | 47 | |------|-------------| 48 | | [alias](#output\_alias) | The alias of the S3 Access Point. | 49 | | [arn](#output\_arn) | The ARN of the S3 Access Point. | 50 | | [block\_public\_access](#output\_block\_public\_access) | The configuration for the S3 bucket access control. | 51 | | [bucket](#output\_bucket) | The bucket assoicated to this Access Point. | 52 | | [domain\_name](#output\_domain\_name) | The DNS domain name of the S3 Access Point in the format `{name-account_id}.s3-accesspoint.{region}.amazonaws.com`. Note: S3 access points only support secure access by HTTPS. HTTP isn't supported. | 53 | | [endpoints](#output\_endpoints) | The VPC endpoints for the S3 Access Point. | 54 | | [id](#output\_id) | The ID of the S3 Access Point. | 55 | | [name](#output\_name) | The name of the S3 Access Point. | 56 | | [network\_origin](#output\_network\_origin) | Indicates whether this access point allows access from the public Internet. Values are `VPC` (the access point doesn't allow access from the public Internet) and `INTERNET` (the access point allows access from the public Internet, subject to the access point and bucket access policies). | 57 | | [vpc\_id](#output\_vpc\_id) | The VPC ID is to be only allowed connections to this access point if `network_origin` is `VPC`. | 58 | 59 | -------------------------------------------------------------------------------- /modules/glue-database/README.md: -------------------------------------------------------------------------------- 1 | # glue-database 2 | 3 | This module creates following resources. 4 | 5 | - `aws_glue_catalog_database` 6 | - `aws_ram_resource_share` (optional) 7 | - `aws_ram_principal_association` (optional) 8 | - `aws_ram_resource_association` (optional) 9 | 10 | 11 | ## Requirements 12 | 13 | | Name | Version | 14 | |------|---------| 15 | | [terraform](#requirement\_terraform) | >= 1.5 | 16 | | [aws](#requirement\_aws) | >= 4.65 | 17 | 18 | ## Providers 19 | 20 | | Name | Version | 21 | |------|---------| 22 | | [aws](#provider\_aws) | 5.19.0 | 23 | 24 | ## Modules 25 | 26 | | Name | Source | Version | 27 | |------|--------|---------| 28 | | [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | 29 | | [share](#module\_share) | tedilabs/account/aws//modules/ram-share | ~> 0.22.0 | 30 | 31 | ## Resources 32 | 33 | | Name | Type | 34 | |------|------| 35 | | [aws_glue_catalog_database.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_catalog_database) | resource | 36 | 37 | ## Inputs 38 | 39 | | Name | Description | Type | Default | Required | 40 | |------|-------------|------|---------|:--------:| 41 | | [name](#input\_name) | (Required) The name of the database. For Hive compatibility, this is folded to lowercase when it is stored. If you plan to access the database from Amazon Athena, then provide a name with only alphanumeric and underscore characters. The acceptable characters are lowercase letters, numbers, and the underscore character. | `string` | n/a | yes | 42 | | [catalog](#input\_catalog) | (Optional) The ID of the Data Catalog in which to create the database. If omitted, this defaults to the AWS Account ID. | `string` | `null` | no | 43 | | [description](#input\_description) | (Optional) The description of the database. | `string` | `"Managed by Terraform."` | no | 44 | | [location\_uri](#input\_location\_uri) | (Optional) The location of the database. For example, an HDFS path. | `string` | `""` | no | 45 | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | 46 | | [parameters](#input\_parameters) | (Optional) A set of key value pairs that specifies the Lambda function or functions to use for creating the data catalog. The mapping used depends on the catalog type.

`LAMBDA`
Use one of the following sets of required parameters, but not both.
- If you have one Lambda function that processes metadata and another for reading the actual data.
(Required) `metadata-function` - The ARN of Lambda function to process metadata.
(Required) `record-function` - The ARN of Lambda function to read the actual data.
- If you have a composite Lambda function that processes both metadata and data.
(Required) `function` - The ARN of a composite Lambda function to process both metadata and data.

`GLUE`
(Required) `catalog_id` - The account ID of the AWS account to which the Glue Data Catalog belongs.

`HIVE`
(Required) `metadata-function` - The ARN of Lambda function to process metadata.
(Optional) `sdk-version` - Defaults to the currently supported version. | `map(string)` | `{}` | no | 47 | | [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | 48 | | [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | 49 | | [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | 50 | | [shares](#input\_shares) | (Optional) A list of resource shares via RAM (Resource Access Manager). | 
list(object({
    name = optional(string)

    # INFO
    # - `AWSRAMDefaultPermissionGlueDatabase`
    # - `AWSRAMPermissionGlueAllTablesReadWriteForDatabase`
    # - `AWSRAMPermissionGlueDatabaseReadWrite`
    # - `AWSRAMPermissionGlueTableReadWriteForDatabase`
    # - `AWSRAMPermissionLFTagGlueDatabaseReadWrite`
    # - `AWSRAMPermissionLFTagGlueTableReadWriteForDatabase`
    permissions = optional(set(string), ["AWSRAMDefaultPermissionGlueDatabase"])

    external_principals_allowed = optional(bool, false)
    principals                  = optional(set(string), [])

    tags = optional(map(string), {})
  }))
| `[]` | no | 51 | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | 52 | 53 | ## Outputs 54 | 55 | | Name | Description | 56 | |------|-------------| 57 | | [arn](#output\_arn) | The Amazon Resource Name (ARN) of the Glue Catalog Database. | 58 | | [catalog](#output\_catalog) | The ID of the Glue Catalog of the database. | 59 | | [description](#output\_description) | The description of the database. | 60 | | [id](#output\_id) | The ID of the database. | 61 | | [location\_uri](#output\_location\_uri) | The description of the database. | 62 | | [name](#output\_name) | The name of the database. | 63 | | [parameters](#output\_parameters) | A set of key value pairs that specifies the Lambda function or functions to use for creating the data catalog. | 64 | | [sharing](#output\_sharing) | The configuration for sharing of the Glue Database.
`status` - An indication of whether the database is shared with other AWS accounts, or was shared with the current account by another AWS account. Sharing is configured through AWS Resource Access Manager (AWS RAM). Values are `NOT_SHARED`, `SHARED_BY_ME` or `SHARED_WITH_ME`.
`shares` - The list of resource shares via RAM (Resource Access Manager). | 65 | 66 | -------------------------------------------------------------------------------- /modules/glue-crawler/outputs.tf: -------------------------------------------------------------------------------- 1 | output "arn" { 2 | description = "The Amazon Resource Name (ARN) of the Glue crawler." 3 | value = aws_glue_crawler.this.arn 4 | } 5 | 6 | output "id" { 7 | description = "The ID of the Glue crawler." 8 | value = aws_glue_crawler.this.id 9 | } 10 | 11 | output "name" { 12 | description = "The name of the Glue crawler." 13 | value = aws_glue_crawler.this.name 14 | } 15 | 16 | output "description" { 17 | description = "The description of the Glue crawler." 18 | value = aws_glue_crawler.this.description 19 | } 20 | 21 | output "configuration" { 22 | description = "The configuration information of the crawler as JSON string." 23 | value = aws_glue_crawler.this.configuration 24 | } 25 | 26 | output "data_lineage_enabled" { 27 | description = "Whether data lineage is enabled for the crawler." 28 | value = aws_glue_crawler.this.lineage_configuration[0].crawler_lineage_settings == "ENABLE" 29 | } 30 | 31 | output "database" { 32 | description = "The Glue database where results are written." 33 | value = aws_glue_crawler.this.database_name 34 | } 35 | 36 | output "table_prefix" { 37 | description = "The table prefix used for catalog tables that are created." 38 | value = aws_glue_crawler.this.table_prefix 39 | } 40 | 41 | output "classifiers" { 42 | description = "A list of custom classifiers to use with this crawler." 43 | value = aws_glue_crawler.this.classifiers 44 | } 45 | 46 | output "data_sources" { 47 | description = < 12 | ## Requirements 13 | 14 | | Name | Version | 15 | |------|---------| 16 | | [terraform](#requirement\_terraform) | >= 1.5 | 17 | | [aws](#requirement\_aws) | >= 4.65 | 18 | 19 | ## Providers 20 | 21 | | Name | Version | 22 | |------|---------| 23 | | [aws](#provider\_aws) | 5.19.0 | 24 | 25 | ## Modules 26 | 27 | | Name | Source | Version | 28 | |------|--------|---------| 29 | | [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | 30 | | [share](#module\_share) | tedilabs/account/aws//modules/ram-share | ~> 0.22.0 | 31 | 32 | ## Resources 33 | 34 | | Name | Type | 35 | |------|------| 36 | | [aws_glue_data_catalog_encryption_settings.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_data_catalog_encryption_settings) | resource | 37 | | [aws_glue_resource_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_resource_policy) | resource | 38 | | [aws_caller_identity.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | 39 | | [aws_partition.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source | 40 | | [aws_region.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | 41 | 42 | ## Inputs 43 | 44 | | Name | Description | Type | Default | Required | 45 | |------|-------------|------|---------|:--------:| 46 | | [catalog](#input\_catalog) | (Optional) The ID of the Data Catalog in which to set the security configuration for. If omitted, this defaults to the AWS Account ID. | `string` | `null` | no | 47 | | [encryption\_at\_rest](#input\_encryption\_at\_rest) | (Optional) A configuration to encrypt at rest in the data catalog. When enabled, metadata stored in the data catalog is encrypted at rest.
(Optional) `enabled` - Whether to enable encryption at rest. Defaults to `false`.
(Optional) `kms_key` - The ID of AWS KMS key used for the encryption. | 
object({
    enabled = optional(bool, false)
    kms_key = optional(string)
  })
| `{}` | no | 48 | | [encryption\_for\_connection\_passwords](#input\_encryption\_for\_connection\_passwords) | (Optional) A configuration to encrypt connection passwords in the data catalog. When enabled, the password you provide when you create a connection is encrypted with the given AWS KMS key.
(Optional) `enabled` - Whether to enable encryption for connection passwords. Defaults to `false`.
(Optional) `kms_key` - The ID of AWS KMS key used for the encryption. If connection password protection is enabled, the caller of `CreateConnection` and `UpdateConnection` needs at least `kms:Encrypt` permission on the specified KMS key, to encrypt passwords before storing them in the Data Catalog. You can set the decrypt permission to enable or restrict access on the password key according to your security requirements. | 
object({
    enabled = optional(bool, false)
    kms_key = optional(string)
  })
| `{}` | no | 49 | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | 50 | | [policy](#input\_policy) | (Optional) The policy to be applied to the aws glue data catalog for access control. | `string` | `""` | no | 51 | | [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | 52 | | [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | 53 | | [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | 54 | | [shares](#input\_shares) | (Optional) A list of resource shares via RAM (Resource Access Manager). | 
list(object({
    name = optional(string)

    # INFO
    # - `AWSRAMDefaultPermissionGlueCatalog`
    # - `AWSRAMPermissionGlueAllTablesReadWriteForCatalog`
    # - `AWSRAMPermissionGlueDatabaseReadWriteForCatalog`
    # - `AWSRAMPermissionGlueTableReadWriteForCatalog`
    # - `AWSRAMPermissionLFTagGlueDatabaseReadWriteForCatalog`
    # - `AWSRAMPermissionLFTagGlueTableReadWriteForCatalog`
    permissions = optional(set(string), ["AWSRAMDefaultPermissionGlueCatalog"])

    external_principals_allowed = optional(bool, false)
    principals                  = optional(set(string), [])

    tags = optional(map(string), {})
  }))
| `[]` | no | 55 | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | 56 | 57 | ## Outputs 58 | 59 | | Name | Description | 60 | |------|-------------| 61 | | [arn](#output\_arn) | The Amazon Resource Name (ARN) of the Glue Catalog. | 62 | | [catalog](#output\_catalog) | The ID of the Glue Catalog. | 63 | | [encryption\_at\_rest](#output\_encryption\_at\_rest) | A configuration to encrypt at rest in the data catalog. | 64 | | [encryption\_for\_connection\_passwords](#output\_encryption\_for\_connection\_passwords) | A configuration to encrypt connection passwords in the data catalog. | 65 | | [policy](#output\_policy) | The policy to be applied to the aws glue data catalog for access control. | 66 | | [sharing](#output\_sharing) | The configuration for sharing of the Glue Data Catalog.
`status` - An indication of whether the data catalog is shared with other AWS accounts, or was shared with the current account by another AWS account. Sharing is configured through AWS Resource Access Manager (AWS RAM). Values are `NOT_SHARED`, `SHARED_BY_ME` or `SHARED_WITH_ME`.
`shares` - The list of resource shares via RAM (Resource Access Manager). | 67 | 68 | -------------------------------------------------------------------------------- /modules/glue-connection/README.md: -------------------------------------------------------------------------------- 1 | # glue-connection 2 | 3 | This module creates following resources. 4 | 5 | - `aws_glue_connection` 6 | 7 | 8 | ## Requirements 9 | 10 | | Name | Version | 11 | |------|---------| 12 | | [terraform](#requirement\_terraform) | >= 1.5 | 13 | | [aws](#requirement\_aws) | >= 4.65 | 14 | 15 | ## Providers 16 | 17 | | Name | Version | 18 | |------|---------| 19 | | [aws](#provider\_aws) | 5.19.0 | 20 | 21 | ## Modules 22 | 23 | | Name | Source | Version | 24 | |------|--------|---------| 25 | | [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | 26 | 27 | ## Resources 28 | 29 | | Name | Type | 30 | |------|------| 31 | | [aws_glue_connection.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/glue_connection) | resource | 32 | | [aws_subnet.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | 33 | 34 | ## Inputs 35 | 36 | | Name | Description | Type | Default | Required | 37 | |------|-------------|------|---------|:--------:| 38 | | [name](#input\_name) | (Required) The name of the Glue connection. | `string` | n/a | yes | 39 | | [properties](#input\_properties) | (Optional) A map of key-value pairs used as parameters for this connection.
## `JDBC`
- JDBC Connections use the following parameters.
(Required) All of (`HOST`, `PORT`, `JDBC_ENGINE`) or `JDBC_CONNECTION_URL`.
(Required) All of (`USERNAME`, `PASSWORD`) or `SECRET_ID`.
(Optional) `JDBC_ENFORCE_SSL`, `CUSTOM_JDBC_CERT`, `CUSTOM_JDBC_CERT_STRING`, `SKIP_CUSTOM_JDBC_CERT_VALIDATION`

## `KAFKA`
- KAFKA Connections use the following parameters.
(Required) `KAFKA_BOOTSTRAP_SERVERS`.
(Optional) `KAFKA_SSL_ENABLED`, `KAFKA_CUSTOM_CERT`, `KAFKA_SKIP_CUSTOM_CERT_VALIDATION`.
(Optional) `KAFKA_CLIENT_KEYSTORE`, `KAFKA_CLIENT_KEYSTORE_PASSWORD`, `KAFKA_CLIENT_KEY_PASSWORD`, `ENCRYPTED_KAFKA_CLIENT_KEYSTORE_PASSWORD`, `ENCRYPTED_KAFKA_CLIENT_KEY_PASSWORD`
(Optional) `KAFKA_SASL_MECHANISM` - Valid values are `SCRAM-SHA-512`, `GSSAPI`, or `AWS_MSK_IAM`.
(Optional) `KAFKA_SASL_SCRAM_USERNAME`, `KAFKA_SASL_SCRAM_PASSWORD`, `ENCRYPTED_KAFKA_SASL_SCRAM_PASSWORD` - Parameters are used to configure SASL/SCRAM-SHA-512 authentication.
(Optional) `KAFKA_SASL_GSSAPI_KEYTAB`, `KAFKA_SASL_GSSAPI_KRB5_CONF`, `KAFKA_SASL_GSSAPI_SERVICE`, `KAFKA_SASL_GSSAPI_PRINCIPAL` - Parameters are used to configure SASL/GSSAPI authentication.

## `MONGODB`
- MONGODB Connections use the following parameters.
(Required) `CONNECTION_URL`.
(Required) All of (`USERNAME`, `PASSWORD`) or `SECRET_ID`.

## `NETWORK`
- NETWORK Connections do not require parameters. Instead, provide a `physical_connection_requirements`.

## `MARKETPLACE`
- MARKETPLACE Connections use the following parameters.
(Required) `CONNECTOR_TYPE`, `CONNECTOR_URL`, `CONNECTOR_CLASS_NAME`, `CONNECTION_URL`.
(Required) All of (`USERNAME`, `PASSWORD`) or `SECRET_ID`. Only required if `CONNECTOR_TYPE` is `JDBC`.

## `CUSTOM`
- Use configuration settings contained in a custom connector to read from and write to data stores that are not natively supported by Glue. | `map(string)` | n/a | yes | 40 | | [catalog](#input\_catalog) | (Optional) The ID of the Data Catalog in which to create the connection. If omitted, this defaults to the AWS Account ID. | `string` | `null` | no | 41 | | [description](#input\_description) | (Optional) The description of the connection. Can be up to 2048 characters long. | `string` | `"Managed by Terraform."` | no | 42 | | [match\_criteria](#input\_match\_criteria) | (Optional) A list of criteria that can be used in selecting this connection. | `list(string)` | `[]` | no | 43 | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | 44 | | [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | 45 | | [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | 46 | | [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | 47 | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | 48 | | [type](#input\_type) | (Optional) The type of the connection. Valid values are `CUSTOM`, `JDBC`, `KAFKA`, `MARKETPLACE`, `MONGODB`, and `NETWORK`. Defaults to `JBDC`. | `string` | `"JDBC"` | no | 49 | | [vpc\_association](#input\_vpc\_association) | (Optional) A configuration for additional VPC association of the Glue connection when your AWS Glue job needs to run on Amazon Elastic Compute Cloud (EC2) instances in a virtual private cloud (VPC) subnet. `vpc_association` as defined below.
(Optional) `enabled` - Whether to associate Glue connection to VPC subnet.
(Optional) `subnet` - The subnet ID used by the connection.
(Optional) `security_groups` - A list of IDs of security groups to allow access to the data store in your VPC subnet. Security groups are associated to the ENI attached to your subnet. You must choose at least one security group with a self-referencing inbound rule for all TCP ports. | 
object({
    enabled         = optional(bool, false)
    subnet          = optional(string)
    security_groups = optional(set(string), [])
  })
| `{}` | no | 50 | 51 | ## Outputs 52 | 53 | | Name | Description | 54 | |------|-------------| 55 | | [arn](#output\_arn) | The Amazon Resource Name (ARN) of the Glue connection. | 56 | | [catalog](#output\_catalog) | The ID of the Glue Catalog. | 57 | | [description](#output\_description) | The description of the Glue connection. | 58 | | [id](#output\_id) | The ID of the Glue connection. | 59 | | [match\_criteria](#output\_match\_criteria) | A list of criteria that can be used in selecting this connection. | 60 | | [name](#output\_name) | The name of the Glue connection. | 61 | | [properties](#output\_properties) | A map of key-value pairs used as parameters for this connection. | 62 | | [type](#output\_type) | The type of the connection. | 63 | | [vpc\_association](#output\_vpc\_association) | A configuration for additional VPC association of the Glue connection when your AWS Glue job needs to run on Amazon Elastic Compute Cloud (EC2) instances in a virtual private cloud (VPC) subnet. | 64 | 65 | -------------------------------------------------------------------------------- /modules/s3-bucket/outputs.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | replication_rules_by_id = { 3 | for rule in var.replication_rules : 4 | rule.id => rule 5 | } 6 | replication_rules = [ 7 | for rule in try(aws_s3_bucket_replication_configuration.this[0].rule, []) : { 8 | id = rule.id 9 | priority = rule.priority 10 | enabled = rule.status == "Enabled" 11 | 12 | prefix = local.replication_rules_by_id[rule.id].prefix 13 | tags = local.replication_rules_by_id[rule.id].tags 14 | 15 | destination = { 16 | account = rule.destination[0].account 17 | bucket = { 18 | arn = rule.destination[0].bucket 19 | name = local.replication_rules_by_id[rule.id].destination.bucket 20 | } 21 | storage_class = local.replication_rules_by_id[rule.id].destination.storage_class 22 | } 23 | 24 | ownership_translation_enabled = local.replication_rules_by_id[rule.id].ownership_translation_enabled 25 | delete_marker_replication_enabled = local.replication_rules_by_id[rule.id].delete_marker_replication_enabled 26 | replica_modification_sync_enabled = rule.source_selection_criteria[0].replica_modifications[0].status == "Enabled" 27 | replication_time_control = { 28 | enabled = rule.destination[0].replication_time[0].status == "Enabled" 29 | time_threshold = rule.destination[0].replication_time[0].time[0].minutes 30 | } 31 | metrics = { 32 | enabled = rule.destination[0].metrics[0].status == "Enabled" 33 | time_threshold = rule.destination[0].metrics[0].event_threshold[0].minutes 34 | } 35 | sse_kms_encrypted_objects_replication = local.replication_rules_by_id[rule.id].sse_kms_encrypted_objects_replication 36 | } 37 | ] 38 | } 39 | 40 | output "region" { 41 | description = "The AWS region this module resources resides in." 42 | value = aws_s3_bucket.this.region 43 | } 44 | 45 | output "name" { 46 | description = "The name of the bucket." 47 | value = aws_s3_bucket.this.bucket 48 | } 49 | 50 | output "id" { 51 | description = "The ID of the bucket." 52 | value = aws_s3_bucket.this.id 53 | } 54 | 55 | output "arn" { 56 | description = "The ARN of the bucket." 57 | value = aws_s3_bucket.this.arn 58 | } 59 | 60 | output "hosted_zone_id" { 61 | description = "The Route 53 Hosted Zone ID for this bucket's region." 62 | value = aws_s3_bucket.this.hosted_zone_id 63 | } 64 | output "domain_name" { 65 | description = "The bucket domain name. Will be of format `bucketname.s3.amazonaws.com`." 66 | value = aws_s3_bucket.this.bucket_domain_name 67 | } 68 | 69 | output "regional_domain_name" { 70 | description = "The bucket region-specific domain name. The bucket domain name including the region name." 71 | value = aws_s3_bucket.this.bucket_regional_domain_name 72 | } 73 | 74 | output "object_lock" { 75 | description = "The configuration for the S3 Object Lock of the bucket." 76 | value = { 77 | enabled = var.object_lock.enabled 78 | default_retention = var.object_lock.default_retention 79 | } 80 | } 81 | 82 | output "versioning" { 83 | description = "The versioning configuration for the bucket." 84 | value = { 85 | status = { 86 | for k, v in local.versioning_status : 87 | v => k 88 | }[aws_s3_bucket_versioning.this.versioning_configuration[0].status] 89 | mfa_deletion = var.versioning.mfa_deletion 90 | } 91 | } 92 | 93 | output "lifecycle" { 94 | description = "The lifecycle configuration for the bucket." 95 | value = { 96 | transition_default_min_object_size_strategy = var.lifecycle_transition_default_min_object_size_strategy 97 | rules = { 98 | for rule in try(aws_s3_bucket_lifecycle_configuration.this[0].rule, []) : 99 | rule.id => { 100 | id = rule.id 101 | enabled = rule.status == "Enabled" 102 | 103 | filter = { 104 | prefix = try(local.lifecycle_rules[rule.id].prefix, null) 105 | tags = try(local.lifecycle_rules[rule.id].tags, {}) 106 | min_object_size = try(local.lifecycle_rules[rule.id].min_object_size, null) 107 | max_object_size = try(local.lifecycle_rules[rule.id].max_object_size, null) 108 | } 109 | } 110 | } 111 | } 112 | } 113 | 114 | output "replication" { 115 | description = "The replication configuration for the bucket." 116 | value = { 117 | enabled = length(var.replication_rules) > 0 118 | iam_role = local.replication_iam_role 119 | rules = local.replication_rules 120 | } 121 | } 122 | 123 | output "encryption" { 124 | description = "The configuration for the S3 bucket Server-Side Encryption." 125 | value = { 126 | type = var.encryption.type 127 | kms_key = try(one(aws_s3_bucket_server_side_encryption_configuration.this.rule[*]).apply_server_side_encryption_by_default[0].kms_master_key_id, null) 128 | bucket_key_enabled = one(aws_s3_bucket_server_side_encryption_configuration.this.rule[*]).bucket_key_enabled 129 | } 130 | } 131 | 132 | output "access_control" { 133 | description = "The configuration for the S3 bucket access control." 134 | value = { 135 | object_ownership = aws_s3_bucket_ownership_controls.this.rule[0].object_ownership 136 | acl = { 137 | enabled = aws_s3_bucket_ownership_controls.this.rule[0].object_ownership != "BucketOwnerEnforced" 138 | grants = local.grants 139 | } 140 | block_public_access = { 141 | block_public_acls_enabled = aws_s3_bucket_public_access_block.this.block_public_acls 142 | ignore_public_acls_enabled = aws_s3_bucket_public_access_block.this.ignore_public_acls 143 | block_public_policy_enabled = aws_s3_bucket_public_access_block.this.block_public_policy 144 | restrict_public_buckets_enabled = aws_s3_bucket_public_access_block.this.restrict_public_buckets 145 | } 146 | cors_rules = { 147 | for rule in try(one(aws_s3_bucket_cors_configuration.this[*]).cors_rule, []) : 148 | rule.id => { 149 | allowed_headers = rule.allowed_headers 150 | allowed_methods = rule.allowed_methods 151 | allowed_origins = rule.allowed_origins 152 | expose_headers = rule.expose_headers 153 | max_age = rule.max_age_seconds 154 | } 155 | } 156 | } 157 | } 158 | 159 | output "logging" { 160 | description = "The logging configuration for the bucket." 161 | value = { 162 | enabled = var.logging.enabled 163 | s3_bucket = one(aws_s3_bucket_logging.this[*].target_bucket) 164 | s3_key_prefix = one(aws_s3_bucket_logging.this[*].target_prefix) 165 | s3_key_format = var.logging.s3_key_format 166 | 167 | is_target_bucket = var.logging.is_target_bucket 168 | allowed_source_buckets = var.logging.allowed_source_buckets 169 | } 170 | } 171 | 172 | output "monitoring" { 173 | description = "The monitoring configuration for the bucket." 174 | value = { 175 | request_metrics = [ 176 | for name, metric in aws_s3_bucket_metric.this : { 177 | name = name 178 | filter = one(metric.filter[*]) 179 | } 180 | ] 181 | } 182 | } 183 | 184 | output "requester_payment" { 185 | description = "The configuration for the S3 bucket request payment." 186 | value = { 187 | enabled = aws_s3_bucket_request_payment_configuration.this.payer == "Requester" 188 | } 189 | } 190 | 191 | output "transfer_acceleration" { 192 | description = "The configuration for the S3 Transfer Acceleration of the bucket." 193 | value = { 194 | enabled = var.transfer_acceleration_enabled 195 | endpoints = { 196 | ipv4 = "${aws_s3_bucket.this.bucket}.s3-accelerate.amazonaws.com" 197 | dualstack = "${aws_s3_bucket.this.bucket}.s3-accelerate.dualstack.amazonaws.com" 198 | } 199 | } 200 | } 201 | 202 | output "resource_group" { 203 | description = "The resource group created to manage resources in this module." 204 | value = merge( 205 | { 206 | enabled = var.resource_group.enabled && var.module_tags_enabled 207 | }, 208 | (var.resource_group.enabled && var.module_tags_enabled 209 | ? { 210 | arn = module.resource_group[0].arn 211 | name = module.resource_group[0].name 212 | } 213 | : {} 214 | ) 215 | ) 216 | } 217 | -------------------------------------------------------------------------------- /modules/athena-workgroup/README.md: -------------------------------------------------------------------------------- 1 | # athena-workgroup 2 | 3 | This module creates following resources. 4 | 5 | - `aws_athena_workgroup` 6 | - `aws_athena_named_query` (optional) 7 | 8 | 9 | ## Requirements 10 | 11 | | Name | Version | 12 | |------|---------| 13 | | [terraform](#requirement\_terraform) | >= 1.5 | 14 | | [aws](#requirement\_aws) | >= 4.7 | 15 | 16 | ## Providers 17 | 18 | | Name | Version | 19 | |------|---------| 20 | | [aws](#provider\_aws) | 5.19.0 | 21 | 22 | ## Modules 23 | 24 | | Name | Source | Version | 25 | |------|--------|---------| 26 | | [resource\_group](#module\_resource\_group) | tedilabs/misc/aws//modules/resource-group | ~> 0.10.0 | 27 | 28 | ## Resources 29 | 30 | | Name | Type | 31 | |------|------| 32 | | [aws_athena_named_query.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_named_query) | resource | 33 | | [aws_athena_workgroup.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/athena_workgroup) | resource | 34 | 35 | ## Inputs 36 | 37 | | Name | Description | Type | Default | Required | 38 | |------|-------------|------|---------|:--------:| 39 | | [name](#input\_name) | (Required) The name of the workgroup. | `string` | n/a | yes | 40 | | [client\_config\_enabled](#input\_client\_config\_enabled) | (Optional) Whether overriding workgroup configurations with client-side configurations is allowed. Defaults to `false`. | `bool` | `false` | no | 41 | | [cloudwatch\_metrics\_enabled](#input\_cloudwatch\_metrics\_enabled) | (Optional) Whether Amazon CloudWatch metrics are enabled for the workgroup. Defaults to `true`. | `bool` | `true` | no | 42 | | [description](#input\_description) | (Optional) The description of the workgroup. | `string` | `"Managed by Terraform."` | no | 43 | | [enabled](#input\_enabled) | (Optional) Whether to enable the workgroup. Defaults to `true`. | `bool` | `true` | no | 44 | | [engine\_version](#input\_engine\_version) | (Optional) Requested engine version. Valid values are `AUTO`, `ATHENA_v2`, `ATHENA_V3`. Defaults to `AUTO`. | `string` | `"AUTO"` | no | 45 | | [force\_destroy](#input\_force\_destroy) | (Optional) Whether to delete the workgroup and its contents even if the workgroup contains any named queries. | `bool` | `false` | no | 46 | | [module\_tags\_enabled](#input\_module\_tags\_enabled) | (Optional) Whether to create AWS Resource Tags for the module informations. | `bool` | `true` | no | 47 | | [named\_queries](#input\_named\_queries) | (Optional) Save named queries to reuse later. A `named_queries` block as defined below.
(Required) `name` - The plain language name for the query. Maximum length of 128.
(Optional) `description` - A brief explanation of the query. Maximum length of 1024.
(Required) `database` - The database to which the query belongs.
(Required) `query` - The text of the query itself. In other words, all query statements. Maximum length of 262144. | 
list(object({
    name        = string
    description = optional(string, "Managed by Terraform.")

    database = string
    query    = string
  }))
| `[]` | no | 48 | | [per\_query\_data\_usage\_limit](#input\_per\_query\_data\_usage\_limit) | (Optional) Sets the limit in bytes for the maximum amount of data a query is allowed to scan. You can set only one per query limit for a workgroup. The limit applies to all queries in the workgroup and if query exceeds the limit, it will be cancelled. Minimum limit is 10 MB and maximum limit is 7EB per workgroup. | `number` | `null` | no | 49 | | [query\_on\_s3\_requester\_pays\_bucket\_enabled](#input\_query\_on\_s3\_requester\_pays\_bucket\_enabled) | (Optional) Whether to allow members assigned to a workgroup to reference Amazon S3 Requester Pays buckets in queries. If set to false, workgroup members cannot query data from Requester Pays buckets, and queries that retrieve data from Requester Pays buckets cause an error. Defaults to `false`. | `bool` | `false` | no | 50 | | [query\_result](#input\_query\_result) | (Optional) The configuration for query result location and encryption. A `query_result` block as defined below.
(Required) `s3_bucket` - The name of the S3 bucket used to store the query result.
(Optional) `s3_key_prefix` - The key prefix for the specified S3 bucket. Defaults to `null`.
(Optional) `s3_bucket_expected_owner` - The AWS account ID that you expect to be the owner of the Amazon S3 bucket.
(Optional) `s3_bucket_owner_full_control_enabled` - Enabling this option grants the owner of the S3 query results bucket full control over the query results. This means that if your query result location is owned by another account, you grant full control over your query results to the other account.
(Optional) `encryption_enabled` - Whether to encrypt query results on S3 bucket.
(Optional) `encryption_mode` - Indicates whether Amazon S3 server-side encryption with Amazon S3-managed keys (SSE\_S3), server-side encryption with KMS-managed keys (SSE\_KMS), or client-side encryption with KMS-managed keys (CSE\_KMS) is used. If a query runs in a workgroup and the workgroup overrides client-side settings, then the workgroup's setting for encryption is used.
(Optional) `encryption_kms_key` - For `SSE_KMS` and `CSE_KMS` encryption modes, this is the KMS key Amazon Resource Name (ARN). | `map(any)` | `null` | no | 51 | | [resource\_group\_description](#input\_resource\_group\_description) | (Optional) The description of Resource Group. | `string` | `"Managed by Terraform."` | no | 52 | | [resource\_group\_enabled](#input\_resource\_group\_enabled) | (Optional) Whether to create Resource Group to find and group AWS resources which are created by this module. | `bool` | `true` | no | 53 | | [resource\_group\_name](#input\_resource\_group\_name) | (Optional) The name of Resource Group. A Resource Group name can have a maximum of 127 characters, including letters, numbers, hyphens, dots, and underscores. The name cannot start with `AWS` or `aws`. | `string` | `""` | no | 54 | | [tags](#input\_tags) | (Optional) A map of tags to add to all resources. | `map(string)` | `{}` | no | 55 | 56 | ## Outputs 57 | 58 | | Name | Description | 59 | |------|-------------| 60 | | [arn](#output\_arn) | The Amazon Resource Name (ARN) of the workgroup. | 61 | | [client\_config\_enabled](#output\_client\_config\_enabled) | Whether overriding workgroup configurations with client-side configurations is allowed. | 62 | | [cloudwatch\_metrics\_enabled](#output\_cloudwatch\_metrics\_enabled) | Whether Amazon CloudWatch metrics are enabled for the workgroup. | 63 | | [description](#output\_description) | The description of the workgroup. | 64 | | [enabled](#output\_enabled) | Whether to enable the workgroup. | 65 | | [force\_destroy](#output\_force\_destroy) | Whether to delete the workgroup and its contents even if the workgroup contains any named queries. | 66 | | [id](#output\_id) | The workgroup name. | 67 | | [name](#output\_name) | The name of the workgroup. | 68 | | [named\_queries](#output\_named\_queries) | The list of Athena named queries. | 69 | | [per\_query\_data\_usage\_limit](#output\_per\_query\_data\_usage\_limit) | The limit in bytes for the maximum amount of data a query is allowed to scan | 70 | | [query\_on\_s3\_requester\_pays\_bucket\_enabled](#output\_query\_on\_s3\_requester\_pays\_bucket\_enabled) | Whether members assigned to a workgroup to reference Amazon S3 Requester Pays buckets in queries are allowed. | 71 | | [query\_result](#output\_query\_result) | The configuration for query result location and encryption. | 72 | 73 | -------------------------------------------------------------------------------- /modules/s3-bucket/lifecycle.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | versioning_status = { 3 | "ENABLED" = "Enabled" 4 | "DISABLED" = "Disabled" 5 | "SUSPENDED" = "Suspended" 6 | } 7 | 8 | lifecycle_rules = { 9 | for rule in var.lifecycle_rules : 10 | rule.id => rule 11 | } 12 | } 13 | 14 | 15 | ################################################### 16 | # Versioning for S3 Bucket 17 | ################################################### 18 | 19 | # INFO: Not supported attributes 20 | # - `expected_bucket_owner` 21 | resource "aws_s3_bucket_versioning" "this" { 22 | region = var.region 23 | 24 | bucket = aws_s3_bucket.this.bucket 25 | 26 | mfa = (var.versioning.status == "ENABLED" 27 | ? var.versioning.mfa_deletion.device 28 | : null 29 | ) 30 | 31 | versioning_configuration { 32 | status = local.versioning_status[var.versioning.status] 33 | mfa_delete = (var.versioning.status == "ENABLED" 34 | ? (var.versioning.mfa_deletion.enabled ? "Enabled" : "Disabled") 35 | : null 36 | ) 37 | } 38 | } 39 | 40 | 41 | ################################################### 42 | # Object Lock for S3 Bucket 43 | ################################################### 44 | 45 | # INFO: Not supported attributes 46 | # - `expected_bucket_owner` 47 | # INFO: Deprecated attributes 48 | # - `token` 49 | resource "aws_s3_bucket_object_lock_configuration" "this" { 50 | count = var.object_lock.enabled ? 1 : 0 51 | 52 | region = var.region 53 | 54 | bucket = aws_s3_bucket_versioning.this.bucket 55 | object_lock_enabled = "Enabled" 56 | 57 | dynamic "rule" { 58 | for_each = var.object_lock.default_retention != null ? [var.object_lock.default_retention] : [] 59 | 60 | content { 61 | default_retention { 62 | mode = rule.value.mode 63 | days = rule.value.unit != "YEARS" ? rule.value.value : null 64 | years = rule.value.unit == "YEARS" ? rule.value.value : null 65 | } 66 | } 67 | } 68 | 69 | lifecycle { 70 | ignore_changes = [ 71 | token, 72 | ] 73 | } 74 | } 75 | 76 | 77 | ################################################### 78 | # Lifecycle Rules for S3 Bucket 79 | ################################################### 80 | 81 | # INFO: Not supported attributes 82 | # - `expected_bucket_owner` 83 | resource "aws_s3_bucket_lifecycle_configuration" "this" { 84 | count = length(var.lifecycle_rules) > 0 ? 1 : 0 85 | 86 | region = var.region 87 | 88 | bucket = aws_s3_bucket_versioning.this.bucket 89 | 90 | transition_default_minimum_object_size = var.lifecycle_transition_default_min_object_size_strategy 91 | 92 | dynamic "rule" { 93 | for_each = var.lifecycle_rules 94 | 95 | content { 96 | id = rule.value.id 97 | status = rule.value.enabled ? "Enabled" : "Disabled" 98 | 99 | dynamic "abort_incomplete_multipart_upload" { 100 | for_each = rule.value.abort_incomplete_multipart_upload != null ? [rule.value.abort_incomplete_multipart_upload] : [] 101 | 102 | content { 103 | days_after_initiation = abort_incomplete_multipart_upload.value.days 104 | } 105 | } 106 | 107 | ## Empty Filter 108 | dynamic "filter" { 109 | for_each = sum([ 110 | rule.value.prefix != "" ? 1 : 0, 111 | length(rule.value.tags) != 0 ? 1 : 0, 112 | rule.value.min_object_size != null ? 1 : 0, 113 | rule.value.max_object_size != null ? 1 : 0, 114 | ]) == 0 ? ["go"] : [] 115 | 116 | content { 117 | prefix = rule.value.prefix 118 | } 119 | } 120 | 121 | ## Single Filter 122 | dynamic "filter" { 123 | for_each = length(rule.value.tags) == 0 && sum([ 124 | rule.value.prefix != "" ? 1 : 0, 125 | rule.value.min_object_size != null ? 1 : 0, 126 | rule.value.max_object_size != null ? 1 : 0, 127 | ]) == 1 ? ["go"] : [] 128 | 129 | content { 130 | prefix = rule.value.prefix 131 | 132 | object_size_greater_than = rule.value.min_object_size 133 | object_size_less_than = rule.value.max_object_size 134 | } 135 | } 136 | 137 | ## Multi Filter 138 | dynamic "filter" { 139 | for_each = sum([ 140 | rule.value.prefix != "" ? 1 : 0, 141 | length(rule.value.tags) != 0 ? 2 : 0, 142 | rule.value.min_object_size != null ? 1 : 0, 143 | rule.value.max_object_size != null ? 1 : 0, 144 | ]) > 1 ? ["go"] : [] 145 | 146 | content { 147 | and { 148 | prefix = rule.value.prefix 149 | tags = rule.value.tags != {} ? rule.value.tags : null 150 | 151 | object_size_greater_than = rule.value.min_object_size 152 | object_size_less_than = rule.value.max_object_size 153 | } 154 | } 155 | } 156 | 157 | dynamic "transition" { 158 | for_each = rule.value.transitions 159 | 160 | content { 161 | date = transition.value.date 162 | days = transition.value.days 163 | 164 | storage_class = transition.value.storage_class 165 | } 166 | } 167 | 168 | dynamic "noncurrent_version_transition" { 169 | for_each = rule.value.noncurrent_version_transitions 170 | 171 | content { 172 | noncurrent_days = noncurrent_version_transition.value.days 173 | newer_noncurrent_versions = noncurrent_version_transition.value.count 174 | 175 | storage_class = noncurrent_version_transition.value.storage_class 176 | } 177 | } 178 | 179 | dynamic "expiration" { 180 | for_each = rule.value.expiration != null ? [rule.value.expiration] : [] 181 | 182 | content { 183 | date = expiration.value.date 184 | days = expiration.value.days 185 | 186 | expired_object_delete_marker = expiration.value.expired_object_delete_marker ? true : null 187 | } 188 | } 189 | 190 | dynamic "noncurrent_version_expiration" { 191 | for_each = rule.value.noncurrent_version_expiration != null ? [rule.value.noncurrent_version_expiration] : [] 192 | 193 | content { 194 | noncurrent_days = noncurrent_version_expiration.value.days 195 | newer_noncurrent_versions = noncurrent_version_expiration.value.count 196 | } 197 | } 198 | } 199 | } 200 | } 201 | 202 | 203 | ################################################### 204 | # Replication Rules for S3 Bucket 205 | ################################################### 206 | 207 | # INFO: Not supported attributes 208 | # - `prefix` 209 | # - `token` 210 | # - `rule.existing_object_replication`: Not supported by Amazon S3 at this time and should not be included in your rule configurations. Specifying this parameter will result in MalformedXML errors 211 | resource "aws_s3_bucket_replication_configuration" "this" { 212 | count = length(var.replication_rules) > 0 ? 1 : 0 213 | 214 | region = var.region 215 | 216 | role = local.replication_iam_role 217 | bucket = aws_s3_bucket_versioning.this.bucket 218 | 219 | dynamic "rule" { 220 | for_each = var.replication_rules 221 | 222 | content { 223 | id = rule.value.id 224 | priority = rule.value.priority 225 | status = rule.value.enabled ? "Enabled" : "Disabled" 226 | 227 | delete_marker_replication { 228 | status = rule.value.delete_marker_replication_enabled ? "Enabled" : "Disabled" 229 | } 230 | 231 | source_selection_criteria { 232 | dynamic "sse_kms_encrypted_objects" { 233 | for_each = rule.value.sse_kms_encrypted_objects_replication.enabled ? ["go"] : [] 234 | 235 | content { 236 | status = "Enabled" 237 | } 238 | } 239 | 240 | replica_modifications { 241 | status = rule.value.replica_modification_sync_enabled ? "Enabled" : "Disabled" 242 | } 243 | } 244 | 245 | # INFO: Not supported yet. 246 | # dynamic "existing_object_replication" { 247 | # for_each = rule.value.existing_object_replication_enabled ? ["go"] : [] 248 | # 249 | # content { 250 | # status = "Enabled" 251 | # } 252 | # } 253 | 254 | ## Empty Filter 255 | dynamic "filter" { 256 | for_each = sum([ 257 | rule.value.prefix != "" ? 1 : 0, 258 | length(rule.value.tags) != 0 ? 1 : 0, 259 | ]) == 0 ? ["go"] : [] 260 | 261 | content {} 262 | } 263 | 264 | ## Single Filter (Prefix) 265 | dynamic "filter" { 266 | for_each = anytrue([ 267 | length(rule.value.tags) == 0 && rule.value.prefix != "", 268 | ]) ? ["go"] : [] 269 | 270 | content { 271 | prefix = rule.value.prefix 272 | } 273 | } 274 | 275 | ## Single Filter (Tag) 276 | dynamic "filter" { 277 | for_each = anytrue([ 278 | length(rule.value.tags) == 1 && rule.value.prefix == "", 279 | ]) ? ["go"] : [] 280 | 281 | content { 282 | dynamic "tag" { 283 | for_each = rule.value.tags 284 | 285 | content { 286 | key = tag.key 287 | value = tag.value == null ? "" : tag.value 288 | } 289 | } 290 | } 291 | } 292 | 293 | ## Multi Filter 294 | dynamic "filter" { 295 | for_each = anytrue([ 296 | length(rule.value.tags) >= 2, 297 | length(rule.value.tags) != 0 && rule.value.prefix != "", 298 | ]) ? ["go"] : [] 299 | 300 | content { 301 | and { 302 | prefix = rule.value.prefix 303 | tags = rule.value.tags 304 | } 305 | } 306 | } 307 | 308 | destination { 309 | bucket = provider::aws::arn_build("aws", "s3", "", "", rule.value.destination.bucket) 310 | account = coalesce(rule.value.destination.account, local.account_id) 311 | storage_class = rule.value.destination.storage_class 312 | 313 | dynamic "access_control_translation" { 314 | for_each = rule.value.ownership_translation_enabled ? ["go"] : [] 315 | 316 | content { 317 | owner = "Destination" 318 | } 319 | } 320 | 321 | dynamic "encryption_configuration" { 322 | for_each = rule.value.sse_kms_encrypted_objects_replication.enabled ? ["go"] : [] 323 | 324 | content { 325 | replica_kms_key_id = rule.value.sse_kms_encrypted_objects_replication.kms_key 326 | } 327 | } 328 | 329 | replication_time { 330 | status = rule.value.replication_time_control.enabled ? "Enabled" : "Disabled" 331 | 332 | time { 333 | minutes = rule.value.replication_time_control.time_threshold 334 | } 335 | } 336 | 337 | metrics { 338 | status = rule.value.metrics.enabled ? "Enabled" : "Disabled" 339 | 340 | event_threshold { 341 | minutes = rule.value.metrics.time_threshold 342 | } 343 | } 344 | } 345 | } 346 | } 347 | 348 | lifecycle { 349 | precondition { 350 | condition = aws_s3_bucket_versioning.this.versioning_configuration[0].status == "Enabled" 351 | error_message = "Replication rules require versioning to be enabled on the source bucket." 352 | } 353 | } 354 | } 355 | --------------------------------------------------------------------------------