├── .gitignore ├── README.md ├── main.tf ├── msk-cluster ├── locals.tf ├── main.tf ├── outputs.tf ├── variables.tf └── versions.tf ├── msk-connetors ├── main.tf ├── outputs.tf ├── variables.tf └── versions.tf ├── msk-custom-plugin ├── main.tf ├── outputs.tf ├── variables.tf └── versions.tf └── msk-dashboard ├── main.tf ├── variables.tf └── versions.tf /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | **/.terraform/* 3 | 4 | # .tfstate files 5 | *.tfstate 6 | *.tfstate.* 7 | 8 | # terraform lockfile 9 | .terraform.lock.hcl 10 | 11 | # Crash log files 12 | crash.log 13 | 14 | # Exclude all .tfvars files, which are likely to contain sentitive data, such as 15 | # password, private keys, and other secrets. These should not be part of version 16 | # control as they are data points which are potentially sensitive and subject 17 | # to change depending on the environment. 18 | # 19 | *.tfvars 20 | 21 | # Ignore override files as they are usually used to override resources locally and so 22 | # are not checked in 23 | override.tf 24 | override.tf.json 25 | *_override.tf 26 | *_override.tf.json 27 | 28 | # Include override files you do wish to add to version control using negated pattern 29 | # 30 | # !example_override.tf 31 | 32 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 33 | # example: *tfplan* 34 | 35 | # Ignore CLI configuration files 36 | .terraformrc 37 | terraform.rc 38 | 39 | # Examples 40 | debezium* 41 | *.tar.gz 42 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS MSK Kafka Cluster Terraform module 2 | 3 | Terraform module which creates AWS MSK (Managed Streaming for Kafka) resources. 4 | 5 | ## Usage 6 | 7 | See [`examples`](https://github.com/ahmadalibagheri/terraform-aws-msk/tree/main/examples) directory for working examples to reference: 8 | 9 | ```hcl 10 | module "kafka-cluster" { 11 | source = "./msk-cluster" 12 | name = "Kafka" 13 | number_of_broker_nodes = 3 14 | enhanced_monitoring = "PER_TOPIC_PER_PARTITION" 15 | environment = "dev" 16 | 17 | broker_node_client_subnets = ["subnet-a", "subnet-b", "subnet-c"] #Adding your subnet 18 | broker_node_ebs_volume_size = 20 19 | broker_node_instance_type = "kafka.t3.small" 20 | broker_node_security_groups = ["sg-default"] #Adding your security group 21 | 22 | encryption_in_transit_client_broker = "TLS" 23 | encryption_in_transit_in_cluster = true 24 | 25 | configuration_name = "example-configuration" 26 | configuration_description = "Example configuration" 27 | configuration_server_properties = { 28 | "auto.create.topics.enable" = true 29 | "delete.topic.enable" = true 30 | } 31 | 32 | jmx_exporter_enabled = true 33 | node_exporter_enabled = true 34 | cloudwatch_logs_enabled = true 35 | # s3_logs_enabled = true 36 | # s3_logs_bucket = "S3-bucket-msk-log" 37 | 38 | } 39 | ``` 40 | 41 | ## Examples 42 | 43 | Examples codified under the [`examples`](https://github.com/ahmadalibagheri/terraform-aws-msk/tree/main/examples) are intended to give users references for how to use the module(s) as well as testing/validating changes to the source code of the module. If contributing to the project, please be sure to make any appropriate updates to the relevant examples to allow maintainers to test your changes and to keep the examples up to date for users. Thank you! 44 | 45 | - [Basic](https://github.com/ahmadalibagheri/terraform-aws-msk/tree/main/examples/basic) 46 | - [Complete](https://github.com/ahmadalibagheri/terraform-aws-msk/tree/main/examples/complete) 47 | 48 | 49 | 50 | ## Requirements 51 | 52 | | Name | Version | 53 | |------|---------| 54 | | [terraform](#requirement\_terraform) | >= 0.13.1 | 55 | | [aws](#requirement\_aws) | >= 3.71 | 56 | 57 | ## Providers 58 | 59 | | Name | Version | 60 | |------|---------| 61 | | [aws](#provider\_aws) | >= 3.71 | 62 | 63 | ## Modules 64 | 65 | No modules. 66 | 67 | ## Resources 68 | 69 | | Name | Type | 70 | |------|------| 71 | | [aws_appautoscaling_policy.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/appautoscaling_policy) | resource | 72 | | [aws_appautoscaling_target.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/appautoscaling_target) | resource | 73 | | [aws_cloudwatch_log_group.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/cloudwatch_log_group) | resource | 74 | | [aws_msk_cluster.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_cluster) | resource | 75 | | [aws_msk_configuration.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_configuration) | resource | 76 | | [aws_msk_scram_secret_association.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/msk_scram_secret_association) | resource | 77 | | [aws_mskconnect_custom_plugin.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/mskconnect_custom_plugin) | resource | 78 | | [aws_mskconnect_worker_configuration.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/mskconnect_worker_configuration) | resource | 79 | 80 | ## Inputs 81 | 82 | | Name | Description | Type | Default | Required | 83 | |------|-------------|------|---------|:--------:| 84 | | [broker\_node\_client\_subnets](#input\_broker\_node\_client\_subnets) | A list of subnets to connect to in client VPC ([documentation](https://docs.aws.amazon.com/msk/1.0/apireference/clusters.html#clusters-prop-brokernodegroupinfo-clientsubnets)) | `list(string)` | `[]` | no | 85 | | [broker\_node\_ebs\_volume\_size](#input\_broker\_node\_ebs\_volume\_size) | The size in GiB of the EBS volume for the data drive on each broker node | `number` | `null` | no | 86 | | [broker\_node\_instance\_type](#input\_broker\_node\_instance\_type) | Specify the instance type to use for the kafka brokers. e.g. kafka.m5.large. ([Pricing info](https://aws.amazon.com/msk/pricing/)) | `string` | `null` | no | 87 | | [broker\_node\_security\_groups](#input\_broker\_node\_security\_groups) | A list of the security groups to associate with the elastic network interfaces to control who can communicate with the cluster | `list(string)` | `[]` | no | 88 | | [client\_authentication\_sasl\_iam](#input\_client\_authentication\_sasl\_iam) | Enables IAM client authentication | `bool` | `false` | no | 89 | | [client\_authentication\_sasl\_scram](#input\_client\_authentication\_sasl\_scram) | Enables SCRAM client authentication via AWS Secrets Manager | `bool` | `false` | no | 90 | | [client\_authentication\_tls\_certificate\_authority\_arns](#input\_client\_authentication\_tls\_certificate\_authority\_arns) | List of ACM Certificate Authority Amazon Resource Names (ARNs) | `list(string)` | `[]` | no | 91 | | [cloudwatch\_log\_group\_kms\_key\_id](#input\_cloudwatch\_log\_group\_kms\_key\_id) | The ARN of the KMS Key to use when encrypting log data | `string` | `null` | no | 92 | | [cloudwatch\_log\_group\_name](#input\_cloudwatch\_log\_group\_name) | Name of the Cloudwatch Log Group to deliver logs to | `string` | `null` | no | 93 | | [cloudwatch\_log\_group\_retention\_in\_days](#input\_cloudwatch\_log\_group\_retention\_in\_days) | Specifies the number of days you want to retain log events in the log group | `number` | `0` | no | 94 | | [cloudwatch\_logs\_enabled](#input\_cloudwatch\_logs\_enabled) | Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs | `bool` | `false` | no | 95 | | [configuration\_description](#input\_configuration\_description) | Description of the configuration | `string` | `null` | no | 96 | | [configuration\_name](#input\_configuration\_name) | Name of the configuration | `string` | `null` | no | 97 | | [configuration\_server\_properties](#input\_configuration\_server\_properties) | Contents of the server.properties file. Supported properties are documented in the [MSK Developer Guide](https://docs.aws.amazon.com/msk/latest/developerguide/msk-configuration-properties.html) | `map(string)` | `{}` | no | 98 | | [connect\_custom\_plugin\_timeouts](#input\_connect\_custom\_plugin\_timeouts) | Timeout configurations for the connect custom plugins | `map(string)` | 
{
  "create": null
}
| no | 99 | | [connect\_custom\_plugins](#input\_connect\_custom\_plugins) | Map of custom plugin configuration details (map of maps) | `any` | `{}` | no | 100 | | [connect\_worker\_config\_description](#input\_connect\_worker\_config\_description) | A summary description of the worker configuration | `string` | `null` | no | 101 | | [connect\_worker\_config\_name](#input\_connect\_worker\_config\_name) | The name of the worker configuration | `string` | `null` | no | 102 | | [connect\_worker\_config\_properties\_file\_content](#input\_connect\_worker\_config\_properties\_file\_content) | Contents of connect-distributed.properties file. The value can be either base64 encoded or in raw format | `string` | `null` | no | 103 | | [create](#input\_create) | Determines whether cluster resources will be created | `bool` | `true` | no | 104 | | [create\_cloudwatch\_log\_group](#input\_create\_cloudwatch\_log\_group) | Determines whether to create a CloudWatch log group | `bool` | `true` | no | 105 | | [create\_connect\_worker\_configuration](#input\_create\_connect\_worker\_configuration) | Determines whether to create connect worker configuration | `bool` | `false` | no | 106 | | [create\_schema\_registry](#input\_create\_schema\_registry) | Determines whether to create a Glue schema registry for managing Avro schemas for the cluster | `bool` | `true` | no | 107 | | [create\_scram\_secret\_association](#input\_create\_scram\_secret\_association) | Determines whether to create SASL/SCRAM secret association | `bool` | `false` | no | 108 | | [encryption\_at\_rest\_kms\_key\_arn](#input\_encryption\_at\_rest\_kms\_key\_arn) | You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest | `string` | `null` | no | 109 | | [encryption\_in\_transit\_client\_broker](#input\_encryption\_in\_transit\_client\_broker) | Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT`. Default value is `TLS` | `string` | `null` | no | 110 | | [encryption\_in\_transit\_in\_cluster](#input\_encryption\_in\_transit\_in\_cluster) | Whether data communication among broker nodes is encrypted. Default value: `true` | `bool` | `null` | no | 111 | | [enhanced\_monitoring](#input\_enhanced\_monitoring) | Specify the desired enhanced MSK CloudWatch monitoring level. See [Monitoring Amazon MSK with Amazon CloudWatch](https://docs.aws.amazon.com/msk/latest/developerguide/monitoring.html) | `string` | `null` | no | 112 | | [firehose\_delivery\_stream](#input\_firehose\_delivery\_stream) | Name of the Kinesis Data Firehose delivery stream to deliver logs to | `string` | `null` | no | 113 | | [firehose\_logs\_enabled](#input\_firehose\_logs\_enabled) | Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose | `bool` | `false` | no | 114 | | [jmx\_exporter\_enabled](#input\_jmx\_exporter\_enabled) | Indicates whether you want to enable or disable the JMX Exporter | `bool` | `false` | no | 115 | | [kafka\_version](#input\_kafka\_version) | Specify the desired Kafka software version | `string` | `null` | no | 116 | | [name](#input\_name) | Name of the MSK cluster | `string` | `"msk"` | no | 117 | | [node\_exporter\_enabled](#input\_node\_exporter\_enabled) | Indicates whether you want to enable or disable the Node Exporter | `bool` | `false` | no | 118 | | [number\_of\_broker\_nodes](#input\_number\_of\_broker\_nodes) | The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets | `number` | `null` | no | 119 | | [s3\_logs\_bucket](#input\_s3\_logs\_bucket) | Name of the S3 bucket to deliver logs to | `string` | `null` | no | 120 | | [s3\_logs\_enabled](#input\_s3\_logs\_enabled) | Indicates whether you want to enable or disable streaming broker logs to S3 | `bool` | `false` | no | 121 | | [s3\_logs\_prefix](#input\_s3\_logs\_prefix) | Prefix to append to the folder name | `string` | `null` | no | 122 | | [scaling\_max\_capacity](#input\_scaling\_max\_capacity) | Max storage capacity for Kafka broker autoscaling | `number` | `250` | no | 123 | | [scaling\_role\_arn](#input\_scaling\_role\_arn) | The ARN of the IAM role that allows Application AutoScaling to modify your scalable target on your behalf. This defaults to an IAM Service-Linked Role | `string` | `null` | no | 124 | | [scaling\_target\_value](#input\_scaling\_target\_value) | The Kafka broker storage utilization at which scaling is initiated | `number` | `70` | no | 125 | | [schema\_registries](#input\_schema\_registries) | A map of schema registries to be created | `map(any)` | `{}` | no | 126 | | [schemas](#input\_schemas) | A map schemas to be created within the schema registry | `map(any)` | `{}` | no | 127 | | [scram\_secret\_association\_secret\_arn\_list](#input\_scram\_secret\_association\_secret\_arn\_list) | List of AWS Secrets Manager secret ARNs to associate with SCRAM | `list(string)` | `[]` | no | 128 | | [tags](#input\_tags) | A map of tags to assign to the resources created | `map(string)` | `{}` | no | 129 | | [timeouts](#input\_timeouts) | Create, update, and delete timeout configurations for the cluster | `map(string)` | `{}` | no | 130 | 131 | ## Outputs 132 | 133 | | Name | Description | 134 | |------|-------------| 135 | | [appautoscaling\_policy\_arn](#output\_appautoscaling\_policy\_arn) | The ARN assigned by AWS to the scaling policy | 136 | | [appautoscaling\_policy\_name](#output\_appautoscaling\_policy\_name) | The scaling policy's name | 137 | | [appautoscaling\_policy\_policy\_type](#output\_appautoscaling\_policy\_policy\_type) | The scaling policy's type | 138 | | [arn](#output\_arn) | Amazon Resource Name (ARN) of the MSK cluster | 139 | | [bootstrap\_brokers](#output\_bootstrap\_brokers) | Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster | 140 | | [bootstrap\_brokers\_plaintext](#output\_bootstrap\_brokers\_plaintext) | Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster. Contains a value if `encryption_in_transit_client_broker` is set to `PLAINTEXT` or `TLS_PLAINTEXT` | 141 | | [bootstrap\_brokers\_sasl\_iam](#output\_bootstrap\_brokers\_sasl\_iam) | One or more DNS names (or IP addresses) and SASL IAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_iam` is set to `true` | 142 | | [bootstrap\_brokers\_sasl\_scram](#output\_bootstrap\_brokers\_sasl\_scram) | One or more DNS names (or IP addresses) and SASL SCRAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_scram` is set to `true` | 143 | | [bootstrap\_brokers\_tls](#output\_bootstrap\_brokers\_tls) | One or more DNS names (or IP addresses) and TLS port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` | 144 | | [configuration\_arn](#output\_configuration\_arn) | Amazon Resource Name (ARN) of the configuration | 145 | | [configuration\_latest\_revision](#output\_configuration\_latest\_revision) | Latest revision of the configuration | 146 | | [connect\_custom\_plugins](#output\_connect\_custom\_plugins) | A map of output attributes for the connect custom plugins created | 147 | | [connect\_worker\_configuration\_arn](#output\_connect\_worker\_configuration\_arn) | The Amazon Resource Name (ARN) of the worker configuration | 148 | | [connect\_worker\_configuration\_latest\_revision](#output\_connect\_worker\_configuration\_latest\_revision) | An ID of the latest successfully created revision of the worker configuration | 149 | | [current\_version](#output\_current\_version) | Current version of the MSK Cluster used for updates, e.g. `K13V1IB3VIYZZH` | 150 | | [log\_group\_arn](#output\_log\_group\_arn) | The Amazon Resource Name (ARN) specifying the log group | 151 | | [schema\_registries](#output\_schema\_registries) | A map of output attributes for the schema registries created | 152 | | [schemas](#output\_schemas) | A map of output attributes for the schemas created | 153 | | [scram\_secret\_association\_id](#output\_scram\_secret\_association\_id) | Amazon Resource Name (ARN) of the MSK cluster | 154 | | [zookeeper\_connect\_string](#output\_zookeeper\_connect\_string) | A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster. The returned values are sorted alphabetically | 155 | | [zookeeper\_connect\_string\_tls](#output\_zookeeper\_connect\_string\_tls) | A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster via TLS. The returned values are sorted alphabetically | 156 | 157 | -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | module "kafka-cluster" { 2 | source = "./msk-cluster" 3 | name = "Kafka" 4 | number_of_broker_nodes = 3 5 | enhanced_monitoring = "PER_TOPIC_PER_PARTITION" 6 | environment = "dev" 7 | 8 | broker_node_client_subnets = ["subnet-a", "subnet-b", "subnet-c"] #Adding your subnet 9 | broker_node_ebs_volume_size = 20 10 | broker_node_instance_type = "kafka.t3.small" 11 | broker_node_security_groups = ["sg-default"] #Adding your security group 12 | 13 | encryption_in_transit_client_broker = "TLS" 14 | encryption_in_transit_in_cluster = true 15 | 16 | configuration_name = "example-configuration" 17 | configuration_description = "Example configuration" 18 | configuration_server_properties = { 19 | "auto.create.topics.enable" = true 20 | "delete.topic.enable" = true 21 | } 22 | 23 | jmx_exporter_enabled = true 24 | node_exporter_enabled = true 25 | cloudwatch_logs_enabled = true 26 | # s3_logs_enabled = true 27 | # s3_logs_bucket = "S3-bucket-msk-log" 28 | 29 | } 30 | -------------------------------------------------------------------------------- /msk-cluster/locals.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | cloudwatch_log_group = var.create_msk_configuration && var.create_cloudwatch_log_group ? aws_cloudwatch_log_group.this[0].name : var.cloudwatch_log_group_name 3 | } 4 | -------------------------------------------------------------------------------- /msk-cluster/main.tf: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Configuration 3 | ################################################################################ 4 | 5 | resource "aws_msk_configuration" "this" { 6 | count = var.create_msk_configuration ? 1 : 0 7 | 8 | name = coalesce(var.configuration_name, var.name) 9 | description = var.configuration_description 10 | kafka_versions = [var.kafka_version] 11 | server_properties = join("\n", local.server_properties) 12 | } 13 | 14 | locals { 15 | server_properties = [ 16 | "num.partitions=${lookup(var.kafka_config, "num.partitions", "15")}", 17 | "default.replication.factor=${lookup(var.kafka_config, "default.replication.factor", "3")}", 18 | "auto.create.topics.enable=${lookup(var.kafka_config, "auto.create.topics.enable", "true")}", 19 | "delete.topic.enable=${lookup(var.kafka_config, "delete.topic.enable", "true")}" 20 | ] 21 | } 22 | 23 | # ################################################################################ 24 | # # MSK Cluster 25 | # ################################################################################ 26 | 27 | resource "aws_msk_cluster" "this" { 28 | depends_on = [aws_msk_configuration.this,aws_cloudwatch_log_group,this] 29 | 30 | count = var.create_kafka_cluster ? 1 : 0 31 | 32 | cluster_name = var.name 33 | kafka_version = var.kafka_version 34 | number_of_broker_nodes = var.number_of_broker_nodes 35 | enhanced_monitoring = var.enhanced_monitoring 36 | 37 | broker_node_group_info { 38 | client_subnets = var.broker_node_client_subnets 39 | ebs_volume_size = var.broker_node_ebs_volume_size 40 | instance_type = var.broker_node_instance_type 41 | security_groups = var.broker_node_security_groups 42 | } 43 | 44 | configuration_info { 45 | arn = aws_msk_configuration.this[0].arn 46 | revision = aws_msk_configuration.this[0].latest_revision 47 | } 48 | 49 | encryption_info { 50 | encryption_in_transit { 51 | client_broker = var.encryption_in_transit_client_broker 52 | in_cluster = var.encryption_in_transit_in_cluster 53 | } 54 | encryption_at_rest_kms_key_arn = var.encryption_at_rest_kms_key_arn 55 | } 56 | 57 | open_monitoring { 58 | prometheus { 59 | jmx_exporter { 60 | enabled_in_broker = var.jmx_exporter_enabled 61 | } 62 | node_exporter { 63 | enabled_in_broker = var.node_exporter_enabled 64 | } 65 | } 66 | } 67 | 68 | logging_info { 69 | broker_logs { 70 | cloudwatch_logs { 71 | enabled = var.cloudwatch_logs_enabled 72 | log_group = var.cloudwatch_logs_enabled ? local.cloudwatch_log_group : null 73 | } 74 | firehose { 75 | enabled = var.firehose_logs_enabled 76 | delivery_stream = var.firehose_delivery_stream 77 | } 78 | s3 { 79 | enabled = var.s3_logs_enabled 80 | bucket = var.s3_logs_bucket 81 | prefix = var.s3_logs_prefix 82 | } 83 | } 84 | } 85 | 86 | timeouts { 87 | create = lookup(var.timeouts, "create", null) 88 | update = lookup(var.timeouts, "update", null) 89 | delete = lookup(var.timeouts, "delete", null) 90 | } 91 | 92 | # required for appautoscaling 93 | lifecycle { 94 | ignore_changes = [broker_node_group_info[0].ebs_volume_size] 95 | } 96 | 97 | tags = { 98 | Name = var.name 99 | Environment = var.environment 100 | } 101 | } 102 | 103 | ################################################################################ 104 | # CloudWatch Log Group 105 | ################################################################################ 106 | 107 | resource "aws_cloudwatch_log_group" "this" { 108 | count = var.create_cloudwatch_log_group ? 1 : 0 109 | 110 | name = coalesce(var.cloudwatch_log_group_name, "/aws/msk/${var.name}") 111 | retention_in_days = var.cloudwatch_log_group_retention_in_days 112 | kms_key_id = var.cloudwatch_log_group_kms_key_id 113 | 114 | tags = var.tags 115 | } 116 | -------------------------------------------------------------------------------- /msk-cluster/outputs.tf: -------------------------------------------------------------------------------- 1 | # ################################################################################ 2 | # # Cluster 3 | # ################################################################################ 4 | 5 | output "arn" { 6 | description = "Amazon Resource Name (ARN) of the MSK cluster" 7 | value = try(aws_msk_cluster.this[0].arn, "") 8 | } 9 | 10 | output "bootstrap_brokers" { 11 | description = "Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster" 12 | value = compact([ 13 | try(aws_msk_cluster.this[0].bootstrap_brokers, ""), 14 | try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_iam, ""), 15 | try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_scram, ""), 16 | try(aws_msk_cluster.this[0].bootstrap_brokers_tls, ""), 17 | ]) 18 | } 19 | 20 | output "bootstrap_brokers_plaintext" { 21 | description = "Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster. Contains a value if `encryption_in_transit_client_broker` is set to `PLAINTEXT` or `TLS_PLAINTEXT`" 22 | value = try(aws_msk_cluster.this[0].bootstrap_brokers, "") 23 | } 24 | 25 | output "bootstrap_brokers_sasl_iam" { 26 | description = "One or more DNS names (or IP addresses) and SASL IAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_iam` is set to `true`" 27 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_iam, "") 28 | } 29 | 30 | output "bootstrap_brokers_sasl_scram" { 31 | description = "One or more DNS names (or IP addresses) and SASL SCRAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_scram` is set to `true`" 32 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_scram, "") 33 | } 34 | 35 | output "bootstrap_brokers_tls" { 36 | description = "One or more DNS names (or IP addresses) and TLS port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS`" 37 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_tls, "") 38 | } 39 | 40 | output "current_version" { 41 | description = "Current version of the MSK Cluster used for updates, e.g. `K13V1IB3VIYZZH`" 42 | value = try(aws_msk_cluster.this[0].current_version, "") 43 | } 44 | 45 | output "zookeeper_connect_string" { 46 | description = "A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster. The returned values are sorted alphabetically" 47 | value = try(aws_msk_cluster.this[0].zookeeper_connect_string, "") 48 | } 49 | 50 | output "zookeeper_connect_string_tls" { 51 | description = "A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster via TLS. The returned values are sorted alphabetically" 52 | value = try(aws_msk_cluster.this[0].zookeeper_connect_string_tls, "") 53 | } 54 | 55 | ################################################################################ 56 | # Configuration 57 | ################################################################################ 58 | 59 | output "configuration_arn" { 60 | description = "Amazon Resource Name (ARN) of the configuration" 61 | value = try(aws_msk_configuration.this[0].arn, "") 62 | } 63 | 64 | output "configuration_latest_revision" { 65 | description = "Latest revision of the configuration" 66 | value = try(aws_msk_configuration.this[0].latest_revision, "") 67 | } 68 | 69 | ################################################################################ 70 | # CloudWatch Log Group 71 | ################################################################################ 72 | 73 | output "log_group_arn" { 74 | description = "The Amazon Resource Name (ARN) specifying the log group" 75 | value = try(aws_cloudwatch_log_group.this[0].arn, "") 76 | } 77 | -------------------------------------------------------------------------------- /msk-cluster/variables.tf: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Cluster 3 | ################################################################################ 4 | variable "create_kafka_cluster" { 5 | description = "Determines whether cluster resources will be created" 6 | type = bool 7 | default = true 8 | } 9 | 10 | variable "name" { 11 | description = "Name of the MSK cluster" 12 | type = string 13 | default = "msk" # to avoid: Error: cluster_name must be 1 characters or higher 14 | } 15 | 16 | variable "environment" { 17 | description = "Name of the environment" 18 | type = string 19 | default = "test" 20 | } 21 | 22 | variable "kafka_version" { 23 | description = "Specify the desired Kafka software version" 24 | type = string 25 | default = "2.6.2" 26 | } 27 | 28 | variable "number_of_broker_nodes" { 29 | description = "The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets" 30 | type = number 31 | default = null 32 | } 33 | 34 | variable "enhanced_monitoring" { 35 | description = "Specify the desired enhanced MSK CloudWatch monitoring level. See [Monitoring Amazon MSK with Amazon CloudWatch](https://docs.aws.amazon.com/msk/latest/developerguide/monitoring.html)" 36 | type = string 37 | default = null 38 | } 39 | 40 | variable "broker_node_client_subnets" { 41 | description = "A list of subnets to connect to in client VPC ([documentation](https://docs.aws.amazon.com/msk/1.0/apireference/clusters.html#clusters-prop-brokernodegroupinfo-clientsubnets))" 42 | type = list(string) 43 | default = [] 44 | } 45 | 46 | variable "broker_node_ebs_volume_size" { 47 | description = "The size in GiB of the EBS volume for the data drive on each broker node" 48 | type = number 49 | default = null 50 | } 51 | 52 | variable "broker_node_instance_type" { 53 | description = "Specify the instance type to use for the kafka brokers. e.g. kafka.m5.large. ([Pricing info](https://aws.amazon.com/msk/pricing/))" 54 | type = string 55 | default = null 56 | } 57 | 58 | variable "broker_node_security_groups" { 59 | description = "A list of the security groups to associate with the elastic network interfaces to control who can communicate with the cluster" 60 | type = list(string) 61 | default = [] 62 | } 63 | 64 | variable "client_authentication_tls_certificate_authority_arns" { 65 | description = "List of ACM Certificate Authority Amazon Resource Names (ARNs)" 66 | type = list(string) 67 | default = [] 68 | } 69 | 70 | variable "client_authentication_sasl_iam" { 71 | description = "Enables IAM client authentication" 72 | type = bool 73 | default = false 74 | } 75 | 76 | variable "client_authentication_sasl_scram" { 77 | description = "Enables SCRAM client authentication via AWS Secrets Manager" 78 | type = bool 79 | default = false 80 | } 81 | 82 | variable "encryption_in_transit_client_broker" { 83 | description = "Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT`. Default value is `TLS`" 84 | type = string 85 | default = null 86 | } 87 | 88 | variable "encryption_in_transit_in_cluster" { 89 | description = "Whether data communication among broker nodes is encrypted. Default value: `true`" 90 | type = bool 91 | default = null 92 | } 93 | 94 | variable "encryption_at_rest_kms_key_arn" { 95 | description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest" 96 | type = string 97 | default = null 98 | } 99 | 100 | variable "jmx_exporter_enabled" { 101 | description = "Indicates whether you want to enable or disable the JMX Exporter" 102 | type = bool 103 | default = false 104 | } 105 | 106 | variable "node_exporter_enabled" { 107 | description = "Indicates whether you want to enable or disable the Node Exporter" 108 | type = bool 109 | default = false 110 | } 111 | 112 | variable "cloudwatch_logs_enabled" { 113 | description = "Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs" 114 | type = bool 115 | default = false 116 | } 117 | 118 | variable "firehose_logs_enabled" { 119 | description = "Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose" 120 | type = bool 121 | default = false 122 | } 123 | 124 | variable "firehose_delivery_stream" { 125 | description = "Name of the Kinesis Data Firehose delivery stream to deliver logs to" 126 | type = string 127 | default = null 128 | } 129 | 130 | variable "s3_logs_enabled" { 131 | description = "Indicates whether you want to enable or disable streaming broker logs to S3" 132 | type = bool 133 | default = false 134 | } 135 | 136 | variable "s3_logs_bucket" { 137 | description = "Name of the S3 bucket to deliver logs to" 138 | type = string 139 | default = null 140 | } 141 | 142 | variable "s3_logs_prefix" { 143 | description = "Prefix to append to the folder name" 144 | type = string 145 | default = null 146 | } 147 | 148 | variable "timeouts" { 149 | description = "Create, update, and delete timeout configurations for the cluster" 150 | type = map(string) 151 | default = {} 152 | } 153 | 154 | variable "tags" { 155 | description = "A map of tags to assign to the resources created" 156 | type = map(string) 157 | default = {} 158 | } 159 | 160 | ################################################################################ 161 | # Configuration 162 | ################################################################################ 163 | variable "create_msk_configuration" { 164 | description = "Determines whether cluster resources will be created" 165 | type = bool 166 | default = true 167 | } 168 | 169 | variable "configuration_name" { 170 | description = "Name of the configuration" 171 | type = string 172 | default = null 173 | } 174 | 175 | variable "configuration_description" { 176 | description = "Description of the configuration" 177 | type = string 178 | default = null 179 | } 180 | 181 | variable "configuration_server_properties" { 182 | description = "Contents of the server.properties file. Supported properties are documented in the [MSK Developer Guide](https://docs.aws.amazon.com/msk/latest/developerguide/msk-configuration-properties.html)" 183 | type = map(string) 184 | default = {} 185 | } 186 | 187 | variable "kafka_config" { 188 | description = "Tho configuration of kafak" 189 | default = {} 190 | } 191 | 192 | ################################################################################ 193 | # CloudWatch Log Group 194 | ################################################################################ 195 | 196 | variable "create_cloudwatch_log_group" { 197 | description = "Determines whether to create a CloudWatch log group" 198 | type = bool 199 | default = true 200 | } 201 | 202 | variable "cloudwatch_log_group_name" { 203 | description = "Name of the Cloudwatch Log Group to deliver logs to" 204 | type = string 205 | default = null 206 | } 207 | 208 | variable "cloudwatch_log_group_retention_in_days" { 209 | description = "Specifies the number of days you want to retain log events in the log group" 210 | type = number 211 | default = 0 212 | } 213 | 214 | variable "cloudwatch_log_group_kms_key_id" { 215 | description = "The ARN of the KMS Key to use when encrypting log data" 216 | type = string 217 | default = null 218 | } 219 | -------------------------------------------------------------------------------- /msk-cluster/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13.1" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 3.71" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /msk-connetors/main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_cloudwatch_log_group" "kafka_logging" { 2 | name = "${data.terraform_remote_state.infra.outputs.environment_name}-${var.name}" 3 | retention_in_days = 30 4 | } 5 | 6 | ################################################################################ 7 | # Configuration 8 | ################################################################################ 9 | 10 | resource "aws_msk_configuration" "kafka_configuration" { 11 | count = var.create ? 1 : 0 12 | 13 | name = coalesce(var.configuration_name, var.name) 14 | description = var.configuration_description 15 | kafka_versions = [var.kafka_version] 16 | server_properties = join("\n", local.server_properties) 17 | } 18 | 19 | locals { 20 | server_properties = [ 21 | "num.partitions=${lookup(var.kafka_config, "num.partitions", "15")}", 22 | "default.replication.factor=${lookup(var.kafka_config, "default.replication.factor", "3")}", 23 | "auto.create.topics.enable=${lookup(var.kafka_config, "auto.create.topics.enable", "true")}", 24 | "delete.topic.enable=${lookup(var.kafka_config, "delete.topic.enable", "true")}" 25 | ] 26 | } 27 | 28 | ################################################################################ 29 | # MSK Cluster 30 | ################################################################################ 31 | 32 | resource "aws_msk_cluster" "kafka_cluster" { 33 | cluster_name = "${data.terraform_remote_state.infra.outputs.environment_name}-${var.name}" 34 | kafka_version = var.kafka_version 35 | number_of_broker_nodes = var.instance_count 36 | 37 | broker_node_group_info { 38 | instance_type = var.instance_type 39 | ebs_volume_size = var.kafka_data_disk 40 | client_subnets = length(var.subnets) > 0 ? var.subnets : data.terraform_remote_state.infra.outputs.subnet_ids 41 | security_groups = length(var.security_groups) > 0 ? var.security_groups : [data.terraform_remote_state.infra.outputs.default_sg_id] 42 | } 43 | 44 | configuration_info { 45 | arn = aws_msk_configuration.kafka_configuration.arn 46 | revision = aws_msk_configuration.kafka_configuration.latest_revision 47 | } 48 | 49 | encryption_info { 50 | encryption_in_transit { 51 | client_broker = var.client_broker_encryption 52 | in_cluster = true 53 | } 54 | } 55 | 56 | enhanced_monitoring = var.monitoring_type 57 | 58 | logging_info { 59 | broker_logs { 60 | cloudwatch_logs { 61 | enabled = true 62 | log_group = aws_cloudwatch_log_group.kafka_logging.name 63 | } 64 | } 65 | } 66 | 67 | tags = { 68 | Name = "${data.terraform_remote_state.infra.outputs.environment_name}-${var.name}" 69 | Environment = data.terraform_remote_state.infra.outputs.environment_name 70 | } 71 | } 72 | 73 | ################################################################################ 74 | # CloudWatch Log Group 75 | ################################################################################ 76 | 77 | resource "aws_cloudwatch_log_group" "this" { 78 | count = var.create && var.create_cloudwatch_log_group ? 1 : 0 79 | 80 | name = coalesce(var.cloudwatch_log_group_name, "/aws/msk/${var.name}") 81 | retention_in_days = var.cloudwatch_log_group_retention_in_days 82 | kms_key_id = var.cloudwatch_log_group_kms_key_id 83 | 84 | tags = var.tags 85 | } 86 | 87 | ################################################################################ 88 | # Connect Custom Plugin 89 | ################################################################################ 90 | 91 | resource "aws_mskconnect_custom_plugin" "this" { 92 | for_each = { for k, v in var.connect_custom_plugins : k => v if var.create } 93 | 94 | name = each.value.name 95 | description = lookup(each.value, "description", null) 96 | content_type = each.value.content_type 97 | 98 | location { 99 | s3 { 100 | bucket_arn = each.value.s3_bucket_arn 101 | file_key = each.value.s3_file_key 102 | object_version = lookup(each.value, "s3_object_version", null) 103 | } 104 | } 105 | 106 | timeouts { 107 | create = try(each.value.timeouts.create, var.connect_custom_plugin_timeouts.create, null) 108 | } 109 | } 110 | 111 | ################################################################################ 112 | # Connect Worker Configuration 113 | ################################################################################ 114 | 115 | resource "aws_mskconnect_worker_configuration" "this" { 116 | count = var.create && var.create_connect_worker_configuration ? 1 : 0 117 | 118 | name = var.connect_worker_config_name 119 | description = var.connect_worker_config_description 120 | properties_file_content = var.connect_worker_config_properties_file_content 121 | } 122 | 123 | resource "aws_route53_record" "kafka_dns_record" { 124 | for_each = var.no_dns ? {} : { for v in toset(split(",", replace(aws_msk_cluster.kafka_cluster.bootstrap_brokers, ":9092", ""))) : v => v } 125 | zone_id = data.terraform_remote_state.infra.outputs.route53_zone_id 126 | name = "${lower(var.name)}-broker${split("-", split(".", each.value)[0])[1]}.${data.terraform_remote_state.infra.outputs.route53_zone_name}" 127 | type = "CNAME" 128 | ttl = "30" 129 | records = [each.value] 130 | } 131 | 132 | 133 | 134 | -------------------------------------------------------------------------------- /msk-connetors/outputs.tf: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Cluster 3 | ################################################################################ 4 | 5 | output "arn" { 6 | description = "Amazon Resource Name (ARN) of the MSK cluster" 7 | value = try(aws_msk_cluster.this[0].arn, "") 8 | } 9 | 10 | output "bootstrap_brokers" { 11 | description = "Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster" 12 | value = compact([ 13 | try(aws_msk_cluster.this[0].bootstrap_brokers, ""), 14 | try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_iam, ""), 15 | try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_scram, ""), 16 | try(aws_msk_cluster.this[0].bootstrap_brokers_tls, ""), 17 | ]) 18 | } 19 | 20 | output "bootstrap_brokers_plaintext" { 21 | description = "Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster. Contains a value if `encryption_in_transit_client_broker` is set to `PLAINTEXT` or `TLS_PLAINTEXT`" 22 | value = try(aws_msk_cluster.this[0].bootstrap_brokers, "") 23 | } 24 | 25 | output "bootstrap_brokers_sasl_iam" { 26 | description = "One or more DNS names (or IP addresses) and SASL IAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_iam` is set to `true`" 27 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_iam, "") 28 | } 29 | 30 | output "bootstrap_brokers_sasl_scram" { 31 | description = "One or more DNS names (or IP addresses) and SASL SCRAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_scram` is set to `true`" 32 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_scram, "") 33 | } 34 | 35 | output "bootstrap_brokers_tls" { 36 | description = "One or more DNS names (or IP addresses) and TLS port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS`" 37 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_tls, "") 38 | } 39 | 40 | output "current_version" { 41 | description = "Current version of the MSK Cluster used for updates, e.g. `K13V1IB3VIYZZH`" 42 | value = try(aws_msk_cluster.this[0].current_version, "") 43 | } 44 | 45 | output "zookeeper_connect_string" { 46 | description = "A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster. The returned values are sorted alphabetically" 47 | value = try(aws_msk_cluster.this[0].zookeeper_connect_string, "") 48 | } 49 | 50 | output "zookeeper_connect_string_tls" { 51 | description = "A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster via TLS. The returned values are sorted alphabetically" 52 | value = try(aws_msk_cluster.this[0].zookeeper_connect_string_tls, "") 53 | } 54 | 55 | ################################################################################ 56 | # Configuration 57 | ################################################################################ 58 | 59 | output "configuration_arn" { 60 | description = "Amazon Resource Name (ARN) of the configuration" 61 | value = try(aws_msk_configuration.this[0].arn, "") 62 | } 63 | 64 | output "configuration_latest_revision" { 65 | description = "Latest revision of the configuration" 66 | value = try(aws_msk_configuration.this[0].latest_revision, "") 67 | } 68 | 69 | ################################################################################ 70 | # Secret(s) 71 | ################################################################################ 72 | 73 | output "scram_secret_association_id" { 74 | description = "Amazon Resource Name (ARN) of the MSK cluster" 75 | value = try(aws_msk_scram_secret_association.this[0].id, "") 76 | } 77 | 78 | ################################################################################ 79 | # CloudWatch Log Group 80 | ################################################################################ 81 | 82 | output "log_group_arn" { 83 | description = "The Amazon Resource Name (ARN) specifying the log group" 84 | value = try(aws_cloudwatch_log_group.this[0].arn, "") 85 | } 86 | 87 | -------------------------------------------------------------------------------- /msk-connetors/variables.tf: -------------------------------------------------------------------------------- 1 | variable "create" { 2 | description = "Determines whether cluster resources will be created" 3 | type = bool 4 | default = true 5 | } 6 | 7 | ################################################################################ 8 | # Cluster 9 | ################################################################################ 10 | 11 | variable "name" { 12 | description = "Name of the MSK cluster" 13 | type = string 14 | default = "msk" # to avoid: Error: cluster_name must be 1 characters or higher 15 | } 16 | 17 | variable "kafka_version" { 18 | description = "Specify the desired Kafka software version" 19 | type = string 20 | default = null 21 | } 22 | 23 | variable "number_of_broker_nodes" { 24 | description = "The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets" 25 | type = number 26 | default = null 27 | } 28 | 29 | variable "enhanced_monitoring" { 30 | description = "Specify the desired enhanced MSK CloudWatch monitoring level. See [Monitoring Amazon MSK with Amazon CloudWatch](https://docs.aws.amazon.com/msk/latest/developerguide/monitoring.html)" 31 | type = string 32 | default = null 33 | } 34 | 35 | variable "broker_node_client_subnets" { 36 | description = "A list of subnets to connect to in client VPC ([documentation](https://docs.aws.amazon.com/msk/1.0/apireference/clusters.html#clusters-prop-brokernodegroupinfo-clientsubnets))" 37 | type = list(string) 38 | default = [] 39 | } 40 | 41 | variable "broker_node_ebs_volume_size" { 42 | description = "The size in GiB of the EBS volume for the data drive on each broker node" 43 | type = number 44 | default = null 45 | } 46 | 47 | variable "broker_node_instance_type" { 48 | description = "Specify the instance type to use for the kafka brokers. e.g. kafka.m5.large. ([Pricing info](https://aws.amazon.com/msk/pricing/))" 49 | type = string 50 | default = null 51 | } 52 | 53 | variable "broker_node_security_groups" { 54 | description = "A list of the security groups to associate with the elastic network interfaces to control who can communicate with the cluster" 55 | type = list(string) 56 | default = [] 57 | } 58 | 59 | variable "client_authentication_tls_certificate_authority_arns" { 60 | description = "List of ACM Certificate Authority Amazon Resource Names (ARNs)" 61 | type = list(string) 62 | default = [] 63 | } 64 | 65 | variable "client_authentication_sasl_iam" { 66 | description = "Enables IAM client authentication" 67 | type = bool 68 | default = false 69 | } 70 | 71 | variable "client_authentication_sasl_scram" { 72 | description = "Enables SCRAM client authentication via AWS Secrets Manager" 73 | type = bool 74 | default = false 75 | } 76 | 77 | variable "encryption_in_transit_client_broker" { 78 | description = "Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT`. Default value is `TLS`" 79 | type = string 80 | default = null 81 | } 82 | 83 | variable "encryption_in_transit_in_cluster" { 84 | description = "Whether data communication among broker nodes is encrypted. Default value: `true`" 85 | type = bool 86 | default = null 87 | } 88 | 89 | variable "encryption_at_rest_kms_key_arn" { 90 | description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest" 91 | type = string 92 | default = null 93 | } 94 | 95 | variable "jmx_exporter_enabled" { 96 | description = "Indicates whether you want to enable or disable the JMX Exporter" 97 | type = bool 98 | default = false 99 | } 100 | 101 | variable "node_exporter_enabled" { 102 | description = "Indicates whether you want to enable or disable the Node Exporter" 103 | type = bool 104 | default = false 105 | } 106 | 107 | variable "cloudwatch_logs_enabled" { 108 | description = "Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs" 109 | type = bool 110 | default = false 111 | } 112 | 113 | variable "firehose_logs_enabled" { 114 | description = "Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose" 115 | type = bool 116 | default = false 117 | } 118 | 119 | variable "firehose_delivery_stream" { 120 | description = "Name of the Kinesis Data Firehose delivery stream to deliver logs to" 121 | type = string 122 | default = null 123 | } 124 | 125 | variable "s3_logs_enabled" { 126 | description = "Indicates whether you want to enable or disable streaming broker logs to S3" 127 | type = bool 128 | default = false 129 | } 130 | 131 | variable "s3_logs_bucket" { 132 | description = "Name of the S3 bucket to deliver logs to" 133 | type = string 134 | default = null 135 | } 136 | 137 | variable "s3_logs_prefix" { 138 | description = "Prefix to append to the folder name" 139 | type = string 140 | default = null 141 | } 142 | 143 | variable "timeouts" { 144 | description = "Create, update, and delete timeout configurations for the cluster" 145 | type = map(string) 146 | default = {} 147 | } 148 | 149 | variable "tags" { 150 | description = "A map of tags to assign to the resources created" 151 | type = map(string) 152 | default = {} 153 | } 154 | 155 | ################################################################################ 156 | # Configuration 157 | ################################################################################ 158 | 159 | variable "configuration_name" { 160 | description = "Name of the configuration" 161 | type = string 162 | default = null 163 | } 164 | 165 | variable "configuration_description" { 166 | description = "Description of the configuration" 167 | type = string 168 | default = null 169 | } 170 | 171 | variable "configuration_server_properties" { 172 | description = "Contents of the server.properties file. Supported properties are documented in the [MSK Developer Guide](https://docs.aws.amazon.com/msk/latest/developerguide/msk-configuration-properties.html)" 173 | type = map(string) 174 | default = {} 175 | } 176 | -------------------------------------------------------------------------------- /msk-connetors/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13.1" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 3.71" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /msk-custom-plugin/main.tf: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Custom Plugin 3 | ################################################################################ 4 | 5 | resource "aws_mskconnect_custom_plugin" "this" { 6 | for_each = { for k, v in var.connect_custom_plugins : k => v if var.create } 7 | 8 | name = each.value.name 9 | description = lookup(each.value, "description", null) 10 | content_type = each.value.content_type 11 | 12 | location { 13 | s3 { 14 | bucket_arn = each.value.s3_bucket_arn 15 | file_key = each.value.s3_file_key 16 | object_version = lookup(each.value, "s3_object_version", null) 17 | } 18 | } 19 | 20 | timeouts { 21 | create = try(each.value.timeouts.create, var.connect_custom_plugin_timeouts.create, null) 22 | } 23 | } 24 | 25 | ################################################################################ 26 | # Worker Configuration 27 | ################################################################################ 28 | 29 | resource "aws_mskconnect_worker_configuration" "this" { 30 | count = var.create && var.create_connect_worker_configuration ? 1 : 0 31 | 32 | name = var.connect_worker_config_name 33 | description = var.connect_worker_config_description 34 | properties_file_content = var.connect_worker_config_properties_file_content 35 | } 36 | -------------------------------------------------------------------------------- /msk-custom-plugin/outputs.tf: -------------------------------------------------------------------------------- 1 | ################################################################################ 2 | # Cluster 3 | ################################################################################ 4 | 5 | output "arn" { 6 | description = "Amazon Resource Name (ARN) of the MSK cluster" 7 | value = try(aws_msk_cluster.this[0].arn, "") 8 | } 9 | 10 | output "bootstrap_brokers" { 11 | description = "Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster" 12 | value = compact([ 13 | try(aws_msk_cluster.this[0].bootstrap_brokers, ""), 14 | try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_iam, ""), 15 | try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_scram, ""), 16 | try(aws_msk_cluster.this[0].bootstrap_brokers_tls, ""), 17 | ]) 18 | } 19 | 20 | output "bootstrap_brokers_plaintext" { 21 | description = "Comma separated list of one or more hostname:port pairs of kafka brokers suitable to bootstrap connectivity to the kafka cluster. Contains a value if `encryption_in_transit_client_broker` is set to `PLAINTEXT` or `TLS_PLAINTEXT`" 22 | value = try(aws_msk_cluster.this[0].bootstrap_brokers, "") 23 | } 24 | 25 | output "bootstrap_brokers_sasl_iam" { 26 | description = "One or more DNS names (or IP addresses) and SASL IAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_iam` is set to `true`" 27 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_iam, "") 28 | } 29 | 30 | output "bootstrap_brokers_sasl_scram" { 31 | description = "One or more DNS names (or IP addresses) and SASL SCRAM port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS` and `client_authentication_sasl_scram` is set to `true`" 32 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_sasl_scram, "") 33 | } 34 | 35 | output "bootstrap_brokers_tls" { 36 | description = "One or more DNS names (or IP addresses) and TLS port pairs. This attribute will have a value if `encryption_in_transit_client_broker` is set to `TLS_PLAINTEXT` or `TLS`" 37 | value = try(aws_msk_cluster.this[0].bootstrap_brokers_tls, "") 38 | } 39 | 40 | output "current_version" { 41 | description = "Current version of the MSK Cluster used for updates, e.g. `K13V1IB3VIYZZH`" 42 | value = try(aws_msk_cluster.this[0].current_version, "") 43 | } 44 | 45 | output "zookeeper_connect_string" { 46 | description = "A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster. The returned values are sorted alphabetically" 47 | value = try(aws_msk_cluster.this[0].zookeeper_connect_string, "") 48 | } 49 | 50 | output "zookeeper_connect_string_tls" { 51 | description = "A comma separated list of one or more hostname:port pairs to use to connect to the Apache Zookeeper cluster via TLS. The returned values are sorted alphabetically" 52 | value = try(aws_msk_cluster.this[0].zookeeper_connect_string_tls, "") 53 | } 54 | 55 | ################################################################################ 56 | # Configuration 57 | ################################################################################ 58 | 59 | output "configuration_arn" { 60 | description = "Amazon Resource Name (ARN) of the configuration" 61 | value = try(aws_msk_configuration.this[0].arn, "") 62 | } 63 | 64 | output "configuration_latest_revision" { 65 | description = "Latest revision of the configuration" 66 | value = try(aws_msk_configuration.this[0].latest_revision, "") 67 | } 68 | 69 | ################################################################################ 70 | # Secret(s) 71 | ################################################################################ 72 | 73 | output "scram_secret_association_id" { 74 | description = "Amazon Resource Name (ARN) of the MSK cluster" 75 | value = try(aws_msk_scram_secret_association.this[0].id, "") 76 | } 77 | 78 | ################################################################################ 79 | # CloudWatch Log Group 80 | ################################################################################ 81 | 82 | output "log_group_arn" { 83 | description = "The Amazon Resource Name (ARN) specifying the log group" 84 | value = try(aws_cloudwatch_log_group.this[0].arn, "") 85 | } 86 | -------------------------------------------------------------------------------- /msk-custom-plugin/variables.tf: -------------------------------------------------------------------------------- 1 | variable "create" { 2 | description = "Determines whether cluster resources will be created" 3 | type = bool 4 | default = true 5 | } 6 | 7 | ################################################################################ 8 | # Cluster 9 | ################################################################################ 10 | 11 | variable "name" { 12 | description = "Name of the MSK cluster" 13 | type = string 14 | default = "msk" # to avoid: Error: cluster_name must be 1 characters or higher 15 | } 16 | 17 | variable "kafka_version" { 18 | description = "Specify the desired Kafka software version" 19 | type = string 20 | default = null 21 | } 22 | 23 | variable "number_of_broker_nodes" { 24 | description = "The desired total number of broker nodes in the kafka cluster. It must be a multiple of the number of specified client subnets" 25 | type = number 26 | default = null 27 | } 28 | 29 | variable "enhanced_monitoring" { 30 | description = "Specify the desired enhanced MSK CloudWatch monitoring level. See [Monitoring Amazon MSK with Amazon CloudWatch](https://docs.aws.amazon.com/msk/latest/developerguide/monitoring.html)" 31 | type = string 32 | default = null 33 | } 34 | 35 | variable "broker_node_client_subnets" { 36 | description = "A list of subnets to connect to in client VPC ([documentation](https://docs.aws.amazon.com/msk/1.0/apireference/clusters.html#clusters-prop-brokernodegroupinfo-clientsubnets))" 37 | type = list(string) 38 | default = [] 39 | } 40 | 41 | variable "broker_node_ebs_volume_size" { 42 | description = "The size in GiB of the EBS volume for the data drive on each broker node" 43 | type = number 44 | default = null 45 | } 46 | 47 | variable "broker_node_instance_type" { 48 | description = "Specify the instance type to use for the kafka brokers. e.g. kafka.m5.large. ([Pricing info](https://aws.amazon.com/msk/pricing/))" 49 | type = string 50 | default = null 51 | } 52 | 53 | variable "broker_node_security_groups" { 54 | description = "A list of the security groups to associate with the elastic network interfaces to control who can communicate with the cluster" 55 | type = list(string) 56 | default = [] 57 | } 58 | 59 | variable "client_authentication_tls_certificate_authority_arns" { 60 | description = "List of ACM Certificate Authority Amazon Resource Names (ARNs)" 61 | type = list(string) 62 | default = [] 63 | } 64 | 65 | variable "client_authentication_sasl_iam" { 66 | description = "Enables IAM client authentication" 67 | type = bool 68 | default = false 69 | } 70 | 71 | variable "client_authentication_sasl_scram" { 72 | description = "Enables SCRAM client authentication via AWS Secrets Manager" 73 | type = bool 74 | default = false 75 | } 76 | 77 | variable "encryption_in_transit_client_broker" { 78 | description = "Encryption setting for data in transit between clients and brokers. Valid values: `TLS`, `TLS_PLAINTEXT`, and `PLAINTEXT`. Default value is `TLS`" 79 | type = string 80 | default = null 81 | } 82 | 83 | variable "encryption_in_transit_in_cluster" { 84 | description = "Whether data communication among broker nodes is encrypted. Default value: `true`" 85 | type = bool 86 | default = null 87 | } 88 | 89 | variable "encryption_at_rest_kms_key_arn" { 90 | description = "You may specify a KMS key short ID or ARN (it will always output an ARN) to use for encrypting your data at rest. If no key is specified, an AWS managed KMS ('aws/msk' managed service) key will be used for encrypting the data at rest" 91 | type = string 92 | default = null 93 | } 94 | 95 | variable "jmx_exporter_enabled" { 96 | description = "Indicates whether you want to enable or disable the JMX Exporter" 97 | type = bool 98 | default = false 99 | } 100 | 101 | variable "node_exporter_enabled" { 102 | description = "Indicates whether you want to enable or disable the Node Exporter" 103 | type = bool 104 | default = false 105 | } 106 | 107 | variable "cloudwatch_logs_enabled" { 108 | description = "Indicates whether you want to enable or disable streaming broker logs to Cloudwatch Logs" 109 | type = bool 110 | default = false 111 | } 112 | 113 | variable "firehose_logs_enabled" { 114 | description = "Indicates whether you want to enable or disable streaming broker logs to Kinesis Data Firehose" 115 | type = bool 116 | default = false 117 | } 118 | 119 | variable "firehose_delivery_stream" { 120 | description = "Name of the Kinesis Data Firehose delivery stream to deliver logs to" 121 | type = string 122 | default = null 123 | } 124 | 125 | variable "s3_logs_enabled" { 126 | description = "Indicates whether you want to enable or disable streaming broker logs to S3" 127 | type = bool 128 | default = false 129 | } 130 | 131 | variable "s3_logs_bucket" { 132 | description = "Name of the S3 bucket to deliver logs to" 133 | type = string 134 | default = null 135 | } 136 | 137 | variable "s3_logs_prefix" { 138 | description = "Prefix to append to the folder name" 139 | type = string 140 | default = null 141 | } 142 | 143 | variable "timeouts" { 144 | description = "Create, update, and delete timeout configurations for the cluster" 145 | type = map(string) 146 | default = {} 147 | } 148 | 149 | variable "tags" { 150 | description = "A map of tags to assign to the resources created" 151 | type = map(string) 152 | default = {} 153 | } 154 | 155 | ################################################################################ 156 | # Configuration 157 | ################################################################################ 158 | 159 | variable "configuration_name" { 160 | description = "Name of the configuration" 161 | type = string 162 | default = null 163 | } 164 | 165 | variable "configuration_description" { 166 | description = "Description of the configuration" 167 | type = string 168 | default = null 169 | } 170 | 171 | variable "configuration_server_properties" { 172 | description = "Contents of the server.properties file. Supported properties are documented in the [MSK Developer Guide](https://docs.aws.amazon.com/msk/latest/developerguide/msk-configuration-properties.html)" 173 | type = map(string) 174 | default = {} 175 | } 176 | -------------------------------------------------------------------------------- /msk-custom-plugin/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13.1" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 3.71" 8 | } 9 | } 10 | } 11 | -------------------------------------------------------------------------------- /msk-dashboard/main.tf: -------------------------------------------------------------------------------- 1 | data "terraform_remote_state" "infra" { 2 | backend = "s3" 3 | config = { 4 | bucket = var.infra_data_s3_name 5 | key = var.infra_data_s3_key 6 | region = var.infra_data_s3_region 7 | } 8 | } 9 | 10 | resource "kubernetes_service" "kafka-ui" { 11 | count = var.enabled ? 1 : 0 12 | metadata { 13 | name = var.host_name 14 | namespace = var.namespace 15 | labels = { 16 | app = "${var.host_name}" 17 | } 18 | } 19 | 20 | spec { 21 | port { 22 | name = "http" 23 | port = 8080 24 | target_port = "8080" 25 | } 26 | 27 | selector = { 28 | app = "${var.host_name}" 29 | } 30 | 31 | type = "NodePort" 32 | } 33 | } 34 | 35 | 36 | resource "kubernetes_deployment" "kafka-ui" { 37 | count = var.enabled ? 1 : 0 38 | metadata { 39 | name = var.host_name 40 | namespace = var.namespace 41 | } 42 | 43 | spec { 44 | replicas = var.replicas 45 | 46 | selector { 47 | match_labels = { 48 | app = "${var.host_name}" 49 | } 50 | } 51 | 52 | template { 53 | metadata { 54 | labels = { 55 | app = "${var.host_name}" 56 | } 57 | } 58 | 59 | spec { 60 | container { 61 | name = var.host_name 62 | image = "${data.terraform_remote_state.infra.outputs.owner_id}.dkr.ecr.${data.terraform_remote_state.infra.outputs.aws_region}.amazonaws.com/kafkauiimage:${var.kafka_ui_image}" 63 | 64 | port { 65 | name = "http" 66 | container_port = 8080 67 | protocol = "TCP" 68 | } 69 | 70 | env { 71 | name = "KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS" 72 | value = var.kafka_bootstap 73 | } 74 | 75 | env { 76 | name = "KAFKA_CLUSTERS_0_NAME" 77 | value = "local" 78 | } 79 | 80 | } 81 | node_selector = { 82 | "kubernetes.io/role" = "node" 83 | } 84 | } 85 | } 86 | } 87 | } 88 | 89 | resource "kubernetes_ingress" "kafka-ui_internal" { 90 | count = var.public_ingress ? 1 : 0 91 | metadata { 92 | name = var.host_name 93 | namespace = var.namespace 94 | annotations = { 95 | "alb.ingress.kubernetes.io/group.name" = "default" 96 | 97 | "alb.ingress.kubernetes.io/listen-ports" = "[{\"HTTPS\": 443}]" 98 | 99 | "alb.ingress.kubernetes.io/scheme" = "internal" 100 | 101 | "alb.ingress.kubernetes.io/security-groups" = "${var.security_groups}" 102 | 103 | "alb.ingress.kubernetes.io/tags" = "${var.host_name}" 104 | 105 | "kubernetes.io/ingress.class" = "alb" 106 | } 107 | } 108 | spec { 109 | rule { 110 | host = "${var.host_name}.${data.terraform_remote_state.infra.outputs.route53_zone_name}" 111 | 112 | http { 113 | path { 114 | backend { 115 | service_name = var.host_name 116 | service_port = "8080" 117 | } 118 | } 119 | } 120 | } 121 | } 122 | } -------------------------------------------------------------------------------- /msk-dashboard/variables.tf: -------------------------------------------------------------------------------- 1 | variable "infra_data_s3_name" {} 2 | variable "infra_data_s3_region" {} 3 | variable "infra_data_s3_key" {} 4 | variable "enabled" { default = true } 5 | variable "host_name" { default = "kafka-ui" } 6 | variable "eks_name" {} 7 | variable "security_groups" {} 8 | variable "kafka_bootstap" {} 9 | variable "replicas" { default = "1" } 10 | variable "public_ingress" { default = true } 11 | variable "namespace" { default = "default" } 12 | variable "kafka_ui_image" { default = "0.2.1-20211215114334" } 13 | -------------------------------------------------------------------------------- /msk-dashboard/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.13.1" 3 | 4 | required_providers { 5 | aws = { 6 | source = "hashicorp/aws" 7 | version = ">= 3.71" 8 | } 9 | } 10 | } 11 | 12 | provider "aws" { 13 | region = data.terraform_remote_state.infra.outputs.aws_region 14 | } 15 | 16 | data "aws_eks_cluster" "eks" { 17 | name = var.eks_name 18 | } 19 | 20 | data "aws_eks_cluster_auth" "eks" { 21 | name = var.eks_name 22 | } 23 | 24 | provider "kubernetes" { 25 | host = data.aws_eks_cluster.eks.endpoint 26 | cluster_ca_certificate = base64decode(data.aws_eks_cluster.eks.certificate_authority.0.data) 27 | token = data.aws_eks_cluster_auth.eks.token 28 | } 29 | --------------------------------------------------------------------------------