├── .github ├── CODEOWNERS └── workflows │ ├── terraform-docs.yml │ ├── terraform-master.yml │ └── terraform-tests.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── alb.tf ├── ecs.tf ├── examples ├── basic-local-with-requirements │ ├── main.tf │ └── requirements.txt ├── basic-sequential-rbac │ ├── main.tf │ └── requirements.txt └── basic-sequential │ └── main.tf ├── go.mod ├── go.sum ├── iam.tf ├── locals.tf ├── main.tf ├── outputs.tf ├── rds.tf ├── route53.tf ├── s3.tf ├── templates ├── dags │ ├── airflow_seed_dag.py │ └── example_dag.py └── startup │ ├── entrypoint_init.sh │ ├── entrypoint_scheduler.sh │ └── entrypoint_webserver.sh ├── test ├── preexisting │ └── main.tf ├── provider.tf └── terraform_apply_destroy_test.go └── variables.tf /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | .github/* @datarootsio/terraform-modules -------------------------------------------------------------------------------- /.github/workflows/terraform-docs.yml: -------------------------------------------------------------------------------- 1 | name: "docs" 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | - open-pr-here 10 | jobs: 11 | docs: 12 | name: "Docs" 13 | runs-on: ubuntu-latest 14 | steps: 15 | - uses: actions/checkout@v2 16 | with: 17 | ref: ${{ github.event.pull_request.head.ref }} 18 | 19 | - name: Render terraform docs inside the README.md and push changes back to PR branch 20 | uses: Dirrk/terraform-docs@v1.0.8 21 | with: 22 | tf_docs_working_dir: . 23 | tf_docs_output_file: README.md 24 | tf_docs_output_method: inject 25 | tf_docs_git_push: 'true' -------------------------------------------------------------------------------- /.github/workflows/terraform-master.yml: -------------------------------------------------------------------------------- 1 | name: 'publish' 2 | on: 3 | push: 4 | branches: 5 | - main 6 | jobs: 7 | publish: 8 | name: 'Publish' 9 | runs-on: ubuntu-latest 10 | steps: 11 | - name: 'Checkout' 12 | uses: actions/checkout@main 13 | - name: Find Tag 14 | id: get_latest_tag 15 | uses: jimschubert/query-tag-action@v1 16 | with: 17 | include: 'v*' 18 | exclude: '*-rc*' 19 | commit-ish: 'HEAD~' 20 | - name: 'Get next version' 21 | id: next_tag 22 | uses: "WyriHaximus/github-action-next-semvers@master" 23 | with: 24 | version: ${{ steps.get_latest_tag.outputs.tag }} 25 | - name: Create Release 26 | id: create_release 27 | uses: actions/create-release@v1 28 | env: 29 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 30 | with: 31 | tag_name: "${{ steps.next_tag.outputs.v_patch }}" 32 | release_name: "${{ steps.next_tag.outputs.v_patch }}" 33 | body: | 34 | Automatic release for ${{ steps.next_tag.outputs.v_patch }} 35 | draft: false 36 | prerelease: false 37 | - name: Push changes 38 | uses: ad-m/github-push-action@master 39 | with: 40 | github_token: ${{ secrets.GITHUB_TOKEN }} 41 | branch: open-pr-here 42 | force: true -------------------------------------------------------------------------------- /.github/workflows/terraform-tests.yml: -------------------------------------------------------------------------------- 1 | name: "tests" 2 | on: 3 | push: 4 | branches: 5 | - main 6 | pull_request: 7 | branches: 8 | - main 9 | - open-pr-here 10 | jobs: 11 | validate: 12 | name: "Validate" 13 | runs-on: ubuntu-latest 14 | if: ${{ github.event_name == 'pull_request' }} 15 | steps: 16 | - name: "Checkout" 17 | uses: actions/checkout@master 18 | 19 | - uses: hashicorp/setup-terraform@v1 20 | with: 21 | terraform_version: 0.15.4 22 | 23 | - name: Terraform Init 24 | run: terraform init 25 | 26 | - name: Terraform Format 27 | run: terraform fmt -check 28 | 29 | - name: Terraform Validate 30 | run: terraform validate 31 | 32 | - name: tflint 33 | run: docker run --rm -v $(pwd):/data -t wata727/tflint 34 | test: 35 | env: 36 | tests_timeout: "2h" 37 | golangci_lint_version: "v1.32" 38 | AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} 39 | AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} 40 | AWS_DEFAULT_REGION: eu-west-1 41 | name: "Tests" 42 | runs-on: ubuntu-latest 43 | if: ${{ github.event_name == 'pull_request' && github.base_ref == 'main' && github.head_ref == 'open-pr-here' }} 44 | steps: 45 | - name: "Checkout" 46 | uses: actions/checkout@master 47 | 48 | - name: "go vet" 49 | run: go vet ./... 50 | 51 | - name: golangci-lint 52 | uses: golangci/golangci-lint-action@v2 53 | with: 54 | # Optional: golangci-lint command line arguments. 55 | args: --timeout=3m0s 56 | version: ${{ env.golangci_lint_version }} 57 | 58 | - uses: hashicorp/setup-terraform@v1 59 | with: 60 | terraform_version: 0.15.4 61 | 62 | - name: "go test" 63 | run: | 64 | go test -v -timeout ${{ env.tests_timeout }} ./... 65 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | 2 | # Local .terraform directories 3 | **/.terraform/* 4 | 5 | # .tfstate files 6 | *.tfstate 7 | *.tfstate.* 8 | 9 | # Crash log files 10 | crash.log 11 | 12 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 13 | # .tfvars files are managed as part of configuration and so should be included in 14 | # version control. 15 | # 16 | # example.tfvars 17 | 18 | # Ignore override files as they are usually used to override resources locally and so 19 | # are not checked in 20 | override.tf 21 | override.tf.json 22 | *_override.tf 23 | *_override.tf.json 24 | 25 | # Include override files you do wish to add to version control using negated pattern 26 | # 27 | # !example_override.tf 28 | 29 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 30 | # example: *tfplan* 31 | .idea 32 | .vscode 33 | 34 | *.tfvars -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 dataroots 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | 2 | default: lint 3 | 4 | tools: 5 | go install gotest.tools/gotestsum 6 | terraform init 7 | 8 | fmt: 9 | terraform fmt 10 | go mod tidy 11 | gofmt -w -s test 12 | 13 | lint-tf: tools 14 | terraform fmt -check 15 | terraform validate 16 | tflint 17 | 18 | lint-go: 19 | test -z $(gofmt -l -s test) 20 | go vet ./... 21 | 22 | lint: lint-tf lint-go 23 | 24 | test: tools lint 25 | go test -timeout 2h ./... 26 | 27 | testverbose: tools lint 28 | go test -v -timeout 2h ./... -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | [![Maintained by dataroots](https://img.shields.io/badge/maintained%20by-dataroots-%2300b189)](https://dataroots.io) 2 | [![Airflow version](https://img.shields.io/badge/Apache%20Airflow-2.0.1-e27d60.svg)](https://airflow.apache.org/) 3 | [![Terraform 0.15](https://img.shields.io/badge/terraform-0.15-%23623CE4)](https://www.terraform.io) 4 | [![Terraform Registry](https://img.shields.io/badge/terraform-registry-%23623CE4)](https://registry.terraform.io/modules/datarootsio/ecs-airflow/aws) 5 | [![Tests](https://github.com/datarootsio/terraform-aws-ecs-airflow/workflows/tests/badge.svg?branch=main)](https://github.com/datarootsio/terraform-aws-ecs-airflow/actions) 6 | [![Go Report Card](https://goreportcard.com/badge/github.com/datarootsio/terraform-aws-ecs-airflow)](https://goreportcard.com/report/github.com/datarootsio/terraform-aws-ecs-airflow) 7 | 8 | ![](https://scontent.fbru1-1.fna.fbcdn.net/v/t1.0-9/94305647_112517570431823_3318660558911176704_o.png?_nc_cat=111&_nc_sid=e3f864&_nc_ohc=-spbrtnzSpQAX_qi7iI&_nc_ht=scontent.fbru1-1.fna&oh=483d147a29972c72dfb588b91d57ac3c&oe=5F99368A "Logo") 9 | 10 | ### DEPRECATED: We are no longer actively maintaining this module, instead we recommend that you look into [AWS MWAA](https://aws.amazon.com/managed-workflows-for-apache-airflow/) as a replacement. 11 | 12 | # Terraform module Airflow on AWS ECS 13 | 14 | This is a module for Terraform that deploys Airflow in AWS. 15 | 16 | ## Setup 17 | 18 | - An ECS Cluster with: 19 | - Sidecar injection container 20 | - Airflow init container 21 | - Airflow webserver container 22 | - Airflow scheduler container 23 | - An ALB 24 | - A RDS instance (optional but recommended) 25 | - A DNS Record (optional but recommended) 26 | - A S3 Bucket (optional) 27 | 28 | Average cost of the minimal setup (with RDS): ~50$/Month 29 | 30 | Why do I need a RDS instance? 31 | 1. This makes Airflow statefull, you will be able to rerun failed dags, keep history of failed/succeeded dags, ... 32 | 2. It allows for dags to run concurrently, otherwise two dags will not be able to run at the same time 33 | 3. The state of your dags persists, even if the Airflow container fails or if you update the container definition (this will trigger an update of the ECS task) 34 | 35 | ## Intend 36 | 37 | The Airflow setup provided with this module, is a setup where the only task of Airflow is to manage your jobs/workflows. So not to do actually heavy lifting like SQL queries, Spark jobs, ... . Offload as many task to AWS Lambda, AWS EMR, AWS Glue, ... . If you want Airflow to have access to these services, use the output role and give it permissions to these services through IAM. 38 | 39 | ## Usage 40 | 41 | ```hcl 42 | module "airflow" { 43 | source = "datarootsio/ecs-airflow/aws" 44 | 45 | resource_prefix = "my-awesome-company" 46 | resource_suffix = "env" 47 | 48 | vpc_id = "vpc-123456" 49 | public_subnet_ids = ["subnet-456789", "subnet-098765"] 50 | 51 | rds_password = "super-secret-pass" 52 | } 53 | ``` 54 | (This will create Airflow, backed up by an RDS (both in a public subnet) and without https) 55 | 56 | [Press here to see more examples](https://github.com/datarootsio/terraform-aws-ecs-airflow/tree/main/examples) 57 | 58 | Note: After that Terraform is done deploying everything, it can take up to a minute for Airflow to be available through HTTP(S) 59 | 60 | ## Adding DAGs 61 | 62 | To add dags, upload them to the created S3 bucket in the subdir "dags/". After you uploaded them run the seed dag. This will sync the s3 bucket with the local dags folder of the ECS container. 63 | 64 | ## Authentication 65 | 66 | For now the only authentication option is 'RBAC'. When enabling this, this module will create a default admin role (only if there are no users in the database). This default role is just a one time entrypoint in to the airflow web interface. When you log in for the first time immediately change the password! Also with this default admin role you can create any user you want. 67 | 68 | ## Todo 69 | 70 | - [ ] RDS Backup options 71 | - [ ] Option to use SQL instead of Postgres 72 | - [ ] Add a Lambda function that triggers the sync dag (so that you can auto sync through ci/cd) 73 | - [x] RBAC 74 | - [ ] Support for [Google OAUTH](https://airflow.readthedocs.io/en/latest/security.html#google-authentication) 75 | 76 | 77 | ## Requirements 78 | 79 | | Name | Version | 80 | |------|---------| 81 | | terraform | ~> 0.15 | 82 | | aws | ~> 3.12.0 | 83 | 84 | ## Providers 85 | 86 | | Name | Version | 87 | |------|---------| 88 | | aws | ~> 3.12.0 | 89 | 90 | ## Inputs 91 | 92 | | Name | Description | Type | Default | Required | 93 | |------|-------------|------|---------|:--------:| 94 | | airflow\_authentication | Authentication backend to be used, supported backends ["", "rbac"]. When "rbac" is selected an admin role is create if there are no other users in the db, from here you can create all the other users. Make sure to change the admin password directly upon first login! (if you don't change the rbac\_admin options the default login is => username: admin, password: admin) | `string` | `""` | no | 95 | | airflow\_container\_home | Working dir for airflow (only change if you are using a different image) | `string` | `"/opt/airflow"` | no | 96 | | airflow\_example\_dag | Add an example dag on startup (mostly for sanity check) | `bool` | `true` | no | 97 | | airflow\_executor | The executor mode that airflow will use. Only allowed values are ["Local", "Sequential"]. "Local": Run DAGs in parallel (will created a RDS); "Sequential": You can not run DAGs in parallel (will NOT created a RDS); | `string` | `"Local"` | no | 98 | | airflow\_image\_name | The name of the airflow image | `string` | `"apache/airflow"` | no | 99 | | airflow\_image\_tag | The tag of the airflow image | `string` | `"2.0.1"` | no | 100 | | airflow\_log\_region | The region you want your airflow logs in, defaults to the region variable | `string` | `""` | no | 101 | | airflow\_log\_retention | The number of days you want to keep the log of airflow container | `string` | `"7"` | no | 102 | | airflow\_py\_requirements\_path | The relative path to a python requirements.txt file to install extra packages in the container that you can use in your DAGs. | `string` | `""` | no | 103 | | airflow\_variables | The variables passed to airflow as an environment variable (see airflow docs for more info https://airflow.apache.org/docs/). You can not specify "AIRFLOW\_\_CORE\_\_SQL\_ALCHEMY\_CONN" and "AIRFLOW\_\_CORE\_\_EXECUTOR" (managed by this module) | `map(string)` | `{}` | no | 104 | | certificate\_arn | The ARN of the certificate that will be used | `string` | `""` | no | 105 | | dns\_name | The DNS name that will be used to expose Airflow. Optional if not serving over HTTPS. Will be autogenerated if not provided | `string` | `""` | no | 106 | | ecs\_cpu | The allocated cpu for your airflow instance | `number` | `1024` | no | 107 | | ecs\_memory | The allocated memory for your airflow instance | `number` | `2048` | no | 108 | | extra\_tags | Extra tags that you would like to add to all created resources | `map(string)` | `{}` | no | 109 | | ip\_allow\_list | A list of ip ranges that are allowed to access the airflow webserver, default: full access | `list(string)` | 
[
  "0.0.0.0/0"
]
| no | 110 | | postgres\_uri | The postgres uri of your postgres db, if none provided a postgres db in rds is made. Format ":@:/" | `string` | `""` | no | 111 | | private\_subnet\_ids | A list of subnet ids of where the ECS and RDS reside, this will only work if you have a NAT Gateway in your VPC | `list(string)` | `[]` | no | 112 | | public\_subnet\_ids | A list of subnet ids of where the ALB will reside, if the "private\_subnet\_ids" variable is not provided ECS and RDS will also reside in these subnets | `list(string)` | n/a | yes | 113 | | rbac\_admin\_email | RBAC Email (only when airflow\_authentication = 'rbac') | `string` | `"admin@admin.com"` | no | 114 | | rbac\_admin\_firstname | RBAC Firstname (only when airflow\_authentication = 'rbac') | `string` | `"admin"` | no | 115 | | rbac\_admin\_lastname | RBAC Lastname (only when airflow\_authentication = 'rbac') | `string` | `"airflow"` | no | 116 | | rbac\_admin\_password | RBAC Password (only when airflow\_authentication = 'rbac') | `string` | `"admin"` | no | 117 | | rbac\_admin\_username | RBAC Username (only when airflow\_authentication = 'rbac') | `string` | `"admin"` | no | 118 | | rds\_allocated\_storage | The allocated storage for the rds db in gibibytes | `number` | `20` | no | 119 | | rds\_availability\_zone | Availability zone for the rds instance | `string` | `"eu-west-1a"` | no | 120 | | rds\_deletion\_protection | Deletion protection for the rds instance | `bool` | `false` | no | 121 | | rds\_engine | The database engine to use. For supported values, see the Engine parameter in [API action CreateDBInstance](https://docs.aws.amazon.com/AmazonRDS/latest/APIReference/API_CreateDBInstance.html) | `string` | `"postgres"` | no | 122 | | rds\_instance\_class | The class of instance you want to give to your rds db | `string` | `"db.t2.micro"` | no | 123 | | rds\_password | Password of rds | `string` | `""` | no | 124 | | rds\_skip\_final\_snapshot | Whether or not to skip the final snapshot before deleting (mainly for tests) | `bool` | `false` | no | 125 | | rds\_storage\_type | One of `"standard"` (magnetic), `"gp2"` (general purpose SSD), or `"io1"` (provisioned IOPS SSD) | `string` | `"standard"` | no | 126 | | rds\_username | Username of rds | `string` | `"airflow"` | no | 127 | | rds\_version | The DB version to use for the RDS instance | `string` | `"12.7"` | no | 128 | | region | The region to deploy your solution to | `string` | `"eu-west-1"` | no | 129 | | resource\_prefix | A prefix for the create resources, example your company name (be aware of the resource name length) | `string` | n/a | yes | 130 | | resource\_suffix | A suffix for the created resources, example the environment for airflow to run in (be aware of the resource name length) | `string` | n/a | yes | 131 | | route53\_zone\_name | The name of a Route53 zone that will be used for the certificate validation. | `string` | `""` | no | 132 | | s3\_bucket\_name | The S3 bucket name where the DAGs and startup scripts will be stored, leave this blank to let this module create a s3 bucket for you. WARNING: this module will put files into the path "dags/" and "startup/" of the bucket | `string` | `""` | no | 133 | | use\_https | Expose traffic using HTTPS or not | `bool` | `false` | no | 134 | | vpc\_id | The id of the vpc where you will run ECS/RDS | `string` | n/a | yes | 135 | 136 | ## Outputs 137 | 138 | | Name | Description | 139 | |------|-------------| 140 | | airflow\_alb\_dns | The DNS name of the ALB, with this you can access the Airflow webserver | 141 | | airflow\_connection\_sg | The security group with which you can connect other instance to Airflow, for example EMR Livy | 142 | | airflow\_dns\_record | The created DNS record (only if "use\_https" = true) | 143 | | airflow\_task\_iam\_role | The IAM role of the airflow task, use this to give Airflow more permissions | 144 | 145 | 146 | 147 | ## Makefile Targets 148 | 149 | ```text 150 | Available targets: 151 | 152 | tools Pull Go and Terraform dependencies 153 | fmt Format Go and Terraform code 154 | lint/lint-tf/lint-go Lint Go and Terraform code 155 | test/testverbose Run tests 156 | 157 | ``` 158 | 159 | ## Contributing 160 | 161 | Contributions to this repository are very welcome! Found a bug or do you have a suggestion? Please open an issue. Do you know how to fix it? Pull requests are welcome as well! To get you started faster, a Makefile is provided. 162 | 163 | Make sure to install [Terraform](https://learn.hashicorp.com/terraform/getting-started/install.html), [Go](https://golang.org/doc/install) (for automated testing) and Make (optional, if you want to use the Makefile) on your computer. Install [tflint](https://github.com/terraform-linters/tflint) to be able to run the linting. 164 | 165 | * Setup tools & dependencies: `make tools` 166 | * Format your code: `make fmt` 167 | * Linting: `make lint` 168 | * Run tests: `make test` (or `go test -timeout 2h ./...` without Make) 169 | 170 | Make sure you branch from the 'open-pr-here' branch, and submit a PR back to the 'open-pr-here' branch. 171 | 172 | ## License 173 | 174 | MIT license. Please see [LICENSE](LICENSE.md) for details. 175 | -------------------------------------------------------------------------------- /alb.tf: -------------------------------------------------------------------------------- 1 | // SG only meant for the alb to connect to the outside world 2 | resource "aws_security_group" "alb" { 3 | vpc_id = var.vpc_id 4 | name = "${var.resource_prefix}-alb-${var.resource_suffix}" 5 | description = "Security group for the alb attached to the airflow ecs task" 6 | 7 | egress { 8 | description = "Allow all traffic out" 9 | from_port = 0 10 | to_port = 0 11 | protocol = "-1" 12 | cidr_blocks = ["0.0.0.0/0"] 13 | } 14 | 15 | tags = local.common_tags 16 | } 17 | 18 | resource "aws_security_group_rule" "alb_outside_http" { 19 | for_each = local.inbound_ports 20 | security_group_id = aws_security_group.alb.id 21 | type = "ingress" 22 | protocol = "TCP" 23 | from_port = each.value 24 | to_port = each.value 25 | cidr_blocks = var.ip_allow_list 26 | } 27 | 28 | 29 | // Give this SG to all the instances that want to connect to 30 | // the airflow ecs task. For example rds and the alb 31 | resource "aws_security_group" "airflow" { 32 | vpc_id = var.vpc_id 33 | name = "${var.resource_prefix}-airflow-${var.resource_suffix}" 34 | description = "Security group to connect to the airflow instance" 35 | 36 | egress { 37 | description = "Allow all traffic out" 38 | from_port = 0 39 | to_port = 0 40 | protocol = "-1" 41 | cidr_blocks = ["0.0.0.0/0"] 42 | } 43 | 44 | tags = local.common_tags 45 | 46 | } 47 | 48 | resource "aws_security_group_rule" "airflow_connection" { 49 | security_group_id = aws_security_group.airflow.id 50 | type = "ingress" 51 | protocol = "-1" 52 | from_port = 0 53 | to_port = 0 54 | source_security_group_id = aws_security_group.airflow.id 55 | } 56 | 57 | // ALB 58 | resource "aws_lb" "airflow" { 59 | name = "${var.resource_prefix}-airflow-${var.resource_suffix}" 60 | internal = false 61 | load_balancer_type = "application" 62 | security_groups = [aws_security_group.alb.id, aws_security_group.airflow.id] 63 | subnets = var.public_subnet_ids 64 | 65 | enable_deletion_protection = false 66 | 67 | tags = local.common_tags 68 | } 69 | 70 | resource "aws_lb_listener" "airflow" { 71 | load_balancer_arn = aws_lb.airflow.arn 72 | port = var.use_https ? "443" : "80" 73 | protocol = var.use_https ? "HTTPS" : "HTTP" 74 | certificate_arn = var.use_https ? local.certificate_arn : "" 75 | 76 | default_action { 77 | type = "forward" 78 | target_group_arn = aws_lb_target_group.airflow.arn 79 | } 80 | } 81 | 82 | resource "aws_lb_listener" "airflow_http_redirect" { 83 | count = var.use_https ? 1 : 0 84 | load_balancer_arn = aws_lb.airflow.arn 85 | port = "80" 86 | protocol = "HTTP" 87 | 88 | default_action { 89 | type = "redirect" 90 | 91 | redirect { 92 | port = "443" 93 | protocol = "HTTPS" 94 | status_code = "HTTP_301" 95 | } 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /ecs.tf: -------------------------------------------------------------------------------- 1 | resource "aws_cloudwatch_log_group" "airflow" { 2 | name = "${var.resource_prefix}-airflow-${var.resource_suffix}" 3 | retention_in_days = var.airflow_log_retention 4 | 5 | tags = local.common_tags 6 | } 7 | 8 | resource "aws_ecs_cluster" "airflow" { 9 | name = "${var.resource_prefix}-airflow-${var.resource_suffix}" 10 | capacity_providers = ["FARGATE_SPOT", "FARGATE"] 11 | 12 | default_capacity_provider_strategy { 13 | capacity_provider = "FARGATE_SPOT" 14 | } 15 | 16 | tags = local.common_tags 17 | } 18 | 19 | resource "aws_ecs_task_definition" "airflow" { 20 | family = "${var.resource_prefix}-airflow-${var.resource_suffix}" 21 | requires_compatibilities = ["FARGATE"] 22 | cpu = var.ecs_cpu 23 | memory = var.ecs_memory 24 | network_mode = "awsvpc" 25 | task_role_arn = aws_iam_role.task.arn 26 | execution_role_arn = aws_iam_role.execution.arn 27 | 28 | volume { 29 | name = local.airflow_volume_name 30 | } 31 | 32 | container_definitions = < { 24 | name = dvo.resource_record_name 25 | record = dvo.resource_record_value 26 | type = dvo.resource_record_type 27 | } 28 | } : {} 29 | 30 | allow_overwrite = true 31 | name = each.value.name 32 | records = [each.value.record] 33 | ttl = 60 34 | type = each.value.type 35 | zone_id = data.aws_route53_zone.zone[0].zone_id 36 | } 37 | 38 | resource "aws_acm_certificate" "cert" { 39 | count = var.use_https && var.certificate_arn == "" ? 1 : 0 40 | domain_name = local.dns_record 41 | validation_method = "DNS" 42 | 43 | lifecycle { 44 | create_before_destroy = true 45 | } 46 | 47 | tags = local.common_tags 48 | } 49 | 50 | resource "aws_acm_certificate_validation" "cert" { 51 | count = var.use_https && var.certificate_arn == "" ? 1 : 0 52 | certificate_arn = aws_acm_certificate.cert[0].arn 53 | validation_record_fqdns = [for record in aws_route53_record.validation : record.fqdn] 54 | } 55 | -------------------------------------------------------------------------------- /s3.tf: -------------------------------------------------------------------------------- 1 | resource "aws_s3_bucket" "airflow" { 2 | count = var.s3_bucket_name == "" ? 1 : 0 3 | bucket = "${var.resource_prefix}-airflow-${var.resource_suffix}" 4 | acl = "private" 5 | 6 | versioning { 7 | enabled = true 8 | } 9 | 10 | server_side_encryption_configuration { 11 | rule { 12 | apply_server_side_encryption_by_default { 13 | sse_algorithm = "aws:kms" 14 | } 15 | } 16 | } 17 | 18 | tags = local.common_tags 19 | } 20 | 21 | resource "aws_s3_bucket_public_access_block" "airflow" { 22 | count = var.s3_bucket_name == "" ? 1 : 0 23 | bucket = aws_s3_bucket.airflow[0].id 24 | 25 | block_public_acls = true 26 | block_public_policy = true 27 | ignore_public_acls = true 28 | restrict_public_buckets = true 29 | } 30 | 31 | resource "aws_s3_bucket_object" "airflow_seed_dag" { 32 | bucket = local.s3_bucket_name 33 | key = "dags/airflow_seed_dag.py" 34 | content = templatefile("${path.module}/templates/dags/airflow_seed_dag.py", { 35 | BUCKET_NAME = local.s3_bucket_name, 36 | KEY = local.s3_key, 37 | AIRFLOW_HOME = var.airflow_container_home 38 | YEAR = local.year 39 | MONTH = local.month 40 | DAY = local.day 41 | }) 42 | } 43 | 44 | resource "aws_s3_bucket_object" "airflow_example_dag" { 45 | count = var.airflow_example_dag ? 1 : 0 46 | bucket = local.s3_bucket_name 47 | key = "dags/example_dag.py" 48 | content = templatefile("${path.module}/templates/dags/example_dag.py", {}) 49 | } 50 | 51 | resource "aws_s3_bucket_object" "airflow_scheduler_entrypoint" { 52 | bucket = local.s3_bucket_name 53 | key = "startup/entrypoint_scheduler.sh" 54 | content = templatefile("${path.module}/templates/startup/entrypoint_scheduler.sh", { AIRFLOW_HOME = var.airflow_container_home }) 55 | } 56 | 57 | resource "aws_s3_bucket_object" "airflow_webserver_entrypoint" { 58 | bucket = local.s3_bucket_name 59 | key = "startup/entrypoint_webserver.sh" 60 | content = templatefile("${path.module}/templates/startup/entrypoint_webserver.sh", { AIRFLOW_HOME = var.airflow_container_home }) 61 | } 62 | 63 | resource "aws_s3_bucket_object" "airflow_init_entrypoint" { 64 | bucket = local.s3_bucket_name 65 | key = "startup/entrypoint_init.sh" 66 | content = templatefile("${path.module}/templates/startup/entrypoint_init.sh", { 67 | RBAC_AUTH = var.airflow_authentication == "rbac" ? "true" : "false", 68 | RBAC_USERNAME = var.rbac_admin_username, 69 | RBAC_EMAIL = var.rbac_admin_email, 70 | RBAC_FIRSTNAME = var.rbac_admin_firstname, 71 | RBAC_LASTNAME = var.rbac_admin_lastname, 72 | RBAC_PASSWORD = var.rbac_admin_password, 73 | AIRFLOW_VERSION = var.airflow_image_tag 74 | }) 75 | } 76 | 77 | resource "aws_s3_bucket_object" "airflow_requirements" { 78 | count = var.airflow_py_requirements_path == "" ? 0 : 1 79 | bucket = local.s3_bucket_name 80 | key = "startup/requirements.txt" 81 | content = templatefile(local.airflow_py_requirements_path, {}) 82 | } -------------------------------------------------------------------------------- /templates/dags/airflow_seed_dag.py: -------------------------------------------------------------------------------- 1 | import os 2 | from os import listdir 3 | from os.path import isfile, join 4 | 5 | import datetime 6 | from typing import Dict 7 | 8 | from airflow import DAG 9 | from airflow.operators.python_operator import PythonOperator 10 | from airflow.operators.bash_operator import BashOperator 11 | 12 | import boto3 13 | 14 | # The bucket name and key the of where dags are stored in S3 15 | S3_BUCKET_NAME = "${BUCKET_NAME}" 16 | # airflow home directory where dags & plugins reside 17 | AIRFLOW_HOME = "${AIRFLOW_HOME}" 18 | 19 | args = { 20 | "start_date": datetime.datetime(${YEAR}, ${MONTH}, ${DAY}), 21 | } 22 | 23 | # we prefix the dag with '0' to make it the first dag 24 | with DAG( 25 | dag_id="0_sync_dags_in_s3_to_local_airflow_dags_folder", 26 | default_args=args, 27 | schedule_interval=None 28 | ) as dag: 29 | list_dags_before = BashOperator( 30 | task_id="list_dags_before", 31 | bash_command="find ${AIRFLOW_HOME}/dags -not -path '*__pycache__*'", 32 | ) 33 | 34 | sync_dags = BashOperator( 35 | task_id="sync_dag_s3_to_airflow", 36 | bash_command=f"python -m awscli s3 sync --exclude='*' --include='*.py' --size-only --delete s3://{S3_BUCKET_NAME}/dags/ {AIRFLOW_HOME}/dags/" 37 | ) 38 | 39 | sync_plugins = BashOperator( 40 | task_id="sync_plugins_s3_to_airflow", 41 | bash_command=f"python -m awscli s3 sync --exclude='*' --include='*.py' --size-only --delete s3://{S3_BUCKET_NAME}/plugins/ {AIRFLOW_HOME}/plugins/" 42 | ) 43 | 44 | refresh_dag_bag = BashOperator( 45 | task_id="refresh_dag_bag", 46 | bash_command="python -c 'from airflow.models import DagBag; d = DagBag();'", 47 | ) 48 | 49 | list_dags_after = BashOperator( 50 | task_id="list_dags_after", 51 | bash_command="find ${AIRFLOW_HOME}/dags -not -path '*__pycache__*'", 52 | ) 53 | 54 | ( 55 | list_dags_before >> 56 | [sync_dags, sync_plugins] >> 57 | refresh_dag_bag >> 58 | list_dags_after 59 | ) 60 | -------------------------------------------------------------------------------- /templates/dags/example_dag.py: -------------------------------------------------------------------------------- 1 | from datetime import timedelta, datetime 2 | 3 | from airflow import DAG 4 | from airflow.operators.dummy_operator import DummyOperator 5 | from airflow import AirflowException 6 | 7 | args = { 8 | "owner": "dataroots", 9 | "start_date": datetime(2020, 10, 12), 10 | } 11 | 12 | with DAG( 13 | dag_id="example_dag", 14 | catchup=False, 15 | max_active_runs=1, 16 | default_args=args, 17 | schedule_interval="*/5 * * * *" 18 | ) as dag: 19 | task_a = DummyOperator( 20 | task_id="task_a" 21 | ) 22 | 23 | task_b = DummyOperator( 24 | task_id="task_b" 25 | ) 26 | task_c = DummyOperator( 27 | task_id="task_c" 28 | ) 29 | 30 | task_d = DummyOperator( 31 | task_id="task_d" 32 | ) 33 | 34 | task_a >> [task_b, task_c] >> task_d 35 | -------------------------------------------------------------------------------- /templates/startup/entrypoint_init.sh: -------------------------------------------------------------------------------- 1 | echo "Starting up airflow init" 2 | 3 | # commands change between version so get the major version here 4 | airflow_major_version=$(echo ${AIRFLOW_VERSION} | awk -F. '{ print $1 }') 5 | 6 | # airflow 7 | if [[ "$airflow_major_version" == "1" ]]; then 8 | airflow initdb 9 | else 10 | airflow db init 11 | fi 12 | 13 | # add admin user if rbac enabled and not exists 14 | if [[ "${RBAC_AUTH}" == "true" ]]; then 15 | # get the amount of users to see if we need to add a default user 16 | amount_of_users="-9999" 17 | if [[ "$airflow_major_version" == "1" ]]; then 18 | amount_of_users=$(python -c 'import sys;print((sys.argv.count("│") // 7) - 1)' $(airflow list_users)) 19 | else 20 | amount_of_users=$(python -c 'import sys;cmd_in = " ".join(sys.argv);print((cmd_in.count("|") // 5) - 1 if "No data found" not in cmd_in else 0)' $(airflow users list)) 21 | fi 22 | 23 | if [[ "$amount_of_users" == "0" ]]; then 24 | echo "Adding admin users, users list is empty!" 25 | if [[ "$airflow_major_version" == "1" ]]; then 26 | airflow create_user -r Admin -u ${RBAC_USERNAME} -e ${RBAC_EMAIL} -f ${RBAC_FIRSTNAME} -l ${RBAC_LASTNAME} -p ${RBAC_PASSWORD} 27 | else 28 | airflow users create -r Admin -u ${RBAC_USERNAME} -e ${RBAC_EMAIL} -f ${RBAC_FIRSTNAME} -l ${RBAC_LASTNAME} -p ${RBAC_PASSWORD} 29 | fi 30 | else 31 | echo "No admin user added, users already exists!" 32 | fi 33 | fi -------------------------------------------------------------------------------- /templates/startup/entrypoint_scheduler.sh: -------------------------------------------------------------------------------- 1 | echo "Starting up airflow scheduler" 2 | # Sanity check the dags 3 | ls /opt/airflow/dags 4 | 5 | # Install boto and awscli for the seed dag 6 | python -m pip install awscli --user 7 | 8 | # Intall python packages through req.txt and pip (if exists) 9 | if [[ -f "${AIRFLOW_HOME}/startup/requirements.txt" ]]; then 10 | echo "requirements.txt provided, installing it with pip" 11 | python -m pip install -r ${AIRFLOW_HOME}/startup/requirements.txt --user 12 | fi 13 | # Run the airflow webserver 14 | airflow scheduler -------------------------------------------------------------------------------- /templates/startup/entrypoint_webserver.sh: -------------------------------------------------------------------------------- 1 | echo "Starting up airflow webserver" 2 | # sanity check the dags 3 | ls /opt/airflow/dags 4 | 5 | # Install boto and awscli for the seed dag 6 | python -m pip install awscli --user 7 | 8 | # Intall python packages through req.txt and pip (if exists) 9 | if [[ -f "${AIRFLOW_HOME}/startup/requirements.txt" ]]; then 10 | echo "requirements.txt provided, installing it with pip" 11 | python -m pip install -r ${AIRFLOW_HOME}/startup/requirements.txt --user 12 | fi 13 | 14 | export AIRFLOW__WEBSERVER__SECRET_KEY=$(openssl rand -hex 30) 15 | 16 | # Run the airflow webserver 17 | airflow webserver -------------------------------------------------------------------------------- /test/preexisting/main.tf: -------------------------------------------------------------------------------- 1 | variable "rds_name" { 2 | type = string 3 | } 4 | 5 | variable "region" { 6 | type = string 7 | description = "The region to deploy your solution to" 8 | default = "eu-west-1" 9 | } 10 | 11 | variable "resource_prefix" { 12 | type = string 13 | description = "A prefix for the create resources, example your company name (be aware of the resource name length)" 14 | } 15 | 16 | variable "resource_suffix" { 17 | type = string 18 | description = "A suffix for the created resources, example the environment for airflow to run in (be aware of the resource name length)" 19 | } 20 | 21 | variable "extra_tags" { 22 | description = "Extra tags that you would like to add to all created resources" 23 | type = map(string) 24 | default = {} 25 | } 26 | 27 | variable "vpc_id" { 28 | type = string 29 | description = "The id of the vpc where you will run ecs/rds" 30 | 31 | validation { 32 | condition = can(regex("^vpc-", var.vpc_id)) 33 | error_message = "The vpc_id value must be a valid VPC id, starting with \"vpc-\"." 34 | } 35 | } 36 | 37 | variable "public_subnet_ids" { 38 | type = list(string) 39 | description = "A list of subnet ids of where the ALB will reside, if the \"private_subnet_ids\" variable is not provided ECS and RDS will also reside in these subnets" 40 | 41 | validation { 42 | condition = length(var.public_subnet_ids) >= 2 43 | error_message = "The size of the list \"public_subnet_ids\" must be at least 2." 44 | } 45 | } 46 | 47 | variable "private_subnet_ids" { 48 | type = list(string) 49 | description = "A list of subnet ids of where the ECS and RDS reside, this will only work if you have a NAT Gateway in your VPC" 50 | default = [] 51 | 52 | validation { 53 | condition = length(var.private_subnet_ids) >= 2 || length(var.private_subnet_ids) == 0 54 | error_message = "The size of the list \"private_subnet_ids\" must be at least 2 or empty." 55 | } 56 | } 57 | 58 | variable "route53_zone_name" { 59 | type = string 60 | description = "The name of a Route53 zone that will be used for the certificate validation." 61 | default = "" 62 | } 63 | 64 | locals { 65 | own_tags = { 66 | Name = "${var.resource_prefix}-airflow-${var.resource_suffix}" 67 | CreatedBy = "Terraform" 68 | Module = "terraform-aws-ecs-airflow" 69 | } 70 | common_tags = merge(local.own_tags, var.extra_tags) 71 | 72 | timestamp_sanitized = replace(timestamp(), "/[- TZ:]/", "") 73 | 74 | rds_ecs_subnet_ids = length(var.private_subnet_ids) == 0 ? var.public_subnet_ids : var.private_subnet_ids 75 | dns_record = "${var.resource_prefix}-airflow-${var.resource_suffix}.${data.aws_route53_zone.zone.name}" 76 | } 77 | 78 | terraform { 79 | required_version = "~> 0.15" 80 | required_providers { 81 | aws = { 82 | source = "hashicorp/aws" 83 | version = "~> 3.12.0" 84 | } 85 | } 86 | } 87 | 88 | provider "aws" { 89 | version = "~> 3.12.0" 90 | region = var.region 91 | } 92 | 93 | resource "aws_security_group" "rds" { 94 | vpc_id = var.vpc_id 95 | name = "${var.resource_prefix}-rds-${var.resource_suffix}" 96 | description = "Security group for the RDS" 97 | 98 | egress { 99 | description = "Allow all traffic out" 100 | from_port = 0 101 | to_port = 0 102 | protocol = "-1" 103 | cidr_blocks = ["0.0.0.0/0"] 104 | } 105 | 106 | ingress { 107 | description = "Allow PostgreSQL traffic in" 108 | from_port = 5432 109 | to_port = 5432 110 | protocol = "TCP" 111 | cidr_blocks = ["10.150.0.0/16"] 112 | } 113 | 114 | tags = local.common_tags 115 | 116 | } 117 | 118 | resource "aws_db_instance" "airflow" { 119 | name = replace(title(var.rds_name), "-", "") 120 | allocated_storage = 20 121 | storage_type = "standard" 122 | engine = "postgres" 123 | engine_version = "11.8" 124 | instance_class = "db.t2.micro" 125 | username = "dataroots" 126 | password = "dataroots" 127 | multi_az = false 128 | availability_zone = "eu-west-1a" 129 | publicly_accessible = false 130 | skip_final_snapshot = true 131 | deletion_protection = false 132 | final_snapshot_identifier = "${var.resource_prefix}-airflow-${var.resource_suffix}-${local.timestamp_sanitized}" 133 | identifier = var.rds_name 134 | vpc_security_group_ids = [aws_security_group.rds.id] 135 | db_subnet_group_name = aws_db_subnet_group.airflow.name 136 | 137 | tags = local.common_tags 138 | } 139 | 140 | resource "aws_db_subnet_group" "airflow" { 141 | name = "${var.resource_prefix}-airflow-${var.resource_suffix}" 142 | subnet_ids = local.rds_ecs_subnet_ids 143 | 144 | tags = local.common_tags 145 | } 146 | 147 | data "aws_route53_zone" "zone" { 148 | name = var.route53_zone_name 149 | } 150 | 151 | resource "aws_route53_record" "validation" { 152 | for_each = { 153 | for dvo in aws_acm_certificate.cert.domain_validation_options : dvo.domain_name => { 154 | name = dvo.resource_record_name 155 | record = dvo.resource_record_value 156 | type = dvo.resource_record_type 157 | } 158 | } 159 | 160 | allow_overwrite = true 161 | name = each.value.name 162 | records = [each.value.record] 163 | ttl = 60 164 | type = each.value.type 165 | zone_id = data.aws_route53_zone.zone.zone_id 166 | } 167 | 168 | resource "aws_acm_certificate" "cert" { 169 | domain_name = local.dns_record 170 | validation_method = "DNS" 171 | 172 | lifecycle { 173 | create_before_destroy = true 174 | } 175 | 176 | tags = local.common_tags 177 | } 178 | 179 | resource "aws_acm_certificate_validation" "cert" { 180 | certificate_arn = aws_acm_certificate.cert.arn 181 | validation_record_fqdns = [for record in aws_route53_record.validation : record.fqdn] 182 | } 183 | 184 | output "certificate_arn" { 185 | value = aws_acm_certificate.cert.arn 186 | } 187 | 188 | output "postgres_uri" { 189 | value = "${aws_db_instance.airflow.address}:${aws_db_instance.airflow.port}/${aws_db_instance.airflow.name}" 190 | } 191 | -------------------------------------------------------------------------------- /test/provider.tf: -------------------------------------------------------------------------------- 1 | // Provider to initialize tests 2 | 3 | provider "aws" { 4 | region = var.region 5 | } 6 | -------------------------------------------------------------------------------- /test/terraform_apply_destroy_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "fmt" 5 | "io/ioutil" 6 | "net/http" 7 | "net/http/cookiejar" 8 | "net/url" 9 | "path/filepath" 10 | "strings" 11 | "testing" 12 | "time" 13 | 14 | "github.com/PuerkitoBio/goquery" 15 | 16 | "github.com/aws/aws-sdk-go/service/ecs" 17 | "github.com/aws/aws-sdk-go/service/iam" 18 | 19 | "github.com/gruntwork-io/terratest/modules/aws" 20 | "github.com/gruntwork-io/terratest/modules/files" 21 | "github.com/gruntwork-io/terratest/modules/logger" 22 | "github.com/gruntwork-io/terratest/modules/random" 23 | "github.com/gruntwork-io/terratest/modules/terraform" 24 | testStructure "github.com/gruntwork-io/terratest/modules/test-structure" 25 | "github.com/stretchr/testify/assert" 26 | ) 27 | 28 | func AddPreAndSuffix(resourceName string, resourcePrefix string, resourceSuffix string) string { 29 | if resourcePrefix == "" { 30 | resourcePrefix = "dataroots" 31 | } 32 | if resourceSuffix == "" { 33 | resourceSuffix = "dev" 34 | } 35 | return fmt.Sprintf("%s-%s-%s", resourcePrefix, resourceName, resourceSuffix) 36 | } 37 | 38 | func GetContainerWithName(containerName string, containers []*ecs.Container) *ecs.Container { 39 | for _, container := range containers { 40 | if *container.Name == containerName { 41 | return container 42 | } 43 | } 44 | return nil 45 | } 46 | 47 | func validateCluster(t *testing.T, options *terraform.Options, region string, resourcePrefix string, resourceSuffix string) { 48 | retrySleepTime := time.Duration(10) * time.Second 49 | ecsGetTaskArnMaxRetries := 20 50 | ecsGetTaskStatusMaxRetries := 50 51 | httpStatusCodeMaxRetries := 15 52 | amountOfConsecutiveGetsToBeHealthy := 3 53 | desiredStatusRunning := "RUNNING" 54 | clusterName := AddPreAndSuffix("airflow", resourcePrefix, resourceSuffix) 55 | serviceName := AddPreAndSuffix("airflow", resourcePrefix, resourceSuffix) 56 | webserverContainerName := AddPreAndSuffix("airflow-webserver", resourcePrefix, resourceSuffix) 57 | schedulerContainerName := AddPreAndSuffix("airflow-webserver", resourcePrefix, resourceSuffix) 58 | sidecarContainerName := AddPreAndSuffix("airflow-sidecar", resourcePrefix, resourceSuffix) 59 | 60 | expectedNavbarColor := "#e27d60" 61 | 62 | iamClient := aws.NewIamClient(t, region) 63 | 64 | fmt.Println("Checking if roles exists") 65 | rolesToCheck := []string{ 66 | AddPreAndSuffix("airflow-task-execution-role", resourcePrefix, resourceSuffix), 67 | AddPreAndSuffix("airflow-task-role", resourcePrefix, resourceSuffix), 68 | } 69 | for _, roleName := range rolesToCheck { 70 | roleInput := &iam.GetRoleInput{RoleName: &roleName} 71 | _, err := iamClient.GetRole(roleInput) 72 | assert.NoError(t, err) 73 | } 74 | 75 | fmt.Println("Checking if ecs cluster exists") 76 | _, err := aws.GetEcsClusterE(t, region, clusterName) 77 | assert.NoError(t, err) 78 | 79 | fmt.Println("Checking if the service is ACTIVE") 80 | airflowEcsService, err := aws.GetEcsServiceE(t, region, clusterName, serviceName) 81 | assert.NoError(t, err) 82 | assert.Equal(t, "ACTIVE", *airflowEcsService.Status) 83 | 84 | fmt.Println("Checking if there is 1 deployment namely the airflow one") 85 | assert.Equal(t, 1, len(airflowEcsService.Deployments)) 86 | 87 | ecsClient := aws.NewEcsClient(t, region) 88 | 89 | // Get all the arns of the task that are running. 90 | // There should only be one task running, the airflow task 91 | fmt.Println("Getting task arns") 92 | listRunningTasksInput := &ecs.ListTasksInput{ 93 | Cluster: &clusterName, 94 | ServiceName: &serviceName, 95 | DesiredStatus: &desiredStatusRunning, 96 | } 97 | 98 | var taskArns []*string 99 | for i := 0; i < ecsGetTaskArnMaxRetries; i++ { 100 | fmt.Printf("Getting task arns, try... %d\n", i) 101 | 102 | runningTasks, _ := ecsClient.ListTasks(listRunningTasksInput) 103 | if len(runningTasks.TaskArns) == 1 { 104 | taskArns = runningTasks.TaskArns 105 | break 106 | } 107 | time.Sleep(retrySleepTime) 108 | } 109 | fmt.Println("Getting that there is only one task running") 110 | assert.Equal(t, 1, len(taskArns)) 111 | 112 | // If there is no task running you can't do the following tests so skip them 113 | if len(taskArns) == 1 { 114 | fmt.Println("Task is running, continuing") 115 | describeTasksInput := &ecs.DescribeTasksInput{ 116 | Cluster: &clusterName, 117 | Tasks: taskArns, 118 | } 119 | 120 | // Wait until the 3 containers are in there desired state 121 | // - Sidecar container must be STOPPED to be healthy 122 | // (only runs once and then stops it's an "init container") 123 | // - Webserver container must be RUNNING to be healthy 124 | // - Scheduler container must be RUNNING to be healthy 125 | fmt.Println("Getting container statuses") 126 | var webserverContainer ecs.Container 127 | var schedulerContainer ecs.Container 128 | var sidecarContainer ecs.Container 129 | for i := 0; i < ecsGetTaskStatusMaxRetries; i++ { 130 | fmt.Printf("Getting container statuses, try... %d\n", i) 131 | 132 | describeTasks, _ := ecsClient.DescribeTasks(describeTasksInput) 133 | airflowTask := describeTasks.Tasks[0] 134 | containers := airflowTask.Containers 135 | 136 | webserverContainer = *GetContainerWithName(webserverContainerName, containers) 137 | schedulerContainer = *GetContainerWithName(schedulerContainerName, containers) 138 | sidecarContainer = *GetContainerWithName(sidecarContainerName, containers) 139 | 140 | if *webserverContainer.LastStatus == "RUNNING" && 141 | *schedulerContainer.LastStatus == "RUNNING" && 142 | *sidecarContainer.LastStatus == "STOPPED" { 143 | break 144 | } 145 | time.Sleep(retrySleepTime) 146 | } 147 | assert.Equal(t, "RUNNING", *webserverContainer.LastStatus) 148 | assert.Equal(t, "RUNNING", *schedulerContainer.LastStatus) 149 | assert.Equal(t, "STOPPED", *sidecarContainer.LastStatus) 150 | 151 | // We do consecutive checks because sometime it could be that 152 | // the webserver is available for a short amount of time and crashes 153 | // a couple of seconds later 154 | fmt.Println("Doing HTTP request/checking health") 155 | 156 | protocol := "https" 157 | airflowAlbDNS := terraform.Output(t, options, "airflow_dns_record") 158 | 159 | if options.Vars["use_https"] == false { 160 | protocol = "http" 161 | } 162 | 163 | if options.Vars["route53_zone_name"] == "" { 164 | airflowAlbDNS = terraform.Output(t, options, "airflow_alb_dns") 165 | } 166 | airflowURL := fmt.Sprintf("%s://%s", protocol, airflowAlbDNS) 167 | 168 | var amountOfConsecutiveHealthyChecks int 169 | var res *http.Response 170 | for i := 0; i < httpStatusCodeMaxRetries; i++ { 171 | fmt.Printf("Doing HTTP request to airflow webservice, try... %d\n", i) 172 | res, err = http.Get(airflowURL) 173 | if res != nil && err == nil { 174 | fmt.Println(res.StatusCode) 175 | if res.StatusCode >= 200 && res.StatusCode < 400 { 176 | amountOfConsecutiveHealthyChecks++ 177 | fmt.Println("Webservice is healthy") 178 | } else { 179 | amountOfConsecutiveHealthyChecks = 0 180 | fmt.Println("Webservice is NOT healthy") 181 | } 182 | 183 | if amountOfConsecutiveHealthyChecks == amountOfConsecutiveGetsToBeHealthy { 184 | break 185 | } 186 | } 187 | time.Sleep(retrySleepTime) 188 | } 189 | 190 | if res != nil { 191 | assert.Equal(t, true, res.StatusCode >= 200 && res.StatusCode < 400) 192 | assert.Equal(t, amountOfConsecutiveGetsToBeHealthy, amountOfConsecutiveHealthyChecks) 193 | 194 | if res.StatusCode >= 200 && res.StatusCode < 400 { 195 | fmt.Println("Getting the actual HTML code") 196 | defer res.Body.Close() 197 | doc, err := goquery.NewDocumentFromReader(res.Body) 198 | assert.NoError(t, err) 199 | 200 | fmt.Println("Checking if the navbar has the correct color") 201 | navbarStyle, exists := doc.Find(".navbar.navbar-inverse.navbar-fixed-top").First().Attr("style") 202 | assert.Equal(t, true, exists) 203 | assert.Contains(t, navbarStyle, fmt.Sprintf("background-color: %s", expectedNavbarColor)) 204 | 205 | // if rbac is enabled check if you can log in 206 | // this is to prevent 'issue #9' of happening again 207 | // ref: https://github.com/datarootsio/terraform-aws-ecs-airflow/issues/9 208 | if options.Vars["airflow_authentication"] == "rbac" { 209 | loginToAirflow(t, airflowURL) 210 | } 211 | } 212 | } 213 | } 214 | } 215 | 216 | func loginToAirflow(t *testing.T, airflowURL string) { 217 | username := "admin" 218 | password := "admin" 219 | airflowLoginURL := fmt.Sprintf("%s/login/", airflowURL) 220 | 221 | // we create a client/session to persist some headers 222 | // throughout the calls that we do 223 | jar, _ := cookiejar.New(nil) 224 | client := &http.Client{ 225 | Jar: jar, 226 | } 227 | 228 | // get the page to fill in some headers in the http session 229 | // and to parse out the csrfToken 230 | res, err := client.Get(airflowLoginURL) 231 | assert.NoError(t, err) 232 | assert.Equal(t, true, res.StatusCode >= 200 && res.StatusCode < 400) 233 | defer res.Body.Close() 234 | 235 | // read the plain body html and get the csrfToken 236 | bodyBytes, _ := ioutil.ReadAll(res.Body) 237 | bodyLines := strings.Split(string(bodyBytes), "\n") 238 | 239 | // parse out the csrfToken 240 | // TODO: replace this with regex 241 | var csrfToken string 242 | for _, bodyLine := range bodyLines { 243 | if strings.Contains(bodyLine, "csrfToken") { 244 | stringNoSpaces := strings.ReplaceAll(bodyLine, " ", "") 245 | stringRemovedLeft := strings.ReplaceAll(stringNoSpaces, "varcsrfToken='", "") 246 | csrfToken = strings.ReplaceAll(stringRemovedLeft, "';", "") 247 | break 248 | } 249 | } 250 | 251 | // create the url vals to be posted, so that we can login 252 | values := make(url.Values) 253 | values.Set("username", username) 254 | values.Set("password", password) 255 | values.Set("csrf_token", csrfToken) 256 | 257 | // try to login into airflow with given creds 258 | // if we can't login w'll come back to the login page 259 | // if we can w'll see the dags table 260 | res, err = client.PostForm(airflowLoginURL, values) 261 | assert.NoError(t, err) 262 | assert.Equal(t, true, res.StatusCode >= 200 && res.StatusCode < 400) 263 | defer res.Body.Close() 264 | 265 | // check on which page we are login or dags 266 | doc, _ := goquery.NewDocumentFromReader(res.Body) 267 | loginBoxExists := doc.Find("div#loginbox").Length() == 1 268 | dagsTableExists := doc.Find("table#dags").Length() == 1 269 | 270 | assert.Equal(t, false, loginBoxExists) 271 | assert.Equal(t, true, dagsTableExists) 272 | } 273 | 274 | // func getPreexistingTerraformOptions(t *testing.T, region string, resourcePrefix string, resourceSuffix string) (*terraform.Options, error) { 275 | // tempTestFolder := testStructure.CopyTerraformFolderToTemp(t, "preexisting", ".") 276 | 277 | // terraformOptions := &terraform.Options{ 278 | // TerraformDir: tempTestFolder, 279 | // Vars: map[string]interface{}{}, 280 | // MaxRetries: 5, 281 | // TimeBetweenRetries: 5 * time.Minute, 282 | // NoColor: true, 283 | // Logger: logger.TestingT, 284 | // } 285 | 286 | // terraformOptions.Vars["region"] = region 287 | // terraformOptions.Vars["resource_prefix"] = resourcePrefix 288 | // terraformOptions.Vars["resource_suffix"] = resourceSuffix 289 | // terraformOptions.Vars["extra_tags"] = map[string]interface{}{ 290 | // "ATestTag": "a_test_tag", 291 | // "ResourcePrefix": resourcePrefix, 292 | // "ResourceSuffix": resourceSuffix, 293 | // } 294 | 295 | // terraformOptions.Vars["vpc_id"] = "vpc-0eafa6867cb3bdaa3" 296 | // terraformOptions.Vars["public_subnet_ids"] = []string{ 297 | // "subnet-08da686d46e99872d", 298 | // "subnet-0e5bb83f963f8df0f", 299 | // } 300 | // terraformOptions.Vars["private_subnet_ids"] = []string{ 301 | // "subnet-03c2a3885cfc8a740", 302 | // "subnet-09c0ce0aff676904a", 303 | // } 304 | 305 | // terraformOptions.Vars["rds_name"] = AddPreAndSuffix("preexisting", resourcePrefix, resourceSuffix) 306 | // terraformOptions.Vars["route53_zone_name"] = "" 307 | 308 | // return terraformOptions, nil 309 | // } 310 | 311 | func getDefaultTerraformOptions(t *testing.T, region string, resourcePrefix string, resourceSuffix string) (*terraform.Options, error) { 312 | tempTestFolder := testStructure.CopyTerraformFolderToTemp(t, "..", ".") 313 | err := files.CopyFile("./provider.tf", filepath.Join(tempTestFolder, "provider.tf")) 314 | if err != nil { 315 | t.Fatal(err) 316 | } 317 | 318 | terraformOptions := &terraform.Options{ 319 | TerraformDir: tempTestFolder, 320 | Vars: map[string]interface{}{}, 321 | MaxRetries: 5, 322 | TimeBetweenRetries: 5 * time.Minute, 323 | NoColor: true, 324 | Logger: logger.TestingT, 325 | } 326 | 327 | terraformOptions.Vars["region"] = region 328 | terraformOptions.Vars["resource_prefix"] = resourcePrefix 329 | terraformOptions.Vars["resource_suffix"] = resourceSuffix 330 | terraformOptions.Vars["extra_tags"] = map[string]interface{}{ 331 | "ATestTag": "a_test_tag", 332 | "ResourcePrefix": resourcePrefix, 333 | "ResourceSuffix": resourceSuffix, 334 | } 335 | 336 | terraformOptions.Vars["airflow_image_name"] = "apache/airflow" 337 | terraformOptions.Vars["airflow_image_tag"] = "1.10.12" 338 | terraformOptions.Vars["airflow_log_region"] = region 339 | terraformOptions.Vars["airflow_log_retention"] = "7" 340 | terraformOptions.Vars["airflow_example_dag"] = true 341 | terraformOptions.Vars["airflow_variables"] = map[string]interface{}{ 342 | "AIRFLOW__WEBSERVER__NAVBAR_COLOR": "#e27d60", 343 | } 344 | terraformOptions.Vars["airflow_executor"] = "Local" 345 | 346 | terraformOptions.Vars["ecs_cpu"] = 2048 347 | terraformOptions.Vars["ecs_memory"] = 4096 348 | 349 | terraformOptions.Vars["ip_allow_list"] = []string{ 350 | "0.0.0.0/0", 351 | } 352 | terraformOptions.Vars["vpc_id"] = "vpc-0eafa6867cb3bdaa3" 353 | terraformOptions.Vars["public_subnet_ids"] = []string{ 354 | "subnet-08da686d46e99872d", 355 | "subnet-0e5bb83f963f8df0f", 356 | } 357 | terraformOptions.Vars["private_subnet_ids"] = []string{ 358 | "subnet-03c2a3885cfc8a740", 359 | "subnet-09c0ce0aff676904a", 360 | } 361 | 362 | // Get password and username from env vars 363 | terraformOptions.Vars["postgres_uri"] = "" 364 | terraformOptions.Vars["rds_username"] = "dataroots" 365 | terraformOptions.Vars["rds_password"] = "dataroots" 366 | terraformOptions.Vars["rds_instance_class"] = "db.t2.micro" 367 | terraformOptions.Vars["rds_availability_zone"] = fmt.Sprintf("%sa", region) 368 | terraformOptions.Vars["rds_skip_final_snapshot"] = true 369 | terraformOptions.Vars["rds_deletion_protection"] = false 370 | 371 | terraformOptions.Vars["use_https"] = true 372 | terraformOptions.Vars["route53_zone_name"] = "aws-sandbox.dataroots.io" 373 | 374 | return terraformOptions, nil 375 | } 376 | 377 | func TestApplyAndDestroyWithDefaultValues(t *testing.T) { 378 | fmt.Println("Starting test: TestApplyAndDestroyWithDefaultValues") 379 | // 'GLOBAL' test vars 380 | region := "eu-west-1" 381 | resourcePrefix := "dtr" 382 | resourceSuffix := strings.ToLower(random.UniqueId()) 383 | 384 | // TODO: Check the task def rev number before and after apply and see if the rev num has increased by 1 385 | 386 | // t.Parallel() 387 | 388 | options, err := getDefaultTerraformOptions(t, region, resourcePrefix, resourceSuffix) 389 | assert.NoError(t, err) 390 | 391 | // terraform destroy => when test completes 392 | defer terraform.Destroy(t, options) 393 | fmt.Println("Running: terraform init && terraform apply") 394 | _, err = terraform.InitE(t, options) 395 | assert.NoError(t, err) 396 | _, err = terraform.PlanE(t, options) 397 | assert.NoError(t, err) 398 | _, err = terraform.ApplyE(t, options) 399 | assert.NoError(t, err) 400 | 401 | // if there are terraform errors, do nothing 402 | if err == nil { 403 | fmt.Println("Terraform apply returned no error, continuing") 404 | validateCluster(t, options, region, resourcePrefix, resourceSuffix) 405 | } 406 | } 407 | 408 | func TestApplyAndDestroyWithPlainHTTP(t *testing.T) { 409 | fmt.Println("Starting test: TestApplyAndDestroyWithPlainHTTP") 410 | // 'GLOBAL' test vars 411 | region := "eu-west-1" 412 | resourcePrefix := "dtr" 413 | resourceSuffix := strings.ToLower(random.UniqueId()) 414 | 415 | // TODO: Check the task def rev number before and after apply and see if the rev num has increased by 1 416 | 417 | t.Parallel() 418 | 419 | options, err := getDefaultTerraformOptions(t, region, resourcePrefix, resourceSuffix) 420 | assert.NoError(t, err) 421 | options.Vars["use_https"] = false 422 | options.Vars["route53_zone_name"] = "" 423 | 424 | // terraform destroy => when test completes 425 | defer terraform.Destroy(t, options) 426 | fmt.Println("Running: terraform init && terraform apply") 427 | _, err = terraform.InitE(t, options) 428 | assert.NoError(t, err) 429 | _, err = terraform.PlanE(t, options) 430 | assert.NoError(t, err) 431 | _, err = terraform.ApplyE(t, options) 432 | assert.NoError(t, err) 433 | 434 | // if there are terraform errors, do nothing 435 | if err == nil { 436 | fmt.Println("Terraform apply returned no error, continuing") 437 | validateCluster(t, options, region, resourcePrefix, resourceSuffix) 438 | } 439 | } 440 | 441 | func TestApplyAndDestroyWithPlainHTTPAndSequentialExecutor(t *testing.T) { 442 | fmt.Println("Starting test: TestApplyAndDestroyWithPlainHTTPAndSequentialExecutor") 443 | // 'GLOBAL' test vars 444 | region := "eu-west-1" 445 | resourcePrefix := "dtr" 446 | resourceSuffix := strings.ToLower(random.UniqueId()) 447 | 448 | // TODO: Check the task def rev number before and after apply and see if the rev num has increased by 1 449 | 450 | t.Parallel() 451 | 452 | options, err := getDefaultTerraformOptions(t, region, resourcePrefix, resourceSuffix) 453 | assert.NoError(t, err) 454 | options.Vars["airflow_executor"] = "Sequential" 455 | 456 | options.Vars["use_https"] = false 457 | options.Vars["route53_zone_name"] = "" 458 | 459 | // terraform destroy => when test completes 460 | defer terraform.Destroy(t, options) 461 | fmt.Println("Running: terraform init && terraform apply") 462 | _, err = terraform.InitE(t, options) 463 | assert.NoError(t, err) 464 | _, err = terraform.PlanE(t, options) 465 | assert.NoError(t, err) 466 | _, err = terraform.ApplyE(t, options) 467 | assert.NoError(t, err) 468 | 469 | // if there are terraform errors, do nothing 470 | if err == nil { 471 | fmt.Println("Terraform apply returned no error, continuing") 472 | validateCluster(t, options, region, resourcePrefix, resourceSuffix) 473 | } 474 | } 475 | 476 | func TestApplyAndDestroyWithPlainHTTPAndSequentialExecutorOnlyPublicSubnet(t *testing.T) { 477 | fmt.Println("Starting test: TestApplyAndDestroyWithPlainHTTPAndSequentialExecutorOnlyPublicSubnet") 478 | // 'GLOBAL' test vars 479 | region := "eu-west-1" 480 | resourcePrefix := "dtr" 481 | resourceSuffix := strings.ToLower(random.UniqueId()) 482 | 483 | // TODO: Check the task def rev number before and after apply and see if the rev num has increased by 1 484 | 485 | t.Parallel() 486 | 487 | options, err := getDefaultTerraformOptions(t, region, resourcePrefix, resourceSuffix) 488 | assert.NoError(t, err) 489 | options.Vars["airflow_executor"] = "Sequential" 490 | 491 | options.Vars["private_subnet_ids"] = []string{} 492 | 493 | options.Vars["use_https"] = false 494 | options.Vars["route53_zone_name"] = "" 495 | 496 | // terraform destroy => when test completes 497 | defer terraform.Destroy(t, options) 498 | fmt.Println("Running: terraform init && terraform apply") 499 | _, err = terraform.InitE(t, options) 500 | assert.NoError(t, err) 501 | _, err = terraform.PlanE(t, options) 502 | assert.NoError(t, err) 503 | _, err = terraform.ApplyE(t, options) 504 | assert.NoError(t, err) 505 | 506 | // if there are terraform errors, do nothing 507 | if err == nil { 508 | fmt.Println("Terraform apply returned no error, continuing") 509 | validateCluster(t, options, region, resourcePrefix, resourceSuffix) 510 | } 511 | } 512 | 513 | func TestApplyAndDestroyWithPlainHTTPAndSequentialExecutorUsingRBAC(t *testing.T) { 514 | fmt.Println("Starting test: TestApplyAndDestroyWithPlainHTTPAndSequentialExecutorUsingRBAC") 515 | // 'GLOBAL' test vars 516 | region := "eu-west-1" 517 | resourcePrefix := "dtr" 518 | resourceSuffix := strings.ToLower(random.UniqueId()) 519 | 520 | // TODO: Check the task def rev number before and after apply and see if the rev num has increased by 1 521 | 522 | t.Parallel() 523 | 524 | options, err := getDefaultTerraformOptions(t, region, resourcePrefix, resourceSuffix) 525 | assert.NoError(t, err) 526 | options.Vars["airflow_executor"] = "Sequential" 527 | options.Vars["airflow_authentication"] = "rbac" 528 | 529 | options.Vars["use_https"] = false 530 | options.Vars["route53_zone_name"] = "" 531 | 532 | // terraform destroy => when test completes 533 | defer terraform.Destroy(t, options) 534 | fmt.Println("Running: terraform init && terraform apply") 535 | _, err = terraform.InitE(t, options) 536 | assert.NoError(t, err) 537 | _, err = terraform.PlanE(t, options) 538 | assert.NoError(t, err) 539 | _, err = terraform.ApplyE(t, options) 540 | assert.NoError(t, err) 541 | 542 | // if there are terraform errors, do nothing 543 | if err == nil { 544 | fmt.Println("Terraform apply returned no error, continuing") 545 | validateCluster(t, options, region, resourcePrefix, resourceSuffix) 546 | } 547 | } 548 | 549 | // func TestApplyAndDestroyWithPlainHTTPAndPreexistingRDS(t *testing.T) { 550 | // fmt.Println("Starting test: TestApplyAndDestroyWithPlainHTTPAndPreexistingRDS") 551 | // // 'GLOBAL' test vars 552 | // region := "eu-west-1" 553 | // resourcePrefix := "dtr" 554 | // resourceSuffix := strings.ToLower(random.UniqueId()) 555 | 556 | // // TODO: Check the task def rev number before and after apply and see if the rev num has increased by 1 557 | 558 | // // t.Parallel() 559 | 560 | // preExistingOptions, err := getPreexistingTerraformOptions(t, region, resourcePrefix, resourceSuffix) 561 | // assert.NoError(t, err) 562 | 563 | // // terraform destroy => when test completes 564 | // defer terraform.Destroy(t, preExistingOptions) 565 | // fmt.Println("Running: terraform init && terraform apply") 566 | // _, err = terraform.InitE(t, preExistingOptions) 567 | // assert.NoError(t, err) 568 | // _, err = terraform.PlanE(t, preExistingOptions) 569 | // assert.NoError(t, err) 570 | // _, err = terraform.ApplyE(t, preExistingOptions) 571 | // assert.NoError(t, err) 572 | 573 | // options, err := getDefaultTerraformOptions(t, region, resourcePrefix, resourceSuffix) 574 | // assert.NoError(t, err) 575 | // options.Vars["postgres_uri"] = terraform.Output(t, preExistingOptions, "postgres_uri") 576 | // options.Vars["certificate_arn"] = "" 577 | // options.Vars["use_https"] = false 578 | 579 | // // terraform destroy => when test completes 580 | // defer terraform.Destroy(t, options) 581 | // fmt.Println("Running: terraform init && terraform apply") 582 | // _, err = terraform.InitE(t, options) 583 | // assert.NoError(t, err) 584 | // _, err = terraform.PlanE(t, options) 585 | // assert.NoError(t, err) 586 | // _, err = terraform.ApplyE(t, options) 587 | // assert.NoError(t, err) 588 | // // if there are terraform errors, do nothing 589 | // if err == nil { 590 | // fmt.Println("Terraform apply returned no error, continuing") 591 | // validateCluster(t, options, region, resourcePrefix, resourceSuffix) 592 | // } 593 | // } 594 | -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | description = "The region to deploy your solution to" 4 | default = "eu-west-1" 5 | } 6 | 7 | variable "resource_prefix" { 8 | type = string 9 | description = "A prefix for the create resources, example your company name (be aware of the resource name length)" 10 | } 11 | 12 | variable "resource_suffix" { 13 | type = string 14 | description = "A suffix for the created resources, example the environment for airflow to run in (be aware of the resource name length)" 15 | } 16 | 17 | variable "extra_tags" { 18 | description = "Extra tags that you would like to add to all created resources" 19 | type = map(string) 20 | default = {} 21 | } 22 | 23 | // Airflow variables 24 | variable "airflow_image_name" { 25 | type = string 26 | description = "The name of the airflow image" 27 | default = "apache/airflow" 28 | } 29 | 30 | variable "airflow_image_tag" { 31 | type = string 32 | description = "The tag of the airflow image" 33 | default = "2.0.1" 34 | } 35 | 36 | variable "airflow_executor" { 37 | type = string 38 | description = "The executor mode that airflow will use. Only allowed values are [\"Local\", \"Sequential\"]. \"Local\": Run DAGs in parallel (will created a RDS); \"Sequential\": You can not run DAGs in parallel (will NOT created a RDS);" 39 | default = "Local" 40 | 41 | validation { 42 | condition = contains(["Local", "Sequential"], var.airflow_executor) 43 | error_message = "The only values that are allowed for \"airflow_executor\" are [\"Local\", \"Sequential\"]." 44 | } 45 | } 46 | 47 | variable "airflow_authentication" { 48 | type = string 49 | description = "Authentication backend to be used, supported backends [\"\", \"rbac\"]. When \"rbac\" is selected an admin role is create if there are no other users in the db, from here you can create all the other users. Make sure to change the admin password directly upon first login! (if you don't change the rbac_admin options the default login is => username: admin, password: admin)" 50 | default = "" 51 | 52 | validation { 53 | condition = contains(["", "rbac"], var.airflow_authentication) 54 | error_message = "The only values that are allowed for \"airflow_executor\" are [\"\", \"rbac\"]." 55 | } 56 | } 57 | 58 | variable "airflow_py_requirements_path" { 59 | type = string 60 | description = "The relative path to a python requirements.txt file to install extra packages in the container that you can use in your DAGs." 61 | default = "" 62 | } 63 | 64 | variable "airflow_variables" { 65 | type = map(string) 66 | description = "The variables passed to airflow as an environment variable (see airflow docs for more info https://airflow.apache.org/docs/). You can not specify \"AIRFLOW__CORE__SQL_ALCHEMY_CONN\" and \"AIRFLOW__CORE__EXECUTOR\" (managed by this module)" 67 | default = {} 68 | } 69 | 70 | variable "airflow_container_home" { 71 | type = string 72 | description = "Working dir for airflow (only change if you are using a different image)" 73 | default = "/opt/airflow" 74 | } 75 | 76 | variable "airflow_log_region" { 77 | type = string 78 | description = "The region you want your airflow logs in, defaults to the region variable" 79 | default = "" 80 | } 81 | 82 | variable "airflow_log_retention" { 83 | type = string 84 | description = "The number of days you want to keep the log of airflow container" 85 | default = "7" 86 | } 87 | 88 | variable "airflow_example_dag" { 89 | type = bool 90 | description = "Add an example dag on startup (mostly for sanity check)" 91 | default = true 92 | } 93 | 94 | // RBAC 95 | variable "rbac_admin_username" { 96 | type = string 97 | description = "RBAC Username (only when airflow_authentication = 'rbac')" 98 | default = "admin" 99 | } 100 | 101 | variable "rbac_admin_password" { 102 | type = string 103 | description = "RBAC Password (only when airflow_authentication = 'rbac')" 104 | default = "admin" 105 | } 106 | 107 | variable "rbac_admin_email" { 108 | type = string 109 | description = "RBAC Email (only when airflow_authentication = 'rbac')" 110 | default = "admin@admin.com" 111 | } 112 | 113 | variable "rbac_admin_firstname" { 114 | type = string 115 | description = "RBAC Firstname (only when airflow_authentication = 'rbac')" 116 | default = "admin" 117 | } 118 | 119 | variable "rbac_admin_lastname" { 120 | type = string 121 | description = "RBAC Lastname (only when airflow_authentication = 'rbac')" 122 | default = "airflow" 123 | } 124 | 125 | // ECS variables 126 | variable "ecs_cpu" { 127 | type = number 128 | description = "The allocated cpu for your airflow instance" 129 | default = 1024 130 | } 131 | 132 | variable "ecs_memory" { 133 | type = number 134 | description = "The allocated memory for your airflow instance" 135 | default = 2048 136 | } 137 | 138 | // Networking variables 139 | variable "ip_allow_list" { 140 | type = list(string) 141 | description = "A list of ip ranges that are allowed to access the airflow webserver, default: full access" 142 | default = ["0.0.0.0/0"] 143 | } 144 | 145 | variable "vpc_id" { 146 | type = string 147 | description = "The id of the vpc where you will run ECS/RDS" 148 | 149 | validation { 150 | condition = can(regex("^vpc-", var.vpc_id)) 151 | error_message = "The vpc_id value must be a valid VPC id, starting with \"vpc-\"." 152 | } 153 | } 154 | 155 | variable "public_subnet_ids" { 156 | type = list(string) 157 | description = "A list of subnet ids of where the ALB will reside, if the \"private_subnet_ids\" variable is not provided ECS and RDS will also reside in these subnets" 158 | 159 | validation { 160 | condition = length(var.public_subnet_ids) >= 2 161 | error_message = "The size of the list \"public_subnet_ids\" must be at least 2." 162 | } 163 | } 164 | 165 | variable "private_subnet_ids" { 166 | type = list(string) 167 | description = "A list of subnet ids of where the ECS and RDS reside, this will only work if you have a NAT Gateway in your VPC" 168 | default = [] 169 | 170 | validation { 171 | condition = length(var.private_subnet_ids) >= 2 || length(var.private_subnet_ids) == 0 172 | error_message = "The size of the list \"private_subnet_ids\" must be at least 2 or empty." 173 | } 174 | } 175 | 176 | // ACM + Route53 177 | variable "use_https" { 178 | type = bool 179 | description = "Expose traffic using HTTPS or not" 180 | default = false 181 | } 182 | 183 | variable "dns_name" { 184 | type = string 185 | description = "The DNS name that will be used to expose Airflow. Optional if not serving over HTTPS. Will be autogenerated if not provided" 186 | default = "" 187 | } 188 | 189 | variable "certificate_arn" { 190 | type = string 191 | description = "The ARN of the certificate that will be used" 192 | default = "" 193 | } 194 | 195 | variable "route53_zone_name" { 196 | type = string 197 | description = "The name of a Route53 zone that will be used for the certificate validation." 198 | default = "" 199 | } 200 | 201 | 202 | // Database variables 203 | variable "postgres_uri" { 204 | type = string 205 | description = "The postgres uri of your postgres db, if none provided a postgres db in rds is made. Format \":@:/\"" 206 | default = "" 207 | } 208 | 209 | variable "rds_allocated_storage" { 210 | type = number 211 | description = "The allocated storage for the rds db in gibibytes" 212 | default = 20 213 | } 214 | 215 | variable "rds_storage_type" { 216 | type = string 217 | description = <