├── .gitignore ├── .header.md ├── .pre-commit-config.yaml ├── .terraform-docs.yaml ├── .terraform.lock.hcl ├── .tflint.hcl ├── .tfsec ├── launch_configuration_imdsv2_tfchecks.json ├── launch_template_imdsv2_tfchecks.json ├── no_launch_config_tfchecks.json ├── sg_no_embedded_egress_rules_tfchecks.json └── sg_no_embedded_ingress_rules_tfchecks.json ├── CODEOWNERS ├── LICENSE ├── NOTICE.txt ├── README.md ├── data.tf ├── examples ├── additional-iam-permissions │ ├── .header.md │ ├── .terraform.lock.hcl │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf ├── agent-configuration-options │ ├── .header.md │ ├── .terraform.lock.hcl │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf ├── basic │ ├── .header.md │ ├── .terraform.lock.hcl │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf └── bring-your-own-network │ ├── .header.md │ ├── .terraform.lock.hcl │ ├── README.md │ ├── main.tf │ ├── outputs.tf │ ├── variables.tf │ └── versions.tf ├── images ├── basic.png └── bring-your-own-network.png ├── main.tf ├── outputs.tf ├── prefect-agent.sh.tpl ├── providers.tf ├── test ├── examples_additional_iam_permissions_test.go ├── examples_agent_configuration_options_test.go ├── examples_basic_test.go └── examples_bring_your_own_network_test.go ├── variables.tf └── vpc.tf /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | plan.out 3 | plan.out.json 4 | 5 | # Local .terraform directories 6 | **/.terraform/* 7 | 8 | # .tfstate files 9 | *.tfstate 10 | *.tfstate.* 11 | 12 | # Crash log files 13 | crash.log 14 | 15 | # Exclude all .tfvars files, which are likely to contain sentitive data, such as 16 | # password, private keys, and other secrets. These should not be part of version 17 | # control as they are data points which are potentially sensitive and subject 18 | # to change depending on the environment. 19 | # 20 | *.tfvars 21 | 22 | # Ignore override files as they are usually used to override resources locally and so 23 | # are not checked in 24 | override.tf 25 | override.tf.json 26 | *_override.tf 27 | *_override.tf.json 28 | 29 | # Include override files you do wish to add to version control using negated pattern 30 | # 31 | # !example_override.tf 32 | 33 | # Include tfplan files to ignore the plan output of command: terraform plan -out=tfplan 34 | # example: *tfplan* 35 | 36 | # Ignore CLI configuration files 37 | .terraformrc 38 | terraform.rc 39 | go.mod 40 | go.sum 41 | -------------------------------------------------------------------------------- /.header.md: -------------------------------------------------------------------------------- 1 | # Prefect Docker Agent on AWS EC2 Module 2 | 3 | The [Prefect Agent](https://docs.prefect.io/orchestration/agents/overview.html) is a lightweight process that orchestrates [flow runs](https://docs.prefect.io/core/concepts/flows.html). It is responsible for starting and monitoring flow runs. During operation the agent process queries the Prefect API for any scheduled flow runs, and allocates resources for them on their respective deployment platforms. 4 | 5 | This Terraform module deploys the infrastructure required to run the Prefect Docker agent on AWS EC2. By default, it deploys AWS VPC Networking resources, which are best practice for Prefect configuration. You can disable the default networking as show in the [bring your own network](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/bring-your-own-network) example. 6 | 7 | ![architecture diagram](https://github.com/aws-ia/terraform-prefect-agent-ec2/blob/bcdbb31cd7ffb55926ccd78c60e2f35c28fc3450/images/basic.png) 8 | 9 | ## Prerequisites 10 | 11 | 1. Generate an [API service account key](https://docs.prefect.io/orchestration/concepts/api_keys.html#using-api-keys) for the agent 12 | 2. Store the API key in AWS Secrets Manager in the console, or using the following CLI command. The secret is not created by this Terraform module intentionally, as Terraform would store the API key in plaintext within the state file. 13 | ``` 14 | aws secretsmanager create-secret --name prefect-api-key --secret-string "{\"key\":\"API_KEY_HERE\"}" 15 | ``` 16 | > Note - if you receive the following error from Terraform, the secret has not been created or the name of the secret provided to Terraform was incorrect. 17 | ``` 18 | Error: Secrets Manager Secret "prefect-api-key" not found 19 | ``` 20 | 21 | ## Examples 22 | 23 | Review the `examples/` directory for several specific deployment patterns: 24 | * [Agent Configuration Options](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/agent-configuration-options) - Demonstrates common agent configuration options that can be passed to the module 25 | * [Additional IAM Permissions](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/additional-iam-permissions) - Uses the IAM role built within the module to add additional permissions to the Prefect Agent EC2 Instance 26 | * [Basic](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/basic) - Simple deployment of the module with **no** inputs provided 27 | * [Bring Your Own Network](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/bring-your-own-network) - Demonstrates using network resources that were built outside of the scope of this module 28 | 29 | ## Agent Configuration 30 | 31 | Several agent configuration options are exposed through this module. Please find more documentation on the following configuration options [here](https://docs.prefect.io/orchestration/agents/overview.html#common-configuration-options). 32 | * [Prefect API address](https://docs.prefect.io/orchestration/agents/overview.html#prefect-api-address) 33 | * [Labels](https://docs.prefect.io/orchestration/agents/overview.html#labels) 34 | * [Agent Automations](https://docs.prefect.io/orchestration/agents/overview.html#agent-automations) 35 | * [Streaming Flow Run Logs](https://docs.prefect.io/orchestration/agents/docker.html#streaming-flow-run-logs) 36 | * [Disabling Image Pulling](https://docs.prefect.io/orchestration/agents/docker.html#disabling-image-pulling) 37 | -------------------------------------------------------------------------------- /.pre-commit-config.yaml: -------------------------------------------------------------------------------- 1 | --- 2 | fail_fast: false 3 | minimum_pre_commit_version: "2.6.0" 4 | repos: 5 | - 6 | repo: https://github.com/aws-ia/pre-commit-configs 7 | # To update run: 8 | # pre-commit autoupdate --freeze 9 | rev: 80ed3f0a164f282afaac0b6aec70e20f7e541932 # frozen: v1.5.0 10 | hooks: 11 | - id: aws-ia-meta-hook -------------------------------------------------------------------------------- /.terraform-docs.yaml: -------------------------------------------------------------------------------- 1 | formatter: markdown 2 | header-from: .header.md 3 | settings: 4 | anchor: true 5 | color: true 6 | default: true 7 | escape: true 8 | html: true 9 | indent: 2 10 | required: true 11 | sensitive: true 12 | type: true 13 | 14 | sort: 15 | enabled: true 16 | by: required 17 | 18 | output: 19 | file: README.md 20 | mode: replace -------------------------------------------------------------------------------- /.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.2.0" 6 | constraints = ">= 3.28.0, >= 3.63.0, >= 3.72.0" 7 | hashes = [ 8 | "h1:7xPC2b+Plr514HPRf837t3YFzlSSIY03StrScaVIfw0=", 9 | "zh:297d6462055eac8eb5c6735bd1a0fec23574e27d56c4c14a39efd8f3931ce4ed", 10 | "zh:457319839adca3638fd76f49fd65e15756717f97ac99bd1805a1c9387a62a250", 11 | "zh:57377384fa28abc4211a0916fc0fb590af238d096ad0490434ffeb89f568df9b", 12 | "zh:578e1d21bd6d38bdaef0909b30959b884e84e6c464796a50e516822955db162a", 13 | "zh:5e7ff13cc976f609aee4ada3c1967ba1f0ce5d276f3102a0aeaedc586d25ea80", 14 | "zh:5e94f09fe1874a2365bd566fecab8f676cd720da1c0bf70875392679549ebf20", 15 | "zh:93da14d7ffb8550b161cb79fe2cfc0f66848dd5022974399ae2bf88da7b9e9c5", 16 | "zh:c51e4541f3d29627974dcb7f5919012a762391accb574ade9e28bdb3c92bada5", 17 | "zh:eff58c1680e3f29e514919346d937bbe47278434ae03ed62443c77e878e267b1", 18 | "zh:f2b749e6c6b77b26e643bbecc829977270cfefab106d5ea57e5a83e96d49cbdd", 19 | "zh:fcc17e60e55c278535c332469727cf215eaea9ec81d38e2b5f05be127ee39a5b", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /.tflint.hcl: -------------------------------------------------------------------------------- 1 | # https://github.com/terraform-linters/tflint/blob/master/docs/user-guide/module-inspection.md 2 | # borrowed & modified indefinitely from https://github.com/ksatirli/building-infrastructure-you-can-mostly-trust/blob/main/.tflint.hcl 3 | 4 | plugin "aws" { 5 | enabled = true 6 | version = "0.12.0" 7 | source = "github.com/terraform-linters/tflint-ruleset-aws" 8 | } 9 | 10 | config { 11 | module = true 12 | force = false 13 | } 14 | 15 | rule "terraform_required_providers" { 16 | enabled = true 17 | } 18 | 19 | rule "terraform_required_version" { 20 | enabled = true 21 | } 22 | 23 | rule "terraform_naming_convention" { 24 | enabled = true 25 | format = "snake_case" 26 | } 27 | 28 | rule "terraform_typed_variables" { 29 | enabled = true 30 | } 31 | 32 | rule "terraform_unused_declarations" { 33 | enabled = true 34 | } 35 | 36 | rule "terraform_comment_syntax" { 37 | enabled = true 38 | } 39 | 40 | rule "terraform_deprecated_index" { 41 | enabled = true 42 | } 43 | 44 | rule "terraform_deprecated_interpolation" { 45 | enabled = true 46 | } 47 | 48 | rule "terraform_documented_outputs" { 49 | enabled = true 50 | } 51 | 52 | rule "terraform_documented_variables" { 53 | enabled = true 54 | } 55 | 56 | rule "terraform_module_pinned_source" { 57 | enabled = true 58 | } 59 | 60 | rule "terraform_standard_module_structure" { 61 | enabled = true 62 | } 63 | 64 | rule "terraform_workspace_remote" { 65 | enabled = true 66 | } 67 | -------------------------------------------------------------------------------- /.tfsec/launch_configuration_imdsv2_tfchecks.json: -------------------------------------------------------------------------------- 1 | { 2 | "checks": [ 3 | { 4 | "code": "CUS002", 5 | "description": "Check to IMDSv2 is required on EC2 instances created by this Launch Template", 6 | "impact": "Instance metadata service can be interacted with freely", 7 | "resolution": "Enable HTTP token requirement for IMDS", 8 | "requiredTypes": [ 9 | "resource" 10 | ], 11 | "requiredLabels": [ 12 | "aws_launch_configuration" 13 | ], 14 | "severity": "CRITICAL", 15 | "matchSpec": { 16 | "action": "isPresent", 17 | "name": "metadata_options", 18 | "subMatch": { 19 | "action": "and", 20 | "predicateMatchSpec": [ 21 | { 22 | "action": "equals", 23 | "name": "http_tokens", 24 | "value": "required" 25 | 26 | } 27 | ] 28 | } 29 | }, 30 | 31 | "errorMessage": "is missing `metadata_options` block - it is required with `http_tokens` set to `required` to make Instance Metadata Service more secure.", 32 | "relatedLinks": [ 33 | "https://tfsec.dev/docs/aws/ec2/enforce-http-token-imds#aws/ec2", 34 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_configuration#metadata-options", 35 | "https://aws.amazon.com/blogs/security/defense-in-depth-open-firewalls-reverse-proxies-ssrf-vulnerabilities-ec2-instance-metadata-service" 36 | ] 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /.tfsec/launch_template_imdsv2_tfchecks.json: -------------------------------------------------------------------------------- 1 | { 2 | "checks": [ 3 | { 4 | "code": "CUS001", 5 | "description": "Check to IMDSv2 is required on EC2 instances created by this Launch Template", 6 | "impact": "Instance metadata service can be interacted with freely", 7 | "resolution": "Enable HTTP token requirement for IMDS", 8 | "requiredTypes": [ 9 | "resource" 10 | ], 11 | "requiredLabels": [ 12 | "aws_launch_template" 13 | ], 14 | "severity": "CRITICAL", 15 | "matchSpec": { 16 | "action": "isPresent", 17 | "name": "metadata_options", 18 | "subMatch": { 19 | "action": "and", 20 | "predicateMatchSpec": [ 21 | { 22 | "action": "equals", 23 | "name": "http_tokens", 24 | "value": "required" 25 | 26 | } 27 | ] 28 | } 29 | }, 30 | 31 | "errorMessage": "is missing `metadata_options` block - it is required with `http_tokens` set to `required` to make Instance Metadata Service more secure.", 32 | "relatedLinks": [ 33 | "https://tfsec.dev/docs/aws/ec2/enforce-http-token-imds#aws/ec2", 34 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template#metadata-options", 35 | "https://aws.amazon.com/blogs/security/defense-in-depth-open-firewalls-reverse-proxies-ssrf-vulnerabilities-ec2-instance-metadata-service" 36 | ] 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /.tfsec/no_launch_config_tfchecks.json: -------------------------------------------------------------------------------- 1 | { 2 | "checks": [ 3 | { 4 | "code": "CUS003", 5 | "description": "Use `aws_launch_template` over `aws_launch_configuration", 6 | "impact": "Launch configurations are not capable of versions", 7 | "resolution": "Convert resource type and attributes to `aws_launch_template`", 8 | "requiredTypes": [ 9 | "resource" 10 | ], 11 | "requiredLabels": [ 12 | "aws_launch_configuration" 13 | ], 14 | "severity": "MEDIUM", 15 | "matchSpec": { 16 | "action": "notPresent", 17 | "name": "image_id" 18 | }, 19 | 20 | "errorMessage": "should be changed to `aws_launch_template` since the functionality is the same but templates can be versioned.", 21 | "relatedLinks": [ 22 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template", 23 | "https://aws.amazon.com/blogs/security/defense-in-depth-open-firewalls-reverse-proxies-ssrf-vulnerabilities-ec2-instance-metadata-service" 24 | ] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /.tfsec/sg_no_embedded_egress_rules_tfchecks.json: -------------------------------------------------------------------------------- 1 | { 2 | "checks": [ 3 | { 4 | "code": "CUS005", 5 | "description": "Security group rules should be defined with `aws_security_group_rule` instead of embedded.", 6 | "impact": "Embedded security group rules can cause issues during configuration updates.", 7 | "resolution": "Move `egress` rules to `aws_security_group_rule` and attach to `aws_security_group`.", 8 | "requiredTypes": [ 9 | "resource" 10 | ], 11 | "requiredLabels": [ 12 | "aws_security_group" 13 | ], 14 | "severity": "MEDIUM", 15 | "matchSpec": { 16 | "action": "notPresent", 17 | "name": "egress" 18 | }, 19 | 20 | "errorMessage": "`egress` rules should be moved to `aws_security_group_rule` and attached to `aws_security_group` instead of embedded.", 21 | "relatedLinks": [ 22 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule", 23 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group" 24 | ] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /.tfsec/sg_no_embedded_ingress_rules_tfchecks.json: -------------------------------------------------------------------------------- 1 | { 2 | "checks": [ 3 | { 4 | "code": "CUS004", 5 | "description": "Security group rules should be defined with `aws_security_group_rule` instead of embedded.", 6 | "impact": "Embedded security group rules can cause issues during configuration updates.", 7 | "resolution": "Move `ingress` rules to `aws_security_group_rule` and attach to `aws_security_group`.", 8 | "requiredTypes": [ 9 | "resource" 10 | ], 11 | "requiredLabels": [ 12 | "aws_security_group" 13 | ], 14 | "severity": "MEDIUM", 15 | "matchSpec": { 16 | "action": "notPresent", 17 | "name": "ingress" 18 | }, 19 | 20 | "errorMessage": "`ingress` rules should be moved to `aws_security_group_rule` and attached to `aws_security_group` instead of embedded.", 21 | "relatedLinks": [ 22 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule", 23 | "https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group" 24 | ] 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @drewmullen @aws-ia/aws-ia-terraform-core -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /NOTICE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2016-2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"). You may not use this file except in compliance with the License. A copy of the License is located at 4 | 5 | http://aws.amazon.com/apache2.0/ 6 | 7 | or in the "license" file accompanying this file. This file is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Prefect Docker Agent on AWS EC2 Module 3 | 4 | The [Prefect Agent](https://docs.prefect.io/orchestration/agents/overview.html) is a lightweight process that orchestrates [flow runs](https://docs.prefect.io/core/concepts/flows.html). It is responsible for starting and monitoring flow runs. During operation the agent process queries the Prefect API for any scheduled flow runs, and allocates resources for them on their respective deployment platforms. 5 | 6 | This Terraform module deploys the infrastructure required to run the Prefect Docker agent on AWS EC2. By default, it deploys AWS VPC Networking resources, which are best practice for Prefect configuration. You can disable the default networking as show in the [bring your own network](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/bring-your-own-network) example. 7 | 8 | ![architecture diagram](https://github.com/aws-ia/terraform-prefect-agent-ec2/blob/bcdbb31cd7ffb55926ccd78c60e2f35c28fc3450/images/basic.png) 9 | 10 | ## Prerequisites 11 | 12 | 1. Generate an [API service account key](https://docs.prefect.io/orchestration/concepts/api_keys.html#using-api-keys) for the agent 13 | 2. Store the API key in AWS Secrets Manager in the console, or using the following CLI command. The secret is not created by this Terraform module intentionally, as Terraform would store the API key in plaintext within the state file. 14 | ``` 15 | aws secretsmanager create-secret --name prefect-api-key --secret-string "{\"key\":\"API_KEY_HERE\"}" 16 | ``` 17 | > Note - if you receive the following error from Terraform, the secret has not been created or the name of the secret provided to Terraform was incorrect. 18 | ``` 19 | Error: Secrets Manager Secret "prefect-api-key" not found 20 | ``` 21 | 22 | ## Examples 23 | 24 | Review the `examples/` directory for several specific deployment patterns: 25 | * [Agent Configuration Options](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/agent-configuration-options) - Demonstrates common agent configuration options that can be passed to the module 26 | * [Additional IAM Permissions](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/additional-iam-permissions) - Uses the IAM role built within the module to add additional permissions to the Prefect Agent EC2 Instance 27 | * [Basic](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/basic) - Simple deployment of the module with **no** inputs provided 28 | * [Bring Your Own Network](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/examples/bring-your-own-network) - Demonstrates using network resources that were built outside of the scope of this module 29 | 30 | ## Agent Configuration 31 | 32 | Several agent configuration options are exposed through this module. Please find more documentation on the following configuration options [here](https://docs.prefect.io/orchestration/agents/overview.html#common-configuration-options). 33 | * [Prefect API address](https://docs.prefect.io/orchestration/agents/overview.html#prefect-api-address) 34 | * [Labels](https://docs.prefect.io/orchestration/agents/overview.html#labels) 35 | * [Agent Automations](https://docs.prefect.io/orchestration/agents/overview.html#agent-automations) 36 | * [Streaming Flow Run Logs](https://docs.prefect.io/orchestration/agents/docker.html#streaming-flow-run-logs) 37 | * [Disabling Image Pulling](https://docs.prefect.io/orchestration/agents/docker.html#disabling-image-pulling) 38 | 39 | ## Requirements 40 | 41 | | Name | Version | 42 | |------|---------| 43 | | [terraform](#requirement\_terraform) | >= 0.14.0 | 44 | | [aws](#requirement\_aws) | >= 3.72.0, < 4.0.0 | 45 | 46 | ## Providers 47 | 48 | | Name | Version | 49 | |------|---------| 50 | | [aws](#provider\_aws) | >= 3.72.0, < 4.0.0 | 51 | 52 | ## Modules 53 | 54 | | Name | Source | Version | 55 | |------|--------|---------| 56 | | [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | 3.11.4 | 57 | | [vpc\_endpoints](#module\_vpc\_endpoints) | terraform-aws-modules/vpc/aws//modules/vpc-endpoints | 3.11.4 | 58 | 59 | ## Resources 60 | 61 | | Name | Type | 62 | |------|------| 63 | | [aws_autoscaling_group.prefect](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/autoscaling_group) | resource | 64 | | [aws_iam_instance_profile.instance_profile](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_instance_profile) | resource | 65 | | [aws_iam_policy_attachment.ssm_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource | 66 | | [aws_iam_role.role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | 67 | | [aws_iam_role_policy.policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | 68 | | [aws_launch_template.prefect](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/launch_template) | resource | 69 | | [aws_security_group.endpoints](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | 70 | | [aws_security_group.sg](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | 71 | | [aws_security_group_rule.prefect_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | 72 | | [aws_security_group_rule.vpce_egress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | 73 | | [aws_security_group_rule.vpce_ingress](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | 74 | | [aws_ami.amazon_linux_2](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/ami) | data source | 75 | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | 76 | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | 77 | | [aws_secretsmanager_secret.prefect](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/secretsmanager_secret) | data source | 78 | 79 | ## Inputs 80 | 81 | | Name | Description | Type | Default | Required | 82 | |------|-------------|------|---------|:--------:| 83 | | [agent\_automation\_config](#input\_agent\_automation\_config) | config id to apply to the prefect agent to enable cloud automations | `string` | `""` | no | 84 | | [ami\_id](#input\_ami\_id) | ami to launch the ec2 instance from, windows images not supported | `string` | `""` | no | 85 | | [attach\_ssm\_policy](#input\_attach\_ssm\_policy) | Attach ssm policy to the prefect iam role | `bool` | `true` | no | 86 | | [custom\_tags](#input\_custom\_tags) | custom tags which can be passed on to the AWS resources. they should be key value pairs having distinct keys. | `map(any)` | `{}` | no | 87 | | [deploy\_network](#input\_deploy\_network) | deploy lightweight network to host the prefect agent | `bool` | `true` | no | 88 | | [desired\_capacity](#input\_desired\_capacity) | the number of prefect agents that should be running in the auto scaling group | `number` | `1` | no | 89 | | [disable\_image\_pulling](#input\_disable\_image\_pulling) | disables the prefect agents ability to pull non-local images | `string` | `false` | no | 90 | | [enable\_detailed\_monitoring](#input\_enable\_detailed\_monitoring) | enable detailed monitoring on the prefect agent (1 min intervals) | `bool` | `true` | no | 91 | | [enable\_local\_flow\_logs](#input\_enable\_local\_flow\_logs) | enables flow logs to output locally on the agent | `bool` | `false` | no | 92 | | [enable\_single\_nat\_gateway](#input\_enable\_single\_nat\_gateway) | enable a shared nat gateway within your vpc | `bool` | `false` | no | 93 | | [iam\_role\_id](#input\_iam\_role\_id) | iam role to attach to the prefect launch template, if not provided, a default one will be created | `string` | `null` | no | 94 | | [instance\_type](#input\_instance\_type) | AWS instance type | `string` | `"t3.medium"` | no | 95 | | [key\_name](#input\_key\_name) | private pem key to apply to the prefect instances | `string` | `null` | no | 96 | | [linux\_type](#input\_linux\_type) | type of linux instance | `string` | `"linux_amd64"` | no | 97 | | [max\_capacity](#input\_max\_capacity) | the maximum size of the auto scaling group | `number` | `1` | no | 98 | | [min\_capacity](#input\_min\_capacity) | the minimum size of the auto scaling group | `number` | `1` | no | 99 | | [prefect\_api\_address](#input\_prefect\_api\_address) | the api address that the prefect agent queries for pending flow runs | `string` | `"https://api.prefect.io"` | no | 100 | | [prefect\_api\_key\_secret\_name](#input\_prefect\_api\_key\_secret\_name) | id of aws secrets manager secret for prefect api key | `string` | `"prefect-api-key"` | no | 101 | | [prefect\_labels](#input\_prefect\_labels) | labels to apply to the prefect agent | `string` | `"[]"` | no | 102 | | [prefect\_secret\_key](#input\_prefect\_secret\_key) | key of aws secrets manager secret for prefect api key | `string` | `"key"` | no | 103 | | [security\_group\_ids](#input\_security\_group\_ids) | security group(s) to attach to the prefect launch template, if not provided, a default one will be created | `list(string)` | `null` | no | 104 | | [subnet\_ids](#input\_subnet\_ids) | ids of the subnets to assign to the autoscaling group | `list(string)` | `[]` | no | 105 | | [vpc\_cidr](#input\_vpc\_cidr) | cidr range to apply to your vpc | `string` | `"192.168.0.0/24"` | no | 106 | | [vpc\_id](#input\_vpc\_id) | id of the vpc to deploy the prefect agent into | `string` | `""` | no | 107 | 108 | ## Outputs 109 | 110 | | Name | Description | 111 | |------|-------------| 112 | | [prefect\_role\_id](#output\_prefect\_role\_id) | iam role id of the role attached to the prefect launch template | 113 | -------------------------------------------------------------------------------- /data.tf: -------------------------------------------------------------------------------- 1 | data "aws_caller_identity" "current" {} 2 | 3 | data "aws_region" "current" {} 4 | 5 | data "aws_ami" "amazon_linux_2" { 6 | most_recent = true 7 | owners = ["amazon"] 8 | filter { 9 | name = "owner-alias" 10 | values = ["amazon"] 11 | } 12 | 13 | filter { 14 | name = "name" 15 | values = ["amzn2-ami-hvm-*-x86_64-ebs"] 16 | } 17 | } 18 | 19 | data "aws_secretsmanager_secret" "prefect" { 20 | name = var.prefect_api_key_secret_name 21 | } 22 | 23 | locals { 24 | image_pulling = var.disable_image_pulling ? "--no-pull" : "" 25 | flow_logs = var.enable_local_flow_logs ? "--show-flow-logs" : "" 26 | subnet_cidrs = var.deploy_network ? cidrsubnets(var.vpc_cidr, 2, 2, 2, 2) : null 27 | config_id = var.agent_automation_config != "" ? "--agent-config-id ${var.agent_automation_config}" : "" 28 | } -------------------------------------------------------------------------------- /examples/additional-iam-permissions/.header.md: -------------------------------------------------------------------------------- 1 | # Additional IAM Permissions 2 | 3 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group as well as a lightweight network to host the agent. It also creates an additional IAM policy and attaches it to the existing IAM role for the agent. 4 | 5 | ## Usage 6 | 7 | To run this example you need to execute: 8 | ``` 9 | $ terraform init 10 | $ terraform plan 11 | $ terraform apply 12 | ``` 13 | Note that this example may create resources which can cost money (AWS EC2, VPC endpoints, NAT gateway, for example). Run terraform destroy when you don't need these resources. -------------------------------------------------------------------------------- /examples/additional-iam-permissions/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.2.0" 6 | constraints = ">= 3.28.0, >= 3.63.0, >= 3.72.0" 7 | hashes = [ 8 | "h1:7xPC2b+Plr514HPRf837t3YFzlSSIY03StrScaVIfw0=", 9 | "zh:297d6462055eac8eb5c6735bd1a0fec23574e27d56c4c14a39efd8f3931ce4ed", 10 | "zh:457319839adca3638fd76f49fd65e15756717f97ac99bd1805a1c9387a62a250", 11 | "zh:57377384fa28abc4211a0916fc0fb590af238d096ad0490434ffeb89f568df9b", 12 | "zh:578e1d21bd6d38bdaef0909b30959b884e84e6c464796a50e516822955db162a", 13 | "zh:5e7ff13cc976f609aee4ada3c1967ba1f0ce5d276f3102a0aeaedc586d25ea80", 14 | "zh:5e94f09fe1874a2365bd566fecab8f676cd720da1c0bf70875392679549ebf20", 15 | "zh:93da14d7ffb8550b161cb79fe2cfc0f66848dd5022974399ae2bf88da7b9e9c5", 16 | "zh:c51e4541f3d29627974dcb7f5919012a762391accb574ade9e28bdb3c92bada5", 17 | "zh:eff58c1680e3f29e514919346d937bbe47278434ae03ed62443c77e878e267b1", 18 | "zh:f2b749e6c6b77b26e643bbecc829977270cfefab106d5ea57e5a83e96d49cbdd", 19 | "zh:fcc17e60e55c278535c332469727cf215eaea9ec81d38e2b5f05be127ee39a5b", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/additional-iam-permissions/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Additional IAM Permissions 3 | 4 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group as well as a lightweight network to host the agent. It also creates an additional IAM policy and attaches it to the existing IAM role for the agent. 5 | 6 | ## Usage 7 | 8 | To run this example you need to execute: 9 | ``` 10 | $ terraform init 11 | $ terraform plan 12 | $ terraform apply 13 | ``` 14 | Note that this example may create resources which can cost money (AWS EC2, VPC endpoints, NAT gateway, for example). Run terraform destroy when you don't need these resources. 15 | 16 | ## Requirements 17 | 18 | | Name | Version | 19 | |------|---------| 20 | | [terraform](#requirement\_terraform) | >= 0.14.0 | 21 | | [aws](#requirement\_aws) | >= 3.72.0 | 22 | 23 | ## Providers 24 | 25 | | Name | Version | 26 | |------|---------| 27 | | [aws](#provider\_aws) | 4.2.0 | 28 | 29 | ## Modules 30 | 31 | | Name | Source | Version | 32 | |------|--------|---------| 33 | | [prefect](#module\_prefect) | ../../ | n/a | 34 | 35 | ## Resources 36 | 37 | | Name | Type | 38 | |------|------| 39 | | [aws_iam_role_policy.additional_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy) | resource | 40 | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | 41 | 42 | ## Inputs 43 | 44 | No inputs. 45 | 46 | ## Outputs 47 | 48 | No outputs. 49 | -------------------------------------------------------------------------------- /examples/additional-iam-permissions/main.tf: -------------------------------------------------------------------------------- 1 | module "prefect" { 2 | source = "../../" 3 | } 4 | 5 | data "aws_caller_identity" "current" {} 6 | 7 | resource "aws_iam_role_policy" "additional_policy" { 8 | name_prefix = "prefect-agent-additional" 9 | role = module.prefect.prefect_role_id 10 | 11 | policy = jsonencode({ 12 | Version = "2012-10-17" 13 | Statement = [ 14 | { 15 | Action = [ 16 | "ecr:Batch*" 17 | ] 18 | Effect = "Allow" 19 | Resource = ["arn:aws:ecr:*:${data.aws_caller_identity.current.account_id}:repository/*"] 20 | }, 21 | ] 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /examples/additional-iam-permissions/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/examples/additional-iam-permissions/outputs.tf -------------------------------------------------------------------------------- /examples/additional-iam-permissions/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/examples/additional-iam-permissions/variables.tf -------------------------------------------------------------------------------- /examples/additional-iam-permissions/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.14.0" 3 | required_providers { 4 | aws = { 5 | source = "hashicorp/aws" 6 | version = ">= 3.72.0" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/agent-configuration-options/.header.md: -------------------------------------------------------------------------------- 1 | # Agent Configuration Options 2 | 3 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group as well as a lightweight network to host the agent. It also provides additional configuration to the agent like [labels](https://docs.prefect.io/orchestration/agents/overview.html#labels) & an [automation id](https://docs.prefect.io/orchestration/concepts/automations.html#overview). 4 | 5 | ## Usage 6 | 7 | To run this example you need to execute: 8 | ``` 9 | $ terraform init 10 | $ terraform plan 11 | $ terraform apply 12 | ``` 13 | Note that this example may create resources which can cost money (AWS EC2, VPC endpoints, NAT gateway, for example). Run terraform destroy when you don't need these resources. -------------------------------------------------------------------------------- /examples/agent-configuration-options/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.2.0" 6 | constraints = ">= 3.28.0, >= 3.63.0, >= 3.72.0" 7 | hashes = [ 8 | "h1:7xPC2b+Plr514HPRf837t3YFzlSSIY03StrScaVIfw0=", 9 | "zh:297d6462055eac8eb5c6735bd1a0fec23574e27d56c4c14a39efd8f3931ce4ed", 10 | "zh:457319839adca3638fd76f49fd65e15756717f97ac99bd1805a1c9387a62a250", 11 | "zh:57377384fa28abc4211a0916fc0fb590af238d096ad0490434ffeb89f568df9b", 12 | "zh:578e1d21bd6d38bdaef0909b30959b884e84e6c464796a50e516822955db162a", 13 | "zh:5e7ff13cc976f609aee4ada3c1967ba1f0ce5d276f3102a0aeaedc586d25ea80", 14 | "zh:5e94f09fe1874a2365bd566fecab8f676cd720da1c0bf70875392679549ebf20", 15 | "zh:93da14d7ffb8550b161cb79fe2cfc0f66848dd5022974399ae2bf88da7b9e9c5", 16 | "zh:c51e4541f3d29627974dcb7f5919012a762391accb574ade9e28bdb3c92bada5", 17 | "zh:eff58c1680e3f29e514919346d937bbe47278434ae03ed62443c77e878e267b1", 18 | "zh:f2b749e6c6b77b26e643bbecc829977270cfefab106d5ea57e5a83e96d49cbdd", 19 | "zh:fcc17e60e55c278535c332469727cf215eaea9ec81d38e2b5f05be127ee39a5b", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/agent-configuration-options/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Agent Configuration Options 3 | 4 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group as well as a lightweight network to host the agent. It also provides additional configuration to the agent like [labels](https://docs.prefect.io/orchestration/agents/overview.html#labels) & an [automation id](https://docs.prefect.io/orchestration/concepts/automations.html#overview). 5 | 6 | ## Usage 7 | 8 | To run this example you need to execute: 9 | ``` 10 | $ terraform init 11 | $ terraform plan 12 | $ terraform apply 13 | ``` 14 | Note that this example may create resources which can cost money (AWS EC2, VPC endpoints, NAT gateway, for example). Run terraform destroy when you don't need these resources. 15 | 16 | ## Requirements 17 | 18 | | Name | Version | 19 | |------|---------| 20 | | [terraform](#requirement\_terraform) | >= 0.14.0 | 21 | | [aws](#requirement\_aws) | >= 3.72.0 | 22 | 23 | ## Providers 24 | 25 | No providers. 26 | 27 | ## Modules 28 | 29 | | Name | Source | Version | 30 | |------|--------|---------| 31 | | [prefect](#module\_prefect) | ../../ | n/a | 32 | 33 | ## Resources 34 | 35 | No resources. 36 | 37 | ## Inputs 38 | 39 | | Name | Description | Type | Default | Required | 40 | |------|-------------|------|---------|:--------:| 41 | | [agent\_automation\_config](#input\_agent\_automation\_config) | config id to apply to the prefect agent to enable cloud automations | `string` | n/a | yes | 42 | 43 | ## Outputs 44 | 45 | No outputs. 46 | -------------------------------------------------------------------------------- /examples/agent-configuration-options/main.tf: -------------------------------------------------------------------------------- 1 | module "prefect" { 2 | source = "../../" 3 | 4 | prefect_api_key_secret_name = "prefect-api-key" #tfsec:ignore:general-secrets-no-plaintext-exposure tfsec:ignore:general-secrets-sensitive-in-attribute 5 | prefect_secret_key = "key" #tfsec:ignore:general-secrets-no-plaintext-exposure tfsec:ignore:general-secrets-sensitive-in-attribute 6 | 7 | prefect_labels = "['prod']" 8 | agent_automation_config = var.agent_automation_config 9 | 10 | enable_local_flow_logs = true 11 | } 12 | -------------------------------------------------------------------------------- /examples/agent-configuration-options/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/examples/agent-configuration-options/outputs.tf -------------------------------------------------------------------------------- /examples/agent-configuration-options/variables.tf: -------------------------------------------------------------------------------- 1 | variable "agent_automation_config" { 2 | type = string 3 | description = "config id to apply to the prefect agent to enable cloud automations" 4 | } -------------------------------------------------------------------------------- /examples/agent-configuration-options/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.14.0" 3 | required_providers { 4 | aws = { 5 | source = "hashicorp/aws" 6 | version = ">= 3.72.0" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/basic/.header.md: -------------------------------------------------------------------------------- 1 | # Basic 2 | 3 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group. It also creates a lightweight network to host the agent. 4 | 5 | ![architecture diagram](../../images/basic.png "architecture diagram") 6 | 7 | ## Usage 8 | 9 | To run this example you need to execute: 10 | ``` 11 | $ terraform init 12 | $ terraform plan 13 | $ terraform apply 14 | ``` 15 | Note that this example may create resources which can cost money (AWS EC2, VPC endpoints, NAT gateway, for example). Run terraform destroy when you don't need these resources. 16 | 17 | -------------------------------------------------------------------------------- /examples/basic/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.2.0" 6 | constraints = ">= 3.28.0, >= 3.63.0, >= 3.72.0" 7 | hashes = [ 8 | "h1:7xPC2b+Plr514HPRf837t3YFzlSSIY03StrScaVIfw0=", 9 | "zh:297d6462055eac8eb5c6735bd1a0fec23574e27d56c4c14a39efd8f3931ce4ed", 10 | "zh:457319839adca3638fd76f49fd65e15756717f97ac99bd1805a1c9387a62a250", 11 | "zh:57377384fa28abc4211a0916fc0fb590af238d096ad0490434ffeb89f568df9b", 12 | "zh:578e1d21bd6d38bdaef0909b30959b884e84e6c464796a50e516822955db162a", 13 | "zh:5e7ff13cc976f609aee4ada3c1967ba1f0ce5d276f3102a0aeaedc586d25ea80", 14 | "zh:5e94f09fe1874a2365bd566fecab8f676cd720da1c0bf70875392679549ebf20", 15 | "zh:93da14d7ffb8550b161cb79fe2cfc0f66848dd5022974399ae2bf88da7b9e9c5", 16 | "zh:c51e4541f3d29627974dcb7f5919012a762391accb574ade9e28bdb3c92bada5", 17 | "zh:eff58c1680e3f29e514919346d937bbe47278434ae03ed62443c77e878e267b1", 18 | "zh:f2b749e6c6b77b26e643bbecc829977270cfefab106d5ea57e5a83e96d49cbdd", 19 | "zh:fcc17e60e55c278535c332469727cf215eaea9ec81d38e2b5f05be127ee39a5b", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/basic/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Basic 3 | 4 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group. It also creates a lightweight network to host the agent. 5 | 6 | ![architecture diagram](../../images/basic.png "architecture diagram") 7 | 8 | ## Usage 9 | 10 | To run this example you need to execute: 11 | ``` 12 | $ terraform init 13 | $ terraform plan 14 | $ terraform apply 15 | ``` 16 | Note that this example may create resources which can cost money (AWS EC2, VPC endpoints, NAT gateway, for example). Run terraform destroy when you don't need these resources. 17 | 18 | ## Requirements 19 | 20 | | Name | Version | 21 | |------|---------| 22 | | [terraform](#requirement\_terraform) | >= 0.14.0 | 23 | | [aws](#requirement\_aws) | >= 3.72.0 | 24 | 25 | ## Providers 26 | 27 | No providers. 28 | 29 | ## Modules 30 | 31 | | Name | Source | Version | 32 | |------|--------|---------| 33 | | [prefect](#module\_prefect) | ../../ | n/a | 34 | 35 | ## Resources 36 | 37 | No resources. 38 | 39 | ## Inputs 40 | 41 | No inputs. 42 | 43 | ## Outputs 44 | 45 | No outputs. 46 | -------------------------------------------------------------------------------- /examples/basic/main.tf: -------------------------------------------------------------------------------- 1 | module "prefect" { 2 | source = "../../" 3 | } -------------------------------------------------------------------------------- /examples/basic/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/examples/basic/outputs.tf -------------------------------------------------------------------------------- /examples/basic/variables.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/examples/basic/variables.tf -------------------------------------------------------------------------------- /examples/basic/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.14.0" 3 | required_providers { 4 | aws = { 5 | source = "hashicorp/aws" 6 | version = ">= 3.72.0" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /examples/bring-your-own-network/.header.md: -------------------------------------------------------------------------------- 1 | # Bring Your Own Network 2 | 3 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group. 4 | 5 | ![architecture diagram](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/images/bring-your-own-network.png) 6 | 7 | ## Usage 8 | 9 | To run this example you need to execute: 10 | ``` 11 | $ terraform init 12 | $ terraform plan 13 | $ terraform apply 14 | ``` 15 | Note that this example may create resources which can cost money (AWS EC2, for example). Run terraform destroy when you don't need these resources. -------------------------------------------------------------------------------- /examples/bring-your-own-network/.terraform.lock.hcl: -------------------------------------------------------------------------------- 1 | # This file is maintained automatically by "terraform init". 2 | # Manual edits may be lost in future updates. 3 | 4 | provider "registry.terraform.io/hashicorp/aws" { 5 | version = "4.2.0" 6 | constraints = ">= 3.28.0, >= 3.63.0, >= 3.72.0" 7 | hashes = [ 8 | "h1:7xPC2b+Plr514HPRf837t3YFzlSSIY03StrScaVIfw0=", 9 | "zh:297d6462055eac8eb5c6735bd1a0fec23574e27d56c4c14a39efd8f3931ce4ed", 10 | "zh:457319839adca3638fd76f49fd65e15756717f97ac99bd1805a1c9387a62a250", 11 | "zh:57377384fa28abc4211a0916fc0fb590af238d096ad0490434ffeb89f568df9b", 12 | "zh:578e1d21bd6d38bdaef0909b30959b884e84e6c464796a50e516822955db162a", 13 | "zh:5e7ff13cc976f609aee4ada3c1967ba1f0ce5d276f3102a0aeaedc586d25ea80", 14 | "zh:5e94f09fe1874a2365bd566fecab8f676cd720da1c0bf70875392679549ebf20", 15 | "zh:93da14d7ffb8550b161cb79fe2cfc0f66848dd5022974399ae2bf88da7b9e9c5", 16 | "zh:c51e4541f3d29627974dcb7f5919012a762391accb574ade9e28bdb3c92bada5", 17 | "zh:eff58c1680e3f29e514919346d937bbe47278434ae03ed62443c77e878e267b1", 18 | "zh:f2b749e6c6b77b26e643bbecc829977270cfefab106d5ea57e5a83e96d49cbdd", 19 | "zh:fcc17e60e55c278535c332469727cf215eaea9ec81d38e2b5f05be127ee39a5b", 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /examples/bring-your-own-network/README.md: -------------------------------------------------------------------------------- 1 | 2 | # Bring Your Own Network 3 | 4 | Configuration in this directory creates the Prefect Docker agent deployed on a single EC2 instance in an autoscaling group. 5 | 6 | ![architecture diagram](https://github.com/aws-ia/terraform-prefect-agent-ec2/tree/main/images/bring-your-own-network.png) 7 | 8 | ## Usage 9 | 10 | To run this example you need to execute: 11 | ``` 12 | $ terraform init 13 | $ terraform plan 14 | $ terraform apply 15 | ``` 16 | Note that this example may create resources which can cost money (AWS EC2, for example). Run terraform destroy when you don't need these resources. 17 | 18 | ## Requirements 19 | 20 | | Name | Version | 21 | |------|---------| 22 | | [terraform](#requirement\_terraform) | >= 0.14.0 | 23 | | [aws](#requirement\_aws) | >= 3.72.0 | 24 | 25 | ## Providers 26 | 27 | No providers. 28 | 29 | ## Modules 30 | 31 | | Name | Source | Version | 32 | |------|--------|---------| 33 | | [prefect](#module\_prefect) | ../../ | n/a | 34 | 35 | ## Resources 36 | 37 | No resources. 38 | 39 | ## Inputs 40 | 41 | | Name | Description | Type | Default | Required | 42 | |------|-------------|------|---------|:--------:| 43 | | [key\_name](#input\_key\_name) | private pem key to apply to the prefect instances | `string` | n/a | yes | 44 | | [subnet\_ids](#input\_subnet\_ids) | ids of the subnets to assign to the autoscaling group | `list(string)` | n/a | yes | 45 | | [vpc\_id](#input\_vpc\_id) | id of the vpc to deploy the prefect agent into | `string` | n/a | yes | 46 | 47 | ## Outputs 48 | 49 | No outputs. 50 | -------------------------------------------------------------------------------- /examples/bring-your-own-network/main.tf: -------------------------------------------------------------------------------- 1 | module "prefect" { 2 | source = "../../" 3 | 4 | key_name = var.key_name 5 | 6 | deploy_network = false 7 | vpc_id = var.vpc_id 8 | subnet_ids = var.subnet_ids 9 | } 10 | -------------------------------------------------------------------------------- /examples/bring-your-own-network/outputs.tf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/examples/bring-your-own-network/outputs.tf -------------------------------------------------------------------------------- /examples/bring-your-own-network/variables.tf: -------------------------------------------------------------------------------- 1 | variable "vpc_id" { 2 | type = string 3 | description = "id of the vpc to deploy the prefect agent into" 4 | } 5 | variable "subnet_ids" { 6 | type = list(string) 7 | description = "ids of the subnets to assign to the autoscaling group" 8 | } 9 | variable "key_name" { 10 | type = string 11 | description = "private pem key to apply to the prefect instances" 12 | } -------------------------------------------------------------------------------- /examples/bring-your-own-network/versions.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.14.0" 3 | required_providers { 4 | aws = { 5 | source = "hashicorp/aws" 6 | version = ">= 3.72.0" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /images/basic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/images/basic.png -------------------------------------------------------------------------------- /images/bring-your-own-network.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-ia/terraform-prefect-agent-ec2/c4ed62c3250377b41739495f102cfe0758dee443/images/bring-your-own-network.png -------------------------------------------------------------------------------- /main.tf: -------------------------------------------------------------------------------- 1 | resource "aws_launch_template" "prefect" { 2 | name_prefix = "prefect-agent" 3 | description = "launches a prefect agent on a specified image" 4 | 5 | image_id = var.ami_id == "" ? data.aws_ami.amazon_linux_2.id : var.ami_id 6 | instance_type = var.instance_type 7 | key_name = var.key_name != null ? var.key_name : null 8 | 9 | vpc_security_group_ids = var.security_group_ids == null ? [aws_security_group.sg[0].id] : var.security_group_ids 10 | 11 | iam_instance_profile { 12 | name = aws_iam_instance_profile.instance_profile.name 13 | } 14 | 15 | metadata_options { 16 | http_endpoint = "enabled" 17 | http_tokens = "required" 18 | } 19 | 20 | monitoring { 21 | enabled = var.enable_detailed_monitoring 22 | } 23 | 24 | tag_specifications { 25 | resource_type = "instance" 26 | 27 | tags = merge({ 28 | Name = "prefect-agent" 29 | managed-by = "terraform" 30 | }, var.custom_tags) 31 | } 32 | 33 | user_data = base64encode(templatefile("${path.module}/prefect-agent.sh.tpl", 34 | { 35 | region = data.aws_region.current.name 36 | linux_type = var.linux_type 37 | prefect_secret_name = var.prefect_api_key_secret_name 38 | prefect_secret_key = var.prefect_secret_key 39 | prefect_api_address = var.prefect_api_address 40 | prefect_labels = var.prefect_labels 41 | image_pulling = local.image_pulling 42 | flow_logs = local.flow_logs 43 | config_id = local.config_id 44 | } 45 | )) 46 | } 47 | 48 | resource "aws_autoscaling_group" "prefect" { 49 | name_prefix = "prefect-agent" 50 | max_size = var.max_capacity 51 | min_size = var.min_capacity 52 | health_check_type = "EC2" 53 | desired_capacity = var.desired_capacity 54 | vpc_zone_identifier = var.deploy_network ? module.vpc[0].private_subnets : var.subnet_ids 55 | 56 | lifecycle { 57 | create_before_destroy = true 58 | } 59 | 60 | launch_template { 61 | id = aws_launch_template.prefect.id 62 | version = "$Latest" # support other versions? 63 | } 64 | } 65 | 66 | resource "aws_security_group" "sg" { 67 | count = var.security_group_ids == null ? 1 : 0 68 | name_prefix = "prefect-agent" 69 | description = "allow all outbound traffic from the prefect agent" 70 | vpc_id = var.deploy_network ? module.vpc[0].vpc_id : var.vpc_id 71 | 72 | tags = merge({ 73 | Name = "prefect-agent" 74 | "managed-by" = "terraform" 75 | }, var.custom_tags) 76 | } 77 | 78 | resource "aws_security_group_rule" "prefect_egress" { 79 | count = var.security_group_ids == null ? 1 : 0 80 | description = "allow all egress traffic to the internet" 81 | type = "egress" 82 | from_port = 0 83 | to_port = 0 84 | protocol = -1 85 | cidr_blocks = ["0.0.0.0/0"] #tfsec:ignore:aws-vpc-no-public-egress-sgr 86 | # this is neccessary as the IP of Prefect Cloud (or Prefect server) is not static 87 | # also, ec2 may need to interact with other resources outside of the vpc 88 | # i.e snowflake, on-prem data sources, etc. 89 | security_group_id = var.security_group_ids == null ? aws_security_group.sg[0].id : null 90 | } 91 | 92 | ########################################## 93 | # IAM Policies 94 | ########################################## 95 | resource "aws_iam_instance_profile" "instance_profile" { 96 | name_prefix = "prefect-agent" 97 | role = var.iam_role_id == null ? aws_iam_role.role[0].name : var.iam_role_id 98 | } 99 | 100 | resource "aws_iam_role" "role" { 101 | count = var.iam_role_id == null ? 1 : 0 102 | name_prefix = "prefect-agent" 103 | path = "/" 104 | 105 | assume_role_policy = < /etc/awslogs/awscli.conf 27 | 28 | # start the logs service 29 | systemctl start awslogsd 30 | systemctl enable awslogsd.service 31 | 32 | # prefect agent install 33 | pip3 install prefect 34 | 35 | # get API key 36 | result=$(aws secretsmanager get-secret-value --secret-id ${prefect_secret_name} --region ${region}) 37 | secret=$(echo $result | jq -r '.SecretString') 38 | PREFECT_API_KEY=$(echo $secret | jq -r '.${prefect_secret_key}') 39 | 40 | # create prefect config file 41 | mkdir ~/.prefect 42 | touch ~/.prefect/config.toml 43 | echo " 44 | [cloud.agent] 45 | labels = ${prefect_labels} 46 | " > ~/.prefect/config.toml 47 | 48 | # create systemd config 49 | touch /etc/systemd/system/prefect-agent.service 50 | echo "[Unit] 51 | Description=Prefect Docker Agent 52 | After=network.target 53 | StartLimitIntervalSec=0 54 | [Service] 55 | Type=simple 56 | Restart=on-failure 57 | RestartSec=5 58 | User=root 59 | ExecStart=/usr/local/bin/prefect agent docker start -k $PREFECT_API_KEY --api ${prefect_api_address} ${image_pulling} ${flow_logs} ${config_id} 60 | [Install] 61 | WantedBy=multi-user.target " >> /etc/systemd/system/prefect-agent.service 62 | 63 | # start prefect agent 64 | systemctl start prefect-agent 65 | 66 | # install cred helper 67 | amazon-linux-extras enable docker 68 | yum install amazon-ecr-credential-helper -y 69 | 70 | mkdir ~/.docker 71 | touch ~/.docker/config.json 72 | echo '{ 73 | "credsStore": "ecr-login" 74 | }' >> ~/.docker/config.json -------------------------------------------------------------------------------- /providers.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_version = ">= 0.14.0" 3 | required_providers { 4 | aws = { 5 | source = "hashicorp/aws" 6 | version = ">= 3.72.0, < 4.0.0" 7 | } 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /test/examples_additional_iam_permissions_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | ) 8 | 9 | func TestExamplesAdditionalIamPermissions(t *testing.T) { 10 | 11 | terraformOptions := &terraform.Options{ 12 | TerraformDir: "../examples/additional-iam-permissions", 13 | 14 | Vars: map[string]interface{}{ 15 | "key_name": "prefect", 16 | "vpc_cidr": "10.0.0.0/24", 17 | }, 18 | } 19 | 20 | defer terraform.Destroy(t, terraformOptions) 21 | terraform.InitAndApply(t, terraformOptions) 22 | } -------------------------------------------------------------------------------- /test/examples_agent_configuration_options_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | ) 8 | 9 | func TestExamplesAgentConfigurationOptions(t *testing.T) { 10 | 11 | terraformOptions := &terraform.Options{ 12 | TerraformDir: "../examples/agent-configuration-options", 13 | 14 | Vars: map[string]interface{}{ 15 | "agent_automation_config": "98dn3ks8-aafg-98s8-j37d-k9sn4uf8sn3k", 16 | }, 17 | } 18 | 19 | defer terraform.Destroy(t, terraformOptions) 20 | terraform.InitAndApply(t, terraformOptions) 21 | } -------------------------------------------------------------------------------- /test/examples_basic_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | ) 8 | 9 | func TestExamplesBasic(t *testing.T) { 10 | 11 | terraformOptions := &terraform.Options{ 12 | TerraformDir: "../examples/basic", 13 | } 14 | 15 | defer terraform.Destroy(t, terraformOptions) 16 | terraform.InitAndApply(t, terraformOptions) 17 | } -------------------------------------------------------------------------------- /test/examples_bring_your_own_network_test.go: -------------------------------------------------------------------------------- 1 | package test 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/gruntwork-io/terratest/modules/terraform" 7 | ) 8 | 9 | func TestExamplesBringYourOwnNetwork(t *testing.T) { 10 | 11 | terraformOptions := &terraform.Options{ 12 | TerraformDir: "../examples/bring-your-own-network", 13 | 14 | Vars: map[string]interface{}{ 15 | "key_name": "prefect", 16 | "vpc_id": "vpc-06abcaf941d62184c", 17 | "subnet_ids": []string{"subnet-01a839cb67a879e99","subnet-08df3dd9fb4c1d937"}, 18 | }, 19 | } 20 | 21 | defer terraform.Destroy(t, terraformOptions) 22 | terraform.InitAndApply(t, terraformOptions) 23 | } -------------------------------------------------------------------------------- /variables.tf: -------------------------------------------------------------------------------- 1 | ## general parameters 2 | variable "instance_type" { 3 | type = string 4 | description = "AWS instance type" 5 | default = "t3.medium" 6 | } 7 | variable "ami_id" { 8 | type = string 9 | description = "ami to launch the ec2 instance from, windows images not supported" 10 | default = "" 11 | } 12 | variable "security_group_ids" { 13 | type = list(string) 14 | description = "security group(s) to attach to the prefect launch template, if not provided, a default one will be created" 15 | default = null 16 | } 17 | variable "linux_type" { 18 | type = string 19 | description = "type of linux instance" 20 | default = "linux_amd64" 21 | } 22 | variable "min_capacity" { 23 | type = number 24 | description = "the minimum size of the auto scaling group" 25 | default = 1 26 | } 27 | variable "max_capacity" { 28 | type = number 29 | description = "the maximum size of the auto scaling group" 30 | default = 1 31 | } 32 | variable "desired_capacity" { 33 | type = number 34 | description = "the number of prefect agents that should be running in the auto scaling group" 35 | default = 1 36 | } 37 | variable "enable_detailed_monitoring" { 38 | type = bool 39 | description = "enable detailed monitoring on the prefect agent (1 min intervals)" 40 | default = true 41 | } 42 | variable "key_name" { 43 | type = string 44 | description = "private pem key to apply to the prefect instances" 45 | default = null 46 | } 47 | variable "custom_tags" { 48 | description = "custom tags which can be passed on to the AWS resources. they should be key value pairs having distinct keys." 49 | type = map(any) 50 | default = {} 51 | } 52 | ## parameters for prefect bootstrap script 53 | variable "prefect_api_key_secret_name" { 54 | type = string 55 | description = "id of aws secrets manager secret for prefect api key" 56 | default = "prefect-api-key" #tfsec:ignore:general-secrets-no-plaintext-exposure tfsec:ignore:general-secrets-sensitive-in-variable 57 | # this is not sensitive, it is the default secret name, used so that userdata can pull secret value 58 | } 59 | variable "prefect_secret_key" { 60 | type = string 61 | description = "key of aws secrets manager secret for prefect api key" 62 | default = "key" #tfsec:ignore:general-secrets-no-plaintext-exposure tfsec:ignore:general-secrets-sensitive-in-variable 63 | # this is not sensitive, it is the default key of the secret used so that userdata can pull secret value 64 | } 65 | variable "prefect_api_address" { 66 | type = string 67 | description = "the api address that the prefect agent queries for pending flow runs" 68 | default = "https://api.prefect.io" 69 | } 70 | variable "prefect_labels" { 71 | type = string 72 | description = "labels to apply to the prefect agent" # Use the following format: "['us-east-1']" 73 | default = "[]" 74 | } 75 | variable "agent_automation_config" { 76 | type = string 77 | description = "config id to apply to the prefect agent to enable cloud automations" 78 | default = "" 79 | } 80 | variable "disable_image_pulling" { 81 | type = string 82 | description = "disables the prefect agents ability to pull non-local images" 83 | default = false 84 | } 85 | variable "enable_local_flow_logs" { 86 | type = bool 87 | description = "enables flow logs to output locally on the agent" 88 | default = false 89 | } 90 | ## parameters for network configuration 91 | variable "deploy_network" { 92 | type = bool 93 | description = "deploy lightweight network to host the prefect agent" 94 | default = true 95 | } 96 | variable "vpc_cidr" { 97 | type = string 98 | description = "cidr range to apply to your vpc" 99 | default = "192.168.0.0/24" 100 | } 101 | variable "vpc_id" { 102 | type = string 103 | description = "id of the vpc to deploy the prefect agent into" 104 | default = "" 105 | } 106 | variable "subnet_ids" { 107 | type = list(string) 108 | description = "ids of the subnets to assign to the autoscaling group" 109 | default = [] 110 | } 111 | variable "enable_single_nat_gateway" { 112 | type = bool 113 | description = "enable a shared nat gateway within your vpc" 114 | default = false 115 | } 116 | ## parameters for iam 117 | variable "iam_role_id" { 118 | type = string 119 | description = "iam role to attach to the prefect launch template, if not provided, a default one will be created" 120 | default = null 121 | } 122 | variable "attach_ssm_policy" { 123 | type = bool 124 | description = "Attach ssm policy to the prefect iam role" 125 | default = true 126 | } 127 | -------------------------------------------------------------------------------- /vpc.tf: -------------------------------------------------------------------------------- 1 | module "vpc" { 2 | count = var.deploy_network ? 1 : 0 3 | source = "terraform-aws-modules/vpc/aws" 4 | version = "3.11.4" 5 | 6 | name = "prefect-vpc" 7 | cidr = var.vpc_cidr 8 | 9 | azs = ["${data.aws_region.current.id}a", "${data.aws_region.current.id}b"] 10 | private_subnets = [local.subnet_cidrs[0], local.subnet_cidrs[1]] 11 | public_subnets = [local.subnet_cidrs[2], local.subnet_cidrs[3]] 12 | 13 | manage_default_route_table = true 14 | 15 | enable_dns_hostnames = true 16 | enable_dns_support = true 17 | 18 | enable_nat_gateway = true 19 | single_nat_gateway = var.enable_single_nat_gateway 20 | 21 | manage_default_security_group = true 22 | default_security_group_ingress = [] 23 | default_security_group_egress = [] 24 | 25 | enable_flow_log = true 26 | create_flow_log_cloudwatch_log_group = true 27 | create_flow_log_cloudwatch_iam_role = true 28 | flow_log_max_aggregation_interval = 60 29 | 30 | 31 | tags = { 32 | managed-by = "terraform" 33 | service = "prefect" 34 | } 35 | } 36 | 37 | module "vpc_endpoints" { 38 | count = var.deploy_network ? 1 : 0 39 | source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" 40 | version = "3.11.4" 41 | 42 | vpc_id = var.deploy_network ? module.vpc[0].vpc_id : null 43 | subnet_ids = var.deploy_network ? module.vpc[0].private_subnets : null 44 | 45 | security_group_ids = [aws_security_group.endpoints[0].id] 46 | 47 | endpoints = { 48 | ecr = { 49 | service_name = "com.amazonaws.${data.aws_region.current.name}.ecr.dkr" 50 | tags = { Name = "ecr-vpc-endpoint" } 51 | private_dns_enabled = true 52 | } 53 | s3 = { 54 | service = "s3" 55 | service_type = "Gateway" 56 | route_table_ids = var.deploy_network ? module.vpc[0].private_route_table_ids : null 57 | tags = { Name = "s3-vpc-endpoint" } 58 | } 59 | } 60 | } 61 | 62 | resource "aws_security_group" "endpoints" { 63 | count = var.deploy_network ? 1 : 0 64 | name_prefix = "vpc-endpoints" 65 | description = "Allow HTTPS traffic to/from vpc endpoints within the vpc" 66 | vpc_id = module.vpc[0].vpc_id 67 | 68 | tags = { 69 | Name = "vpc-endpoints" 70 | managed-by = "terraform" 71 | } 72 | } 73 | 74 | resource "aws_security_group_rule" "vpce_egress" { 75 | count = var.deploy_network ? 1 : 0 76 | description = "allow egress https traffic to the local vpc" 77 | type = "egress" 78 | from_port = 443 79 | to_port = 443 80 | protocol = "TCP" 81 | cidr_blocks = [var.vpc_cidr] 82 | security_group_id = var.deploy_network ? aws_security_group.endpoints[0].id : null 83 | } 84 | 85 | resource "aws_security_group_rule" "vpce_ingress" { 86 | count = var.deploy_network ? 1 : 0 87 | description = "allow ingress https traffic to the local vpc" 88 | type = "ingress" 89 | from_port = 443 90 | to_port = 443 91 | protocol = "TCP" 92 | cidr_blocks = [var.vpc_cidr] 93 | security_group_id = var.deploy_network ? aws_security_group.endpoints[0].id : null 94 | } 95 | --------------------------------------------------------------------------------