├── .gitignore ├── LICENSE ├── README.md ├── reference_images ├── full_arch_multi_workspace_mono_repo.png ├── multi_workspace_mono_repo.png └── simple_multi_workspace_mono_repo.png ├── repo ├── common_infrastructure │ ├── logging │ │ ├── logging.tf │ │ ├── provider.tf │ │ └── variables.tf │ └── unity_catalog │ │ ├── metastore_creation.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf ├── common_modules_account │ ├── identity_assignment │ │ ├── identity assignment.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── metastore_assignment │ │ ├── metastore_assignment.tf │ │ ├── provider.tf │ │ └── variables.tf │ └── workspace_creation │ │ ├── outputs.tf │ │ ├── provider.tf │ │ ├── variables.tf │ │ └── workspace_creation.tf ├── common_modules_cloud_provider │ ├── cloud_provider_credential │ │ ├── cloud_provider_credential.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── cloud_provider_network │ │ ├── cloud_provider_network.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ └── cloud_provider_storage │ │ ├── cloud_provider_storage.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf ├── common_modules_workspace │ ├── cluster_policy │ │ ├── cluster_policy.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── uc_workspace_catalog │ │ ├── provider.tf │ │ ├── uc_workspace_catalog.tf │ │ └── variables.tf │ └── workspace_confs │ │ ├── provider.tf │ │ └── workspace_confs.tf ├── databricks_dev │ ├── cloud_provider │ │ ├── cloud_provider.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── databricks_account │ │ ├── databricks_account.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── databricks_workspace │ │ ├── databricks_workspace.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── dev.tf │ ├── provider.tf │ └── variables.tf ├── databricks_prod │ ├── cloud_provider │ │ ├── cloud_provider.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── databricks_account │ │ ├── databricks_account.tf │ │ ├── outputs.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── databricks_workspace │ │ ├── databricks_workspace.tf │ │ ├── provider.tf │ │ └── variables.tf │ ├── prod.tf │ ├── provider.tf │ └── variables.tf └── databricks_qa │ ├── cloud_provider │ ├── cloud_provider.tf │ ├── outputs.tf │ ├── provider.tf │ └── variables.tf │ ├── databricks_account │ ├── databricks_account.tf │ ├── outputs.tf │ ├── provider.tf │ └── variables.tf │ ├── databricks_workspace │ ├── databricks_workspace.tf │ ├── provider.tf │ └── variables.tf │ ├── provider.tf │ ├── qa.tf │ └── variables.tf └── tfvars_examples ├── logging.md ├── unity_catalog.md └── workspace_env.md /.gitignore: -------------------------------------------------------------------------------- 1 | # Local .terraform directories 2 | */.terraform/* 3 | */.terraform 4 | .terraform.lock.hcl 5 | .terraform/* 6 | **/terraform.tfstate 7 | **/terraform.tfstate.* 8 | **/.terraform/* 9 | */.terraform/* 10 | 11 | # .tfstate files 12 | *.tfstate 13 | *.tfstate.* 14 | */*/terraform.tfstate 15 | 16 | # env files 17 | *.tfvars 18 | 19 | # state files 20 | *.tfstate -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [2023] [JD Braun] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ⚠️ This repository is unmaintained and should be used for reference purposes only. For up-to-date Terraform code tailored for security-forward organizations, see the [Security Reference Architecture – Terraform Templates repository](https://github.com/databricks/terraform-databricks-sra). 2 | 3 | ## Databricks Multiple Workspace Repository Example 4 | 5 | ### Folder Structure - Modules: 6 | - **Common Modules: Cloud Provider**: Reusable modules for underlying cloud resources related to a workspace 7 | - Cloud Provider Credential: Module to create the underlying cross-account role 8 | - Cloud Provider Network: Module to create the underlying network (VPC, subnet, security groups, etc.) 9 | - Cloud Provider Storage: Module to create the underlying workspace root bucket 10 |   11 | 12 | - **Common Modules: Account**: Reusable modules for account-level resources 13 | - Metastore Assignment: Module to assign the calling workspace to the metastore 14 | - Workspace Creation: Module to create the workspace based on the outputs of the previous modules 15 | - Identity assignment: Module to assign users and groups to the workspace 16 |   17 | 18 | - **Common Modules: Workspace**: Reusable modules for workspace-level resources 19 | - Workspace Catalog: Module to create a catalog, and underlying cloud resources, isolated to the individual workspace 20 | - Cluster Policy: Module to create a parameterized cluster policy per environment 21 | - Workspace Confs: Module to create consistent workspace configurations across workspace 22 |   23 | 24 | ___ 25 | 26 | ### Folder Structure - Individual Pipelines: 27 | - **Common Infrastructure/Unity Catalog**: 28 | - Creation of the Unity Catalog metastore with no root storage, isolating it from other environments 29 |   30 | 31 | - **Common Infrastructure/Logging**: 32 | - Creation of logging resources and underlying cloud resources 33 |   34 | 35 | - **Environments**: 36 | - Development: Creation of cloud and Databricks Resources for a development environment 37 | - QA: Creation of cloud and Databricks Resources for a QA environment 38 | - Production: Creation of cloud and Databricks Resources for a production environment 39 |   40 | 41 | ___ 42 | 43 | ### How to set-up: 44 | - Create a .tfvars file based on the examples found in the tfvars_example folder 45 | - **Recommended**: Set environment variables for your AWS and Databricks credentials 46 | - Perform the following steps in: **common_infrastructure/unity_catalog**, **common_infrastructure/logging**, **databricks_dev**, **databricks_qa**, and **databricks_qa** 47 | - *Add the required .tfvars file* 48 | - Terraform init 49 | - Terraform plan 50 | - Terraform apply 51 | 52 | **Note**: Please raise a git issues with any problems or concerns about the repo 53 | 54 | ___ 55 | 56 | ### Architecture Diagrams: 57 | - [Full Multi-Workspace Architecture](https://github.com/JDBraun/dbx_mws_example/blob/main/reference_images/full_arch_multi_workspace_mono_repo.png) 58 | - [Multi-Workspace Architecture](https://github.com/JDBraun/dbx_mws_example/blob/main/reference_images/multi_workspace_mono_repo.png) 59 | - [Simple Multi-Workspace Architecture](https://github.com/JDBraun/dbx_mws_example/blob/main/reference_images/simple_multi_workspace_mono_repo.png) 60 | 61 | ___ 62 | 63 | ### FAQ: 64 | - **"I get an Error: Please use a valid IAM role. What do I do?"** 65 | - This occurs after the networking configured is finalized. This is due to a race condition between the IAM role and the logging of it to the Databricks endpoint. Please re-plan and apply and it will go through. It can be mitigated with a sleep condition. 66 | 67 | - **"What do I do with identities?"**. 68 | - Identities should be integrated with SCIM. Once they are integrated with SCIM, reference them as data sources, similar to the identity assignment example. Then continue to assign permissions through the workspace provider. 69 | -------------------------------------------------------------------------------- /reference_images/full_arch_multi_workspace_mono_repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JDBraun/dbx_mws_example/119f65b11bc345f19ccf797ba207bbe030a322df/reference_images/full_arch_multi_workspace_mono_repo.png -------------------------------------------------------------------------------- /reference_images/multi_workspace_mono_repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JDBraun/dbx_mws_example/119f65b11bc345f19ccf797ba207bbe030a322df/reference_images/multi_workspace_mono_repo.png -------------------------------------------------------------------------------- /reference_images/simple_multi_workspace_mono_repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/JDBraun/dbx_mws_example/119f65b11bc345f19ccf797ba207bbe030a322df/reference_images/simple_multi_workspace_mono_repo.png -------------------------------------------------------------------------------- /repo/common_infrastructure/logging/logging.tf: -------------------------------------------------------------------------------- 1 | // Terraform Documentation: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mws_log_delivery 2 | 3 | // S3 Log Bucket 4 | resource "aws_s3_bucket" "log_delivery" { 5 | bucket = "${var.resource_prefix}-log-delivery" 6 | force_destroy = true 7 | tags = { 8 | Name = "${var.resource_prefix}-log-delivery" 9 | } 10 | } 11 | 12 | // S3 Bucket Versioning 13 | resource "aws_s3_bucket_versioning" "log_delivery" { 14 | bucket = aws_s3_bucket.log_delivery.id 15 | versioning_configuration { 16 | status = "Disabled" 17 | } 18 | } 19 | 20 | // S3 Public Access Block 21 | resource "aws_s3_bucket_public_access_block" "log_delivery" { 22 | bucket = aws_s3_bucket.log_delivery.id 23 | block_public_acls = true 24 | block_public_policy = true 25 | ignore_public_acls = true 26 | restrict_public_buckets = true 27 | depends_on = [aws_s3_bucket.log_delivery] 28 | } 29 | 30 | // S3 Policy for Log Delivery Data 31 | data "databricks_aws_bucket_policy" "log_delivery" { 32 | full_access_role = aws_iam_role.log_delivery.arn 33 | bucket = aws_s3_bucket.log_delivery.bucket 34 | } 35 | 36 | // S3 Policy for Log Delivery Resources 37 | resource "aws_s3_bucket_policy" "log_delivery" { 38 | bucket = aws_s3_bucket.log_delivery.id 39 | policy = jsonencode({ 40 | "Version" : "2012-10-17", 41 | "Statement" : [ 42 | { 43 | "Effect" : "Allow", 44 | "Principal" : { 45 | "AWS" : ["${aws_iam_role.log_delivery.arn}"] 46 | }, 47 | "Action" : "s3:GetBucketLocation", 48 | "Resource" : "arn:aws:s3:::${var.resource_prefix}-log-delivery" 49 | }, 50 | { 51 | "Effect" : "Allow", 52 | "Principal" : { 53 | "AWS" : ["${aws_iam_role.log_delivery.arn}"] 54 | }, 55 | "Action" : [ 56 | "s3:PutObject", 57 | "s3:GetObject", 58 | "s3:DeleteObject", 59 | "s3:PutObjectAcl", 60 | "s3:AbortMultipartUpload", 61 | "s3:ListMultipartUploadParts" 62 | ], 63 | "Resource" : [ 64 | "arn:aws:s3:::${var.resource_prefix}-log-delivery", 65 | "arn:aws:s3:::${var.resource_prefix}-log-delivery/*" 66 | ] 67 | }, 68 | { 69 | "Effect" : "Allow", 70 | "Principal" : { 71 | "AWS" : ["${aws_iam_role.log_delivery.arn}"] 72 | }, 73 | "Action" : "s3:ListBucket", 74 | "Resource" : "arn:aws:s3:::${var.resource_prefix}-log-delivery" 75 | } 76 | ] 77 | } 78 | ) 79 | depends_on = [ 80 | aws_s3_bucket.log_delivery 81 | ] 82 | } 83 | 84 | // IAM Role 85 | 86 | // Assume Role Policy Log Delivery 87 | data "databricks_aws_assume_role_policy" "log_delivery" { 88 | external_id = var.databricks_account_id 89 | for_log_delivery = true 90 | } 91 | 92 | 93 | // Log Delivery IAM Role 94 | resource "aws_iam_role" "log_delivery" { 95 | name = "${var.resource_prefix}-log-delivery" 96 | description = "(${var.resource_prefix}) Log Delivery Role" 97 | assume_role_policy = data.databricks_aws_assume_role_policy.log_delivery.json 98 | tags = { 99 | Name = "${var.resource_prefix}-log-delivery-role" 100 | } 101 | } 102 | 103 | // Databricks Configurations 104 | 105 | // Databricks Credential Configuration for Logs 106 | resource "databricks_mws_credentials" "log_writer" { 107 | account_id = var.databricks_account_id 108 | credentials_name = "${var.resource_prefix}-log-delivery-credential" 109 | role_arn = aws_iam_role.log_delivery.arn 110 | depends_on = [ 111 | aws_s3_bucket_policy.log_delivery 112 | ] 113 | } 114 | 115 | // Databricks Storage Configuration for Logs 116 | resource "databricks_mws_storage_configurations" "log_bucket" { 117 | account_id = var.databricks_account_id 118 | storage_configuration_name = "${var.resource_prefix}-log-delivery-bucket" 119 | bucket_name = aws_s3_bucket.log_delivery.bucket 120 | depends_on = [ 121 | aws_s3_bucket_policy.log_delivery 122 | ] 123 | } 124 | 125 | // Databricks Billable Usage Logs Configurations 126 | resource "databricks_mws_log_delivery" "billable_usage_logs" { 127 | account_id = var.databricks_account_id 128 | credentials_id = databricks_mws_credentials.log_writer.credentials_id 129 | storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id 130 | delivery_path_prefix = "billable-usage-logs" 131 | config_name = "Billable Usage Logs" 132 | log_type = "BILLABLE_USAGE" 133 | output_format = "CSV" 134 | depends_on = [ 135 | aws_s3_bucket_policy.log_delivery 136 | ] 137 | } 138 | 139 | // Databricks Audit Logs Configurations 140 | resource "databricks_mws_log_delivery" "audit_logs" { 141 | account_id = var.databricks_account_id 142 | credentials_id = databricks_mws_credentials.log_writer.credentials_id 143 | storage_configuration_id = databricks_mws_storage_configurations.log_bucket.storage_configuration_id 144 | delivery_path_prefix = "audit-logs" 145 | config_name = "Audit Logs" 146 | log_type = "AUDIT_LOGS" 147 | output_format = "JSON" 148 | depends_on = [ 149 | aws_s3_bucket_policy.log_delivery 150 | ] 151 | } 152 | -------------------------------------------------------------------------------- /repo/common_infrastructure/logging/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | version = "~> 1.35.0" 6 | } 7 | aws = { 8 | source = "hashicorp/aws" 9 | } 10 | } 11 | } 12 | 13 | provider "aws" { 14 | region = var.region 15 | default_tags { 16 | tags = { 17 | Owner = var.resource_owner 18 | Resource = var.resource_prefix 19 | } 20 | } 21 | } 22 | 23 | provider "databricks" { 24 | host = "https://accounts.cloud.databricks.com" 25 | account_id = var.databricks_account_id 26 | client_id = var.client_id 27 | client_secret = var.client_secret 28 | } -------------------------------------------------------------------------------- /repo/common_infrastructure/logging/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | } 4 | 5 | variable "databricks_account_id" { 6 | type = string 7 | } 8 | 9 | variable "client_id" { 10 | type = string 11 | } 12 | 13 | variable "client_secret" { 14 | type = string 15 | } 16 | 17 | variable "resource_owner" { 18 | type = string 19 | } 20 | 21 | variable "resource_prefix" { 22 | type = string 23 | } 24 | -------------------------------------------------------------------------------- /repo/common_infrastructure/unity_catalog/metastore_creation.tf: -------------------------------------------------------------------------------- 1 | // Metastore 2 | resource "databricks_metastore" "this" { 3 | name = "unity-catalog-${var.resource_prefix}" 4 | region = var.region 5 | force_destroy = true 6 | } -------------------------------------------------------------------------------- /repo/common_infrastructure/unity_catalog/outputs.tf: -------------------------------------------------------------------------------- 1 | output "metastore_id" { 2 | value = databricks_metastore.this.id 3 | } -------------------------------------------------------------------------------- /repo/common_infrastructure/unity_catalog/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | version = "~> 1.35.0" 6 | } 7 | aws = { 8 | source = "hashicorp/aws" 9 | } 10 | } 11 | } 12 | 13 | provider "aws" { 14 | region = var.region 15 | default_tags { 16 | tags = { 17 | Owner = var.resource_owner 18 | Resource = var.resource_prefix 19 | } 20 | } 21 | } 22 | 23 | provider "databricks" { 24 | host = "https://accounts.cloud.databricks.com" 25 | account_id = var.databricks_account_id 26 | client_id = var.client_id 27 | client_secret = var.client_secret 28 | } -------------------------------------------------------------------------------- /repo/common_infrastructure/unity_catalog/variables.tf: -------------------------------------------------------------------------------- 1 | variable "region" { 2 | type = string 3 | } 4 | 5 | variable "resource_owner" { 6 | type = string 7 | } 8 | 9 | variable "resource_prefix" { 10 | type = string 11 | } 12 | 13 | variable "aws_account_id" { 14 | type = string 15 | } 16 | 17 | variable "databricks_account_id" { 18 | type = string 19 | } 20 | 21 | variable "client_id" { 22 | type = string 23 | } 24 | 25 | variable "client_secret" { 26 | type = string 27 | } 28 | 29 | 30 | variable "ucname" { 31 | type = string 32 | } -------------------------------------------------------------------------------- /repo/common_modules_account/identity_assignment/identity assignment.tf: -------------------------------------------------------------------------------- 1 | data "databricks_user" "user" { 2 | user_name = var.user_name 3 | } 4 | 5 | resource "databricks_mws_permission_assignment" "user" { 6 | workspace_id = var.workspace_id 7 | principal_id = data.databricks_user.user.id 8 | permissions = ["ADMIN"] 9 | } -------------------------------------------------------------------------------- /repo/common_modules_account/identity_assignment/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/common_modules_account/identity_assignment/variables.tf: -------------------------------------------------------------------------------- 1 | variable "user_name" { 2 | type = string 3 | } 4 | 5 | variable "workspace_id" { 6 | type = string 7 | } -------------------------------------------------------------------------------- /repo/common_modules_account/metastore_assignment/metastore_assignment.tf: -------------------------------------------------------------------------------- 1 | resource "databricks_metastore_assignment" "this" { 2 | metastore_id = var.metastore_id 3 | workspace_id = var.workspace_id 4 | } -------------------------------------------------------------------------------- /repo/common_modules_account/metastore_assignment/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/common_modules_account/metastore_assignment/variables.tf: -------------------------------------------------------------------------------- 1 | variable "metastore_id" { 2 | type = string 3 | } 4 | 5 | variable "workspace_id" { 6 | type = string 7 | } -------------------------------------------------------------------------------- /repo/common_modules_account/workspace_creation/outputs.tf: -------------------------------------------------------------------------------- 1 | output "workspace_url" { 2 | value = databricks_mws_workspaces.this.workspace_url 3 | } 4 | 5 | output "workspace_id" { 6 | value = databricks_mws_workspaces.this.workspace_id 7 | } -------------------------------------------------------------------------------- /repo/common_modules_account/workspace_creation/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/common_modules_account/workspace_creation/variables.tf: -------------------------------------------------------------------------------- 1 | variable "databricks_account_id" { 2 | type = string 3 | } 4 | 5 | variable "region" { 6 | type = string 7 | } 8 | 9 | variable "cross_account_role_arn" { 10 | type = string 11 | } 12 | 13 | variable "resource_prefix" { 14 | type = string 15 | } 16 | 17 | variable "bucket_name" { 18 | type = string 19 | } 20 | 21 | variable "security_group_ids" { 22 | type = list(any) 23 | } 24 | 25 | variable "subnet_ids" { 26 | type = list(any) 27 | } 28 | 29 | variable "vpc_id" { 30 | type = string 31 | } -------------------------------------------------------------------------------- /repo/common_modules_account/workspace_creation/workspace_creation.tf: -------------------------------------------------------------------------------- 1 | // Terraform Documentation: https://registry.terraform.io/providers/databricks/databricks/latest/docs/resources/mws_workspaces 2 | 3 | 4 | // Wait on Credential Due to Race Condition 5 | // https://kb.databricks.com/en_US/terraform/failed-credential-validation-checks-error-with-terraform 6 | resource "null_resource" "previous" {} 7 | 8 | resource "time_sleep" "wait_30_seconds" { 9 | depends_on = [null_resource.previous] 10 | 11 | create_duration = "30s" 12 | } 13 | 14 | // Credential Configuration 15 | resource "databricks_mws_credentials" "this" { 16 | role_arn = var.cross_account_role_arn 17 | credentials_name = "${var.resource_prefix}-credentials" 18 | depends_on = [time_sleep.wait_30_seconds] 19 | } 20 | 21 | // Storage Configuration 22 | resource "databricks_mws_storage_configurations" "this" { 23 | account_id = var.databricks_account_id 24 | bucket_name = var.bucket_name 25 | storage_configuration_name = "${var.resource_prefix}-storage" 26 | } 27 | 28 | // Network Configuration 29 | resource "databricks_mws_networks" "this" { 30 | account_id = var.databricks_account_id 31 | network_name = "${var.resource_prefix}-network" 32 | security_group_ids = var.security_group_ids 33 | subnet_ids = var.subnet_ids 34 | vpc_id = var.vpc_id 35 | } 36 | 37 | // Workspace Configuration 38 | resource "databricks_mws_workspaces" "this" { 39 | account_id = var.databricks_account_id 40 | aws_region = var.region 41 | workspace_name = var.resource_prefix 42 | credentials_id = databricks_mws_credentials.this.credentials_id 43 | storage_configuration_id = databricks_mws_storage_configurations.this.storage_configuration_id 44 | network_id = databricks_mws_networks.this.network_id 45 | pricing_tier = "ENTERPRISE" 46 | depends_on = [databricks_mws_networks.this] 47 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_credential/cloud_provider_credential.tf: -------------------------------------------------------------------------------- 1 | // EXPLANATION: The cross-account role for the Databricks workspace 2 | 3 | // Cross Account Role 4 | data "databricks_aws_assume_role_policy" "this" { 5 | external_id = var.databricks_account_id 6 | } 7 | 8 | resource "aws_iam_role" "cross_account_role" { 9 | name = "${var.resource_prefix}-crossaccount" 10 | assume_role_policy = data.databricks_aws_assume_role_policy.this.json 11 | tags = { 12 | Name = "${var.resource_prefix}-crossaccount-role" 13 | } 14 | } 15 | 16 | resource "aws_iam_role_policy" "cross_account" { 17 | name = "${var.resource_prefix}-crossaccount-policy" 18 | role = aws_iam_role.cross_account_role.id 19 | policy = jsonencode({ 20 | "Version" : "2012-10-17", 21 | "Statement" : [ 22 | { 23 | "Sid" : "NonResourceBasedPermissions", 24 | "Effect" : "Allow", 25 | "Action" : [ 26 | "ec2:CancelSpotInstanceRequests", 27 | "ec2:DescribeAvailabilityZones", 28 | "ec2:DescribeIamInstanceProfileAssociations", 29 | "ec2:DescribeInstanceStatus", 30 | "ec2:DescribeInstances", 31 | "ec2:DescribeInternetGateways", 32 | "ec2:DescribeNatGateways", 33 | "ec2:DescribeNetworkAcls", 34 | "ec2:DescribePrefixLists", 35 | "ec2:DescribeReservedInstancesOfferings", 36 | "ec2:DescribeRouteTables", 37 | "ec2:DescribeSecurityGroups", 38 | "ec2:DescribeSpotInstanceRequests", 39 | "ec2:DescribeSpotPriceHistory", 40 | "ec2:DescribeSubnets", 41 | "ec2:DescribeVolumes", 42 | "ec2:DescribeVpcAttribute", 43 | "ec2:DescribeVpcs", 44 | "ec2:CreateTags", 45 | "ec2:DeleteTags", 46 | "ec2:RequestSpotInstances" 47 | ], 48 | "Resource" : [ 49 | "*" 50 | ] 51 | }, 52 | { 53 | "Sid" : "InstancePoolsSupport", 54 | "Effect" : "Allow", 55 | "Action" : [ 56 | "ec2:AssociateIamInstanceProfile", 57 | "ec2:DisassociateIamInstanceProfile", 58 | "ec2:ReplaceIamInstanceProfileAssociation" 59 | ], 60 | "Resource" : "arn:aws:ec2:${var.region}:${var.aws_account_id}:instance/*", 61 | "Condition" : { 62 | "StringEquals" : { 63 | "ec2:ResourceTag/Vendor" : "Databricks" 64 | } 65 | } 66 | }, 67 | { 68 | "Sid" : "AllowEc2RunInstancePerTag", 69 | "Effect" : "Allow", 70 | "Action" : "ec2:RunInstances", 71 | "Resource" : [ 72 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:volume/*", 73 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:instance/*" 74 | ], 75 | "Condition" : { 76 | "StringEquals" : { 77 | "aws:RequestTag/Vendor" : "Databricks" 78 | } 79 | } 80 | }, 81 | { 82 | "Sid" : "AllowEc2RunInstanceImagePerTag", 83 | "Effect" : "Allow", 84 | "Action" : "ec2:RunInstances", 85 | "Resource" : [ 86 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:image/*" 87 | ], 88 | "Condition" : { 89 | "StringEquals" : { 90 | "aws:ResourceTag/Vendor" : "Databricks" 91 | } 92 | } 93 | }, 94 | { 95 | "Sid" : "AllowEc2RunInstancePerVPCid", 96 | "Effect" : "Allow", 97 | "Action" : "ec2:RunInstances", 98 | "Resource" : [ 99 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:network-interface/*", 100 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:subnet/*", 101 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:security-group/*" 102 | ], 103 | "Condition" : { 104 | "StringEquals" : { 105 | "ec2:vpc" : "arn:aws:ec2:${var.region}:${var.aws_account_id}:vpc/${var.vpc_id}" 106 | } 107 | } 108 | }, 109 | { 110 | "Sid" : "AllowEc2RunInstanceOtherResources", 111 | "Effect" : "Allow", 112 | "Action" : "ec2:RunInstances", 113 | "NotResource" : [ 114 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:image/*", 115 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:network-interface/*", 116 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:subnet/*", 117 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:security-group/*", 118 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:volume/*", 119 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:instance/*" 120 | ] 121 | }, 122 | { 123 | "Sid" : "EC2TerminateInstancesTag", 124 | "Effect" : "Allow", 125 | "Action" : [ 126 | "ec2:TerminateInstances" 127 | ], 128 | "Resource" : [ 129 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:instance/*" 130 | ], 131 | "Condition" : { 132 | "StringEquals" : { 133 | "ec2:ResourceTag/Vendor" : "Databricks" 134 | } 135 | } 136 | }, 137 | { 138 | "Sid" : "EC2AttachDetachVolumeTag", 139 | "Effect" : "Allow", 140 | "Action" : [ 141 | "ec2:AttachVolume", 142 | "ec2:DetachVolume" 143 | ], 144 | "Resource" : [ 145 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:instance/*", 146 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:volume/*" 147 | ], 148 | "Condition" : { 149 | "StringEquals" : { 150 | "ec2:ResourceTag/Vendor" : "Databricks" 151 | } 152 | } 153 | }, 154 | { 155 | "Sid" : "EC2CreateVolumeByTag", 156 | "Effect" : "Allow", 157 | "Action" : [ 158 | "ec2:CreateVolume" 159 | ], 160 | "Resource" : [ 161 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:volume/*" 162 | ], 163 | "Condition" : { 164 | "StringEquals" : { 165 | "aws:RequestTag/Vendor" : "Databricks" 166 | } 167 | } 168 | }, 169 | { 170 | "Sid" : "EC2DeleteVolumeByTag", 171 | "Effect" : "Allow", 172 | "Action" : [ 173 | "ec2:DeleteVolume" 174 | ], 175 | "Resource" : [ 176 | "arn:aws:ec2:${var.region}:${var.aws_account_id}:volume/*" 177 | ], 178 | "Condition" : { 179 | "StringEquals" : { 180 | "ec2:ResourceTag/Vendor" : "Databricks" 181 | } 182 | } 183 | }, 184 | { 185 | "Effect" : "Allow", 186 | "Action" : [ 187 | "iam:CreateServiceLinkedRole", 188 | "iam:PutRolePolicy" 189 | ], 190 | "Resource" : "arn:aws:iam::*:role/aws-service-role/spot.amazonaws.com/AWSServiceRoleForEC2Spot", 191 | "Condition" : { 192 | "StringLike" : { 193 | "iam:AWSServiceName" : "spot.amazonaws.com" 194 | } 195 | } 196 | }, 197 | { 198 | "Sid" : "VpcNonresourceSpecificActions", 199 | "Effect" : "Allow", 200 | "Action" : [ 201 | "ec2:AuthorizeSecurityGroupEgress", 202 | "ec2:AuthorizeSecurityGroupIngress", 203 | "ec2:RevokeSecurityGroupEgress", 204 | "ec2:RevokeSecurityGroupIngress" 205 | ], 206 | "Resource" : "arn:aws:ec2:${var.region}:${var.aws_account_id}:security-group/${var.security_group_ids[0]}", 207 | "Condition" : { 208 | "StringEquals" : { 209 | "ec2:vpc" : "arn:aws:ec2:${var.region}:${var.aws_account_id}:vpc/${var.vpc_id}" 210 | } 211 | } 212 | }, 213 | { 214 | "Effect" : "Allow", 215 | "Action" : "iam:PassRole", 216 | "Resource" : "arn:aws:iam::${var.aws_account_id}:role/${var.resource_prefix}-s3-instance-profile" 217 | } 218 | ] 219 | } 220 | ) 221 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_credential/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_provider_credential" { 2 | value = aws_iam_role.cross_account_role.arn 3 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_credential/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | aws = { 7 | source = "hashicorp/aws" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_credential/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "databricks_account_id" { 6 | type = string 7 | } 8 | 9 | variable "resource_prefix" { 10 | type = string 11 | } 12 | 13 | variable "region" { 14 | type = string 15 | } 16 | 17 | variable "vpc_id" { 18 | type = string 19 | } 20 | 21 | variable "security_group_ids" { 22 | type = list(string) 23 | } 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_network/cloud_provider_network.tf: -------------------------------------------------------------------------------- 1 | // EXPLANATION: Create the customer managed-vpc and security group rules 2 | 3 | module "vpc" { 4 | source = "terraform-aws-modules/vpc/aws" 5 | version = "5.1.1" 6 | 7 | name = "${var.resource_prefix}-data-plane-VPC" 8 | cidr = var.vpc_cidr_range 9 | azs = var.availability_zones 10 | 11 | enable_dns_hostnames = true 12 | enable_nat_gateway = true 13 | single_nat_gateway = false 14 | one_nat_gateway_per_az = true 15 | create_igw = true 16 | 17 | public_subnet_names = [for az in var.availability_zones : format("%s-public-%s", var.resource_prefix, az)] 18 | public_subnets = var.public_subnets_cidr 19 | 20 | private_subnet_names = [for az in var.availability_zones : format("%s-private-%s", var.resource_prefix, az)] 21 | private_subnets = var.private_subnets_cidr 22 | } 23 | 24 | // SG 25 | resource "aws_security_group" "sg" { 26 | vpc_id = module.vpc.vpc_id 27 | depends_on = [module.vpc] 28 | 29 | dynamic "ingress" { 30 | for_each = var.sg_ingress_protocol 31 | content { 32 | from_port = 0 33 | to_port = 65535 34 | protocol = ingress.value 35 | self = true 36 | } 37 | } 38 | 39 | dynamic "egress" { 40 | for_each = var.sg_egress_protocol 41 | content { 42 | from_port = 0 43 | to_port = 65535 44 | protocol = egress.value 45 | self = true 46 | } 47 | } 48 | 49 | dynamic "egress" { 50 | for_each = var.sg_egress_ports 51 | content { 52 | from_port = egress.value 53 | to_port = egress.value 54 | protocol = "tcp" 55 | cidr_blocks = ["0.0.0.0/0"] 56 | } 57 | } 58 | tags = { 59 | Name = "${var.resource_prefix}-data-plane-sg" 60 | } 61 | } 62 | 63 | module "vpc_endpoints" { 64 | source = "terraform-aws-modules/vpc/aws//modules/vpc-endpoints" 65 | version = "3.11.0" 66 | 67 | vpc_id = module.vpc.vpc_id 68 | security_group_ids = [aws_security_group.sg.id] 69 | 70 | endpoints = { 71 | s3 = { 72 | service = "s3" 73 | service_type = "Gateway" 74 | route_table_ids = module.vpc.private_route_table_ids 75 | tags = { 76 | Name = "${var.resource_prefix}-s3-vpc-endpoint" 77 | } 78 | }, 79 | sts = { 80 | service = "sts" 81 | private_dns_enabled = true 82 | subnet_ids = length(module.vpc.private_subnets) > 0 ? slice(module.vpc.private_subnets, 0, min(2, length(module.vpc.private_subnets))) : [] 83 | tags = { 84 | Name = "${var.resource_prefix}-sts-vpc-endpoint" 85 | } 86 | }, 87 | kinesis-streams = { 88 | service = "kinesis-streams" 89 | private_dns_enabled = true 90 | subnet_ids = length(module.vpc.private_subnets) > 0 ? slice(module.vpc.private_subnets, 0, min(2, length(module.vpc.private_subnets))) : [] 91 | tags = { 92 | Name = "${var.resource_prefix}-kinesis-vpc-endpoint" 93 | } 94 | } 95 | } 96 | depends_on = [ 97 | module.vpc 98 | ] 99 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_network/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_provider_network_vpc" { 2 | value = module.vpc.vpc_id 3 | } 4 | 5 | output "cloud_provider_network_subnets" { 6 | value = module.vpc.private_subnets 7 | } 8 | 9 | output "cloud_provider_network_security_groups" { 10 | value = [aws_security_group.sg.id] 11 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_network/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | aws = { 4 | source = "hashicorp/aws" 5 | } 6 | } 7 | } 8 | -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_network/variables.tf: -------------------------------------------------------------------------------- 1 | variable "vpc_cidr_range" { 2 | type = string 3 | } 4 | 5 | variable "availability_zones" { 6 | type = list(string) 7 | } 8 | 9 | variable "resource_prefix" { 10 | type = string 11 | } 12 | 13 | variable "public_subnets_cidr" { 14 | type = list(string) 15 | } 16 | 17 | variable "private_subnets_cidr" { 18 | type = list(string) 19 | } 20 | 21 | variable "sg_ingress_protocol" { 22 | type = list(any) 23 | } 24 | 25 | variable "sg_egress_ports" { 26 | type = list(any) 27 | } 28 | 29 | variable "sg_egress_protocol" { 30 | type = list(any) 31 | } 32 | 33 | -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_storage/cloud_provider_storage.tf: -------------------------------------------------------------------------------- 1 | // EXPLANATION: Create the workspace root bucket 2 | 3 | resource "aws_s3_bucket" "root_storage_bucket" { 4 | bucket = var.dbfsname 5 | force_destroy = true 6 | tags = { 7 | Name = var.dbfsname 8 | } 9 | } 10 | 11 | resource "aws_s3_bucket_versioning" "root_bucket_versioning" { 12 | bucket = aws_s3_bucket.root_storage_bucket.id 13 | versioning_configuration { 14 | status = "Disabled" 15 | } 16 | } 17 | 18 | resource "aws_s3_bucket_public_access_block" "root_storage_bucket" { 19 | bucket = aws_s3_bucket.root_storage_bucket.id 20 | block_public_acls = true 21 | block_public_policy = true 22 | ignore_public_acls = true 23 | restrict_public_buckets = true 24 | depends_on = [aws_s3_bucket.root_storage_bucket] 25 | } 26 | 27 | data "databricks_aws_bucket_policy" "this" { 28 | databricks_e2_account_id = var.databricks_account_id 29 | bucket = aws_s3_bucket.root_storage_bucket.bucket 30 | } 31 | 32 | resource "aws_s3_bucket_policy" "root_bucket_policy" { 33 | bucket = aws_s3_bucket.root_storage_bucket.id 34 | policy = data.databricks_aws_bucket_policy.this.json 35 | depends_on = [aws_s3_bucket_public_access_block.root_storage_bucket] 36 | } 37 | -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_storage/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_provider_storage" { 2 | value = aws_s3_bucket.root_storage_bucket.id 3 | } 4 | -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_storage/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | aws = { 7 | source = "hashicorp/aws" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /repo/common_modules_cloud_provider/cloud_provider_storage/variables.tf: -------------------------------------------------------------------------------- 1 | variable "dbfsname" { 2 | type = string 3 | } 4 | 5 | variable "databricks_account_id" { 6 | type = string 7 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/cluster_policy/cluster_policy.tf: -------------------------------------------------------------------------------- 1 | locals { 2 | default_policy = { 3 | "dbus_per_hour" : { 4 | "type" : "range", 5 | "maxValue" : 10 6 | }, 7 | "autotermination_minutes" : { 8 | "type" : "fixed", 9 | "value" : 20, 10 | "hidden" : true 11 | }, 12 | "custom_tags.Team" : { 13 | "type" : "fixed", 14 | "value" : var.team 15 | } 16 | } 17 | } 18 | 19 | resource "databricks_cluster_policy" "fair_use" { 20 | name = "${var.team} cluster policy" 21 | definition = jsonencode(merge(local.default_policy, var.policy_overrides)) 22 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/cluster_policy/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/cluster_policy/variables.tf: -------------------------------------------------------------------------------- 1 | variable "team" { 2 | type = string 3 | } 4 | 5 | variable "policy_overrides" { 6 | type = map(object({type = string, value = string})) 7 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/uc_workspace_catalog/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/uc_workspace_catalog/uc_workspace_catalog.tf: -------------------------------------------------------------------------------- 1 | resource "null_resource" "previous" {} 2 | 3 | resource "time_sleep" "wait_30_seconds" { 4 | depends_on = [null_resource.previous] 5 | 6 | create_duration = "30s" 7 | } 8 | 9 | 10 | // Unity Catalog Trust Policy 11 | data "aws_iam_policy_document" "passrole_for_unity_catalog_catalog" { 12 | statement { 13 | effect = "Allow" 14 | actions = ["sts:AssumeRole"] 15 | principals { 16 | identifiers = ["arn:aws:iam::414351767826:role/unity-catalog-prod-UCMasterRole-14S5ZJVKOTYTL"] 17 | type = "AWS" 18 | } 19 | condition { 20 | test = "StringEquals" 21 | variable = "sts:ExternalId" 22 | values = [var.databricks_account_id] 23 | } 24 | } 25 | statement { 26 | sid = "ExplicitSelfRoleAssumption" 27 | effect = "Allow" 28 | actions = ["sts:AssumeRole"] 29 | principals { 30 | type = "AWS" 31 | identifiers = ["arn:aws:iam::${var.aws_account_id}:root"] 32 | } 33 | condition { 34 | test = "ArnLike" 35 | variable = "aws:PrincipalArn" 36 | values = ["arn:aws:iam::${var.aws_account_id}:role/${var.resource_prefix}-unity-catalog-${var.workspace_id}"] 37 | } 38 | condition { 39 | test = "StringEquals" 40 | variable = "sts:ExternalId" 41 | values = [var.databricks_account_id] 42 | } 43 | } 44 | } 45 | 46 | // Unity Catalog Role 47 | resource "aws_iam_role" "unity_catalog_role" { 48 | name = "${var.resource_prefix}-unity-catalog-${var.workspace_id}" 49 | assume_role_policy = data.aws_iam_policy_document.passrole_for_unity_catalog_catalog.json 50 | tags = { 51 | Name = "${var.resource_prefix}-unity-catalog" 52 | } 53 | } 54 | 55 | // Unity Catalog IAM Policy 56 | data "aws_iam_policy_document" "unity_catalog_iam_policy" { 57 | statement { 58 | actions = [ 59 | "s3:GetObject", 60 | "s3:GetObjectVersion", 61 | "s3:PutObject", 62 | "s3:PutObjectAcl", 63 | "s3:DeleteObject", 64 | "s3:ListBucket", 65 | "s3:GetBucketLocation" 66 | ] 67 | 68 | resources = [ 69 | "arn:aws:s3:::${var.uc_catalog_name}/*", 70 | "arn:aws:s3:::${var.uc_catalog_name}" 71 | ] 72 | 73 | effect = "Allow" 74 | } 75 | 76 | statement { 77 | actions = ["sts:AssumeRole"] 78 | resources = ["arn:aws:iam::${var.aws_account_id}:role/${var.resource_prefix}-unity-catalog-${var.workspace_id}"] 79 | effect = "Allow" 80 | } 81 | } 82 | 83 | // Unity Catalog Policy 84 | resource "aws_iam_role_policy" "unity_catalog" { 85 | name = "${var.resource_prefix}-unity-catalog-policy-${var.workspace_id}" 86 | role = aws_iam_role.unity_catalog_role.id 87 | policy = data.aws_iam_policy_document.unity_catalog_iam_policy.json 88 | } 89 | 90 | 91 | // Unity Catalog S3 92 | resource "aws_s3_bucket" "unity_catalog_bucket" { 93 | bucket = var.uc_catalog_name 94 | force_destroy = true 95 | tags = { 96 | Name = var.uc_catalog_name 97 | } 98 | } 99 | 100 | resource "aws_s3_bucket_versioning" "unity_catalog_versioning" { 101 | bucket = aws_s3_bucket.unity_catalog_bucket.id 102 | versioning_configuration { 103 | status = "Disabled" 104 | } 105 | } 106 | 107 | resource "aws_s3_bucket_server_side_encryption_configuration" "unity_catalog" { 108 | bucket = aws_s3_bucket.unity_catalog_bucket.bucket 109 | 110 | rule { 111 | apply_server_side_encryption_by_default { 112 | sse_algorithm = "AES256" 113 | } 114 | } 115 | } 116 | 117 | resource "aws_s3_bucket_public_access_block" "unity_catalog" { 118 | bucket = aws_s3_bucket.unity_catalog_bucket.id 119 | block_public_acls = true 120 | block_public_policy = true 121 | ignore_public_acls = true 122 | restrict_public_buckets = true 123 | depends_on = [aws_s3_bucket.unity_catalog_bucket] 124 | } 125 | 126 | // Storage Credential 127 | resource "databricks_storage_credential" "workspace_catalog_storage_credential" { 128 | name = aws_iam_role.unity_catalog_role.name 129 | aws_iam_role { 130 | role_arn = aws_iam_role.unity_catalog_role.arn 131 | } 132 | depends_on = [aws_iam_role.unity_catalog_role, time_sleep.wait_30_seconds] 133 | } 134 | 135 | // External Location 136 | resource "databricks_external_location" "workspace_catalog_external_location" { 137 | name = var.uc_catalog_name 138 | url = "s3://${var.uc_catalog_name}/catalog/" 139 | credential_name = databricks_storage_credential.workspace_catalog_storage_credential.id 140 | skip_validation = true 141 | read_only = false 142 | comment = "Managed by TF" 143 | } 144 | 145 | 146 | // Workspace Catalog 147 | resource "databricks_catalog" "workspace_catalog" { 148 | name = var.uc_catalog_name 149 | comment = "This catalog is for workspace - ${var.workspace_id}" 150 | isolation_mode = "ISOLATED" 151 | storage_root = "s3://${var.uc_catalog_name}/catalog/" 152 | properties = { 153 | purpose = "Catalog for workspace - ${var.workspace_id}" 154 | } 155 | depends_on = [databricks_external_location.workspace_catalog_external_location] 156 | } 157 | 158 | // Grant Admin Catalog Perms 159 | resource "databricks_grant" "workspace_catalog" { 160 | catalog = databricks_catalog.workspace_catalog.name 161 | 162 | principal = var.workspace_catalog_admin 163 | privileges = ["ALL_PRIVILEGES"] 164 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/uc_workspace_catalog/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "resource_prefix" { 6 | type = string 7 | } 8 | 9 | variable "databricks_account_id" { 10 | type = string 11 | } 12 | 13 | variable "workspace_id" { 14 | type = string 15 | } 16 | 17 | variable "uc_catalog_name" { 18 | type = string 19 | } 20 | 21 | variable "workspace_catalog_admin" { 22 | type = string 23 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/workspace_confs/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/common_modules_workspace/workspace_confs/workspace_confs.tf: -------------------------------------------------------------------------------- 1 | resource "databricks_workspace_conf" "just_config_map" { 2 | custom_config = { 3 | "enableResultsDownloading" = "false", 4 | "enableNotebookTableClipboard" = "false", 5 | "enableVerboseAuditLogs" = "true", 6 | "enable-X-Frame-Options" = "true", 7 | "enable-X-Content-Type-Options" = "true", 8 | "enable-X-XSS-Protection" = "true", 9 | "enableWebTerminal" = "false", 10 | "enableDbfsFileBrowser" = "false", 11 | "enforceUserIsolation" = "true", 12 | "enableNotebookGitVersioning" = "true" 13 | } 14 | } -------------------------------------------------------------------------------- /repo/databricks_dev/cloud_provider/cloud_provider.tf: -------------------------------------------------------------------------------- 1 | module "cloud_provider_network" { 2 | source = "../../common_modules_cloud_provider/cloud_provider_network" 3 | 4 | vpc_cidr_range = var.vpc_cidr_range 5 | availability_zones = var.availability_zones 6 | resource_prefix = var.resource_prefix 7 | public_subnets_cidr = var.public_subnets_cidr 8 | private_subnets_cidr = var.private_subnets_cidr 9 | sg_ingress_protocol = var.sg_ingress_protocol 10 | sg_egress_ports = var.sg_egress_ports 11 | sg_egress_protocol = var.sg_egress_protocol 12 | 13 | } 14 | 15 | module "cloud_provider_credential" { 16 | source = "../../common_modules_cloud_provider/cloud_provider_credential" 17 | 18 | aws_account_id = var.aws_account_id 19 | databricks_account_id = var.databricks_account_id 20 | resource_prefix = var.resource_prefix 21 | region = var.region 22 | vpc_id = module.cloud_provider_network.cloud_provider_network_vpc 23 | security_group_ids = module.cloud_provider_network.cloud_provider_network_security_groups 24 | 25 | depends_on = [ module.cloud_provider_network ] 26 | } 27 | 28 | module "cloud_provider_storage" { 29 | source = "../../common_modules_cloud_provider/cloud_provider_storage" 30 | databricks_account_id = var.databricks_account_id 31 | dbfsname = var.dbfsname 32 | } -------------------------------------------------------------------------------- /repo/databricks_dev/cloud_provider/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_provider_credential" { 2 | value = module.cloud_provider_credential.cloud_provider_credential 3 | } 4 | 5 | output "cloud_provider_network_vpc" { 6 | value = module.cloud_provider_network.cloud_provider_network_vpc 7 | } 8 | 9 | output "cloud_provider_network_subnets" { 10 | value = module.cloud_provider_network.cloud_provider_network_subnets 11 | } 12 | 13 | output "cloud_provider_network_security_groups" { 14 | value = module.cloud_provider_network.cloud_provider_network_security_groups 15 | } 16 | 17 | output "cloud_provider_storage" { 18 | value = module.cloud_provider_storage.cloud_provider_storage 19 | } -------------------------------------------------------------------------------- /repo/databricks_dev/cloud_provider/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | aws = { 7 | source = "hashicorp/aws" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /repo/databricks_dev/cloud_provider/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "databricks_account_id" { 6 | type = string 7 | } 8 | 9 | variable "resource_prefix" { 10 | type = string 11 | } 12 | 13 | variable "region" { 14 | type = string 15 | } 16 | 17 | variable "vpc_cidr_range" { 18 | type = string 19 | } 20 | 21 | variable "availability_zones" { 22 | type = list(string) 23 | } 24 | 25 | variable "public_subnets_cidr" { 26 | type = list(string) 27 | } 28 | 29 | variable "private_subnets_cidr" { 30 | type = list(string) 31 | } 32 | 33 | variable "sg_ingress_protocol" { 34 | type = list(any) 35 | } 36 | 37 | variable "sg_egress_ports" { 38 | type = list(any) 39 | } 40 | 41 | variable "sg_egress_protocol" { 42 | type = list(any) 43 | } 44 | 45 | variable "dbfsname" { 46 | type = string 47 | } 48 | -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_account/databricks_account.tf: -------------------------------------------------------------------------------- 1 | module "workspace" { 2 | source = "../../common_modules_account/workspace_creation" 3 | 4 | databricks_account_id = var.databricks_account_id 5 | region = var.region 6 | resource_prefix = var.resource_prefix 7 | cross_account_role_arn = var.cross_account_role_arn 8 | bucket_name = var.bucket_name 9 | vpc_id = var.vpc_id 10 | subnet_ids = var.subnet_ids 11 | security_group_ids = var.security_group_ids 12 | 13 | } 14 | 15 | module "metastore_assignment" { 16 | source = "../../common_modules_account/metastore_assignment" 17 | 18 | metastore_id = var.metastore_id 19 | workspace_id = module.workspace.workspace_id 20 | 21 | depends_on = [module.workspace] 22 | 23 | } 24 | 25 | module "identity_assignment" { 26 | source = "../../common_modules_account/identity_assignment" 27 | 28 | workspace_id = module.workspace.workspace_id 29 | user_name = var.user_name 30 | 31 | depends_on = [module.metastore_assignment] 32 | 33 | } -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_account/outputs.tf: -------------------------------------------------------------------------------- 1 | output "workspace_url" { 2 | value = module.workspace.workspace_url 3 | } 4 | 5 | output "workspace_id" { 6 | value = module.workspace.workspace_id 7 | } -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_account/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_account/variables.tf: -------------------------------------------------------------------------------- 1 | variable "databricks_account_id" { 2 | type = string 3 | } 4 | 5 | variable "region" { 6 | type = string 7 | } 8 | 9 | variable "cross_account_role_arn" { 10 | type = string 11 | } 12 | 13 | variable "resource_prefix" { 14 | type = string 15 | } 16 | 17 | variable "bucket_name" { 18 | type = string 19 | } 20 | 21 | variable "security_group_ids" { 22 | type = list(string) 23 | } 24 | 25 | variable "subnet_ids" { 26 | type = list(string) 27 | } 28 | 29 | variable "vpc_id" { 30 | type = string 31 | } 32 | 33 | variable "user_name" { 34 | type = string 35 | } 36 | 37 | variable "metastore_id" { 38 | type = string 39 | } -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_workspace/databricks_workspace.tf: -------------------------------------------------------------------------------- 1 | module "uc_catalog" { 2 | source = "../../common_modules_workspace/uc_workspace_catalog" 3 | 4 | databricks_account_id = var.databricks_account_id 5 | aws_account_id = var.aws_account_id 6 | resource_prefix = var.resource_prefix 7 | uc_catalog_name = "${var.resource_prefix}-catalog-${var.workspace_id}" 8 | workspace_id = var.workspace_id 9 | workspace_catalog_admin = var.workspace_catalog_admin 10 | 11 | } 12 | 13 | module "dev_compute_policy" { 14 | source = "../../common_modules_workspace/cluster_policy" 15 | team = var.team 16 | policy_overrides = { 17 | "spark_conf.spark.databricks.io.cache.enabled" : { 18 | "type" : "fixed", 19 | "value" : "true" 20 | }, 21 | } 22 | } 23 | 24 | module "workspace_config" { 25 | source = "../../common_modules_workspace/workspace_confs" 26 | } -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_workspace/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | aws = { 7 | source = "hashicorp/aws" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /repo/databricks_dev/databricks_workspace/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "resource_prefix" { 6 | type = string 7 | } 8 | 9 | variable "databricks_account_id" { 10 | type = string 11 | } 12 | 13 | variable "workspace_id" { 14 | type = string 15 | } 16 | 17 | variable "uc_catalog_name" { 18 | type = string 19 | } 20 | 21 | variable "workspace_catalog_admin" { 22 | type = string 23 | } 24 | 25 | variable "team" { 26 | type = string 27 | } -------------------------------------------------------------------------------- /repo/databricks_dev/dev.tf: -------------------------------------------------------------------------------- 1 | module "cloud_provider" { 2 | source = "./cloud_provider" 3 | providers = { 4 | aws = aws 5 | databricks = databricks.mws 6 | } 7 | aws_account_id = var.aws_account_id 8 | databricks_account_id = var.databricks_account_id 9 | resource_prefix = var.resource_prefix 10 | region = var.region 11 | vpc_cidr_range = var.vpc_cidr_range 12 | availability_zones = var.availability_zones 13 | public_subnets_cidr = var.public_subnets_cidr 14 | private_subnets_cidr = var.private_subnets_cidr 15 | sg_ingress_protocol = var.sg_ingress_protocol 16 | sg_egress_ports = var.sg_egress_ports 17 | sg_egress_protocol = var.sg_egress_protocol 18 | dbfsname = var.dbfsname 19 | 20 | } 21 | 22 | module "databricks_account" { 23 | source = "./databricks_account" 24 | providers = { 25 | databricks = databricks.mws 26 | } 27 | 28 | databricks_account_id = var.databricks_account_id 29 | region = var.region 30 | resource_prefix = var.resource_prefix 31 | cross_account_role_arn = module.cloud_provider.cloud_provider_credential 32 | bucket_name = module.cloud_provider.cloud_provider_storage 33 | vpc_id = module.cloud_provider.cloud_provider_network_vpc 34 | subnet_ids = module.cloud_provider.cloud_provider_network_subnets 35 | security_group_ids = module.cloud_provider.cloud_provider_network_security_groups 36 | metastore_id = var.metastore_id 37 | user_name = var.user_name 38 | 39 | } 40 | 41 | module "databricks_workspace" { 42 | source = "./databricks_workspace" 43 | providers = { 44 | aws = aws 45 | databricks = databricks.workspace 46 | } 47 | 48 | aws_account_id = var.aws_account_id 49 | databricks_account_id = var.databricks_account_id 50 | resource_prefix = var.resource_prefix 51 | workspace_id = module.databricks_account.workspace_id 52 | uc_catalog_name = "${var.resource_prefix}-catalog-${module.databricks_account.workspace_id}" 53 | workspace_catalog_admin = var.user_name 54 | team = var.team 55 | 56 | } 57 | 58 | -------------------------------------------------------------------------------- /repo/databricks_dev/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | version = "~> 1.35.0" 6 | } 7 | aws = { 8 | source = "hashicorp/aws" 9 | } 10 | } 11 | } 12 | 13 | provider "aws" { 14 | region = var.region 15 | default_tags { 16 | tags = { 17 | Owner = var.resource_owner 18 | Resource = var.resource_prefix 19 | } 20 | } 21 | } 22 | 23 | provider "databricks" { 24 | alias = "mws" 25 | host = "https://accounts.cloud.databricks.com" 26 | account_id = var.databricks_account_id 27 | client_id = var.client_id 28 | client_secret = var.client_secret 29 | } 30 | 31 | provider "databricks" { 32 | alias = "workspace" 33 | host = module.databricks_account.workspace_url 34 | account_id = var.databricks_account_id 35 | client_id = var.client_id 36 | client_secret = var.client_secret 37 | } -------------------------------------------------------------------------------- /repo/databricks_dev/variables.tf: -------------------------------------------------------------------------------- 1 | variable "availability_zones" { 2 | type = list(string) 3 | } 4 | 5 | variable "aws_account_id" { 6 | type = string 7 | } 8 | 9 | variable "client_id" { 10 | type = string 11 | } 12 | 13 | variable "client_secret" { 14 | type = string 15 | } 16 | 17 | variable "databricks_account_id" { 18 | type = string 19 | } 20 | 21 | variable "dbfsname" { 22 | type = string 23 | } 24 | 25 | variable "metastore_id" { 26 | type = string 27 | } 28 | 29 | variable "private_subnets_cidr" { 30 | type = list(string) 31 | } 32 | 33 | variable "public_subnets_cidr" { 34 | type = list(string) 35 | } 36 | 37 | variable "region" { 38 | type = string 39 | } 40 | 41 | variable "resource_prefix" { 42 | type = string 43 | } 44 | 45 | variable "resource_owner" { 46 | type = string 47 | } 48 | 49 | variable "sg_egress_ports" { 50 | type = list(any) 51 | } 52 | 53 | variable "sg_egress_protocol" { 54 | type = list(any) 55 | } 56 | 57 | variable "sg_ingress_protocol" { 58 | type = list(any) 59 | } 60 | 61 | variable "team" { 62 | type = string 63 | } 64 | 65 | variable "user_name" { 66 | type = string 67 | } 68 | 69 | variable "vpc_cidr_range" { 70 | type = string 71 | } 72 | 73 | 74 | -------------------------------------------------------------------------------- /repo/databricks_prod/cloud_provider/cloud_provider.tf: -------------------------------------------------------------------------------- 1 | module "cloud_provider_network" { 2 | source = "../../common_modules_cloud_provider/cloud_provider_network" 3 | 4 | vpc_cidr_range = var.vpc_cidr_range 5 | availability_zones = var.availability_zones 6 | resource_prefix = var.resource_prefix 7 | public_subnets_cidr = var.public_subnets_cidr 8 | private_subnets_cidr = var.private_subnets_cidr 9 | sg_ingress_protocol = var.sg_ingress_protocol 10 | sg_egress_ports = var.sg_egress_ports 11 | sg_egress_protocol = var.sg_egress_protocol 12 | 13 | } 14 | 15 | module "cloud_provider_credential" { 16 | source = "../../common_modules_cloud_provider/cloud_provider_credential" 17 | 18 | aws_account_id = var.aws_account_id 19 | databricks_account_id = var.databricks_account_id 20 | resource_prefix = var.resource_prefix 21 | region = var.region 22 | vpc_id = module.cloud_provider_network.cloud_provider_network_vpc 23 | security_group_ids = module.cloud_provider_network.cloud_provider_network_security_groups 24 | 25 | depends_on = [ module.cloud_provider_network ] 26 | } 27 | 28 | module "cloud_provider_storage" { 29 | source = "../../common_modules_cloud_provider/cloud_provider_storage" 30 | databricks_account_id = var.databricks_account_id 31 | dbfsname = var.dbfsname 32 | } -------------------------------------------------------------------------------- /repo/databricks_prod/cloud_provider/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_provider_credential" { 2 | value = module.cloud_provider_credential.cloud_provider_credential 3 | } 4 | 5 | output "cloud_provider_network_vpc" { 6 | value = module.cloud_provider_network.cloud_provider_network_vpc 7 | } 8 | 9 | output "cloud_provider_network_subnets" { 10 | value = module.cloud_provider_network.cloud_provider_network_subnets 11 | } 12 | 13 | output "cloud_provider_network_security_groups" { 14 | value = module.cloud_provider_network.cloud_provider_network_security_groups 15 | } 16 | 17 | output "cloud_provider_storage" { 18 | value = module.cloud_provider_storage.cloud_provider_storage 19 | } -------------------------------------------------------------------------------- /repo/databricks_prod/cloud_provider/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | aws = { 7 | source = "hashicorp/aws" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /repo/databricks_prod/cloud_provider/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "databricks_account_id" { 6 | type = string 7 | } 8 | 9 | variable "resource_prefix" { 10 | type = string 11 | } 12 | 13 | variable "region" { 14 | type = string 15 | } 16 | 17 | variable "vpc_cidr_range" { 18 | type = string 19 | } 20 | 21 | variable "availability_zones" { 22 | type = list(string) 23 | } 24 | 25 | variable "public_subnets_cidr" { 26 | type = list(string) 27 | } 28 | 29 | variable "private_subnets_cidr" { 30 | type = list(string) 31 | } 32 | 33 | variable "sg_ingress_protocol" { 34 | type = list(any) 35 | } 36 | 37 | variable "sg_egress_ports" { 38 | type = list(any) 39 | } 40 | 41 | variable "sg_egress_protocol" { 42 | type = list(any) 43 | } 44 | 45 | variable "dbfsname" { 46 | type = string 47 | } 48 | -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_account/databricks_account.tf: -------------------------------------------------------------------------------- 1 | module "workspace" { 2 | source = "../../common_modules_account/workspace_creation" 3 | 4 | databricks_account_id = var.databricks_account_id 5 | region = var.region 6 | resource_prefix = var.resource_prefix 7 | cross_account_role_arn = var.cross_account_role_arn 8 | bucket_name = var.bucket_name 9 | vpc_id = var.vpc_id 10 | subnet_ids = var.subnet_ids 11 | security_group_ids = var.security_group_ids 12 | 13 | } 14 | 15 | module "metastore_assignment" { 16 | source = "../../common_modules_account/metastore_assignment" 17 | 18 | metastore_id = var.metastore_id 19 | workspace_id = module.workspace.workspace_id 20 | 21 | depends_on = [module.workspace] 22 | 23 | } 24 | 25 | module "identity_assignment" { 26 | source = "../../common_modules_account/identity_assignment" 27 | 28 | workspace_id = module.workspace.workspace_id 29 | user_name = var.user_name 30 | 31 | depends_on = [module.metastore_assignment] 32 | 33 | } -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_account/outputs.tf: -------------------------------------------------------------------------------- 1 | output "workspace_url" { 2 | value = module.workspace.workspace_url 3 | } 4 | 5 | output "workspace_id" { 6 | value = module.workspace.workspace_id 7 | } -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_account/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_account/variables.tf: -------------------------------------------------------------------------------- 1 | variable "databricks_account_id" { 2 | type = string 3 | } 4 | 5 | variable "region" { 6 | type = string 7 | } 8 | 9 | variable "cross_account_role_arn" { 10 | type = string 11 | } 12 | 13 | variable "resource_prefix" { 14 | type = string 15 | } 16 | 17 | variable "bucket_name" { 18 | type = string 19 | } 20 | 21 | variable "security_group_ids" { 22 | type = list(string) 23 | } 24 | 25 | variable "subnet_ids" { 26 | type = list(string) 27 | } 28 | 29 | variable "vpc_id" { 30 | type = string 31 | } 32 | 33 | variable "user_name" { 34 | type = string 35 | } 36 | 37 | variable "metastore_id" { 38 | type = string 39 | } -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_workspace/databricks_workspace.tf: -------------------------------------------------------------------------------- 1 | module "uc_catalog" { 2 | source = "../../common_modules_workspace/uc_workspace_catalog" 3 | 4 | databricks_account_id = var.databricks_account_id 5 | aws_account_id = var.aws_account_id 6 | resource_prefix = var.resource_prefix 7 | uc_catalog_name = "${var.resource_prefix}-catalog-${var.workspace_id}" 8 | workspace_id = var.workspace_id 9 | workspace_catalog_admin = var.workspace_catalog_admin 10 | 11 | } 12 | 13 | module "dev_compute_policy" { 14 | source = "../../common_modules_workspace/cluster_policy" 15 | team = var.team 16 | policy_overrides = { 17 | "spark_conf.spark.databricks.io.cache.enabled" : { 18 | "type" : "fixed", 19 | "value" : "true" 20 | }, 21 | } 22 | } 23 | 24 | module "workspace_config" { 25 | source = "../../common_modules_workspace/workspace_confs" 26 | } -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_workspace/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/databricks_prod/databricks_workspace/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "resource_prefix" { 6 | type = string 7 | } 8 | 9 | variable "databricks_account_id" { 10 | type = string 11 | } 12 | 13 | variable "workspace_id" { 14 | type = string 15 | } 16 | 17 | variable "uc_catalog_name" { 18 | type = string 19 | } 20 | 21 | variable "workspace_catalog_admin" { 22 | type = string 23 | } 24 | 25 | variable "team" { 26 | type = string 27 | } -------------------------------------------------------------------------------- /repo/databricks_prod/prod.tf: -------------------------------------------------------------------------------- 1 | module "cloud_provider" { 2 | source = "./cloud_provider" 3 | providers = { 4 | aws = aws 5 | databricks = databricks.mws 6 | } 7 | aws_account_id = var.aws_account_id 8 | databricks_account_id = var.databricks_account_id 9 | resource_prefix = var.resource_prefix 10 | region = var.region 11 | vpc_cidr_range = var.vpc_cidr_range 12 | availability_zones = var.availability_zones 13 | public_subnets_cidr = var.public_subnets_cidr 14 | private_subnets_cidr = var.private_subnets_cidr 15 | sg_ingress_protocol = var.sg_ingress_protocol 16 | sg_egress_ports = var.sg_egress_ports 17 | sg_egress_protocol = var.sg_egress_protocol 18 | dbfsname = var.dbfsname 19 | 20 | } 21 | 22 | module "databricks_account" { 23 | source = "./databricks_account" 24 | providers = { 25 | databricks = databricks.mws 26 | } 27 | 28 | databricks_account_id = var.databricks_account_id 29 | region = var.region 30 | resource_prefix = var.resource_prefix 31 | cross_account_role_arn = module.cloud_provider.cloud_provider_credential 32 | bucket_name = module.cloud_provider.cloud_provider_storage 33 | vpc_id = module.cloud_provider.cloud_provider_network_vpc 34 | subnet_ids = module.cloud_provider.cloud_provider_network_subnets 35 | security_group_ids = module.cloud_provider.cloud_provider_network_security_groups 36 | metastore_id = var.metastore_id 37 | user_name = var.user_name 38 | 39 | } 40 | 41 | module "databricks_workspace" { 42 | source = "./databricks_workspace" 43 | providers = { 44 | aws = aws 45 | databricks = databricks.workspace 46 | } 47 | 48 | aws_account_id = var.aws_account_id 49 | databricks_account_id = var.databricks_account_id 50 | resource_prefix = var.resource_prefix 51 | workspace_id = module.databricks_account.workspace_id 52 | uc_catalog_name = "${var.resource_prefix}-catalog-${module.databricks_account.workspace_id}" 53 | workspace_catalog_admin = var.user_name 54 | team = var.team 55 | 56 | } 57 | -------------------------------------------------------------------------------- /repo/databricks_prod/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | version = "~> 1.35.0" 6 | } 7 | aws = { 8 | source = "hashicorp/aws" 9 | } 10 | } 11 | } 12 | 13 | provider "aws" { 14 | region = var.region 15 | default_tags { 16 | tags = { 17 | Owner = var.resource_owner 18 | Resource = var.resource_prefix 19 | } 20 | } 21 | } 22 | 23 | provider "databricks" { 24 | alias = "mws" 25 | host = "https://accounts.cloud.databricks.com" 26 | account_id = var.databricks_account_id 27 | client_id = var.client_id 28 | client_secret = var.client_secret 29 | } 30 | 31 | provider "databricks" { 32 | alias = "workspace" 33 | host = module.databricks_account.workspace_url 34 | account_id = var.databricks_account_id 35 | client_id = var.client_id 36 | client_secret = var.client_secret 37 | } -------------------------------------------------------------------------------- /repo/databricks_prod/variables.tf: -------------------------------------------------------------------------------- 1 | variable "availability_zones" { 2 | type = list(string) 3 | } 4 | 5 | variable "aws_account_id" { 6 | type = string 7 | } 8 | 9 | variable "client_id" { 10 | type = string 11 | } 12 | 13 | variable "client_secret" { 14 | type = string 15 | } 16 | 17 | variable "databricks_account_id" { 18 | type = string 19 | } 20 | 21 | variable "dbfsname" { 22 | type = string 23 | } 24 | 25 | variable "metastore_id" { 26 | type = string 27 | } 28 | 29 | variable "private_subnets_cidr" { 30 | type = list(string) 31 | } 32 | 33 | variable "public_subnets_cidr" { 34 | type = list(string) 35 | } 36 | 37 | variable "region" { 38 | type = string 39 | } 40 | 41 | variable "resource_prefix" { 42 | type = string 43 | } 44 | 45 | variable "resource_owner" { 46 | type = string 47 | } 48 | 49 | variable "sg_egress_ports" { 50 | type = list(any) 51 | } 52 | 53 | variable "sg_egress_protocol" { 54 | type = list(any) 55 | } 56 | 57 | variable "sg_ingress_protocol" { 58 | type = list(any) 59 | } 60 | 61 | variable "team" { 62 | type = string 63 | } 64 | 65 | variable "user_name" { 66 | type = string 67 | } 68 | 69 | variable "vpc_cidr_range" { 70 | type = string 71 | } 72 | -------------------------------------------------------------------------------- /repo/databricks_qa/cloud_provider/cloud_provider.tf: -------------------------------------------------------------------------------- 1 | module "cloud_provider_network" { 2 | source = "../../common_modules_cloud_provider/cloud_provider_network" 3 | 4 | vpc_cidr_range = var.vpc_cidr_range 5 | availability_zones = var.availability_zones 6 | resource_prefix = var.resource_prefix 7 | public_subnets_cidr = var.public_subnets_cidr 8 | private_subnets_cidr = var.private_subnets_cidr 9 | sg_ingress_protocol = var.sg_ingress_protocol 10 | sg_egress_ports = var.sg_egress_ports 11 | sg_egress_protocol = var.sg_egress_protocol 12 | 13 | } 14 | 15 | module "cloud_provider_credential" { 16 | source = "../../common_modules_cloud_provider/cloud_provider_credential" 17 | 18 | aws_account_id = var.aws_account_id 19 | databricks_account_id = var.databricks_account_id 20 | resource_prefix = var.resource_prefix 21 | region = var.region 22 | vpc_id = module.cloud_provider_network.cloud_provider_network_vpc 23 | security_group_ids = module.cloud_provider_network.cloud_provider_network_security_groups 24 | 25 | depends_on = [ module.cloud_provider_network ] 26 | } 27 | 28 | module "cloud_provider_storage" { 29 | source = "../../common_modules_cloud_provider/cloud_provider_storage" 30 | databricks_account_id = var.databricks_account_id 31 | dbfsname = var.dbfsname 32 | } -------------------------------------------------------------------------------- /repo/databricks_qa/cloud_provider/outputs.tf: -------------------------------------------------------------------------------- 1 | output "cloud_provider_credential" { 2 | value = module.cloud_provider_credential.cloud_provider_credential 3 | } 4 | 5 | output "cloud_provider_network_vpc" { 6 | value = module.cloud_provider_network.cloud_provider_network_vpc 7 | } 8 | 9 | output "cloud_provider_network_subnets" { 10 | value = module.cloud_provider_network.cloud_provider_network_subnets 11 | } 12 | 13 | output "cloud_provider_network_security_groups" { 14 | value = module.cloud_provider_network.cloud_provider_network_security_groups 15 | } 16 | 17 | output "cloud_provider_storage" { 18 | value = module.cloud_provider_storage.cloud_provider_storage 19 | } -------------------------------------------------------------------------------- /repo/databricks_qa/cloud_provider/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | aws = { 7 | source = "hashicorp/aws" 8 | } 9 | } 10 | } -------------------------------------------------------------------------------- /repo/databricks_qa/cloud_provider/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "databricks_account_id" { 6 | type = string 7 | } 8 | 9 | variable "resource_prefix" { 10 | type = string 11 | } 12 | 13 | variable "region" { 14 | type = string 15 | } 16 | 17 | variable "vpc_cidr_range" { 18 | type = string 19 | } 20 | 21 | variable "availability_zones" { 22 | type = list(string) 23 | } 24 | 25 | variable "public_subnets_cidr" { 26 | type = list(string) 27 | } 28 | 29 | variable "private_subnets_cidr" { 30 | type = list(string) 31 | } 32 | 33 | variable "sg_ingress_protocol" { 34 | type = list(any) 35 | } 36 | 37 | variable "sg_egress_ports" { 38 | type = list(any) 39 | } 40 | 41 | variable "sg_egress_protocol" { 42 | type = list(any) 43 | } 44 | 45 | variable "dbfsname" { 46 | type = string 47 | } 48 | -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_account/databricks_account.tf: -------------------------------------------------------------------------------- 1 | module "workspace" { 2 | source = "../../common_modules_account/workspace_creation" 3 | 4 | databricks_account_id = var.databricks_account_id 5 | region = var.region 6 | resource_prefix = var.resource_prefix 7 | cross_account_role_arn = var.cross_account_role_arn 8 | bucket_name = var.bucket_name 9 | vpc_id = var.vpc_id 10 | subnet_ids = var.subnet_ids 11 | security_group_ids = var.security_group_ids 12 | 13 | } 14 | 15 | module "metastore_assignment" { 16 | source = "../../common_modules_account/metastore_assignment" 17 | 18 | metastore_id = var.metastore_id 19 | workspace_id = module.workspace.workspace_id 20 | 21 | depends_on = [module.workspace] 22 | 23 | } 24 | 25 | module "identity_assignment" { 26 | source = "../../common_modules_account/identity_assignment" 27 | 28 | workspace_id = module.workspace.workspace_id 29 | user_name = var.user_name 30 | 31 | depends_on = [module.metastore_assignment] 32 | 33 | } -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_account/outputs.tf: -------------------------------------------------------------------------------- 1 | output "workspace_url" { 2 | value = module.workspace.workspace_url 3 | } 4 | 5 | output "workspace_id" { 6 | value = module.workspace.workspace_id 7 | } -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_account/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_account/variables.tf: -------------------------------------------------------------------------------- 1 | variable "databricks_account_id" { 2 | type = string 3 | } 4 | 5 | variable "region" { 6 | type = string 7 | } 8 | 9 | variable "cross_account_role_arn" { 10 | type = string 11 | } 12 | 13 | variable "resource_prefix" { 14 | type = string 15 | } 16 | 17 | variable "bucket_name" { 18 | type = string 19 | } 20 | 21 | variable "security_group_ids" { 22 | type = list(string) 23 | } 24 | 25 | variable "subnet_ids" { 26 | type = list(string) 27 | } 28 | 29 | variable "vpc_id" { 30 | type = string 31 | } 32 | 33 | variable "user_name" { 34 | type = string 35 | } 36 | 37 | variable "metastore_id" { 38 | type = string 39 | } -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_workspace/databricks_workspace.tf: -------------------------------------------------------------------------------- 1 | module "uc_catalog" { 2 | source = "../../common_modules_workspace/uc_workspace_catalog" 3 | 4 | databricks_account_id = var.databricks_account_id 5 | aws_account_id = var.aws_account_id 6 | resource_prefix = var.resource_prefix 7 | uc_catalog_name = "${var.resource_prefix}-catalog-${var.workspace_id}" 8 | workspace_id = var.workspace_id 9 | workspace_catalog_admin = var.workspace_catalog_admin 10 | 11 | } 12 | 13 | module "dev_compute_policy" { 14 | source = "../../common_modules_workspace/cluster_policy" 15 | team = var.team 16 | policy_overrides = { 17 | "spark_conf.spark.databricks.io.cache.enabled" : { 18 | "type" : "fixed", 19 | "value" : "true" 20 | }, 21 | } 22 | } 23 | 24 | module "workspace_config" { 25 | source = "../../common_modules_workspace/workspace_confs" 26 | } -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_workspace/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | } 6 | } 7 | } -------------------------------------------------------------------------------- /repo/databricks_qa/databricks_workspace/variables.tf: -------------------------------------------------------------------------------- 1 | variable "aws_account_id" { 2 | type = string 3 | } 4 | 5 | variable "resource_prefix" { 6 | type = string 7 | } 8 | 9 | variable "databricks_account_id" { 10 | type = string 11 | } 12 | 13 | variable "workspace_id" { 14 | type = string 15 | } 16 | 17 | variable "uc_catalog_name" { 18 | type = string 19 | } 20 | 21 | variable "workspace_catalog_admin" { 22 | type = string 23 | } 24 | 25 | variable "team" { 26 | type = string 27 | } -------------------------------------------------------------------------------- /repo/databricks_qa/provider.tf: -------------------------------------------------------------------------------- 1 | terraform { 2 | required_providers { 3 | databricks = { 4 | source = "databricks/databricks" 5 | version = "~> 1.35.0" 6 | } 7 | aws = { 8 | source = "hashicorp/aws" 9 | } 10 | } 11 | } 12 | 13 | provider "aws" { 14 | region = var.region 15 | default_tags { 16 | tags = { 17 | Owner = var.resource_owner 18 | Resource = var.resource_prefix 19 | } 20 | } 21 | } 22 | 23 | provider "databricks" { 24 | alias = "mws" 25 | host = "https://accounts.cloud.databricks.com" 26 | account_id = var.databricks_account_id 27 | client_id = var.client_id 28 | client_secret = var.client_secret 29 | } 30 | 31 | provider "databricks" { 32 | alias = "workspace" 33 | host = module.databricks_account.workspace_url 34 | account_id = var.databricks_account_id 35 | client_id = var.client_id 36 | client_secret = var.client_secret 37 | } -------------------------------------------------------------------------------- /repo/databricks_qa/qa.tf: -------------------------------------------------------------------------------- 1 | module "cloud_provider" { 2 | source = "./cloud_provider" 3 | providers = { 4 | aws = aws 5 | databricks = databricks.mws 6 | } 7 | aws_account_id = var.aws_account_id 8 | databricks_account_id = var.databricks_account_id 9 | resource_prefix = var.resource_prefix 10 | region = var.region 11 | vpc_cidr_range = var.vpc_cidr_range 12 | availability_zones = var.availability_zones 13 | public_subnets_cidr = var.public_subnets_cidr 14 | private_subnets_cidr = var.private_subnets_cidr 15 | sg_ingress_protocol = var.sg_ingress_protocol 16 | sg_egress_ports = var.sg_egress_ports 17 | sg_egress_protocol = var.sg_egress_protocol 18 | dbfsname = var.dbfsname 19 | 20 | } 21 | 22 | module "databricks_account" { 23 | source = "./databricks_account" 24 | providers = { 25 | databricks = databricks.mws 26 | } 27 | 28 | databricks_account_id = var.databricks_account_id 29 | region = var.region 30 | resource_prefix = var.resource_prefix 31 | cross_account_role_arn = module.cloud_provider.cloud_provider_credential 32 | bucket_name = module.cloud_provider.cloud_provider_storage 33 | vpc_id = module.cloud_provider.cloud_provider_network_vpc 34 | subnet_ids = module.cloud_provider.cloud_provider_network_subnets 35 | security_group_ids = module.cloud_provider.cloud_provider_network_security_groups 36 | metastore_id = var.metastore_id 37 | user_name = var.user_name 38 | 39 | } 40 | 41 | module "databricks_workspace" { 42 | source = "./databricks_workspace" 43 | providers = { 44 | aws = aws 45 | databricks = databricks.workspace 46 | } 47 | 48 | aws_account_id = var.aws_account_id 49 | databricks_account_id = var.databricks_account_id 50 | resource_prefix = var.resource_prefix 51 | workspace_id = module.databricks_account.workspace_id 52 | uc_catalog_name = "${var.resource_prefix}-catalog-${module.databricks_account.workspace_id}" 53 | workspace_catalog_admin = var.user_name 54 | team = var.team 55 | 56 | } -------------------------------------------------------------------------------- /repo/databricks_qa/variables.tf: -------------------------------------------------------------------------------- 1 | variable "availability_zones" { 2 | type = list(string) 3 | } 4 | 5 | variable "aws_account_id" { 6 | type = string 7 | } 8 | 9 | variable "client_id" { 10 | type = string 11 | } 12 | 13 | variable "client_secret" { 14 | type = string 15 | } 16 | 17 | variable "databricks_account_id" { 18 | type = string 19 | } 20 | 21 | variable "dbfsname" { 22 | type = string 23 | } 24 | 25 | variable "metastore_id" { 26 | type = string 27 | } 28 | 29 | variable "private_subnets_cidr" { 30 | type = list(string) 31 | } 32 | 33 | variable "public_subnets_cidr" { 34 | type = list(string) 35 | } 36 | 37 | variable "region" { 38 | type = string 39 | } 40 | 41 | variable "resource_prefix" { 42 | type = string 43 | } 44 | 45 | variable "resource_owner" { 46 | type = string 47 | } 48 | 49 | variable "sg_egress_ports" { 50 | type = list(any) 51 | } 52 | 53 | variable "sg_egress_protocol" { 54 | type = list(any) 55 | } 56 | 57 | variable "sg_ingress_protocol" { 58 | type = list(any) 59 | } 60 | 61 | variable "team" { 62 | type = string 63 | } 64 | 65 | variable "user_name" { 66 | type = string 67 | } 68 | 69 | variable "vpc_cidr_range" { 70 | type = string 71 | } 72 | -------------------------------------------------------------------------------- /tfvars_examples/logging.md: -------------------------------------------------------------------------------- 1 | // Databricks Variables 2 | client_id = "" 3 | client_secret = "" 4 | databricks_account_id = "" 5 | resource_prefix = "" 6 | resource_owner = "" 7 | 8 | // AWS Variables 9 | region = "" -------------------------------------------------------------------------------- /tfvars_examples/unity_catalog.md: -------------------------------------------------------------------------------- 1 | // General - AWS 2 | aws_account_id = "" 3 | region = "" 4 | 5 | // General - Databricks 6 | client_id = "" 7 | client_secret = "" 8 | databricks_account_id = "" 9 | resource_prefix = "" 10 | resource_owner = "" 11 | ucname = "" -------------------------------------------------------------------------------- /tfvars_examples/workspace_env.md: -------------------------------------------------------------------------------- 1 | // General 2 | resource_prefix = "" 3 | resource_owner = "" 4 | 5 | // General - AWS 6 | aws_account_id = "" 7 | region = "" 8 | 9 | // General - Databricks 10 | databricks_account_id = "" 11 | client_id = "" 12 | client_secret = "" 13 | metastore_id = "" 14 | 15 | // Cloud Provider Network 16 | availability_zones = "" // List 17 | private_subnets_cidr = "" // List 18 | public_subnets_cidr = "" // List 19 | sg_egress_ports = "" // List 20 | sg_egress_protocol = "" // List 21 | sg_ingress_protocol = "" // List 22 | vpc_cidr_range = "" 23 | 24 | // Cloud Provider Storage (Root Storage) 25 | dbfsname = "" 26 | 27 | // Workspace and Catalog Admin 28 | user_name = "" 29 | 30 | // Cluster Policy 31 | team = "" --------------------------------------------------------------------------------