├── .terraform.lock ├── docs ├── TROUBLESHOOTING.md ├── upgrading_to_bigquery_v8.0.md ├── upgrading_to_bigquery_v2.0.md ├── upgrading_to_bigquery_v3.0.md └── upgrading_to_bigquery_v4.0.md ├── test ├── .gitignore ├── setup │ ├── .gitignore │ ├── versions.tf │ ├── outputs.tf │ ├── variables.tf │ ├── main.tf │ └── iam.tf ├── fixtures │ └── full │ │ ├── terraform.tfvars │ │ ├── sample_bq_schema.json │ │ ├── main.tf │ │ ├── variables.tf │ │ └── outputs.tf └── integration │ ├── discover_test.go │ ├── full │ ├── inspec.yml │ └── controls │ │ └── big_query.rb │ ├── go.mod │ └── data_warehouse │ └── data_warehouse_test.go ├── modules ├── data_warehouse │ ├── .gitignore │ ├── src │ │ ├── function │ │ │ ├── requirements.txt │ │ │ └── main.py │ │ ├── schema │ │ │ ├── distribution_centers_schema.json │ │ │ ├── products_schema.json │ │ │ ├── orders_schema.json │ │ │ ├── order_items_schema.json │ │ │ ├── inventory_items_schema.json │ │ │ ├── events_schema.json │ │ │ └── users_schema.json │ │ └── sql │ │ │ ├── sp_bigqueryml_generate_create.sql │ │ │ ├── sp_provision_lookup_tables.sql │ │ │ ├── sp_sample_translation_queries.sql │ │ │ ├── sp_bigqueryml_model.sql │ │ │ └── sp_bigqueryml_generate_describe.sql │ ├── assets │ │ └── data-warehouse-architecture.png │ ├── workflow_polling │ │ ├── variables.tf │ │ ├── versions.tf │ │ └── main.tf │ ├── versions.tf │ ├── outputs.tf │ ├── metadata.display.yaml │ ├── main.tf │ ├── variables.tf │ ├── templates │ │ └── workflow.tftpl │ ├── README.md │ └── workflows.tf ├── udf │ ├── outputs.tf │ ├── versions.tf │ ├── variables.tf │ ├── metadata.display.yaml │ ├── README.md │ ├── metadata.yaml │ ├── scripts │ │ └── persistent_udfs.sh │ └── main.tf ├── scheduled_queries │ ├── outputs.tf │ ├── variables.tf │ ├── versions.tf │ ├── metadata.display.yaml │ ├── README.md │ ├── main.tf │ └── metadata.yaml └── authorization │ ├── versions.tf │ ├── outputs.tf │ ├── metadata.display.yaml │ ├── variables.tf │ ├── main.tf │ ├── README.md │ └── metadata.yaml ├── .github ├── renovate.json ├── release-please.yml ├── trusted-contribution.yml └── workflows │ ├── stale.yml │ └── lint.yaml ├── examples ├── multiple_tables │ ├── terraform.tfvars │ ├── versions.tf │ ├── sample_bq_schema.json │ ├── provider.tf │ ├── variables.tf │ ├── outputs.tf │ └── README.md ├── data_warehouse │ ├── variables.tf │ ├── versions.tf │ ├── README.md │ ├── main.tf │ └── outputs.tf ├── basic_bq │ ├── versions.tf │ ├── sample_bq_schema.json │ ├── outputs.tf │ ├── variables.tf │ ├── README.md │ └── main.tf ├── scheduled_queries │ ├── outputs.tf │ ├── versions.tf │ ├── README.md │ └── main.tf └── basic_view │ ├── versions.tf │ ├── terraform.tfvars │ ├── outputs.tf │ ├── variables.tf │ ├── README.md │ └── main.tf ├── SECURITY.md ├── CODEOWNERS ├── .gitignore ├── versions.tf ├── .kitchen.yml ├── helpers ├── version-repo.sh └── setup-sa.sh ├── Makefile ├── CONTRIBUTING.md └── outputs.tf /.terraform.lock: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /docs/TROUBLESHOOTING.md: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | source.sh 2 | -------------------------------------------------------------------------------- /test/setup/.gitignore: -------------------------------------------------------------------------------- 1 | terraform.tfvars 2 | source.sh 3 | -------------------------------------------------------------------------------- /modules/data_warehouse/.gitignore: -------------------------------------------------------------------------------- 1 | *.zip 2 | terraform.tfvars 3 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/function/requirements.txt: -------------------------------------------------------------------------------- 1 | functions-framework==3.* 2 | google-cloud-dataform==0.5.10 3 | -------------------------------------------------------------------------------- /test/fixtures/full/terraform.tfvars: -------------------------------------------------------------------------------- 1 | default_table_expiration_ms = 3600000 2 | dataset_labels = { 3 | env = "dev" 4 | billable = "true" 5 | owner = "janesmith" 6 | } 7 | -------------------------------------------------------------------------------- /modules/data_warehouse/assets/data-warehouse-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/terraform-google-modules/terraform-google-bigquery/HEAD/modules/data_warehouse/assets/data-warehouse-architecture.png -------------------------------------------------------------------------------- /.github/renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": ["github>GoogleCloudPlatform/cloud-foundation-toolkit//infra/terraform/test-org/github/resources/renovate"] 4 | } 5 | -------------------------------------------------------------------------------- /examples/multiple_tables/terraform.tfvars: -------------------------------------------------------------------------------- 1 | project_id = "example-project" 2 | delete_contents_on_destroy = true 3 | default_table_expiration_ms = 3600000 4 | dataset_labels = { 5 | env = "dev" 6 | billable = "true" 7 | owner = "janesmith" 8 | } 9 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | To report a security issue, please use http://g.co/vulnz. We use 2 | http://g.co/vulnz for our intake, and do coordination and disclosure here on 3 | GitHub (including using GitHub Security Advisory). The Google Security Team will 4 | respond within 5 working days of your report on g.co/vulnz. 5 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/schema/distribution_centers_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "id", 5 | "type": "INTEGER" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "name", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "latitude", 15 | "type": "FLOAT" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "longitude", 20 | "type": "FLOAT" 21 | } 22 | ] 23 | -------------------------------------------------------------------------------- /.github/release-please.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2021 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | releaseType: terraform-module 16 | handleGHRelease: true 17 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # NOTE: This file is automatically generated from values at: 2 | # https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/blob/main/infra/terraform/test-org/org/locals.tf 3 | 4 | * @terraform-google-modules/cft-admins @ayushmjain @bradmiro @davenportjw @q2w @terraform-google-modules/jump-start-solutions-admins 5 | 6 | # NOTE: GitHub CODEOWNERS locations: 7 | # https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#codeowners-and-branch-protection 8 | 9 | CODEOWNERS @terraform-google-modules/cft-admins 10 | .github/CODEOWNERS @terraform-google-modules/cft-admins 11 | docs/CODEOWNERS @terraform-google-modules/cft-admins 12 | 13 | -------------------------------------------------------------------------------- /examples/data_warehouse/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "The ID of the project in which to provision resources." 19 | type = string 20 | } 21 | -------------------------------------------------------------------------------- /examples/basic_bq/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | } 22 | } 23 | required_version = ">= 1.3" 24 | } 25 | -------------------------------------------------------------------------------- /examples/scheduled_queries/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "query_names" { 18 | value = module.scheduled_queries.query_names 19 | description = "The resource names of the transfer config" 20 | } 21 | -------------------------------------------------------------------------------- /examples/basic_view/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | } 22 | } 23 | required_version = ">= 1.3" 24 | } 25 | -------------------------------------------------------------------------------- /examples/data_warehouse/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | } 22 | } 23 | required_version = ">= 1.3" 24 | } 25 | -------------------------------------------------------------------------------- /examples/multiple_tables/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | } 22 | } 23 | required_version = ">= 1.3" 24 | } 25 | -------------------------------------------------------------------------------- /examples/scheduled_queries/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | terraform { 17 | required_version = ">= 1.3" 18 | required_providers { 19 | 20 | google = { 21 | source = "hashicorp/google" 22 | } 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/sql/sp_bigqueryml_generate_create.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | CREATE OR REPLACE MODEL `${project_id}.${dataset_id}.${model_name}` 16 | REMOTE WITH CONNECTION `${connection_id}` 17 | OPTIONS (ENDPOINT = 'gemini-2.0-flash'); 18 | -------------------------------------------------------------------------------- /modules/udf/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "added_udfs" { 18 | description = "List of UDFs utility functions added." 19 | value = var.add_udfs ? ["find_in_set", "check_protocol", "parse_url", "csv_to_struct"] : [] 20 | } 21 | -------------------------------------------------------------------------------- /examples/basic_bq/sample_bq_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "Full visitor ID", 4 | "mode": "NULLABLE", 5 | "name": "fullVisitorId", 6 | "type": "STRING" 7 | }, 8 | { 9 | "description": "Visit number", 10 | "mode": "NULLABLE", 11 | "name": "visitNumber", 12 | "type": "INTEGER" 13 | }, 14 | { 15 | "description": "Visit ID", 16 | "mode": "NULLABLE", 17 | "name": "visitId", 18 | "type": "INTEGER" 19 | }, 20 | { 21 | "description": "Visit Start Time", 22 | "mode": "NULLABLE", 23 | "name": "visitStartTime", 24 | "type": "INTEGER" 25 | }, 26 | { 27 | "description": "Full Date of Visit", 28 | "mode": "NULLABLE", 29 | "name": "fullDate", 30 | "type": "DATE" 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /test/fixtures/full/sample_bq_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "Full visitor ID", 4 | "mode": "NULLABLE", 5 | "name": "fullVisitorId", 6 | "type": "STRING" 7 | }, 8 | { 9 | "description": "Visit number", 10 | "mode": "NULLABLE", 11 | "name": "visitNumber", 12 | "type": "INTEGER" 13 | }, 14 | { 15 | "description": "Visit ID", 16 | "mode": "NULLABLE", 17 | "name": "visitId", 18 | "type": "INTEGER" 19 | }, 20 | { 21 | "description": "Visit Start Time", 22 | "mode": "NULLABLE", 23 | "name": "visitStartTime", 24 | "type": "INTEGER" 25 | }, 26 | { 27 | "description": "Full Date of Visit", 28 | "mode": "NULLABLE", 29 | "name": "fullDate", 30 | "type": "DATE" 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /examples/multiple_tables/sample_bq_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "description": "Full visitor ID", 4 | "mode": "NULLABLE", 5 | "name": "fullVisitorId", 6 | "type": "STRING" 7 | }, 8 | { 9 | "description": "Visit number", 10 | "mode": "NULLABLE", 11 | "name": "visitNumber", 12 | "type": "INTEGER" 13 | }, 14 | { 15 | "description": "Visit ID", 16 | "mode": "NULLABLE", 17 | "name": "visitId", 18 | "type": "INTEGER" 19 | }, 20 | { 21 | "description": "Visit Start Time", 22 | "mode": "NULLABLE", 23 | "name": "visitStartTime", 24 | "type": "INTEGER" 25 | }, 26 | { 27 | "description": "Full Date of Visit", 28 | "mode": "NULLABLE", 29 | "name": "fullDate", 30 | "type": "DATE" 31 | } 32 | ] 33 | -------------------------------------------------------------------------------- /test/integration/discover_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package test 16 | 17 | import ( 18 | "testing" 19 | 20 | "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" 21 | ) 22 | 23 | func TestAll(t *testing.T) { 24 | tft.AutoDiscoverAndTest(t) 25 | } 26 | -------------------------------------------------------------------------------- /test/setup/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">= 1.3" 19 | required_providers { 20 | google = { 21 | source = "hashicorp/google" 22 | } 23 | google-beta = { 24 | source = "hashicorp/google-beta" 25 | } 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /modules/scheduled_queries/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "query_names" { 18 | value = concat( 19 | values({ for k, v in google_bigquery_data_transfer_config.query_config : k => v.name }), 20 | ) 21 | 22 | description = "The resource names of the transfer config" 23 | } 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # OSX leaves these everywhere on SMB shares 2 | ._* 3 | 4 | # OSX trash 5 | .DS_Store 6 | *.pyc* 7 | 8 | # Emacs save files 9 | *~ 10 | \#*\# 11 | .\#* 12 | .idea 13 | # Vim-related files 14 | [._]*.s[a-w][a-z] 15 | [._]s[a-w][a-z] 16 | *.un~ 17 | Session.vim 18 | .netrwhist 19 | 20 | ### https://raw.github.com/github/gitignore/90f149de451a5433aebd94d02d11b0e28843a1af/Terraform.gitignore 21 | 22 | # Local .terraform directories 23 | **/.terraform/* 24 | 25 | # Local .kitchen directories 26 | **/.kitchen/* 27 | 28 | # .tfstate files 29 | *.tfstate 30 | *.tfstate.* 31 | 32 | # Crash log files 33 | crash.log 34 | 35 | # Ignore any .tfvars files that are generated automatically for each Terraform run. Most 36 | # .tfvars files are managed as part of configuration and so should be included in 37 | # version control. 38 | # 39 | # example.tfvars 40 | credentials.json 41 | 42 | # tf lock file 43 | .terraform.lock.hcl 44 | -------------------------------------------------------------------------------- /test/setup/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "project_id" { 18 | value = module.project.project_id 19 | } 20 | 21 | output "sa_key" { 22 | value = google_service_account_key.int_test.private_key 23 | sensitive = true 24 | } 25 | 26 | output "kms_keys" { 27 | value = module.kms_keyring.keys 28 | } 29 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/schema/products_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "id", 5 | "type": "INTEGER" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "cost", 10 | "type": "FLOAT" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "category", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "name", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "brand", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "retail_price", 30 | "type": "FLOAT" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "department", 35 | "type": "STRING" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "sku", 40 | "type": "STRING" 41 | }, 42 | { 43 | "mode": "NULLABLE", 44 | "name": "distribution_center_id", 45 | "type": "INTEGER" 46 | } 47 | ] 48 | -------------------------------------------------------------------------------- /modules/scheduled_queries/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "The project where scheduled queries are created" 19 | type = string 20 | } 21 | 22 | variable "queries" { 23 | description = "Data transfer configuration for creating scheduled queries" 24 | type = list(any) 25 | } 26 | -------------------------------------------------------------------------------- /examples/data_warehouse/README.md: -------------------------------------------------------------------------------- 1 | # Simple Example 2 | 3 | This example illustrates how to use the `data_warehouse` module. 4 | 5 | 6 | ## Inputs 7 | 8 | | Name | Description | Type | Default | Required | 9 | |------|-------------|------|---------|:--------:| 10 | | project\_id | The ID of the project in which to provision resources. | `string` | n/a | yes | 11 | 12 | ## Outputs 13 | 14 | | Name | Description | 15 | |------|-------------| 16 | | bigquery\_editor\_url | BQ editor URL | 17 | | lookerstudio\_report\_url | Looker Studio URL | 18 | | raw\_bucket | Raw bucket name | 19 | 20 | 21 | 22 | To provision this example, run the following from within this directory: 23 | - `terraform init` to get the plugins 24 | - `terraform plan` to see the infrastructure plan 25 | - `terraform apply` to apply the infrastructure build 26 | - `terraform destroy` to destroy the built infrastructure 27 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/schema/orders_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "order_id", 5 | "type": "INTEGER" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "user_id", 10 | "type": "INTEGER" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "status", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "gender", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "created_at", 25 | "type": "TIMESTAMP" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "returned_at", 30 | "type": "TIMESTAMP" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "shipped_at", 35 | "type": "TIMESTAMP" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "delivered_at", 40 | "type": "TIMESTAMP" 41 | }, 42 | { 43 | "mode": "NULLABLE", 44 | "name": "num_of_item", 45 | "type": "INTEGER" 46 | } 47 | ] 48 | -------------------------------------------------------------------------------- /test/setup/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | variable "org_id" { 17 | description = "The numeric organization id" 18 | } 19 | 20 | variable "folder_id" { 21 | description = "The folder to deploy in" 22 | } 23 | 24 | variable "billing_account" { 25 | description = "The billing account id associated with the project, e.g. XXXXXX-YYYYYY-ZZZZZZ" 26 | } 27 | -------------------------------------------------------------------------------- /examples/basic_bq/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "bigquery_dataset" { 18 | value = module.bigquery.bigquery_dataset 19 | description = "Bigquery dataset resource." 20 | } 21 | 22 | output "bigquery_tables" { 23 | value = module.bigquery.bigquery_tables 24 | description = "Map of bigquery table resources being provisioned." 25 | } 26 | -------------------------------------------------------------------------------- /examples/data_warehouse/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "data_warehouse" { 18 | source = "terraform-google-modules/bigquery/google//modules/data_warehouse" 19 | version = "~> 9.0" 20 | 21 | project_id = var.project_id 22 | region = "us-central1" 23 | deletion_protection = false 24 | force_destroy = true 25 | } 26 | -------------------------------------------------------------------------------- /versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">= 1.3" 19 | required_providers { 20 | 21 | google = { 22 | source = "hashicorp/google" 23 | version = ">= 5.39, < 8" 24 | } 25 | } 26 | 27 | provider_meta "google" { 28 | module_name = "blueprints/terraform/terraform-google-bigquery/v10.2.1" 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /modules/udf/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">= 1.3" 19 | required_providers { 20 | 21 | google = { 22 | source = "hashicorp/google" 23 | version = ">= 3.53, < 8" 24 | } 25 | } 26 | 27 | provider_meta "google" { 28 | module_name = "blueprints/terraform/terraform-google-bigquery:udf/v10.2.1" 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /test/fixtures/full/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "example" { 18 | source = "../../../examples/multiple_tables" 19 | default_table_expiration_ms = var.default_table_expiration_ms 20 | project_id = var.project_id 21 | dataset_labels = var.dataset_labels 22 | kms_key = jsondecode(var.kms_keys)["foo"] 23 | } 24 | -------------------------------------------------------------------------------- /modules/authorization/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">= 1.3" 19 | required_providers { 20 | 21 | google = { 22 | source = "hashicorp/google" 23 | version = ">= 4.44, < 8" 24 | } 25 | } 26 | 27 | provider_meta "google" { 28 | module_name = "blueprints/terraform/terraform-google-bigquery:authorization/v10.2.1" 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /modules/scheduled_queries/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_version = ">= 1.3" 19 | required_providers { 20 | 21 | google = { 22 | source = "hashicorp/google" 23 | version = ">= 4.0, < 8" 24 | } 25 | } 26 | 27 | provider_meta "google" { 28 | module_name = "blueprints/terraform/terraform-google-bigquery:scheduled_queries/v10.2.1" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /.github/trusted-contribution.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023-2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # NOTE: This file is automatically generated from: 16 | # https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/blob/main/infra/terraform/test-org/github 17 | 18 | annotations: 19 | - type: comment 20 | text: "/gcbrun" 21 | trustedContributors: 22 | - release-please[bot] 23 | - renovate[bot] 24 | - renovate-bot 25 | - forking-renovate[bot] 26 | - dependabot[bot] 27 | -------------------------------------------------------------------------------- /modules/udf/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "dataset_id" { 18 | description = "Dataset id" 19 | type = string 20 | } 21 | 22 | variable "project_id" { 23 | description = "Project ID that contains the dataset" 24 | type = string 25 | } 26 | 27 | variable "add_udfs" { 28 | description = "Whether or not this module should be enabled." 29 | type = string 30 | default = false 31 | } 32 | -------------------------------------------------------------------------------- /test/integration/full/inspec.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | name: local 16 | depends: 17 | - name: inspec-gcp 18 | git: https://github.com/inspec/inspec-gcp.git 19 | tag: v1.10.0 20 | supports: 21 | - platform: gcp 22 | attributes: 23 | - name: bigquery_dataset 24 | required: true 25 | type: hash 26 | - name: bigquery_tables 27 | required: true 28 | type: hash 29 | - name: bigquery_external_tables 30 | required: true 31 | type: hash 32 | -------------------------------------------------------------------------------- /examples/basic_bq/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "Project where the dataset and table are created." 19 | } 20 | 21 | variable "delete_contents_on_destroy" { 22 | description = "(Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present." 23 | type = bool 24 | default = null 25 | } 26 | -------------------------------------------------------------------------------- /docs/upgrading_to_bigquery_v8.0.md: -------------------------------------------------------------------------------- 1 | # Upgrading to BigQuery v8.0 2 | 3 | The v8.0 release of BigQuery is a backwards incompatible release. 4 | 5 | - The supported provider has been updated to v5.3 6 | - `require_partition_filter` has been deprecated under the `time_partitioning` 7 | block and can be used at the top level with the same name instead. 8 | 9 | ## Migration Instructions 10 | 11 | 1. Upgrade version 12 | 13 | ```diff 14 | module "bigquery" { 15 | source = "terraform-google-modules/bigquery/google" 16 | - version = "~> 7.0" 17 | + version = "~> 8.0" 18 | .... 19 | } 20 | ``` 21 | 22 | 2. Remove `require_partition_filter` from within the `time_partitioning` block 23 | and set it at the top level, if required. 24 | 25 | ```diff 26 | module "bigquery" { 27 | source = "terraform-google-modules/bigquery/google" 28 | .... 29 | tables = [ 30 | { 31 | .... 32 | + require_partition_filter = true, 33 | time_partitioning = { 34 | .... 35 | - require_partition_filter = true, 36 | .... 37 | }, 38 | .... 39 | } 40 | ] 41 | ... 42 | } 43 | ``` 44 | -------------------------------------------------------------------------------- /examples/data_warehouse/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2021 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "lookerstudio_report_url" { 18 | description = "Looker Studio URL" 19 | value = module.data_warehouse.lookerstudio_report_url 20 | } 21 | 22 | output "bigquery_editor_url" { 23 | description = "BQ editor URL" 24 | value = module.data_warehouse.bigquery_editor_url 25 | } 26 | 27 | output "raw_bucket" { 28 | description = "Raw bucket name" 29 | value = module.data_warehouse.raw_bucket 30 | } 31 | -------------------------------------------------------------------------------- /modules/data_warehouse/workflow_polling/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "input_workflow_state" { 18 | type = string 19 | description = "Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation" 20 | } 21 | 22 | variable "workflow_id" { 23 | type = string 24 | description = "The identifer of a workflow created by Terraform. Format is projects/{project ID}/locations/{region}/workflows/{workflow name}" 25 | } 26 | 27 | 28 | -------------------------------------------------------------------------------- /.kitchen.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | --- 16 | provisioner: 17 | name: terraform 18 | 19 | platforms: 20 | - name: local 21 | 22 | verifier: 23 | name: terraform 24 | systems: 25 | - name: system 26 | backend: gcp 27 | 28 | suites: 29 | - name: full 30 | driver: 31 | name: terraform 32 | command_timeout: 1800 33 | root_module_directory: test/fixtures/full 34 | # setting version verification to false since it requires TF to be less than v1.1 35 | verify_version: false 36 | -------------------------------------------------------------------------------- /examples/multiple_tables/provider.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | provider "google" { 18 | scopes = [ 19 | # To configure an external table with a Google Sheet you must pass this scope 20 | # see: https://registry.terraform.io/providers/hashicorp/google/latest/docs/resources/bigquery_table#source_format 21 | "https://www.googleapis.com/auth/drive.readonly", 22 | # Because we are using scopes, you must also give it the BigQuery scope 23 | "https://www.googleapis.com/auth/bigquery", 24 | ] 25 | } 26 | -------------------------------------------------------------------------------- /test/fixtures/full/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "default_table_expiration_ms" { 18 | description = "Default TTL of tables using the dataset in MS" 19 | } 20 | 21 | variable "project_id" { 22 | description = "Project where the dataset and table are created" 23 | } 24 | 25 | variable "kms_keys" { 26 | description = "The KMS key module output" 27 | default = null 28 | } 29 | 30 | variable "dataset_labels" { 31 | description = "Key value pairs in a map for dataset labels" 32 | type = map(string) 33 | } 34 | -------------------------------------------------------------------------------- /modules/authorization/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "authorized_views" { 18 | value = google_bigquery_dataset_access.authorized_view 19 | description = "Authorized views for the dataset" 20 | } 21 | 22 | output "authorized_roles" { 23 | value = google_bigquery_dataset_access.access_role 24 | description = "Authorized roles for the dataset" 25 | } 26 | 27 | output "authorized_dataset" { 28 | value = google_bigquery_dataset_access.authorized_dataset 29 | description = "Authorized datasets for the BQ dataset" 30 | } 31 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/schema/order_items_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "id", 5 | "type": "INTEGER" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "order_id", 10 | "type": "INTEGER" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "user_id", 15 | "type": "INTEGER" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "product_id", 20 | "type": "INTEGER" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "inventory_item_id", 25 | "type": "INTEGER" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "status", 30 | "type": "STRING" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "created_at", 35 | "type": "TIMESTAMP" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "shipped_at", 40 | "type": "TIMESTAMP" 41 | }, 42 | { 43 | "mode": "NULLABLE", 44 | "name": "delivered_at", 45 | "type": "TIMESTAMP" 46 | }, 47 | { 48 | "mode": "NULLABLE", 49 | "name": "returned_at", 50 | "type": "TIMESTAMP" 51 | }, 52 | { 53 | "mode": "NULLABLE", 54 | "name": "sale_price", 55 | "type": "FLOAT" 56 | } 57 | ] 58 | -------------------------------------------------------------------------------- /examples/basic_bq/README.md: -------------------------------------------------------------------------------- 1 | # terraform-google-bigquery basic_bq 2 | The basic_bq example uses the root terraform-google-bigquery module to deploy a single dataset and a table with a basic schema. This example is a good reference to understand and test the module usage. 3 | 4 | 5 | ## Inputs 6 | 7 | | Name | Description | Type | Default | Required | 8 | |------|-------------|------|---------|:--------:| 9 | | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no | 10 | | project\_id | Project where the dataset and table are created. | `any` | n/a | yes | 11 | 12 | ## Outputs 13 | 14 | | Name | Description | 15 | |------|-------------| 16 | | bigquery\_dataset | Bigquery dataset resource. | 17 | | bigquery\_tables | Map of bigquery table resources being provisioned. | 18 | 19 | 20 | 21 | ## Setup 22 | Update the contents of `terraform.tfvars` to match your test environment. 23 | 24 | ## Run example 25 | `terraform init` 26 | `terraform plan` 27 | `terraform apply -var-file terraform.tfvars` 28 | -------------------------------------------------------------------------------- /examples/scheduled_queries/README.md: -------------------------------------------------------------------------------- 1 | # terraform-google-bigquery scheduled_queries 2 | 3 | This example illustrates how to create scheduled queries using `scheduled_queries` module. 4 | 5 | ## Inputs 6 | 7 | | Name | Description | Type | Default | Required | 8 | |------|-------------|------|---------|:--------:| 9 | | project\_id | The project where scheduled queries are created | `string` | n/a | yes | 10 | | queries | Data transfer configuration for creating scheduled queries | `list(any)` | n/a | yes | 11 | 12 | ## Outputs 13 | 14 | | Name | Description | 15 | |------|-------------| 16 | | query\_names | The resource names of the transfer config | 17 | 18 | ## Usage 19 | 20 | Run the following commands within this directory: 21 | - `terraform init` to get the plugins 22 | - `terraform plan` to see the infrastructure plan 23 | - `terraform apply` to apply the infrastructure build 24 | - `terraform destroy` to destroy the built infrastructure 25 | 26 | 27 | ## Inputs 28 | 29 | No inputs. 30 | 31 | ## Outputs 32 | 33 | | Name | Description | 34 | |------|-------------| 35 | | query\_names | The resource names of the transfer config | 36 | 37 | 38 | -------------------------------------------------------------------------------- /examples/basic_view/terraform.tfvars: -------------------------------------------------------------------------------- 1 | delete_contents_on_destroy = true 2 | table_project_id = "example-table-project" 3 | table_dataset_labels = { 4 | env = "dev" 5 | billable = "true" 6 | owner = "janesmith" 7 | } 8 | view_project_id = "example-view-project" 9 | view_dataset_labels = { 10 | env = "dev" 11 | billable = "true" 12 | owner = "janesmith" 13 | } 14 | tables = [ 15 | { 16 | table_id = "bar", 17 | schema = "sample_bq_schema.json", 18 | time_partitioning = null, 19 | range_partitioning = null, 20 | expiration_time = 2524604400000, # 2050/01/01 21 | clustering = [], 22 | labels = { 23 | env = "devops" 24 | billable = "true" 25 | owner = "joedoe" 26 | }, 27 | } 28 | ] 29 | 30 | views = [ 31 | { 32 | view_id = "bar", 33 | use_legacy_sql = false, 34 | query = < 4.0" 11 | 12 | dataset_id = "example_dataset" 13 | dataset_name = "example_dataset" 14 | description = "example description" 15 | project_id = "example-project" 16 | location = "US" 17 | } 18 | 19 | module "add_udfs" { 20 | source = "terraform-google-modules/bigquery/google//modules/udf" 21 | version = "~> 10.2" 22 | 23 | dataset_id = module.dataset.bigquery_dataset.dataset_id 24 | project_id = module.dataset.bigquery_dataset.project 25 | } 26 | ``` 27 | 28 | 29 | ## Inputs 30 | 31 | | Name | Description | Type | Default | Required | 32 | |------|-------------|------|---------|:--------:| 33 | | add\_udfs | Whether or not this module should be enabled. | `string` | `false` | no | 34 | | dataset\_id | Dataset id | `string` | n/a | yes | 35 | | project\_id | Project ID that contains the dataset | `string` | n/a | yes | 36 | 37 | ## Outputs 38 | 39 | | Name | Description | 40 | |------|-------------| 41 | | added\_udfs | List of UDFs utility functions added. | 42 | 43 | 44 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/schema/users_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "id", 5 | "type": "INTEGER" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "first_name", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "last_name", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "email", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "age", 25 | "type": "INTEGER" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "gender", 30 | "type": "STRING" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "state", 35 | "type": "STRING" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "street_address", 40 | "type": "STRING" 41 | }, 42 | { 43 | "mode": "NULLABLE", 44 | "name": "postal_code", 45 | "type": "STRING" 46 | }, 47 | { 48 | "mode": "NULLABLE", 49 | "name": "city", 50 | "type": "STRING" 51 | }, 52 | { 53 | "mode": "NULLABLE", 54 | "name": "country", 55 | "type": "STRING" 56 | }, 57 | { 58 | "mode": "NULLABLE", 59 | "name": "latitude", 60 | "type": "FLOAT" 61 | }, 62 | { 63 | "mode": "NULLABLE", 64 | "name": "longitude", 65 | "type": "FLOAT" 66 | }, 67 | { 68 | "mode": "NULLABLE", 69 | "name": "traffic_source", 70 | "type": "STRING" 71 | }, 72 | { 73 | "mode": "NULLABLE", 74 | "name": "created_at", 75 | "type": "TIMESTAMP" 76 | } 77 | ] 78 | -------------------------------------------------------------------------------- /examples/multiple_tables/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "bigquery_dataset" { 18 | value = module.bigquery.bigquery_dataset 19 | description = "Bigquery dataset resource." 20 | } 21 | 22 | output "bigquery_tables" { 23 | value = module.bigquery.bigquery_tables 24 | description = "Map of bigquery table resources being provisioned." 25 | } 26 | 27 | output "bigquery_external_tables" { 28 | value = module.bigquery.bigquery_external_tables 29 | description = "Map of bigquery table resources being provisioned." 30 | } 31 | 32 | output "bigquery_auth_dataset" { 33 | value = module.auth_dataset.bigquery_dataset 34 | description = "Authorized Bigquery dataset resource." 35 | } 36 | 37 | output "authorization" { 38 | value = module.add_authorization.authorized_dataset["${module.auth_dataset.bigquery_dataset.project}_${module.auth_dataset.bigquery_dataset.dataset_id}"] 39 | description = "Authorization Bigquery dataset resource." 40 | } 41 | -------------------------------------------------------------------------------- /modules/data_warehouse/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | version = ">= 6.11, < 8" 22 | } 23 | google-beta = { 24 | source = "hashicorp/google-beta" 25 | version = ">= 6.11, < 8" 26 | } 27 | archive = { 28 | source = "hashicorp/archive" 29 | version = ">= 2.4.2" 30 | } 31 | time = { 32 | source = "hashicorp/time" 33 | version = ">= 0.9.1" 34 | } 35 | http = { 36 | source = "hashicorp/http" 37 | version = ">= 2" 38 | } 39 | local = { 40 | source = "hashicorp/local" 41 | version = ">= 2.4" 42 | } 43 | random = { 44 | source = "hashicorp/random" 45 | version = ">= 3.6.2" 46 | } 47 | } 48 | required_version = ">= 1.3" 49 | 50 | provider_meta "google" { 51 | module_name = "blueprints/terraform/terraform-google-bigquery:data_warehouse/v10.2.1" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /modules/data_warehouse/workflow_polling/versions.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | source = "hashicorp/google" 21 | version = ">= 4.52, < 8" 22 | } 23 | google-beta = { 24 | source = "hashicorp/google-beta" 25 | version = ">= 4.52, < 8" 26 | } 27 | archive = { 28 | source = "hashicorp/archive" 29 | version = ">= 2" 30 | } 31 | random = { 32 | source = "hashicorp/random" 33 | version = ">= 2" 34 | } 35 | time = { 36 | source = "hashicorp/time" 37 | version = ">= 0.9.1" 38 | } 39 | http = { 40 | source = "hashicorp/http" 41 | version = ">= 2" 42 | } 43 | local = { 44 | source = "hashicorp/local" 45 | version = ">=2.4" 46 | } 47 | } 48 | required_version = ">= 1.3" 49 | 50 | provider_meta "google" { 51 | module_name = "blueprints/terraform/terraform-google-bigquery:data_warehouse/v10.2.1" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /examples/basic_bq/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "bigquery" { 18 | source = "terraform-google-modules/bigquery/google" 19 | version = "~> 9.0" 20 | 21 | dataset_id = "foo" 22 | dataset_name = "foo" 23 | description = "some description" 24 | project_id = var.project_id 25 | location = "US" 26 | delete_contents_on_destroy = var.delete_contents_on_destroy 27 | tables = [ 28 | { 29 | table_id = "bar", 30 | schema = file("sample_bq_schema.json"), 31 | time_partitioning = null, 32 | range_partitioning = null, 33 | expiration_time = 2524604400000, # 2050/01/01 34 | clustering = [], 35 | labels = { 36 | env = "devops" 37 | billable = "true" 38 | owner = "joedoe" 39 | }, 40 | } 41 | ] 42 | dataset_labels = { 43 | env = "dev" 44 | billable = "true" 45 | owner = "janesmith" 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /examples/basic_view/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "bigquery_dataset" { 18 | value = module.bigquery_tables.bigquery_dataset 19 | description = "Bigquery dataset resource." 20 | } 21 | 22 | output "bigquery_tables" { 23 | value = module.bigquery_tables.bigquery_tables 24 | description = "Map of bigquery table resources being provisioned." 25 | } 26 | 27 | output "bigquery_dataset_view" { 28 | value = module.bigquery_views_without_pii.bigquery_dataset 29 | description = "Bigquery dataset resource." 30 | } 31 | 32 | output "bigquery_views" { 33 | value = module.bigquery_views_without_pii.bigquery_views 34 | description = "Map of bigquery table/view resources being provisioned." 35 | } 36 | 37 | output "authorized_views" { 38 | value = module.authorization.authorized_views 39 | description = "Map of authorized views created" 40 | } 41 | 42 | output "access_roles" { 43 | value = module.authorization.authorized_roles 44 | description = "Map of roles assigned to identities" 45 | } 46 | -------------------------------------------------------------------------------- /examples/scheduled_queries/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "dataset" { 18 | source = "terraform-google-modules/bigquery/google" 19 | version = "~> 9.0" 20 | 21 | dataset_id = "example_dataset" 22 | dataset_name = "example_dataset" 23 | description = "Example description" 24 | project_id = "example-project" 25 | location = "EU" 26 | } 27 | 28 | module "scheduled_queries" { 29 | source = "terraform-google-modules/bigquery/google//modules/scheduled_queries" 30 | version = "~> 9.0" 31 | 32 | project_id = module.dataset.project 33 | 34 | queries = [ 35 | { 36 | name = "my-query" 37 | location = "EU" 38 | data_source_id = "scheduled_query" 39 | destination_dataset_id = module.dataset.bigquery_dataset.dataset_id 40 | params = { 41 | destination_table_name_template = "my_table" 42 | write_disposition = "WRITE_APPEND" 43 | query = "SELECT name FROM tabl WHERE x = 'y'" 44 | } 45 | } 46 | ] 47 | } 48 | -------------------------------------------------------------------------------- /modules/authorization/metadata.display.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: blueprints.cloud.google.com/v1alpha1 16 | kind: BlueprintMetadata 17 | metadata: 18 | name: terraform-google-bigquery-authorization-display 19 | annotations: 20 | config.kubernetes.io/local-config: "true" 21 | spec: 22 | info: 23 | title: BigQuery Authorized Datasets, Views and Routines 24 | source: 25 | repo: https://github.com/terraform-google-modules/terraform-google-bigquery 26 | sourceType: git 27 | dir: /modules/authorization 28 | ui: 29 | input: 30 | variables: 31 | authorized_datasets: 32 | name: authorized_datasets 33 | title: Authorized Datasets 34 | authorized_routines: 35 | name: authorized_routines 36 | title: Authorized Routines 37 | authorized_views: 38 | name: authorized_views 39 | title: Authorized Views 40 | dataset_id: 41 | name: dataset_id 42 | title: Dataset Id 43 | project_id: 44 | name: project_id 45 | title: Project Id 46 | roles: 47 | name: roles 48 | title: Roles 49 | -------------------------------------------------------------------------------- /.github/workflows/stale.yml: -------------------------------------------------------------------------------- 1 | # Copyright 2022-2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # NOTE: This file is automatically generated from: 16 | # https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/blob/main/infra/terraform/test-org/github 17 | 18 | name: "Close stale issues" 19 | on: 20 | schedule: 21 | - cron: "0 23 * * *" 22 | 23 | permissions: 24 | contents: read 25 | issues: write 26 | pull-requests: write 27 | actions: write 28 | 29 | jobs: 30 | stale: 31 | if: github.repository_owner == 'GoogleCloudPlatform' || github.repository_owner == 'terraform-google-modules' 32 | runs-on: ubuntu-latest 33 | steps: 34 | - uses: actions/stale@v10 35 | with: 36 | repo-token: ${{ secrets.GITHUB_TOKEN }} 37 | stale-issue-message: 'This issue is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days' 38 | stale-pr-message: 'This PR is stale because it has been open 60 days with no activity. Remove stale label or comment or this will be closed in 7 days' 39 | exempt-issue-labels: 'triaged' 40 | exempt-pr-labels: 'dependencies,autorelease: pending' 41 | operations-per-run: 100 42 | -------------------------------------------------------------------------------- /examples/multiple_tables/README.md: -------------------------------------------------------------------------------- 1 | # terraform-google-bigquery multiple_tables 2 | The multiple_tables example uses the root terraform-google-bigquery module 3 | to deploy a single dataset and two tables with basic schemas. 4 | This example is a good reference to understand and test the module usage. 5 | 6 | 7 | ## Inputs 8 | 9 | | Name | Description | Type | Default | Required | 10 | |------|-------------|------|---------|:--------:| 11 | | dataset\_labels | A mapping of labels to assign to the table. | `map(string)` | n/a | yes | 12 | | default\_table\_expiration\_ms | Default TTL of tables using the dataset in MS. | `any` | `null` | no | 13 | | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no | 14 | | kms\_key | The KMS key to use to encrypt data by default | `string` | `null` | no | 15 | | project\_id | Project where the dataset and table are created. | `any` | n/a | yes | 16 | 17 | ## Outputs 18 | 19 | | Name | Description | 20 | |------|-------------| 21 | | authorization | Authorization Bigquery dataset resource. | 22 | | bigquery\_auth\_dataset | Authorized Bigquery dataset resource. | 23 | | bigquery\_dataset | Bigquery dataset resource. | 24 | | bigquery\_external\_tables | Map of bigquery table resources being provisioned. | 25 | | bigquery\_tables | Map of bigquery table resources being provisioned. | 26 | 27 | 28 | 29 | ## Setup 30 | Update the contents of `terraform.tfvars` to match your test environment. 31 | 32 | ## Run example 33 | `terraform init` 34 | `terraform plan` 35 | `terraform apply -var-file terraform.tfvars` 36 | -------------------------------------------------------------------------------- /modules/scheduled_queries/README.md: -------------------------------------------------------------------------------- 1 | # BigQuery Scheduled Queries 2 | 3 | This submodule is used to create [scheduled queries](https://cloud.google.com/bigquery/docs/scheduling-queries) that can be run on a recurring basis. 4 | 5 | 6 | Example: 7 | ```hcl 8 | module "dataset" { 9 | source = "terraform-google-modules/bigquery/google" 10 | version = "~> 10.2" 11 | 12 | dataset_id = "example_dataset" 13 | dataset_name = "example_dataset" 14 | description = "example description" 15 | project_id = "example-project" 16 | location = "EU" 17 | } 18 | 19 | module "scheduled_queries" { 20 | source = "terraform-google-modules/bigquery/google//modules/scheduled_queries" 21 | version = "~> 5.3.0" 22 | 23 | project_id = module.dataset.bigquery_dataset.project_id 24 | 25 | queries = [ 26 | { 27 | name = "my-query" 28 | location = "EU" 29 | data_source_id = "scheduled_query" 30 | destination_dataset_id = module.dataset.bigquery_dataset.dataset_id 31 | params = { 32 | destination_table_name_template = "my_table" 33 | write_disposition = "WRITE_APPEND" 34 | query = "SELECT name FROM tabl WHERE x = 'y'" 35 | } 36 | } 37 | ] 38 | } 39 | ``` 40 | 41 | 42 | ## Inputs 43 | 44 | | Name | Description | Type | Default | Required | 45 | |------|-------------|------|---------|:--------:| 46 | | project\_id | The project where scheduled queries are created | `string` | n/a | yes | 47 | | queries | Data transfer configuration for creating scheduled queries | `list(any)` | n/a | yes | 48 | 49 | ## Outputs 50 | 51 | | Name | Description | 52 | |------|-------------| 53 | | query\_names | The resource names of the transfer config | 54 | 55 | 56 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/sql/sp_provision_lookup_tables.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | CREATE OR REPLACE TABLE `${project_id}.${dataset_id}.distribution_centers` 16 | 17 | ( 18 | id INTEGER, 19 | name STRING, 20 | longitude FLOAT64, 21 | latitude FLOAT64, 22 | distribution_center_geom GEOGRAPHY 23 | ) 24 | OPTIONS( 25 | labels=[("data-warehouse","true")] 26 | ) 27 | AS 28 | SELECT 1, 'Memphis TN', -89.9711, 35.1174, ST_GEOGPOINT(-89.9711, 35.1174) 29 | UNION ALL 30 | SELECT 2, 'Chicago IL', -87.6847, 41.8369, ST_GEOGPOINT(-87.6847, 41.8369) 31 | UNION ALL 32 | SELECT 3, 'Houston TX', -95.3698, 29.7604, ST_GEOGPOINT(-95.3698, 29.7604) 33 | UNION ALL 34 | SELECT 4, 'Los Angeles CA', -118.25, 34.05, ST_GEOGPOINT(-118.25, 34.05) 35 | UNION ALL 36 | SELECT 5, 'New Orleans LA', -90.0667, 29.95, ST_GEOGPOINT(-90.0667, 29.95) 37 | UNION ALL 38 | SELECT 6, 'Port Authority of New York/New Jersey NY/NJ', -73.7834, 40.634, ST_GEOGPOINT(-73.7834, 40.634) 39 | UNION ALL 40 | SELECT 7, 'Philadelphia PA', -75.1667, 39.95, ST_GEOGPOINT(-75.1667, 39.95) 41 | UNION ALL 42 | SELECT 8, 'Mobile AL', -88.0431, 30.6944, ST_GEOGPOINT(-88.0431, 30.6944) 43 | UNION ALL 44 | SELECT 9, 'Charleston SC', -79.9333, 32.7833, ST_GEOGPOINT(-79.9333, 32.7833) 45 | UNION ALL 46 | SELECT 10, 'Savannah GA', -81.1167, 32.0167, ST_GEOGPOINT(-81.1167, 32.0167) 47 | ; 48 | -------------------------------------------------------------------------------- /modules/authorization/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "dataset_id" { 18 | description = "Unique ID for the dataset being provisioned." 19 | type = string 20 | } 21 | 22 | variable "project_id" { 23 | description = "Project where the dataset and table are created" 24 | type = string 25 | } 26 | 27 | # Format: list(objects) 28 | # domain: A domain to grant access to. 29 | # group_by_email: An email address of a Google Group to grant access to. 30 | # user_by_email: An email address of a user to grant access to. 31 | # group_by_email: An email address of a Google Group to grant access to. 32 | # special_group: A special group to grant access to. 33 | variable "roles" { 34 | description = "An array of objects that define dataset access for one or more entities." 35 | type = any 36 | default = [] 37 | } 38 | 39 | variable "authorized_views" { 40 | description = "An array of views to give authorize for the dataset" 41 | type = list(object({ 42 | dataset_id = string, 43 | project_id = string, 44 | table_id = string # this is the view id, but we keep table_id to stay consistent as the resource 45 | })) 46 | default = [] 47 | } 48 | 49 | variable "authorized_datasets" { 50 | description = "An array of datasets to be authorized on the dataset" 51 | type = list(object({ 52 | dataset_id = string, 53 | project_id = string, 54 | })) 55 | default = [] 56 | } 57 | 58 | variable "authorized_routines" { 59 | description = "An array of authorized routine to be authorized on the dataset" 60 | type = list(object({ 61 | project_id = string, 62 | dataset_id = string, 63 | routine_id = string 64 | })) 65 | default = [] 66 | } 67 | -------------------------------------------------------------------------------- /helpers/setup-sa.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Copyright 2022 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | 17 | set -e 18 | set -u 19 | 20 | # check for input variables 21 | if [ $# -ne 2 ]; then 22 | echo 23 | echo "Usage: $0 " 24 | echo 25 | exit 1 26 | fi 27 | 28 | # Organization ID 29 | ORG_ID="$(gcloud organizations list --format="value(ID)" --filter="$1")" 30 | 31 | if [[ $ORG_ID == "" ]]; 32 | then 33 | echo "The organization id provided does not exist. Exiting." 34 | exit 1; 35 | fi 36 | 37 | # Host project 38 | HOST_PROJECT="$(gcloud projects list --format="value(projectId)" --filter="$2")" 39 | 40 | if [[ $HOST_PROJECT == "" ]]; 41 | then 42 | echo "The host project does not exist. Exiting." 43 | exit 1; 44 | fi 45 | 46 | # Service Account creation 47 | SA_NAME="bq-${RANDOM}" 48 | SA_ID="${SA_NAME}@${HOST_PROJECT}.iam.gserviceaccount.com" 49 | STAGING_DIR="${PWD}" 50 | KEY_FILE="${STAGING_DIR}/credentials.json" 51 | 52 | gcloud iam service-accounts \ 53 | --project "${HOST_PROJECT}" create "${SA_NAME}" \ 54 | --display-name "${SA_NAME}" 55 | 56 | echo "Downloading key to credentials.json..." 57 | 58 | gcloud iam service-accounts keys create "${KEY_FILE}" \ 59 | --iam-account "${SA_ID}" \ 60 | --user-output-enabled false 61 | 62 | echo "Applying permissions for org $ORG_ID and project $HOST_PROJECT..." 63 | 64 | # Grant roles/resourcemanager.projectIamAdmin to the service account on the host project 65 | gcloud projects add-iam-policy-binding \ 66 | "${HOST_PROJECT}" \ 67 | --member="serviceAccount:${SA_ID}" \ 68 | --role="roles/bigquery.dataOwner" \ 69 | --user-output-enabled false 70 | 71 | # Enable required API's 72 | gcloud services enable \ 73 | bigquery-json.googleapis.com \ 74 | --project "${HOST_PROJECT}" 75 | 76 | 77 | echo "All done." 78 | -------------------------------------------------------------------------------- /modules/data_warehouse/outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "ds_friendly_name" { 18 | value = google_bigquery_dataset.ds_edw.friendly_name 19 | description = "Dataset name" 20 | } 21 | 22 | output "raw_bucket" { 23 | value = google_storage_bucket.raw_bucket.name 24 | description = "Raw bucket name" 25 | } 26 | 27 | output "lookerstudio_report_url" { 28 | value = "https://lookerstudio.google.com/reporting/create?c.reportId=8a6517b8-8fcd-47a2-a953-9d4fb9ae4794&ds.ds_profit.datasourceName=lookerstudio_report_profit&ds.ds_profit.projectId=${module.project-services.project_id}&ds.ds_profit.type=TABLE&ds.ds_profit.datasetId=${google_bigquery_dataset.ds_edw.dataset_id}&ds.ds_profit.tableId=lookerstudio_report_profit&ds.ds_dc.datasourceName=lookerstudio_report_distribution_centers&ds.ds_dc.projectId=${module.project-services.project_id}&ds.ds_dc.type=TABLE&ds.ds_dc.datasetId=${google_bigquery_dataset.ds_edw.dataset_id}&ds.ds_dc.tableId=lookerstudio_report_distribution_centers" 29 | description = "The URL to create a new Looker Studio report displays a sample dashboard for the e-commerce data analysis" 30 | } 31 | 32 | output "bigquery_editor_url" { 33 | value = "https://console.cloud.google.com/bigquery?project=${module.project-services.project_id}&ws=!1m5!1m4!6m3!1s${module.project-services.project_id}!2s${google_bigquery_dataset.ds_edw.dataset_id}!3ssp_sample_queries" 34 | description = "The URL to launch the BigQuery editor with the sample query procedure opened" 35 | } 36 | 37 | output "neos_tutorial_url" { 38 | value = "https://console.cloud.google.com/products/solutions/deployments?walkthrough_id=panels--sic--data-warehouse_toc" 39 | description = "The URL to launch the in-console tutorial for the EDW solution" 40 | } 41 | -------------------------------------------------------------------------------- /modules/data_warehouse/metadata.display.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: blueprints.cloud.google.com/v1alpha1 16 | kind: BlueprintMetadata 17 | metadata: 18 | name: terraform-google-bigquery-data-warehouse-display 19 | annotations: 20 | config.kubernetes.io/local-config: "true" 21 | spec: 22 | info: 23 | title: terraform-google-bigquery-data-warehouse 24 | source: 25 | repo: https://github.com/terraform-google-modules/terraform-google-bigquery 26 | sourceType: git 27 | dir: /modules/data_warehouse 28 | ui: 29 | input: 30 | variables: 31 | create_ignore_service_accounts: 32 | name: create_ignore_service_accounts 33 | title: Ignore Service Accounts if Exist 34 | dataform_region: 35 | name: dataform_region 36 | title: Dataform Region 37 | deletion_protection: 38 | name: deletion_protection 39 | title: Deletion Protection 40 | invisible: true 41 | enable_apis: 42 | name: enable_apis 43 | title: Enable Apis 44 | force_destroy: 45 | name: force_destroy 46 | title: Force Destroy 47 | invisible: true 48 | labels: 49 | name: labels 50 | title: Labels 51 | project_id: 52 | name: project_id 53 | title: Project Id 54 | region: 55 | name: region 56 | title: Region 57 | text_generation_model_name: 58 | name: text_generation_model_name 59 | title: Text Generation Model Name 60 | runtime: 61 | outputs: 62 | bigquery_editor_url: 63 | openInNewTab: true 64 | showInNotification: true 65 | ds_friendly_name: {} 66 | lookerstudio_report_url: 67 | openInNewTab: true 68 | raw_bucket: {} 69 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/sql/sp_sample_translation_queries.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | /* To use the Migration Service, in the top pane, go to More --> Enable SQL translation. 16 | The queries below are examples of non-BigQuery SQL syntax that can be used with the 17 | interactive translator to see before and after changes performed. 18 | 19 | The sample query below uses PostgreSQL syntax.*/ 20 | 21 | /* Query 1 22 | ------------- 23 | 24 | CREATE TABLE ${project_id}.${dataset_id}.inventory_items (id VARCHAR, product_id VARCHAR, created_at TIMESTAMP, sold_at TIMESTAMP, cost NUMERIC, product_category VARCHAR, product_name VARCHAR, product_brand VARCHAR, product_retail_price NUMERIC, product_department VARCHAR, product_sku VARCHAR, product_distribution_center_id VARCHAR); 25 | CREATE TABLE ${project_id}.${dataset_id}.order_items (id INTEGER, order_id INTEGER, user_id INTEGER, product_id INTEGER, inventory_item_id INTEGER, status VARCHAR, created_at TIMESTAMP, shipped_at TIMESTAMP, delivered_at TIMESTAMP, returned_at TIMESTAMP, sale_price NUMERIC); 26 | 27 | SELECT 28 | EXTRACT(dow from order_items.created_at) AS WeekdayNumber, 29 | TO_CHAR(order_items.created_at, 'DAY') AS WeekdayName, 30 | inventory.product_category AS product_category, 31 | COUNT(DISTINCT order_items.order_id) AS num_high_value_orders 32 | FROM ${project_id}.${dataset_id}.inventory_items AS inventory 33 | INNER JOIN ${project_id}.${dataset_id}.order_items AS order_items 34 | ON inventory.id::int = order_items.inventory_item_id 35 | AND cast(inventory.product_id as int) = order_items.product_id 36 | AND order_items.created_at BETWEEN TO_TIMESTAMP('2022-01-01','YYYY-MM-DD') AND TO_TIMESTAMP('2022-12-31','YYYY-MM-DD') 37 | GROUP BY 1, 2, 3 38 | HAVING AVG(order_items.sale_price) > 85; 39 | */ 40 | 41 | 42 | SELECT 'OPEN THE STORED PROCEDURE FOR MORE DETAILS TO USE THE TRANSLATION SERVICE' as sql_text; 43 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/sql/sp_bigqueryml_model.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | /* 16 | Run a query to see the results of the model 17 | -- 18 | SELECT 19 | CONCAT('cluster ', CAST(centroid_id as STRING)) as cluster, 20 | avg_spend as average_spend, 21 | count_orders as count_of_orders, 22 | days_since_order 23 | FROM ( 24 | SELECT 25 | centroid_id, 26 | feature, 27 | ROUND(numerical_value, 2) as value 28 | FROM 29 | ML.CENTROIDS(MODEL `${dataset_id}.customer_segment_clustering`) 30 | ) 31 | PIVOT ( 32 | SUM(value) 33 | FOR feature IN ('avg_spend', 'count_orders', 'days_since_order') 34 | ) 35 | ORDER BY centroid_id 36 | */ 37 | 38 | --Model Example 39 | CREATE OR REPLACE MODEL 40 | `${project_id}.${dataset_id}.customer_segment_clustering` 41 | OPTIONS( 42 | MODEL_TYPE = 'KMEANS', -- model name 43 | NUM_CLUSTERS = 5, -- how many clusters to create 44 | KMEANS_INIT_METHOD = 'KMEANS++', 45 | STANDARDIZE_FEATURES = TRUE -- note: normalization taking place to scale the range of independent variables (each feature contributes proportionately to the final distance) 46 | ) 47 | AS ( 48 | SELECT 49 | * EXCEPT (user_id) 50 | FROM ( 51 | SELECT 52 | user_id, 53 | DATE_DIFF(CURRENT_DATE(), CAST(MAX(order_created_date) as DATE), day) as days_since_order, ---RECENCY 54 | COUNT(DISTINCT order_id) as count_orders, --FREQUENCY 55 | AVG(sale_price) as avg_spend --MONETARY 56 | FROM ( 57 | SELECT 58 | user_id, 59 | order_id, 60 | sale_price, 61 | created_at as order_created_date 62 | FROM 63 | `${project_id}.${dataset_id}.order_items` 64 | WHERE 65 | created_at BETWEEN TIMESTAMP('2020-07-31 00:00:00') 66 | AND TIMESTAMP('2023-07-31 00:00:00') 67 | ) 68 | GROUP BY user_id 69 | ) 70 | ) 71 | ; 72 | -------------------------------------------------------------------------------- /examples/basic_view/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "delete_contents_on_destroy" { 18 | description = "(Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present." 19 | type = bool 20 | default = null 21 | } 22 | 23 | variable "table_project_id" { 24 | description = "Project where the dataset and table are created." 25 | } 26 | 27 | variable "table_dataset_labels" { 28 | description = "A mapping of labels to assign to the table." 29 | type = map(string) 30 | } 31 | 32 | variable "tables" { 33 | description = "A list of maps that includes table_id, schema, clustering, time_partitioning, range_partitioning, view, expiration_time, labels in each element." 34 | default = [] 35 | type = list(object({ 36 | table_id = string, 37 | schema = string, 38 | clustering = list(string), 39 | time_partitioning = object({ 40 | expiration_ms = string, 41 | field = string, 42 | type = string, 43 | }), 44 | range_partitioning = object({ 45 | field = string, 46 | range = object({ 47 | start = string, 48 | end = string, 49 | interval = string, 50 | }), 51 | }), 52 | expiration_time = string, 53 | labels = map(string), 54 | })) 55 | } 56 | 57 | 58 | ### Views ### 59 | 60 | variable "view_project_id" { 61 | description = "Project where the dataset and table are created." 62 | } 63 | 64 | variable "view_dataset_labels" { 65 | description = "A mapping of labels to assign to the table." 66 | type = map(string) 67 | } 68 | 69 | variable "views" { 70 | description = "A list of objects which include table_id, which is view id, and view query" 71 | default = [] 72 | type = list(object({ 73 | view_id = string, 74 | query = string, 75 | use_legacy_sql = bool, 76 | labels = map(string), 77 | })) 78 | } 79 | -------------------------------------------------------------------------------- /docs/upgrading_to_bigquery_v3.0.md: -------------------------------------------------------------------------------- 1 | # Upgrading to BigQuery v3.0 2 | 3 | The v3.0 release of BigQuery is a backwards incompatible release. 4 | - The `time_partitioning` variable has been inlined to the `table` object structure. 5 | - The `clustering` and `expiration_time` keys were added to the `table` object structure. 6 | - The `expiration` variable renamed to `default_table_expiration_ms` 7 | - `count` replaced with `for_each`. 8 | - Outputs changed to output raw resources. 9 | 10 | ## Migration Instructions 11 | 12 | 1. Rename `expiration` variable to `default_table_expiration_ms`. 13 | ```diff 14 | module "bigquery" { 15 | source = "terraform-google-modules/bigquery/google" 16 | - version = "~> 2.0" 17 | - version = "~> 3.0" 18 | 19 | - expiration = 3600000 20 | + default_table_expiration_ms = 3600000 21 | .... 22 | } 23 | ``` 24 | 25 | 2. Delete `time_partitioning` variable and update `tables` variable. 26 | ```diff 27 | - time_partitioning = "DAY" 28 | tables = [ 29 | { 30 | table_id = "foo", 31 | schema = "", 32 | + time_partitioning = { 33 | + type = "DAY", 34 | + field = null, 35 | + require_partition_filter = false, 36 | + expiration_ms = null, 37 | + }, 38 | + expiration_time = null, 39 | + clustering = [], 40 | labels = { 41 | env = "dev" 42 | billable = "true" 43 | }, 44 | }, 45 | { 46 | table_id = "bar", 47 | schema = "", 48 | + time_partitioning = { 49 | + type = "DAY", 50 | + field = null, 51 | + require_partition_filter = false, 52 | + expiration_ms = null, 53 | + }, 54 | + expiration_time = null, 55 | + clustering = [], 56 | labels = { 57 | env = "devops" 58 | billable = "true" 59 | }, 60 | } 61 | ] 62 | ... 63 | } 64 | ``` 65 | 66 | 3. Replacing `count` with `for_each` requires `removing/importing` resources from/to terraform state. 67 | Make a backup of terraform state: 68 | ```bash 69 | terraform init 70 | terraform state pull > state-backup.json 71 | ``` 72 | Remove table resources from the state: 73 | ```bash 74 | terraform state rm 'module.bigquery.google_bigquery_table.main[0]' 75 | terraform state rm 'module.bigquery.google_bigquery_table.main[1]' 76 | ... 77 | ``` 78 | 79 | Import table resources using table ids: 80 | ```bash 81 | terraform import 'module.bigquery.google_bigquery_table.main["foo"]' ':.foo' 82 | terraform import 'module.bigquery.google_bigquery_table.main["bar"]' ':.bar' 83 | ``` 84 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/function/main.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from google.cloud import dataform_v1beta1 16 | import os 17 | 18 | # Commit the notebook files to the repositories created by Terraform 19 | 20 | 21 | def commit_repository_changes(client, project, region) -> str: 22 | directory = f"{os.path.dirname(__file__)}/notebooks/" 23 | for file in os.listdir(directory): 24 | with open(os.path.join(directory, file), 'rb') as f: 25 | encoded_string = f.read() 26 | file_base_name = os.path.basename(file).removesuffix(".ipynb") 27 | repo_id = f"projects/{project}/locations/{region}/repositories/{file_base_name}" # ignore line too long error # noqa: E501 28 | print(repo_id) 29 | request = dataform_v1beta1.CommitRepositoryChangesRequest() 30 | request.name = repo_id 31 | request.commit_metadata = dataform_v1beta1.CommitMetadata( 32 | author=dataform_v1beta1.CommitAuthor( 33 | name="Google JSS", 34 | email_address="no-reply@google.com" 35 | ), 36 | commit_message="Committing Jump Start Solution notebooks" 37 | ) 38 | request.file_operations = {} 39 | request.file_operations["content.ipynb"] = \ 40 | dataform_v1beta1.\ 41 | CommitRepositoryChangesRequest.\ 42 | FileOperation(write_file=dataform_v1beta1. 43 | CommitRepositoryChangesRequest. 44 | FileOperation. 45 | WriteFile(contents=encoded_string) 46 | ) 47 | print(request.file_operations) 48 | client.commit_repository_changes(request=request) 49 | print(f"Committed changes to {repo_id}") 50 | return ("Committed changes to all repos") 51 | 52 | 53 | def run_it(request) -> str: 54 | dataform_client = dataform_v1beta1.DataformClient() 55 | project_id = os.environ.get("PROJECT_ID") 56 | region_id = os.environ.get("REGION") 57 | commit_changes = commit_repository_changes( 58 | dataform_client, project_id, region_id) 59 | print("Notebooks created!") 60 | return commit_changes 61 | -------------------------------------------------------------------------------- /examples/basic_view/README.md: -------------------------------------------------------------------------------- 1 | # terraform-google-bigquery basic_view 2 | The basic_view example uses the root terraform-google-bigquery module to deploy a dataset and a table with a basic schema. 3 | Additionally, it creates another dataset with a view on the table. 4 | This is a common practice for providing limited data in a different dataset. 5 | 6 | 7 | ## Inputs 8 | 9 | | Name | Description | Type | Default | Required | 10 | |------|-------------|------|---------|:--------:| 11 | | delete\_contents\_on\_destroy | (Optional) If set to true, delete all the tables in the dataset when destroying the resource; otherwise, destroying the resource will fail if tables are present. | `bool` | `null` | no | 12 | | table\_dataset\_labels | A mapping of labels to assign to the table. | `map(string)` | n/a | yes | 13 | | table\_project\_id | Project where the dataset and table are created. | `any` | n/a | yes | 14 | | tables | A list of maps that includes table\_id, schema, clustering, time\_partitioning, range\_partitioning, view, expiration\_time, labels in each element. | 
list(object({
    table_id   = string,
    schema     = string,
    clustering = list(string),
    time_partitioning = object({
      expiration_ms = string,
      field         = string,
      type          = string,
    }),
    range_partitioning = object({
      field = string,
      range = object({
        start    = string,
        end      = string,
        interval = string,
      }),
    }),
    expiration_time = string,
    labels          = map(string),
  }))
| `[]` | no | 15 | | view\_dataset\_labels | A mapping of labels to assign to the table. | `map(string)` | n/a | yes | 16 | | view\_project\_id | Project where the dataset and table are created. | `any` | n/a | yes | 17 | | views | A list of objects which include table\_id, which is view id, and view query | 
list(object({
    view_id        = string,
    query          = string,
    use_legacy_sql = bool,
    labels         = map(string),
  }))
| `[]` | no | 18 | 19 | ## Outputs 20 | 21 | | Name | Description | 22 | |------|-------------| 23 | | access\_roles | Map of roles assigned to identities | 24 | | authorized\_views | Map of authorized views created | 25 | | bigquery\_dataset | Bigquery dataset resource. | 26 | | bigquery\_dataset\_view | Bigquery dataset resource. | 27 | | bigquery\_tables | Map of bigquery table resources being provisioned. | 28 | | bigquery\_views | Map of bigquery table/view resources being provisioned. | 29 | 30 | 31 | 32 | ## Setup 33 | Update the contents of `terraform.tfvars` to match your test environment. 34 | 35 | ## Run example 36 | ``` 37 | terraform init 38 | terraform plan 39 | terraform apply -var-file terraform.tfvars 40 | ``` 41 | -------------------------------------------------------------------------------- /modules/scheduled_queries/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | data "google_project" "project" { 18 | project_id = var.project_id 19 | } 20 | 21 | resource "google_project_iam_member" "bq_transfer_permission" { 22 | project = data.google_project.project.project_id 23 | role = "roles/iam.serviceAccountShortTermTokenMinter" 24 | member = "serviceAccount:service-${data.google_project.project.number}@gcp-sa-bigquerydatatransfer.iam.gserviceaccount.com" 25 | } 26 | 27 | resource "google_bigquery_data_transfer_config" "query_config" { 28 | for_each = { for i in var.queries : i.name => i } 29 | 30 | project = var.project_id 31 | display_name = each.value.name 32 | location = lookup(each.value, "location", null) 33 | data_source_id = each.value.data_source_id 34 | schedule = lookup(each.value, "schedule", "") 35 | destination_dataset_id = lookup(each.value, "destination_dataset_id", null) 36 | notification_pubsub_topic = lookup(each.value, "notification_pubsub_topic", null) 37 | data_refresh_window_days = lookup(each.value, "data_refresh_window_days", null) 38 | disabled = lookup(each.value, "disabled", null) 39 | service_account_name = lookup(each.value, "service_account_name", null) 40 | params = each.value.params 41 | 42 | dynamic "schedule_options" { 43 | for_each = (lookup(each.value, "disable_auto_scheduling", null) != null) ? [each.value.disable_auto_scheduling] : [] 44 | content { 45 | disable_auto_scheduling = lookup(each.value, "disable_auto_scheduling", null) 46 | start_time = lookup(each.value, "start_time", null) 47 | end_time = lookup(each.value, "end_time", null) 48 | } 49 | } 50 | 51 | dynamic "email_preferences" { 52 | for_each = (lookup(each.value, "enable_failure_email", false) != false) ? [each.value.enable_failure_email] : [] 53 | content { 54 | enable_failure_email = lookup(each.value, "enable_failure_email", false) 55 | } 56 | } 57 | 58 | dynamic "sensitive_params" { 59 | for_each = (lookup(each.value, "secret_access_key", null) != null) ? [each.value.secret_access_key] : [] 60 | content { 61 | secret_access_key = lookup(each.value, "secret_access_key", null) 62 | } 63 | } 64 | 65 | depends_on = [google_project_iam_member.bq_transfer_permission] 66 | } 67 | -------------------------------------------------------------------------------- /docs/upgrading_to_bigquery_v4.0.md: -------------------------------------------------------------------------------- 1 | # Upgrading to BigQuery v4.0 2 | 3 | The v4.0 release of BigQuery is a backwards incompatible release. 4 | - The supported provider has been updated to v3.0. 5 | - Add support for setting custom access. The default has been changed to be more 6 | secure. 7 | - The UDFS submodule has been isolated from the main module. 8 | - Trimmed outputs and made their names more consistent. 9 | 10 | ## Access 11 | 12 | The major logic change in this release is a new default for the `access` field. 13 | 14 | Datasets are created with the following 15 | [default](https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets): 16 | 17 | 18 | > If unspecified at dataset creation time, BigQuery adds default dataset access for the following entities: access.specialGroup: projectReaders; access.role: READER; access.specialGroup: projectWriters; access.role: WRITER; access.specialGroup: projectOwners; access.role: OWNER; access.userByEmail: [dataset creator email]; access.role: OWNER; 19 | 20 | The problematic default is 21 | `access.userByEmail: [dataset creator email]; access.role: OWNER`. 22 | This will subtly grant owner access to the creating user and leave that access 23 | on, even if the access to the project, etc is revoked. 24 | 25 | The new default will only set 26 | `specialGroup: projectOwners; access.role: roles/bigquery.admin`. This is done 27 | because the BigQuery API requires at least one owner on the dataset, and this is 28 | the safest and most generic owner that can be granted. 29 | 30 | Do note that standard GCP IAM logic still apply to bigquery datasets. 31 | For example, setting an IAM role at the project level will inherit to all 32 | datasets within the project. 33 | 34 | ## Migration Instructions 35 | 36 | 1. Upgrade version 37 | ```diff 38 | module "bigquery" { 39 | source = "terraform-google-modules/bigquery/google" 40 | - version = "~> 3.0" 41 | + version = "~> 4.0" 42 | .... 43 | } 44 | ``` 45 | 46 | 2. Set access to an empty slice if you wish to preserve existing behaviour 47 | ```diff 48 | module "bigquery" { 49 | source = "terraform-google-modules/bigquery/google" 50 | + access = [] 51 | ... 52 | } 53 | ``` 54 | 55 | 3. Move add_udf field to its own module call 56 | ```diff 57 | module "bigquery" { 58 | source = "terraform-google-modules/bigquery/google" 59 | - add_udfs: true 60 | ... 61 | } 62 | 63 | + module "add_udfs" { 64 | + source = ""terraform-google-modules/bigquery/google//modules/udf" 65 | + version = "~> 4.0" 66 | + dataset_id = module.bigquery.bigquery_dataset.dataset_id 67 | + project_id = module.bigquery.bigquery_dataset.project 68 | } 69 | ``` 70 | 71 | 4. Adjust output references 72 | 73 | - The following outputs have been deleted: 74 | - dataset_labels 75 | - table_labels 76 | - added_udfs 77 | - The following outputs have been renamed: 78 | - data_project -> project 79 | - table_id -> table_ids 80 | - table_name -> table_names 81 | 82 | See https://www.terraform.io/docs/configuration/outputs.html#accessing-child-module-outputs 83 | on how to adjust your usage. 84 | -------------------------------------------------------------------------------- /modules/scheduled_queries/metadata.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: blueprints.cloud.google.com/v1alpha1 16 | kind: BlueprintMetadata 17 | metadata: 18 | name: terraform-google-bigquery-scheduled-queries 19 | annotations: 20 | config.kubernetes.io/local-config: "true" 21 | spec: 22 | info: 23 | title: BigQuery Scheduled Queries 24 | source: 25 | repo: https://github.com/terraform-google-modules/terraform-google-bigquery 26 | sourceType: git 27 | dir: /modules/scheduled_queries 28 | version: 10.2.1 29 | actuationTool: 30 | flavor: Terraform 31 | version: ">= 1.3" 32 | description: {} 33 | content: 34 | examples: 35 | - name: basic_bq 36 | location: examples/basic_bq 37 | - name: basic_view 38 | location: examples/basic_view 39 | - name: data_warehouse 40 | location: examples/data_warehouse 41 | - name: multiple_tables 42 | location: examples/multiple_tables 43 | - name: scheduled_queries 44 | location: examples/scheduled_queries 45 | interfaces: 46 | variables: 47 | - name: project_id 48 | description: The project where scheduled queries are created 49 | varType: string 50 | required: true 51 | - name: queries 52 | description: Data transfer configuration for creating scheduled queries 53 | varType: list(any) 54 | required: true 55 | outputs: 56 | - name: query_names 57 | description: The resource names of the transfer config 58 | requirements: 59 | roles: 60 | - level: Project 61 | roles: 62 | - roles/cloudfunctions.admin 63 | - roles/dataform.admin 64 | - roles/datalineage.viewer 65 | - roles/resourcemanager.projectIamAdmin 66 | - roles/run.invoker 67 | - roles/workflows.admin 68 | - roles/bigquery.admin 69 | - roles/aiplatform.admin 70 | - roles/iam.serviceAccountAdmin 71 | - roles/iam.serviceAccountTokenCreator 72 | - roles/iam.serviceAccountUser 73 | - roles/logging.configWriter 74 | - roles/serviceusage.serviceUsageAdmin 75 | - roles/storage.admin 76 | services: 77 | - bigquery.googleapis.com 78 | - bigqueryconnection.googleapis.com 79 | - bigquerystorage.googleapis.com 80 | - cloudkms.googleapis.com 81 | - cloudresourcemanager.googleapis.com 82 | - iam.googleapis.com 83 | - serviceusage.googleapis.com 84 | providerVersions: 85 | - source: hashicorp/google 86 | version: ">= 4.0, < 7" 87 | -------------------------------------------------------------------------------- /.github/workflows/lint.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2023-2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # NOTE: This file is automatically generated from values at: 16 | # https://github.com/GoogleCloudPlatform/cloud-foundation-toolkit/blob/main/infra/terraform/test-org/org/locals.tf 17 | 18 | name: 'lint' 19 | 20 | on: 21 | workflow_dispatch: 22 | pull_request: 23 | types: [opened, edited, reopened, synchronize] 24 | branches: [main] 25 | 26 | permissions: 27 | contents: read 28 | 29 | concurrency: 30 | group: '${{ github.workflow }}-${{ github.head_ref || github.ref }}' 31 | cancel-in-progress: true 32 | 33 | jobs: 34 | lint: 35 | name: 'lint' 36 | runs-on: 'ubuntu-latest' 37 | steps: 38 | - uses: 'actions/checkout@v6' 39 | - id: variables 40 | run: | 41 | MAKEFILE=$(find . -name Makefile -print -quit) 42 | if [ -z "$MAKEFILE" ]; then 43 | echo dev-tools=gcr.io/cloud-foundation-cicd/cft/developer-tools:1 >> "$GITHUB_OUTPUT" 44 | else 45 | VERSION=$(grep "DOCKER_TAG_VERSION_DEVELOPER_TOOLS := " $MAKEFILE | cut -d\ -f3) 46 | IMAGE=$(grep "DOCKER_IMAGE_DEVELOPER_TOOLS := " $MAKEFILE | cut -d\ -f3) 47 | REGISTRY=$(grep "REGISTRY_URL := " $MAKEFILE | cut -d\ -f3) 48 | echo dev-tools=${REGISTRY}/${IMAGE}:${VERSION} >> "$GITHUB_OUTPUT" 49 | fi 50 | - run: docker run --rm -e ENABLE_BPMETADATA -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} module-swapper 51 | env: 52 | ENABLE_BPMETADATA: 1 53 | 54 | - run: docker run --rm -e ENABLE_BPMETADATA -v ${{ github.workspace }}:/workspace ${{ steps.variables.outputs.dev-tools }} /usr/local/bin/test_lint.sh 55 | env: 56 | ENABLE_BPMETADATA: 1 57 | 58 | commitlint: 59 | runs-on: ubuntu-latest 60 | steps: 61 | - uses: actions/checkout@v6 62 | with: 63 | fetch-depth: 0 64 | - name: Setup node 65 | uses: actions/setup-node@v6 66 | with: 67 | node-version: lts/* 68 | - name: Install commitlint 69 | run: | 70 | npm install -D @commitlint/cli@20.2.0 @commitlint/config-conventional@20.2.0 71 | echo "module.exports = { extends: ['@commitlint/config-conventional'], rules: {'subject-case': [0], 'header-max-length': [0]} };" > commitlint.config.js 72 | npx commitlint --version 73 | - name: Validate PR commits with commitlint 74 | if: github.event_name == 'pull_request' 75 | env: 76 | TITLE: ${{ github.event.pull_request.title }} 77 | run: 'echo "$TITLE" | npx commitlint --verbose' 78 | -------------------------------------------------------------------------------- /test/setup/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | locals { 17 | per_module_services = { 18 | root = [ 19 | "iam.googleapis.com", 20 | "cloudkms.googleapis.com", 21 | "cloudresourcemanager.googleapis.com", 22 | "bigquery.googleapis.com", 23 | "bigquerystorage.googleapis.com", 24 | ] 25 | authorization = [ 26 | "cloudkms.googleapis.com", 27 | "cloudresourcemanager.googleapis.com", 28 | "bigquery.googleapis.com", 29 | "bigquerystorage.googleapis.com", 30 | "bigqueryconnection.googleapis.com", 31 | "serviceusage.googleapis.com", 32 | "iam.googleapis.com", 33 | ] 34 | data_warehouse = [ 35 | "cloudkms.googleapis.com", 36 | "cloudresourcemanager.googleapis.com", 37 | "bigquery.googleapis.com", 38 | "bigquerystorage.googleapis.com", 39 | "bigqueryconnection.googleapis.com", 40 | "serviceusage.googleapis.com", 41 | "iam.googleapis.com", 42 | ] 43 | scheduled_queries = [ 44 | "cloudkms.googleapis.com", 45 | "cloudresourcemanager.googleapis.com", 46 | "bigquery.googleapis.com", 47 | "bigquerystorage.googleapis.com", 48 | "bigqueryconnection.googleapis.com", 49 | "serviceusage.googleapis.com", 50 | "iam.googleapis.com", 51 | ] 52 | udf = [ 53 | "cloudkms.googleapis.com", 54 | "cloudresourcemanager.googleapis.com", 55 | "bigquery.googleapis.com", 56 | "bigquerystorage.googleapis.com", 57 | "bigqueryconnection.googleapis.com", 58 | "serviceusage.googleapis.com", 59 | "iam.googleapis.com", 60 | ] 61 | } 62 | } 63 | 64 | module "project" { 65 | source = "terraform-google-modules/project-factory/google" 66 | version = "~> 18.0" 67 | 68 | name = "ci-bigquery" 69 | random_project_id = "true" 70 | org_id = var.org_id 71 | folder_id = var.folder_id 72 | billing_account = var.billing_account 73 | default_service_account = "keep" 74 | 75 | activate_apis = tolist(toset(flatten(values(local.per_module_services)))) 76 | } 77 | 78 | module "kms_keyring" { 79 | source = "terraform-google-modules/kms/google" 80 | version = "~> 3.0" 81 | 82 | project_id = module.project.project_id 83 | location = "us" 84 | keyring = "ci-bigquery-keyring" 85 | keys = ["foo"] 86 | prevent_destroy = "false" 87 | depends_on = [ 88 | module.project 89 | ] 90 | } 91 | 92 | data "google_bigquery_default_service_account" "initialize_encryption_account" { 93 | project = module.project.project_id 94 | } 95 | -------------------------------------------------------------------------------- /examples/basic_view/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | 18 | module "bigquery_tables" { 19 | source = "terraform-google-modules/bigquery/google" 20 | version = "~> 9.0" 21 | 22 | dataset_id = "foo" 23 | dataset_name = "foo" 24 | description = "some description" 25 | project_id = var.table_project_id 26 | location = "US" 27 | delete_contents_on_destroy = var.delete_contents_on_destroy 28 | tables = var.tables 29 | dataset_labels = var.table_dataset_labels 30 | 31 | # we provide the access control separately with another module, see bottom. 32 | # Authorization module has the capability of authorizing views 33 | # since access block in here conflicts with that, we only use the module. 34 | access = [] 35 | } 36 | 37 | module "bigquery_views_without_pii" { 38 | source = "terraform-google-modules/bigquery/google" 39 | version = "~> 9.0" 40 | 41 | dataset_id = "${module.bigquery_tables.bigquery_dataset.dataset_id}_view_without_pii" # this creates a dependency so that we have the tables first 42 | dataset_name = "foo view" 43 | description = "some description" 44 | project_id = var.view_project_id 45 | delete_contents_on_destroy = var.delete_contents_on_destroy 46 | location = "US" 47 | views = var.views 48 | dataset_labels = var.view_dataset_labels 49 | 50 | access = [ 51 | { 52 | role = "roles/bigquery.dataOwner" 53 | special_group = "projectOwners" 54 | } 55 | ] 56 | } 57 | 58 | # it is possible to pass the view access to a dataset resource but then we have a chicken-egg problem. 59 | # the view wants first the tables are created, while the view access control needs the views 60 | # so we create the authorized views after creating tables and views. 61 | module "authorization" { 62 | source = "terraform-google-modules/bigquery/google//modules/authorization" 63 | version = "~> 9.0" 64 | 65 | project_id = var.table_project_id 66 | dataset_id = module.bigquery_tables.bigquery_dataset.dataset_id 67 | 68 | roles = [] 69 | # roles = [ 70 | # { 71 | # role = "roles/bigquery.dataEditor" 72 | # group_by_email = "ops@mycompany.com" 73 | # } 74 | # ] 75 | 76 | authorized_views = [ 77 | for view in var.views : 78 | { 79 | project_id = var.view_project_id, 80 | dataset_id = module.bigquery_views_without_pii.bigquery_dataset.dataset_id, 81 | table_id = view.view_id 82 | } 83 | ] 84 | } 85 | -------------------------------------------------------------------------------- /modules/udf/metadata.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: blueprints.cloud.google.com/v1alpha1 16 | kind: BlueprintMetadata 17 | metadata: 18 | name: terraform-google-bigquery-udf 19 | annotations: 20 | config.kubernetes.io/local-config: "true" 21 | spec: 22 | info: 23 | title: BigQuery Persistent UDFs 24 | source: 25 | repo: https://github.com/terraform-google-modules/terraform-google-bigquery 26 | sourceType: git 27 | dir: /modules/udf 28 | version: 10.2.1 29 | actuationTool: 30 | flavor: Terraform 31 | version: ">= 1.3" 32 | description: {} 33 | content: 34 | examples: 35 | - name: basic_bq 36 | location: examples/basic_bq 37 | - name: basic_view 38 | location: examples/basic_view 39 | - name: data_warehouse 40 | location: examples/data_warehouse 41 | - name: multiple_tables 42 | location: examples/multiple_tables 43 | - name: scheduled_queries 44 | location: examples/scheduled_queries 45 | interfaces: 46 | variables: 47 | - name: dataset_id 48 | description: Dataset id 49 | varType: string 50 | required: true 51 | - name: project_id 52 | description: Project ID that contains the dataset 53 | varType: string 54 | required: true 55 | - name: add_udfs 56 | description: Whether or not this module should be enabled. 57 | varType: string 58 | defaultValue: false 59 | outputs: 60 | - name: added_udfs 61 | description: List of UDFs utility functions added. 62 | requirements: 63 | roles: 64 | - level: Project 65 | roles: 66 | - roles/iam.serviceAccountAdmin 67 | - roles/iam.serviceAccountTokenCreator 68 | - roles/iam.serviceAccountUser 69 | - roles/resourcemanager.projectIamAdmin 70 | - roles/serviceusage.serviceUsageAdmin 71 | - roles/bigquery.admin 72 | - roles/cloudfunctions.admin 73 | - roles/logging.configWriter 74 | - roles/run.invoker 75 | - roles/storage.admin 76 | - roles/workflows.admin 77 | - roles/aiplatform.admin 78 | - roles/dataform.admin 79 | - roles/datalineage.viewer 80 | services: 81 | - bigquery.googleapis.com 82 | - bigqueryconnection.googleapis.com 83 | - bigquerystorage.googleapis.com 84 | - cloudkms.googleapis.com 85 | - cloudresourcemanager.googleapis.com 86 | - iam.googleapis.com 87 | - serviceusage.googleapis.com 88 | providerVersions: 89 | - source: hashicorp/google 90 | version: ">= 3.53, < 7" 91 | -------------------------------------------------------------------------------- /modules/data_warehouse/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "project-services" { 18 | source = "terraform-google-modules/project-factory/google//modules/project_services" 19 | version = "~> 18.0" 20 | disable_services_on_destroy = false 21 | 22 | project_id = var.project_id 23 | enable_apis = var.enable_apis 24 | 25 | activate_apis = [ 26 | "aiplatform.googleapis.com", 27 | "artifactregistry.googleapis.com", 28 | "bigquery.googleapis.com", 29 | "bigqueryconnection.googleapis.com", 30 | "bigquerydatapolicy.googleapis.com", 31 | "bigquerydatatransfer.googleapis.com", 32 | "bigquerymigration.googleapis.com", 33 | "bigqueryreservation.googleapis.com", 34 | "bigquerystorage.googleapis.com", 35 | "cloudapis.googleapis.com", 36 | "cloudbuild.googleapis.com", 37 | "cloudfunctions.googleapis.com", 38 | "compute.googleapis.com", 39 | "config.googleapis.com", 40 | "datacatalog.googleapis.com", 41 | "dataform.googleapis.com", 42 | "datalineage.googleapis.com", 43 | "notebooks.googleapis.com", 44 | "run.googleapis.com", 45 | "serviceusage.googleapis.com", 46 | "storage.googleapis.com", 47 | "storage-api.googleapis.com", 48 | "workflows.googleapis.com", 49 | "workflowexecutions.googleapis.com" 50 | ] 51 | 52 | activate_api_identities = [ 53 | { 54 | api = "workflows.googleapis.com" 55 | roles = [ 56 | "roles/workflows.viewer" 57 | ], 58 | api = "bigquerydatatransfer.googleapis.com" 59 | roles = [ 60 | "roles/bigquerydatatransfer.serviceAgent" 61 | ] 62 | } 63 | ] 64 | } 65 | 66 | # Wait after APIs are enabled to give time for them to spin up 67 | resource "time_sleep" "wait_after_apis" { 68 | create_duration = "30s" 69 | depends_on = [module.project-services] 70 | } 71 | 72 | # resource "google_project_service_identity" "default" { 73 | # provider = google-beta 74 | # project = module.project-services.project_id 75 | # service = "workflows.googleapis.com" 76 | # } 77 | 78 | # Create random ID to be used for deployment uniqueness 79 | resource "random_id" "id" { 80 | byte_length = 4 81 | } 82 | 83 | # Set up Storage Buckets 84 | ## Set up the raw storage bucket for data 85 | resource "google_storage_bucket" "raw_bucket" { 86 | name = "ds-edw-raw-${random_id.id.hex}" 87 | project = module.project-services.project_id 88 | location = var.region 89 | uniform_bucket_level_access = true 90 | force_destroy = var.force_destroy 91 | 92 | public_access_prevention = "enforced" 93 | 94 | depends_on = [time_sleep.wait_after_apis] 95 | 96 | labels = var.labels 97 | } 98 | -------------------------------------------------------------------------------- /modules/authorization/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | role_keys = [ 19 | for role in var.roles : 20 | join("_", compact([ 21 | role["role"], 22 | lookup(role, "domain", null), 23 | lookup(role, "group_by_email", null), 24 | lookup(role, "user_by_email", null), 25 | lookup(role, "special_group", null) 26 | ])) 27 | ] 28 | roles = zipmap(local.role_keys, var.roles) 29 | views = { for view in var.authorized_views : "${view["project_id"]}_${view["dataset_id"]}_${view["table_id"]}" => view } 30 | datasets = { for dataset in var.authorized_datasets : "${dataset["project_id"]}_${dataset["dataset_id"]}" => dataset } 31 | routines = { for routine in var.authorized_routines : "${routine["project_id"]}_${routine["dataset_id"]}_${routine["routine_id"]}" => routine } 32 | 33 | iam_to_primitive = { 34 | "roles/bigquery.dataOwner" : "OWNER" 35 | "roles/bigquery.dataEditor" : "WRITER" 36 | "roles/bigquery.dataViewer" : "READER" 37 | } 38 | } 39 | 40 | resource "google_bigquery_dataset_access" "authorized_view" { 41 | for_each = local.views 42 | dataset_id = var.dataset_id 43 | project = var.project_id 44 | view { 45 | project_id = each.value.project_id 46 | dataset_id = each.value.dataset_id 47 | table_id = each.value.table_id 48 | } 49 | } 50 | 51 | resource "google_bigquery_dataset_access" "access_role" { 52 | for_each = local.roles 53 | dataset_id = var.dataset_id 54 | project = var.project_id 55 | # BigQuery API converts IAM to primitive roles in its backend. 56 | # This causes Terraform to show a diff on every plan that uses IAM equivalent roles. 57 | # Thus, do the conversion between IAM to primitive role here to prevent the diff. 58 | role = lookup(local.iam_to_primitive, each.value.role, each.value.role) 59 | domain = lookup(each.value, "domain", null) 60 | group_by_email = lookup(each.value, "group_by_email", null) 61 | user_by_email = lookup(each.value, "user_by_email", null) 62 | special_group = lookup(each.value, "special_group", null) 63 | } 64 | 65 | resource "google_bigquery_dataset_access" "authorized_dataset" { 66 | for_each = local.datasets 67 | dataset_id = var.dataset_id 68 | project = var.project_id 69 | dataset { 70 | dataset { 71 | project_id = each.value.project_id 72 | dataset_id = each.value.dataset_id 73 | } 74 | target_types = ["VIEWS"] 75 | } 76 | } 77 | 78 | resource "google_bigquery_dataset_access" "authorized_routine" { 79 | for_each = local.routines 80 | dataset_id = var.dataset_id 81 | project = var.project_id 82 | routine { 83 | project_id = each.value.project_id 84 | dataset_id = each.value.dataset_id 85 | routine_id = each.value.routine_id 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # Copyright 2023 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # Please note that this file was generated from [terraform-google-module-template](https://github.com/terraform-google-modules/terraform-google-module-template). 16 | # Please make sure to contribute relevant changes upstream! 17 | 18 | # Make will use bash instead of sh 19 | SHELL := /usr/bin/env bash 20 | 21 | DOCKER_TAG_VERSION_DEVELOPER_TOOLS := 1.25 22 | DOCKER_IMAGE_DEVELOPER_TOOLS := cft/developer-tools 23 | REGISTRY_URL := gcr.io/cloud-foundation-cicd 24 | ENABLE_BPMETADATA := 1 25 | export ENABLE_BPMETADATA 26 | 27 | # Enter docker container for local development 28 | .PHONY: docker_run 29 | docker_run: 30 | docker run --rm -it \ 31 | -e SERVICE_ACCOUNT_JSON \ 32 | -v $(CURDIR):/workspace \ 33 | $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ 34 | /bin/bash 35 | 36 | # Execute prepare tests within the docker container 37 | .PHONY: docker_test_prepare 38 | docker_test_prepare: 39 | docker run --rm -it \ 40 | -e SERVICE_ACCOUNT_JSON \ 41 | -e TF_VAR_org_id \ 42 | -e TF_VAR_folder_id \ 43 | -e TF_VAR_billing_account \ 44 | -v $(CURDIR):/workspace \ 45 | $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ 46 | /usr/local/bin/execute_with_credentials.sh prepare_environment 47 | 48 | # Clean up test environment within the docker container 49 | .PHONY: docker_test_cleanup 50 | docker_test_cleanup: 51 | docker run --rm -it \ 52 | -e SERVICE_ACCOUNT_JSON \ 53 | -e TF_VAR_org_id \ 54 | -e TF_VAR_folder_id \ 55 | -e TF_VAR_billing_account \ 56 | -v $(CURDIR):/workspace \ 57 | $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ 58 | /usr/local/bin/execute_with_credentials.sh cleanup_environment 59 | 60 | # Execute integration tests within the docker container 61 | .PHONY: docker_test_integration 62 | docker_test_integration: 63 | docker run --rm -it \ 64 | -e SERVICE_ACCOUNT_JSON \ 65 | -v $(CURDIR):/workspace \ 66 | $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ 67 | /usr/local/bin/test_integration.sh 68 | 69 | # Execute lint tests within the docker container 70 | .PHONY: docker_test_lint 71 | docker_test_lint: 72 | docker run --rm -it \ 73 | -e ENABLE_BPMETADATA \ 74 | -v $(CURDIR):/workspace \ 75 | $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ 76 | /usr/local/bin/test_lint.sh 77 | 78 | # Generate documentation 79 | .PHONY: docker_generate_docs 80 | docker_generate_docs: 81 | docker run --rm -it \ 82 | -e ENABLE_BPMETADATA \ 83 | -v $(CURDIR):/workspace \ 84 | $(REGISTRY_URL)/${DOCKER_IMAGE_DEVELOPER_TOOLS}:${DOCKER_TAG_VERSION_DEVELOPER_TOOLS} \ 85 | /bin/bash -c 'source /usr/local/bin/task_helper_functions.sh && generate_docs -d --per-module-requirements' 86 | 87 | # Alias for backwards compatibility 88 | .PHONY: generate_docs 89 | generate_docs: docker_generate_docs 90 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This document provides guidelines for contributing to the module. 4 | 5 | ## Dependencies 6 | 7 | The following dependencies must be installed on the development system: 8 | 9 | - [Docker Engine][docker-engine] 10 | - [Google Cloud SDK][google-cloud-sdk] 11 | - [make] 12 | 13 | ## Generating Documentation for Inputs and Outputs 14 | 15 | The Inputs and Outputs tables in the READMEs of the root module, 16 | submodules, and example modules are automatically generated based on 17 | the `variables` and `outputs` of the respective modules. These tables 18 | must be refreshed if the module interfaces are changed. 19 | 20 | ### Execution 21 | 22 | Run `make generate_docs` to generate new Inputs and Outputs tables. 23 | 24 | ## Integration Testing 25 | 26 | Integration tests are used to verify the behaviour of the root module, 27 | submodules, and example modules. Additions, changes, and fixes should 28 | be accompanied with tests. 29 | 30 | The integration tests are run using [Kitchen][kitchen], 31 | [Kitchen-Terraform][kitchen-terraform], and [InSpec][inspec]. These 32 | tools are packaged within a Docker image for convenience. 33 | 34 | The general strategy for these tests is to verify the behaviour of the 35 | [example modules](./examples/), thus ensuring that the root module, 36 | submodules, and example modules are all functionally correct. 37 | 38 | ### Test Environment 39 | The easiest way to test the module is in an isolated test project. The setup for such a project is defined in [test/setup](./test/setup/) directory. 40 | 41 | To use this setup, you need a service account with Project Creator access on a folder. Export the Service Account credentials to your environment like so: 42 | 43 | ``` 44 | export SERVICE_ACCOUNT_JSON=$(< credentials.json) 45 | ``` 46 | 47 | You will also need to set a few environment variables: 48 | ``` 49 | export TF_VAR_org_id="your_org_id" 50 | export TF_VAR_folder_id="your_folder_id" 51 | export TF_VAR_billing_account="your_billing_account_id" 52 | ``` 53 | 54 | With these settings in place, you can prepare a test project using Docker: 55 | ``` 56 | make docker_test_prepare 57 | ``` 58 | 59 | ### Noninteractive Execution 60 | 61 | Run `make docker_test_integration` to test all of the example modules 62 | noninteractively, using the prepared test project. 63 | 64 | ### Interactive Execution 65 | 66 | 1. Run `make docker_run` to start the testing Docker container in 67 | interactive mode. 68 | 69 | 1. Run `kitchen_do create ` to initialize the working 70 | directory for an example module. 71 | 72 | 1. Run `kitchen_do converge ` to apply the example module. 73 | 74 | 1. Run `kitchen_do verify ` to test the example module. 75 | 76 | 1. Run `kitchen_do destroy ` to destroy the example module 77 | state. 78 | 79 | ## Linting and Formatting 80 | 81 | Many of the files in the repository can be linted or formatted to 82 | maintain a standard of quality. 83 | 84 | ### Execution 85 | 86 | Run `make docker_test_lint`. 87 | 88 | [docker-engine]: https://www.docker.com/products/docker-engine 89 | [flake8]: http://flake8.pycqa.org/en/latest/ 90 | [gofmt]: https://golang.org/cmd/gofmt/ 91 | [google-cloud-sdk]: https://cloud.google.com/sdk/install 92 | [hadolint]: https://github.com/hadolint/hadolint 93 | [inspec]: https://inspec.io/ 94 | [kitchen-terraform]: https://github.com/newcontext-oss/kitchen-terraform 95 | [kitchen]: https://kitchen.ci/ 96 | [make]: https://en.wikipedia.org/wiki/Make_(software) 97 | [shellcheck]: https://www.shellcheck.net/ 98 | [terraform-docs]: https://github.com/segmentio/terraform-docs 99 | [terraform]: https://terraform.io/ 100 | -------------------------------------------------------------------------------- /modules/data_warehouse/workflow_polling/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2024 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | data "google_client_config" "current" { 18 | } 19 | 20 | locals { 21 | is_active = var.input_workflow_state == null || var.input_workflow_state == "FAILED" ? "no" : "yes" 22 | } 23 | 24 | ## Trigger the execution of the setup workflow with an API call 25 | data "http" "call_workflows_setup" { 26 | url = "https://workflowexecutions.googleapis.com/v1/${var.workflow_id}/executions" 27 | method = local.is_active == "no" ? "POST" : "GET" 28 | # method = "POST" 29 | request_headers = { 30 | Accept = "application/json" 31 | Authorization = "Bearer ${data.google_client_config.current.access_token}" } 32 | } 33 | 34 | resource "time_sleep" "workflow_execution_wait" { 35 | create_duration = "120s" 36 | depends_on = [ 37 | data.http.call_workflows_setup 38 | ] 39 | } 40 | 41 | # locals { 42 | # is_active = var.input_workflow_state == null || var.input_workflow_state == "FAILED" ? "no" : "yes" 43 | # } 44 | 45 | # ## Trigger the execution of the setup workflow with an API call 46 | # data "http" "call_workflows_setup" { 47 | # url = "https://workflowexecutions.googleapis.com/v1/${var.workflow_id}/executions" 48 | # method = local.is_active == "no" ? "POST" : "GET" 49 | # request_headers = { 50 | # Accept = "application/json" 51 | # Authorization = "Bearer ${data.google_client_config.current.access_token}" } 52 | # } 53 | 54 | # ## If the workflow last failed, sleep for 30 seconds before checking the workflow execution status. 55 | # ## If last execution did not fail, exit as quickly as possible (1 second) 56 | # resource "time_sleep" "workflow_execution_wait" { 57 | # create_duration = local.is_active == "no" ? "30s" : "1s" 58 | # depends_on = [ 59 | # data.http.call_workflows_setup, 60 | # ] 61 | # } 62 | 63 | ## Check the state of the setup workflow execution with an API call 64 | data "http" "call_workflows_state" { 65 | url = "https://workflowexecutions.googleapis.com/v1/${var.workflow_id}/executions" 66 | method = "GET" 67 | request_headers = { 68 | Accept = "application/json" 69 | Authorization = "Bearer ${data.google_client_config.current.access_token}" } 70 | depends_on = [ 71 | data.http.call_workflows_setup, 72 | time_sleep.workflow_execution_wait 73 | ] 74 | } 75 | 76 | ## Parse out the workflow execution state from the API call response 77 | locals { 78 | response_body = jsondecode(data.http.call_workflows_state.response_body) 79 | workflow_state = local.response_body.executions[0].state 80 | } 81 | 82 | ## Output the workflow state to use as input for subsequent invocations 83 | output "workflow_state" { 84 | description = "State of the most recent workflow execution. Used to determine how to proceed with next polling run." 85 | value = local.workflow_state 86 | } 87 | 88 | # ## If workflow execution is actively running, sleep for 90 seconds to allow it to finish 89 | # ## If not, exit as quickly as possible (1 second) 90 | # resource "time_sleep" "complete_workflow" { 91 | # create_duration = local.workflow_state == "ACTIVE" ? "90s" : "1s" 92 | # depends_on = [ 93 | # data.http.call_workflows_setup, 94 | # time_sleep.workflow_execution_wait, 95 | # data.http.call_workflows_state, 96 | # local.workflow_state 97 | # ] 98 | # } 99 | -------------------------------------------------------------------------------- /modules/data_warehouse/src/sql/sp_bigqueryml_generate_describe.sql: -------------------------------------------------------------------------------- 1 | -- Copyright 2023 Google LLC 2 | -- 3 | -- Licensed under the Apache License, Version 2.0 (the "License"); 4 | -- you may not use this file except in compliance with the License. 5 | -- You may obtain a copy of the License at 6 | -- 7 | -- http://www.apache.org/licenses/LICENSE-2.0 8 | -- 9 | -- Unless required by applicable law or agreed to in writing, software 10 | -- distributed under the License is distributed on an "AS IS" BASIS, 11 | -- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | -- See the License for the specific language governing permissions and 13 | -- limitations under the License. 14 | 15 | /* 16 | This sample query demonstrates how you can use a remote model from Vertex AI to use Generative AI to help describe the results of your BigQuery Machine Learning model. This query requires the sp_bigqueryml_generate_create routine to run first, since that creates the model connection between BigQuery and Vertex AI so you can leverage Vertex's GenAI capabilities. This runs as part of the Workflow created in the Jump Start Solution Deployment, but may take some time to generate. 17 | 18 | It also requires the sp_bigqueryml_model routine to be run, since this query analyzes the output of the K-means model that clusters customers into buying patterns. You can verify if this routine has successfully run by running the following query that describes its output: 19 | 20 | /* 21 | Run a query to see the results of the model 22 | --------------------------------------------- 23 | 24 | SELECT 25 | CONCAT('cluster ', CAST(centroid_id as STRING)) as cluster, 26 | avg_spend as average_spend, 27 | count_orders as count_of_orders, 28 | days_since_order 29 | FROM ( 30 | SELECT 31 | centroid_id, 32 | feature, 33 | ROUND(numerical_value, 2) as value 34 | FROM 35 | ML.CENTROIDS(MODEL `${dataset_id}.customer_segment_clustering`) 36 | ) 37 | PIVOT ( 38 | SUM(value) 39 | FOR feature IN ('avg_spend', 'count_orders', 'days_since_order') 40 | ) 41 | ORDER BY centroid_id 42 | */ 43 | 44 | /* 45 | If this query fails because the model doesn't yet exist, you can manually run this routine to create the K-means model with the query below: 46 | 47 | Call sp_bigqueryml_model routine 48 | ----------------------------------- 49 | 50 | CALL `thelook.sp_bigqueryml_model`(); 51 | */ 52 | 53 | /* 54 | SELECT * 55 | FROM ML.GENERATE_TEXT( 56 | MODEL `${project_id}.${dataset_id}.${model_name}`, 57 | ( 58 | with clusters AS( 59 | SELECT 60 | CONCAT('cluster ', CAST(centroid_id as STRING)) as centroid, 61 | avg_spend as average_spend, 62 | count_orders as count_of_orders, 63 | days_since_order 64 | FROM ( 65 | SELECT centroid_id, feature, ROUND(numerical_value, 2) as value 66 | FROM ML.CENTROIDS(MODEL `${project_id}.${dataset_id}.customer_segment_clustering`) 67 | ) 68 | PIVOT ( 69 | SUM(value) 70 | FOR feature IN ('avg_spend', 'count_orders', 'days_since_order') 71 | ) 72 | ORDER BY centroid_id 73 | ) 74 | 75 | SELECT 76 | "Pretend you're a creative strategist, given the following clusters come up with creative brand persona and title labels for each of these clusters, and explain step by step; what would be the next marketing step for these clusters" || " " || clusters.centroid || ", Average Spend $" || clusters.average_spend || ", Count of orders per person " || clusters.count_of_orders || ", Days since last order " || clusters.days_since_order AS prompt 77 | FROM 78 | clusters 79 | ), 80 | -- See the BigQuery "Generate Text" docs to better understand how changing these inputs will impact your results: https://cloud.google.com/bigquery/docs/generate-text#generate_text 81 | STRUCT( 82 | 800 AS max_output_tokens, 83 | 0.8 AS temperature, 84 | 40 AS top_k, 85 | 0.8 AS top_p, 86 | TRUE AS flatten_json_output 87 | ) 88 | ) 89 | ; 90 | */ 91 | 92 | SELECT 'OPEN THE STORED PROCEDURE FOR MORE DETAILS' as sql_text; 93 | -------------------------------------------------------------------------------- /outputs.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | output "bigquery_dataset" { 18 | value = google_bigquery_dataset.main 19 | description = "Bigquery dataset resource." 20 | } 21 | 22 | output "bigquery_tables" { 23 | value = google_bigquery_table.main 24 | description = "Map of bigquery table resources being provisioned." 25 | } 26 | 27 | output "bigquery_views" { 28 | value = google_bigquery_table.view 29 | description = "Map of bigquery view resources being provisioned." 30 | } 31 | 32 | output "bigquery_external_tables" { 33 | value = google_bigquery_table.external_table 34 | description = "Map of BigQuery external table resources being provisioned." 35 | } 36 | 37 | output "project" { 38 | value = google_bigquery_dataset.main.project 39 | description = "Project where the dataset and tables are created" 40 | } 41 | 42 | output "table_ids" { 43 | value = [ 44 | for table in google_bigquery_table.main : 45 | table.table_id 46 | ] 47 | description = "Unique id for the table being provisioned" 48 | } 49 | 50 | output "table_fqns" { 51 | value = [ 52 | for table in google_bigquery_table.main : 53 | table.id 54 | ] 55 | description = "Fully qualified names for the table with format projects/{{project}}/datasets/{{dataset}}/tables/{{name}}" 56 | } 57 | 58 | output "table_names" { 59 | value = [ 60 | for table in google_bigquery_table.main : 61 | table.friendly_name 62 | ] 63 | description = "Friendly name for the table being provisioned" 64 | } 65 | 66 | output "view_ids" { 67 | value = [ 68 | for view in google_bigquery_table.view : 69 | view.table_id 70 | ] 71 | description = "Unique id for the view being provisioned" 72 | } 73 | 74 | output "view_names" { 75 | value = [ 76 | for view in google_bigquery_table.view : 77 | view.friendly_name 78 | ] 79 | description = "friendlyname for the view being provisioned" 80 | } 81 | 82 | output "external_table_ids" { 83 | value = [ 84 | for external_table in google_bigquery_table.external_table : 85 | external_table.table_id 86 | ] 87 | description = "Unique IDs for any external tables being provisioned" 88 | } 89 | 90 | output "external_table_names" { 91 | value = [ 92 | for table in google_bigquery_table.external_table : 93 | table.friendly_name 94 | ] 95 | description = "Friendly names for any external tables being provisioned" 96 | } 97 | 98 | output "routine_ids" { 99 | value = [ 100 | for routine in google_bigquery_routine.routine : 101 | routine.routine_id 102 | ] 103 | description = "Unique IDs for any routine being provisioned" 104 | } 105 | 106 | output "env_vars" { 107 | value = { 108 | "BIGQUERY_DATASET" = google_bigquery_dataset.main.dataset_id 109 | "BIGQUERY_TABLES" = jsonencode([for table in google_bigquery_table.main : table.table_id]) 110 | "BIGQUERY_VIEWS" = jsonencode([for table in google_bigquery_table.view : table.table_id]) 111 | "BIGQUERY_MATERIALIZED_VIEWS" = jsonencode([for table in google_bigquery_table.materialized_view : table.table_id]) 112 | "BIGQUERY_EXTERNAL_TABLES" = jsonencode([for table in google_bigquery_table.external_table : table.table_id]) 113 | "BIGQUERY_ROUTINES" = jsonencode([for routine in google_bigquery_routine.routine : routine.routine_id]) 114 | } 115 | description = "Exported environment variables" 116 | } 117 | -------------------------------------------------------------------------------- /modules/udf/scripts/persistent_udfs.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # Copyright 2023 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | set -e 18 | 19 | action=$1 20 | dataset_name=$2 21 | 22 | function create() { 23 | drop; 24 | 25 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "CREATE FUNCTION IF NOT EXISTS ${dataset_name}.find_in_set(str STRING, strList STRING) 26 | AS ( 27 | CASE 28 | WHEN STRPOS(str, ',') > 0 THEN 0 29 | ELSE 30 | ( 31 | WITH list AS ( 32 | SELECT ROW_NUMBER() OVER() id, l FROM UNNEST(SPLIT(strList, ',')) l 33 | ) 34 | (SELECT id FROM list WHERE l = str) 35 | ) 36 | END 37 | );" 38 | 39 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "CREATE FUNCTION IF NOT EXISTS ${dataset_name}.check_protocol(url STRING) 40 | AS ( 41 | CASE 42 | WHEN REGEXP_CONTAINS(url, '^[a-zA-Z]+://') THEN url 43 | ELSE CONCAT('http://', url) 44 | END 45 | );" 46 | 47 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "CREATE FUNCTION IF NOT EXISTS ${dataset_name}.parse_url(url STRING, part STRING) 48 | AS ( 49 | CASE 50 | -- Return HOST part of the URL. 51 | WHEN part = 'HOST' THEN SPLIT(\`${PROJECT_ID}\`.${dataset_name}.check_protocol(url), '/')[OFFSET(2)] 52 | WHEN part = 'REF' THEN 53 | IF(REGEXP_CONTAINS(url, '#'), SPLIT(\`${PROJECT_ID}\`.${dataset_name}.check_protocol 54 | (url), '#')[OFFSET(1)], NULL) 55 | WHEN part = 'PROTOCOL' THEN RTRIM(REGEXP_EXTRACT(url, '^[a-zA-Z]+://'), '://') 56 | ELSE '' 57 | END 58 | );" 59 | 60 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "CREATE FUNCTION IF NOT EXISTS ${dataset_name}.csv_to_struct(strList STRING) 61 | AS ( 62 | CASE 63 | WHEN REGEXP_CONTAINS(strList, ',') OR REGEXP_CONTAINS(strList, ':') THEN 64 | (ARRAY( 65 | WITH list AS ( 66 | SELECT l FROM UNNEST(SPLIT(TRIM(strList), ',')) l WHERE REGEXP_CONTAINS(l, ':') 67 | ) 68 | SELECT AS STRUCT 69 | TRIM(SPLIT(l, ':')[OFFSET(0)]) AS key, TRIM(SPLIT(l, ':')[OFFSET(1)]) as value 70 | FROM list 71 | )) 72 | ELSE NULL 73 | END 74 | );" 75 | } 76 | 77 | function list() { 78 | ${BQ_PATH} --project_id "${PROJECT_ID}" ls --routines "${dataset_name}" 79 | } 80 | 81 | function drop() { 82 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "DROP FUNCTION IF EXISTS ${dataset_name}.find_in_set" 83 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "DROP FUNCTION IF EXISTS ${dataset_name}.parse_url" 84 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "DROP FUNCTION IF EXISTS ${dataset_name}.check_protocol" 85 | ${BQ_PATH} --project_id "${PROJECT_ID}" query --use_legacy_sql=false "DROP FUNCTION IF EXISTS ${dataset_name}.csv_to_struct" 86 | } 87 | 88 | function usage() { 89 | echo "" 90 | echo "Shell script for creating and dropping " 91 | echo "" 92 | echo "" 93 | echo "USAGE: ./persistent_udf.sh " 94 | echo "All parameters are mandatory" 95 | echo "action can have the following values: create, drop" 96 | echo "Example: ./persistent_udf.sh create test_dataset" 97 | } 98 | 99 | if [ "${action}" = "create" ] 100 | then 101 | create; 102 | elif [ "${action}" = "drop" ] 103 | then 104 | drop; 105 | else 106 | echo 'Invalid action '"${action}" 107 | usage; 108 | exit 1; 109 | fi 110 | 111 | -------------------------------------------------------------------------------- /modules/authorization/README.md: -------------------------------------------------------------------------------- 1 | # BigQuery Authorized Datasets, Views and Routines 2 | 3 | This submodule is used to add [authorized datasets](https://cloud.google.com/bigquery/docs/authorized-datasets), [authorized views](https://cloud.google.com/bigquery/docs/share-access-views#authorize_the_view_to_access_the_source_dataset) and [authorized routines](https://cloud.google.com/bigquery/docs/authorized-functions). 4 | An `authorized dataset` lets you authorize all of the views in a specified dataset to access the data in a second dataset. An `authorized view` lets you share query results with particular users and groups without giving them access to the underlying source data. `Authorized Routine (Function)` let you share query results with particular users or groups without giving those users or groups access to the underlying tables 5 | 6 | ## Background 7 | It is possible to define authorized views while creating a dataset. However, we have a chicken&egg problem if we create all at the same time. This module has the goal of solving that. 8 | See [basic_view](../../examples/basic_view/main.tf) as an example. 9 | 10 | ## Caveat 11 | This module creates [bigquery_dataset_access](https://www.terraform.io/docs/providers/google/r/bigquery_dataset_access.html) resources, which conflict with the 12 | access blocks in a [bigquery_dataset](https://www.terraform.io/docs/providers/google/r/bigquery_dataset.html) resource. Therefore, when using together with a dataset, 13 | you should pass empty access block to the dataset. 14 | 15 | 16 | Example: 17 | ```hcl 18 | module "dataset" { 19 | source = "terraform-google-modules/bigquery/google" 20 | version = "~> 10.2" 21 | 22 | dataset_id = "example_dataset" 23 | dataset_name = "example_dataset" 24 | description = "example description" 25 | project_id = "example-project" 26 | location = "US" 27 | 28 | access = [] # pass empty not to conflict with below 29 | } 30 | 31 | module "add_authorization" { 32 | source = "terraform-google-modules/bigquery/google//modules/authorization" 33 | version = "~> 4.1" 34 | 35 | dataset_id = module.dataset.bigquery_dataset.dataset_id 36 | project_id = module.dataset.bigquery_dataset.project 37 | 38 | roles = [ 39 | { 40 | role = "roles/bigquery.dataEditor" 41 | group_by_email = "ops@mycompany.com" 42 | } 43 | ] 44 | 45 | authorized_views = [ 46 | { 47 | project_id = "view_project" 48 | dataset_id = "view_dataset" 49 | table_id = "view_id" 50 | } 51 | ] 52 | authorized_datasets = [ 53 | { 54 | project_id = "auth_dataset_project" 55 | dataset_id = "auth_dataset" 56 | } 57 | ] 58 | } 59 | ``` 60 | 61 | 62 | ## Inputs 63 | 64 | | Name | Description | Type | Default | Required | 65 | |------|-------------|------|---------|:--------:| 66 | | authorized\_datasets | An array of datasets to be authorized on the dataset | 
list(object({
    dataset_id = string,
    project_id = string,
  }))
| `[]` | no | 67 | | authorized\_routines | An array of authorized routine to be authorized on the dataset | 
list(object({
    project_id = string,
    dataset_id = string,
    routine_id = string
  }))
| `[]` | no | 68 | | authorized\_views | An array of views to give authorize for the dataset | 
list(object({
    dataset_id = string,
    project_id = string,
    table_id   = string # this is the view id, but we keep table_id to stay consistent as the resource
  }))
| `[]` | no | 69 | | dataset\_id | Unique ID for the dataset being provisioned. | `string` | n/a | yes | 70 | | project\_id | Project where the dataset and table are created | `string` | n/a | yes | 71 | | roles | An array of objects that define dataset access for one or more entities. | `any` | `[]` | no | 72 | 73 | ## Outputs 74 | 75 | | Name | Description | 76 | |------|-------------| 77 | | authorized\_dataset | Authorized datasets for the BQ dataset | 78 | | authorized\_roles | Authorized roles for the dataset | 79 | | authorized\_views | Authorized views for the dataset | 80 | 81 | 82 | -------------------------------------------------------------------------------- /test/setup/iam.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | locals { 18 | per_module_roles = { 19 | root = [ 20 | "roles/bigquery.admin", 21 | "roles/storage.admin", 22 | "roles/cloudkms.cryptoKeyEncrypterDecrypter", 23 | ] 24 | authorization = [ 25 | "roles/bigquery.admin", 26 | "roles/aiplatform.admin", 27 | "roles/cloudfunctions.admin", 28 | "roles/dataform.admin", 29 | "roles/datalineage.viewer", 30 | "roles/iam.serviceAccountAdmin", 31 | "roles/iam.serviceAccountTokenCreator", 32 | "roles/iam.serviceAccountUser", 33 | "roles/logging.configWriter", 34 | "roles/resourcemanager.projectIamAdmin", 35 | "roles/run.invoker", 36 | "roles/serviceusage.serviceUsageAdmin", 37 | "roles/storage.admin", 38 | "roles/workflows.admin" 39 | ] 40 | data_warehouse = [ 41 | "roles/bigquery.admin", 42 | "roles/aiplatform.admin", 43 | "roles/cloudfunctions.admin", 44 | "roles/dataform.admin", 45 | "roles/datalineage.viewer", 46 | "roles/iam.serviceAccountAdmin", 47 | "roles/iam.serviceAccountTokenCreator", 48 | "roles/iam.serviceAccountUser", 49 | "roles/logging.configWriter", 50 | "roles/resourcemanager.projectIamAdmin", 51 | "roles/run.invoker", 52 | "roles/serviceusage.serviceUsageAdmin", 53 | "roles/storage.admin", 54 | "roles/workflows.admin" 55 | ] 56 | scheduled_queries = [ 57 | "roles/bigquery.admin", 58 | "roles/aiplatform.admin", 59 | "roles/cloudfunctions.admin", 60 | "roles/dataform.admin", 61 | "roles/datalineage.viewer", 62 | "roles/iam.serviceAccountAdmin", 63 | "roles/iam.serviceAccountTokenCreator", 64 | "roles/iam.serviceAccountUser", 65 | "roles/logging.configWriter", 66 | "roles/resourcemanager.projectIamAdmin", 67 | "roles/run.invoker", 68 | "roles/serviceusage.serviceUsageAdmin", 69 | "roles/storage.admin", 70 | "roles/workflows.admin" 71 | ] 72 | udf = [ 73 | "roles/bigquery.admin", 74 | "roles/aiplatform.admin", 75 | "roles/cloudfunctions.admin", 76 | "roles/dataform.admin", 77 | "roles/datalineage.viewer", 78 | "roles/iam.serviceAccountAdmin", 79 | "roles/iam.serviceAccountTokenCreator", 80 | "roles/iam.serviceAccountUser", 81 | "roles/logging.configWriter", 82 | "roles/resourcemanager.projectIamAdmin", 83 | "roles/run.invoker", 84 | "roles/serviceusage.serviceUsageAdmin", 85 | "roles/storage.admin", 86 | "roles/workflows.admin" 87 | ] 88 | } 89 | 90 | int_required_roles = tolist(toset(flatten(values(local.per_module_roles)))) 91 | } 92 | 93 | resource "google_service_account" "int_test" { 94 | project = module.project.project_id 95 | account_id = "ci-account" 96 | display_name = "ci-account" 97 | } 98 | 99 | resource "google_project_iam_member" "int_test" { 100 | count = length(local.int_required_roles) 101 | 102 | project = module.project.project_id 103 | role = local.int_required_roles[count.index] 104 | member = "serviceAccount:${google_service_account.int_test.email}" 105 | } 106 | 107 | resource "google_service_account_key" "int_test" { 108 | service_account_id = google_service_account.int_test.id 109 | } 110 | 111 | resource "google_project_iam_member" "bq_encryption_account" { 112 | depends_on = [data.google_bigquery_default_service_account.initialize_encryption_account] #waits for account initialization 113 | project = module.project.project_id 114 | role = "roles/cloudkms.cryptoKeyEncrypterDecrypter" 115 | member = "serviceAccount:${data.google_bigquery_default_service_account.initialize_encryption_account.email}" 116 | } 117 | -------------------------------------------------------------------------------- /modules/data_warehouse/variables.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | type = string 19 | description = "Google Cloud Project ID" 20 | } 21 | 22 | variable "region" { 23 | type = string 24 | description = "Google Cloud Region" 25 | 26 | /** 27 | * This variable list assumes you are using the same region for both Dataform and all other assets. 28 | * If you want to deploy your Dataform respositories in a different region, set the default value 29 | * for var.dataform_region to one of the regions in the Dataform validation list. 30 | * You can then set this variable value to any of the following: 31 | * "asia-northeast3" 32 | * "asia-southeast1" 33 | * "europe-west1" 34 | * "europe-west2" 35 | * "europe-west3" 36 | * "europe-west4" 37 | * "europe-west9" 38 | * "us-central1" 39 | * "us-west4" 40 | * 41 | * Be sure to update the validation list below to include these additional values! 42 | */ 43 | 44 | validation { 45 | condition = contains([ 46 | "asia-southeast1", 47 | "europe-west1", 48 | "europe-west2", 49 | "europe-west3", 50 | "europe-west4", 51 | "us-central1", 52 | ], 53 | var.region) 54 | error_message = "This region is not supported. Region must be one of: asia-southeast1, europe-west1, europe-west2, europe-west3, europe-west4, us-central1" 55 | } 56 | } 57 | 58 | 59 | variable "dataform_region" { 60 | type = string 61 | description = "Region that is used to deploy Dataform resources. This does not limit where resources can be run or what region data must be located in." 62 | default = null 63 | nullable = true 64 | 65 | validation { 66 | condition = anytrue([var.dataform_region == null, try(contains( 67 | [ 68 | "asia-east1", 69 | "asia-northeast1", 70 | "asia-south1", 71 | "asia-southeast1", 72 | "australia-southeast1", 73 | "europe-west1", 74 | "europe-west2", 75 | "europe-west3", 76 | "europe-west4", 77 | "europe-west6", 78 | "southamerica-east1", 79 | "us-central1", 80 | "us-east1", 81 | "us-west1", 82 | ], var.dataform_region), true)]) 83 | error_message = "This region is not supported for Dataform. Region must be one of: asia-east1, asia-northeast1, asia-south1, asia-southeast1, australia-southeast1, europe-west1, europe-west2, europe-west3, europe-west4, europe-west6, southamerica-east1, us-central1, us-east1, us-west1." 84 | } 85 | } 86 | 87 | 88 | variable "text_generation_model_name" { 89 | type = string 90 | description = "Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation" 91 | default = "text_generate_model" 92 | 93 | } 94 | variable "labels" { 95 | type = map(string) 96 | description = "A map of labels to apply to contained resources." 97 | default = { "data-warehouse" = true } 98 | } 99 | 100 | variable "enable_apis" { 101 | type = string 102 | description = "Whether or not to enable underlying apis in this solution." 103 | default = true 104 | } 105 | 106 | variable "force_destroy" { 107 | type = string 108 | description = "Whether or not to protect BigQuery resources from deletion when solution is modified or changed." 109 | default = true 110 | } 111 | 112 | variable "deletion_protection" { 113 | type = string 114 | description = "Whether or not to protect GCS resources from deletion when solution is modified or changed." 115 | default = false 116 | } 117 | 118 | variable "create_ignore_service_accounts" { 119 | type = string 120 | description = "Whether or not to ignore creation of a service account if an account of the same name already exists" 121 | default = true 122 | } 123 | -------------------------------------------------------------------------------- /test/integration/go.mod: -------------------------------------------------------------------------------- 1 | module github.com/terraform-google-modules/terraform-google-bigquery/test/integration 2 | 3 | go 1.22.7 4 | 5 | toolchain go1.23.5 6 | 7 | require ( 8 | github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test v0.17.4 9 | github.com/stretchr/testify v1.10.0 10 | ) 11 | 12 | require ( 13 | cloud.google.com/go v0.116.0 // indirect 14 | cloud.google.com/go/compute v1.28.3 // indirect 15 | cloud.google.com/go/compute/metadata v0.5.2 // indirect 16 | cloud.google.com/go/iam v1.2.2 // indirect 17 | cloud.google.com/go/storage v1.47.0 // indirect 18 | github.com/agext/levenshtein v1.2.3 // indirect 19 | github.com/alexflint/go-filemutex v1.3.0 // indirect 20 | github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect 21 | github.com/aws/aws-sdk-go v1.47.13 // indirect 22 | github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect 23 | github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect 24 | github.com/go-errors/errors v1.5.1 // indirect 25 | github.com/go-openapi/jsonpointer v0.20.0 // indirect 26 | github.com/go-openapi/jsonreference v0.20.2 // indirect 27 | github.com/go-openapi/swag v0.22.4 // indirect 28 | github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect 29 | github.com/golang/protobuf v1.5.4 // indirect 30 | github.com/google/gnostic-models v0.6.8 // indirect 31 | github.com/google/go-cmp v0.6.0 // indirect 32 | github.com/google/s2a-go v0.1.8 // indirect 33 | github.com/google/uuid v1.6.0 // indirect 34 | github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect 35 | github.com/googleapis/gax-go/v2 v2.14.0 // indirect 36 | github.com/gruntwork-io/terratest v0.48.1 // indirect 37 | github.com/hashicorp/errwrap v1.1.0 // indirect 38 | github.com/hashicorp/go-cleanhttp v0.5.2 // indirect 39 | github.com/hashicorp/go-getter v1.7.6 // indirect 40 | github.com/hashicorp/go-getter/v2 v2.2.3 // indirect 41 | github.com/hashicorp/go-multierror v1.1.1 // indirect 42 | github.com/hashicorp/go-safetemp v1.0.0 // indirect 43 | github.com/hashicorp/go-version v1.7.0 // indirect 44 | github.com/hashicorp/hcl v0.0.0-20170504190234-a4b07c25de5f // indirect 45 | github.com/hashicorp/hcl/v2 v2.22.0 // indirect 46 | github.com/hashicorp/terraform-config-inspect v0.0.0-20241129133400-c404f8227ea6 // indirect 47 | github.com/hashicorp/terraform-json v0.24.0 // indirect 48 | github.com/jinzhu/copier v0.4.0 // indirect 49 | github.com/jmespath/go-jmespath v0.4.0 // indirect 50 | github.com/josharian/intern v1.0.0 // indirect 51 | github.com/klauspost/compress v1.17.3 // indirect 52 | github.com/mailru/easyjson v0.7.7 // indirect 53 | github.com/mattn/go-shellwords v1.0.12 // indirect 54 | github.com/mattn/go-zglob v0.0.4 // indirect 55 | github.com/mitchellh/go-homedir v1.1.0 // indirect 56 | github.com/mitchellh/go-testing-interface v1.14.2-0.20210821155943-2d9075ca8770 // indirect 57 | github.com/mitchellh/go-wordwrap v1.0.1 // indirect 58 | github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect 59 | github.com/tidwall/gjson v1.18.0 // indirect 60 | github.com/tidwall/match v1.1.1 // indirect 61 | github.com/tidwall/pretty v1.2.1 // indirect 62 | github.com/tidwall/sjson v1.2.5 // indirect 63 | github.com/tmccombs/hcl2json v0.6.4 // indirect 64 | github.com/ulikunitz/xz v0.5.11 // indirect 65 | github.com/zclconf/go-cty v1.15.1 // indirect 66 | go.opencensus.io v0.24.0 // indirect 67 | golang.org/x/crypto v0.31.0 // indirect 68 | golang.org/x/mod v0.22.0 // indirect 69 | golang.org/x/net v0.33.0 // indirect 70 | golang.org/x/oauth2 v0.24.0 // indirect 71 | golang.org/x/sync v0.10.0 // indirect 72 | golang.org/x/sys v0.28.0 // indirect 73 | golang.org/x/text v0.21.0 // indirect 74 | golang.org/x/time v0.8.0 // indirect 75 | golang.org/x/tools v0.22.0 // indirect 76 | golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect 77 | google.golang.org/api v0.206.0 // indirect 78 | google.golang.org/appengine v1.6.8 // indirect 79 | google.golang.org/genproto v0.0.0-20241113202542-65e8d215514f // indirect 80 | google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 // indirect 81 | google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 // indirect 82 | google.golang.org/grpc v1.67.1 // indirect 83 | google.golang.org/protobuf v1.35.1 // indirect 84 | gopkg.in/yaml.v3 v3.0.1 // indirect 85 | k8s.io/kube-openapi v0.0.0-20231113174909-778a5567bc1e // indirect 86 | sigs.k8s.io/kustomize/kyaml v0.18.1 // indirect 87 | sigs.k8s.io/yaml v1.4.0 // indirect 88 | ) 89 | -------------------------------------------------------------------------------- /modules/authorization/metadata.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2025 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | apiVersion: blueprints.cloud.google.com/v1alpha1 16 | kind: BlueprintMetadata 17 | metadata: 18 | name: terraform-google-bigquery-authorization 19 | annotations: 20 | config.kubernetes.io/local-config: "true" 21 | spec: 22 | info: 23 | title: BigQuery Authorized Datasets, Views and Routines 24 | source: 25 | repo: https://github.com/terraform-google-modules/terraform-google-bigquery 26 | sourceType: git 27 | dir: /modules/authorization 28 | version: 10.2.1 29 | actuationTool: 30 | flavor: Terraform 31 | version: ">= 1.3" 32 | description: {} 33 | content: 34 | examples: 35 | - name: basic_bq 36 | location: examples/basic_bq 37 | - name: basic_view 38 | location: examples/basic_view 39 | - name: data_warehouse 40 | location: examples/data_warehouse 41 | - name: multiple_tables 42 | location: examples/multiple_tables 43 | - name: scheduled_queries 44 | location: examples/scheduled_queries 45 | interfaces: 46 | variables: 47 | - name: dataset_id 48 | description: Unique ID for the dataset being provisioned. 49 | varType: string 50 | required: true 51 | - name: project_id 52 | description: Project where the dataset and table are created 53 | varType: string 54 | required: true 55 | - name: roles 56 | description: An array of objects that define dataset access for one or more entities. 57 | varType: any 58 | defaultValue: [] 59 | - name: authorized_views 60 | description: An array of views to give authorize for the dataset 61 | varType: |- 62 | list(object({ 63 | dataset_id = string, 64 | project_id = string, 65 | table_id = string # this is the view id, but we keep table_id to stay consistent as the resource 66 | })) 67 | defaultValue: [] 68 | - name: authorized_datasets 69 | description: An array of datasets to be authorized on the dataset 70 | varType: |- 71 | list(object({ 72 | dataset_id = string, 73 | project_id = string, 74 | })) 75 | defaultValue: [] 76 | - name: authorized_routines 77 | description: An array of authorized routine to be authorized on the dataset 78 | varType: |- 79 | list(object({ 80 | project_id = string, 81 | dataset_id = string, 82 | routine_id = string 83 | })) 84 | defaultValue: [] 85 | outputs: 86 | - name: authorized_dataset 87 | description: Authorized datasets for the BQ dataset 88 | - name: authorized_roles 89 | description: Authorized roles for the dataset 90 | - name: authorized_views 91 | description: Authorized views for the dataset 92 | requirements: 93 | roles: 94 | - level: Project 95 | roles: 96 | - roles/run.invoker 97 | - roles/storage.admin 98 | - roles/workflows.admin 99 | - roles/bigquery.admin 100 | - roles/aiplatform.admin 101 | - roles/cloudfunctions.admin 102 | - roles/dataform.admin 103 | - roles/iam.serviceAccountTokenCreator 104 | - roles/iam.serviceAccountUser 105 | - roles/logging.configWriter 106 | - roles/serviceusage.serviceUsageAdmin 107 | - roles/datalineage.viewer 108 | - roles/iam.serviceAccountAdmin 109 | - roles/resourcemanager.projectIamAdmin 110 | services: 111 | - bigquery.googleapis.com 112 | - bigqueryconnection.googleapis.com 113 | - bigquerystorage.googleapis.com 114 | - cloudkms.googleapis.com 115 | - cloudresourcemanager.googleapis.com 116 | - iam.googleapis.com 117 | - serviceusage.googleapis.com 118 | providerVersions: 119 | - source: hashicorp/google 120 | version: ">= 4.44, < 7" 121 | -------------------------------------------------------------------------------- /modules/udf/main.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | module "bq_find_in_set" { 18 | source = "terraform-google-modules/gcloud/google" 19 | version = "~> 3.0" 20 | enabled = var.add_udfs 21 | 22 | platform = "linux" 23 | additional_components = ["bq"] 24 | 25 | create_cmd_entrypoint = "bq" 26 | destroy_cmd_entrypoint = "bq" 27 | 28 | create_cmd_body = < 0 THEN 0 33 | ELSE 34 | ( 35 | WITH list AS ( 36 | SELECT ROW_NUMBER() OVER() id, l FROM UNNEST(SPLIT(strList, ',')) l 37 | ) 38 | (SELECT id FROM list WHERE l = str) 39 | ) 40 | END 41 | );" 42 | EOT 43 | 44 | destroy_cmd_body = "--project_id ${var.project_id} query --use_legacy_sql=false \"DROP FUNCTION IF EXISTS ${var.dataset_id}.find_in_set\"" 45 | } 46 | 47 | module "bq_check_protocol" { 48 | source = "terraform-google-modules/gcloud/google" 49 | version = "~> 3.0" 50 | enabled = var.add_udfs 51 | 52 | platform = "linux" 53 | additional_components = ["bq"] 54 | 55 | create_cmd_entrypoint = "bq" 56 | destroy_cmd_entrypoint = "bq" 57 | 58 | create_cmd_body = < 35 | ## Inputs 36 | 37 | | Name | Description | Type | Default | Required | 38 | |------|-------------|------|---------|:--------:| 39 | | create\_ignore\_service\_accounts | Whether or not to ignore creation of a service account if an account of the same name already exists | `string` | `true` | no | 40 | | dataform\_region | Region that is used to deploy Dataform resources. This does not limit where resources can be run or what region data must be located in. | `string` | `null` | no | 41 | | deletion\_protection | Whether or not to protect GCS resources from deletion when solution is modified or changed. | `string` | `false` | no | 42 | | enable\_apis | Whether or not to enable underlying apis in this solution. | `string` | `true` | no | 43 | | force\_destroy | Whether or not to protect BigQuery resources from deletion when solution is modified or changed. | `string` | `true` | no | 44 | | labels | A map of labels to apply to contained resources. | `map(string)` | 
{
  "data-warehouse": true
}
| no | 45 | | project\_id | Google Cloud Project ID | `string` | n/a | yes | 46 | | region | Google Cloud Region | `string` | n/a | yes | 47 | | text\_generation\_model\_name | Name of the BigQuery ML GenAI remote model that connects to the LLM used for text generation | `string` | `"text_generate_model"` | no | 48 | 49 | ## Outputs 50 | 51 | | Name | Description | 52 | |------|-------------| 53 | | bigquery\_editor\_url | The URL to launch the BigQuery editor with the sample query procedure opened | 54 | | ds\_friendly\_name | Dataset name | 55 | | lookerstudio\_report\_url | The URL to create a new Looker Studio report displays a sample dashboard for the e-commerce data analysis | 56 | | neos\_tutorial\_url | The URL to launch the in-console tutorial for the EDW solution | 57 | | raw\_bucket | Raw bucket name | 58 | 59 | 60 | 61 | ## Requirements 62 | 63 | These sections describe requirements for using this module. 64 | 65 | ### Software 66 | 67 | The following dependencies must be available: 68 | 69 | - [Terraform](https://github.com/hashicorp/terraform) v0.13 70 | - [Terraform Provider for GCP](https://github.com/hashicorp/terraform-provider-google) plugin v3.0 71 | 72 | ### Service Account 73 | 74 | A service account with the following roles must be used to provision 75 | the resources of this module: 76 | 77 | - Storage Admin: `roles/storage.admin` 78 | - BigQuery Admin: `roles/bigquery.admin` 79 | - Workflows Admin: `roles/workflows.admin` 80 | - Dataplex Admin: `roles/dataplex.admin` 81 | 82 | The [Project Factory module](./.terraform/modules/project-services/README.md) and the 83 | [IAM module](https://github.com/terraform-google-modules/terraform-google-iam) may be used in combination to provision a 84 | service account with the necessary roles applied. 85 | 86 | ### APIs 87 | 88 | A project with the following APIs enabled must be used to host the 89 | resources of this module: 90 | 91 | - Vertex AI API: `aiplatform.googleapis.com` 92 | - Artifact Registry API: `artifactregistry.googleapis.com` 93 | - BigQuery API: `bigquery.googleapis.com` 94 | - BigQuery Connection API: `bigqueryconnection.googleapis.com` 95 | - BigQuery Data Policy API: `bigquerydatapolicy.googleapis.com` 96 | - BigQuery Data Transfer Service API: `bigquerydatatransfer.googleapis.com` 97 | - BigQuery Migration API: `bigquerymigration.googleapis.com` 98 | - BigQuery Reservations API: `bigqueryreservation.googleapis.com` 99 | - BigQuery Storage API: `bigquerystorage.googleapis.com` 100 | - Google Cloud APIs: `cloudapis.googleapis.com` 101 | - Cloud Build API: `cloudbuild.googleapis.com` 102 | - Cloud Functions API: `cloudfunctions.googleapis.com` 103 | - Compute Engine API: `compute.googleapis.com` 104 | - Infrastructure Manager API: `config.googleapis.com` 105 | - Data Catalog API: `datacatalog.googleapis.com` 106 | - Dataform API: `dataform.googleapis.com` 107 | - Data Lineage API: `datalineage.googleapis.com` 108 | - Notebooks API: `notebooks.googleapis.com` 109 | - Cloud Run API: `run.googleapis.com` 110 | - Service Usage API: `serviceusage.googleapis.com` 111 | - Google Cloud Storage API: `storage.googleapis.com` 112 | - Google Cloud Storage JSON API: `storage-api.googleapis.com` 113 | - Google Cloud Workflows API: `workflows.googleapis.com` 114 | 115 | The [Project Factory module](./.terraform/modules/project-services/README.md) can be used to 116 | provision a project with the necessary APIs enabled. 117 | 118 | 119 | ## Security Disclosures 120 | 121 | Please see our [security disclosure process](../../SECURITY.md). 122 | -------------------------------------------------------------------------------- /modules/data_warehouse/workflows.tf: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2023 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | # Set up the Workflow 18 | ## Create the Workflows service account 19 | resource "google_service_account" "workflow_manage_sa" { 20 | project = module.project-services.project_id 21 | account_id = "cloud-workflow-sa-${random_id.id.hex}" 22 | display_name = "Service Account for Cloud Workflows" 23 | description = "Service account used to manage Cloud Workflows" 24 | create_ignore_already_exists = var.create_ignore_service_accounts 25 | 26 | 27 | depends_on = [time_sleep.wait_after_apis] 28 | } 29 | 30 | ## Define the IAM roles granted to the Workflows service account 31 | locals { 32 | workflow_roles = [ 33 | "roles/bigquery.connectionUser", 34 | "roles/bigquery.dataEditor", 35 | "roles/bigquery.jobUser", 36 | "roles/iam.serviceAccountTokenCreator", 37 | "roles/iam.serviceAccountUser", 38 | "roles/run.invoker", 39 | "roles/storage.objectAdmin", 40 | "roles/workflows.admin", 41 | ] 42 | } 43 | 44 | ## Grant the Workflow service account access 45 | resource "google_project_iam_member" "workflow_manage_sa_roles" { 46 | count = length(local.workflow_roles) 47 | project = module.project-services.project_id 48 | member = "serviceAccount:${google_service_account.workflow_manage_sa.email}" 49 | role = local.workflow_roles[count.index] 50 | 51 | depends_on = [google_service_account.workflow_manage_sa] 52 | } 53 | 54 | ## Create the workflow 55 | resource "google_workflows_workflow" "workflow" { 56 | name = "initial-workflow" 57 | project = module.project-services.project_id 58 | region = var.region 59 | description = "Runs post Terraform setup steps for Solution in Console" 60 | service_account = google_service_account.workflow_manage_sa.id 61 | 62 | source_contents = templatefile("${path.module}/templates/workflow.tftpl", { 63 | raw_bucket = google_storage_bucket.raw_bucket.name, 64 | dataset_id = google_bigquery_dataset.ds_edw.dataset_id, 65 | function_url = google_cloudfunctions2_function.notebook_deploy_function.url 66 | function_name = google_cloudfunctions2_function.notebook_deploy_function.name 67 | }) 68 | 69 | deletion_protection = var.deletion_protection 70 | 71 | labels = var.labels 72 | 73 | depends_on = [ 74 | google_project_iam_member.workflow_manage_sa_roles, 75 | time_sleep.wait_after_function 76 | ] 77 | } 78 | 79 | module "workflow_polling_1" { 80 | source = "./workflow_polling" 81 | workflow_id = google_workflows_workflow.workflow.id 82 | input_workflow_state = null 83 | 84 | depends_on = [ 85 | google_storage_bucket.raw_bucket, 86 | google_bigquery_routine.sp_bigqueryml_generate_create, 87 | google_bigquery_routine.sp_bigqueryml_model, 88 | google_bigquery_routine.sproc_sp_demo_lookerstudio_report, 89 | google_bigquery_routine.sp_provision_lookup_tables, 90 | google_workflows_workflow.workflow, 91 | google_storage_bucket.raw_bucket, 92 | google_cloudfunctions2_function.notebook_deploy_function, 93 | time_sleep.wait_after_function, 94 | google_service_account_iam_member.workflow_auth_function 95 | ] 96 | } 97 | 98 | 99 | module "workflow_polling_4" { 100 | source = "./workflow_polling" 101 | workflow_id = google_workflows_workflow.workflow.id 102 | input_workflow_state = module.workflow_polling_1.workflow_state 103 | 104 | depends_on = [ 105 | module.workflow_polling_1 106 | ] 107 | } 108 | 109 | # TODO: Expand testing to support more dynamic polling of workflow state 110 | # module "workflow_polling_1" { 111 | # source = "./workflow_polling" 112 | 113 | # workflow_id = google_workflows_workflow.workflow.id 114 | # input_workflow_state = null 115 | 116 | # depends_on = [ 117 | # google_storage_bucket.raw_bucket, 118 | # google_bigquery_routine.sp_bigqueryml_generate_create, 119 | # google_bigquery_routine.sp_bigqueryml_model, 120 | # google_bigquery_routine.sproc_sp_demo_lookerstudio_report, 121 | # google_bigquery_routine.sp_provision_lookup_tables, 122 | # google_workflows_workflow.workflow, 123 | # google_storage_bucket.raw_bucket, 124 | # google_cloudfunctions2_function.notebook_deploy_function, 125 | # time_sleep.wait_after_function, 126 | # google_service_account_iam_member.workflow_auth_function 127 | # ] 128 | # } 129 | 130 | # module "workflow_polling_2" { 131 | # source = "./workflow_polling" 132 | # workflow_id = google_workflows_workflow.workflow.id 133 | 134 | # input_workflow_state = module.workflow_polling_1.workflow_state 135 | 136 | # depends_on = [ 137 | # module.workflow_polling_1 138 | # ] 139 | # } 140 | 141 | # module "workflow_polling_3" { 142 | # source = "./workflow_polling" 143 | # workflow_id = google_workflows_workflow.workflow.id 144 | 145 | # input_workflow_state = module.workflow_polling_2.workflow_state 146 | 147 | # depends_on = [ 148 | # module.workflow_polling_2 149 | # ] 150 | # } 151 | 152 | # module "workflow_polling_4" { 153 | # source = "./workflow_polling" 154 | # workflow_id = google_workflows_workflow.workflow.id 155 | 156 | # input_workflow_state = module.workflow_polling_3.workflow_state 157 | 158 | # depends_on = [ 159 | # module.workflow_polling_3 160 | # ] 161 | # } 162 | -------------------------------------------------------------------------------- /test/integration/data_warehouse/data_warehouse_test.go: -------------------------------------------------------------------------------- 1 | // Copyright 2023 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | package multiple_buckets 16 | 17 | import ( 18 | "fmt" 19 | "log" 20 | "os" 21 | "reflect" 22 | "strings" 23 | "testing" 24 | "time" 25 | 26 | "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/bq" 27 | "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/gcloud" 28 | "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/tft" 29 | "github.com/GoogleCloudPlatform/cloud-foundation-toolkit/infra/blueprint-test/pkg/utils" 30 | "github.com/stretchr/testify/assert" 31 | ) 32 | 33 | // TODO: Remove because Eventarc is no longer in use 34 | // Retry if these errors are encountered. 35 | var retryErrors = map[string]string{ 36 | // IAM for Eventarc service agent is eventually consistent 37 | ".*Permission denied while using the Eventarc Service Agent.*": "Eventarc Service Agent IAM is eventually consistent", 38 | } 39 | 40 | func TestDataWarehouse(t *testing.T) { 41 | dwh := tft.NewTFBlueprintTest(t, tft.WithRetryableTerraformErrors(retryErrors, 10, time.Minute)) 42 | 43 | dwh.DefineVerify(func(assert *assert.Assertions) { 44 | dwh.DefaultVerify(assert) 45 | 46 | projectID := dwh.GetTFSetupStringOutput("project_id") 47 | bucket := dwh.GetStringOutput("raw_bucket") 48 | workflow := "initial-workflow" 49 | location := "us-central1" 50 | 51 | // Assert that the bucket is in location defined above 52 | bucketOP := gcloud.Runf(t, "storage buckets describe gs://%s --project %s", bucket, projectID) 53 | assert.Equal("US-CENTRAL1", bucketOP.Get("location").String(), "Bucket should be in %[1]s \n", strings.ToUpper(location)) 54 | 55 | // Assert that Workflow ran successfully 56 | verifyWorkflows := func() (bool, error) { 57 | workflowState := gcloud.Runf(t, "workflows executions list %s --project %s --location=%s --limit=1", workflow, projectID, location).Array() 58 | state := workflowState[0].Get("state").String() 59 | assert.NotEqual(t, state, "FAILED") 60 | if state == "SUCCEEDED" { 61 | return false, nil 62 | } else { 63 | return true, nil 64 | } 65 | } 66 | utils.Poll(t, verifyWorkflows, 8, 30*time.Second) 67 | 68 | homeDir, err := os.UserHomeDir() 69 | if err != nil { 70 | log.Fatal(err) 71 | } 72 | file, err := os.Create(homeDir + "/.bigqueryrc") 73 | if err != nil { 74 | log.Fatal(err) 75 | } 76 | file.Close() 77 | 78 | // Assert BigQuery tables & views are not empty 79 | test_tables := func (){ 80 | 81 | tables := []string{ 82 | "thelook.distribution_centers", 83 | "thelook.events", 84 | "thelook.inventory_items", 85 | "thelook.order_items", 86 | "thelook.orders", 87 | "thelook.products", 88 | "thelook.users", 89 | "thelook.lookerstudio_report_distribution_centers", 90 | "thelook.lookerstudio_report_profit", 91 | } 92 | 93 | query_template := "SELECT COUNT(*) AS count_rows FROM `%[1]s.%[2]s`;" 94 | for _, table := range tables { 95 | query := fmt.Sprintf(query_template, projectID, table) 96 | op := bq.Runf(t, "--project_id=%[1]s --headless=true query --nouse_legacy_sql %[2]s", projectID, query) 97 | 98 | count := op.Get("0.count_rows").Int() 99 | count_kind := reflect.TypeOf(count).Kind() 100 | test_result := assert.Greater(count, int64(0)) 101 | if test_result == true { 102 | } else { 103 | fmt.Printf("Some kind of error occurred while running the count query for the %[1]s table. We think it has %[2]d rows. Test failed. Here's some additional details: \n Query results. If this number is greater than 0, then there is probably an issue in the comparison: %[3]s \n Variable type for the count. This should be INT64: %[4]s \n ", table, count, op, count_kind) 104 | } 105 | } 106 | } 107 | test_tables() 108 | 109 | // Assert BigQuery connection to Vertex GenAI was successfully created and works as expected 110 | test_llms := func() { 111 | 112 | llm_query_template := "SELECT COUNT(*) AS count_rows FROM ML.GENERATE_TEXT(MODEL `%[1]s.thelook.text_generate_model`, (with clusters AS(SELECT CONCAT('cluster', CAST(centroid_id as STRING)) as centroid, avg_spend as average_spend, count_orders as count_of_orders, days_since_order FROM (SELECT centroid_id, feature, ROUND(numerical_value, 2) as value FROM ML.CENTROIDS(MODEL `%[1]s.thelook.customer_segment_clustering`)) PIVOT (SUM(value) FOR feature IN ('avg_spend', 'count_orders', 'days_since_order')) ORDER BY centroid_id) SELECT 'Pretend you are a creative strategist, given the following clusters come up with creative brand persona and title labels for each of these clusters, and explain step by step; what would be the next marketing step for these clusters' || ' ' || clusters.centroid || ', Average Spend $' || clusters.average_spend || ', Count of orders per person ' || clusters.count_of_orders || ', Days since last order ' || clusters.days_since_order AS prompt FROM clusters), STRUCT(800 AS max_output_tokens, 0.8 AS temperature, 40 AS top_k, 0.8 AS top_p, TRUE AS flatten_json_output));" 113 | query := fmt.Sprintf(llm_query_template, projectID) 114 | llm_op := bq.Runf(t, "--project_id=%[1]s --headless=true query --nouse_legacy_sql %[2]s", projectID, query) 115 | 116 | llm_count := llm_op.Get("0.count_rows").Int() 117 | count_llm_kind := reflect.TypeOf(llm_count).Kind() 118 | llm_test_result := assert.Greater(llm_count, int64(0)) 119 | if llm_test_result == true { 120 | } else { 121 | fmt.Printf("Some kind of error occurred while running the llm_count query. We think it has %[1]d rows. Here's some additional details: \n Query results. If this number is greater than 0, then there is probably an issue in the comparison: %[2]s \n Variable type for the count. This should be INT64: %[3]s \n ", llm_count, llm_op, count_llm_kind) 122 | } 123 | } 124 | test_llms() 125 | }) 126 | dwh.Test() 127 | } 128 | --------------------------------------------------------------------------------