├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── proactive-governance ├── README.md ├── main.tf └── policies │ └── constraints │ └── enforce_label.yaml └── reactive-governance ├── .gitignore ├── README.md ├── alert ├── deploy │ ├── main.tf │ └── variables.tf └── src │ ├── cas_alert.py │ └── requirements.txt ├── img ├── cas-dashboard.png ├── cas-reactive-alerting-architecture.png ├── cas-reactive-reporting-architecture.png └── cas-scheduler.png ├── label-automation └── project_label_automation │ ├── README.md │ ├── deploy │ ├── main.tf │ └── variables.tf │ ├── img │ ├── apply_label_output.png │ ├── csv_valid_1.png │ ├── csv_valid_2.png │ └── csv_with_errors.png │ └── src │ ├── automate_project_label.py │ └── requirements.txt ├── report ├── deploy │ ├── asset_table_schema.txt │ ├── asset_table_view_schema.txt │ ├── main.tf │ ├── variables.tf │ └── view_query.tftpl └── src │ ├── cas_report.py │ └── requirements.txt └── sample-deployment ├── main.tf ├── terraform.tfvars.example └── variables.tf /.gitignore: -------------------------------------------------------------------------------- 1 | sendgrid.env 2 | target 3 | *.json 4 | terraform.tfstate* 5 | terraform/example/*.zip 6 | reactive-governance/sample-deployment/*.zip 7 | terraform.tfvars 8 | 9 | .* 10 | !/.gitignore 11 | !/.github 12 | !/.markdownlint.json 13 | *.iml 14 | .DS_Store 15 | .DS_Store 16 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. 4 | 5 | ## Before you begin 6 | 7 | ### Sign our Contributor License Agreement 8 | 9 | Contributions to this project must be accompanied by a 10 | [Contributor License Agreement](https://cla.developers.google.com/about) (CLA). 11 | You (or your employer) retain the copyright to your contribution; this simply 12 | gives us permission to use and redistribute your contributions as part of the 13 | project. 14 | 15 | If you or your current employer have already signed the Google CLA (even if it 16 | was for a different project), you probably don't need to do it again. 17 | 18 | Visit to see your current agreements or to 19 | sign a new one. 20 | 21 | ### Review our Community Guidelines 22 | 23 | This project follows 24 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 25 | 26 | ## Contribution process 27 | 28 | ### Code Reviews 29 | 30 | All submissions, including submissions by project members, require review. We 31 | use GitHub pull requests for this purpose. Consult 32 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 33 | information on using pull requests. 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GCP Labels for Enterprises 2 | > This repository consists solutions and tools to effectively use labels for managing Google Cloud 3 | > resources at scale. 4 | 5 | Do you run or administer applications on Google Cloud? Do you want to identify costs for individuals 6 | or application teams on Google Cloud? Do you want to automate billing for the enterprise with hundreds 7 | and thousands of employees using Google Cloud? If your answer is yes, this repository is for you. Google 8 | Cloud provides Labels to help enterprises identify and manage costs. Labels are a powerful tool to track 9 | your GCP usage and resources at scale, and with the granularity you need. 10 | 11 | ## What are Labels? 12 | [Labels](https://cloud.google.com/resource-manager/docs/creating-managing-labels) are key-value pairs 13 | that are supported by a number of GCP resources. You can attach a label to each resource, then filter the 14 | resources based on their labels. Labels provide a convenient way for developers and administrators to 15 | organize resources at scale. 16 | 17 | ## Using labels to understand costs 18 | Information about labels is forwarded to the billing system, so you can 19 | [break down your billed charges](https://cloud.google.com/billing/docs/how-to/bq-examples) by label. 20 | When you enable the [export of billing data](https://cloud.google.com/billing/docs/how-to/export-data-bigquery) 21 | to [BigQuery](https://cloud.google.com/bigquery/), labels are exported to BigQuery with all corresponding 22 | GCP resources and their usage. This makes it easier for CIOs and managers to answer questions such as: 23 | * What does the shopping cart service in my application cost to run? 24 | * How much do I spend on developer test machines? 25 | 26 | ## Enforce labels: proactive and reactive governance 27 | It is highly recommended to enforce labels and make sure each resource is labelled appropriately. Use 28 | proactive and reactive governance strategies with tools to enforce labels. Create a list of standard 29 | labels that have to be set to each resource and with proactive governance, ensure all newly created 30 | resources have this set of labels. You can use a streamlined pipeline to create a resource. Another 31 | approach is to have a tools in place that checks for mandatory labels and informs you or even shuts 32 | down the resources until labels are set as reactive governance on labels. 33 | 34 | This repository provides tools 35 | for [Proactive](https://github.com/google/cost-attribution-solution/tree/main/proactive-governance/) and 36 | [Reactive](https://github.com/google/cost-attribution-solution/tree/main/reactive-governance/) governance 37 | of labels. 38 | 39 | ## What is Next? 40 | 41 | 1. Cost Attribution for resources when labels/tags are not applied 42 | 2. Support for tags 43 | 44 | ## Getting Support 45 | 46 | Cost Attribution Solution is a project based on open source contributions. We'd 47 | love for you to [report issues, file feature requests][new-issue], and 48 | [send pull requests][new-pr] (see [Contributing](README.md#7-contributing)). For questions or feedback 49 | contact [cost-attribution-solution@google.com](mailto:cost-attribution-solution@google.com). Cost Attribution Solution is not 50 | officially covered by the Google Cloud product support. 51 | 52 | ## Contributing 53 | 54 | * [Contributing guidelines][contributing-guidelines] 55 | * [Code of conduct][code-of-conduct] 56 | 57 | 58 | 59 | [code-of-conduct]: code-of-conduct.md 60 | [contributing-guidelines]: CONTRIBUTING.md 61 | [new-issue]: https://github.com/google/quota-monitoring-solution/issues/new 62 | [new-pr]: https://github.com/google/quota-monitoring-solution/compare 63 | 64 | 65 | -------------------------------------------------------------------------------- /proactive-governance/README.md: -------------------------------------------------------------------------------- 1 | # Proactive governance: Terraform Validator 2 | Automate label creation using centralized CI/CD pipeline and enforce labels using 3 | [Terraform Policy Validation](https://cloud.google.com/docs/terraform/policy-validation). Terraform 4 | policy validator automates and ensures that you never miss GCP labels on the resource while provisioning 5 | it. You can take labeling input while each resource gets provisioned using Terraform and CI/CD pipeline. 6 | 7 | Businesses are shifting towards infrastructure-as-code, and with that change comes a concern that 8 | configuration errors can cause security and governance violations. To address this, security and cloud 9 | administrators need to be able to set up guardrails that make sure everyone in their organization follows 10 | security best practices. These guardrails are in the form of *constraints*. 11 | 12 | Constraints define your organization's source of truth for security and governance requirements. 13 | The constraints must be compatible with tools across every stage of the application lifecycle, from 14 | development, to deployment, and even to an audit of deployed resources. 15 | 16 | ## One way to define constraints 17 | 18 | Constraints are defined so that they can work across an ecosystem of pre-deployment and monitoring tools. 19 | These constraints live in your organization's repository as the source of truth for your security and 20 | governance requirements. You can obtain constraints from the Policy Library, or build your own constraint 21 | templates. 22 | 23 | ## Pre-deployment check 24 | 25 | Check for constraint violations during pre-deployment and provide warnings or halt invalid deployments 26 | before they reach production. 27 | 28 | ## Integrate Terraform Policy Validator 29 | Follow these steps to integrate validator with your CI/CD pipeline. These constraint will enforce labeling 30 | resources during provisioning. If resources are not validated as per the guidelines configured in the 31 | labels.yaml file, deployment using terraform will fail. 32 | 1. Go to Cloud Shell and clone the policy library. 33 | ```sh 34 | git clone https://github.com/GoogleCloudPlatform/policy-library.git 35 | ``` 36 | 2. Copy the sample Enforce Labels constraint into the policies/constraints directory. 37 | ```sh 38 | cp samples/enforce_label.yaml policies/constraints/enforce_label.yaml 39 | ``` 40 | 3. Update enforce labels constraint with a sample constraint provided with this repository file 41 | [enforce_label.yaml](/policies/constraints/enforce_label.yaml) 42 | This sample is based on the [Best Practices for Labels](https://cloud.google.com/resource-manager/docs/best-practices-labels#example_labels_table) on GCP. 43 | 4. To verify that the policy works as expected, create the following Terraform main.tf file in the 44 | current directory and copy contents with the sample [main.tf](/proactive-governance/main.tf) provided 45 | with this repository 46 | ```sh 47 | vi main.tf 48 | ``` 49 | 5. Initialize Terraform and generate a Terraform plan using the following: 50 | ```sh 51 | terraform init 52 | ``` 53 | 6. Export the Terraform plan, if asked, click Authorize when prompted: 54 | ```sh 55 | terraform plan -out=test.tfplan 56 | ``` 57 | 7. Convert the Terraform plan to JSON: 58 | ```sh 59 | terraform show -json ./test.tfplan > ./tfplan.json 60 | ``` 61 | 8. Install the terraform-tools component: 62 | ```sh 63 | sudo apt-get install google-cloud-sdk-terraform-tools 64 | ``` 65 | 9. Enter the following command to validate that your Terraform plan complies with your policies: 66 | ```sh 67 | gcloud beta terraform vet tfplan.json --policy-library=. --format=json 68 | ``` 69 | Since the labels are not applied to the resource, the plan violates the constraint you set up and 70 | throws error. 71 | 10. Now apply label on the resource in the main.tf 72 | ```sh 73 | resource "google_compute_instance" "default" { 74 | name = "my-instance" 75 | machine_type = "n2-standard-2" 76 | zone = "us-central1-a" 77 | 78 | tags = ["foo", "bar"] 79 | labels ={ 80 | altostrat-environment = "dev" 81 | altostrat-data-classification = "na" 82 | altostrat-cost-center = "it-jp" 83 | altostrat-team = "shopping-cart" 84 | altostrat-component = "frontend" 85 | altostrat-app = "shopping-cart-payment" 86 | altostrat-compliance = "pci" 87 | } 88 | ... 89 | ``` 90 | 11. Now validate your Terraform plan again, and this should result in no violations found: 91 | ```sh 92 | gcloud beta terraform vet tfplan.json --policy-library=. --format=json 93 | ``` 94 | Expected output: 95 | ```sh 96 | [] 97 | ``` 98 | 99 | >Note: In this sample use case, only non-compliant projects, buckets and VMs would be flagged. Read 100 | instructions and update resource list to be scanned for labels in policies/constraints/enforce_label.yaml. 101 | Also, update replace 'alostrat' with your organization name and label keys/values as per your labeling strategy. 102 | 103 | ## CI/CD Example 104 | A bash script for using gcloud beta terraform vet in a CI/CD pipeline might look like this 105 | [CI/CD example](https://cloud.google.com/docs/terraform/policy-validation/validate-policies#cicd_example). 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | -------------------------------------------------------------------------------- /proactive-governance/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | terraform { 16 | required_providers { 17 | google = { 18 | source = "hashicorp/google" 19 | version = "~> 3.84" 20 | } 21 | } 22 | } 23 | 24 | resource "google_service_account" "default" { 25 | account_id = "my-custom-sa" 26 | display_name = "Custom SA for VM Instance" 27 | } 28 | 29 | // Sample compute instance 30 | resource "google_compute_instance" "default" { 31 | name = "my-instance" 32 | machine_type = "n2-standard-2" 33 | zone = "us-central1-a" 34 | 35 | tags = ["foo", "bar"] 36 | 37 | boot_disk { 38 | initialize_params { 39 | image = "debian-cloud/debian-11" 40 | labels = { 41 | my_label = "value" 42 | } 43 | } 44 | } 45 | 46 | // Local SSD disk 47 | scratch_disk { 48 | interface = "NVME" 49 | } 50 | 51 | network_interface { 52 | network = "default" 53 | 54 | access_config { 55 | // Ephemeral public IP 56 | } 57 | } 58 | 59 | metadata = { 60 | foo = "bar" 61 | } 62 | 63 | metadata_startup_script = "echo hi > /test.txt" 64 | 65 | service_account { 66 | # Google recommends custom service accounts that have cloud-platform scope and permissions granted via IAM Roles. 67 | email = google_service_account.default.email 68 | scopes = ["cloud-platform"] 69 | } 70 | } -------------------------------------------------------------------------------- /proactive-governance/policies/constraints/enforce_label.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | apiVersion: constraints.gatekeeper.sh/v1alpha1 16 | kind: GCPEnforceLabelConstraintV1 17 | metadata: 18 | name: require_labels 19 | annotations: 20 | description: Checks that labels are set for all resources (or a subset of resources) 21 | and that they match a certain regular expression pattern. 22 | spec: 23 | severity: high 24 | match: 25 | ancestries: 26 | - "organizations/**" 27 | parameters: 28 | # required parameter: list of label objects that resources should have. 29 | # A label object is composed of a key value pair like: 30 | # 31 | # "label_key": "label_value_regex_to_match" 32 | # 33 | # Any missing label results in a violation. For instance a resource with no label1 or label2 label, 34 | # in this sample case, would raise 2 violations: one for label1 being absent and one for label2. 35 | # 36 | # In the same spirit, a resource with label1 or label2 present, but with values not matching their respective regex 37 | # would also raise one violation per mismatch. 38 | # 39 | # In the following example, valid values for a label named "label1" would be only "label1-value", 40 | # but a label named label2 could have various values like "label2-value", "label2-valueOK" etc. 41 | # 42 | # A violation is raised if the label value does not match the pattern passed as a parameter here. 43 | mandatory_labels: 44 | - "altostrat-environment": "^(?i)(prod|staging|dev|tools|shared)" 45 | - "altostrat-data-classification": "^(?i)(na|low|moderate|high )" 46 | - "altostrat-cost-center": "^(?i)(fin-us|mkt-eu|it-jp)" 47 | - "altostrat-team": "" 48 | - "altostrat-component": "^(?i)(frontend|cache|application|database)" 49 | - "altostrat-app": "" 50 | - "altostrat-compliance": "^(?i)(pci|hipaa)" 51 | 52 | # optional parameter: list of resource types to scan for labels 53 | # any resource that is not of these types will not raise any violation. 54 | # In this sample use case, only non-compliant projects and buckets would be flagged. 55 | # If not passed, all tested resource types would be scanned for (see template for full list) 56 | resource_types_to_scan: 57 | - "cloudresourcemanager.googleapis.com/Project" 58 | - "storage.googleapis.com/Bucket" 59 | - "compute.googleapis.com/Instance" 60 | -------------------------------------------------------------------------------- /reactive-governance/.gitignore: -------------------------------------------------------------------------------- 1 | sendgrid.env 2 | target 3 | *.json 4 | terraform.tfstate* 5 | terraform/example/*.zip 6 | 7 | .gitignore 8 | !/.gitignore 9 | !/.github 10 | !/.markdownlint.json 11 | *.iml 12 | -------------------------------------------------------------------------------- /reactive-governance/README.md: -------------------------------------------------------------------------------- 1 | # Reactive Governance - Reporting and Alerting 2 | If your enterprise is in the phase of running workloads on GCP and there are resources that are missing labels, 3 | use reactive governance. As a best practice, it is recommended to use reactive governance along with 4 | [proactive governance](https://github.com/google/cost-attribution-solution/tree/main/proactive-governance) 5 | to ensure all resources are labeled appropriately. To implement reactive governance, frequently 6 | scan the platform for constraint violations on labels and send notifications when a violation is found. 7 | 8 | There are 3 parts of the reactive governance solution: 9 | * **Reporting** resources with missing labels. 10 | * **Alerting** when resource is created or updated without labels. 11 | * **Label Automation** efficiently and accurately applies labels to projects, streamlining workflows and reducing errors. 12 | 13 | ## Reporting Architecture 14 | ![architecture](../reactive-governance/img/cas-reactive-reporting-architecture.png) 15 | 16 | The architecture is built using Google Cloud managed services - Cloud Scheduler, 17 | Functions, Pub/Sub, BigQuery and Looker studio. 18 | 19 | * The solution is architected to scale using Pub/Sub. 20 | * Cloud Scheduler is used to trigger Cloud Functions. This is also an user 21 | interface to configure frequency, parent nodes etc. 22 | * Cloud Function is used to export list of organization assets into BigQuery table. 23 | * BigQuery is used to store data and create view with resources that do not have labels. 24 | * Easy to get started and deploy with Looker Studio Dashboard. In addition to 25 | Looker Studio, other visualization tools can be configured. 26 | * The Looker Studio report can be scheduled to be emailed to appropriate team 27 | for weekly/daily reporting. 28 | 29 | ## Alerting Architecture 30 | ![architecture](../reactive-governance/img/cas-reactive-alerting-architecture.png) 31 | 32 | The architecture is built using Google Cloud managed services - Asset Feed, 33 | Functions, Pub/Sub and Cloud Monitoring. 34 | 35 | * Asset Feeds are subscribed using Pub/Sub. 36 | * Cloud Function is used to filter and log asset feed. 37 | * Log metric is configured to trigger an alert for asset feed. 38 | 39 | ### 1. Prerequisites 40 | 41 | 1. Host Project - A project where the BigQuery instance, Cloud Functions and 42 | Cloud Schedulers will be deployed. For example Project A. 43 | 2. Target Node - The Organization or folder or project which will be scanned 44 | for Assets. For example Org A or Folder A. 45 | 3. Project Owner role on host Project A. 46 | 4. Google Cloud SDK is installed. Detailed instructions to install the SDK 47 | [here](https://cloud.google.com/sdk/docs/install#mac). See the Getting Started 48 | page for an introduction to using gcloud and terraform. 49 | 5. Terraform version >= 0.14.6 installed. Instructions to install terraform here 50 | * Verify terraform version after installing. 51 | 52 | ```sh 53 | terraform -version 54 | ``` 55 | 56 | The output should look like: 57 | 58 | ```sh 59 | Terraform v0.14.6 60 | + provider registry.terraform.io/hashicorp/google v3.57.0 61 | ``` 62 | 63 | *Note - Minimum required version v0.14.6. Lower terraform versions may not work.* 64 | 65 | ## 2. Initial Setup 66 | 67 | 1. In local workstation or create a new directory to run terraform and store 68 | credential file 69 | 70 | ```sh 71 | mkdir 72 | cd 73 | ``` 74 | 75 | 2. Set default project in config to host project A 76 | 77 | ```sh 78 | gcloud config set project 79 | ``` 80 | 81 | The output should look like: 82 | 83 | ```sh 84 | Updated property [core/project]. 85 | ``` 86 | 87 | 3. Ensure that the latest version of all installed components is installed on 88 | the local workstation. 89 | 90 | ```sh 91 | gcloud components update 92 | ``` 93 | 94 | 4. Cloud Scheduler depends on the App Engine application. Create an App Engine 95 | application in the host project. Replace the region. List of regions where 96 | App Engine is available can be found 97 | [here](https://cloud.google.com/about/locations#region). 98 | 99 | ```sh 100 | gcloud app create --region= 101 | ``` 102 | 103 | Note: Cloud Scheduler (below) needs to be in the same region as App Engine. 104 | Use the same region in terraform as mentioned here. 105 | 106 | The output should look like: 107 | 108 | ```sh 109 | You are creating an app for project [quota-monitoring-project-3]. 110 | WARNING: Creating an App Engine application for a project is irreversible and the region 111 | cannot be changed. More information about regions is at 112 | . 113 | 114 | Creating App Engine application in project [quota-monitoring-project-1] and region [us-east1]....done. 115 | 116 | Success! The app is now created. Please use `gcloud app deploy` to deploy your first app. 117 | ``` 118 | 119 | ## 3. Create Service Account 120 | 121 | 1. In local workstation, setup environment variables. Replace the name of the 122 | Service Account in the commands below 123 | 124 | ```sh 125 | export DEFAULT_PROJECT_ID=$(gcloud config get-value core/project 2> /dev/null) 126 | export SERVICE_ACCOUNT_ID="sa-"$DEFAULT_PROJECT_ID 127 | export DISPLAY_NAME="sa-"$DEFAULT_PROJECT_ID 128 | ``` 129 | 130 | 2. Verify host project Id. 131 | 132 | ```sh 133 | echo $DEFAULT_PROJECT_ID 134 | ``` 135 | 136 | 3. Create Service Account 137 | 138 | ```sh 139 | gcloud iam service-accounts create $SERVICE_ACCOUNT_ID --description="Service Account to deploy cost attribution solution and export assets" --display-name=$DISPLAY_NAME 140 | ``` 141 | 142 | The output should look like: 143 | 144 | ```sh 145 | Created service account [sa-cost-attribution-solution-1]. 146 | ``` 147 | 148 | ## 4. Grant Roles to Service Account 149 | 150 | ### 4.1 Grant Roles in the Host Project 151 | 152 | The following roles need to be added to the Service Account in the host 153 | project i.e. Project A: 154 | 155 | * BigQuery 156 | * BigQuery Admin 157 | * Cloud Functions 158 | * Cloud Functions Admin 159 | * Cloud Scheduler 160 | * Cloud Scheduler Admin 161 | * Pub/Sub 162 | * Pub/Sub Admin 163 | * Run Terraform 164 | * Service Account User 165 | * Service Usage Admin 166 | * Monitoring 167 | * Notification Channel Editor 168 | * Alert Policy Editor 169 | * Viewer 170 | * Metric Writer 171 | * Logs 172 | * Logs Configuration Writer 173 | * Log Writer 174 | * Cloud Asset 175 | * Cloud Asset Owner 176 | 177 | 1. Run following commands to assign the roles: 178 | 179 | ```sh 180 | export SA_ROLES="roles/bigquery.admin \ 181 | roles/cloudfunctions.admin \ 182 | roles/cloudscheduler.admin \ 183 | roles/pubsub.admin \ 184 | roles/iam.serviceAccountUser \ 185 | roles/serviceusage.serviceUsageAdmin \ 186 | roles/monitoring.notificationChannelEditor \ 187 | roles/monitoring.alertPolicyEditor \ 188 | roles/logging.configWriter \ 189 | roles/logging.logWriter \ 190 | roles/monitoring.viewer \ 191 | roles/monitoring.metricWriter \ 192 | roles/cloudasset.owner \ 193 | roles/storage.admin" 194 | 195 | for role in $SA_ROLES; do 196 | echo "Assigning role: $role" 197 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID \ 198 | --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" \ 199 | --role $role --condition=None 200 | done 201 | ``` 202 | 203 | ### 4.2 Grant Roles in the Target Organization 204 | 205 | If you want to scan projects in the org, add following roles to the Service 206 | Account created in the previous step at the Org A: 207 | 208 | * Reporting 209 | * Cloud Asset Viewer 210 | * Alerting 211 | * Cloud Asset Owner - This permission allows solution to subscribe to asset feed. This solution does not 212 | modify assets at the org level 213 | 214 | 1. Set target organization id 215 | 216 | ```sh 217 | export TARGET_ORG_ID= 218 | ``` 219 | 220 | 2. Run the following commands to add to the roles to the service account 221 | * Reporting: 222 | ```sh 223 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudasset.viewer" --condition=None 224 | ``` 225 | * Alerting: 226 | ```sh 227 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudasset.owner" --condition=None 228 | ``` 229 | 230 | ### 4.3 Download the Source Code 231 | 232 | 1. Clone the Cost attribute Solution repo 233 | 234 | ```sh 235 | git clone https://github.com/google/cost-attribution-solution.git cost-attribution-solution 236 | ``` 237 | 238 | 2. Change directories into the Terraform example 239 | 240 | ```sh 241 | cd cost-attribution-solution/reactive-governance/sample-deployment 242 | ``` 243 | 244 | ### 4.4 Set OAuth Token Using Service Account Impersonization 245 | 246 | Impersonate your host project service account and set environment variable 247 | using temporary token to authenticate terraform. You will need to make 248 | sure your user has the 249 | [Service Account Token Creator role](https://cloud.google.com/iam/docs/service-account-permissions#token-creator-role) 250 | to create short-lived credentials. 251 | 252 | ```sh 253 | gcloud config set auth/impersonate_service_account \ 254 | $SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com 255 | 256 | export GOOGLE_OAUTH_ACCESS_TOKEN=$(gcloud auth print-access-token) 257 | ``` 258 | 259 | * **TIP**: If you get an error saying *unable to impersonate*, you will 260 | need to unset the impersonation. Have the role added similar to below, then 261 | try again. 262 | 263 | ```sh 264 | # unset impersonation 265 | gcloud config unset auth/impersonate_service_account 266 | 267 | # set your current authenticated user as var 268 | PROJECT_USER=$(gcloud config get-value core/account) 269 | 270 | # grant IAM role serviceAccountTokenCreator 271 | gcloud iam service-accounts add-iam-policy-binding $SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com \ 272 | --member user:$PROJECT_USER \ 273 | --role roles/iam.serviceAccountTokenCreator \ 274 | --condition=None 275 | ``` 276 | 277 | ## 5. Configure Terraform 278 | 279 | 1. Verify that you have these 3 files in your local directory: 280 | * main.tf 281 | * variables.tf 282 | * terraform.tfvars.example 283 | 284 | 2. Open [terraform.tfvars](sample-deployment/terraform.tfvars.example) file in your 285 | favourite editor and change values for the variables. 286 | 287 | ```sh 288 | cp terraform.tfvars.example terraform.tfvars 289 | vi terraform.tfvars 290 | ``` 291 | 292 | 3. For `region`, use the same region as used for App Engine in earlier steps. 293 | 294 | ## 6. Run Terraform 295 | 296 | 1. Run terraform commands 297 | * `terraform init` 298 | * `terraform plan` 299 | * `terraform apply` 300 | * On Prompt Enter a value: `yes` 301 | 302 | 2. This will: 303 | * Enable required APIs 304 | * Create all resources and connect them. 305 | 306 | Note: In case terraform fails, run terraform plan and terraform apply again 307 | 308 | 3. Stop impersonating service account (when finished with terraform) 309 | 310 | ```sh 311 | gcloud config unset auth/impersonate_service_account 312 | ``` 313 | 314 | ## 7. Test Reporting 315 | 316 | 1. Initiate first job run in Cloud Scheduler. 317 | 318 | **Console** 319 | 320 | Click 'Force Run' on Cloud Job scheduler. 321 | 322 | *Note: The status of the ‘Run Now’ button changes to ‘Running’ for a fraction 323 | of seconds.* 324 | 325 | ![run-cloud-scheduler](../reactive-governance/img/cas-scheduler.png) 326 | 327 | **Terminal** 328 | 329 | ```sh 330 | gcloud scheduler jobs run cas-scheduler --location 331 | ``` 332 | 333 | 2. To verify that the program ran successfully, check the BigQuery Table. The 334 | time to load data in BigQuery might take a few minutes. The execution time 335 | depends on the number of projects to scan. 336 | 337 | 338 | ## 8. Reporting Dashboard 339 | Any visualization tool that works with BigQuery can be used. For this setup, Looker Studio has been used. 340 | 341 | ### 8.1 Looker Studio Dashboard Setup 342 | 343 | 1. Go to the [Looker Studio dashboard template](https://lookerstudio.google.com/s/l2haE0mW5cc). 344 | A Looker Studio dashboard will look like this: 345 | ![ds-cas-reporting-dashboard](../reactive-governance/img/cas-dashboard.png) 346 | 2. Make a copy of the template from the copy icon at the top bar (top - right 347 | corner) 348 | 3. Click on ‘Copy Report’ button **without changing datasource options** 349 | 4. This will create a copy of the report and open in Edit mode. If not click on 350 | ‘Edit’ button on top right corner in copied template 351 | 5. Select any one table. On the 352 | right panel in ‘Data’ tab, click on icon ‘edit data source’ 353 | It will open the data source details 354 | 6. Replace the BigQuery with view created by the solutions. 355 | 7. After making sure that query is returning results, replace it in the Data 356 | Studio, click on the ‘Reconnect’ button in the data source pane. 357 | 8. In the next window, click on the ‘Done’ button. 358 | 9. Once the data source is configured, click on the ‘View’ button on the top 359 | right corner. 360 | 361 | Note: make additional changes in the layout like which metrics to be displayed 362 | on Dashboard, color shades for consumption column, number of rows for each 363 | table etc in the ‘Edit’ mode. 364 | 365 | ### 8.1 Scheduled Reporting 366 | 367 | Cost Attribution reports can be scheduled from the Looker Studio dashboard using 368 | ‘Schedule email delivery’. The screenshot of the Looker Studio dashboard will be 369 | delivered as a pdf report to the configured email Ids. 370 | 371 | ## 9. Test Alerting 372 | Make sure email id is provided in the terraform.tfvars file during terraform setup in the 373 | earlier steps. By default, following services are configured to receive alerts: 374 | * Compute Instance 375 | * Cloud Storage Bucket 376 | * Bigquery Dataset 377 | * Bigquery Table 378 | * PubSub Topic 379 | * PubSub Subscription 380 | 381 | For additional resources, add services in the main.tf file under "project_feed". 382 | 383 | For testing, add a compute instance and do not add label, you should receive alerts via email. 384 | Also, if a compute instance which had label, remove label and save. You should receive 385 | an alert via email. 386 | 387 | ## 10. Label Automation (Reconciliation) 388 | 389 | Go beyond just reporting and alerting by actively enforcing your labeling policies on 390 | existing projects. This empowers you to automate the application of correct labels to 391 | unlabeled or mislabeled resources, for comprehensive cost visibility and data accuracy 392 | across your entire Google Cloud landscape. Streamline your labeling workflow with 393 | [these steps](https://github.com/google/cost-attribution-solution/tree/main/reactive-governance/label-automation/project_label_automation/) for automated labeling. 394 | 395 | -------------------------------------------------------------------------------- /reactive-governance/alert/deploy/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | ## RESOURCES TO SETUP ALERTING WHEN RESOURCE IS CREATED OR UPDATED AND LABEL IS MISSING ## 18 | 19 | terraform { 20 | provider_meta "google" { 21 | module_name = "cloud-solutions/cost-attribution-solution-alert-v1.1" 22 | } 23 | } 24 | 25 | locals { 26 | expanded_region = var.region == "us-central" || var.region == "europe-west" ? "${var.region}1" : var.region 27 | } 28 | 29 | # Create Pub/Sub topic to list projects in the parent node 30 | resource "google_pubsub_topic" "cas_alerting_topic" { 31 | name = var.cas_alerting_topic 32 | } 33 | 34 | # Create a feed that sends notifications on resource creation or update events for project. 35 | resource "google_cloud_asset_project_feed" "project_feed" { 36 | count = length(var.organization_id) == 0 ? 1 : 0 37 | 38 | project = var.project_id 39 | feed_id = "missing-labels-feed" 40 | content_type = "RESOURCE" 41 | asset_types = var.asset_types 42 | 43 | feed_output_config { 44 | pubsub_destination { 45 | topic = google_pubsub_topic.cas_alerting_topic.id 46 | } 47 | } 48 | 49 | condition { 50 | expression = <<-EOT 51 | !temporal_asset.deleted 52 | EOT 53 | title = "created or updated" 54 | description = "Send notifications on creation or update events" 55 | } 56 | depends_on = [google_pubsub_topic.cas_alerting_topic] 57 | } 58 | 59 | # Create a feed that sends notifications on resource creation or update events for organization. 60 | resource "google_cloud_asset_organization_feed" "organization_feed" { 61 | count = length(var.organization_id) == 0 ? 0 : 1 62 | 63 | org_id = var.organization_id 64 | billing_project = var.project_id 65 | feed_id = "missing-labels-feed" 66 | content_type = "RESOURCE" 67 | asset_types = var.asset_types 68 | 69 | feed_output_config { 70 | pubsub_destination { 71 | topic = google_pubsub_topic.cas_alerting_topic.id 72 | } 73 | } 74 | 75 | condition { 76 | expression = <<-EOT 77 | !temporal_asset.deleted 78 | EOT 79 | title = "created or updated" 80 | description = "Send notifications on creation or update events" 81 | } 82 | depends_on = [google_pubsub_topic.cas_alerting_topic] 83 | } 84 | 85 | data "archive_file" "alert_source_code_zip" { 86 | type = "zip" 87 | source_dir = abspath("${path.module}/../../alert/src/") 88 | output_path = "./alert_src.zip" 89 | } 90 | 91 | resource "google_storage_bucket_object" "alert_source_code_object" { 92 | name = "src.${data.archive_file.alert_source_code_zip.output_md5}.zip" 93 | bucket = var.bucket_gcf_source_name 94 | source = data.archive_file.alert_source_code_zip.output_path 95 | 96 | depends_on = [ 97 | data.archive_file.alert_source_code_zip 98 | ] 99 | } 100 | 101 | resource "google_cloudfunctions2_function" "cas_alert_function" { 102 | name = var.cloud_function_cas_alerting 103 | location = local.expanded_region 104 | description = var.cloud_function_cas_alerting_desc 105 | 106 | build_config { 107 | runtime = "python310" 108 | entry_point = "cas_alert" 109 | environment_variables = { 110 | GOOGLE_FUNCTION_SOURCE = "cas_alert.py" 111 | } 112 | source { 113 | storage_source { 114 | bucket = var.bucket_gcf_source_name 115 | object = google_storage_bucket_object.alert_source_code_object.name 116 | } 117 | } 118 | } 119 | 120 | service_config { 121 | available_memory = var.cloud_function_cas_reporting_memory 122 | timeout_seconds = var.cloud_function_cas_reporting_timeout 123 | service_account_email = var.service_account_email 124 | environment_variables = { 125 | LOG_EXECUTION_ID = "true" 126 | } 127 | } 128 | 129 | event_trigger { 130 | trigger_region = local.expanded_region 131 | event_type = "google.cloud.pubsub.topic.v1.messagePublished" 132 | pubsub_topic = google_pubsub_topic.cas_alerting_topic.id 133 | retry_policy = "RETRY_POLICY_RETRY" 134 | } 135 | 136 | depends_on = [google_pubsub_topic.cas_alerting_topic] 137 | } 138 | 139 | # Custom log-based metric to send quota alert data through 140 | resource "google_logging_metric" "quota_logging_metric" { 141 | name = var.cas_alert_log_metric 142 | description = "Tracks logs for resources missing labels" 143 | # filter = "logName:\"projects/${var.project_id}/logs/\" jsonPayload.message:\"Resource with missing Label - Name: \"" 144 | filter = "resource.type=\"cloud_run_revision\" AND resource.labels.service_name=\"cas-alert\" AND SEARCH(textPayload, \"Resource with missing Label\")" 145 | metric_descriptor { 146 | metric_kind = "DELTA" 147 | value_type = "INT64" 148 | unit = "1" 149 | labels { 150 | key = "data" 151 | value_type = "STRING" 152 | } 153 | } 154 | label_extractors = { 155 | "data" = "EXTRACT(jsonPayload.message)" 156 | } 157 | } 158 | 159 | #Set notification channels below 160 | #Add Notification channel - Email 161 | resource "google_monitoring_notification_channel" "email0" { 162 | display_name = "Oncall" 163 | type = "email" 164 | labels = { 165 | email_address = var.notification_email_address 166 | } 167 | } 168 | 169 | #Alert policy for log-based metric 170 | # Condition display name can be changed based on user's quota range 171 | resource "google_monitoring_alert_policy" "alert_policy_quota" { 172 | display_name = "Resources missing label" 173 | combiner = "OR" 174 | conditions { 175 | display_name = "Resources missing label" 176 | condition_threshold { 177 | filter = "metric.type=\"logging.googleapis.com/user/${google_logging_metric.quota_logging_metric.name}\" AND resource.type=\"cloud_run_revision\"" 178 | duration = "60s" 179 | comparison = "COMPARISON_GT" 180 | threshold_value = 0 181 | trigger { 182 | count = 1 183 | } 184 | aggregations { 185 | per_series_aligner = "ALIGN_COUNT" 186 | alignment_period = "60s" 187 | } 188 | } 189 | } 190 | documentation { 191 | mime_type = "text/markdown" 192 | content = "$${metric.label.data}" 193 | } 194 | notification_channels = [ 195 | google_monitoring_notification_channel.email0.name 196 | ] 197 | depends_on = [ 198 | google_logging_metric.quota_logging_metric, 199 | google_monitoring_notification_channel.email0 200 | ] 201 | } -------------------------------------------------------------------------------- /reactive-governance/alert/deploy/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | variable "organization_id" { 18 | description = "Value of the Organization Id to export assets and build report" 19 | type = string 20 | } 21 | 22 | variable "project_id" { 23 | description = "Value of the Project Id to deploy the solution" 24 | type = string 25 | } 26 | 27 | variable "region" { 28 | description = "Value of the region to deploy the solution" 29 | type = string 30 | 31 | validation { 32 | condition = var.region != "us-central1" && var.region != "europe-west1" 33 | error_message = "Region must be set to an App Engine location. us-central1 and europe-west1 should be specified as us-central and europe-west respectively." 34 | } 35 | } 36 | 37 | variable "cas_alerting_topic" { 38 | description = "Value of the Pub/Sub topic Id subscribed to asset feed and triggers Cloud Function" 39 | type = string 40 | default = "cas_alerting_topic" 41 | } 42 | 43 | variable "cloud_function_cas_alerting" { 44 | description = "Value of the name for the Cloud Function to Alert for missing labels" 45 | type = string 46 | default = "cas_alert" 47 | } 48 | 49 | variable "cloud_function_cas_alerting_desc" { 50 | description = "Value of the description for the Cloud Function to Alert missing labels" 51 | type = string 52 | default = "Alert when resources created or updated and are missing labels" 53 | } 54 | 55 | variable "cas_alert_log_metric" { 56 | description = "Value of the name for custom log metric" 57 | type = string 58 | default = "cas_alert_log_metric" 59 | } 60 | 61 | variable "notification_email_address" { 62 | description = "Email to receive alerts when resources with missing labels" 63 | type = string 64 | } 65 | 66 | variable "bucket_gcf_source_name" { 67 | description = "Bucket to upload source code to Cloud Function" 68 | type = string 69 | } 70 | 71 | variable "cloud_function_cas_reporting_memory" { 72 | description = "Value of the memory for the Cloud Function to Export Assets in Bigquery" 73 | type = string 74 | default = "512M" 75 | } 76 | 77 | variable "cloud_function_cas_reporting_timeout" { 78 | description = "Value of the timeout for the Cloud Function to Export Assets in Bigquery" 79 | type = number 80 | default = 540 81 | } 82 | 83 | variable "service_account_email" { 84 | description = "Value of the Service Account" 85 | type = string 86 | } 87 | 88 | variable "asset_types" { 89 | type = list(string) 90 | description = "List of asset types to include in the feed. See https://cloud.google.com/asset-inventory/docs/supported-asset-types for supported types." 91 | default = [ 92 | "cloudresourcemanager.googleapis.com/Project", 93 | "compute.googleapis.com/Instance", 94 | "storage.googleapis.com/Bucket", 95 | "bigquery.googleapis.com/Dataset", 96 | "bigquery.googleapis.com/Table", 97 | "pubsub.googleapis.com/Topic", 98 | "pubsub.googleapis.com/Subscription", 99 | ] 100 | } -------------------------------------------------------------------------------- /reactive-governance/alert/src/cas_alert.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import base64 16 | import json 17 | import logging 18 | 19 | import base64 20 | import json 21 | import logging 22 | 23 | def cas_alert(event, context): 24 | """Cloud Function to process CAS alerts from Pub/Sub. 25 | 26 | Args: 27 | event (dict): The dictionary with data that will be passed to the function by the trigger. 28 | context (google.cloud.functions.Context): The Cloud Functions event metadata. 29 | """ 30 | logger = logging.getLogger(__name__) 31 | logger.setLevel(logging.INFO) # Or logging.DEBUG for even lower level 32 | 33 | # Get the Pub/Sub message data 34 | pubsub_message = base64.b64decode(event['data']).decode('utf-8') 35 | # logger.info(f"Pub/Sub message: {pubsub_message}") 36 | 37 | # Check if the message is valid JSON 38 | if pubsub_message.startswith('{') or pubsub_message.startswith('['): 39 | try: 40 | # Parse the JSON message 41 | message_data = json.loads(pubsub_message) 42 | asset = message_data.get('asset', {}) 43 | resource = asset.get('resource', {}) 44 | labels = resource.get('data', {}).get('labels', {}) 45 | 46 | # Log asset and resource information 47 | logger.info(f"Asset Type: {asset.get('assetType')}") 48 | logger.info(f"Name: {asset.get('name')}") 49 | logger.info(f"Parent: {resource.get('parent')}") 50 | logger.info(f"Labels count: {len(labels) if labels else 0}") 51 | 52 | # Check if labels are missing and log a warning 53 | if not labels: 54 | logger.warning( 55 | f"Resource with missing Label - Name: {asset.get('name')} | " 56 | f"Asset Type: {asset.get('assetType')} | " 57 | f"Parent: {resource.get('parent')}" 58 | ) 59 | 60 | except json.JSONDecodeError as e: 61 | logger.error(f"Error decoding JSON: {e}. Raw message: {pubsub_message}") 62 | else: 63 | logger.info(f"Received non-JSON message: {pubsub_message}") -------------------------------------------------------------------------------- /reactive-governance/alert/src/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-functions==1.1.0 2 | google-events==0.10.0 3 | google-cloud-pubsub==2.19.0 -------------------------------------------------------------------------------- /reactive-governance/img/cas-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/img/cas-dashboard.png -------------------------------------------------------------------------------- /reactive-governance/img/cas-reactive-alerting-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/img/cas-reactive-alerting-architecture.png -------------------------------------------------------------------------------- /reactive-governance/img/cas-reactive-reporting-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/img/cas-reactive-reporting-architecture.png -------------------------------------------------------------------------------- /reactive-governance/img/cas-scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/img/cas-scheduler.png -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/README.md: -------------------------------------------------------------------------------- 1 | # Label Automation 2 | Go beyond just reporting and alerting by actively enforcing your labeling policies on 3 | existing projects. This empowers you to automate the application of correct labels to 4 | unlabeled or mislabeled resources, for comprehensive cost visibility and data accuracy 5 | across your entire Google Cloud landscape. 6 | 7 | This Python-based CLI tool provides a solution for clearing and applying labels to Google Cloud projects. 8 | 9 | ## Features 10 | 11 | * **Automated Label Management:** Efficiently clears existing labels and applies new ones to specified projects. 12 | * **GCS Integration:** Reads label data from a CSV file stored in Google Cloud Storage (GCS). 13 | * **Manual Intervention:** Includes a confirmation step before any changes are made, allowing for review and preventing accidental modifications. 14 | * **Error Handling:** Error handling for empty columns or invalid key-value pairs in the input CSV. 15 | * **Secure Execution:** Designed to run on a Compute Engine instance with SSH access, eliminating the need to download Service Account keys locally. 16 | 17 | ## Prerequisites 18 | 19 | * **Python 3.7 or higher:** Ensure you have a compatible Python version installed. 20 | * **Google Cloud Project:** An active GCP project with the necessary APIs enabled (e.g., Cloud Resource Manager). 21 | * **Terraform:** Terraform installed and configured to interact with your GCP project. 22 | 23 | ## Getting Started 24 | 25 | **1. Prepare Label Data:** 26 | 27 | * **Create a CSV file:** Create a CSV file containing the project IDs and the new label key-value pairs you want to apply. 28 | * **CSV File Structure:** Ensure your CSV adheres to the following structure: 29 | ``` 30 | project_id,cost-center,environment,app 31 | my-project-123,fin-ops,prod,shopping-cart 32 | my-project-456,sre,dev,catalog 33 | ``` 34 | * The **first column** must contain the `project_id`. 35 | * **Subsequent column headers** represent the `label keys` (e.g., "cost-center", "environment", "app"). 36 | * The **cells** in each row represent the corresponding `label values` for each project. 37 | Example of a valid CSVs: 38 | 39 |
40 | valid_csv_1 41 | 42 |
valid_csv_2 43 |
44 |
45 | Example of a CSV with errors in values: 46 |
47 | invalid_valid 48 | 49 | 50 | In this example, the following errors are present: 51 | * `shopping cart`: Contains a space. 52 | * `cata%log`: Contains a special character (`%`). 53 | 54 | * **Upload to GCS:** Upload the CSV file to a GCS bucket. 55 | 56 | **1. Setup:** 57 | 58 | * **Terraform Variables:** 59 | * In the cloned repo, move to the current directory and update terraform variables as per your setup: 60 | 61 | cd terraform 62 | vi variables.tf 63 | 64 | **3. Provision Infrastructure:** 65 | 66 | terraform init 67 | terraform plan 68 | terraform apply 69 | This will create the Compute Engine VM and Service Account with the required permissions. 70 | 71 | **4. Execute the Script:** 72 | 73 | * **SSH into the VM:** Connect to the newly created VM instance using SSH. 74 | * Download `automate_project_labels.py` and `requirements.txt` on the VM 75 | * **Prepare the environment:** 76 | ```bash 77 | sudo apt-get install python3-venv 78 | python3 -m venv env 79 | source env/bin/activate 80 | pip install -r requirements.txt 81 | ``` 82 | * **Run the script:** 83 | * The script will prompt you to enter the full GCS path to your CSV file (e.g., `gs://your-bucket-name/labels.csv`). 84 | ```bash 85 | python3 automate_project_labels.py 86 | ``` 87 | 88 | 89 | **5. Sample CLI Output:** 90 | 91 | ![label_output](../../../reactive-governance/label-automation/project_label_automation/img/apply_label_output.png) -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/deploy/main.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | terraform { 16 | provider_meta "google" { 17 | module_name = "cloud-solutions/cost-attribution-solution-automation-prj-v1.1" 18 | } 19 | } 20 | 21 | provider "google" { 22 | project = var.project_id 23 | region = var.region 24 | zone = var.zone 25 | } 26 | 27 | resource "google_service_account" "cas_ca_service_account" { 28 | account_id = var.service_account_id_clear_apply_labels 29 | display_name = var.service_account_name_clear_apply_labels 30 | } 31 | 32 | resource "google_folder_iam_member" "service_account_folder_viewer" { 33 | folder = var.folder_id 34 | role = "roles/resourcemanager.folderViewer" 35 | member = "serviceAccount:${google_service_account.cas_ca_service_account.email}" 36 | } 37 | 38 | resource "google_folder_iam_member" "service_account_project_mover" { 39 | folder = var.folder_id 40 | role = "roles/resourcemanager.projectMover" 41 | member = "serviceAccount:${google_service_account.cas_ca_service_account.email}" 42 | } 43 | 44 | resource "google_storage_bucket_iam_member" "member" { 45 | bucket = var.gcs_bucket 46 | role = "roles/storage.objectViewer" 47 | member = "serviceAccount:${google_service_account.cas_ca_service_account.email}" 48 | } 49 | 50 | resource "google_compute_instance" "default" { 51 | name = var.compute_instance_id 52 | machine_type = "n2-standard-2" 53 | zone = "us-central1-a" 54 | 55 | boot_disk { 56 | initialize_params { 57 | image = "debian-cloud/debian-11" 58 | labels = { 59 | cas = "clear-apply-labels" 60 | } 61 | } 62 | } 63 | 64 | // Local SSD disk 65 | scratch_disk { 66 | interface = "NVME" 67 | } 68 | 69 | network_interface { 70 | network = "default" 71 | 72 | access_config { 73 | // Ephemeral public IP 74 | } 75 | } 76 | 77 | service_account { 78 | email = google_service_account.cas_ca_service_account.email 79 | scopes = ["cloud-platform"] 80 | } 81 | } -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/deploy/variables.tf: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | variable "project_id" { 16 | description = "Value of the Project Id to deploy the solution" 17 | type = string 18 | default = "prj-22-376417" 19 | } 20 | 21 | variable "folder_id" { 22 | description = "Value of the Folder Id to deploy the solution" 23 | type = string 24 | default = "665019192319" 25 | } 26 | 27 | variable "service_account_id_clear_apply_labels" { 28 | description = "Value of the Service Account Id to clear and apply labels" 29 | type = string 30 | default = "cas-apply-labels-sa" 31 | } 32 | 33 | variable "service_account_name_clear_apply_labels" { 34 | description = "Value of the Service Account name to clear and apply labels" 35 | type = string 36 | default = "service account name for clear and apply labels" 37 | } 38 | 39 | variable "region" { 40 | description = "Value of the region to deploy the solution" 41 | type = string 42 | default = "us-central" 43 | 44 | validation { 45 | condition = var.region != "us-central1" && var.region != "europe-west1" 46 | error_message = "us-central1 and europe-west1 should be specified as us-central and europe-west respectively." 47 | } 48 | } 49 | 50 | variable "zone" { 51 | description = "Value of the zone to deploy the solution" 52 | type = string 53 | default = "us-central-c" 54 | } 55 | 56 | variable "gcs_bucket" { 57 | description = "Value of the GCS bucket where csv file is stored with new labeling strategy" 58 | type = string 59 | default = "cost_attribution" 60 | } 61 | 62 | variable "compute_instance_id" { 63 | description = "Value of the compute instance id" 64 | type = string 65 | default = "cas-apply-label-instance" 66 | } -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/img/apply_label_output.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/label-automation/project_label_automation/img/apply_label_output.png -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/img/csv_valid_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/label-automation/project_label_automation/img/csv_valid_1.png -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/img/csv_valid_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/label-automation/project_label_automation/img/csv_valid_2.png -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/img/csv_with_errors.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/cost-attribution-solution/ebf7acf49733624aae83eee2bad764698c85d0b2/reactive-governance/label-automation/project_label_automation/img/csv_with_errors.png -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/src/automate_project_label.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | from google.cloud import resourcemanager_v3 16 | from google.cloud import storage 17 | import googleapiclient.discovery 18 | import pandas as pd 19 | from tabulate import tabulate 20 | import json 21 | import re 22 | import sys 23 | import logging 24 | 25 | 26 | # Create resource manager client 27 | client = resourcemanager_v3.ProjectsClient() 28 | 29 | 30 | def clear_and_apply_project_labels(): 31 | """ 32 | Clear and Apply labels on project with data from CSV. 33 | The program first parses CSV file and validates the data for labels key:value pair 34 | Parameters: 35 | csv_url (str): GCS location of the csv file 36 | """ 37 | # example url - gs://cost_attribution/project_labels.csv 38 | csv_uri = input("\nEnter the CSV URI: ") 39 | 40 | # validate is CSV path is valid and CSV exists. 41 | if not is_valid_gcs_uri_and_exists(csv_uri): 42 | return 43 | 44 | # read projects and labels from csv file and convert into data frame 45 | logging.info("Reading csv file..") 46 | df = read_csv(csv_uri) 47 | logging.info("Clearing labels from projects listed in the csv file") 48 | print("\n=== Clearing Labels ===") 49 | print("========================") 50 | print("\nProjects with labels:") 51 | projects_with_labels = list_project_id(df) 52 | logging.info(f"Total {len(projects_with_labels)} " 53 | f"projects found to clear labels.") 54 | print(f"Total {len(projects_with_labels)} " 55 | f"projects found to clear labels.\n") 56 | 57 | # clear labels from projects that has labels 58 | if len(projects_with_labels) > 0: 59 | cont = input("---> Do you want to clear labels? ENTER Y: ") 60 | if cont.lower() == "y": 61 | print("Deleting current labels...\n") 62 | for project_id in projects_with_labels: 63 | delete_labels(project_id) 64 | 65 | # apply new labels 66 | logging.info("Applying labels on projects listed in the csv file") 67 | print("\n=== New Labels to be applied ===") 68 | print("================================") 69 | all_project_labels = build_labels_dict(df) 70 | cont = input("---> Do you want to apply labels? ENTER Y: ") 71 | if cont.lower() == 'y': 72 | print("Applying new labels...\n") 73 | apply_labels(all_project_labels) 74 | logging.info("Applying labels on all projects completed successfully") 75 | 76 | 77 | def is_valid_gcs_uri_and_exists(uri): 78 | """Checks if a given URI is a valid gs:// URI and if the object exists. 79 | 80 | Args: 81 | uri: The URI to validate. 82 | """ 83 | storage_client = storage.Client() 84 | bucket_name = uri.split("/")[2] 85 | name = "/".join(uri.split("/")[3:]) 86 | bucket = storage_client.bucket(bucket_name) 87 | stats = storage.Blob(bucket=bucket, name=name).exists(storage_client) 88 | if stats: 89 | print("\nCSV found successfully!") 90 | return True 91 | else: 92 | print(f"\nError: Object not found at {uri}") 93 | print("Please provide valid URI in pattern: gs:///") 94 | return False 95 | 96 | 97 | def read_csv(csv_uri): 98 | """ 99 | Read csv file from GCS bucket. The csv file contains project ids and new 100 | labels to be applied 101 | Parameters: 102 | csv_url (str): GCS location of the csv file 103 | Returns: 104 | data frame (data frame): Data frame with csv data 105 | """ 106 | 107 | # Access csv from gcs directly using url 108 | df = pd.read_csv(csv_uri) 109 | df = df.astype(str) # Convert DataFrame to string 110 | # print csv file content as table on console 111 | print("\n=== Print CSV Snippet (This may not display full csv) ===") 112 | print("=========================================================") 113 | print(tabulate(df, headers='keys', tablefmt='psql')) 114 | # Parse CSV and if there is any error in the data, exit program 115 | csv_error_count = parse_csv(df) 116 | if csv_error_count > 0: 117 | exit_program() 118 | else: 119 | logging.info("CSV parsing completed successfully") 120 | print("CSV parsing completed successfully:\n") 121 | return df 122 | 123 | 124 | def list_project_id(df): 125 | """ 126 | Build list of projects from which label needs to be cleared 127 | Parameters: 128 | df (data frame): CSV data converted to data frame 129 | Returns: 130 | projects with labels (list): returns all projects with labels 131 | """ 132 | projects_with_labels = [] 133 | for index, row in df.iterrows(): 134 | # get all rows from column[0] of csv (or dataframe) 135 | # considering column[0] is 'project_id' 136 | project_id = row[df.columns[0]] 137 | # add project on the list if the project has labels 138 | if project_labeled(project_id): 139 | projects_with_labels.append(project_id) 140 | logging.info(project_id) 141 | print(project_id) 142 | return projects_with_labels 143 | 144 | 145 | def project_labeled(project_id): 146 | """ 147 | Check if the project has any labels 148 | Parameters: 149 | project_id (str): project id 150 | Returns: 151 | TRUE or FALSE (boolean): returns TRUE if project has any label, 152 | else FALSE 153 | """ 154 | request = resourcemanager_v3.GetProjectRequest( 155 | name=f"projects/{project_id}" 156 | ) 157 | response = client.get_project(request=request) 158 | labels = response.labels 159 | if not labels: 160 | return False 161 | else: 162 | return True 163 | 164 | 165 | def delete_labels(project_id): 166 | """ 167 | Clear all labels from a project 168 | Parameters: 169 | project_id (str): project id to delete all labels from the project 170 | """ 171 | manager = googleapiclient.discovery.build('cloudresourcemanager', 'v1') 172 | request = manager.projects().get(projectId=project_id) 173 | project = request.execute() 174 | del project['labels'] 175 | request = manager.projects().update(projectId=project_id, body=project) 176 | project = request.execute() 177 | logging.info(f"Labels deleted successfully for project_id: {project_id}") 178 | print(f"Labels deleted successfully for project_id: {project_id}") 179 | 180 | 181 | def apply_project_labels(project_id, new_labels): 182 | """ 183 | Apply new Labels to a project 184 | Parameters: 185 | project_id (str): project id 186 | new_labels (dict): new labels as dictionary 187 | """ 188 | request = resourcemanager_v3.GetProjectRequest( 189 | name=f"projects/{project_id}" 190 | ) 191 | # Make the request 192 | response = client.get_project(request=request) 193 | labels = response.labels 194 | labels.update(new_labels) 195 | response.labels = labels 196 | update_request = resourcemanager_v3.UpdateProjectRequest( 197 | project=response, 198 | update_mask="labels" 199 | ) 200 | update_response = client.update_project(request=update_request) 201 | # todo - error handling 202 | logging.info(f"Labels applied successfully for project_id: {project_id}") 203 | print(f"Labels applied successfully for project_id: {project_id}") 204 | 205 | 206 | def build_labels_dict(df): 207 | """ 208 | Build dictionary object with project id as key and dictionary of labels as value 209 | Parameters: 210 | df (data frame): CSV data converted to data frame 211 | Returns: 212 | all project labels (dict): returns dict with project_id as key and 213 | nested dict of labels as key:value pair for each project_id 214 | """ 215 | all_projects_labels = {} 216 | columns = df.columns 217 | for index, row in df.iterrows(): 218 | project_labels = {} 219 | for column in columns: 220 | # skip project columns and cells where value is empty 221 | if column != "project_id" and not pd.isnull(row[column]): 222 | project_labels[column] = row[column] 223 | json_str = json.dumps(project_labels, indent=4) 224 | print(f"project_id: {row[columns[0]]}, \nlabels: {json_str}\n") 225 | all_projects_labels[row[columns[0]]] = project_labels 226 | return all_projects_labels 227 | 228 | 229 | def apply_labels(all_project_labels): 230 | """ 231 | Apply labels for all project_ids extracted from the csv 232 | Parameters: 233 | all_project_labels (nested dictionary): dictionary contains project_ids 234 | and corresponding labels 235 | """ 236 | for project_id in all_project_labels.keys(): 237 | labels = all_project_labels[project_id] 238 | # print(f"key: {project_id}, \nvalue: {labels}\n") 239 | apply_project_labels(project_id, labels) 240 | 241 | 242 | def parse_csv(df): 243 | columns = df.columns 244 | print("\n=== Validating CSV ===") 245 | print("========================") 246 | key_pattern = r'^([a-z])([a-z0-9_-]{0,62})$' 247 | value_pattern = r'^([a-z0-9_-]{0,62})?$' 248 | warning_count = 0 249 | error_count = 0 250 | 251 | for key in columns: 252 | if "Unnamed" in key: 253 | warning_count += 1 254 | logging.warning("Found empty column in the csv") 255 | print("WARNING: Found empty column in the csv") 256 | if not re.match(key_pattern, key): 257 | error_count += 1 258 | logging.error(f"Label Key '{key}' does not follow recommended pattern.") 259 | print(f"ERROR: Label Key '{key}' does not follow recommended pattern. " 260 | f"Refer https://cloud.google.com/resource-manager/docs/labels-overview#requirements") 261 | for index, row in df.iterrows(): 262 | for column in columns: 263 | if not pd.isnull(row[column]) and not re.match(value_pattern, row[column]): 264 | error_count += 1 265 | logging.error(f" Label Value '{row[column]}' does not follow recommended pattern.") 266 | print(f"ERROR: Label Value '{row[column]}' does not follow recommended pattern. " 267 | f"Refer https://cloud.google.com/resource-manager/docs/labels-overview#requirements") 268 | return error_count 269 | 270 | 271 | def exit_program(): 272 | print("Exiting the program...") 273 | sys.exit(0) 274 | 275 | 276 | 277 | if __name__ == "__main__": 278 | clear_and_apply_project_labels() -------------------------------------------------------------------------------- /reactive-governance/label-automation/project_label_automation/src/requirements.txt: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # https://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | google-cloud-resource-manager 16 | google-cloud-storage 17 | google-api-python-client 18 | pandas 19 | fsspec 20 | gcsfs 21 | tabulate -------------------------------------------------------------------------------- /reactive-governance/report/deploy/asset_table_schema.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "name", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "name": "asset_type", 9 | "type": "STRING", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "name": "resource", 14 | "type": "RECORD", 15 | "mode": "NULLABLE", 16 | "fields": [ 17 | { 18 | "name": "version", 19 | "type": "STRING", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "name": "discovery_document_uri", 24 | "type": "STRING", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "discovery_name", 29 | "type": "STRING", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "name": "resource_url", 34 | "type": "STRING", 35 | "mode": "NULLABLE" 36 | }, 37 | { 38 | "name": "parent", 39 | "type": "STRING", 40 | "mode": "NULLABLE" 41 | }, 42 | { 43 | "name": "data", 44 | "type": "STRING", 45 | "mode": "NULLABLE" 46 | }, 47 | { 48 | "name": "location", 49 | "type": "STRING", 50 | "mode": "NULLABLE" 51 | } 52 | ] 53 | }, 54 | { 55 | "name": "ancestors", 56 | "type": "STRING", 57 | "mode": "REPEATED" 58 | }, 59 | { 60 | "name": "update_time", 61 | "type": "TIMESTAMP", 62 | "mode": "NULLABLE" 63 | } 64 | ] -------------------------------------------------------------------------------- /reactive-governance/report/deploy/asset_table_view_schema.txt: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "name": "url", 4 | "type": "STRING", 5 | "mode": "NULLABLE" 6 | }, 7 | { 8 | "name": "project_id", 9 | "type": "STRING", 10 | "mode": "NULLABLE" 11 | }, 12 | { 13 | "name": "name", 14 | "type": "STRING", 15 | "mode": "NULLABLE" 16 | }, 17 | { 18 | "name": "asset_type", 19 | "type": "STRING", 20 | "mode": "NULLABLE" 21 | }, 22 | { 23 | "name": "discovery_name", 24 | "type": "STRING", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "location", 29 | "type": "STRING", 30 | "mode": "NULLABLE" 31 | }, 32 | { 33 | "name": "ancestors", 34 | "type": "STRING", 35 | "mode": "REPEATED" 36 | } 37 | ] -------------------------------------------------------------------------------- /reactive-governance/report/deploy/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | ## RESOURCES TO CREATE MISSING LABELS DASHBOARD ## 18 | 19 | terraform { 20 | provider_meta "google" { 21 | module_name = "cloud-solutions/cost-attribution-solution-report-v1.1" 22 | } 23 | } 24 | 25 | locals { 26 | expanded_region = var.region == "us-central" || var.region == "europe-west" ? "${var.region}1" : var.region 27 | } 28 | 29 | # Create Pub/Sub topic to list projects in the parent node 30 | resource "google_pubsub_topic" "cas_topic" { 31 | name = var.cas_topic 32 | } 33 | 34 | resource "google_cloud_scheduler_job" "cas_job" { 35 | name = var.scheduler_cas_job_name 36 | description = var.scheduler_cas_job_description 37 | schedule = var.scheduler_cas_job_frequency 38 | region = local.expanded_region 39 | 40 | pubsub_target { 41 | topic_name = google_pubsub_topic.cas_topic.id 42 | data = base64encode("hello") 43 | } 44 | 45 | depends_on = [google_pubsub_topic.cas_topic] 46 | } 47 | 48 | data "archive_file" "report_source_code_zip" { 49 | type = "zip" 50 | source_dir = abspath("${path.module}/../../report/src/") 51 | output_path = "./report_src.zip" 52 | } 53 | 54 | resource "google_storage_bucket_object" "report_source_code_object" { 55 | name = "src.${data.archive_file.report_source_code_zip.output_md5}.zip" 56 | bucket = var.bucket_gcf_source_name 57 | source = data.archive_file.report_source_code_zip.output_path 58 | 59 | depends_on = [ 60 | data.archive_file.report_source_code_zip 61 | ] 62 | } 63 | 64 | resource "google_cloudfunctions2_function" "cas_report_function" { 65 | name = var.cloud_function_cas_reporting 66 | location = local.expanded_region 67 | description = var.scheduler_cas_job_description 68 | 69 | build_config { 70 | runtime = "python310" 71 | entry_point = "cas_report" 72 | environment_variables = { 73 | GOOGLE_FUNCTION_SOURCE = "cas_report.py" 74 | } 75 | source { 76 | storage_source { 77 | bucket = var.bucket_gcf_source_name 78 | object = google_storage_bucket_object.report_source_code_object.name 79 | } 80 | } 81 | } 82 | 83 | service_config { 84 | available_memory = var.cloud_function_cas_reporting_memory 85 | timeout_seconds = var.cloud_function_cas_reporting_timeout 86 | environment_variables = { 87 | PARENT = var.organization_id 88 | PROJECT_ID = var.project_id 89 | BIGQUERY_DATASET = var.bigquery_dataset 90 | BIGQUERY_TABLE = var.bigquery_table 91 | LOG_EXECUTION_ID = "true" 92 | } 93 | service_account_email = var.service_account_email 94 | } 95 | 96 | event_trigger { 97 | trigger_region = local.expanded_region 98 | event_type = "google.cloud.pubsub.topic.v1.messagePublished" 99 | pubsub_topic = google_pubsub_topic.cas_topic.id 100 | retry_policy = "RETRY_POLICY_RETRY" 101 | } 102 | 103 | depends_on = [google_pubsub_topic.cas_topic] 104 | } 105 | 106 | # BigQuery Dataset 107 | resource "google_bigquery_dataset" "dataset" { 108 | dataset_id = var.bigquery_dataset 109 | friendly_name = var.bigquery_dataset 110 | description = var.bigquery_dataset_desc 111 | location = var.location 112 | default_partition_expiration_ms = var.bigquery_dataset_default_partition_expiration_ms 113 | labels = { 114 | solution = "cost-attribute-solution" 115 | } 116 | } 117 | 118 | resource "google_bigquery_table" "default" { 119 | dataset_id = google_bigquery_dataset.dataset.dataset_id 120 | table_id = var.bigquery_table 121 | deletion_protection = false 122 | 123 | #time_partitioning { 124 | # type = var.bigquery_table_partition 125 | #} 126 | 127 | #labels = { 128 | # solution = "cost-attribute-solution" 129 | #} 130 | 131 | schema = file("${path.module}/asset_table_schema.txt") 132 | depends_on = [google_bigquery_dataset.dataset] 133 | } 134 | 135 | resource "google_bigquery_table" "cas_table_view" { 136 | dataset_id = google_bigquery_dataset.dataset.dataset_id 137 | table_id = var.bigquery_table_view 138 | 139 | labels = { 140 | solution = "cost-attribute-solution" 141 | } 142 | 143 | schema = file("${path.module}/asset_table_view_schema.txt") 144 | 145 | view { 146 | query = templatefile("${path.module}/view_query.tftpl", { 147 | cas_table = "${var.project_id}.${google_bigquery_dataset.dataset.dataset_id}.${google_bigquery_table.default.table_id}", 148 | }) 149 | use_legacy_sql = false 150 | } 151 | depends_on = [google_bigquery_table.default] 152 | } -------------------------------------------------------------------------------- /reactive-governance/report/deploy/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | variable "organization_id" { 18 | description = "Value of the Organization Id to export assets and build report" 19 | type = string 20 | } 21 | 22 | variable "project_id" { 23 | description = "Value of the Project Id to deploy the solution" 24 | type = string 25 | } 26 | 27 | variable "region" { 28 | description = "Value of the region to deploy the solution" 29 | type = string 30 | 31 | validation { 32 | condition = var.region != "us-central1" && var.region != "europe-west1" 33 | error_message = "Region must be set to an App Engine location. us-central1 and europe-west1 should be specified as us-central and europe-west respectively." 34 | } 35 | } 36 | 37 | ## RESOURCES TO CREATE MISSING LABELS DASHBOARD ## 38 | variable "bigquery_dataset" { 39 | description = "Value of the BigQuery dataset id to load assets" 40 | type = string 41 | default = "cas_report_dataset" 42 | } 43 | 44 | variable "bigquery_dataset_desc" { 45 | description = "Value of the BigQuery dataset description to load assets" 46 | type = string 47 | default = "Dataset to load organization assets data" 48 | } 49 | 50 | variable "bigquery_dataset_default_partition_expiration_ms" { 51 | description = "Value of the Big Query Dataset default partition expiration" 52 | type = number 53 | default = 86400000 54 | } 55 | 56 | variable "bigquery_table" { 57 | description = "Value of the BigQuery table to load assets" 58 | type = string 59 | default = "cas_table" 60 | } 61 | 62 | variable "bigquery_table_view" { 63 | description = "Value of the BigQuery view to extract assets without labels" 64 | type = string 65 | default = "cas_view_missing_labels" 66 | } 67 | 68 | variable "bigquery_table_partition" { 69 | description = "Value of the Big Query Table time partitioning" 70 | type = string 71 | default = "DAY" 72 | } 73 | 74 | variable "location" { 75 | description = "Location of where the BigQuery data should be stored, use US or EU for multi-region storage, or use any other region id for single region storage." 76 | type = string 77 | } 78 | 79 | variable "cas_topic" { 80 | description = "Value of the Pub/Sub topic Id to trigger Cloud Function" 81 | type = string 82 | default = "cas_report_topic" 83 | } 84 | 85 | variable "scheduler_cas_job_name" { 86 | description = "Value of name of job scheduler" 87 | type = string 88 | default = "cas_report_job" 89 | } 90 | 91 | variable "scheduler_cas_job_description" { 92 | description = "Value of description of job scheduler" 93 | type = string 94 | default = "trigger cost attribution monitoring cloud function" 95 | } 96 | 97 | variable "scheduler_cas_job_frequency" { 98 | description = "Value of the cas job frequency to trigger the solution" 99 | type = string 100 | } 101 | 102 | variable "bucket_gcf_source_name" { 103 | description = "Bucket to upload source code to Cloud Function" 104 | type = string 105 | } 106 | 107 | variable "cloud_function_cas_reporting" { 108 | description = "Value of the name for the Cloud Function to Export Assets in Bigquery" 109 | type = string 110 | default = "cas_report" 111 | } 112 | 113 | variable "cloud_function_cas_reporting_desc" { 114 | description = "Value of the description for the Cloud Function to Export Assets in Bigquery" 115 | type = string 116 | default = "Export Assets in Bigquery" 117 | } 118 | 119 | variable "cloud_function_cas_reporting_memory" { 120 | description = "Value of the memory for the Cloud Function to Export Assets in Bigquery" 121 | type = string 122 | default = "512M" 123 | } 124 | 125 | variable "cloud_function_cas_reporting_timeout" { 126 | description = "Value of the timeout for the Cloud Function to Export Assets in Bigquery" 127 | type = number 128 | default = 540 129 | } 130 | 131 | variable "service_account_email" { 132 | description = "Value of the Service Account" 133 | type = string 134 | } -------------------------------------------------------------------------------- /reactive-governance/report/deploy/view_query.tftpl: -------------------------------------------------------------------------------- 1 | SELECT 2 | name AS url, 3 | CASE 4 | WHEN REGEXP_EXTRACT(name, r'projects/([^/]+)/') is null THEN array_reverse(ancestors)[offset(0)] 5 | ELSE REGEXP_EXTRACT(name, r'projects/([^/]+)/') 6 | END 7 | AS project_id, 8 | REGEXP_EXTRACT(name, r'[^/]+$') AS name, 9 | asset_type, 10 | resource.discovery_name, 11 | resource.location, 12 | ancestors 13 | FROM 14 | `${cas_table}` 15 | WHERE 16 | JSON_EXTRACT(resource.data, '$.labels') IS NULL 17 | AND ( asset_type LIKE '%appengine%' 18 | OR asset_type LIKE '%artifactregistry%' 19 | OR asset_type LIKE '%bigquery%' 20 | OR asset_type LIKE '%privateca%' 21 | OR asset_type LIKE '%bigtable%' 22 | OR asset_type LIKE '%composer%' 23 | OR asset_type LIKE '%cloudfunctions%' 24 | OR asset_type LIKE '%cloudkms%' 25 | OR asset_type LIKE '%run%' 26 | OR asset_type LIKE '%spanner%' 27 | OR asset_type LIKE '%sqladmin%' 28 | OR asset_type LIKE '%storage%' 29 | OR asset_type LIKE '%translate%' 30 | OR asset_type LIKE '%compute%' 31 | OR asset_type LIKE '%dataflow%' 32 | OR asset_type LIKE '%dataproc%' 33 | OR asset_type LIKE '%file%' 34 | OR asset_type LIKE '%container%' 35 | OR asset_type LIKE '%redis%' 36 | OR asset_type LIKE '%pubsub%' 37 | OR asset_type LIKE '%recaptchaenterprise%' 38 | OR asset_type LIKE '%cloudresourcemanager.googleapis.com/v3/projects%' 39 | OR asset_type LIKE '%transcoder%' 40 | OR asset_type LIKE '%recaptchaenterprise%' 41 | OR asset_type LIKE '%aiplatform%' 42 | OR asset_type LIKE '%workflows%' ) 43 | AND asset_type NOT IN ('compute.googleapis.com/Route', 44 | 'compute.googleapis.com/Subnetwork', 45 | 'compute.googleapis.com/Network', 46 | 'compute.googleapis.com/Firewall', 47 | 'compute.googleapis.com/HttpHealthCheck', 48 | 'compute.googleapis.com/InstanceTemplate', 49 | 'run.googleapis.com/Revision', 50 | 'compute.googleapis.com/Router', 51 | 'compute.googleapis.com/TargetPool', 52 | 'artifactregistry.googleapis.com/DockerImage', 53 | 'storage.k8s.io/StorageClass') -------------------------------------------------------------------------------- /reactive-governance/report/src/cas_report.py: -------------------------------------------------------------------------------- 1 | # Copyright 2024 Google LLC 2 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | import os 16 | import logging 17 | 18 | from google.cloud import asset_v1 19 | 20 | # Configure environment variables 21 | PROJECT_ID = os.getenv("PROJECT_ID") 22 | BIGQUERY_DATASET = os.getenv("BIGQUERY_DATASET") 23 | BIGQUERY_TABLE = os.getenv("BIGQUERY_TABLE") 24 | PARENT = os.getenv("PARENT") 25 | 26 | def cas_report(event, context): 27 | """ 28 | Cloud Function to export organization assets to BigQuery table 29 | """ 30 | logger = logging.getLogger(__name__) 31 | logger.setLevel(logging.INFO) # Or logging.DEBUG for even lower level 32 | logger.info("Asset export start") 33 | 34 | client = asset_v1.AssetServiceClient() 35 | 36 | # If org ID is provided, scan all projects in the org or scan the host project 37 | if PARENT and PARENT.strip(): 38 | parent_id = f"organizations/{PARENT}" 39 | else: 40 | parent_id = f"projects/{PROJECT_ID}" 41 | 42 | output_config = asset_v1.OutputConfig( 43 | bigquery_destination=asset_v1.BigQueryDestination( 44 | dataset=f"projects/{PROJECT_ID}/datasets/{BIGQUERY_DATASET}", 45 | table=BIGQUERY_TABLE, 46 | force=True, 47 | ) 48 | ) 49 | 50 | request = asset_v1.ExportAssetsRequest( 51 | parent=parent_id, 52 | content_type=asset_v1.ContentType.RESOURCE, 53 | output_config=output_config, 54 | ) 55 | 56 | operation = client.export_assets(request=request) 57 | response = operation.result() 58 | 59 | logger.info(response) 60 | logger.info("Asset export complete") 61 | 62 | -------------------------------------------------------------------------------- /reactive-governance/report/src/requirements.txt: -------------------------------------------------------------------------------- 1 | google-cloud-asset==3.10.0 -------------------------------------------------------------------------------- /reactive-governance/sample-deployment/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | locals { 18 | expanded_region = var.region == "us-central" || var.region == "europe-west" ? "${var.region}1" : var.region 19 | } 20 | 21 | provider "google" { 22 | project = var.project_id 23 | region = var.region 24 | 25 | // These might be required if the quota project is different than the host project and are not associated with your credentials. 26 | // See https://registry.terraform.io/providers/hashicorp/google/latest/docs/guides/provider_reference#quota-management-configuration for more details. 27 | //user_project_override = true 28 | //billing_project = var.project_id 29 | } 30 | 31 | # Enable Cloud Resource Manager API 32 | module "project-service-cloudresourcemanager" { 33 | source = "terraform-google-modules/project-factory/google//modules/project_services" 34 | version = "4.0.0" 35 | 36 | project_id = var.project_id 37 | activate_apis = [ 38 | "cloudresourcemanager.googleapis.com" 39 | ] 40 | } 41 | 42 | # Enable APIs 43 | module "project-services" { 44 | source = "terraform-google-modules/project-factory/google//modules/project_services" 45 | version = "4.0.0" 46 | 47 | project_id = var.project_id 48 | activate_apis = var.activate_apis 49 | depends_on = [module.project-service-cloudresourcemanager] 50 | } 51 | 52 | resource "google_storage_bucket" "bucket_gcf_source" { 53 | name = "${var.project_id}-gcf-source" 54 | storage_class = "REGIONAL" 55 | location = local.expanded_region 56 | force_destroy = "true" 57 | uniform_bucket_level_access = "true" 58 | depends_on = [module.project-services] 59 | } 60 | 61 | module "cas_alert" { 62 | source = "../alert/deploy" 63 | 64 | organization_id = var.organization_id 65 | project_id = var.project_id 66 | region = var.region 67 | bucket_gcf_source_name = google_storage_bucket.bucket_gcf_source.name 68 | service_account_email = var.service_account_email 69 | notification_email_address = var.notification_email_address 70 | asset_types = var.alert_asset_types 71 | depends_on = [module.project-services] 72 | } 73 | 74 | module "cas_report" { 75 | source = "../report/deploy" 76 | 77 | organization_id = var.organization_id 78 | project_id = var.project_id 79 | region = var.region 80 | location = var.location 81 | bucket_gcf_source_name = google_storage_bucket.bucket_gcf_source.name 82 | service_account_email = var.service_account_email 83 | scheduler_cas_job_frequency = var.scheduler_cas_job_frequency 84 | depends_on = [module.project-services] 85 | } -------------------------------------------------------------------------------- /reactive-governance/sample-deployment/terraform.tfvars.example: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | # Update values 18 | organization_id = "" //update (leave empty if scanning only host project) 19 | project_id = "" //update 20 | region = "us-west1" //update 21 | location = "US" //update 22 | service_account_email = "" //update 23 | notification_email_address = "" //update 24 | scheduler_cas_job_frequency = "0 * * * *" //every hour 25 | # Asset types to be included in the alert if they are missing label. See https://cloud.google.com/asset-inventory/docs/supported-asset-types for supported types 26 | alert_asset_types = ["cloudresourcemanager.googleapis.com/Project", "compute.googleapis.com/Instance",] -------------------------------------------------------------------------------- /reactive-governance/sample-deployment/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2024 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | variable "organization_id" { 18 | description = "Value of the Organization Id to export assets and build report" 19 | type = string 20 | } 21 | 22 | variable "project_id" { 23 | description = "Value of the Project Id to deploy the solution" 24 | type = string 25 | } 26 | 27 | variable "region" { 28 | description = "Value of the region to deploy the solution" 29 | type = string 30 | 31 | validation { 32 | condition = var.region != "us-central1" && var.region != "europe-west1" 33 | error_message = "Region must be set to an App Engine location. us-central1 and europe-west1 should be specified as us-central and europe-west respectively." 34 | } 35 | } 36 | 37 | variable "location" { 38 | description = "Value of the location if region id is not used" 39 | type = string 40 | } 41 | 42 | variable "service_account_email" { 43 | description = "Value of the Service Account" 44 | type = string 45 | } 46 | 47 | variable "notification_email_address" { 48 | description = "Email to receive alerts when resources with missing labels" 49 | type = string 50 | } 51 | 52 | variable "scheduler_cas_job_frequency" { 53 | description = "Value of the cas job frequency to trigger the solution" 54 | type = string 55 | } 56 | 57 | variable "alert_asset_types" { 58 | type = list(string) 59 | description = "List of asset types to include in the alert. See https://cloud.google.com/asset-inventory/docs/supported-asset-types for supported types." 60 | default = [ 61 | "cloudresourcemanager.googleapis.com/Project", 62 | "compute.googleapis.com/Instance", 63 | "storage.googleapis.com/Bucket", 64 | "bigquery.googleapis.com/Dataset", 65 | "bigquery.googleapis.com/Table", 66 | "pubsub.googleapis.com/Topic", 67 | "pubsub.googleapis.com/Subscription", 68 | ] 69 | } 70 | 71 | variable "activate_apis" { 72 | type = list(string) 73 | description = "List of APIs to enable for the project. This is necessary for some asset types to be correctly ingested by the feed." 74 | default = [ 75 | "compute.googleapis.com", 76 | "bigquery.googleapis.com", 77 | "pubsub.googleapis.com", 78 | "cloudscheduler.googleapis.com", 79 | "cloudfunctions.googleapis.com", 80 | "cloudasset.googleapis.com", 81 | "cloudbuild.googleapis.com", 82 | "run.googleapis.com", 83 | "eventarc.googleapis.com", 84 | "storage.googleapis.com", 85 | "logging.googleapis.com", 86 | "monitoring.googleapis.com", 87 | "iam.googleapis.com", 88 | "iamcredentials.googleapis.com", 89 | ] 90 | } 91 | --------------------------------------------------------------------------------