├── .github └── workflows │ ├── linters.yaml │ └── release-please.yaml ├── .gitignore ├── .markdownlint.json ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── QMS_app_alerting.csv ├── README.md ├── code-of-conduct.md ├── img ├── bigquery-orderby-consumption.png ├── ds-copy-report-fixed-new-data-source.png ├── ds-dropdown-copy.png ├── ds-edit-mode-updated.png ├── ds-schedule-email-button.png ├── ds-switch-to-view-mode.png ├── ds-updated-quotas-dashboard.png ├── ds_data_source_config_step_2.png ├── ds_data_source_config_step_3.png ├── ds_datasource_config_step_1.png ├── ds_edit_data_source.png ├── quota-monitoring-alerting-architecture.png ├── quota-monitoring-config-flow.png ├── quota_monitoring_key_features.png ├── run_cloud_scheduler.png ├── service_account_roles.png └── test_bigquery_table.png ├── pom.xml ├── quota-notification ├── pom.xml └── src │ ├── main │ └── java │ │ └── functions │ │ ├── ConfigureAppAlert.java │ │ ├── ConfigureAppAlertHelper.java │ │ ├── SendNotification.java │ │ └── eventpojos │ │ ├── Alert.java │ │ ├── AppAlert.java │ │ └── PubSubMessage.java │ └── test │ └── java │ └── functions │ └── SendNotificationTest.java ├── quota-scan ├── pom.xml └── src │ └── main │ ├── java │ └── functions │ │ ├── ListProjects.java │ │ ├── ScanProjectQuotas.java │ │ ├── ScanProjectQuotasHelper.java │ │ └── eventpojos │ │ ├── GCPProject.java │ │ ├── GCPResourceClient.java │ │ ├── ProjectQuota.java │ │ ├── PubSubMessage.java │ │ └── TimeSeriesQuery.java │ └── resources │ └── config.properties └── terraform ├── example ├── main.tf ├── terraform.tfvars └── variables.tf └── modules └── qms ├── main.tf ├── outputs.tf └── variables.tf /.github/workflows/linters.yaml: -------------------------------------------------------------------------------- 1 | name: linters 2 | on: [push, pull_request] 3 | jobs: 4 | lint: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - name: Check out code 8 | uses: actions/checkout@v3 9 | with: 10 | fetch-depth: 0 11 | - name: Lint the codebase 12 | uses: github/super-linter@v4 13 | env: 14 | VALIDATE_ALL_CODEBASE: false 15 | VALIDATE_MARKDOWN: true 16 | DEFAULT_BRANCH: main 17 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 18 | LINTER_RULES_PATH: / 19 | MARKDOWN_CONFIG_FILE: .markdownlint.json 20 | FILTER_REGEX_EXCLUDE: CHANGELOG.md -------------------------------------------------------------------------------- /.github/workflows/release-please.yaml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - main 5 | name: release-please 6 | jobs: 7 | release-please: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - uses: google-github-actions/release-please-action@v3 11 | id: release 12 | with: 13 | release-type: terraform-module 14 | package-name: ${{env.ACTION_NAME}} 15 | token: ${{ secrets.RELEASE_PLEASE }} 16 | extra-files: | 17 | terraform/modules/qms/variables.tf 18 | terraform/modules/qms/main.tf 19 | - uses: actions/checkout@v3 20 | if: ${{ steps.release.outputs.release_created }} 21 | with: 22 | fetch-depth: 0 23 | - uses: thedoctor0/zip-release@0.6.2 24 | if: ${{ steps.release.outputs.release_created }} 25 | name: Create Zip for quota-scan 26 | with: 27 | type: 'zip' 28 | filename: 'quota-monitoring-solution.zip' 29 | directory: quota-scan 30 | - uses: thedoctor0/zip-release@0.6.2 31 | if: ${{ steps.release.outputs.release_created }} 32 | name: Create Zip for quota-notification 33 | with: 34 | type: 'zip' 35 | filename: 'quota-monitoring-notification.zip' 36 | directory: quota-notification 37 | - uses: ncipollo/release-action@v1 38 | if: ${{ steps.release.outputs.release_created }} 39 | name: Upload 40 | with: 41 | allowUpdates: true 42 | artifacts: "quota-scan/quota-monitoring-solution.zip,quota-notification/quota-monitoring-notification.zip" 43 | omitBodyDuringUpdate: true 44 | tag: ${{ steps.release.outputs.tag_name }} 45 | token: ${{ secrets.RELEASE_PLEASE }} 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | sendgrid.env 2 | quota-notification/target 3 | quota-scan/target 4 | key.json 5 | terraform.tfstate* 6 | terraform/example/*.zip 7 | 8 | .* 9 | !/.gitignore 10 | !/.github 11 | !/.markdownlint.json 12 | *.iml 13 | -------------------------------------------------------------------------------- /.markdownlint.json: -------------------------------------------------------------------------------- 1 | { 2 | "default": true, 3 | "MD007": { 4 | "indent": 4 5 | }, 6 | "MD013": { 7 | "code_blocks": false, 8 | "tables": false 9 | }, 10 | "MD029": { 11 | "style": "ordered" 12 | }, 13 | "MD030": { 14 | "ul_single": 3, 15 | "ol_single": 2, 16 | "ul_multi": 3, 17 | "ol_multi": 2 18 | } 19 | } -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [5.1.0](https://github.com/google/quota-monitoring-solution/compare/v5.0.2...v5.1.0) (2023-09-21) 4 | 5 | 6 | ### Features 7 | 8 | * Add user-agent string to the terraform module. ([#95](https://github.com/google/quota-monitoring-solution/issues/95)) ([87780ec](https://github.com/google/quota-monitoring-solution/commit/87780ecdfce90217e1b822891c30e2b7722734bb)) 9 | 10 | 11 | ### Bug Fixes 12 | 13 | * adding main.tf to the extra files for release-please ([#96](https://github.com/google/quota-monitoring-solution/issues/96)) ([0c48faa](https://github.com/google/quota-monitoring-solution/commit/0c48faa13f185e00329443472cfbab696a563bec)) 14 | * Fix for Issue [#86](https://github.com/google/quota-monitoring-solution/issues/86) (per second quotas) ([#89](https://github.com/google/quota-monitoring-solution/issues/89)) ([c2e5d47](https://github.com/google/quota-monitoring-solution/commit/c2e5d477574bfed4e77be2b6369a1f8d1b94a9ea)) 15 | * Removing the leading pipe from the log filter. ([#90](https://github.com/google/quota-monitoring-solution/issues/90)) ([eab522e](https://github.com/google/quota-monitoring-solution/commit/eab522e9620e9fefdb31639f5ea656b0f10a1cd5)) 16 | * Updating the README to clarify the preferred paths to get support. ([#93](https://github.com/google/quota-monitoring-solution/issues/93)) ([dcdeafb](https://github.com/google/quota-monitoring-solution/commit/dcdeafbfbc578f28e182eff59ab827e5c6d4853d)) 17 | 18 | ## [5.0.2](https://github.com/google/quota-monitoring-solution/compare/v5.0.1...v5.0.2) (2023-05-22) 19 | 20 | 21 | ### Bug Fixes 22 | 23 | * Add quotes to BQ table location ([#84](https://github.com/google/quota-monitoring-solution/issues/84)) ([dcd52c3](https://github.com/google/quota-monitoring-solution/commit/dcd52c30918a5293a07c163158f24faa4456c216)) 24 | 25 | ## [5.0.1](https://github.com/google/quota-monitoring-solution/compare/v5.0.0...v5.0.1) (2023-04-26) 26 | 27 | 28 | ### Bug Fixes 29 | 30 | * Feature/looker studio template public ([#80](https://github.com/google/quota-monitoring-solution/issues/80)) ([4d423e3](https://github.com/google/quota-monitoring-solution/commit/4d423e32b80af4061908704bb8fb89458fabbbbd)) 31 | * Implemented [#67](https://github.com/google/quota-monitoring-solution/issues/67) - use short-lived tokens ([#69](https://github.com/google/quota-monitoring-solution/issues/69)) ([d08475a](https://github.com/google/quota-monitoring-solution/commit/d08475a622b82d21cb125a35571720fb69fe53d3)) 32 | 33 | ## [5.0.0](https://github.com/google/quota-monitoring-solution/compare/v4.5.1...v5.0.0) (2023-01-18) 34 | 35 | 36 | ### ⚠ BREAKING CHANGES 37 | 38 | * App Level Alerting along with centralized alerting ([#61](https://github.com/google/quota-monitoring-solution/issues/61)) 39 | 40 | ### Features 41 | 42 | * App Level Alerting along with centralized alerting ([#61](https://github.com/google/quota-monitoring-solution/issues/61)) ([339a8b6](https://github.com/google/quota-monitoring-solution/commit/339a8b6972f085e944c7b225b1bfc1df0af1f5ff)) 43 | 44 | ## [4.5.1](https://github.com/google/quota-monitoring-solution/compare/v4.5.0...v4.5.1) (2022-12-30) 45 | 46 | 47 | ### Bug Fixes 48 | 49 | * removed redundant code ([415ffbf](https://github.com/google/quota-monitoring-solution/commit/415ffbf33d8c5f9848ed21fe25ec96c6ed3a5d46)) 50 | 51 | ## [4.5.0](https://github.com/google/quota-monitoring-solution/compare/v4.4.0...v4.5.0) (2022-12-23) 52 | 53 | 54 | ### Features 55 | 56 | * Alerting notification in html ([13f5754](https://github.com/google/quota-monitoring-solution/commit/13f5754e07c50e9e77d2640b6f29bddbd25bcc80)) 57 | 58 | 59 | ### Bug Fixes 60 | 61 | * Alerting fixes ([1a11ce8](https://github.com/google/quota-monitoring-solution/commit/1a11ce8393cb187a0db46ed14709a45988066272)) 62 | 63 | ## [4.4.0](https://github.com/google/quota-monitoring-solution/compare/v4.3.0...v4.4.0) (2022-12-22) 64 | 65 | 66 | ### Features 67 | 68 | * Adding release-please to manage version numbers and updating Terraform to pull from GitHub releases. ([56b42a5](https://github.com/google/quota-monitoring-solution/commit/56b42a5b3c5fb4d676e39f1caafe53fa27bd51a7)) 69 | * updating the README to include upgrade steps for Issue 18 ([5c5a784](https://github.com/google/quota-monitoring-solution/commit/5c5a784ce118cf7fb82e80ec60ab51eee792d8c4)) 70 | 71 | 72 | ### Bug Fixes 73 | 74 | * adding end of line to fix linter error. ([7293c18](https://github.com/google/quota-monitoring-solution/commit/7293c18ad4909171d239463eea1d26f4b43f5565)) 75 | * correcting the path for GitHub workflows. ([4baf80a](https://github.com/google/quota-monitoring-solution/commit/4baf80ac121ef2a7c39c07465cc17dcd35a536d9)) 76 | * do not lint CHANGELOG.md ([e8959a5](https://github.com/google/quota-monitoring-solution/commit/e8959a5a89bba6cb0b1b37d71b04dc2eea453af2)) 77 | * removing old markdown linter config files. ([00fc7f4](https://github.com/google/quota-monitoring-solution/commit/00fc7f4694c0ff9a037c769c35ae38e64dce1cbe)) 78 | * resolving some missing dependencies between resources. ([be13e64](https://github.com/google/quota-monitoring-solution/commit/be13e64bc92d105afc291a86c36c5abcbe4e79cb)) 79 | * updating release workflow to use a different token and pinning versions. ([e68a9d0](https://github.com/google/quota-monitoring-solution/commit/e68a9d05273bbf3fe24dac2e9f86d8f03420ab28)) 80 | * updating the markdown linter to use the GitHub super linter. ([e5b2a4f](https://github.com/google/quota-monitoring-solution/commit/e5b2a4fc32151f915726b4918e19916bffd12b22)) 81 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement (CLA). You (or your employer) retain the copyright to your 10 | contribution; this simply gives us permission to use and redistribute your 11 | contributions as part of the project. Head over to 12 | to see your current agreements on file or 13 | to sign a new one. 14 | 15 | You generally only need to submit a CLA once, so if you've already submitted one 16 | (even if it was for a different project), you probably don't need to do it 17 | again. 18 | 19 | ## Code Reviews 20 | 21 | All submissions, including submissions by project members, require review. We 22 | use GitHub pull requests for this purpose. Consult 23 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 24 | information on using pull requests. 25 | 26 | ## Community Guidelines 27 | 28 | This project follows 29 | [Google's Open Source Community Guidelines](https://opensource.google/conduct/). 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /QMS_app_alerting.csv: -------------------------------------------------------------------------------- 1 | project_id,email_id,app_code,dashboard_url 2 | edge-retail-374401|pub-sub-example-394521,admin@yannipeng.altostrat.com,app-1,https://lookerstudio.google.com/reporting/1fb0594e-67b1-4173-920a-8ac6318f2e14/page/xxWVB 3 | data-flow-pubsub-bigtable,admin@yannipeng.altostrat.com,app-2,https://lookerstudio.google.com/reporting/1fb0594e-67b1-4173-920a-8ac6318f2e14/page/xxWVB 4 | epam-394023,admin@yannipeng.altostrat.com,app-3,https://lookerstudio.google.com/reporting/1fb0594e-67b1-4173-920a-8ac6318f2e14/page/xxWVB 5 | yanni-test3,admin@yannipeng.altostrat.com,app-4,https://lookerstudio.google.com/reporting/1fb0594e-67b1-4173-920a-8ac6318f2e14/page/xxWVB -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Quota Monitoring and Alerting 2 | 3 | > An easy-to-deploy Looker Studio Dashboard with alerting capabilities, showing 4 | usage and quota limits in an organization or folder. 5 | 6 | Google Cloud enforces [quotas](https://cloud.google.com/docs/quota) on resource 7 | usage for project owners, setting a limit on how much of a particular Google 8 | Cloud resource your project can use. Each quota limit represents a specific 9 | countable resource, such as the number of API requests made per day to the 10 | number of load balancers used concurrently by your application. 11 | 12 | Quotas are enforced for a variety of reasons: 13 | 14 | * To protect the community of Google Cloud users by preventing unforeseen 15 | spikes in usage. 16 | * To help you manage resources. For example, you can set your own limits on 17 | service usage while developing and testing your applications. 18 | 19 | We are introducing a new custom quota monitoring and alerting solution for 20 | Google Cloud customers. 21 | 22 | ## 1. Summary 23 | 24 | Quota Monitoring Solution is a stand-alone application of an easy-to-deploy 25 | Looker Studio dashboard with alerting capabilities showing all usage and quota 26 | limits in an organization or folder. 27 | 28 | ### 1.1 Four Initial Features 29 | 30 | ![key-features](img/quota_monitoring_key_features.png) 31 | 32 | *The data refresh rate depends on the configured frequency to run the 33 | application. 34 | 35 | ## 2. Architecture 36 | 37 | ![architecture](img/quota-monitoring-alerting-architecture.png) 38 | 39 | The architecture is built using Google Cloud managed services - Cloud 40 | Functions, Pub/Sub, Dataflow and BigQuery. 41 | 42 | * The solution is architected to scale using Pub/Sub. 43 | * Cloud Scheduler is used to trigger Cloud Functions. This is also an user 44 | interface to configure frequency, parent nodes, alert threshold and email Ids. 45 | Parent node could be an organization Id, folder id, list of organization Ids 46 | or list of folder Ids. 47 | * Cloud Functions are used to scan quotas across projects for the configured 48 | parent node. 49 | * BigQuery is used to store data. 50 | * Alert threshold will be applicable across all metrics. 51 | * Alerts can be received by Email, Mobile App, PagerDuty, SMS, Slack, 52 | Webhooks and Pub/Sub. Cloud Monitoring custom log metric has been leveraged to 53 | create Alerts. 54 | * Easy to get started and deploy with Looker Studio Dashboard. In addition to 55 | Looker Studio, other visualization tools can be configured. 56 | * The Looker Studio report can be scheduled to be emailed to appropriate team 57 | for weekly/daily reporting. 58 | 59 | ## 3. Configuring Quota Monitoring and Alerting 60 | 61 | ![configuration](img/quota-monitoring-config-flow.png) 62 | 63 | 1. Upload csv file with columns: project_id,email_id,app_code,dashboard_url 64 | 2. For applications with more than 1 projects the project_id column can take 65 | a string with more than project. Reference: [CSV file](./QMS_app_alerting.csv) 66 | 3. \*Note 1 project will only have 1 app-code, but app-code can have more 67 | than 1 project. 68 | 69 | e.g. If you have two rows in the csv file: 70 | 71 | edge-retail-374401|pub-sub-example-394521, appcode1 72 | 73 | edge-retail-374401, appcode2 74 | 75 | edge-retail-374401 will end up with appcode2. 76 | 4. Cloud scheduler will trigger configAppAlerts for each app code in csv: 77 | 5. Create custom log metric 78 | 6. Create notification channel 79 | 7. Create Alert using custom log metric & notification channel 80 | 8. Upload all data to big query 81 | 82 | ## 4. Deployment Guide 83 | 84 | ### Content 85 | 86 | 87 | - [Quota Monitoring and Alerting](#quota-monitoring-and-alerting) 88 | - [1. Summary](#1-summary) 89 | - [1.1 Four Initial Features](#11-four-initial-features) 90 | - [2. Architecture](#2-architecture) 91 | - [3. Deployment Guide](#3-deployment-guide) 92 | - [Content](#content) 93 | - [3.1 Prerequisites](#31-prerequisites) 94 | - [3.2 Initial Setup](#32-initial-setup) 95 | - [3.3 Create Service Account](#33-create-service-account) 96 | - [3.4 Grant Roles to Service Account](#34-grant-roles-to-service-account) 97 | - [3.4.1 Grant Roles in the Host Project](#341-grant-roles-in-the-host-project) 98 | - [3.4.2 Grant Roles in the Target Folder](#342-grant-roles-in-the-target-folder) 99 | - [3.4.3 Grant Roles in the Target Organization](#343-grant-roles-in-the-target-organization) 100 | - [3.5 Download the Source Code](#35-download-the-source-code) 101 | - [3.6 Download Service Account Key File](#36-download-service-account-key-file) 102 | - [3.7 Configure Terraform](#37-configure-terraform) 103 | - [3.8 Run Terraform](#38-run-terraform) 104 | - [3.9 Testing](#39-testing) 105 | - [3.10 Looker Studio Dashboard setup](#310-looker-studio-dashboard-setup) 106 | - [3.11 Scheduled Reporting](#311-scheduled-reporting) 107 | - [3.11 Alerting](#311-alerting) 108 | - [3.11.1 Slack Configuration](#3111-slack-configuration) 109 | - [3.11.1.1 Create Notification Channel](#31111-create-notification-channel) 110 | - [3.11.1.2 Configuring Alerting Policy](#31112-configuring-alerting-policy) 111 | - [4. Release Note](#4-release-note) 112 | - [v4.0.0: Quota Monitoring across GCP services](#v400-quota-monitoring-across-gcp-services) 113 | - [New](#new) 114 | - [Known Limitations](#known-limitations) 115 | - [v4.4.0](#v440) 116 | - [New in v4.4.0](#new-in-v440) 117 | - [5. What is Next](#5-what-is-next) 118 | - [6. Getting Support](#6-getting-support) 119 | - [7. Contributing](#7-contributing) 120 | 121 | 122 | ### 4.1 Prerequisites 123 | 124 | 1. Host Project - A project where the BigQuery instance, Cloud Function and 125 | Cloud Scheduler will be deployed. For example Project A. 126 | 2. Target Node - The Organization or folder or project which will be scanned 127 | for Quota Metrics. For example Org A and Folder A. 128 | 3. Project Owner role on host Project A. IAM Admin role in target Org A and 129 | target Folder A. 130 | 4. Google Cloud SDK is installed. Detailed instructions to install the SDK 131 | [here](https://cloud.google.com/sdk/docs/install#mac). See the Getting Started 132 | page for an introduction to using gcloud and terraform. 133 | 5. Terraform version >= 0.14.6 installed. Instructions to install terraform here 134 | * Verify terraform version after installing. 135 | 136 | ```sh 137 | terraform -version 138 | ``` 139 | 140 | The output should look like: 141 | 142 | ```sh 143 | Terraform v0.14.6 144 | + provider registry.terraform.io/hashicorp/google v3.57.0 145 | ``` 146 | 147 | *Note - Minimum required version v0.14.6. Lower terraform versions may not work.* 148 | 149 | ### 4.2 Initial Setup 150 | 151 | 1. In local workstation create a new directory to run terraform and store 152 | credential file 153 | 154 | ```sh 155 | mkdir 156 | cd 157 | ``` 158 | 159 | 2. Set default project in config to host project A 160 | 161 | ```sh 162 | gcloud config set project 163 | ``` 164 | 165 | The output should look like: 166 | 167 | ```sh 168 | Updated property [core/project]. 169 | ``` 170 | 171 | 3. Ensure that the latest version of all installed components is installed on 172 | the local workstation. 173 | 174 | ```sh 175 | gcloud components update 176 | ``` 177 | 178 | 4. Cloud Scheduler depends on the App Engine application. Create an App Engine 179 | application in the host project. Replace the region. List of regions where 180 | App Engine is available can be found 181 | [here](https://cloud.google.com/about/locations#region). 182 | 183 | ```sh 184 | gcloud app create --region= 185 | ``` 186 | 187 | Note: Cloud Scheduler (below) needs to be in the same region as App Engine. 188 | Use the same region in terraform as mentioned here. 189 | 190 | The output should look like: 191 | 192 | ```sh 193 | You are creating an app for project [quota-monitoring-project-3]. 194 | WARNING: Creating an App Engine application for a project is irreversible and the region 195 | cannot be changed. More information about regions is at 196 | . 197 | 198 | Creating App Engine application in project [quota-monitoring-project-1] and region [us-east1]....done. 199 | 200 | Success! The app is now created. Please use `gcloud app deploy` to deploy your first app. 201 | ``` 202 | 203 | ### 4.3 Create Service Account 204 | 205 | 1. In local workstation, setup environment variables. Replace the name of the 206 | Service Account in the commands below 207 | 208 | ```sh 209 | export DEFAULT_PROJECT_ID=$(gcloud config get-value core/project 2> /dev/null) 210 | export SERVICE_ACCOUNT_ID="sa-"$DEFAULT_PROJECT_ID 211 | export DISPLAY_NAME="sa-"$DEFAULT_PROJECT_ID 212 | ``` 213 | 214 | 2. Verify host project Id. 215 | 216 | ```sh 217 | echo $DEFAULT_PROJECT_ID 218 | ``` 219 | 220 | 3. Create Service Account 221 | 222 | ```sh 223 | gcloud iam service-accounts create $SERVICE_ACCOUNT_ID --description="Service Account to scan quota usage" --display-name=$DISPLAY_NAME 224 | ``` 225 | 226 | The output should look like: 227 | 228 | ```sh 229 | Created service account [sa-quota-monitoring-project-1]. 230 | ``` 231 | 232 | ### 4.4 Grant Roles to Service Account 233 | 234 | #### 4.4.1 Grant Roles in the Host Project 235 | 236 | The following roles need to be added to the Service Account in the host 237 | project i.e. Project A: 238 | 239 | * BigQuery 240 | * BigQuery Data Editor 241 | * BigQuery Job User 242 | * Cloud Functions 243 | * Cloud Functions Admin 244 | * Cloud Scheduler 245 | * Cloud Scheduler Admin 246 | * Pub/Sub 247 | * Pub/Sub Admin 248 | * Run Terraform 249 | * Service Account User 250 | * Enable APIs 251 | * Service Usage Admin 252 | * Storage Bucket 253 | * Storage Admin 254 | * Scan Quotas 255 | * Cloud Asset Viewer 256 | * Compute Network Viewer 257 | * Compute Viewer 258 | * Monitoring 259 | * Notification Channel Editor 260 | * Alert Policy Editor 261 | * Viewer 262 | * Metric Writer 263 | * Logs 264 | * Logs Configuration Writer 265 | * Log Writer 266 | * IAM 267 | * Security Admin 268 | 269 | 1. Run following commands to assign the roles: 270 | 271 | ```sh 272 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.dataEditor" --condition=None 273 | 274 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/bigquery.jobUser" --condition=None 275 | 276 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudfunctions.admin" --condition=None 277 | 278 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudscheduler.admin" --condition=None 279 | 280 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/pubsub.admin" --condition=None 281 | 282 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/iam.serviceAccountUser" --condition=None 283 | 284 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/storage.admin" --condition=None 285 | 286 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/serviceusage.serviceUsageAdmin" --condition=None 287 | 288 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudasset.viewer" --condition=None 289 | 290 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.networkViewer" --condition=None 291 | 292 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.viewer" --condition=None 293 | 294 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.notificationChannelEditor" --condition=None 295 | 296 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.alertPolicyEditor" --condition=None 297 | 298 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/logging.configWriter" --condition=None 299 | 300 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/logging.logWriter" --condition=None 301 | 302 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.viewer" --condition=None 303 | 304 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.metricWriter" --condition=None 305 | 306 | gcloud projects add-iam-policy-binding $DEFAULT_PROJECT_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/iam.securityAdmin" --condition=None 307 | ``` 308 | 309 | #### 4.4.2 Grant Roles in the Target Folder 310 | 311 | SKIP THIS STEP IF THE FOLDER IS NOT THE TARGET TO SCAN QUOTA 312 | 313 | If you want to scan projects in the folder, add following roles to the Service 314 | Account created in the previous step at the target folder A: 315 | 316 | * Cloud Asset Viewer 317 | * Compute Network Viewer 318 | * Compute Viewer 319 | * Folder Viewer 320 | * Monitoring Viewer 321 | 322 | 1. Set target folder id 323 | 324 | ```sh 325 | export TARGET_FOLDER_ID= 326 | ``` 327 | 328 | 2. Run the following commands add to the roles to the service account 329 | 330 | ```sh 331 | gcloud alpha resource-manager folders add-iam-policy-binding $TARGET_FOLDER_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudasset.viewer" 332 | 333 | gcloud alpha resource-manager folders add-iam-policy-binding $TARGET_FOLDER_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.networkViewer" 334 | 335 | gcloud alpha resource-manager folders add-iam-policy-binding $TARGET_FOLDER_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.viewer" 336 | 337 | gcloud alpha resource-manager folders add-iam-policy-binding $TARGET_FOLDER_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/resourcemanager.folderViewer" 338 | 339 | gcloud alpha resource-manager folders add-iam-policy-binding $TARGET_FOLDER_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.viewer" 340 | ``` 341 | 342 | Note: If this fails, run the commands again 343 | 344 | #### 4.4.3 Grant Roles in the Target Organization 345 | 346 | SKIP THIS STEP IF THE ORGANIZATION IS NOT THE TARGET 347 | 348 | If you want to scan projects in the org, add following roles to the Service 349 | Account created in the previous step at the Org A: 350 | 351 | * Cloud Asset Viewer 352 | * Compute Network Viewer 353 | * Compute Viewer 354 | * Org Viewer 355 | * Folder Viewer 356 | * Monitoring Viewer 357 | 358 | ![org-service-acccount-roles](img/service_account_roles.png) 359 | 360 | 1. Set target organization id 361 | 362 | ```sh 363 | export TARGET_ORG_ID= 364 | ``` 365 | 366 | 2. Run the following commands to add to the roles to the service account 367 | 368 | ```sh 369 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/cloudasset.viewer" --condition=None 370 | 371 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.networkViewer" --condition=None 372 | 373 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/compute.viewer" --condition=None 374 | 375 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/resourcemanager.folderViewer" --condition=None 376 | 377 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/resourcemanager.organizationViewer" --condition=None 378 | 379 | gcloud organizations add-iam-policy-binding $TARGET_ORG_ID --member="serviceAccount:$SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com" --role="roles/monitoring.viewer" --condition=None 380 | ``` 381 | 382 | ### 4.5 Download the Source Code 383 | 384 | 1. Clone the Quota Management Solution repo 385 | 386 | ```sh 387 | git clone https://github.com/google/quota-monitoring-solution.git quota-monitorings-solution 388 | ``` 389 | 390 | 2. Change directories into the Terraform example 391 | 392 | ```sh 393 | cd ./quota-monitorings-solution/terraform/example 394 | ``` 395 | 396 | ### 4.6 Set OAuth Token Using Service Account Impersonization 397 | 398 | Impersonate your host project service account and set environment variable 399 | using temporary token to authenticate terraform. You will need to make 400 | sure your user has the 401 | [Service Account Token Creator role](https://cloud.google.com/iam/docs/service-account-permissions#token-creator-role) 402 | to create short-lived credentials. 403 | 404 | ```sh 405 | gcloud config set auth/impersonate_service_account \ 406 | $SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com 407 | 408 | export GOOGLE_OAUTH_ACCESS_TOKEN=$(gcloud auth print-access-token) 409 | ``` 410 | 411 | * **TIP**: If you get an error saying *unable to impersonate*, you will 412 | need to unset the impersonation. Have the role added similar to below, then 413 | try again. 414 | 415 | ```sh 416 | # unset impersonation 417 | gcloud config unset auth/impersonate_service_account 418 | 419 | # set your current authenticated user as var 420 | PROJECT_USER=$(gcloud config get-value core/account) 421 | 422 | # grant IAM role serviceAccountTokenCreator 423 | gcloud iam service-accounts add-iam-policy-binding $SERVICE_ACCOUNT_ID@$DEFAULT_PROJECT_ID.iam.gserviceaccount.com \ 424 | --member user:$PROJECT_USER \ 425 | --role roles/iam.serviceAccountTokenCreator \ 426 | --condition=None 427 | ``` 428 | 429 | ### 4.7 Configure Terraform 430 | 431 | 1. Verify that you have these 3 files in your local directory: 432 | * main.tf 433 | * variables.tf 434 | * terraform.tfvars 435 | 436 | 2. Open [terraform.tfvars](terraform/example/terraform.tfvars) file in your 437 | favourite editor and change values for the variables. 438 | 439 | ```sh 440 | vi terraform.tfvars 441 | ``` 442 | 443 | 3. For `region`, use the same region as used for App Engine in earlier steps. 444 | 445 | The variables `source_code_base_url`, `qms_version`, `source_code_zip` 446 | and `source_code_notification_zip` on the QMS module are used to download 447 | the source for the QMS Cloud Functions from the latest GitHub [release](https://github.com/google/quota-monitoring-solution/releases). 448 | 449 | To deploy the latest unreleased code from a local clone of the QMS 450 | repository, set `qms_version` to `main` 451 | 452 | ### 4.8 Run Terraform 453 | 454 | 1. Run terraform commands 455 | * `terraform init` 456 | * `terraform plan` 457 | * `terraform apply` 458 | * On Prompt Enter a value: `yes` 459 | 460 | 2. This will: 461 | * Enable required APIs 462 | * Create all resources and connect them. 463 | 464 | Note: In case terraform fails, run terraform plan and terraform apply again 465 | 466 | 3. Stop impersonating service account (when finished with terraform) 467 | 468 | ```sh 469 | gcloud config unset auth/impersonate_service_account 470 | ``` 471 | 472 | ### 4.9 Testing 473 | 474 | 1. Initiate first job run in Cloud Scheduler. 475 | 476 | **Console** 477 | 478 | Click 'Run Now' on Cloud Job scheduler. 479 | 480 | *Note: The status of the ‘Run Now’ button changes to ‘Running’ for a fraction 481 | of seconds.* 482 | 483 | ![run-cloud-scheduler](img/run_cloud_scheduler.png) 484 | 485 | **Terminal** 486 | 487 | ```sh 488 | gcloud scheduler jobs run quota-monitoring-cron-job --location 489 | gcloud scheduler jobs run quota-monitoring-app-alert-config --location 490 | ``` 491 | 492 | 2. To verify that the program ran successfully, check the BigQuery Table. The 493 | time to load data in BigQuery might take a few minutes. The execution time 494 | depends on the number of projects to scan. A sample BigQuery table will look 495 | like this: 496 | ![test-bigquery-table](img/test_bigquery_table.png) 497 | 498 | ### 4.10 Looker Studio Dashboard setup 499 | 500 | 1. Go to the [Looker Studio dashboard template](https://lookerstudio.google.com/reporting/f5e179e9-29e1-46c2-a443-97f5e24edd64). 501 | A Looker Studio dashboard will look like this: 502 | ![ds-updated-quotas-dashboard](img/ds-updated-quotas-dashboard.png) 503 | 2. Make a copy of the template from the copy icon at the top bar (top - right 504 | corner) 505 | ![ds-dropdown-copy](img/ds-dropdown-copy.png) 506 | 3. Click on ‘Copy Report’ button **without changing datasource options** 507 | ![ds-copy-report-fixed-new-data-source](img/ds-copy-report-fixed-new-data-source.png) 508 | 4. This will create a copy of the report and open in Edit mode. If not click on 509 | ‘Edit’ button on top right corner in copied template: 510 | ![ds-edit-mode-updated](img/ds-edit-mode-updated.png) 511 | 5. Select any one table like below ‘Disks Total GB - Quotas’ is selected. On the 512 | right panel in ‘Data’ tab, click on icon ‘edit data source’ 513 | ![ds_edit_data_source](img/ds_edit_data_source.png) 514 | It will open the data source details 515 | ![ds_datasource_config_step_1]img/ds_datasource_config_step_1.png 516 | 6. Replace the BigQuery Project Id of your bq table, Dataset Id and Table Name to 517 | match your deployment. If you assigned app codes add a list of project ids in 518 | where clause from the csv file upload. Verify the query by running in BigQuery 519 | Editor for accuracy & syntax: 520 | 521 | ```sql 522 | #For org level dashboard use the following query 523 | SELECT 524 | project_id, 525 | added_at, 526 | region, 527 | quota_metric, 528 | CASE 529 | WHEN CAST(quota_limit AS STRING) ='9223372036854775807' THEN 'unlimited' 530 | ELSE 531 | CAST(quota_limit AS STRING) 532 | END AS str_quota_limit, 533 | SUM(current_usage) AS current_usage, 534 | ROUND((SAFE_DIVIDE(CAST(SUM(current_usage) AS BIGNUMERIC), CAST(quota_limit AS BIGNUMERIC))*100),2) AS current_consumption, 535 | SUM(max_usage) AS max_usage, 536 | ROUND((SAFE_DIVIDE(CAST(SUM(max_usage) AS BIGNUMERIC), CAST(quota_limit AS BIGNUMERIC))*100),2) AS max_consumption 537 | FROM 538 | ( 539 | SELECT 540 | *, 541 | RANK() OVER (PARTITION BY project_id, region, quota_metric ORDER BY added_at DESC) AS latest_row 542 | FROM 543 | `[YOUR_PROJECT_ID].quota_monitoring_dataset.quota_monitoring_table` 544 | ) t 545 | WHERE 546 | latest_row=1 547 | AND current_usage IS NOT NULL 548 | AND quota_limit IS NOT NULL 549 | AND current_usage != 0 550 | AND quota_limit != 0 551 | GROUP BY 552 | project_id, 553 | region, 554 | quota_metric, 555 | added_at, 556 | quota_limit 557 | 558 | # For app level dashboard use the following query replace PROJECT_ID with project_ids from csv file upload 559 | SELECT 560 | project_id, 561 | added_at, 562 | region, 563 | quota_metric, 564 | CASE 565 | WHEN CAST(quota_limit AS STRING) ='9223372036854775807' THEN 'unlimited' 566 | ELSE 567 | CAST(quota_limit AS STRING) 568 | END AS str_quota_limit, 569 | SUM(current_usage) AS current_usage, 570 | ROUND((SAFE_DIVIDE(CAST(SUM(current_usage) AS BIGNUMERIC), CAST(quota_limit AS BIGNUMERIC))*100),2) AS current_consumption, 571 | SUM(max_usage) AS max_usage, 572 | ROUND((SAFE_DIVIDE(CAST(SUM(max_usage) AS BIGNUMERIC), CAST(quota_limit AS BIGNUMERIC))*100),2) AS max_consumption 573 | FROM 574 | ( 575 | SELECT 576 | *, 577 | RANK() OVER (PARTITION BY project_id, region, quota_metric ORDER BY added_at DESC) AS latest_row 578 | FROM 579 | `[YOUR_PROJECT_ID].quota_monitoring_dataset.quota_monitoring_table` 580 | ) t 581 | WHERE 582 | latest_row=1 583 | AND current_usage IS NOT NULL 584 | AND quota_limit IS NOT NULL 585 | AND current_usage != 0 586 | AND quota_limit != 0 587 | AND project-id IN ([PROJECT_ID1], [PROJECT_ID2]..) 588 | GROUP BY 589 | project_id, 590 | region, 591 | quota_metric, 592 | added_at, 593 | quota_limit 594 | ``` 595 | 596 | 7. After making sure that query is returning results, replace it in the Data 597 | Studio, click on the ‘Reconnect’ button in the data source pane. 598 | ![ds_data_source_config_step_3](img/ds_data_source_config_step_3.png) 599 | 8. In the next window, click on the ‘Done’ button. 600 | ![ds_data_source_config_step_2](img/ds_data_source_config_step_2.png) 601 | 9. Once the data source is configured, click on the ‘View’ button on the top 602 | right corner. 603 | Note: make additional changes in the layout like which metrics to be displayed 604 | on Dashboard, color shades for consumption column, number of rows for each 605 | table etc in the ‘Edit’ mode. 606 | ![ds-switch-to-view-mode](img/ds-switch-to-view-mode.png) 607 | 608 | ### 4.11 Scheduled Reporting 609 | 610 | Quota monitoring reports can be scheduled from the Looker Studio dashboard using 611 | ‘Schedule email delivery’. The screenshot of the Looker Studio dashboard will be 612 | delivered as a pdf report to the configured email Ids. 613 | 614 | ![ds-schedule-email-button](img/ds-schedule-email-button.png) 615 | 616 | ### 4.11 Alerting 617 | 618 | The alerts about services nearing their quota limits can be configured to be 619 | sent via email as well as following external services: 620 | 621 | * Slack 622 | * PagerDuty 623 | * SMS 624 | * Custom Webhooks 625 | 626 | #### 4.11.1 Slack Configuration 627 | 628 | To configure notifications to be sent to a Slack channel, you must have the 629 | Monitoring Notification Channel Editor role on the host project. 630 | 631 | ##### 4.11.1.1 Create Notification Channel 632 | 633 | 1. In the Cloud Console, use the project picker to select your Google Cloud 634 | project, and then select Monitoring, or click the link here: Go to Monitoring 635 | 2. In the Monitoring navigation pane, click Alerting. 636 | 3. Click Edit notification channels. 637 | 4. In the Slack section, click Add new. This brings you to the Slack sign-in 638 | page: 639 | * Select your Slack workspace. 640 | * Click Allow to enable Google Cloud Monitoring access to your Slack 641 | workspace. This action takes you back to the Monitoring configuration page 642 | for your notification channel. 643 | * Enter the name of the Slack channel you want to use for notifications. 644 | * Enter a display name for the notification channel. 645 | 5. In your Slack workspace: 646 | * Invite the Monitoring app to the channel by sending the following 647 | message in the channel: 648 | * /invite @Google Cloud Monitoring 649 | * Be sure you invite the Monitoring app to the channel you specified when 650 | creating the notification channel in Monitoring. 651 | 652 | ##### 4.11.1.2 Configuring Alerting Policy 653 | 654 | 1. In the Alerting section, click on Policies. 655 | 2. Find the Policy named ‘Resource Reaching Quotas’. This policy was created 656 | via Terraform code above. 657 | 3. Click Edit. 658 | 4. It opens an Edit Alerting Policy page. Leave the current condition metric as 659 | is, and click on Next. 660 | 5. In the Notification Options, Select the Slack Channel that you created above. 661 | 6. Click on Save. 662 | 663 | You should now receive alerts in your Slack channel whenever a quota reaches 664 | the specified threshold limit. 665 | 666 | ## 5. Release Note 667 | 668 | ### v4.0.0: Quota Monitoring across GCP services 669 | 670 | #### New 671 | 672 | * The new version provides visibility into Quotas across various GCP services 673 | beyond the original GCE (Compute). 674 | * New Looker Studio Dashboard template reporting metrics across GCP services 675 | 676 | #### Known Limitations 677 | 678 | * The records are grouped by hour. Scheduler need to be configured to start 679 | running preferably at the beginning of the hour. 680 | * Out of the box solution is configured to scan quotas ‘once every day’. The 681 | SQL query to build the dashboard uses current date to filter the records. If 682 | you change the frequency, make changes to the query to rightly reflect the 683 | latest data. 684 | 685 | ### v4.4.0 686 | 687 | #### New in v4.4.0 688 | 689 | * The new version includes a fix that converts the data pull process to use 690 | the Montoring Query Language (MQL). This allows QMS to pull the limit and 691 | current usage at the exact same time, so reporting queries can be more 692 | tightly scoped, eliminating over reporting problems. 693 | 694 | To upgrade existing installations: 695 | 696 | * Re-run the Terraform, to update the Cloud Functions and Scheduled Query 697 | * Update the SQL used in the Looker Studio dashboard according to Step #7 698 | of [4.10 Looker Studio Dashboard setup](#410-looker-studio-dashboard-setup). 699 | 700 | ## 6. What is Next 701 | 702 | 1. Graphs (Quota utilization over a period of time) 703 | 2. Search project, folder, org, region 704 | 3. Threshold configurable for each metric 705 | 706 | ## 7. Getting Support 707 | 708 | Quota Monitoring Solution is a project based on open source contributions. We'd 709 | love for you to [report issues, file feature requests][new-issue], and 710 | [send pull requests][new-pr] (see [Contributing](README.md#7-contributing)). Quota 711 | Monitoring Solution is not officially covered by the Google Cloud product support. 712 | 713 | ## 8. Contributing 714 | 715 | * [Contributing guidelines][contributing-guidelines] 716 | * [Code of conduct][code-of-conduct] 717 | 718 | 719 | 720 | [code-of-conduct]: code-of-conduct.md 721 | [contributing-guidelines]: CONTRIBUTING.md 722 | [new-issue]: https://github.com/google/quota-monitoring-solution/issues/new 723 | [new-pr]: https://github.com/google/quota-monitoring-solution/compare 724 | -------------------------------------------------------------------------------- /code-of-conduct.md: -------------------------------------------------------------------------------- 1 | # Google Open Source Community Guidelines 2 | 3 | At Google, we recognize and celebrate the creativity and collaboration of open 4 | source contributors and the diversity of skills, experiences, cultures, and 5 | opinions they bring to the projects and communities they participate in. 6 | 7 | Every one of Google's open source projects and communities are inclusive 8 | environments, based on treating all individuals respectfully, regardless of 9 | gender identity and expression, sexual orientation, disabilities, 10 | neurodiversity, physical appearance, body size, ethnicity, nationality, race, 11 | age, religion, or similar personal characteristic. 12 | 13 | We value diverse opinions, but we value respectful behavior more. 14 | 15 | Respectful behavior includes: 16 | 17 | * Being considerate, kind, constructive, and helpful. 18 | * Not engaging in demeaning, discriminatory, harassing, hateful, sexualized, or 19 | physically threatening behavior, speech, and imagery. 20 | * Not engaging in unwanted physical contact. 21 | 22 | Some Google open source projects [may adopt][] an explicit project code of 23 | conduct, which may have additional detailed expectations for participants. Most 24 | of those projects will use our [modified Contributor Covenant][]. 25 | 26 | [may adopt]: https://opensource.google/docs/releasing/preparing/#conduct 27 | [modified Contributor Covenant]: https://opensource.google/docs/releasing/template/CODE_OF_CONDUCT/ 28 | 29 | ## Resolve peacefully 30 | 31 | We do not believe that all conflict is necessarily bad; healthy debate and 32 | disagreement often yields positive results. However, it is never okay to be 33 | disrespectful. 34 | 35 | If you see someone behaving disrespectfully, you are encouraged to address the 36 | behavior directly with those involved. Many issues can be resolved quickly and 37 | easily, and this gives people more control over the outcome of their dispute. 38 | If you are unable to resolve the matter for any reason, or if the behavior is 39 | threatening or harassing, report it. We are dedicated to providing an 40 | environment where participants feel welcome and safe. 41 | 42 | ## Reporting problems 43 | 44 | Some Google open source projects may adopt a project-specific code of conduct. 45 | In those cases, a Google employee will be identified as the Project Steward, 46 | who will receive and handle reports of code of conduct violations. In the event 47 | that a project hasn’t identified a Project Steward, you can report problems by 48 | emailing opensource@google.com. 49 | 50 | We will investigate every complaint, but you may not receive a direct response. 51 | We will use our discretion in determining when and how to follow up on reported 52 | incidents, which may range from not taking action to permanent expulsion from 53 | the project and project-sponsored spaces. We will notify the accused of the 54 | report and provide them an opportunity to discuss it before any action is 55 | taken. The identity of the reporter will be omitted from the details of the 56 | report supplied to the accused. In potentially harmful situations, such as 57 | ongoing harassment or threats to anyone's safety, we may take action without 58 | notice. 59 | 60 | *This document was adapted from the [IndieWeb Code of Conduct][] and can also 61 | be found at .* 62 | 63 | [IndieWeb Code of Conduct]: https://indieweb.org/code-of-conduct 64 | -------------------------------------------------------------------------------- /img/bigquery-orderby-consumption.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/bigquery-orderby-consumption.png -------------------------------------------------------------------------------- /img/ds-copy-report-fixed-new-data-source.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds-copy-report-fixed-new-data-source.png -------------------------------------------------------------------------------- /img/ds-dropdown-copy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds-dropdown-copy.png -------------------------------------------------------------------------------- /img/ds-edit-mode-updated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds-edit-mode-updated.png -------------------------------------------------------------------------------- /img/ds-schedule-email-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds-schedule-email-button.png -------------------------------------------------------------------------------- /img/ds-switch-to-view-mode.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds-switch-to-view-mode.png -------------------------------------------------------------------------------- /img/ds-updated-quotas-dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds-updated-quotas-dashboard.png -------------------------------------------------------------------------------- /img/ds_data_source_config_step_2.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds_data_source_config_step_2.png -------------------------------------------------------------------------------- /img/ds_data_source_config_step_3.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds_data_source_config_step_3.png -------------------------------------------------------------------------------- /img/ds_datasource_config_step_1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds_datasource_config_step_1.png -------------------------------------------------------------------------------- /img/ds_edit_data_source.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/ds_edit_data_source.png -------------------------------------------------------------------------------- /img/quota-monitoring-alerting-architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/quota-monitoring-alerting-architecture.png -------------------------------------------------------------------------------- /img/quota-monitoring-config-flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/quota-monitoring-config-flow.png -------------------------------------------------------------------------------- /img/quota_monitoring_key_features.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/quota_monitoring_key_features.png -------------------------------------------------------------------------------- /img/run_cloud_scheduler.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/run_cloud_scheduler.png -------------------------------------------------------------------------------- /img/service_account_roles.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/service_account_roles.png -------------------------------------------------------------------------------- /img/test_bigquery_table.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/google/quota-monitoring-solution/c0054476abcf4a09bd67899429087b3a4bbc65e2/img/test_bigquery_table.png -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | quota-monitoring-solution 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 11 13 | 11 14 | 15 | 16 | -------------------------------------------------------------------------------- /quota-notification/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.example 8 | quota-notification 9 | 1.0-SNAPSHOT 10 | 11 | 12 | 11 13 | 11 14 | 15 | 16 | 17 | 18 | org.apache.maven.plugins 19 | maven-compiler-plugin 20 | 3.8.1 21 | 22 | 23 | com.google.cloud.functions 24 | functions-framework-api 25 | 1.0.3 26 | provided 27 | 28 | 29 | com.google.truth 30 | truth 31 | 1.1 32 | test 33 | 34 | 35 | com.google.guava 36 | guava-testlib 37 | 30.1-jre 38 | test 39 | 40 | 41 | com.google.cloud 42 | google-cloud-logging 43 | 3.14.0 44 | 45 | 46 | com.google.cloud 47 | google-cloud-monitoring 48 | 49 | 50 | com.google.cloud 51 | google-cloud-bigquery 52 | 1.126.3 53 | 54 | 55 | com.google.cloud 56 | google-cloud-monitoring 57 | 2.0.11 58 | 59 | 60 | com.google.cloud 61 | google-cloud-storage 62 | 2.10.0 63 | 64 | 65 | org.apache.commons 66 | commons-lang3 67 | 3.0 68 | 69 | 70 | 71 | 72 | 73 | 74 | 81 | com.google.cloud.functions 82 | function-maven-plugin 83 | 0.9.6 84 | 85 | functions.ConfigureAppAlert 86 | 87 | 88 | 89 | 90 | 91 | -------------------------------------------------------------------------------- /quota-notification/src/main/java/functions/ConfigureAppAlert.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package functions; 17 | 18 | import static functions.ConfigureAppAlertHelper.configureAppAlerting; 19 | import static functions.ConfigureAppAlertHelper.loadCsvFromGcsToBigQuery; 20 | 21 | import com.google.cloud.bigquery.BigQuery; 22 | import com.google.cloud.bigquery.BigQueryOptions; 23 | import com.google.cloud.bigquery.TableId; 24 | import com.google.cloud.functions.HttpFunction; 25 | import com.google.cloud.functions.HttpRequest; 26 | import com.google.cloud.functions.HttpResponse; 27 | import java.util.logging.Logger; 28 | 29 | public class ConfigureAppAlert implements HttpFunction { 30 | public static final String HOME_PROJECT = System.getenv("HOME_PROJECT"); 31 | public static final String APP_ALERT_DATASET = System.getenv("APP_ALERT_DATASET"); 32 | public static final String APP_ALERT_TABLE = System.getenv("APP_ALERT_TABLE"); 33 | public static final String CSV_SOURCE_URI = System.getenv("CSV_SOURCE_URI"); 34 | 35 | private static final Logger logger = Logger.getLogger(ConfigureAppAlert.class.getName()); 36 | 37 | @Override 38 | public void service(HttpRequest request, HttpResponse response) 39 | throws Exception { 40 | response.getWriter().write("App Notification Configuration starting\n"); 41 | // Initialize client that will be used to send requests. 42 | BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); 43 | // Get table 44 | TableId tableId = TableId.of(HOME_PROJECT, APP_ALERT_DATASET, APP_ALERT_TABLE); 45 | //Initialize App Alert Table - Load data from CSV file 46 | loadCsvFromGcsToBigQuery(bigquery,tableId); 47 | // Configure App Alerting - Custom Log Metrics, Notification Channels and Alert Policies 48 | configureAppAlerting(bigquery); 49 | } 50 | 51 | 52 | } -------------------------------------------------------------------------------- /quota-notification/src/main/java/functions/ConfigureAppAlertHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package functions; 18 | 19 | import com.google.cloud.ReadChannel; 20 | import com.google.cloud.bigquery.BigQuery; 21 | import com.google.cloud.bigquery.BigQueryException; 22 | import com.google.cloud.bigquery.CsvOptions; 23 | import com.google.cloud.bigquery.Field; 24 | import com.google.cloud.bigquery.FieldValueList; 25 | import com.google.cloud.bigquery.Job; 26 | import com.google.cloud.bigquery.JobId; 27 | import com.google.cloud.bigquery.JobInfo; 28 | import com.google.cloud.bigquery.LoadJobConfiguration; 29 | import com.google.cloud.bigquery.QueryJobConfiguration; 30 | import com.google.cloud.bigquery.Schema; 31 | import com.google.cloud.bigquery.StandardSQLTypeName; 32 | import com.google.cloud.bigquery.TableId; 33 | import com.google.cloud.bigquery.TableResult; 34 | import com.google.cloud.logging.Logging; 35 | import com.google.cloud.logging.LoggingOptions; 36 | import com.google.cloud.logging.MetricInfo; 37 | import com.google.cloud.monitoring.v3.AlertPolicyServiceClient; 38 | import com.google.cloud.monitoring.v3.NotificationChannelServiceClient; 39 | import com.google.cloud.storage.Blob; 40 | import com.google.cloud.storage.Storage; 41 | import com.google.cloud.storage.StorageOptions; 42 | import com.google.monitoring.v3.Aggregation; 43 | import com.google.monitoring.v3.Aggregation.Aligner; 44 | import com.google.monitoring.v3.AlertPolicy; 45 | import com.google.monitoring.v3.AlertPolicy.Condition; 46 | import com.google.monitoring.v3.AlertPolicy.Condition.MetricThreshold; 47 | import com.google.monitoring.v3.AlertPolicy.Condition.Trigger; 48 | import com.google.monitoring.v3.AlertPolicy.ConditionCombinerType; 49 | import com.google.monitoring.v3.AlertPolicy.Documentation; 50 | import com.google.monitoring.v3.ComparisonType; 51 | import com.google.monitoring.v3.NotificationChannel; 52 | import com.google.monitoring.v3.UpdateAlertPolicyRequest; 53 | import com.google.monitoring.v3.UpdateNotificationChannelRequest; 54 | import com.google.protobuf.Duration; 55 | import functions.eventpojos.AppAlert; 56 | import java.io.BufferedReader; 57 | import java.io.FileNotFoundException; 58 | import java.io.IOException; 59 | import com.google.monitoring.v3.ProjectName; 60 | import java.nio.channels.Channels; 61 | import java.util.ArrayList; 62 | import java.util.HashSet; 63 | import java.util.List; 64 | import java.util.Set; 65 | import java.util.UUID; 66 | import java.util.logging.Logger; 67 | import org.apache.commons.lang3.StringUtils; 68 | 69 | 70 | public class ConfigureAppAlertHelper { 71 | private static final String HOME_PROJECT = ConfigureAppAlert.HOME_PROJECT; 72 | private static final String DATASET = ConfigureAppAlert.APP_ALERT_DATASET; 73 | private static final String TABLE = ConfigureAppAlert.APP_ALERT_TABLE; 74 | private static final String CSV_SOURCE_URI = ConfigureAppAlert.CSV_SOURCE_URI; 75 | 76 | private static final Logger logger = Logger.getLogger(ConfigureAppAlertHelper.class.getName()); 77 | 78 | 79 | 80 | /* 81 | * API to initialize App Alert table. 82 | * This API loads data from csv file in Cloud Storage bucket to BigQuery table 83 | * */ 84 | public static void loadCsvFromGcsToBigQuery(BigQuery bigquery, TableId tableId){ 85 | // Do not upload data from csv if table already contains data 86 | // This batch upload is used only for initializing the data for the first time 87 | // for subsequent addition/modification of data, use DML queries in BigQuery table 88 | if(!isAppAlertTableEmpty(bigquery) ) 89 | return; 90 | // Parse the csv file and verify that 91 | // 1. File is available, 92 | // 2. There are no records with empty/null project_id, email_id or app_code 93 | // 3. There are no duplicates for app_code 94 | if(isCSVParseSuccess()){ 95 | logger.info("CSV parsed successfully! Loading data in BigQuery..."); 96 | } else { 97 | logger.severe("CSV parsing failed. Fix the CSV to proceed."); 98 | return; 99 | } 100 | 101 | //Upload data from csv if table is empty 102 | Schema schema = 103 | Schema.of( 104 | Field.of("project_id", StandardSQLTypeName.STRING), 105 | Field.of("email_id", StandardSQLTypeName.STRING), 106 | Field.of("app_code", StandardSQLTypeName.STRING), 107 | Field.of("dashboard_url", StandardSQLTypeName.STRING)); 108 | try { 109 | // Skip header row in the file. 110 | CsvOptions csvOptions = CsvOptions.newBuilder().setSkipLeadingRows(1).build(); 111 | 112 | LoadJobConfiguration loadConfig = 113 | LoadJobConfiguration.newBuilder(tableId, CSV_SOURCE_URI, csvOptions).setSchema(schema).build(); 114 | 115 | // Load data from a GCS CSV file into the table 116 | Job job = bigquery.create(JobInfo.of(loadConfig)); 117 | // Blocks until this load table job completes its execution, either failing or succeeding. 118 | job = job.waitFor(); 119 | if (job.isDone()) { 120 | logger.info("CSV from GCS successfully added during load append job"); 121 | } else { 122 | logger.severe( 123 | "BigQuery was unable to load into the table due to an error:" 124 | + job.getStatus().getError()); 125 | } 126 | } catch (BigQueryException | InterruptedException e) { 127 | logger.severe("Column not added during load append \n" + e.toString()); 128 | } 129 | } 130 | 131 | /* 132 | * API to configure App Alerting - Custom log metrics, Notification Channels and Alert Policies for each app 133 | * */ 134 | public static void configureAppAlerting(BigQuery bigquery){ 135 | // Fetch all records for app alert from DB and list 136 | List appAlerts = listAppAlertConfig(bigquery); 137 | 138 | // If any of the appCode is violating unique constraint for appCode, return 139 | if(!appCodeUnique(appAlerts)) 140 | return; 141 | 142 | //Iterate over AppAlerts and create or update configurations for Custom Log Metrics, Notification Channels and Alert Policies 143 | //Update configuration in BigQuery table for each appAlert entity 144 | for(AppAlert appAlert : appAlerts){ 145 | appAlert.setCustomLogMetricId(createCustomLogMetric(appAlert)); 146 | appAlert.setNotificationChannelId(createOrUpdateNotificationChannel(appAlert)); 147 | appAlert.setAlertPolicyId(createOrUpdateAlertPolicy(appAlert)); 148 | updateAppAlertConfig(bigquery, appAlert); 149 | } 150 | logger.info("App Alert configuration completed successfully for all apps!"); 151 | } 152 | 153 | /* 154 | * API to verify that appCodes are unique in DB 155 | * */ 156 | private static boolean appCodeUnique(List appAlerts){ 157 | Set appCodes = new HashSet<>(); 158 | 159 | for(AppAlert appAlert : appAlerts){ 160 | String appCode = appAlert.getAppCode(); 161 | //If AppCode is not unique, return false 162 | if(!appCodes.add(appCode)){ 163 | logger.severe("Found duplicate app code \""+appCode+"\" in BigQuery Table. app_code should be unique for each row" ); 164 | return false; 165 | } 166 | } 167 | return true; 168 | } 169 | 170 | /* 171 | * API to fetch App Alert configurations from BigQuery 172 | * @return - List of App Alerts 173 | * */ 174 | public static List listAppAlertConfig(BigQuery bigquery){ 175 | List appAlerts = new ArrayList<>(); 176 | QueryJobConfiguration queryConfig = 177 | QueryJobConfiguration.newBuilder( 178 | "SELECT * " 179 | + "FROM `" 180 | + HOME_PROJECT 181 | + "." 182 | + DATASET 183 | + "." 184 | + TABLE 185 | + "` ") 186 | .setUseLegacySql(false) 187 | .build(); 188 | 189 | TableResult result = executeBigQueryQuery(bigquery, queryConfig); 190 | 191 | // Get all pages of the results 192 | for (FieldValueList row : result.iterateAll()) { 193 | // Get all values 194 | AppAlert appAlert = new AppAlert(); 195 | appAlert.setProjectId(row.get("project_id").getStringValue()); 196 | appAlert.setEmailId(row.get("email_id").getStringValue()); 197 | appAlert.setAppCode(row.get("app_code").getStringValue()); 198 | appAlert.setDashboardUrl(row.get("dashboard_url").isNull() ? null : row.get("dashboard_url").getStringValue()); 199 | appAlert.setNotificationChannelId(row.get("notification_channel_id").isNull() ? null : row.get("notification_channel_id").getStringValue()); 200 | appAlert.setCustomLogMetricId(row.get("custom_log_metric_id").isNull() ? null : row.get("custom_log_metric_id").getStringValue()); 201 | appAlert.setAlertPolicyId(row.get("alert_policy_id").isNull() ? null : row.get("alert_policy_id").getStringValue()); 202 | appAlerts.add(appAlert); 203 | } 204 | logger.info("Query ran successfully to list App Alerts!"); 205 | 206 | return appAlerts; 207 | } 208 | 209 | /* 210 | * API to update App Alerting by adding Custom log metric Id, Notification Channel Id and Alert Policy Id for each app 211 | * */ 212 | private static void updateAppAlertConfig(BigQuery bigquery, AppAlert appAlert){ 213 | QueryJobConfiguration queryConfig = 214 | QueryJobConfiguration.newBuilder( 215 | "UPDATE " 216 | + "`" 217 | + HOME_PROJECT 218 | + "." 219 | + DATASET 220 | + "." 221 | + TABLE 222 | + "` " 223 | +" SET custom_log_metric_id = \""+appAlert.getCustomLogMetricId()+"\", notification_channel_id = \""+appAlert.getNotificationChannelId()+"\", alert_policy_id = \""+appAlert.getAlertPolicyId()+"\" " 224 | +" WHERE app_code = \""+appAlert.getAppCode()+"\"" 225 | ) 226 | .setUseLegacySql(false) 227 | .build(); 228 | 229 | TableResult result = executeBigQueryQuery(bigquery, queryConfig); 230 | 231 | /*// Print the results. 232 | result.iterateAll().forEach(rows -> rows.forEach(row -> logger.info((String) row.getValue())));*/ 233 | 234 | logger.info("Table updated successfully and updated Alert Config for App Alerts"); 235 | } 236 | 237 | 238 | /* 239 | * API to configure Custom log metrics 240 | * @param appAlert - appCode name from appAlert 241 | * @return CustomLogMetric name 242 | * */ 243 | private static String createCustomLogMetric(AppAlert appAlert){ 244 | // If custom log metric has been created for the appId, then do not create a new custom log metric 245 | if(appAlert.getCustomLogMetricId() != null) 246 | return appAlert.getCustomLogMetricId(); 247 | 248 | // Create a custom log metric for the appId if metric doesn't exist for appCode 249 | LoggingOptions options = LoggingOptions.getDefaultInstance(); 250 | MetricInfo metricInfo = null; 251 | try(Logging logging = options.getService()) { 252 | 253 | metricInfo = MetricInfo.newBuilder("resource_usage_"+appAlert.getAppCode(), "logName:\"projects/"+HOME_PROJECT+"/logs/\" jsonPayload.message:\"|AppCode-"+appAlert.getAppCode()+" | ProjectId | Scope |\"") 254 | .setDescription("Tracks logs for quota usage above threshold for app_code = "+appAlert.getAppCode()) 255 | .build(); 256 | metricInfo = logging.create(metricInfo); 257 | } catch (Exception e) { 258 | logger.severe("Error creating Custom Log Metric for app code - "+appAlert.getAppCode()+e.getMessage()); 259 | } 260 | logger.info("Successfully created custom log metric - "+ ((metricInfo == null ) ? null : metricInfo.getName())); 261 | return metricInfo.getName(); 262 | } 263 | 264 | /* 265 | * API to create or update Notification Channels for each app 266 | * */ 267 | private static String createOrUpdateNotificationChannel(AppAlert appAlert){ 268 | NotificationChannel notificationChannel = null; 269 | try (NotificationChannelServiceClient client = NotificationChannelServiceClient.create()) { 270 | // Create a new notification channel if there is no existing notification channel for the given appCode 271 | if(appAlert.getNotificationChannelId() == null){ 272 | notificationChannel = createNotificationChannel(client, appAlert); 273 | } else { 274 | // Update the notification channel if there is an existing notification channel for the given appCode 275 | // This can be used to update the emailId 276 | notificationChannel = updateNotificationChannel(client, appAlert); 277 | } 278 | } catch (IOException e) { 279 | logger.severe("Can't create Notification channel "+e.toString()); 280 | } 281 | return ((notificationChannel == null ) ? null : notificationChannel.getName()); 282 | } 283 | 284 | /* 285 | * API to create Notification Channels for a given app Code and email id 286 | * */ 287 | private static NotificationChannel createNotificationChannel(NotificationChannelServiceClient client, AppAlert appAlert){ 288 | NotificationChannel notificationChannel = NotificationChannel.newBuilder() 289 | .setType("email") 290 | .setDisplayName("OnCall-"+appAlert.getAppCode()) 291 | .setDescription("Email channel for alert notification on app -"+appAlert.getAppCode()) 292 | .putLabels("email_address",appAlert.getEmailId()) 293 | .build(); 294 | 295 | notificationChannel = client.createNotificationChannel("projects/"+HOME_PROJECT,notificationChannel); 296 | logger.info("Successfully created notification channel - "+notificationChannel.getName()); 297 | return notificationChannel; 298 | } 299 | 300 | /* 301 | * API to update Notification Channels for a given app Code and email id 302 | * */ 303 | private static NotificationChannel updateNotificationChannel(NotificationChannelServiceClient client, AppAlert appAlert){ 304 | NotificationChannel notificationChannel = NotificationChannel.newBuilder() 305 | .setType("email") 306 | .setDisplayName("OnCall-"+appAlert.getAppCode()) 307 | .setDescription("Email channel for alert notification on app -"+appAlert.getAppCode()) 308 | .putLabels("email_address",appAlert.getEmailId()) 309 | .setName(appAlert.getNotificationChannelId()) 310 | .build(); 311 | 312 | UpdateNotificationChannelRequest updateNotificationChannelRequest = 313 | UpdateNotificationChannelRequest.newBuilder().setNotificationChannel(notificationChannel).build(); 314 | notificationChannel = client.updateNotificationChannel(updateNotificationChannelRequest); 315 | logger.info("Successfully updated notification channel - "+notificationChannel.getName()); 316 | return notificationChannel; 317 | } 318 | 319 | /* 320 | * API to create or update Alert Policy for a given app Code 321 | * */ 322 | private static String createOrUpdateAlertPolicy(AppAlert appAlert) { 323 | AlertPolicy actualAlertPolicy = null; 324 | try (AlertPolicyServiceClient alertPolicyServiceClient = AlertPolicyServiceClient.create()) { 325 | 326 | // A Filter that identifies which time series should be compared with the threshold 327 | String metricFilter = 328 | "resource.type = \"cloud_function\" AND metric.type=\"logging.googleapis.com/user/" 329 | + appAlert.getCustomLogMetricId() + "\""; 330 | 331 | // Build Duration 332 | Duration aggregationDuration = Duration.newBuilder().setSeconds(60).build(); 333 | 334 | //Build Documentation 335 | Documentation documentation = Documentation.newBuilder() 336 | .setMimeType("text/markdown") 337 | .setContent("**Resource usage quota is reaching threshold in project - "+appAlert.getProjectId()+"
[See Quota Dashboard for details]("+appAlert.getDashboardUrl()+")**") 338 | .build(); 339 | 340 | // Build Aggregation 341 | Aggregation aggregation = 342 | Aggregation.newBuilder() 343 | .setAlignmentPeriod(aggregationDuration) 344 | .setPerSeriesAligner(Aligner.ALIGN_COUNT) 345 | .build(); 346 | 347 | // Build MetricThreshold 348 | AlertPolicy.Condition.MetricThreshold metricThreshold = 349 | MetricThreshold.newBuilder() 350 | .setComparison(ComparisonType.COMPARISON_GT) 351 | .addAggregations(aggregation) 352 | .setFilter(metricFilter) 353 | .setDuration(aggregationDuration) 354 | .setTrigger(Trigger.newBuilder().setCount(1).build()) 355 | .build(); 356 | 357 | // Construct Condition object 358 | AlertPolicy.Condition alertPolicyCondition = 359 | AlertPolicy.Condition.newBuilder() 360 | .setDisplayName("QuotaExceedAlertPolicy-" + appAlert.getAppCode()) 361 | .setConditionThreshold(metricThreshold) 362 | .build(); 363 | 364 | // Create an alert policy 365 | if(appAlert.getAlertPolicyId() == null){ 366 | actualAlertPolicy = createAlertPolicy(alertPolicyServiceClient, appAlert, documentation, alertPolicyCondition); 367 | } else { 368 | //Update Alert policy 369 | actualAlertPolicy = updateAlertPolicy(alertPolicyServiceClient, appAlert, documentation, alertPolicyCondition); 370 | } 371 | 372 | 373 | } catch (IOException e) { 374 | logger.severe("Error creating or updating Alert Policy for app code - "+appAlert.getAppCode()+e.getMessage()); 375 | } 376 | return ((actualAlertPolicy == null) ? null : actualAlertPolicy.getName()); 377 | 378 | } 379 | 380 | /* 381 | * API to create an Alert Policy for a given app Code 382 | * */ 383 | private static AlertPolicy createAlertPolicy(AlertPolicyServiceClient client, AppAlert appAlert, Documentation documentation, Condition alertPolicyCondition){ 384 | AlertPolicy alertPolicy = 385 | AlertPolicy.newBuilder() 386 | .setDisplayName("QuotaExceedAlertPolicy-" + appAlert.getAppCode()) 387 | .setDocumentation(documentation) 388 | .addConditions(alertPolicyCondition) 389 | .setCombiner(ConditionCombinerType.OR) 390 | .addNotificationChannels(appAlert.getNotificationChannelId()) 391 | .build(); 392 | 393 | AlertPolicy actualAlertPolicy = client.createAlertPolicy(ProjectName.of(HOME_PROJECT), alertPolicy); 394 | logger.info("alert policy created successfully - "+ actualAlertPolicy.getName()); 395 | return actualAlertPolicy; 396 | } 397 | 398 | /* 399 | * API to update an Alert Policy for a given app Code 400 | * */ 401 | private static AlertPolicy updateAlertPolicy(AlertPolicyServiceClient client, AppAlert appAlert, Documentation documentation, Condition alertPolicyCondition){ 402 | AlertPolicy alertPolicy1 = 403 | AlertPolicy.newBuilder() 404 | .setName(appAlert.getAlertPolicyId()) 405 | .setDisplayName("QuotaExceedAlertPolicy-" + appAlert.getAppCode()) 406 | .setDocumentation(documentation) 407 | .addConditions(alertPolicyCondition) 408 | .setCombiner(ConditionCombinerType.OR) 409 | .addNotificationChannels(appAlert.getNotificationChannelId()) 410 | .build(); 411 | 412 | UpdateAlertPolicyRequest updateAlertPolicyRequest = 413 | UpdateAlertPolicyRequest.newBuilder().setAlertPolicy(alertPolicy1).build(); 414 | AlertPolicy actualAlertPolicy = client.updateAlertPolicy(updateAlertPolicyRequest); 415 | logger.info("alert policy updated successfully - "+ actualAlertPolicy.getName()); 416 | return actualAlertPolicy; 417 | } 418 | 419 | /* 420 | * API to check that table is empty before initializing the data 421 | * */ 422 | private static boolean isAppAlertTableEmpty(BigQuery bigquery){ 423 | int count = 0; 424 | QueryJobConfiguration queryConfig = 425 | QueryJobConfiguration.newBuilder( 426 | "SELECT COUNT (*) as count " 427 | + "FROM `" 428 | + HOME_PROJECT 429 | + "." 430 | + DATASET 431 | + "." 432 | + TABLE 433 | + "` ") 434 | .setUseLegacySql(false) 435 | .build(); 436 | 437 | TableResult result = executeBigQueryQuery(bigquery, queryConfig); 438 | 439 | for (FieldValueList row : result.iterateAll()) { 440 | count = (int) row.get("count").getLongValue(); 441 | } 442 | //Return true is table contains zero records else return false 443 | return count <= 0; 444 | } 445 | 446 | /* 447 | * API to execute parse CSV file before uploading to BigQuery for App Alert data initialization 448 | * 1. Check that file exists 449 | * 2. Check if each record contains projectId, emailId and appCode 450 | * 3. Check that appCode is not duplicate 451 | * */ 452 | private static boolean isCSVParseSuccess(){ 453 | 454 | BufferedReader bufferedReader = null; 455 | String row = ""; 456 | Set appCodes = new HashSet<>(); 457 | 458 | Storage storage = StorageOptions.getDefaultInstance().getService(); 459 | String bucketName = StringUtils.substringBetween(CSV_SOURCE_URI, "//", "/"); 460 | String fileName = StringUtils.substringAfterLast(CSV_SOURCE_URI, "/"); 461 | Blob blob = storage.get(bucketName,fileName); 462 | //If csv file is not found in the cloud storage bucket, return 463 | if(blob == null){ 464 | logger.severe("CSV file not found - "+CSV_SOURCE_URI); 465 | return false; 466 | } 467 | ReadChannel readChannel = blob.reader(); 468 | try { 469 | bufferedReader = new BufferedReader(Channels.newReader(readChannel, "UTF-8")); 470 | while ((row = bufferedReader.readLine()) != null) { 471 | String[] cells = row.split(","); 472 | 473 | //2. Check if each record contains projectId, emailId and appCode 474 | if(StringUtils.isEmpty(cells[0]) || StringUtils.isEmpty(cells[1]) || StringUtils.isEmpty(cells[2])){ 475 | logger.severe("Found empty record in CSV. Can't load csv in BigQuery. Required project_id, email_id and app_code in each rows"); 476 | return false; 477 | } 478 | 479 | //3. Check that appCode is not duplicate 480 | if(!appCodes.add(cells[2])) { 481 | logger.severe("Found duplicate app code \""+cells[2]+"\" in CSV. Can't load csv in BigQuery. app_code should be unique for each row"); 482 | return false; 483 | } 484 | } 485 | } catch (FileNotFoundException e) { 486 | logger.severe("File not found - "+CSV_SOURCE_URI+e.getMessage()); 487 | return false; 488 | } catch (IOException e) { 489 | logger.severe("Error parsing CSV file - "+CSV_SOURCE_URI+e.getMessage()); 490 | return false; 491 | } 492 | return true; 493 | } 494 | 495 | 496 | /* 497 | * API to execute DML query on BigQuery table for the given query 498 | * */ 499 | private static TableResult executeBigQueryQuery(BigQuery bigquery, QueryJobConfiguration queryConfig){ 500 | TableResult result = null; 501 | try{ 502 | // Create a job ID so that we can safely retry. 503 | JobId jobId = JobId.of(UUID.randomUUID().toString()); 504 | Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); 505 | 506 | // Wait for the query to complete. 507 | queryJob = queryJob.waitFor(); 508 | 509 | // Check for errors 510 | if (queryJob == null) { 511 | throw new RuntimeException("Job no longer exists"); 512 | } else if (queryJob.getStatus().getError() != null) { 513 | throw new RuntimeException(queryJob.getStatus().getError().toString()); 514 | } 515 | 516 | // Identify the table 517 | result = queryJob.getQueryResults(); 518 | } catch (InterruptedException e) { 519 | logger.severe("Error executing BigQuery query"+e.getMessage()); 520 | } 521 | return result; 522 | } 523 | 524 | } 525 | -------------------------------------------------------------------------------- /quota-notification/src/main/java/functions/SendNotification.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package functions; 17 | 18 | import static functions.ConfigureAppAlertHelper.listAppAlertConfig; 19 | 20 | import com.google.cloud.bigquery.BigQuery; 21 | import com.google.cloud.bigquery.BigQueryException; 22 | import com.google.cloud.bigquery.BigQueryOptions; 23 | import com.google.cloud.bigquery.FieldValueList; 24 | import com.google.cloud.bigquery.Job; 25 | import com.google.cloud.bigquery.JobId; 26 | import com.google.cloud.bigquery.JobInfo; 27 | import com.google.cloud.bigquery.QueryJobConfiguration; 28 | import com.google.cloud.bigquery.TableResult; 29 | import com.google.cloud.functions.BackgroundFunction; 30 | import com.google.cloud.functions.Context; 31 | import functions.eventpojos.Alert; 32 | import functions.eventpojos.AppAlert; 33 | import functions.eventpojos.PubSubMessage; 34 | import java.util.ArrayList; 35 | import java.util.HashMap; 36 | import java.util.List; 37 | import java.util.Map; 38 | import java.util.UUID; 39 | import java.util.logging.Logger; 40 | 41 | /* 42 | * Cloud Function triggered by Pub/Sub topic to send notification 43 | * */ 44 | public class SendNotification implements BackgroundFunction { 45 | private static final String HOME_PROJECT = System.getenv("HOME_PROJECT"); 46 | private static final String DATASET = System.getenv("ALERT_DATASET"); 47 | private static final String TABLE = System.getenv("ALERT_TABLE"); 48 | private static final String APP_ALERT_DATASET = System.getenv("APP_ALERT_DATASET"); 49 | private static final String APP_ALERT_TABLE = System.getenv("APP_ALERT_TABLE"); 50 | 51 | private static final String DELIMITER = "|"; 52 | private static final String DELIMITER_REGEX = String.valueOf("\\|"); 53 | 54 | private static final Logger logger = Logger.getLogger(SendNotification.class.getName()); 55 | 56 | /* 57 | * API to accept notification information and process it 58 | * */ 59 | @Override 60 | public void accept(PubSubMessage message, Context context) { 61 | // Initialize client that will be used to send requests 62 | BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); 63 | // logger.info(String.format(message.getEmailIds())); 64 | logger.info("Successfully made it to sendNotification"); 65 | // get rows from quota_monitoring_notification_table 66 | List alerts = browseAlertTable(bigquery); 67 | logger.info("Successfully got data from alert table"); 68 | String alertMessage = buildAlertMessage(alerts, null); 69 | logger.info(alertMessage); 70 | appAlertLogs(bigquery, alerts); 71 | return; 72 | } 73 | 74 | /* 75 | * API to fetch records which qualifies for alerting from the main table 76 | * */ 77 | private static List browseAlertTable(BigQuery bigquery) { 78 | List alerts = new ArrayList(); 79 | Alert alert = new Alert(); 80 | try { 81 | 82 | QueryJobConfiguration queryConfig = 83 | QueryJobConfiguration.newBuilder( 84 | "SELECT project_id, region, quota_metric, current_usage, quota_limit, current_consumption " 85 | + "FROM `" 86 | + HOME_PROJECT 87 | + "." 88 | + DATASET 89 | + "." 90 | + TABLE 91 | + "` ") 92 | .setUseLegacySql(false) 93 | .build(); 94 | 95 | // Create a job ID so that we can safely retry. 96 | JobId jobId = JobId.of(UUID.randomUUID().toString()); 97 | Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); 98 | 99 | // Wait for the query to complete. 100 | queryJob = queryJob.waitFor(); 101 | 102 | // Check for errors 103 | if (queryJob == null) { 104 | throw new RuntimeException("Job no longer exists"); 105 | } else if (queryJob.getStatus().getError() != null) { 106 | throw new RuntimeException(queryJob.getStatus().getError().toString()); 107 | } 108 | 109 | // Identify the table 110 | TableResult result = queryJob.getQueryResults(); 111 | 112 | // Get all pages of the results 113 | for (FieldValueList row : result.iterateAll()) { 114 | // Get all values 115 | alert = new Alert(); 116 | alert.setProjectId(row.get("project_id").getStringValue()); 117 | alert.setRegion(row.get("region").getStringValue()); 118 | alert.setQuotaMetric(row.get("quota_metric").getStringValue()); 119 | alert.setQuotaLimit(row.get("quota_limit").getStringValue()); 120 | alert.setCurrentUsage(row.get("current_usage").getStringValue()); 121 | alert.setCurrentConsumption(row.get("current_consumption").getNumericValue().floatValue()); 122 | 123 | alerts.add(alert); 124 | } 125 | logger.info("Query ran successfully "); 126 | } catch (BigQueryException | InterruptedException e) { 127 | logger.severe("Query failed to run \n" + e.toString()); 128 | } 129 | return alerts; 130 | } 131 | 132 | /* 133 | * API to build Alert Message for list of Quota metrics 134 | * */ 135 | private static String buildAlertMessage(List alerts, String appCode){ 136 | StringBuilder htmlBuilder = new StringBuilder(); 137 | htmlBuilder.append("Quota metric usage alert details\n\n"); 138 | htmlBuilder.append("## "+alerts.size()+" quota metric usages above threshold\n\n"); 139 | if(appCode == null) 140 | htmlBuilder.append("|ProjectId | Scope | Metric | Consumption(%) |\n"); 141 | else 142 | htmlBuilder.append("|AppCode-"+appCode+" | ProjectId | Scope | Metric | Consumption(%) |\n"); 143 | htmlBuilder.append("|:---------|:------|:--------|:---------------|\n"); 144 | for(Alert alert : alerts){ 145 | htmlBuilder.append(alert.toString()); 146 | htmlBuilder.append("|\n"); 147 | } 148 | String html = htmlBuilder.toString(); 149 | return html; 150 | } 151 | 152 | /* 153 | * API to log App Alert Message for list of Quota metrics 154 | * */ 155 | static void appAlertLogs(BigQuery bigquery, List alerts){ 156 | // quota_monitoring_app_alerting_table populated by csv 157 | List appAlertsConfigs = listAppAlertConfig(bigquery); 158 | Map appAlertsConfigsMap = new HashMap<>(); 159 | Map> appAlerts = new HashMap<>(); 160 | 161 | //Convert List to Map 162 | for(AppAlert appAlertConfig : appAlertsConfigs){ 163 | String projects = appAlertConfig.getProjectId(); 164 | if (projects.contains(DELIMITER)) { 165 | String[] projectIds = projects.split(DELIMITER_REGEX); 166 | for ( String p : projectIds) { 167 | appAlertsConfigsMap.put(p, appAlertConfig.getAppCode()); 168 | } 169 | } else { 170 | appAlertsConfigsMap.put(appAlertConfig.getProjectId(), appAlertConfig.getAppCode()); 171 | } 172 | } 173 | 174 | for(Alert alert : alerts){ 175 | String appCode = appAlertsConfigsMap.get(alert.getProjectId()); 176 | if (appAlertsConfigsMap.containsKey(alert.getProjectId())){ 177 | if(appAlerts.containsKey(appCode)){ 178 | appAlerts.get(appCode).add(alert); 179 | } else { 180 | List appAlert = new ArrayList<>(); 181 | appAlert.add(alert); 182 | appAlerts.put(appCode, appAlert); 183 | } 184 | 185 | } 186 | } 187 | 188 | for(Map.Entry> entry : appAlerts.entrySet() ){ 189 | if(entry.getValue().size() > 0){ 190 | String alertMessage = buildAlertMessage(entry.getValue(), entry.getKey()); 191 | logger.info(alertMessage); 192 | } 193 | } 194 | } 195 | 196 | /* 197 | * API to fetch App Alert configurations from BigQuery 198 | * @return - List of App Alerts 199 | * */ 200 | public static List listAppAlertConfig(BigQuery bigquery){ 201 | List appAlerts = new ArrayList<>(); 202 | QueryJobConfiguration queryConfig = 203 | QueryJobConfiguration.newBuilder( 204 | "SELECT * " 205 | + "FROM `" 206 | + HOME_PROJECT 207 | + "." 208 | + APP_ALERT_DATASET 209 | + "." 210 | + APP_ALERT_TABLE 211 | + "` ") 212 | .setUseLegacySql(false) 213 | .build(); 214 | 215 | TableResult result = executeBigQueryQuery(bigquery, queryConfig); 216 | 217 | // Get all pages of the results 218 | for (FieldValueList row : result.iterateAll()) { 219 | // Get all values 220 | AppAlert appAlert = new AppAlert(); 221 | appAlert.setProjectId(row.get("project_id").getStringValue()); 222 | appAlert.setEmailId(row.get("email_id").getStringValue()); 223 | appAlert.setAppCode(row.get("app_code").getStringValue()); 224 | appAlert.setDashboardUrl(row.get("dashboard_url").isNull() ? null : row.get("dashboard_url").getStringValue()); 225 | appAlert.setNotificationChannelId(row.get("notification_channel_id").isNull() ? null : row.get("notification_channel_id").getStringValue()); 226 | appAlert.setCustomLogMetricId(row.get("custom_log_metric_id").isNull() ? null : row.get("custom_log_metric_id").getStringValue()); 227 | appAlert.setAlertPolicyId(row.get("alert_policy_id").isNull() ? null : row.get("alert_policy_id").getStringValue()); 228 | appAlerts.add(appAlert); 229 | } 230 | logger.info("Query ran successfully to list App Alerts!"); 231 | 232 | return appAlerts; 233 | } 234 | 235 | /* 236 | * API to execute DML query on BigQuery table for the given query 237 | * */ 238 | private static TableResult executeBigQueryQuery(BigQuery bigquery, QueryJobConfiguration queryConfig){ 239 | TableResult result = null; 240 | try{ 241 | // Create a job ID so that we can safely retry. 242 | JobId jobId = JobId.of(UUID.randomUUID().toString()); 243 | Job queryJob = bigquery.create(JobInfo.newBuilder(queryConfig).setJobId(jobId).build()); 244 | 245 | // Wait for the query to complete. 246 | queryJob = queryJob.waitFor(); 247 | 248 | // Check for errors 249 | if (queryJob == null) { 250 | throw new RuntimeException("Job no longer exists"); 251 | } else if (queryJob.getStatus().getError() != null) { 252 | throw new RuntimeException(queryJob.getStatus().getError().toString()); 253 | } 254 | 255 | // Identify the table 256 | result = queryJob.getQueryResults(); 257 | } catch (InterruptedException e) { 258 | logger.severe("Error executing BigQuery query"+e.getMessage()); 259 | } 260 | return result; 261 | } 262 | } 263 | -------------------------------------------------------------------------------- /quota-notification/src/main/java/functions/eventpojos/Alert.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package functions.eventpojos; 17 | 18 | public class Alert { 19 | private String projectId; 20 | private String region; 21 | private String quotaMetric; 22 | private String currentUsage; 23 | private String quotaLimit; 24 | private Float currentConsumption; 25 | 26 | public String getProjectId() { 27 | return projectId; 28 | } 29 | 30 | public void setProjectId(String projectId) { 31 | this.projectId = projectId; 32 | } 33 | 34 | public String getRegion() { 35 | return region; 36 | } 37 | 38 | public void setRegion(String region) { 39 | this.region = region; 40 | } 41 | 42 | public String getQuotaMetric() { 43 | return quotaMetric; 44 | } 45 | 46 | public void setQuotaMetric(String quotaMetric) { 47 | this.quotaMetric = quotaMetric; 48 | } 49 | 50 | public String getCurrentUsage() { 51 | return currentUsage; 52 | } 53 | 54 | public void setCurrentUsage(String currentUsage) { 55 | this.currentUsage = currentUsage; 56 | } 57 | 58 | public String getQuotaLimit() { 59 | return quotaLimit; 60 | } 61 | 62 | public void setQuotaLimit(String quotaLimit) { 63 | this.quotaLimit = quotaLimit; 64 | } 65 | 66 | public Float getCurrentConsumption() { 67 | return currentConsumption; 68 | } 69 | 70 | public void setCurrentConsumption(Float currentConsumption) { 71 | this.currentConsumption = currentConsumption; 72 | } 73 | 74 | public String toString(){ 75 | StringBuilder alertBuilder = new StringBuilder(); 76 | alertBuilder.append("|" + projectId); 77 | alertBuilder.append("|"+region); 78 | alertBuilder.append("|`"+ quotaMetric.replace(".com"," .com")+"`"); 79 | //alertBuilder.append("|"+usage); 80 | //alertBuilder.append("|"+limit); 81 | alertBuilder.append("|"+ currentConsumption); 82 | return alertBuilder.toString(); 83 | } 84 | } 85 | 86 | 87 | -------------------------------------------------------------------------------- /quota-notification/src/main/java/functions/eventpojos/AppAlert.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package functions.eventpojos; 18 | 19 | public class AppAlert { 20 | private String projectId; 21 | private String emailId; 22 | private String appCode; 23 | private String dashboardUrl; 24 | private String notificationChannelId; 25 | private String customLogMetricId; 26 | private String alertPolicyId; 27 | 28 | public String getProjectId() { 29 | return projectId; 30 | } 31 | 32 | public void setProjectId(String projectId) { 33 | this.projectId = projectId; 34 | } 35 | 36 | public String getEmailId() { 37 | return emailId; 38 | } 39 | 40 | public void setEmailId(String emailId) { 41 | this.emailId = emailId; 42 | } 43 | 44 | public String getAppCode() { 45 | return appCode; 46 | } 47 | 48 | public void setAppCode(String appCode) { 49 | this.appCode = appCode; 50 | } 51 | 52 | public String getDashboardUrl() { 53 | return dashboardUrl; 54 | } 55 | 56 | public void setDashboardUrl(String dashboardUrl) { 57 | this.dashboardUrl = dashboardUrl; 58 | } 59 | 60 | public String getNotificationChannelId() { 61 | return notificationChannelId; 62 | } 63 | 64 | public void setNotificationChannelId(String notificationChannelId) { 65 | this.notificationChannelId = notificationChannelId; 66 | } 67 | 68 | public String getCustomLogMetricId() { 69 | return customLogMetricId; 70 | } 71 | 72 | public void setCustomLogMetricId(String customLogMetricId) { 73 | this.customLogMetricId = customLogMetricId; 74 | } 75 | 76 | public String getAlertPolicyId() { 77 | return alertPolicyId; 78 | } 79 | 80 | public void setAlertPolicyId(String alertPolicyId) { 81 | this.alertPolicyId = alertPolicyId; 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /quota-notification/src/main/java/functions/eventpojos/PubSubMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package functions.eventpojos; 17 | 18 | public class PubSubMessage { 19 | // Cloud Functions uses GSON to populate this object. 20 | // Field types/names are specified by Cloud Functions 21 | // Changing them may break your code! 22 | private String metric; 23 | private String limit; 24 | private String usage; 25 | private Float consumption; 26 | private String emailIds; 27 | 28 | public String getMetric() { 29 | return metric; 30 | } 31 | 32 | public void setMetric(String metric) { 33 | this.metric = metric; 34 | } 35 | 36 | public String getLimit() { 37 | return limit; 38 | } 39 | 40 | public void setLimit(String limit) { 41 | this.limit = limit; 42 | } 43 | 44 | public String getUsage() { 45 | return usage; 46 | } 47 | 48 | public void setUsage(String usage) { 49 | this.usage = usage; 50 | } 51 | 52 | public Float getConsumption() { 53 | return consumption; 54 | } 55 | 56 | public void setConsumption(Float consumption) { 57 | this.consumption = consumption; 58 | } 59 | 60 | public String getEmailIds() { 61 | return emailIds; 62 | } 63 | 64 | public void setEmailIds(String emailIds) { 65 | this.emailIds = emailIds; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /quota-notification/src/test/java/functions/SendNotificationTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | package functions; 17 | 18 | import static com.google.common.truth.Truth.assertThat; 19 | 20 | import com.google.common.testing.TestLogHandler; 21 | import functions.eventpojos.PubSubMessage; 22 | import java.util.logging.Logger; 23 | import org.junit.Before; 24 | import org.junit.Test; 25 | import org.junit.runner.RunWith; 26 | import org.junit.runners.JUnit4; 27 | 28 | @RunWith(JUnit4.class) 29 | public class SendNotificationTest { 30 | private SendNotification sampleUnderTest; 31 | private static final Logger logger = Logger.getLogger(SendNotification.class.getName()); 32 | 33 | private static final TestLogHandler LOG_HANDLER = new TestLogHandler(); 34 | 35 | @Before 36 | public void setUp() { 37 | sampleUnderTest = new SendNotification(); 38 | logger.addHandler(LOG_HANDLER); 39 | LOG_HANDLER.clear(); 40 | } 41 | 42 | @Test 43 | public void sendNotification_shouldSendEmail() { 44 | PubSubMessage pubSubMessage = new PubSubMessage(); 45 | pubSubMessage.setEmailIds("anuradhabajpai@google.com"); 46 | pubSubMessage.setLimit("100"); 47 | pubSubMessage.setMetric("VPC"); 48 | pubSubMessage.setUsage("80"); 49 | pubSubMessage.setConsumption(80f); 50 | /*pubSubMessage.setData(Base64.getEncoder().encodeToString( 51 | "anuradha.bajpai@gmail.com,anuradhabajpai@google.com".getBytes(StandardCharsets.UTF_8)));*/ 52 | sampleUnderTest.accept(pubSubMessage, null); 53 | 54 | String logMessage = LOG_HANDLER.getStoredLogRecords().get(0).getMessage(); 55 | assertThat("anuradhabajpai@google.com").isEqualTo(logMessage); 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /quota-scan/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 18 | 19 | 22 | 4.0.0 23 | 24 | com.example.cloud.functions 25 | functions-hello-pub-sub 26 | 27 | 28 | com.google.cloud.samples 29 | shared-configuration 30 | 1.0.21 31 | 32 | 33 | 34 | 11 35 | 11 36 | UTF-8 37 | 38 | 39 | 40 | 41 | 42 | 43 | com.google.cloud 44 | libraries-bom 45 | 26.1.3 46 | pom 47 | import 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | com.google.cloud.functions 56 | functions-framework-api 57 | 1.0.4 58 | provided 59 | 60 | 61 | com.google.cloud 62 | google-cloud-compute 63 | 64 | 65 | com.google.cloud 66 | google-cloud-resourcemanager 67 | 68 | 69 | com.google.auth 70 | google-auth-library-oauth2-http 71 | 1.11.0 72 | 73 | 74 | com.google.cloud 75 | google-cloud-pubsub 76 | 77 | 78 | com.google.cloud 79 | google-cloud-bigquery 80 | 81 | 82 | org.threeten 83 | threetenbp 84 | 1.6.2 85 | 86 | 87 | 88 | 89 | com.google.truth 90 | truth 91 | 1.1.3 92 | test 93 | 94 | 95 | com.google.guava 96 | guava-testlib 97 | 31.1-jre 98 | test 99 | 100 | 101 | org.mockito 102 | mockito-all 103 | 2.0.2-beta 104 | test 105 | 106 | 107 | 108 | 109 | 110 | com.google.cloud 111 | google-cloud-logging 112 | test 113 | 114 | 115 | com.google.code.gson 116 | gson 117 | 2.9.1 118 | 119 | 120 | com.google.cloud 121 | google-cloud-monitoring 122 | 3.4.6 123 | 124 | 125 | 126 | 127 | org.apache.httpcomponents 128 | httpclient 129 | 4.5.13 130 | test 131 | 132 | 133 | 134 | 135 | io.github.resilience4j 136 | resilience4j-core 137 | 1.7.1 138 | test 139 | 140 | 141 | io.github.resilience4j 142 | resilience4j-retry 143 | 1.7.1 144 | test 145 | 146 | 147 | 148 | 149 | com.google.cloud.functions 150 | function-maven-plugin 151 | 0.10.1 152 | test 153 | 154 | 155 | com.google.api.grpc 156 | proto-google-cloud-pubsub-v1 157 | 158 | 159 | 160 | 161 | 162 | 163 | org.apache.maven.plugins 164 | maven-surefire-plugin 165 | 166 | 167 | 3.0.0-M5 168 | 169 | 170 | **/*Test.java 171 | 172 | ${skipTests} 173 | sponge_log 174 | false 175 | 176 | 177 | 178 | 187 | com.google.cloud.functions 188 | function-maven-plugin 189 | 0.9.2 190 | 191 | functions.ScanProjectQuotas 192 | 193 | 194 | 195 | 196 | 197 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/ListProjects.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package functions; 18 | 19 | import com.google.api.client.googleapis.auth.oauth2.GoogleCredential; 20 | import com.google.api.client.googleapis.javanet.GoogleNetHttpTransport; 21 | import com.google.api.client.http.HttpTransport; 22 | import com.google.api.client.json.JsonFactory; 23 | import com.google.api.client.json.jackson2.JacksonFactory; 24 | import com.google.api.core.ApiFuture; 25 | import com.google.api.core.ApiFutures; 26 | import com.google.api.gax.batching.BatchingSettings; 27 | import com.google.api.services.cloudresourcemanager.CloudResourceManager; 28 | import com.google.api.services.cloudresourcemanager.model.ListProjectsResponse; 29 | import com.google.api.services.cloudresourcemanager.model.Project; 30 | import com.google.cloud.functions.HttpFunction; 31 | import com.google.cloud.functions.HttpRequest; 32 | import com.google.cloud.functions.HttpResponse; 33 | import com.google.cloud.pubsub.v1.Publisher; 34 | import com.google.gson.Gson; 35 | import com.google.gson.JsonElement; 36 | import com.google.gson.JsonObject; 37 | import com.google.gson.JsonParseException; 38 | import com.google.protobuf.ByteString; 39 | import com.google.pubsub.v1.ProjectTopicName; 40 | import com.google.pubsub.v1.PubsubMessage; 41 | import com.google.pubsub.v1.TopicName; 42 | import java.io.IOException; 43 | import java.io.PrintWriter; 44 | import java.nio.charset.StandardCharsets; 45 | import java.security.GeneralSecurityException; 46 | import java.util.ArrayList; 47 | import java.util.Arrays; 48 | import java.util.List; 49 | import java.util.concurrent.ExecutionException; 50 | import java.util.concurrent.TimeUnit; 51 | import java.util.logging.Level; 52 | import java.util.logging.Logger; 53 | import org.threeten.bp.Duration; 54 | 55 | /* 56 | * The ListProjects Cloud Function lists project Ids for a given parent node. 57 | * Parent node in this context could be an Organization Id or a folder Id 58 | * */ 59 | public class ListProjects implements HttpFunction { 60 | // Cloud Function Environment variable for Pub/Sub topic name to publish project Ids 61 | private static final String TOPIC_NAME = System.getenv("PUBLISH_TOPIC"); 62 | // Cloud Function Environment variable for Home Project Id 63 | private static final String HOME_PROJECT_ID = System.getenv("HOME_PROJECT"); 64 | // Cloud Function Environment variable for Threshold 65 | private static final String THRESHOLD = System.getenv("THRESHOLD"); 66 | 67 | private static final Logger logger = Logger.getLogger(ListProjects.class.getName()); 68 | 69 | private static final Gson gson = new Gson(); 70 | 71 | /* 72 | * API to accept the Http request to Cloud Function. 73 | * */ 74 | @Override 75 | public void service(HttpRequest request, HttpResponse response) throws IOException { 76 | String projectId = request.getFirstQueryParameter("projectId").orElse(HOME_PROJECT_ID); 77 | String threshold = request.getFirstQueryParameter("threshold").orElse(THRESHOLD); 78 | request.getQueryParameters(); 79 | 80 | // Parse JSON request and check for "organization" and "projectId" fields 81 | String responseMessage = null; 82 | try { 83 | JsonElement requestParsed = gson.fromJson(request.getReader(), JsonElement.class); 84 | JsonObject requestJson = null; 85 | 86 | if (requestParsed != null && requestParsed.isJsonObject()) { 87 | requestJson = requestParsed.getAsJsonObject(); 88 | } 89 | 90 | if (requestJson != null && requestJson.has("organizations")) { 91 | projectId = requestJson.get("projectId").getAsString(); 92 | } 93 | logger.info("Publishing message to topic: " + TOPIC_NAME); 94 | logger.info("ProjectId: " + projectId); 95 | 96 | ByteString byteStr = ByteString.copyFrom(HOME_PROJECT_ID, StandardCharsets.UTF_8); 97 | PubsubMessage pubsubApiMessage = PubsubMessage.newBuilder().setData(byteStr).build(); 98 | Publisher publisher = 99 | Publisher.newBuilder(ProjectTopicName.of(HOME_PROJECT_ID, TOPIC_NAME)).build(); 100 | // Attempt to publish the message 101 | publisher.publish(pubsubApiMessage).get(); 102 | responseMessage = "Message published."; 103 | List projectIds = getProjectIds(); 104 | publishMessages(projectIds); 105 | } catch (JsonParseException e) { 106 | logger.severe("Error parsing JSON: " + e.getMessage()); 107 | } catch (InterruptedException | ExecutionException | GeneralSecurityException e) { 108 | logger.log(Level.SEVERE, "Error publishing Pub/Sub message: " + e.getMessage(), e); 109 | responseMessage = "Error publishing Pub/Sub message; see logs for more info."; 110 | } 111 | 112 | var writer = new PrintWriter(response.getWriter()); 113 | writer.printf("projectId: %s!", HOME_PROJECT_ID); 114 | writer.printf("publish response: %s!", responseMessage); 115 | } 116 | 117 | /* 118 | * API to get accessible project Ids and create a list 119 | * */ 120 | private static List getProjectIds() throws IOException, GeneralSecurityException { 121 | List projectIds = new ArrayList<>(); 122 | // Instantiate Cloud Resource Manager Service and list projects. 123 | CloudResourceManager.Projects.List request = 124 | createCloudResourceManagerService().projects().list(); 125 | // Iterate over the project list and fetch project Ids to create a list of project Ids 126 | ListProjectsResponse projectsResponse; 127 | do { 128 | projectsResponse = request.execute(); 129 | if (projectsResponse.getProjects() == null) { 130 | continue; 131 | } 132 | for (Project project : projectsResponse.getProjects()) { 133 | projectIds.add(project.getProjectId()); 134 | logger.info("Received Project Id: " + project.getProjectId()); 135 | } 136 | request.setPageToken(projectsResponse.getNextPageToken()); 137 | } while (projectsResponse.getNextPageToken() != null); 138 | return projectIds; 139 | } 140 | 141 | /* 142 | * API to publish message to Pub/Sub topic 143 | * */ 144 | public static void publishMessages(List projectIds) 145 | throws IOException, ExecutionException, InterruptedException { 146 | TopicName topicName = TopicName.of(HOME_PROJECT_ID, TOPIC_NAME); 147 | Publisher publisher = null; 148 | List> messageIdFutures = new ArrayList<>(); 149 | 150 | try { 151 | // Batch settings control how the publisher batches messages 152 | long requestBytesThreshold = 5000L; // default : 1 byte 153 | long messageCountBatchSize = 100L; // default : 1 message 154 | 155 | Duration publishDelayThreshold = Duration.ofMillis(100); // default : 1 ms 156 | 157 | // Publish request get triggered based on request size, messages count & time since last 158 | // publish, whichever condition is met first. 159 | BatchingSettings batchingSettings = 160 | BatchingSettings.newBuilder() 161 | .setElementCountThreshold(messageCountBatchSize) 162 | .setRequestByteThreshold(requestBytesThreshold) 163 | .setDelayThreshold(publishDelayThreshold) 164 | .build(); 165 | 166 | // Create a publisher instance with default settings bound to the topic 167 | publisher = Publisher.newBuilder(topicName).setBatchingSettings(batchingSettings).build(); 168 | 169 | // schedule publishing one message at a time : messages get automatically batched 170 | for (String projectId : projectIds) { 171 | String message = projectId; 172 | ByteString data = ByteString.copyFromUtf8(message); 173 | PubsubMessage pubsubMessage = PubsubMessage.newBuilder().setData(data).build(); 174 | 175 | // Once published, returns a server-assigned message id (unique within the topic) 176 | ApiFuture messageIdFuture = publisher.publish(pubsubMessage); 177 | messageIdFutures.add(messageIdFuture); 178 | } 179 | } finally { 180 | // Wait on any pending publish requests. 181 | List messageIds = ApiFutures.allAsList(messageIdFutures).get(); 182 | 183 | logger.info("Published " + messageIds.size() + " messages with batch settings."); 184 | 185 | if (publisher != null) { 186 | // When finished with the publisher, shutdown to free up resources. 187 | publisher.shutdown(); 188 | publisher.awaitTermination(1, TimeUnit.MINUTES); 189 | } 190 | } 191 | } 192 | 193 | /* 194 | * API to get an instance of Cloud Resource Manager Service 195 | * */ 196 | private static CloudResourceManager createCloudResourceManagerService() 197 | throws IOException, GeneralSecurityException { 198 | HttpTransport httpTransport = GoogleNetHttpTransport.newTrustedTransport(); 199 | JsonFactory jsonFactory = JacksonFactory.getDefaultInstance(); 200 | 201 | GoogleCredential credential = GoogleCredential.getApplicationDefault(); 202 | if (credential.createScopedRequired()) { 203 | credential = 204 | credential.createScoped(Arrays.asList("https://www.googleapis.com/auth/cloud-platform")); 205 | } 206 | 207 | return new CloudResourceManager.Builder(httpTransport, jsonFactory, credential) 208 | .setApplicationName("Google-CloudResourceManagerSample/0.1") 209 | .build(); 210 | } 211 | } 212 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/ScanProjectQuotas.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package functions; 15 | 16 | import static functions.ScanProjectQuotasHelper.createGCPResourceClient; 17 | import static functions.ScanProjectQuotasHelper.getQuota; 18 | import static functions.ScanProjectQuotasHelper.loadBigQueryTable; 19 | 20 | import com.google.cloud.functions.BackgroundFunction; 21 | import com.google.cloud.functions.Context; 22 | import com.google.monitoring.v3.ProjectName; 23 | import functions.eventpojos.GCPProject; 24 | import functions.eventpojos.GCPResourceClient; 25 | import functions.eventpojos.PubSubMessage; 26 | import functions.eventpojos.ProjectQuota; 27 | 28 | import java.io.IOException; 29 | import java.nio.charset.StandardCharsets; 30 | import java.util.Base64; 31 | import java.util.List; 32 | import java.util.logging.Level; 33 | import java.util.logging.Logger; 34 | 35 | public class ScanProjectQuotas implements BackgroundFunction { 36 | private static final Logger logger = Logger.getLogger(ScanProjectQuotas.class.getName()); 37 | 38 | // Cloud Function Environment variable for Threshold 39 | public static final String THRESHOLD = System.getenv("THRESHOLD"); 40 | // BigQuery Dataset name 41 | public static final String BIG_QUERY_DATASET = System.getenv("BIG_QUERY_DATASET"); 42 | // BigQuery Table name 43 | public static final String BIG_QUERY_TABLE = System.getenv("BIG_QUERY_TABLE"); 44 | 45 | /* 46 | * API to accept request to Cloud Function 47 | * */ 48 | @Override 49 | public void accept(PubSubMessage message, Context context) { 50 | if (message.getData() == null) { 51 | logger.log(Level.WARNING, "No Project Id provided"); 52 | return; 53 | } 54 | // project Id received from Pub/Sub topic 55 | String projectId = 56 | new String( 57 | Base64.getDecoder().decode(message.getData().getBytes(StandardCharsets.UTF_8)), 58 | StandardCharsets.UTF_8); 59 | try { 60 | GCPProject gcpProject = new GCPProject(); 61 | gcpProject.setProjectId(projectId); 62 | gcpProject.setProjectName(ProjectName.of(projectId).toString()); 63 | GCPResourceClient gcpResourceClient = createGCPResourceClient(); 64 | 65 | // 1. Scan Allocation quota and load in main table in BigQuery 66 | getAllocationUsageQuotas(gcpResourceClient, gcpProject); 67 | // 2. Scan Rate quotas and load in main table 68 | getRateUsageQuotas(gcpResourceClient, gcpProject); 69 | } catch (Exception e) { 70 | logger.log(Level.SEVERE, " " + e.getMessage(), e); 71 | } 72 | } 73 | 74 | /* 75 | * API to get all Allocation quotas usage for this project 76 | * */ 77 | private static void getAllocationUsageQuotas( 78 | GCPResourceClient gcpResourceClient, GCPProject gcpProject) { 79 | try { 80 | scanQuota( 81 | gcpResourceClient, 82 | gcpProject, 83 | ScanProjectQuotasHelper.Quotas.ALLOCATION 84 | ); 85 | } catch (IOException e) { 86 | logger.log(Level.SEVERE, "Error fetching Allocation usage quotas " + e.getMessage(), e); 87 | } 88 | } 89 | 90 | /* 91 | * API to get all Rate quotas usage for this project 92 | * */ 93 | private static void getRateUsageQuotas( 94 | GCPResourceClient gcpResourceClient, GCPProject gcpProject) { 95 | try { 96 | scanQuota( 97 | gcpResourceClient, 98 | gcpProject, 99 | ScanProjectQuotasHelper.Quotas.RATE 100 | ); 101 | } catch (IOException e) { 102 | logger.log(Level.SEVERE, "Error fetching Rate usage quotas " + e.getMessage(), e); 103 | } 104 | } 105 | 106 | /* 107 | * API to get quotas from APIs and load in BigQuery 108 | * */ 109 | private static void scanQuota( 110 | GCPResourceClient gcpResourceClient, 111 | GCPProject gcpProject, 112 | ScanProjectQuotasHelper.Quotas q) 113 | throws IOException { 114 | List projectQuotas = getQuota(gcpProject, q); 115 | loadBigQueryTable(gcpResourceClient, projectQuotas); 116 | logger.log( 117 | Level.INFO, "Quotas loaded successfully for project Id:" + gcpProject.getProjectId()); 118 | } 119 | } 120 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/ScanProjectQuotasHelper.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package functions; 15 | 16 | import static functions.ScanProjectQuotas.THRESHOLD; 17 | 18 | import com.google.cloud.Timestamp; 19 | import com.google.cloud.bigquery.BigQuery; 20 | import com.google.cloud.bigquery.BigQueryError; 21 | import com.google.cloud.bigquery.BigQueryException; 22 | import com.google.cloud.bigquery.BigQueryOptions; 23 | import com.google.cloud.bigquery.InsertAllRequest; 24 | import com.google.cloud.bigquery.InsertAllResponse; 25 | import com.google.cloud.bigquery.TableId; 26 | import com.google.cloud.monitoring.v3.QueryServiceClient.QueryTimeSeriesPagedResponse; 27 | import com.google.cloud.monitoring.v3.QueryServiceClient; 28 | import com.google.monitoring.v3.QueryTimeSeriesRequest; 29 | import com.google.monitoring.v3.TimeSeriesData; 30 | import com.google.monitoring.v3.TimeSeriesDescriptor; 31 | import com.google.monitoring.v3.TimeSeriesData.PointData; 32 | 33 | import functions.eventpojos.GCPProject; 34 | import functions.eventpojos.GCPResourceClient; 35 | import functions.eventpojos.ProjectQuota; 36 | import java.io.IOException; 37 | import java.time.LocalDate; 38 | import java.time.LocalTime; 39 | import java.time.ZoneId; 40 | import java.time.ZonedDateTime; 41 | import java.time.format.DateTimeFormatter; 42 | import java.util.ArrayList; 43 | import java.util.HashMap; 44 | import java.util.List; 45 | import java.util.Map; 46 | import java.util.logging.Level; 47 | import java.util.logging.Logger; 48 | 49 | public class ScanProjectQuotasHelper { 50 | private static final Logger logger = Logger.getLogger(ScanProjectQuotasHelper.class.getName()); 51 | 52 | public static final String MQL_ALLOCATION_ALL = "fetch consumer_quota" + 53 | "| { current: metric serviceruntime.googleapis.com/quota/allocation/usage" + 54 | " | filter resource.project_id = '%1$s'" + 55 | " | align next_older(1w)" + 56 | " | every 1w" + 57 | " ; maximum: metric serviceruntime.googleapis.com/quota/allocation/usage" + 58 | " | filter resource.project_id = '%1$s'" + 59 | " | group_by 1w, [value_usage_max: max(value.usage)]" + 60 | " | every 1w" + 61 | " ; limit: metric 'serviceruntime.googleapis.com/quota/limit'" + 62 | " | filter resource.project_id = '%1$s'" + 63 | " | align next_older(1w)" + 64 | " | every 1w" + 65 | " }" + 66 | "| join" + 67 | "| value [current: val(0), maximum: val(1), limit: val(2)]"; 68 | 69 | // MQL to fetch rate quotas on a per minute basis 70 | public static final String MQL_RATE_QPM = "fetch consumer_quota" + 71 | "| { current: metric serviceruntime.googleapis.com/quota/rate/net_usage" + 72 | " | filter resource.project_id = '%1$s'" + 73 | " | every 1m" + 74 | " | within 1w" + 75 | " ; maximum: metric serviceruntime.googleapis.com/quota/rate/net_usage" + 76 | " | filter resource.project_id = '%1$s'" + 77 | " | group_by 1w, [value_usage_max: max(value.net_usage)]" + 78 | " | every 1m" + 79 | " | within 1w" + 80 | " ; limit: metric 'serviceruntime.googleapis.com/quota/limit'" + 81 | " | filter resource.project_id = '%1$s'" + 82 | " && !(metric.limit_name =~ '.*GoogleEgressBandwidth.*'" + 83 | " || metric.limit_name =~ '.*EGRESS-BANDWIDTH.*'" + 84 | " || metric.limit_name =~ '.*PerDay.*'" + 85 | " || metric.limit_name =~ '.*Qpd.*')" + 86 | " | align next_older(1m)" + 87 | " | every 1m" + 88 | " | within 1w" + 89 | " }" + 90 | "| join" + 91 | "| value [current: val(0), maximum: val(1), limit: val(2)]"; 92 | 93 | // MQL to fetch rate quotas on a per second basis 94 | public static final String MQL_RATE_QPS = "fetch consumer_quota" + 95 | "| { current:" + 96 | " metric serviceruntime.googleapis.com/quota/rate/net_usage" + 97 | " | filter" + 98 | " resource.project_id = '%1$s'" + 99 | " | every 1s" + 100 | " | within 1d" + 101 | " ; maximum:" + 102 | " metric serviceruntime.googleapis.com/quota/rate/net_usage" + 103 | " | filter" + 104 | " resource.project_id = '%1$s'" + 105 | " | group_by 1d, [value_usage_max: max(value.net_usage)]" + 106 | " | every 1s" + 107 | " | within 1d" + 108 | " ; limit:" + 109 | " metric serviceruntime.googleapis.com/quota/limit" + 110 | " | filter" + 111 | " resource.project_id = '%1$s'" + 112 | " && (metric.limit_name =~ '.*GoogleEgressBandwidth.*'" + 113 | " || metric.limit_name =~ '.*EGRESS-BANDWIDTH.*')" + 114 | " | align next_older(1m)" + 115 | " | every 1s" + 116 | " | within 1d" + 117 | " }" + 118 | "| join" + 119 | "| value [current: val(0), maximum: val(1), limit: val(2)]"; 120 | 121 | // MQL to get the usage aggregated on a daily basis 122 | // Based on filter provided at https://cloud.google.com/monitoring/alerts/using-quota-metrics#mql-rate-multiple-limits 123 | public static final String MQL_RATE_QPD = "fetch consumer_quota" + 124 | "| { daily:" + 125 | " metric serviceruntime.googleapis.com/quota/rate/net_usage" + 126 | " | filter" + 127 | " resource.project_id = '%1$s'" + 128 | " | group_by 1d, [value_usage_sum: sum(value.net_usage)]" + 129 | " | every 1d" + 130 | " | within 1w, d'%2$s'" + 131 | " ; limit:" + 132 | " metric serviceruntime.googleapis.com/quota/limit" + 133 | " | filter" + 134 | " resource.project_id = '%1$s'" + 135 | " && (metric.limit_name =~ '.*PerDay.*'" + 136 | " || metric.limit_name =~ '.*Qpd.*')" + 137 | " | align next_older(1d)" + 138 | " | every 1d" + 139 | " | within 1w, d'%2$s'" + 140 | " }" + 141 | "| join" + 142 | "| value [daily: val(0), limit: val(1)]"; 143 | 144 | enum Quotas { 145 | ALLOCATION, 146 | RATE 147 | } 148 | 149 | /* 150 | * API to create GCP Resource Client for BigQuery Tables 151 | * */ 152 | static GCPResourceClient createGCPResourceClient() { 153 | String datasetName = ScanProjectQuotas.BIG_QUERY_DATASET; 154 | String tableName = ScanProjectQuotas.BIG_QUERY_TABLE; 155 | // Initialize client that will be used to send requests. 156 | BigQuery bigquery = BigQueryOptions.getDefaultInstance().getService(); 157 | // Get table 158 | TableId tableId = TableId.of(datasetName, tableName); 159 | GCPResourceClient gcpResourceClient = new GCPResourceClient(); 160 | gcpResourceClient.setBigQuery(bigquery); 161 | gcpResourceClient.setTableId(tableId); 162 | return gcpResourceClient; 163 | } 164 | 165 | public static List getQuota(GCPProject gcpProject, Quotas quota) { 166 | List projectQuotas = new ArrayList<>(); 167 | 168 | try (QueryServiceClient queryServiceClient = QueryServiceClient.create()) { 169 | QueryTimeSeriesRequest request = 170 | QueryTimeSeriesRequest.newBuilder() 171 | .setName(gcpProject.getProjectName()) 172 | .setQuery(String.format(getMql(quota), gcpProject.getProjectId())) 173 | .build(); 174 | 175 | Timestamp ts = Timestamp.now(); 176 | QueryTimeSeriesPagedResponse response = queryServiceClient.queryTimeSeries(request); 177 | HashMap indexMap = buildIndexMap(response.getPage().getResponse().getTimeSeriesDescriptor()); 178 | for (TimeSeriesData data : response.iterateAll()) { 179 | projectQuotas.add(populateProjectQuota(data, null, ts, indexMap, quota)); 180 | } 181 | 182 | // Get the QPD and QPS quotas 183 | if(quota == Quotas.RATE) { 184 | projectQuotas.addAll(getPerDayQuota(gcpProject, ts)); 185 | 186 | projectQuotas.addAll(getPerSecondQuota(gcpProject, ts)); 187 | } 188 | 189 | } catch (IOException e) { 190 | logger.log( 191 | Level.SEVERE, 192 | "Error fetching timeseries data for project: " 193 | + gcpProject.getProjectName() 194 | + e.getMessage(), 195 | e); 196 | } 197 | 198 | return projectQuotas; 199 | } 200 | 201 | private static List getPerSecondQuota(GCPProject gcpProject, Timestamp ts) { 202 | HashMap projectQuotas = new HashMap<>(); 203 | 204 | try (QueryServiceClient queryServiceClient = QueryServiceClient.create()) { 205 | String mql = 206 | String.format(MQL_RATE_QPS, 207 | gcpProject.getProjectId() 208 | ); 209 | 210 | QueryTimeSeriesRequest request = 211 | QueryTimeSeriesRequest.newBuilder() 212 | .setName(gcpProject.getProjectName()) 213 | .setQuery(mql) 214 | .build(); 215 | 216 | QueryTimeSeriesPagedResponse response = queryServiceClient.queryTimeSeries(request); 217 | HashMap indexMap = buildIndexMap(response.getPage().getResponse().getTimeSeriesDescriptor()); 218 | for (TimeSeriesData data : response.iterateAll()) { 219 | String key = data.getLabelValues(indexMap.get("metric.limit_name")).getStringValue() + data.getLabelValues(indexMap.get("resource.location")).getStringValue(); 220 | projectQuotas.put(key, populateProjectQuota(data, null, ts, indexMap, Quotas.RATE)); 221 | } 222 | } catch (IOException e) { 223 | logger.log( 224 | Level.SEVERE, 225 | "Error fetching timeseries data for project: " 226 | + gcpProject.getProjectName() 227 | + e.getMessage(), 228 | e); 229 | } 230 | 231 | return new ArrayList(projectQuotas.values()); 232 | } 233 | 234 | private static List getPerDayQuota(GCPProject gcpProject, Timestamp ts) { 235 | List projectQuotas = new ArrayList<>(); 236 | HashMap values = new HashMap<>() {{ 237 | put("current", (long) 0); 238 | put("max", (long) 0); 239 | }}; 240 | 241 | try (QueryServiceClient queryServiceClient = QueryServiceClient.create()) { 242 | // This needs to align to the day boundaries as closely as possible to that we get an 243 | // accurate view into same window as the quota system. 244 | LocalDate today = LocalDate.now(); 245 | ZonedDateTime endOfDay = ZonedDateTime.of(today, LocalTime.MAX, ZoneId.of("America/Los_Angeles")); 246 | 247 | String mql = 248 | String.format(MQL_RATE_QPD, 249 | gcpProject.getProjectId(), 250 | endOfDay.format(DateTimeFormatter.ofPattern("yyyy/MM/dd HH:mm:ss")) 251 | ); 252 | 253 | QueryTimeSeriesRequest request = 254 | QueryTimeSeriesRequest.newBuilder() 255 | .setName(gcpProject.getProjectName()) 256 | .setQuery(mql) 257 | .build(); 258 | 259 | QueryTimeSeriesPagedResponse response = queryServiceClient.queryTimeSeries(request); 260 | HashMap indexMap = buildIndexMap(response.getPage().getResponse().getTimeSeriesDescriptor()); 261 | for (TimeSeriesData data : response.iterateAll()) { 262 | List stuffs = data.getPointDataList(); 263 | 264 | for (PointData pointData : stuffs) { 265 | logger.info(String.format("Metric: %s, Current: %d, Max %d, Value %d%n", 266 | data.getLabelValues(indexMap.get("metric.quota_metric")).getStringValue(), 267 | values.get("current"), values.get("max"), 268 | pointData.getValues(0).getInt64Value())); 269 | 270 | // Cloud Monitoring returns UTC timestamps so we need to use end of day UTC to match correctly. 271 | if (pointData.getTimeInterval().getEndTime().getSeconds() == ZonedDateTime.of(today, LocalTime.MAX, ZoneId.of("UTC")).toEpochSecond()) { 272 | values.replace("current", pointData.getValues(0).getInt64Value()); 273 | } 274 | 275 | if (pointData.getValues(0).getInt64Value() > values.get("max")) { 276 | values.replace("max", pointData.getValues(0).getInt64Value()); 277 | } 278 | } 279 | 280 | projectQuotas.add(populateProjectQuota(data, values, ts, indexMap, Quotas.RATE)); 281 | } 282 | } catch (IOException e) { 283 | logger.log( 284 | Level.SEVERE, 285 | "Error fetching timeseries data for project: " 286 | + gcpProject.getProjectName() 287 | + e.getMessage(), 288 | e); 289 | } 290 | 291 | return projectQuotas; 292 | } 293 | 294 | private static HashMap buildIndexMap(TimeSeriesDescriptor labels) { 295 | HashMap indexMap = new HashMap<>(); 296 | 297 | for(int i=0; i aggregatedData, Timestamp ts, HashMap indexMap, Quotas q) { 310 | ProjectQuota projectQuota = new ProjectQuota(); 311 | 312 | projectQuota.setProjectId(data.getLabelValues(indexMap.get("resource.project_id")).getStringValue()); 313 | projectQuota.setTimestamp(ts.toString()); 314 | projectQuota.setRegion(data.getLabelValues(indexMap.get("resource.location")).getStringValue()); 315 | projectQuota.setMetric(data.getLabelValues(indexMap.get("metric.quota_metric")).getStringValue()); 316 | 317 | if(q == Quotas.RATE) { 318 | projectQuota.setApiMethod(data.getLabelValues(indexMap.get("metric.method")).getStringValue()); 319 | } 320 | projectQuota.setLimitName(data.getLabelValues(indexMap.get("metric.limit_name")).getStringValue()); 321 | projectQuota.setQuotaType(q.toString()); 322 | 323 | if(aggregatedData == null) { 324 | projectQuota.setCurrentUsage(data.getPointData(0).getValues(indexMap.get("current")).getInt64Value()); 325 | projectQuota.setMaxUsage(data.getPointData(0).getValues(indexMap.get("maximum")).getInt64Value()); 326 | } else { 327 | projectQuota.setCurrentUsage(aggregatedData.get("current")); 328 | projectQuota.setMaxUsage(aggregatedData.get("max")); 329 | } 330 | 331 | projectQuota.setQuotaLimit(data.getPointData(0).getValues(indexMap.get("limit")).getInt64Value()); 332 | projectQuota.setThreshold(Integer.valueOf(THRESHOLD)); 333 | 334 | return projectQuota; 335 | } 336 | 337 | private static String getMql(Quotas q) { 338 | String mql; 339 | 340 | switch (q) { 341 | case ALLOCATION: 342 | mql = MQL_ALLOCATION_ALL; 343 | break; 344 | case RATE: 345 | mql = MQL_RATE_QPM; 346 | break; 347 | default: 348 | mql = ""; 349 | } 350 | 351 | return mql; 352 | } 353 | 354 | /* 355 | * API to load data into BigQuery 356 | * */ 357 | static void loadBigQueryTable( 358 | GCPResourceClient gcpResourceClient, 359 | List projectQuotas) { 360 | for (ProjectQuota pq : projectQuotas) { 361 | Map row = createBQRow(pq); 362 | tableInsertRows(gcpResourceClient, row); 363 | } 364 | } 365 | 366 | /* 367 | * API to build BigQuery row content from ProjectQuota object 368 | * */ 369 | public static Map createBQRow(ProjectQuota projectQuota) { 370 | Map rowContent = new HashMap<>(); 371 | 372 | rowContent.put("project_id", projectQuota.getProjectId()); 373 | rowContent.put("added_at", projectQuota.getTimestamp()); 374 | rowContent.put("region", projectQuota.getRegion()); 375 | rowContent.put("quota_metric", projectQuota.getMetric()); 376 | if(projectQuota.getQuotaType() == Quotas.RATE.toString()) { 377 | rowContent.put("api_method", projectQuota.getApiMethod()); 378 | } 379 | rowContent.put("limit_name", projectQuota.getLimitName()); 380 | rowContent.put("quota_type", projectQuota.getQuotaType()); 381 | rowContent.put("current_usage", projectQuota.getCurrentUsage()); 382 | rowContent.put("max_usage", projectQuota.getMaxUsage()); 383 | rowContent.put("quota_limit", projectQuota.getQuotaLimit()); 384 | rowContent.put("threshold", projectQuota.getThreshold()); 385 | 386 | return rowContent; 387 | } 388 | 389 | /* 390 | * API to insert row in table 391 | * */ 392 | public static void tableInsertRows( 393 | GCPResourceClient gcpResourceClient, Map rowContent) { 394 | 395 | try { 396 | // Initialize client that will be used to send requests. This client only needs to be created 397 | // once, and can be reused for multiple requests. 398 | BigQuery bigquery = gcpResourceClient.getBigQuery(); 399 | // Get table 400 | TableId tableId = gcpResourceClient.getTableId(); 401 | // Inserts rowContent into datasetName:tableId. 402 | InsertAllResponse response = 403 | bigquery.insertAll(InsertAllRequest.newBuilder(tableId).addRow(rowContent).build()); 404 | 405 | if (response.hasErrors()) { 406 | // If any of the insertions failed, this lets you inspect the errors 407 | for (Map.Entry> entry : response.getInsertErrors().entrySet()) { 408 | logger.log(Level.SEVERE, "Bigquery row insert response error: " + entry.getValue()); 409 | } 410 | } 411 | } catch (BigQueryException e) { 412 | logger.log(Level.SEVERE, "Insert operation not performed: " + e.toString()); 413 | } 414 | } 415 | } 416 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/eventpojos/GCPProject.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package functions.eventpojos; 15 | 16 | public class GCPProject { 17 | private String projectId; 18 | private String projectName; 19 | 20 | public String getProjectId() { 21 | return projectId; 22 | } 23 | 24 | public void setProjectId(String projectId) { 25 | this.projectId = projectId; 26 | } 27 | 28 | public String getProjectName() { 29 | return projectName; 30 | } 31 | 32 | public void setProjectName(String projectName) { 33 | this.projectName = projectName; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/eventpojos/GCPResourceClient.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package functions.eventpojos; 18 | 19 | import com.google.cloud.bigquery.BigQuery; 20 | import com.google.cloud.bigquery.TableId; 21 | 22 | /* 23 | * Class to store references of GCP client resources. 24 | * Stores references for BigQuery, Region and Network client. 25 | * Reference for BigQuery table also. 26 | * */ 27 | public class GCPResourceClient { 28 | private BigQuery bigQuery; 29 | private TableId tableId; 30 | 31 | public BigQuery getBigQuery() { 32 | return bigQuery; 33 | } 34 | 35 | public void setBigQuery(BigQuery bigQuery) { 36 | this.bigQuery = bigQuery; 37 | } 38 | 39 | public TableId getTableId() { 40 | return tableId; 41 | } 42 | 43 | public void setTableId(TableId tableId) { 44 | this.tableId = tableId; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/eventpojos/ProjectQuota.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package functions.eventpojos; 18 | 19 | /* 20 | * POJO for ProjectQuota 21 | * */ 22 | public class ProjectQuota { 23 | private String projectId; 24 | private String timestamp; 25 | private String region; 26 | private String metric; 27 | private String apiMethod; 28 | private String limitName; 29 | private String quotaType; 30 | private Long currentUsage; 31 | private Long maxUsage; 32 | private Long quotaLimit; 33 | private Integer threshold; 34 | 35 | public String getProjectId() { 36 | return projectId; 37 | } 38 | 39 | public void setProjectId(String projectId) { 40 | this.projectId = projectId; 41 | } 42 | 43 | public String getTimestamp() { 44 | return timestamp; 45 | } 46 | 47 | public void setTimestamp(String timestamp) { 48 | this.timestamp = timestamp; 49 | } 50 | 51 | public String getRegion() { 52 | return region; 53 | } 54 | 55 | public void setRegion(String region) { 56 | this.region = region; 57 | } 58 | 59 | public String getMetric() { 60 | return metric; 61 | } 62 | 63 | public void setMetric(String metric) { 64 | this.metric = metric; 65 | } 66 | 67 | public String getApiMethod() { 68 | return apiMethod; 69 | } 70 | 71 | public void setApiMethod(String apiMethod) { 72 | this.apiMethod = apiMethod; 73 | } 74 | 75 | public String getLimitName() { 76 | return limitName; 77 | } 78 | 79 | public void setLimitName(String limitName) { 80 | this.limitName = limitName; 81 | } 82 | 83 | public String getQuotaType() { 84 | return quotaType; 85 | } 86 | 87 | public void setQuotaType(String quotaType) { 88 | this.quotaType = quotaType; 89 | } 90 | 91 | public Long getCurrentUsage() { 92 | return currentUsage; 93 | } 94 | 95 | public void setCurrentUsage(Long currentUsage) { 96 | this.currentUsage = currentUsage; 97 | } 98 | 99 | public Long getMaxUsage() { 100 | return maxUsage; 101 | } 102 | 103 | public void setMaxUsage(Long maxUsage) { 104 | this.maxUsage = maxUsage; 105 | } 106 | 107 | public Long getQuotaLimit() { 108 | return quotaLimit; 109 | } 110 | 111 | public void setQuotaLimit(Long quotaLimit) { 112 | this.quotaLimit = quotaLimit; 113 | } 114 | 115 | public Integer getThreshold() { 116 | return threshold; 117 | } 118 | 119 | public void setThreshold(Integer threshold) { 120 | this.threshold = threshold; 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/eventpojos/PubSubMessage.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | package functions.eventpojos; 18 | 19 | import java.util.Map; 20 | 21 | public class PubSubMessage { 22 | // Cloud Functions uses GSON to populate this object. 23 | // Field types/names are specified by Cloud Functions 24 | // Changing them may break your code! 25 | private String data; 26 | private Map attributes; 27 | private String messageId; 28 | private String publishTime; 29 | 30 | public String getData() { 31 | return data; 32 | } 33 | 34 | public void setData(String data) { 35 | this.data = data; 36 | } 37 | 38 | public Map getAttributes() { 39 | return attributes; 40 | } 41 | 42 | public void setAttributes(Map attributes) { 43 | this.attributes = attributes; 44 | } 45 | 46 | public String getMessageId() { 47 | return messageId; 48 | } 49 | 50 | public void setMessageId(String messageId) { 51 | this.messageId = messageId; 52 | } 53 | 54 | public String getPublishTime() { 55 | return publishTime; 56 | } 57 | 58 | public void setPublishTime(String publishTime) { 59 | this.publishTime = publishTime; 60 | } 61 | } 62 | // [END functions_helloworld_pubsub_message] 63 | -------------------------------------------------------------------------------- /quota-scan/src/main/java/functions/eventpojos/TimeSeriesQuery.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2023 Google LLC 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | 14 | package functions.eventpojos; 15 | 16 | public class TimeSeriesQuery { 17 | private String allocationQuotaUsageFilter; 18 | private String rateQuotaUsageFilter; 19 | private String quotaLimitFilter; 20 | 21 | public String getAllocationQuotaUsageFilter() { 22 | return allocationQuotaUsageFilter; 23 | } 24 | 25 | public void setAllocationQuotaUsageFilter(String allocationQuotaUsageFilter) { 26 | this.allocationQuotaUsageFilter = allocationQuotaUsageFilter; 27 | } 28 | 29 | public String getRateQuotaUsageFilter() { 30 | return rateQuotaUsageFilter; 31 | } 32 | 33 | public void setRateQuotaUsageFilter(String rateQuotaUsageFilter) { 34 | this.rateQuotaUsageFilter = rateQuotaUsageFilter; 35 | } 36 | 37 | public String getQuotaLimitFilter() { 38 | return quotaLimitFilter; 39 | } 40 | 41 | public void setQuotaLimitFilter(String quotaLimitFilter) { 42 | this.quotaLimitFilter = quotaLimitFilter; 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /quota-scan/src/main/resources/config.properties: -------------------------------------------------------------------------------- 1 | allocation.quota.usage.filter=metric.type="serviceruntime.googleapis.com/quota/allocation/usage" resource.type="consumer_quota" 2 | rate.quota.usage.filter=metric.type="serviceruntime.googleapis.com/quota/rate/net_usage" resource.type="consumer_quota" 3 | quota.limit.filter=metric.type="serviceruntime.googleapis.com/quota/limit" resource.type="consumer_quota" -------------------------------------------------------------------------------- /terraform/example/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | terraform { 18 | required_providers { 19 | google = { 20 | version = ">3.5.0" 21 | } 22 | } 23 | } 24 | 25 | provider "google" { 26 | project = var.project_id 27 | region = var.region 28 | } 29 | 30 | module "qms" { 31 | source = "../modules/qms" 32 | 33 | project_id = var.project_id 34 | region = var.region 35 | service_account_email = var.service_account_email 36 | folders = var.folders 37 | organizations = var.organizations 38 | alert_log_bucket_name = var.alert_log_bucket_name 39 | notification_email_address = var.notification_email_address 40 | threshold = var.threshold 41 | } -------------------------------------------------------------------------------- /terraform/example/terraform.tfvars: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | # Update values 18 | project_id = "" 19 | region = "" 20 | service_account_email = "" 21 | folders = "[]" 22 | organizations = "[]" 23 | alert_log_bucket_name = "" 24 | notification_email_address = "" 25 | threshold = "" 26 | -------------------------------------------------------------------------------- /terraform/example/variables.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | variable "project_id" { 18 | description = "Value of the Project Id to deploy the solution" 19 | type = string 20 | } 21 | 22 | variable "region" { 23 | description = "Value of the region to deploy the solution. Use the same region as used for App Engine" 24 | type = string 25 | } 26 | 27 | variable "service_account_email" { 28 | description = "Value of the Service Account" 29 | type = string 30 | } 31 | 32 | variable "folders" { 33 | description = "Value of the list of folders to be scanned for quota" 34 | type = string 35 | } 36 | 37 | variable "organizations" { 38 | description = "Value of the list of organization Ids to scanned for quota" 39 | type = string 40 | } 41 | 42 | variable "threshold" { 43 | description = "Value of threshold for all metrics. If any metric usage >= the threshold, notification will be created" 44 | type = string 45 | } 46 | 47 | variable "notification_email_address" { 48 | description = "Email Address to receive email notifications" 49 | type = string 50 | } 51 | 52 | variable "alert_log_bucket_name" { 53 | description = "Bucket Name for alert Log Sink (must be globally unique)" 54 | type = string 55 | } 56 | -------------------------------------------------------------------------------- /terraform/modules/qms/main.tf: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright 2022 Google LLC 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | terraform { 18 | provider_meta "google" { 19 | module_name = "cloud-solutions/quota-monitoring-solution-deploy-v5.1" #x-release-please-minor 20 | } 21 | } 22 | 23 | locals { 24 | expanded_region = var.region == "us-central" || var.region == "europe-west" ? "${var.region}1" : var.region 25 | use_github_release = var.qms_version != "main" ? true : false 26 | } 27 | 28 | # Enable Cloud Resource Manager API 29 | module "project-service-cloudresourcemanager" { 30 | source = "terraform-google-modules/project-factory/google//modules/project_services" 31 | version = "4.0.0" 32 | 33 | project_id = var.project_id 34 | 35 | activate_apis = [ 36 | "cloudresourcemanager.googleapis.com" 37 | ] 38 | } 39 | 40 | # Enable APIs 41 | module "project-services" { 42 | source = "terraform-google-modules/project-factory/google//modules/project_services" 43 | version = "4.0.0" 44 | 45 | project_id = var.project_id 46 | 47 | activate_apis = [ 48 | "compute.googleapis.com", 49 | "iam.googleapis.com", 50 | "monitoring.googleapis.com", 51 | "storage.googleapis.com", 52 | "storage-api.googleapis.com", 53 | "bigquery.googleapis.com", 54 | "pubsub.googleapis.com", 55 | "appengine.googleapis.com", 56 | "cloudscheduler.googleapis.com", 57 | "cloudfunctions.googleapis.com", 58 | "cloudbuild.googleapis.com", 59 | "bigquerydatatransfer.googleapis.com" 60 | ] 61 | depends_on = [module.project-service-cloudresourcemanager] 62 | } 63 | 64 | # Create Pub/Sub topic to list projects in the parent node 65 | resource "google_pubsub_topic" "topic_alert_project_id" { 66 | name = var.topic_alert_project_id 67 | depends_on = [module.project-services] 68 | } 69 | 70 | # Create Pub/Sub topic to send notification 71 | resource "google_pubsub_topic" "topic_alert_notification" { 72 | name = var.topic_alert_notification 73 | depends_on = [module.project-services] 74 | } 75 | 76 | # Cloud scheduler job to invoke list projects cloud function 77 | resource "google_cloud_scheduler_job" "job" { 78 | name = var.scheduler_cron_job_name 79 | description = var.scheduler_cron_job_description 80 | schedule = var.scheduler_cron_job_frequency 81 | time_zone = var.scheduler_cron_job_timezone 82 | attempt_deadline = var.scheduler_cron_job_deadline 83 | region = local.expanded_region 84 | depends_on = [module.project-services] 85 | retry_config { 86 | retry_count = 1 87 | } 88 | 89 | http_target { 90 | http_method = "POST" 91 | uri = google_cloudfunctions_function.function-listProjects.https_trigger_url 92 | body = base64encode("{\"organizations\":\"${var.organizations}\",\"threshold\":\"${var.threshold}\",\"projectId\":\"${var.project_id}\"}") 93 | 94 | oidc_token { 95 | service_account_email = var.service_account_email 96 | } 97 | } 98 | } 99 | 100 | # Cloud scheduler job to invoke config app alert cloud function 101 | resource "google_cloud_scheduler_job" "app_alert_configure_job" { 102 | name = var.scheduler_app_alert_config_job_name 103 | description = var.scheduler_app_alert_job_description 104 | schedule = var.scheduler_app_alert_job_frequency 105 | time_zone = var.scheduler_cron_job_timezone 106 | attempt_deadline = var.scheduler_cron_job_deadline 107 | region = local.expanded_region 108 | depends_on = [module.project-services] 109 | retry_config { 110 | retry_count = 1 111 | } 112 | 113 | http_target { 114 | http_method = "POST" 115 | uri = google_cloudfunctions_function.function-configureAppAlert.https_trigger_url 116 | 117 | oidc_token { 118 | service_account_email = var.service_account_email 119 | } 120 | } 121 | } 122 | 123 | resource "google_storage_bucket" "bucket_gcf_source" { 124 | name = "${var.project_id}-gcf-source" 125 | storage_class = "REGIONAL" 126 | location = local.expanded_region 127 | force_destroy = "true" 128 | uniform_bucket_level_access = "true" 129 | } 130 | 131 | data "archive_file" "local_source_code_zip" { 132 | count = local.use_github_release ? 0 : 1 133 | 134 | type = "zip" 135 | source_dir = abspath("${path.module}/../../../quota-scan") 136 | output_path = var.source_code_zip 137 | } 138 | 139 | resource "null_resource" "source_code_zip" { 140 | count = local.use_github_release ? 1 : 0 141 | 142 | triggers = { 143 | on_version_change = var.qms_version 144 | } 145 | 146 | provisioner "local-exec" { 147 | command = "curl -Lo ${var.source_code_zip} ${var.source_code_base_url}/${var.qms_version}/${var.source_code_zip}" 148 | } 149 | } 150 | 151 | resource "google_storage_bucket_object" "source_code_object" { 152 | name = "${var.qms_version}-${var.source_code_zip}" 153 | bucket = google_storage_bucket.bucket_gcf_source.name 154 | source = var.source_code_zip 155 | 156 | depends_on = [ 157 | null_resource.source_code_zip, 158 | data.archive_file.local_source_code_zip 159 | ] 160 | } 161 | 162 | # cloud function to list projects 163 | resource "google_cloudfunctions_function" "function-listProjects" { 164 | name = var.cloud_function_list_project 165 | description = var.cloud_function_list_project_desc 166 | runtime = "java11" 167 | region = local.expanded_region 168 | 169 | available_memory_mb = var.cloud_function_list_project_memory 170 | source_archive_bucket = google_storage_bucket.bucket_gcf_source.name 171 | source_archive_object = google_storage_bucket_object.source_code_object.name 172 | trigger_http = true 173 | entry_point = "functions.ListProjects" 174 | service_account_email = var.service_account_email 175 | timeout = var.cloud_function_list_project_timeout 176 | depends_on = [module.project-services] 177 | 178 | environment_variables = { 179 | PUBLISH_TOPIC = google_pubsub_topic.topic_alert_project_id.name 180 | HOME_PROJECT = var.project_id 181 | } 182 | } 183 | 184 | # IAM entry for all users to invoke the function 185 | resource "google_cloudfunctions_function_iam_member" "invoker-listProjects" { 186 | project = google_cloudfunctions_function.function-listProjects.project 187 | region = google_cloudfunctions_function.function-listProjects.region 188 | cloud_function = google_cloudfunctions_function.function-listProjects.name 189 | depends_on = [module.project-services] 190 | 191 | role = "roles/cloudfunctions.invoker" 192 | member = "serviceAccount:${var.service_account_email}" 193 | } 194 | 195 | # Second cloud function to scan project 196 | resource "google_cloudfunctions_function" "function-scanProject" { 197 | name = var.cloud_function_scan_project 198 | description = var.cloud_function_scan_project_desc 199 | runtime = "java11" 200 | region = local.expanded_region 201 | 202 | available_memory_mb = var.cloud_function_scan_project_memory 203 | source_archive_bucket = google_storage_bucket.bucket_gcf_source.name 204 | source_archive_object = google_storage_bucket_object.source_code_object.name 205 | entry_point = "functions.ScanProjectQuotas" 206 | service_account_email = var.service_account_email 207 | timeout = var.cloud_function_scan_project_timeout 208 | depends_on = [module.project-services] 209 | 210 | event_trigger { 211 | event_type = "google.pubsub.topic.publish" 212 | resource = var.topic_alert_project_id 213 | } 214 | 215 | environment_variables = { 216 | NOTIFICATION_TOPIC = google_pubsub_topic.topic_alert_notification.name 217 | THRESHOLD = var.threshold 218 | BIG_QUERY_DATASET = var.big_query_dataset_id 219 | BIG_QUERY_TABLE = var.big_query_table_id 220 | } 221 | } 222 | 223 | # IAM entry for all users to invoke the function 224 | resource "google_cloudfunctions_function_iam_member" "invoker-scanProject" { 225 | project = google_cloudfunctions_function.function-scanProject.project 226 | region = google_cloudfunctions_function.function-scanProject.region 227 | cloud_function = google_cloudfunctions_function.function-scanProject.name 228 | depends_on = [module.project-services] 229 | 230 | role = "roles/cloudfunctions.invoker" 231 | member = "serviceAccount:${var.service_account_email}" 232 | } 233 | 234 | data "archive_file" "local_source_code_notification_zip" { 235 | count = local.use_github_release ? 0 : 1 236 | 237 | type = "zip" 238 | source_dir = abspath("${path.module}/../../../quota-notification") 239 | output_path = "./${var.source_code_notification_zip}" 240 | } 241 | 242 | resource "null_resource" "source_code_notification_zip" { 243 | count = local.use_github_release ? 1 : 0 244 | 245 | triggers = { 246 | on_version_change = var.qms_version 247 | } 248 | 249 | provisioner "local-exec" { 250 | command = "curl -Lo ${var.source_code_notification_zip} ${var.source_code_base_url}/${var.qms_version}/${var.source_code_notification_zip}" 251 | } 252 | } 253 | 254 | resource "google_storage_bucket_object" "source_code_notification_object" { 255 | name = "${var.qms_version}-${var.source_code_notification_zip}" 256 | bucket = google_storage_bucket.bucket_gcf_source.name 257 | source = var.source_code_notification_zip 258 | 259 | depends_on = [ 260 | null_resource.source_code_notification_zip, 261 | data.archive_file.local_source_code_notification_zip 262 | ] 263 | } 264 | 265 | # Third cloud function to send notification 266 | resource "google_cloudfunctions_function" "function-notificationProject" { 267 | name = var.cloud_function_notification_project 268 | description = var.cloud_function_notification_project_desc 269 | runtime = "java11" 270 | region = local.expanded_region 271 | 272 | available_memory_mb = var.cloud_function_notification_project_memory 273 | source_archive_bucket = google_storage_bucket.bucket_gcf_source.name 274 | source_archive_object = google_storage_bucket_object.source_code_notification_object.name 275 | entry_point = "functions.SendNotification" 276 | service_account_email = var.service_account_email 277 | timeout = var.cloud_function_notification_project_timeout 278 | depends_on = [module.project-services] 279 | 280 | event_trigger { 281 | event_type = "google.pubsub.topic.publish" 282 | resource = var.topic_alert_notification 283 | } 284 | 285 | environment_variables = { 286 | HOME_PROJECT = var.project_id 287 | ALERT_DATASET = var.big_query_alert_dataset_id 288 | ALERT_TABLE = var.big_query_alert_table_id 289 | APP_ALERT_DATASET = var.big_query_alert_dataset_id 290 | APP_ALERT_TABLE = var.big_query_app_alert_table_id 291 | } 292 | } 293 | 294 | # IAM entry for all users to invoke the function 295 | resource "google_cloudfunctions_function_iam_member" "invoker-notificationProject" { 296 | project = google_cloudfunctions_function.function-notificationProject.project 297 | region = google_cloudfunctions_function.function-notificationProject.region 298 | cloud_function = google_cloudfunctions_function.function-notificationProject.name 299 | depends_on = [module.project-services] 300 | 301 | role = "roles/cloudfunctions.invoker" 302 | member = "serviceAccount:${var.service_account_email}" 303 | } 304 | 305 | # cloud function to configure app alerting 306 | resource "google_cloudfunctions_function" "function-configureAppAlert" { 307 | name = var.cloud_function_config_app_alert 308 | description = var.cloud_function_config_app_alert_desc 309 | runtime = "java11" 310 | region = local.expanded_region 311 | 312 | available_memory_mb = var.cloud_function_config_app_alert_memory 313 | source_archive_bucket = google_storage_bucket.bucket_gcf_source.name 314 | source_archive_object = google_storage_bucket_object.source_code_notification_object.name 315 | trigger_http = true 316 | entry_point = "functions.ConfigureAppAlert" 317 | service_account_email = var.service_account_email 318 | timeout = var.cloud_function_config_app_alert_timeout 319 | depends_on = [module.project-services] 320 | 321 | environment_variables = { 322 | HOME_PROJECT = var.project_id 323 | APP_ALERT_DATASET = var.big_query_alert_dataset_id 324 | APP_ALERT_TABLE = var.big_query_app_alert_table_id 325 | CSV_SOURCE_URI = "gs://${google_storage_bucket.bucket_gcf_source.name}/${var.app_alert_csv_file_name}" 326 | } 327 | } 328 | 329 | # IAM entry for all users to invoke the function 330 | resource "google_cloudfunctions_function_iam_member" "invoker-configureAppAlert" { 331 | project = google_cloudfunctions_function.function-configureAppAlert.project 332 | region = google_cloudfunctions_function.function-configureAppAlert.region 333 | cloud_function = google_cloudfunctions_function.function-configureAppAlert.name 334 | depends_on = [module.project-services] 335 | 336 | role = "roles/cloudfunctions.invoker" 337 | member = "serviceAccount:${var.service_account_email}" 338 | } 339 | 340 | # BigQuery Dataset 341 | resource "google_bigquery_dataset" "dataset" { 342 | dataset_id = var.big_query_dataset_id 343 | friendly_name = var.big_query_dataset_id 344 | description = var.big_query_dataset_desc 345 | location = var.big_query_dataset_location 346 | default_partition_expiration_ms = var.big_query_dataset_default_partition_expiration_ms 347 | depends_on = [module.project-services] 348 | } 349 | 350 | # BigQuery Table 351 | resource "google_bigquery_table" "default" { 352 | dataset_id = google_bigquery_dataset.dataset.dataset_id 353 | table_id = var.big_query_table_id 354 | 355 | time_partitioning { 356 | type = var.big_query_table_partition 357 | } 358 | 359 | labels = { 360 | env = "default" 361 | } 362 | 363 | schema = <