├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bigquery_schemas ├── bigquery_schema.json ├── bigquery_schema_params_table.json └── bigquery_schema_stats_table.json ├── get_timeseries ├── .gitignore ├── app.yaml ├── config.py ├── main.py ├── main_test.py ├── requirements.txt └── sample.json ├── list_metrics ├── .gitignore ├── app.yaml ├── config.py ├── main.py ├── main_test.py └── requirements.txt ├── list_projects ├── config.json ├── index.js ├── package.json └── test │ ├── index.test.js │ ├── test_send_empty_pubsub_msg.sh │ ├── test_send_incorrect_token_pubsub_msg.sh │ └── test_send_pubsub_msg.sh ├── sample_messages ├── bigquery_query_1.sql ├── bigquery_query_2.sql ├── bigquery_query_3.sql ├── metric_descriptor.json ├── timeseries_response_align.json └── timeseries_response_align_and_reduce.json ├── test ├── .gitignore ├── config.py ├── end_to_end_test_run.py └── requirements.txt └── write_metrics ├── .gitignore ├── app.yaml ├── config.py ├── main.py ├── main_test.py ├── requirements.txt └── sample.json /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to become a contributor and submit your own code 2 | 3 | ## Contributor License Agreements 4 | 5 | We'd love to accept your sample apps and patches! Before we can take them, we 6 | have to jump a couple of legal hurdles. 7 | 8 | Please fill out either the individual or corporate Contributor License Agreement 9 | (CLA). 10 | 11 | * If you are an individual writing original source code and you're sure you 12 | own the intellectual property, then you'll need to sign an [individual CLA] 13 | (https://developers.google.com/open-source/cla/individual). 14 | * If you work for a company that wants to allow you to contribute your work, 15 | then you'll need to sign a [corporate CLA] 16 | (https://developers.google.com/open-source/cla/corporate). 17 | 18 | Follow either of the two links above to access the appropriate CLA and 19 | instructions for how to sign and return it. Once we receive it, we'll be able to 20 | accept your pull requests. 21 | 22 | ## Contributing A Patch 23 | 24 | 1. Submit an issue describing your proposed change to the repo in question. 25 | 1. The repo owner will respond to your issue promptly. 26 | 1. If your proposed change is accepted, and you haven't already done so, sign a 27 | Contributor License Agreement (see details above). 28 | 1. Fork the desired repo, develop and test your code changes. 29 | 1. Ensure that your code adheres to the existing style in the sample to which 30 | you are contributing. Refer to the 31 | [Google Cloud Platform Samples Style Guide] 32 | (https://github.com/GoogleCloudPlatform/Template/wiki/style.html) for the 33 | recommended coding standards for this organization. 34 | 1. Ensure that your code has an appropriate set of unit tests which all pass. 35 | 1. Submit a pull request. 36 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Reference Implementation for Stackdriver Metric Export 2 | This implementation is meant to demonstrate how to use the `projects.timeseries.list` API, PubSub and BigQuery to store aggregated Cloud Monitoring 3 | metrics for long-term analysis in BigQuery. 4 | 5 | # Deployment Instructions 6 | 1. Clone the [source repo]() 7 | ```sh 8 | git clone https://github.com/GoogleCloudPlatform/stackdriver-metrics-export 9 | cd stackdriver-metrics-export 10 | ``` 11 | 2. Enable the APIs 12 | ```sh 13 | gcloud services enable compute.googleapis.com \ 14 | cloudscheduler.googleapis.com \ 15 | cloudfunctions.googleapis.com \ 16 | cloudresourcemanager.googleapis.com 17 | ``` 18 | 3. Set your PROJECT_ID variable, by replacing [YOUR_PROJECT_ID] with your GCP project id 19 | ```sh 20 | export PROJECT_ID=[YOUR_PROJECT_ID] 21 | ``` 22 | 23 | 4. Create the BigQuery tables 24 | Create a Dataset and then a table using the schema JSON files 25 | ```sh 26 | bq mk metric_export 27 | bq mk --table --time_partitioning_type=DAY metric_export.sd_metrics_export_fin ./bigquery_schemas/bigquery_schema.json 28 | bq mk --table --time_partitioning_type=DAY metric_export.sd_metrics_stats ./bigquery_schemas/bigquery_schema_stats_table.json 29 | bq mk --table metric_export.sd_metrics_params ./bigquery_schemas/bigquery_schema_params_table.json 30 | ``` 31 | 32 | 5. Replace the JSON token in the config.py files 33 | Generate a new token and then replace that token in the each of config.py files. Use this same token in the Cloud Scheduler. 34 | ```sh 35 | TOKEN=$(python -c "import uuid; msg = uuid.uuid4(); print(msg)") 36 | LIST_PROJECTS_TOKEN=$(python -c "import uuid; msg = uuid.uuid4(); print (msg)") 37 | sed -i s/16b2ecfb-7734-48b9-817d-4ac8bd623c87/$TOKEN/g list_metrics/config.py 38 | sed -i s/16b2ecfb-7734-48b9-817d-4ac8bd623c87/$TOKEN/g get_timeseries/config.py 39 | sed -i s/16b2ecfb-7734-48b9-817d-4ac8bd623c87/$TOKEN/g write_metrics/config.py 40 | sed -i s/16b2ecfb-7734-48b9-817d-4ac8bd623c87/$TOKEN/g list_projects/config.json 41 | sed -ibk "s/99a9ffa8797a629783cb4aa762639e92b098bac5/$LIST_PROJECTS_TOKEN/g" list_projects/config.json 42 | sed -ibk "s/YOUR_PROJECT_ID/$PROJECT_ID/g" list_projects/config.json 43 | ``` 44 | 45 | 6. Deploy the App Engine apps 46 | Run `gcloud app create` if you don't already have an App Engine app in your project and remove the line `service: list-metrics` from app.yaml. 47 | 48 | __Note:__ The default service account for App Engine has the project `Editor` permission. If you don't use the default service account, you need to grant the App Engine service account sufficient permissions for Cloud Monitoring, Pub/Sub, Cloud Storate, and BigQuery. 49 | 50 | ```sh 51 | cd list_metrics 52 | pip install -t lib -r requirements.txt 53 | echo "y" | gcloud app deploy 54 | ``` 55 | 56 | Copy the URL from the `Deployed service` output and add it to the `LIST_METRICS_URL` variable. 57 | The following is an example. Please replace __PROJECT_ID__ and [__REGION_ID__](https://cloud.google.com/appengine/docs/legacy/standard/python/how-requests-are-routed#region-id) with the real values 58 | 59 | ```sh 60 | export LIST_METRICS_URL=https://list-metrics-dot-PROJECT_ID.REGION_ID.r.appspot.com 61 | ``` 62 | 63 | Do the same for the other services: 64 | 65 | ``` 66 | cd ../get_timeseries 67 | pip install -t lib -r requirements.txt 68 | echo "y" | gcloud app deploy 69 | ``` 70 | Copy the URL from the `Deployed service` output and add it to the `GET_TIMESERIES_URL` variable. 71 | The following is an example. Note: __PROJECT_ID__ and [__REGION_ID__](https://cloud.google.com/appengine/docs/legacy/standard/python/how-requests-are-routed#region-id) are replaced with the real values. 72 | 73 | ```sh 74 | export GET_TIMESERIES_URL=https://get-timeseries-dot-PROJECT_ID-REGION_ID.r.appspot.com 75 | ``` 76 | 77 | ``` 78 | cd ../write_metrics 79 | pip install -t lib -r requirements.txt 80 | echo "y" | gcloud app deploy 81 | ``` 82 | Copy the URL from the `Deployed service` output and add it to the `WRITE_METRICS_URL` variable. 83 | The following is an example. Note: __PROJECT_ID__ and [__REGION_ID__](https://cloud.google.com/appengine/docs/legacy/standard/python/how-requests-are-routed#region-id) are replaced with the real values. 84 | 85 | ```sh 86 | export WRITE_METRICS_URL=https://write-metrics-dot-PROJECT_ID-REGION_ID.appspot.com 87 | ``` 88 | 89 | 7. Create the Pub/Sub topics and subscriptions after setting YOUR_PROJECT_ID 90 | 91 | 92 | Now, get the get_timeseries and write_metrics URLs and create the Pub/Sub topics and subscriptions 93 | 94 | ```sh 95 | gcloud pubsub topics create metrics_export_start 96 | gcloud pubsub subscriptions create metrics_export_start_sub --topic metrics_export_start --ack-deadline=60 --message-retention-duration=10m --push-endpoint="$LIST_METRICS_URL/push-handlers/receive_messages" 97 | 98 | gcloud pubsub topics create metrics_list 99 | gcloud pubsub subscriptions create metrics_list_sub --topic metrics_list --ack-deadline=60 --message-retention-duration=30m --push-endpoint="$GET_TIMESERIES_URL/push-handlers/receive_messages" 100 | 101 | gcloud pubsub topics create write_metrics 102 | gcloud pubsub subscriptions create write_metrics_sub --topic write_metrics --ack-deadline=60 --message-retention-duration=30m --push-endpoint="$WRITE_METRICS_URL/push-handlers/receive_messages" 103 | ``` 104 | 105 | 8. Create a service account for the list_projects function 106 | ```sh 107 | gcloud iam service-accounts create \ 108 | gce-list-projects \ 109 | --description "Used for the function that lists the projects for the GCE Footprint Cloud Function" 110 | export LIST_PROJECTS_SERVICE_ACCOUNT=gce-list-projects@$PROJECT_ID.iam.gserviceaccount.com 111 | ``` 112 | 113 | 9 Assign IAM permissions to the service account 114 | ```sh 115 | gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:$LIST_PROJECTS_SERVICE_ACCOUNT" --role="roles/compute.viewer" 116 | gcloud projects add-iam-policy-binding $PROJECT_ID --member="serviceAccount:$LIST_PROJECTS_SERVICE_ACCOUNT" --role="roles/pubsub.publisher" 117 | 118 | ``` 119 | 120 | 10. Deploy the list_projects function 121 | ```sh 122 | cd ../list_projects 123 | gcloud functions deploy list_projects \ 124 | --trigger-topic metric_export_get_project_start \ 125 | --runtime nodejs18 \ 126 | --entry-point list_projects \ 127 | --service-account=$LIST_PROJECTS_SERVICE_ACCOUNT 128 | ``` 129 | 130 | 11. Deploy the Cloud Scheduler job 131 | ```sh 132 | gcloud scheduler jobs create pubsub metric_export \ 133 | --schedule "*/5 * * * *" \ 134 | --topic metric_export_get_project_start \ 135 | --message-body "{ \"token\":\"$(echo $LIST_PROJECTS_TOKEN)\"}" 136 | ``` 137 | 138 | ## Run the tests 139 | 1. Run the job from the scheduler 140 | ```sh 141 | gcloud scheduler jobs run metric_export 142 | ``` 143 | 144 | 2. Test the app by sending a PubSub message to the metrics_export_start topic 145 | ```sh 146 | gcloud pubsub topics publish metrics_export_start --message "{\"token\": \"$TOKEN\"}" 147 | ``` 148 | 149 | You can send in all of the parameters using the following command 150 | ```sh 151 | gcloud pubsub topics publish metrics_export_start --message "{\"token\": \"$TOKEN\"}, \"start_time\": \"2019-03-13T17:30:00.000000Z\", \"end_time\":\"2019-03-13T17:40:00.000000Z\",\"aggregation_alignment_period\":\"3600s\"}" 152 | ``` 153 | 154 | 3. Verify that the app is working appropriately by running the end-to-end testing 155 | 156 | Configure your project_id and lookup the batch_id in the config.py file. 157 | ```sh 158 | cd test 159 | export PROJECT_ID=$(gcloud config get-value project) 160 | export TIMESTAMP=$(date -d "-2 hour" +%Y-%m-%dT%k:%M:00Z) 161 | export BATCH_ID=$(gcloud logging read "resource.type=\"gae_app\" AND resource.labels.module_id=\"list-metrics\" AND logName=\"projects/$PROJECT_ID/logs/appengine.googleapis.com%2Frequest_log\" AND protoPayload.line.logMessage:\"batch_id:\" AND timestamp >= \"$TIMESTAMP\"" --limit 1 --format json | grep "batch_id:" | awk '{ print substr($3,1,32); }') 162 | sed -i s/YOUR_PROJECT_ID/$PROJECT_ID/g config.py 163 | sed -i s/R8BK5S99QU4ZZOGCR1UDPWVH6LPKI5QU/$BATCH_ID/g config.py 164 | 165 | python end_to_end_test_run.py 166 | ``` 167 | -------------------------------------------------------------------------------- /bigquery_schemas/bigquery_schema.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "REQUIRED", 4 | "name": "batch_id", 5 | "type": "STRING" 6 | }, 7 | { 8 | "name": "metric", 9 | "type": "RECORD", 10 | "mode": "REQUIRED", 11 | "fields": [ 12 | { 13 | "name": "type", 14 | "type": "STRING", 15 | "mode": "REQUIRED" 16 | }, 17 | { 18 | "name": "labels", 19 | "type": "RECORD", 20 | "mode": "REPEATED", 21 | "fields": [ 22 | { 23 | "name": "key", 24 | "type": "STRING", 25 | "mode": "NULLABLE" 26 | }, 27 | { 28 | "name": "value", 29 | "type": "STRING", 30 | "mode": "NULLABLE" 31 | } 32 | ] 33 | } 34 | ] 35 | }, 36 | { 37 | "name": "resource", 38 | "type": "RECORD", 39 | "mode": "REQUIRED", 40 | "fields": [ 41 | { 42 | "name": "type", 43 | "type": "STRING", 44 | "mode": "REQUIRED" 45 | }, 46 | { 47 | "name": "labels", 48 | "type": "RECORD", 49 | "mode": "REPEATED", 50 | "fields": [ 51 | { 52 | "name": "key", 53 | "type": "STRING", 54 | "mode": "NULLABLE" 55 | }, 56 | { 57 | "name": "value", 58 | "type": "STRING", 59 | "mode": "NULLABLE" 60 | } 61 | ] 62 | } 63 | ] 64 | }, 65 | { 66 | "name": "metric_metadata", 67 | "type": "RECORD", 68 | "mode": "NULLABLE", 69 | "fields": [ 70 | { 71 | "name": "system_labels", 72 | "type": "RECORD", 73 | "mode": "REPEATED", 74 | "fields": [ 75 | { 76 | "name": "key", 77 | "type": "STRING", 78 | "mode": "NULLABLE" 79 | }, 80 | { 81 | "name": "value", 82 | "type": "STRING", 83 | "mode": "NULLABLE" 84 | }, 85 | { 86 | "mode": "REPEATED", 87 | "name": "value_list", 88 | "type": "RECORD", 89 | "fields": [ 90 | { 91 | "mode": "NULLABLE", 92 | "name": "value", 93 | "type": "STRING" 94 | } 95 | ] 96 | } 97 | ] 98 | }, 99 | { 100 | "name": "user_labels", 101 | "type": "RECORD", 102 | "mode": "REPEATED", 103 | "fields": [ 104 | { 105 | "name": "key", 106 | "type": "STRING", 107 | "mode": "NULLABLE" 108 | }, 109 | { 110 | "name": "value", 111 | "type": "STRING", 112 | "mode": "NULLABLE" 113 | } 114 | ] 115 | } 116 | ] 117 | }, 118 | { 119 | "name": "metric_kind", 120 | "type": "STRING", 121 | "mode": "REQUIRED" 122 | }, 123 | { 124 | "name": "value_type", 125 | "type": "STRING", 126 | "mode": "REQUIRED" 127 | }, 128 | { 129 | "name": "point", 130 | "type": "RECORD", 131 | "mode": "REQUIRED", 132 | "fields": [ 133 | { 134 | "name": "interval", 135 | "type": "RECORD", 136 | "mode": "REQUIRED", 137 | "fields": [ 138 | { 139 | "name": "start_time", 140 | "type": "TIMESTAMP", 141 | "mode": "NULLABLE" 142 | }, 143 | { 144 | "name": "end_time", 145 | "type": "TIMESTAMP", 146 | "mode": "REQUIRED" 147 | } 148 | ] 149 | }, 150 | { 151 | "name": "value", 152 | "type": "RECORD", 153 | "mode": "REQUIRED", 154 | "fields": [ 155 | { 156 | "name": "boolean_value", 157 | "type": "BOOLEAN", 158 | "mode": "NULLABLE" 159 | }, 160 | { 161 | "name": "int64_value", 162 | "type": "INTEGER", 163 | "mode": "NULLABLE" 164 | }, 165 | { 166 | "name": "double_value", 167 | "type": "FLOAT", 168 | "mode": "NULLABLE" 169 | }, 170 | { 171 | "name": "string_value", 172 | "type": "STRING", 173 | "mode": "NULLABLE" 174 | }, 175 | { 176 | "name": "distribution_value", 177 | "type": "RECORD", 178 | "mode": "NULLABLE", 179 | "fields": [ 180 | { 181 | "mode": "NULLABLE", 182 | "name": "count", 183 | "type": "INTEGER" 184 | }, 185 | { 186 | "mode": "NULLABLE", 187 | "name": "mean", 188 | "type": "NUMERIC" 189 | }, 190 | { 191 | "mode": "NULLABLE", 192 | "name": "sumOfSquaredDeviation", 193 | "type": "NUMERIC" 194 | }, 195 | { 196 | "mode": "NULLABLE", 197 | "name": "range", 198 | "type": "RECORD", 199 | "fields": [ 200 | { 201 | "mode": "NULLABLE", 202 | "name": "min", 203 | "type": "NUMERIC" 204 | }, 205 | { 206 | "mode": "NULLABLE", 207 | "name": "max", 208 | "type": "NUMERIC" 209 | } 210 | ] 211 | }, 212 | { 213 | "mode": "NULLABLE", 214 | "name": "bucketOptions", 215 | "type": "RECORD", 216 | "fields": [ 217 | { 218 | "mode": "NULLABLE", 219 | "name": "linearBuckets", 220 | "type": "RECORD", 221 | "fields": [ 222 | { 223 | "mode": "NULLABLE", 224 | "name": "numFiniteBuckets", 225 | "type": "NUMERIC" 226 | }, 227 | { 228 | "mode": "NULLABLE", 229 | "name": "width", 230 | "type": "NUMERIC" 231 | }, 232 | { 233 | "mode": "NULLABLE", 234 | "name": "offset", 235 | "type": "NUMERIC" 236 | } 237 | ] 238 | }, 239 | { 240 | "mode": "NULLABLE", 241 | "name": "exponentialBuckets", 242 | "type": "RECORD", 243 | "fields": [ 244 | { 245 | "mode": "NULLABLE", 246 | "name": "numFiniteBuckets", 247 | "type": "NUMERIC" 248 | }, 249 | { 250 | "mode": "NULLABLE", 251 | "name": "growthFactor", 252 | "type": "NUMERIC" 253 | }, 254 | { 255 | "mode": "NULLABLE", 256 | "name": "scale", 257 | "type": "NUMERIC" 258 | } 259 | ] 260 | }, 261 | { 262 | "mode": "NULLABLE", 263 | "name": "explicitBuckets", 264 | "type": "RECORD", 265 | "fields": [ 266 | { 267 | "mode": "NULLABLE", 268 | "name": "bounds", 269 | "type": "RECORD", 270 | "fields": [ 271 | { 272 | "mode": "REPEATED", 273 | "name": "value", 274 | "type": "NUMERIC" 275 | } 276 | ] 277 | } 278 | ] 279 | } 280 | ] 281 | }, 282 | { 283 | "mode": "NULLABLE", 284 | "name": "bucketCounts", 285 | "type": "RECORD", 286 | "fields": [ 287 | { 288 | "mode": "REPEATED", 289 | "name": "value", 290 | "type": "INTEGER" 291 | } 292 | ] 293 | }, 294 | { 295 | "mode": "NULLABLE", 296 | "name": "exemplars", 297 | "type": "RECORD", 298 | "fields": [ 299 | { 300 | "mode": "NULLABLE", 301 | "name": "value", 302 | "type": "NUMERIC" 303 | }, 304 | { 305 | "mode": "NULLABLE", 306 | "name": "timestamp", 307 | "type": "STRING" 308 | } 309 | ] 310 | } 311 | ] 312 | } 313 | ] 314 | } 315 | ] 316 | } 317 | ] -------------------------------------------------------------------------------- /bigquery_schemas/bigquery_schema_params_table.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "NULLABLE", 4 | "name": "start_time", 5 | "type": "TIMESTAMP" 6 | }, 7 | { 8 | "mode": "NULLABLE", 9 | "name": "end_time", 10 | "type": "TIMESTAMP" 11 | }, 12 | { 13 | "mode": "NULLABLE", 14 | "name": "aggregation_alignment_period", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "message_id", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "project_list", 25 | "type": "RECORD", 26 | "fields": [ 27 | { 28 | "mode": "REPEATED", 29 | "name": "project_id", 30 | "type": "STRING" 31 | } 32 | ] 33 | }, 34 | { 35 | "mode": "REQUIRED", 36 | "name": "batch_id", 37 | "type": "STRING" 38 | }, 39 | { 40 | "mode": "NULLABLE", 41 | "name": "batch_start_time", 42 | "type": "TIMESTAMP" 43 | } 44 | ] -------------------------------------------------------------------------------- /bigquery_schemas/bigquery_schema_stats_table.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "mode": "REQUIRED", 4 | "name": "app_name", 5 | "type": "STRING" 6 | }, 7 | { 8 | "mode": "REQUIRED", 9 | "name": "batch_id", 10 | "type": "STRING" 11 | }, 12 | { 13 | "mode": "REQUIRED", 14 | "name": "message_id", 15 | "type": "STRING" 16 | }, 17 | { 18 | "mode": "NULLABLE", 19 | "name": "src_message_id", 20 | "type": "STRING" 21 | }, 22 | { 23 | "mode": "NULLABLE", 24 | "name": "metric_type", 25 | "type": "STRING" 26 | }, 27 | { 28 | "mode": "NULLABLE", 29 | "name": "error_msg_cnt", 30 | "type": "INTEGER" 31 | }, 32 | { 33 | "mode": "NULLABLE", 34 | "name": "msg_written_cnt", 35 | "type": "INTEGER" 36 | }, 37 | { 38 | "mode": "NULLABLE", 39 | "name": "msg_without_timeseries", 40 | "type": "INTEGER" 41 | }, 42 | { 43 | "mode": "NULLABLE", 44 | "name": "batch_start_time", 45 | "type": "TIMESTAMP" 46 | }, 47 | { 48 | "mode": "NULLABLE", 49 | "name": "processing_end_time", 50 | "type": "TIMESTAMP" 51 | }, 52 | { 53 | "mode": "NULLABLE", 54 | "name": "payload", 55 | "type": "STRING" 56 | } 57 | ] -------------------------------------------------------------------------------- /get_timeseries/.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | -------------------------------------------------------------------------------- /get_timeseries/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | service: get-timeseries 16 | runtime: python310 17 | app_engine_apis: true 18 | entrypoint: gunicorn -b :$PORT -w 2 main:app 19 | 20 | basic_scaling: 21 | max_instances: 25 22 | idle_timeout: 10m 23 | 24 | handlers: 25 | - url: /.* 26 | script: auto 27 | -------------------------------------------------------------------------------- /get_timeseries/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | PUBSUB_TOPIC="write_metrics" 18 | AGGREGATION_ALIGNMENT_PERIOD="3600s" 19 | GROUP_BY_STRING="metric.labels.key" 20 | PAGE_SIZE=500 21 | PUBSUB_VERIFICATION_TOKEN = '16b2ecfb-7734-48b9-817d-4ac8bd623c87' 22 | BIGQUERY_DATASET='metric_export' 23 | BIGQUERY_STATS_TABLE='sd_metrics_stats' 24 | WRITE_BQ_STATS_FLAG=True 25 | 26 | GAUGE="GAUGE" 27 | DELTA="DELTA" 28 | CUMULATIVE="CUMULATIVE" 29 | 30 | BOOL="BOOL" 31 | INT64="INT64" 32 | DOUBLE="DOUBLE" 33 | STRING="STRING" 34 | DISTRIBUTION="DISTRIBUTION" 35 | 36 | ALIGN_DELTA="ALIGN_DELTA" 37 | ALIGN_FRACTION_TRUE="ALIGN_FRACTION_TRUE" 38 | ALIGN_SUM="ALIGN_SUM" 39 | ALIGN_COUNT="ALIGN_COUNT" 40 | ALIGN_NONE="ALIGN_NONE" 41 | REDUCE_FRACTION_TRUE="REDUCE_FRACTION_TRUE" 42 | REDUCE_MEAN="REDUCE_MEAN" 43 | REDUCE_SUM="REDUCE_SUM" 44 | REDUCE_COUNT="REDUCE_COUNT" 45 | REDUCE_NONE="REDUCE_NONE" -------------------------------------------------------------------------------- /get_timeseries/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | from flask import Flask, Response, request 19 | import json 20 | import base64 21 | import config 22 | from datetime import datetime 23 | from googleapiclient.discovery import build 24 | from googleapiclient.discovery import HttpError 25 | from google.appengine.api import app_identity 26 | from google.appengine.api import wrap_wsgi_app 27 | 28 | 29 | app = Flask(__name__) 30 | app.wsgi_app = wrap_wsgi_app(app.wsgi_app) 31 | 32 | # logging.basicConfig(level=logging.DEBUG) 33 | 34 | 35 | def get_aligner_reducer(metric_kind, metric_val_type): 36 | """Returns the appropriate erSeriesAligner and crossSeriesReducer given the inputs""" 37 | if metric_kind == config.GAUGE: 38 | if metric_val_type == config.BOOL: 39 | crossSeriesReducer = config.REDUCE_MEAN 40 | perSeriesAligner = config.ALIGN_FRACTION_TRUE 41 | elif metric_val_type in [config.INT64, config.DOUBLE, config.DISTRIBUTION]: 42 | crossSeriesReducer = config.REDUCE_SUM 43 | perSeriesAligner = config.ALIGN_SUM 44 | elif metric_val_type == config.STRING: 45 | crossSeriesReducer = config.REDUCE_COUNT 46 | perSeriesAligner = config.ALIGN_NONE 47 | else: 48 | logging.debug( 49 | "No match for GAUGE {},{}".format(metric_kind, metric_val_type) 50 | ) 51 | elif metric_kind == config.DELTA: 52 | if metric_val_type in [config.INT64, config.DOUBLE, config.DISTRIBUTION]: 53 | crossSeriesReducer = config.REDUCE_SUM 54 | perSeriesAligner = config.ALIGN_SUM 55 | else: 56 | logging.debug( 57 | "No match for DELTA {},{}".format(metric_kind, metric_val_type) 58 | ) 59 | elif metric_kind == config.CUMULATIVE: 60 | if metric_val_type in [config.INT64, config.DOUBLE, config.DISTRIBUTION]: 61 | crossSeriesReducer = config.REDUCE_SUM 62 | perSeriesAligner = config.ALIGN_DELTA 63 | else: 64 | logging.debug( 65 | "No match for CUMULATIVE {},{}".format(metric_kind, metric_val_type) 66 | ) 67 | else: 68 | logging.debug("No match for {},{}".format(metric_kind, metric_val_type)) 69 | 70 | return crossSeriesReducer, perSeriesAligner 71 | 72 | 73 | def get_and_publish_timeseries(data, metadata): 74 | """Handle the Pub/Sub push messages 75 | Given a metricDescriptor object described below and a batch_id, as input 76 | https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors#MetricDescriptor 77 | 78 | Get the timeseries for the metric and then publish each individual timeseries described below 79 | https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries 80 | as a separate Pub/Sub message. This means there is a fan-out from 1 metric to 1 or more 81 | timeseries objects. 82 | """ 83 | metric_type = data["metric"]["type"] 84 | metric_kind = data["metric"]["metricKind"] 85 | metric_val_type = data["metric"]["valueType"] 86 | end_time_str = data["end_time"] 87 | start_time_str = data["start_time"] 88 | project_id = data["project_id"] 89 | 90 | logging.debug( 91 | "get_timeseries for metric: {},{},{},{},{}".format( 92 | metric_type, metric_kind, metric_val_type, start_time_str, end_time_str 93 | ) 94 | ) 95 | project_name = "projects/{project_id}".format(project_id=project_id) 96 | # Capture the stats 97 | stats = {} 98 | msgs_published = 0 99 | msgs_without_timeseries = 0 100 | metrics_count_from_api = 0 101 | 102 | # get the appropriate aligner based on the metric_kind and value_type 103 | crossSeriesReducer, perSeriesAligner = get_aligner_reducer( 104 | metric_kind, metric_val_type 105 | ) 106 | 107 | # build a dict with the API parameters 108 | api_args = {} 109 | api_args["project_name"] = project_name 110 | api_args["metric_filter"] = 'metric.type="{}" '.format(metric_type) 111 | api_args["end_time_str"] = data["end_time"] 112 | api_args["start_time_str"] = data["start_time"] 113 | api_args["aggregation_alignment_period"] = data["aggregation_alignment_period"] 114 | api_args["group_by"] = config.GROUP_BY_STRING 115 | api_args["crossSeriesReducer"] = crossSeriesReducer 116 | api_args["perSeriesAligner"] = perSeriesAligner 117 | api_args["nextPageToken"] = "" 118 | 119 | # Call the projects.timeseries.list API 120 | response_code = 200 121 | timeseries = {} 122 | while True: 123 | try: 124 | timeseries = get_timeseries(api_args) 125 | except Exception as e: 126 | metadata["error_msg_cnt"] = 1 127 | logging.error(f"Exception calling Monitoring API: {e}") 128 | 129 | if timeseries: 130 | # retryable error codes based on https://developers.google.com/maps-booking/reference/grpc-api/status_codes 131 | if "executionErrors" in timeseries: 132 | if timeseries["executionErrors"]["code"] != 0: 133 | response_code = 500 134 | logging.error( 135 | "Received an error getting the timeseries with code: {} and msg: {}".format( 136 | timeseries["executionErrors"]["code"], 137 | timeseries["executionErrors"]["message"], 138 | ) 139 | ) 140 | break 141 | elif "timeSeries" not in timeseries: 142 | logging.info(f"No timeSeries in {timeseries}") 143 | break 144 | else: 145 | # write the timeseries 146 | msgs_published += publish_timeseries(timeseries, metadata) 147 | metrics_count_from_api += len(timeseries["timeSeries"]) 148 | if "nextPageToken" in timeseries: 149 | api_args["nextPageToken"] = timeseries["nextPageToken"] 150 | else: 151 | break 152 | 153 | else: 154 | logging.debug("No timeseries returned, no publish to pubsub") 155 | 156 | # build a list of stats message to write to BigQuery 157 | metadata["msg_written_cnt"] = 0 158 | metadata["msg_without_timeseries"] = 1 159 | if "error_msg_cnt" not in metadata: 160 | metadata["error_msg_cnt"] = 0 161 | metadata["payload"] = "" 162 | metadata["metric_type"] = metric_type 163 | json_msg = build_bigquery_stats_message(metadata) 164 | json_msg_list = [] 165 | json_msg_list.append(json_msg) 166 | 167 | msgs_published += 1 168 | # write the list of stats messages to BigQuery 169 | write_to_bigquery(json_msg_list) 170 | msgs_without_timeseries = 1 171 | break 172 | 173 | stats["msgs_published"] = msgs_published 174 | stats["msgs_without_timeseries"] = msgs_without_timeseries 175 | stats["metrics_count_from_api"] = metrics_count_from_api 176 | logging.debug("Stats are {}".format(json.dumps(stats))) 177 | 178 | return response_code 179 | 180 | 181 | def get_timeseries(api_args): 182 | """Call the https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.timeSeries/list 183 | using the googleapiclient 184 | """ 185 | logging.debug(f"api_args: {api_args}") 186 | service = build("monitoring", "v3", cache_discovery=True) 187 | timeseries = ( 188 | service.projects() 189 | .timeSeries() 190 | .list( 191 | name=api_args["project_name"], 192 | filter=api_args["metric_filter"], 193 | aggregation_alignmentPeriod=api_args["aggregation_alignment_period"], 194 | # aggregation_crossSeriesReducer=api_args["crossSeriesReducer"], 195 | aggregation_perSeriesAligner=api_args["perSeriesAligner"], 196 | aggregation_groupByFields=api_args["group_by"], 197 | interval_endTime=api_args["end_time_str"], 198 | interval_startTime=api_args["start_time_str"], 199 | pageSize=config.PAGE_SIZE, 200 | pageToken=api_args["nextPageToken"], 201 | ) 202 | .execute() 203 | ) 204 | logging.debug( 205 | "response: {}".format(json.dumps(timeseries, sort_keys=True, indent=4)) 206 | ) 207 | return timeseries 208 | 209 | 210 | def get_pubsub_message(one_timeseries, metadata): 211 | logging.debug( 212 | "pubsub msg is {}".format(json.dumps(one_timeseries, sort_keys=True, indent=4)) 213 | ) 214 | data = json.dumps(one_timeseries).encode("utf-8") 215 | message = { 216 | "data": base64.b64encode(data).decode(), 217 | "attributes": { 218 | "batch_id": metadata["batch_id"], 219 | "token": config.PUBSUB_VERIFICATION_TOKEN, 220 | "src_message_id": metadata["message_id"], 221 | "batch_start_time": metadata["batch_start_time"], 222 | }, 223 | } 224 | 225 | logging.debug( 226 | "pubsub msg is {}".format(json.dumps(message, sort_keys=True, indent=4)) 227 | ) 228 | return message 229 | 230 | 231 | def publish_metrics(msg_list): 232 | """Call the https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish 233 | using the googleapiclient to publish a message to Pub/Sub. 234 | The token and batch_id are included as attributes 235 | """ 236 | service = build("pubsub", "v1", cache_discovery=True) 237 | topic_path = "projects/{project_id}/topics/{topic}".format( 238 | project_id=app_identity.get_application_id(), topic=config.PUBSUB_TOPIC 239 | ) 240 | body = {"messages": msg_list} 241 | logging.debug("pubsub msg is {}".format(body)) 242 | response = ( 243 | service.projects().topics().publish(topic=topic_path, body=body).execute() 244 | ) 245 | logging.debug("response is {}".format(response, sort_keys=True, indent=4)) 246 | 247 | 248 | def publish_timeseries(request, metadata): 249 | """Call the https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish 250 | using the googleapiclient to publish a message to Pub/Sub 251 | """ 252 | 253 | msgs_published = 0 254 | json_msg_list = [] 255 | pubsub_msg_list = [] 256 | # handle >= 1 timeSeries, potentially > 1 returned from Monitoring API call 257 | for one_timeseries in request["timeSeries"]: 258 | message = get_pubsub_message(one_timeseries, metadata) 259 | pubsub_msg_list.append(message) 260 | 261 | # build a list of stats messages to write to BigQuery 262 | metadata["msg_written_cnt"] = 1 263 | metadata["msg_without_timeseries"] = 0 264 | metadata["error_msg_cnt"] = 0 265 | metadata["payload"] = "{}".format(json.dumps(one_timeseries)) 266 | metadata["metric_type"] = one_timeseries["metric"]["type"] 267 | if config.WRITE_BQ_STATS_FLAG: 268 | json_msg = build_bigquery_stats_message(metadata) 269 | json_msg_list.append(json_msg) 270 | 271 | msgs_published += 1 272 | 273 | # Write the messages to pubsub 274 | publish_metrics(pubsub_msg_list) 275 | 276 | # write the list of stats messages to BigQuery 277 | if config.WRITE_BQ_STATS_FLAG: 278 | write_to_bigquery(json_msg_list) 279 | 280 | return msgs_published 281 | 282 | 283 | def build_bigquery_stats_message(metadata): 284 | processing_end_time = datetime.now() 285 | processing_end_time_str = processing_end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 286 | 287 | # Write the stats to the BigQuery stats tabledata 288 | bq_msg = { 289 | "app_name": "get_timeseries", 290 | "batch_id": metadata["batch_id"], 291 | "message_id": metadata["message_id"], 292 | "src_message_id": metadata["src_message_id"], 293 | "metric_type": metadata["metric_type"], 294 | "error_msg_cnt": metadata["error_msg_cnt"], 295 | "msg_written_cnt": metadata["msg_written_cnt"], 296 | "msg_without_timeseries": metadata["msg_without_timeseries"], 297 | "payload": metadata["payload"], 298 | "batch_start_time": metadata["batch_start_time"], 299 | "processing_end_time": processing_end_time_str, 300 | } 301 | json_msg = {"json": bq_msg} 302 | logging.debug("json_msg {}".format(json.dumps(json_msg, sort_keys=True, indent=4))) 303 | return json_msg 304 | 305 | 306 | def write_to_bigquery(json_row_list): 307 | """Write rows to the BigQuery stats table using the googleapiclient and the streaming insertAll method 308 | https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll 309 | """ 310 | logging.debug("write_to_bigquery") 311 | 312 | bigquery = build("bigquery", "v2", cache_discovery=True) 313 | 314 | body = { 315 | "kind": "bigquery#tableDataInsertAllRequest", 316 | "skipInvalidRows": "false", 317 | "rows": json_row_list, 318 | } 319 | logging.debug("body: {}".format(json.dumps(body, sort_keys=True, indent=4))) 320 | 321 | response = ( 322 | bigquery.tabledata() 323 | .insertAll( 324 | projectId=app_identity.get_application_id(), 325 | datasetId=config.BIGQUERY_DATASET, 326 | tableId=config.BIGQUERY_STATS_TABLE, 327 | body=body, 328 | ) 329 | .execute() 330 | ) 331 | logging.debug("BigQuery said... = {}".format(response)) 332 | 333 | bq_msgs_with_errors = 0 334 | if "insertErrors" in response: 335 | if len(response["insertErrors"]) > 0: 336 | logging.error("Error: {}".format(response)) 337 | bq_msgs_with_errors = len(response["insertErrors"]) 338 | logging.debug("bq_msgs_with_errors: {}".format(bq_msgs_with_errors)) 339 | else: 340 | logging.debug( 341 | "By amazing luck, there are no errors, response = {}".format(response) 342 | ) 343 | return response 344 | 345 | 346 | @app.route("/push-handlers/receive_messages", methods=["POST"]) 347 | def post(): 348 | """Receive the Pub/Sub message via POST 349 | Validate the input and then process the message 350 | """ 351 | 352 | logging.debug("received message") 353 | 354 | response_code = 200 355 | try: 356 | if not request.data: 357 | raise ValueError("No request data received") 358 | envelope = json.loads(request.data.decode("utf-8")) 359 | logging.debug("Raw pub/sub message: {}".format(envelope)) 360 | 361 | if "message" not in envelope: 362 | raise ValueError("No message in envelope") 363 | 364 | if "messageId" in envelope["message"]: 365 | logging.debug( 366 | "messageId: {}".format(envelope["message"].get("messageId", "")) 367 | ) 368 | message_id = envelope["message"].get("messageId", "") 369 | 370 | if "attributes" not in envelope["message"]: 371 | raise ValueError( 372 | "Attributes such as token and batch_id missing from request" 373 | ) 374 | 375 | if "data" not in envelope["message"]: 376 | raise ValueError("No data in message") 377 | payload = base64.b64decode(envelope["message"]["data"]) 378 | logging.debug("payload: {} ".format(payload)) 379 | 380 | # if the pubsub PUBSUB_VERIFICATION_TOKEN isn't included or doesn't match, don't continue 381 | if "token" not in envelope["message"]["attributes"]: 382 | raise ValueError("token missing from request") 383 | if ( 384 | not envelope["message"]["attributes"]["token"] 385 | == config.PUBSUB_VERIFICATION_TOKEN 386 | ): 387 | raise ValueError("token from request doesn't match") 388 | 389 | # if the batch_id isn't included, fail immediately 390 | if "batch_id" not in envelope["message"]["attributes"]: 391 | raise ValueError("batch_id missing from request") 392 | batch_id = envelope["message"]["attributes"]["batch_id"] 393 | logging.debug("batch_id: {}".format(batch_id)) 394 | 395 | if "batch_start_time" in envelope["message"]["attributes"]: 396 | batch_start_time = envelope["message"]["attributes"]["batch_start_time"] 397 | else: 398 | batch_start_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") 399 | 400 | if "src_message_id" in envelope["message"]["attributes"]: 401 | src_message_id = envelope["message"]["attributes"]["src_message_id"] 402 | 403 | data = json.loads(payload) 404 | logging.debug("data: {} ".format(data)) 405 | 406 | # Check the input parameters 407 | if not data: 408 | raise ValueError("No data in Pub/Sub Message") 409 | if "metric" not in data: 410 | raise ValueError("Missing metric key in Pub/Sub message") 411 | if "type" not in data["metric"]: 412 | raise ValueError("Missing metric[type] key in Pub/Sub message") 413 | if "metricKind" not in data["metric"]: 414 | raise ValueError("Missing metric[metricKind] key in Pub/Sub message") 415 | if "valueType" not in data["metric"]: 416 | raise ValueError("Missing metric[valueType] key in Pub/Sub message") 417 | if "end_time" not in data: 418 | raise ValueError("Missing end_time key in Pub/Sub message") 419 | if "start_time" not in data: 420 | raise ValueError("Missing start_time key in Pub/Sub message") 421 | if "aggregation_alignment_period" not in data: 422 | raise ValueError( 423 | "Missing aggregation_alignment_period key in Pub/Sub message" 424 | ) 425 | if "project_id" not in data: 426 | data["project_id"] = app_identity.get_application_id() 427 | 428 | metadata = { 429 | "batch_id": batch_id, 430 | "message_id": message_id, 431 | "src_message_id": src_message_id, 432 | "batch_start_time": batch_start_time, 433 | } 434 | # get the metrics and publish to Pub/Sub 435 | response_code = get_and_publish_timeseries(data, metadata) 436 | except Exception as e: 437 | logging.error("Error: {}".format(e)) 438 | return Response(f"{e}", status=500) 439 | 440 | return Response("Ok", status=response_code) 441 | -------------------------------------------------------------------------------- /get_timeseries/main_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import pytest 18 | from main import app as flask_app 19 | 20 | import main 21 | import config 22 | import base64 23 | import json 24 | 25 | 26 | @pytest.fixture 27 | def app(): 28 | yield flask_app 29 | 30 | 31 | @pytest.fixture 32 | def client(app): 33 | return app.test_client() 34 | 35 | 36 | def test_aligner_reducer_values(): 37 | """Test the get_aligner_reducer() function logic""" 38 | crossSeriesReducer, perSeriesAligner = main.get_aligner_reducer( 39 | config.GAUGE, config.BOOL 40 | ) 41 | assert crossSeriesReducer == config.REDUCE_MEAN 42 | assert perSeriesAligner == config.ALIGN_FRACTION_TRUE 43 | 44 | crossSeriesReducer, perSeriesAligner = main.get_aligner_reducer( 45 | config.GAUGE, config.INT64 46 | ) 47 | assert crossSeriesReducer == config.REDUCE_SUM 48 | assert perSeriesAligner == config.ALIGN_SUM 49 | 50 | crossSeriesReducer, perSeriesAligner = main.get_aligner_reducer( 51 | config.DELTA, config.INT64 52 | ) 53 | assert crossSeriesReducer == config.REDUCE_SUM 54 | assert perSeriesAligner == config.ALIGN_SUM 55 | 56 | crossSeriesReducer, perSeriesAligner = main.get_aligner_reducer( 57 | config.CUMULATIVE, config.INT64 58 | ) 59 | assert crossSeriesReducer == config.REDUCE_SUM 60 | assert perSeriesAligner == config.ALIGN_DELTA 61 | 62 | 63 | def test_post_empty_data(app, client): 64 | """Test sending an empty message""" 65 | response = client.post("/push-handlers/receive_messages") 66 | assert response.status_code == 500 67 | assert response.get_data(as_text=True) == "No request data received" 68 | 69 | 70 | def test_incorrect_token_post(app, client): 71 | """Test sending an incorrect token""" 72 | request = build_request(token="incorrect_token") 73 | mimetype = "application/json" 74 | headers = { 75 | "Content-Type": mimetype, 76 | "Accept": mimetype, 77 | } 78 | response = client.post( 79 | "/push-handlers/receive_messages", data=json.dumps(request), headers=headers 80 | ) 81 | assert response.status_code == 500 82 | 83 | 84 | def build_request( 85 | token=config.PUBSUB_VERIFICATION_TOKEN, 86 | batch_id="12h3eldjhwuidjwk222dwd09db5zlaqs", 87 | metric_type="bigquery.googleapis.com/query/count", 88 | metric_kind=config.GAUGE, 89 | value_type=config.INT64, 90 | start_time="2019-02-18T13:00:00.311635Z", 91 | end_time="2019-02-18T14:00:00.311635Z", 92 | aggregation_alignment_period="3600s", 93 | ): 94 | """Build a request to submit""" 95 | 96 | payload = { 97 | "metric": { 98 | "type": metric_type, 99 | "metricKind": metric_kind, 100 | "valueType": value_type, 101 | }, 102 | "start_time": start_time, 103 | "end_time": end_time, 104 | "aggregation_alignment_period": aggregation_alignment_period, 105 | } 106 | request = { 107 | "message": { 108 | "attributes": {"batch_id": batch_id, "token": token}, 109 | "data": base64.b64encode(json.dumps(payload).encode("utf-8")).decode(), 110 | } 111 | } 112 | return request 113 | -------------------------------------------------------------------------------- /get_timeseries/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | gunicorn 3 | appengine-python-standard>=1.0.0 4 | google-api-python-client -------------------------------------------------------------------------------- /get_timeseries/sample.json: -------------------------------------------------------------------------------- 1 | {"timeSeries": [{"metricKind": "DELTA", "metric": {"type": "agent.googleapis.com/agent/api_request_count"}, "points": [{"interval": {"endTime": "2018-12-14T19:58:17.140600Z", "startTime": "2018-12-14T18:58:17.140600Z"}, "value": {"int64Value": "358"}}], "resource": {"labels": {"project_id": "sage-facet-201016"}, "type": "gce_instance"}, "valueType": "INT64"}]} -------------------------------------------------------------------------------- /list_metrics/.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | -------------------------------------------------------------------------------- /list_metrics/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | service: list-metrics 16 | runtime: python310 17 | app_engine_apis: true 18 | entrypoint: gunicorn -b :$PORT -w 2 main:app 19 | 20 | basic_scaling: 21 | max_instances: 25 22 | idle_timeout: 10m 23 | 24 | handlers: 25 | - url: /.* 26 | script: auto -------------------------------------------------------------------------------- /list_metrics/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | PUBSUB_TOPIC="metrics_list" 18 | AGGREGATION_ALIGNMENT_PERIOD="3600s" 19 | PUBSUB_VERIFICATION_TOKEN = '16b2ecfb-7734-48b9-817d-4ac8bd623c87' 20 | LAST_END_TIME_FILENAME="last_end_time.txt" 21 | PAGE_SIZE=500 22 | BIGQUERY_DATASET='metric_export' 23 | BIGQUERY_STATS_TABLE='sd_metrics_stats' 24 | BIGQUERY_PARAMS_TABLE='sd_metrics_params' 25 | WRITE_BQ_STATS_FLAG=True 26 | WRITE_MONITORING_STATS_FLAG=True 27 | ALL="*" 28 | 29 | INCLUSIONS = { 30 | "include_all": "", 31 | "metricTypes":[ 32 | # { "metricType": "compute.googleapis.com/instance/cpu/utilization" }, 33 | # { "metricType": "compute.googleapis.com/instance/disk/write_ops_count" } 34 | ], 35 | "metricTypeGroups": [ 36 | # { "metricTypeGroup": "bigquery.googleapis.com" } 37 | ] 38 | } 39 | 40 | EXCLUSIONS = { 41 | "exclude_all": "", 42 | "metricKinds":[ 43 | { 44 | "metricKind": "GAUGE", 45 | "valueType": "STRING" 46 | } 47 | ], 48 | "metricTypes":[ 49 | ], 50 | "metricTypeGroups": [ 51 | { 52 | "metricTypeGroup": "aws.googleapis.com" 53 | }, 54 | { 55 | "metricTypeGroup": "external.googleapis.com" 56 | } 57 | ] 58 | } -------------------------------------------------------------------------------- /list_metrics/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | from flask import Flask, Response, request 19 | from google.appengine.api import wrap_wsgi_app 20 | from google.appengine.api import app_identity 21 | import json 22 | import base64 23 | from datetime import datetime 24 | from datetime import timedelta 25 | from googleapiclient.discovery import build 26 | from google.cloud import storage 27 | import os 28 | import config 29 | import random 30 | import string 31 | import re 32 | 33 | # logging.basicConfig(level=logging.DEBUG) 34 | 35 | app = Flask(__name__) 36 | app.wsgi_app = wrap_wsgi_app(app.wsgi_app) 37 | 38 | 39 | def set_last_end_time(project_id, bucket_name, end_time_str, offset): 40 | """Write the end_time as a string value in a JSON object in GCS. 41 | This file is used to remember the last end_time in case one isn't provided 42 | """ 43 | # get the datetime object 44 | end_time = datetime.strptime(end_time_str, "%Y-%m-%dT%H:%M:%S.%fZ") 45 | delta = timedelta(seconds=offset) 46 | # Add offset seconds & convert back to str 47 | end_time_calc = end_time + delta 48 | end_time_calc_str = end_time_calc.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 49 | file_name = "{}.{}".format(project_id, config.LAST_END_TIME_FILENAME) 50 | 51 | logging.debug( 52 | "set_last_end_time - end_time_str: {}, end_time_Calc_str: {}".format( 53 | end_time_str, end_time_calc_str 54 | ) 55 | ) 56 | end_time_str_json = {"end_time": end_time_calc_str} 57 | storage_client = storage.Client() 58 | bucket = storage_client.bucket(bucket_name) 59 | blob = bucket.blob(file_name) 60 | with blob.open("w") as f: 61 | f.write(json.dumps(end_time_str_json)) 62 | 63 | return end_time_calc_str 64 | 65 | 66 | def get_last_end_time(project_id, bucket_name): 67 | """Get the end_time as a string value from a JSON object in GCS. 68 | This file is used to remember the last end_time in case one isn't provided 69 | """ 70 | last_end_time_str = "" 71 | file_name = "{}.{}".format(project_id, config.LAST_END_TIME_FILENAME) 72 | logging.debug("get_last_end_time - file_name: {}".format(file_name)) 73 | storage_client = storage.Client() 74 | bucket = storage_client.bucket(bucket_name) 75 | blob = bucket.blob(file_name) 76 | try: 77 | with blob.open("r") as f: 78 | contents = f.read() 79 | logging.debug("GCS FILE CONTENTS: {}".format(contents)) 80 | json_contents = json.loads(contents) 81 | last_end_time_str = json_contents["end_time"] 82 | except Exception as e: 83 | logging.error(f"{e}") 84 | 85 | return last_end_time_str 86 | 87 | 88 | def publish_metrics(msg_list): 89 | """Call the https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish 90 | using the googleapiclient to publish a message to Pub/Sub. 91 | The token and batch_id are included as attributes 92 | """ 93 | if len(msg_list) > 0: 94 | service = build("pubsub", "v1", cache_discovery=True) 95 | topic_path = "projects/{project_id}/topics/{topic}".format( 96 | project_id=app_identity.get_application_id(), topic=config.PUBSUB_TOPIC 97 | ) 98 | body = {"messages": msg_list} 99 | logging.debug("pubsub msg is {}".format(body)) 100 | response = ( 101 | service.projects().topics().publish(topic=topic_path, body=body).execute() 102 | ) 103 | logging.debug("response is {}".format(response, sort_keys=True, indent=4)) 104 | else: 105 | logging.debug("No pubsub messages to publish") 106 | 107 | 108 | def get_message_for_publish_metric(request, metadata): 109 | """Build a message for the https://cloud.google.com/pubsub/docs/reference/rest/v1/projects.topics/publish 110 | using the googleapiclient to publish a message to Pub/Sub. 111 | The token and batch_id are included as attributes 112 | """ 113 | # logging.debug("sending message is {}".format(json.dumps(request, sort_keys=True, indent=4))) 114 | 115 | data = json.dumps(request).encode("utf-8") 116 | 117 | message = { 118 | "data": base64.b64encode(data).decode(), 119 | "attributes": { 120 | "batch_id": metadata["batch_id"], 121 | "token": config.PUBSUB_VERIFICATION_TOKEN, 122 | "batch_start_time": metadata["batch_start_time"], 123 | "src_message_id": metadata["message_id"], 124 | }, 125 | } 126 | # logging.debug("pubsub message is {}".format(json.dumps(message, sort_keys=True, indent=4))) 127 | return message 128 | 129 | 130 | def get_batch_id(): 131 | """Generate a unique id to use across the batches to uniquely identify each one""" 132 | return "".join( 133 | random.choice(string.ascii_uppercase + string.digits) for _ in range(32) 134 | ) 135 | 136 | 137 | def check_date_format(date_str): 138 | """Check the date to ensure that it's in the proper format""" 139 | pattern = re.compile(r"^\d{4}-+\d{2}-+\d{2}T+\d{2}:+\d{2}:+\d{2}.+\d{1,}Z+$") 140 | matched = pattern.match(date_str) 141 | return matched 142 | 143 | 144 | def check_exclusions(metric): 145 | """Check whether to exclude a metric based on the inclusions OR exclusions list. 146 | Note that this checks inclusions first. 147 | returns True for metrics to include 148 | returns False for metrics to exclude 149 | """ 150 | inclusions = config.INCLUSIONS 151 | if "include_all" in inclusions and inclusions["include_all"] == config.ALL: 152 | # logging.debug("including based on include_all setting {},{}".format(metric['type'],inclusions["include_all"])) 153 | return True 154 | 155 | if "metricKinds" in inclusions: 156 | for inclusion in inclusions["metricKinds"]: 157 | # logging.debug("inclusion check: {},{}".format(metric['metricKind'],inclusion['metricKind'])) 158 | if (metric.get("metricKind") == inclusion["metricKind"]) and ( 159 | metric.get("valueType") == inclusion["valueType"] 160 | ): 161 | # logging.debug("including based on metricKind {},{} AND {},{}".format(metric['metricKind'],inclusion['metricKind'],metric['valueType'],inclusion['valueType'])) 162 | return True 163 | 164 | if "metricTypes" in inclusions: 165 | for inclusion in inclusions["metricTypes"]: 166 | # logging.debug("inclusion metricTypes check: {},{}".format(metric['type'],inclusion['metricType'])) 167 | if metric.get("type", "").find(inclusion["metricType"]) != -1: 168 | # logging.debug("including based on metricType {},{}".format(metric['type'],inclusion['metricType'])) 169 | return True 170 | 171 | if "metricTypeGroups" in inclusions: 172 | for inclusion in inclusions["metricTypeGroups"]: 173 | # logging.debug("inclusion metricTypes check: {},{}".format(metric['type'],inclusion['metricTypeGroup'])) 174 | if metric.get("type", "").find(inclusion["metricTypeGroup"]) != -1: 175 | logging.debug( 176 | "including based on metricTypeGroups {},{}".format( 177 | metric.get("type", ""), inclusion["metricTypeGroup"] 178 | ) 179 | ) 180 | return True 181 | 182 | exclusions = config.EXCLUSIONS 183 | if "exclude_all" in exclusions and exclusions["exclude_all"] == config.ALL: 184 | # logging.debug("excluding based on exclude_all setting {},{}".format(metric['type'],exclusions["exclude_all"])) 185 | return False 186 | 187 | if "metricKinds" in exclusions: 188 | for exclusion in exclusions["metricKinds"]: 189 | # logging.debug("exclusion check: {},{}".format(metric['metricKind'],exclusion['metricKind'])) 190 | if (metric.get("metricKind") == exclusion["metricKind"]) and ( 191 | metric.get("valueType") == exclusion["valueType"] 192 | ): 193 | # logging.debug("excluding based on metricKind {},{} AND {},{}".format(metric['metricKind'],exclusion['metricKind'],metric['valueType'],exclusion['valueType'])) 194 | return False 195 | 196 | if "metricTypes" in exclusions: 197 | for exclusion in exclusions["metricTypes"]: 198 | # logging.debug("exclusion metricTypes check: {},{}".format(metric['type'],exclusion['metricType'])) 199 | if metric.get("type", "").find(exclusion["metricType"]) != -1: 200 | # logging.debug("excluding based on metricType {},{}".format(metric['type'],exclusion['metricType'])) 201 | return False 202 | 203 | if "metricTypeGroups" in exclusions: 204 | for exclusion in exclusions["metricTypeGroups"]: 205 | # logging.debug("exclusion metricTypeGroups check: {},{}".format(metric['type'],exclusion['metricTypeGroup'])) 206 | if metric.get("type", "").find(exclusion["metricTypeGroup"]) != -1: 207 | # logging.debug("excluding based on metricTypeGroup {},{}".format(metric['type'],exclusion['metricTypeGroup'])) 208 | return False 209 | return True 210 | 211 | 212 | def get_metrics(project_id, next_page_token): 213 | """Call the https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors/list 214 | using the googleapiclient to get all the metricDescriptors for the project 215 | """ 216 | 217 | service = build("monitoring", "v3", cache_discovery=True) 218 | project_name = "projects/{project_id}".format(project_id=project_id) 219 | 220 | metrics = ( 221 | service.projects() 222 | .metricDescriptors() 223 | .list(name=project_name, pageSize=config.PAGE_SIZE, pageToken=next_page_token) 224 | .execute() 225 | ) 226 | 227 | logging.debug( 228 | "project_id: {}, size: {}".format(project_id, len(metrics["metricDescriptors"])) 229 | ) 230 | return metrics 231 | 232 | 233 | def get_and_publish_metrics(message_to_publish, metadata): 234 | """Publish the direct JSON results of each metricDescriptor as a separate Pub/Sub message""" 235 | 236 | stats = {} 237 | msgs_published = 0 238 | msgs_excluded = 0 239 | metrics_count_from_api = 0 240 | 241 | next_page_token = "" 242 | while True: 243 | json_msg_list = [] 244 | pubsub_msg_list = [] 245 | 246 | project_id = message_to_publish["project_id"] 247 | metric_list = get_metrics(project_id, next_page_token) 248 | 249 | metrics_count_from_api += len(metric_list["metricDescriptors"]) 250 | for metric in metric_list["metricDescriptors"]: 251 | logging.debug("Processing metric {} for publish".format(metric)) 252 | metadata["payload"] = json.dumps(metric) 253 | metadata["error_msg_cnt"] = 0 254 | 255 | message_to_publish["metric"] = metric 256 | if check_exclusions(metric): 257 | pubsub_msg = get_message_for_publish_metric( 258 | message_to_publish, metadata 259 | ) 260 | pubsub_msg_list.append(pubsub_msg) 261 | metadata["msg_written_cnt"] = 1 262 | metadata["msg_without_timeseries"] = 0 263 | msgs_published += 1 264 | else: 265 | # logging.debug("Excluded the metric: {}".format(metric['name'])) 266 | msgs_excluded += 1 267 | metadata["msg_written_cnt"] = 0 268 | metadata["msg_without_timeseries"] = 1 269 | 270 | # build a list of stats messages to write to BigQuery 271 | if config.WRITE_BQ_STATS_FLAG: 272 | json_msg = build_bigquery_stats_message(message_to_publish, metadata) 273 | json_msg_list.append(json_msg) 274 | 275 | # Write to pubsub if there is 1 or more 276 | logging.debug("Start publish_metrics") 277 | publish_metrics(pubsub_msg_list) 278 | 279 | # write the list of stats messages to BigQuery 280 | if config.WRITE_BQ_STATS_FLAG: 281 | write_to_bigquery(json_msg_list) 282 | 283 | if "nextPageToken" in metric_list: 284 | next_page_token = metric_list["nextPageToken"] 285 | else: 286 | break 287 | stats["msgs_published"] = msgs_published 288 | stats["msgs_excluded"] = msgs_excluded 289 | stats["metrics_count_from_api"] = metrics_count_from_api 290 | 291 | return stats 292 | 293 | 294 | def write_stats(stats, stats_project_id, batch_id): 295 | """Write 3 custom monitoring metrics to the Monitoring API""" 296 | logging.debug("write_stats: {}".format(json.dumps(stats))) 297 | service = build("monitoring", "v3", cache_discovery=True) 298 | project_name = "projects/{project_id}".format( 299 | project_id=app_identity.get_application_id() 300 | ) 301 | 302 | end_time = datetime.now() 303 | end_time_str = end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 304 | metric_type = "custom.googleapis.com/stackdriver-monitoring-export/msgs-published" 305 | body = { 306 | "timeSeries": [ 307 | { 308 | "metric": { 309 | "type": metric_type, 310 | "labels": { 311 | "batch_id": batch_id, 312 | "metrics_project_id": stats_project_id, 313 | }, 314 | }, 315 | "resource": { 316 | "type": "generic_node", 317 | "labels": { 318 | "project_id": app_identity.get_application_id(), 319 | "location": "us-central1-a", 320 | "namespace": "stackdriver-metric-export", 321 | "node_id": "list-metrics", 322 | }, 323 | }, 324 | "metricKind": "GAUGE", 325 | "valueType": "INT64", 326 | "points": [ 327 | { 328 | "interval": {"endTime": end_time_str}, 329 | "value": {"int64Value": stats["msgs_published"]}, 330 | } 331 | ], 332 | } 333 | ] 334 | } 335 | 336 | metrics = ( 337 | service.projects().timeSeries().create(name=project_name, body=body).execute() 338 | ) 339 | logging.debug( 340 | "wrote a response is {}".format(json.dumps(metrics, sort_keys=True, indent=4)) 341 | ) 342 | 343 | body["timeSeries"][0]["metric"][ 344 | "type" 345 | ] = "custom.googleapis.com/stackdriver-monitoring-export/msgs-excluded" 346 | body["timeSeries"][0]["points"][0]["value"]["int64Value"] = stats["msgs_excluded"] 347 | metrics = ( 348 | service.projects().timeSeries().create(name=project_name, body=body).execute() 349 | ) 350 | logging.debug( 351 | "response is {}".format(json.dumps(metrics, sort_keys=True, indent=4)) 352 | ) 353 | 354 | body["timeSeries"][0]["metric"][ 355 | "type" 356 | ] = "custom.googleapis.com/stackdriver-monitoring-export/metrics-from-api" 357 | body["timeSeries"][0]["points"][0]["value"]["int64Value"] = stats[ 358 | "metrics_count_from_api" 359 | ] 360 | metrics = ( 361 | service.projects().timeSeries().create(name=project_name, body=body).execute() 362 | ) 363 | logging.debug( 364 | "response is {}".format(json.dumps(metrics, sort_keys=True, indent=4)) 365 | ) 366 | 367 | 368 | def build_bigquery_stats_message(metric, metadata): 369 | processing_end_time = datetime.now() 370 | processing_end_time_str = processing_end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 371 | 372 | # Write the stats to the BigQuery stats tabledata 373 | bq_msg = { 374 | "app_name": "list_metrics", 375 | "batch_id": metadata["batch_id"], 376 | "message_id": metadata["message_id"], 377 | # "src_message_id": src_message_id, 378 | "metric_type": metric["metric"]["type"], 379 | "error_msg_cnt": metadata["error_msg_cnt"], 380 | "msg_written_cnt": metadata["msg_written_cnt"], 381 | "msg_without_timeseries": metadata["msg_without_timeseries"], 382 | "payload": metadata["payload"], 383 | "batch_start_time": metadata["batch_start_time"], 384 | "processing_end_time": processing_end_time_str, 385 | } 386 | json_msg = {"json": bq_msg} 387 | # logging.debug("json_msg {}".format(json.dumps(json_msg, sort_keys=True, indent=4))) 388 | return json_msg 389 | 390 | 391 | def write_to_bigquery(json_row_list): 392 | """Write rows to the BigQuery stats table using the googleapiclient and the streaming insertAll method 393 | https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll 394 | """ 395 | # logging.debug("write_to_bigquery") 396 | 397 | if len(json_row_list) > 0: 398 | bigquery = build("bigquery", "v2", cache_discovery=True) 399 | 400 | body = { 401 | "kind": "bigquery#tableDataInsertAllRequest", 402 | "skipInvalidRows": "false", 403 | "rows": json_row_list, 404 | } 405 | # logging.debug('body: {}'.format(json.dumps(body, sort_keys=True, indent=4))) 406 | 407 | response = ( 408 | bigquery.tabledata() 409 | .insertAll( 410 | projectId=app_identity.get_application_id(), 411 | datasetId=config.BIGQUERY_DATASET, 412 | tableId=config.BIGQUERY_STATS_TABLE, 413 | body=body, 414 | ) 415 | .execute() 416 | ) 417 | # logging.debug("BigQuery said... = {}".format(response)) 418 | 419 | bq_msgs_with_errors = 0 420 | if "insertErrors" in response: 421 | if len(response["insertErrors"]) > 0: 422 | logging.error("Error: {}".format(response)) 423 | bq_msgs_with_errors = len(response["insertErrors"]) 424 | else: 425 | logging.debug( 426 | "By amazing luck, there are no errors, response = {}".format(response) 427 | ) 428 | logging.debug("bq_msgs_written: {}".format(bq_msgs_with_errors)) 429 | return response 430 | else: 431 | logging.debug("No BigQuery records to write") 432 | return None 433 | 434 | 435 | def write_input_parameters_to_bigquery(project_id, metadata, msg): 436 | """Write rows to the BigQuery input parameters table using the 437 | googleapiclient and the streaming insertAll method 438 | https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll 439 | """ 440 | # logging.debug("write_input_parameters_to_bigquery") 441 | 442 | bigquery = build("bigquery", "v2", cache_discovery=True) 443 | 444 | body = { 445 | "kind": "bigquery#tableDataInsertAllRequest", 446 | "skipInvalidRows": "false", 447 | "rows": [ 448 | { 449 | "json": { 450 | "start_time": msg["start_time"], 451 | "end_time": msg["end_time"], 452 | "aggregation_alignment_period": msg["aggregation_alignment_period"], 453 | "message_id": metadata["message_id"], 454 | "project_list": {"project_id": [project_id]}, 455 | "batch_id": metadata["batch_id"], 456 | "batch_start_time": metadata["batch_start_time"], 457 | } 458 | } 459 | ], 460 | } 461 | # logging.debug('body: {}'.format(json.dumps(body, sort_keys=True, indent=4))) 462 | 463 | response = ( 464 | bigquery.tabledata() 465 | .insertAll( 466 | projectId=app_identity.get_application_id(), 467 | datasetId=config.BIGQUERY_DATASET, 468 | tableId=config.BIGQUERY_PARAMS_TABLE, 469 | body=body, 470 | ) 471 | .execute() 472 | ) 473 | # logging.debug("BigQuery said... = {}".format(response)) 474 | 475 | bq_msgs_with_errors = 0 476 | if "insertErrors" in response: 477 | if len(response["insertErrors"]) > 0: 478 | logging.error("Error: {}".format(response)) 479 | bq_msgs_with_errors = len(response["insertErrors"]) 480 | else: 481 | logging.debug( 482 | "By amazing luck, there are no errors, response = {}".format(response) 483 | ) 484 | logging.debug("bq_msgs_written: {}".format(bq_msgs_with_errors)) 485 | return response 486 | 487 | 488 | @app.route("/push-handlers/receive_messages", methods=["POST"]) 489 | def receive_messages_handler(): 490 | """Handle the Pub/Sub push messages 491 | Validate the input and then process the message 492 | """ 493 | logging.debug("received message") 494 | ret_val = "" 495 | ret_code = 200 496 | try: 497 | if not request.data: 498 | raise ValueError("No request data received") 499 | envelope = json.loads(request.data.decode("utf-8")) 500 | logging.debug("Raw pub/sub message: {}".format(envelope)) 501 | if "message" not in envelope: 502 | raise ValueError("No message in envelope") 503 | if "messageId" in envelope["message"]: 504 | logging.debug("messageId: {}".format(envelope["message"]["messageId"])) 505 | message_id = envelope["message"].get("messageId", "") 506 | if "publishTime" in envelope["message"]: 507 | publish_time = envelope["message"]["publishTime"] 508 | if "data" not in envelope["message"]: 509 | raise ValueError("No data in message") 510 | payload = base64.b64decode(envelope["message"]["data"]) 511 | logging.debug("payload: {} ".format(payload)) 512 | data = json.loads(payload.decode("utf-8")) 513 | logging.debug("data: {} ".format(data)) 514 | # Add any of the parameters to the pubsub message to send 515 | message_to_publish = {} 516 | # if the pubsub PUBSUB_VERIFICATION_TOKEN isn't included or doesn't match, don't continue 517 | if "token" not in data: 518 | raise ValueError("token missing from request") 519 | if not data["token"] == config.PUBSUB_VERIFICATION_TOKEN: 520 | raise ValueError( 521 | "token from request doesn't match, received: {}".format(data["token"]) 522 | ) 523 | # if the project has been passed in, use that, otherwise use default project of App Engine app 524 | if "project_id" not in data: 525 | project_id = project_id = app_identity.get_application_id() 526 | else: 527 | project_id = data["project_id"] 528 | message_to_publish["project_id"] = project_id 529 | # if the alignment_period is supplied, use that, otherwise use default 530 | if "aggregation_alignment_period" not in data: 531 | aggregation_alignment_period = config.AGGREGATION_ALIGNMENT_PERIOD 532 | else: 533 | aggregation_alignment_period = data["aggregation_alignment_period"] 534 | pattern = re.compile(r"^\d{1,}s+$") 535 | matched = pattern.match(aggregation_alignment_period) 536 | if not matched: 537 | raise ValueError( 538 | "aggregation_alignment_period needs to be digits followed by an 's' such as 3600s, received: {}".format( 539 | aggregation_alignment_period 540 | ) 541 | ) 542 | alignment_seconds = int( 543 | aggregation_alignment_period[: len(aggregation_alignment_period) - 1] 544 | ) 545 | if alignment_seconds < 60: 546 | raise ValueError( 547 | "aggregation_alignment_period needs to be more than 60s, received: {}".format( 548 | aggregation_alignment_period 549 | ) 550 | ) 551 | message_to_publish[ 552 | "aggregation_alignment_period" 553 | ] = aggregation_alignment_period 554 | # get the App Engine default bucket name to store a GCS file with last end_time 555 | bucket_name = os.environ.get( 556 | "BUCKET_NAME", app_identity.get_default_gcs_bucket_name() 557 | ) 558 | # Calculate the end_time first 559 | if "end_time" not in data: 560 | # the end_time should be set here for all metrics in the batch 561 | # setting later in the architecture would mean that the end_time may vary 562 | end_time = datetime.now() 563 | end_time_str = end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 564 | else: 565 | end_time_str = data["end_time"] 566 | matched = check_date_format(end_time_str) 567 | if not matched: 568 | raise ValueError( 569 | "end_time needs to be in the format 2019-02-08T14:00:00.311635Z, received: {}".format( 570 | end_time_str 571 | ) 572 | ) 573 | message_to_publish["end_time"] = end_time_str 574 | # if the start_time is supplied, use the previous end_time 575 | sent_in_start_time_flag = False 576 | if "start_time" not in data: 577 | start_time_str = get_last_end_time(project_id, bucket_name) 578 | # if the file hasn't been found, then start 1 alignment period in the past 579 | if not start_time_str: 580 | start_time_str = set_last_end_time( 581 | project_id, bucket_name, end_time_str, (alignment_seconds * -1) 582 | ) 583 | # raise ValueError("start_time couldn't be read from GCS, received: {}".format(start_time_str)) 584 | logging.debug( 585 | "start_time_str: {}, end_time_str: {}".format( 586 | start_time_str, end_time_str 587 | ) 588 | ) 589 | else: 590 | sent_in_start_time_flag = True 591 | start_time_str = data["start_time"] 592 | matched = check_date_format(start_time_str) 593 | if not matched: 594 | raise ValueError( 595 | "start_time needs to be in the format 2019-02-08T14:00:00.311635Z, received: {}".format( 596 | start_time_str 597 | ) 598 | ) 599 | message_to_publish["start_time"] = start_time_str 600 | # Create a unique identifier for this batch 601 | batch_id = get_batch_id() 602 | logging.debug("batch_id: {}".format(batch_id)) 603 | # Publish the messages to Pub/Sub 604 | logging.info( 605 | "Running with input parameters - {}".format( 606 | json.dumps(message_to_publish, sort_keys=True, indent=4) 607 | ) 608 | ) 609 | metadata = { 610 | "batch_id": batch_id, 611 | "message_id": message_id, 612 | "batch_start_time": publish_time, 613 | } 614 | if config.WRITE_BQ_STATS_FLAG: 615 | write_input_parameters_to_bigquery(project_id, metadata, message_to_publish) 616 | stats = get_and_publish_metrics(message_to_publish, metadata) 617 | logging.debug("Stats are {}".format(json.dumps(stats))) 618 | """ Write the late end_time_str to GCS to use in a subsequent run, 619 | but only if the start_time was not sent in. If the start_time is 620 | supplied, then we consider that an ad hoc run, and won't set the 621 | previous end_time 622 | """ 623 | if not sent_in_start_time_flag: 624 | set_last_end_time(project_id, bucket_name, end_time_str, 1) 625 | # Write the stats to custom monitoring metrics 626 | if config.WRITE_MONITORING_STATS_FLAG: 627 | write_stats(stats, project_id, batch_id) 628 | ret_val = str(stats) 629 | except Exception as e: 630 | logging.error("Error: {}".format(e)) 631 | ret_val = str(e) 632 | ret_code = 500 633 | return Response(ret_val, status=ret_code, mimetype="text/plain") 634 | -------------------------------------------------------------------------------- /list_metrics/main_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import pytest 18 | from main import app as flask_app 19 | 20 | import main 21 | import config 22 | import base64 23 | import json 24 | 25 | 26 | @pytest.fixture 27 | def app(): 28 | yield flask_app 29 | 30 | 31 | @pytest.fixture 32 | def client(app): 33 | return app.test_client() 34 | 35 | 36 | def test_check_date_format(): 37 | """Test the check_date_format function""" 38 | results = main.check_date_format("23232") 39 | assert results is None 40 | results = main.check_date_format("2019-02-08T14:00:00.311635Z") 41 | assert results is not None 42 | 43 | 44 | def test_post_empty_data(app, client): 45 | """Test sending an empty message""" 46 | response = client.post("/push-handlers/receive_messages") 47 | assert response.status_code == 500 48 | assert response.get_data(as_text=True) == "No request data received" 49 | 50 | 51 | def test_incorrect_aggregation_alignment_period_post(app, client): 52 | mimetype = "application/json" 53 | headers = { 54 | "Content-Type": mimetype, 55 | "Accept": mimetype, 56 | } 57 | """Test sending incorrect aggregation_alignment_period as input""" 58 | request = build_request(aggregation_alignment_period="12") 59 | response = client.post( 60 | "/push-handlers/receive_messages", data=json.dumps(request).encode("utf-8"), headers=headers 61 | ) 62 | assert response.status_code == 500 63 | assert ( 64 | response.get_data(as_text=True) 65 | == "aggregation_alignment_period needs to be digits followed by an 's' such as 3600s, received: 12" 66 | ) 67 | 68 | request = build_request(aggregation_alignment_period="12s") 69 | response = client.post( 70 | "/push-handlers/receive_messages", data=json.dumps(request).encode("utf-8"), headers=headers 71 | ) 72 | assert response.status_code == 500 73 | assert ( 74 | response.get_data(as_text=True) 75 | == "aggregation_alignment_period needs to be more than 60s, received: 12s", 76 | ) 77 | 78 | 79 | def test_exclusions_check(): 80 | """Test the exclusion logic""" 81 | assert ( 82 | main.check_exclusions({"type": "aws.googleapis.com/flex/cpu/utilization"}) 83 | == False 84 | ), "This should be excluded" 85 | assert ( 86 | main.check_exclusions({"type": "appengine.googleapis.com/flex/cpu/utilization"}) 87 | == True 88 | ), "This should not be excluded" 89 | 90 | 91 | def test_incorrect_token_post(app, client): 92 | """Test sending an incorrect token""" 93 | request = build_request(token="incorrect_token") 94 | mimetype = "application/json" 95 | headers = { 96 | "Content-Type": mimetype, 97 | "Accept": mimetype, 98 | } 99 | response = client.post("/push-handlers/receive_messages", data=json.dumps(request), headers=headers) 100 | assert response.status_code == 500 101 | 102 | 103 | def build_request( 104 | token=config.PUBSUB_VERIFICATION_TOKEN, 105 | aggregation_alignment_period="3600s", 106 | ): 107 | """Build a Pub/Sub message as input""" 108 | 109 | payload = { 110 | "token": token, 111 | "aggregation_alignment_period": aggregation_alignment_period, 112 | } 113 | request = { 114 | "message": { 115 | "data": base64.b64encode(json.dumps(payload).encode("utf-8")).decode() 116 | } 117 | } 118 | return request 119 | -------------------------------------------------------------------------------- /list_metrics/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | gunicorn 3 | google-cloud-storage 4 | appengine-python-standard>=1.0.0 5 | google-api-python-client -------------------------------------------------------------------------------- /list_projects/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "PROJECTS_TOPIC" : "projects/YOUR_PROJECT_ID/topics/metrics_export_start", 3 | "TOKEN": "99a9ffa8797a629783cb4aa762639e92b098bac5", 4 | "ACTIVE": "ACTIVE", 5 | "METRIC_EXPORT_PUBSUB_VERIFICATION_TOKEN": "16b2ecfb-7734-48b9-817d-4ac8bd623c87" 6 | } -------------------------------------------------------------------------------- /list_projects/index.js: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * Copyright 2019, Google, Inc. 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | // Based on https://github.com/GoogleCloudPlatform/gcf-gce-usage-monitoring/ 17 | 18 | 'use strict'; 19 | const config = require('./config.json'); 20 | 21 | // Use the Pub/Sub nodejs client library 22 | // https://googleapis.dev/nodejs/pubsub/latest/index.html#reference 23 | const {PubSub} = require('@google-cloud/pubsub'); 24 | const pubsub = new PubSub(); 25 | 26 | // Use the Resource Manae nodejs client library 27 | // https://googleapis.dev/nodejs/resource/latest/index.html#reference 28 | const {Resource} = require('@google-cloud/resource'); 29 | const resource = new Resource(); 30 | 31 | // [START functions_getProjects] 32 | /** 33 | * Calls the Cloud Resource Manager API to get a list of projects. Writes a 34 | * Pub/Sub message for each project 35 | */ 36 | async function getProjects() { 37 | 38 | try { 39 | // Lists all current projects 40 | const [projects] = await resource.getProjects(); 41 | 42 | console.log(`Got past getProjects() call`); 43 | 44 | // Set a uniform endTime for all the resulting messages 45 | const endTime = new Date(); 46 | const endTimeStr = endTime.toISOString(); 47 | 48 | // sample 2019-11-12T17:58:26.068483Z 49 | 50 | for (var i=0; i { 86 | console.log(`messageId: ${context.eventId}`); 87 | const data = Buffer.from(pubSubEvent.data, 'base64').toString(); 88 | var jsonMessage = ""; 89 | try { 90 | jsonMessage = JSON.parse(data); 91 | } catch(err) { 92 | console.error(`Error parsing input message: ${data}`); 93 | console.error(err); 94 | throw err; 95 | } 96 | if ("token" in jsonMessage) { 97 | const token = jsonMessage["token"]; 98 | if (token === config.TOKEN){ 99 | return getProjects(); 100 | } else { 101 | const err = new Error("The token property in the pubsub message does not match, not processing"); 102 | console.error(err); 103 | throw err; 104 | } 105 | } else { 106 | const err = new Error("No token property in the pubsub message, not processing"); 107 | console.error(err); 108 | throw err; 109 | } 110 | }; 111 | // [END functions_list_projects] 112 | -------------------------------------------------------------------------------- /list_projects/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloud-functions-stackdriver-metric-export-nodejs", 3 | "version": "0.0.1", 4 | "private": true, 5 | "license": "Apache-2.0", 6 | "author": "Google Inc.", 7 | "engines": { 8 | "node": ">=4.3.2" 9 | }, 10 | "scripts": { 11 | "test": "mocha test/*.test.js --timeout=60000 --exit" 12 | }, 13 | "dependencies": { 14 | "@google-cloud/compute": "^3.9.1", 15 | "@google-cloud/monitoring": "^3.0.4", 16 | "@google-cloud/pubsub": "^3.5.0", 17 | "@google-cloud/resource": "^1.1.2" 18 | }, 19 | "devDependencies": { 20 | "assert": "^2.0.0", 21 | "child_process": "^1.0.2", 22 | "mocha": "^10.2.0" 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /list_projects/test/index.test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2019, Google, Inc. 3 | * Licensed under the Apache License, Version 2.0 (the "License"); 4 | * you may not use this file except in compliance with the License. 5 | * You may obtain a copy of the License at 6 | * 7 | * http://www.apache.org/licenses/LICENSE-2.0 8 | * 9 | * Unless required by applicable law or agreed to in writing, software 10 | * distributed under the License is distributed on an "AS IS" BASIS, 11 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | * See the License for the specific language governing permissions and 13 | * limitations under the License. 14 | */ 15 | 16 | // [START functions_pubsub_integration_test] 17 | const childProcess = require('child_process'); 18 | const assert = require('assert'); 19 | 20 | it('list_projects: should print the messageId of the Pub/Sub message', done => { 21 | 22 | const response = childProcess 23 | .execSync("export TOKEN_ID=$TOKEN && test/test_send_pubsub_msg.sh") 24 | .toString(); 25 | 26 | // Grab the messageId that was created when sending the Pub/Sub message above 27 | const firstIndex = response.indexOf("'")+1; 28 | const lastIndex = response.lastIndexOf("'"); 29 | const messageId = response.slice(firstIndex, lastIndex); 30 | console.log("messageId: "+messageId); 31 | 32 | // Wait for the Cloud Function logs to be available 33 | childProcess 34 | .execSync("sleep 20"); 35 | 36 | // Check the Cloud Function logs 37 | const logs = childProcess 38 | .execSync(`gcloud beta functions logs read list_projects --execution-id="${messageId}"`) 39 | .toString(); 40 | console.log("logs are: "+logs); 41 | 42 | // Check to see that the message has been received 43 | assert.strictEqual(logs.includes(`${messageId}`), true); 44 | 45 | // Check to see that the messageId is printed 46 | assert.strictEqual(logs.includes(`messageId`), true); 47 | 48 | // Check to see that at least 1 Pub/Sub message has been sent 49 | assert.strictEqual(logs.includes(`Published pubsub message`), true); 50 | assert.strictEqual(logs.includes(`finished with status: 'ok'`), true); 51 | done(); 52 | }); 53 | 54 | it('list_projects: should print the error message when no input is sent in Pub/Sub trigger message ', done => { 55 | 56 | const response = childProcess 57 | .execSync("test/test_send_empty_pubsub_msg.sh") 58 | .toString(); 59 | 60 | // Grab the messageId that was created when sending the Pub/Sub message above 61 | const firstIndex = response.indexOf("'")+1; 62 | const lastIndex = response.lastIndexOf("'"); 63 | const messageId = response.slice(firstIndex, lastIndex); 64 | console.log("messageId: "+messageId); 65 | 66 | // Wait for the Cloud Function logs to be available 67 | childProcess 68 | .execSync("sleep 20"); 69 | 70 | // Check the Cloud Function logs 71 | const logs = childProcess 72 | .execSync(`gcloud beta functions logs read list_projects --execution-id="${messageId}"`) 73 | .toString(); 74 | console.log("logs are: "+logs); 75 | 76 | // Check to see that the message has been received 77 | assert.strictEqual(logs.includes(`${messageId}`), true); 78 | 79 | // Check to see that the messageId is printed 80 | assert.strictEqual(logs.includes(`Error: No token property in the pubsub message`), true); 81 | assert.strictEqual(logs.includes(`finished with status: 'error'`), true); 82 | 83 | done(); 84 | }); 85 | 86 | it('list_projects: should print the error message when token doesnt match config ', done => { 87 | 88 | const response = childProcess 89 | .execSync("test/test_send_incorrect_token_pubsub_msg.sh") 90 | .toString(); 91 | 92 | // Grab the messageId that was created when sending the Pub/Sub message above 93 | const firstIndex = response.indexOf("'")+1; 94 | const lastIndex = response.lastIndexOf("'"); 95 | const messageId = response.slice(firstIndex, lastIndex); 96 | console.log("messageId: "+messageId); 97 | 98 | // Wait for the Cloud Function logs to be available 99 | childProcess 100 | .execSync("sleep 20"); 101 | 102 | // Check the Cloud Function logs 103 | const logs = childProcess 104 | .execSync(`gcloud beta functions logs read list_projects --execution-id="${messageId}"`) 105 | .toString(); 106 | console.log("logs are: "+logs); 107 | 108 | // Check to see that the message has been received 109 | assert.strictEqual(logs.includes(`${messageId}`), true); 110 | 111 | // Check to see that the messageId is printed 112 | assert.strictEqual(logs.includes(`Error: The token property in the pubsub message does not match`), true); 113 | assert.strictEqual(logs.includes(`finished with status: 'error'`), true); 114 | 115 | done(); 116 | }); -------------------------------------------------------------------------------- /list_projects/test/test_send_empty_pubsub_msg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | gcloud pubsub topics publish gce_footprint_start --message="{}" -------------------------------------------------------------------------------- /list_projects/test/test_send_incorrect_token_pubsub_msg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | gcloud pubsub topics publish gce_footprint_start --message="{\"token\":\"123\"}" -------------------------------------------------------------------------------- /list_projects/test/test_send_pubsub_msg.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # Copyright 2019 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # https://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | gcloud pubsub topics publish gce_footprint_start --message="{\"token\":\"$TOKEN\"}" -------------------------------------------------------------------------------- /sample_messages/bigquery_query_1.sql: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # [START metric_export_bigquery_sql_1] 16 | SELECT 17 | metric.type AS metric_type, 18 | EXTRACT(DATE FROM point.INTERVAL.start_time) AS extract_date, 19 | MAX(point.value.distribution_value.mean) AS max_mean, 20 | MIN(point.value.distribution_value.mean) AS min_mean, 21 | AVG(point.value.distribution_value.mean) AS avg_mean 22 | FROM 23 | `sage-facet-201016.metric_export.sd_metrics_export` 24 | CROSS JOIN 25 | UNNEST(resource.labels) AS resource_labels 26 | WHERE 27 | point.interval.end_time > TIMESTAMP(DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)) 28 | AND point.interval.end_time <= CURRENT_TIMESTAMP 29 | AND metric.type = 'appengine.googleapis.com/http/server/response_latencies' 30 | AND resource_labels.key = "project_id" 31 | AND resource_labels.value = "sage-facet-201016" 32 | GROUP BY 33 | metric_type, 34 | extract_date 35 | ORDER BY 36 | extract_date 37 | # [END metric_export_bigquery_sql_1] 38 | -------------------------------------------------------------------------------- /sample_messages/bigquery_query_2.sql: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # [START metric_export_bigquery_sql_2] 16 | SELECT 17 | EXTRACT(DATE FROM point.interval.end_time) AS extract_date, 18 | sum(point.value.int64_value) as query_cnt 19 | FROM 20 | `sage-facet-201016.metric_export.sd_metrics_export` 21 | CROSS JOIN 22 | UNNEST(resource.labels) AS resource_labels 23 | WHERE 24 | point.interval.end_time > TIMESTAMP(DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)) 25 | AND point.interval.end_time <= CURRENT_TIMESTAMP 26 | and metric.type = 'bigquery.googleapis.com/query/count' 27 | AND resource_labels.key = "project_id" 28 | AND resource_labels.value = "sage-facet-201016" 29 | group by extract_date 30 | order by extract_date 31 | # [END metric_export_bigquery_sql_2] -------------------------------------------------------------------------------- /sample_messages/bigquery_query_3.sql: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # [START metric_export_bigquery_sql_3] 16 | SELECT 17 | EXTRACT(WEEK FROM point.interval.end_time) AS extract_date, 18 | min(point.value.double_value) as min_cpu_util, 19 | max(point.value.double_value) as max_cpu_util, 20 | avg(point.value.double_value) as avg_cpu_util 21 | FROM 22 | `sage-facet-201016.metric_export.sd_metrics_export` 23 | WHERE 24 | point.interval.end_time > TIMESTAMP(DATE_SUB(CURRENT_DATE, INTERVAL 30 DAY)) 25 | AND point.interval.end_time <= CURRENT_TIMESTAMP 26 | AND metric.type = 'compute.googleapis.com/instance/cpu/utilization' 27 | group by extract_date 28 | order by extract_date 29 | # [END metric_export_bigquery_sql_3] -------------------------------------------------------------------------------- /sample_messages/metric_descriptor.json: -------------------------------------------------------------------------------- 1 | { 2 | "metricDescriptors": [ 3 | { 4 | "name": "projects/sage-facet-201016/metricDescriptors/pubsub.googleapis.com/subscription/push_request_count", 5 | "labels": [ 6 | { 7 | "key": "response_class", 8 | "description": "A classification group for the response code. It can be one of ['ack', 'deadline_exceeded', 'internal', 'invalid', 'remote_server_4xx', 'remote_server_5xx', 'unreachable']." 9 | }, 10 | { 11 | "key": "response_code", 12 | "description": "Operation response code string, derived as a string representation of a status code (e.g., 'success', 'not_found', 'unavailable')." 13 | }, 14 | { 15 | "key": "delivery_type", 16 | "description": "Push delivery mechanism." 17 | } 18 | ], 19 | "metricKind": "DELTA", 20 | "valueType": "INT64", 21 | "unit": "1", 22 | "description": "Cumulative count of push attempts, grouped by result. Unlike pulls, the push server implementation does not batch user messages. So each request only contains one user message. The push server retries on errors, so a given user message can appear multiple times.", 23 | "displayName": "Push requests", 24 | "type": "pubsub.googleapis.com/subscription/push_request_count", 25 | "metadata": { 26 | "launchStage": "GA", 27 | "samplePeriod": "60s", 28 | "ingestDelay": "120s" 29 | } 30 | } 31 | ] 32 | } 33 | -------------------------------------------------------------------------------- /sample_messages/timeseries_response_align.json: -------------------------------------------------------------------------------- 1 | { 2 | "timeSeries": [ 3 | { 4 | "metric": { 5 | "labels": { 6 | "delivery_type": "gae", 7 | "response_class": "ack", 8 | "response_code": "success" 9 | }, 10 | "type": "pubsub.googleapis.com/subscription/push_request_count" 11 | }, 12 | "metricKind": "DELTA", 13 | "points": [ 14 | { 15 | "interval": { 16 | "endTime": "2019-02-19T21:00:00.829121Z", 17 | "startTime": "2019-02-19T20:00:00.829121Z" 18 | }, 19 | "value": { 20 | "int64Value": "1" 21 | } 22 | } 23 | ], 24 | "resource": { 25 | "labels": { 26 | "project_id": "sage-facet-201016", 27 | "subscription_id": "metric_export_init_pub" 28 | }, 29 | "type": "pubsub_subscription" 30 | }, 31 | "valueType": "INT64" 32 | }, 33 | { 34 | "metric": { 35 | "labels": { 36 | "delivery_type": "gae", 37 | "response_class": "ack", 38 | "response_code": "success" 39 | }, 40 | "type": "pubsub.googleapis.com/subscription/push_request_count" 41 | }, 42 | "metricKind": "DELTA", 43 | "points": [ 44 | { 45 | "interval": { 46 | "endTime": "2019-02-19T21:00:00.829121Z", 47 | "startTime": "2019-02-19T20:00:00.829121Z" 48 | }, 49 | "value": { 50 | "int64Value": "803" 51 | } 52 | } 53 | ], 54 | "resource": { 55 | "labels": { 56 | "project_id": "sage-facet-201016", 57 | "subscription_id": "metrics_list" 58 | }, 59 | "type": "pubsub_subscription" 60 | }, 61 | "valueType": "INT64" 62 | } 63 | ] 64 | } -------------------------------------------------------------------------------- /sample_messages/timeseries_response_align_and_reduce.json: -------------------------------------------------------------------------------- 1 | { 2 | "timeSeries": [ 3 | { 4 | "metric": { 5 | "type": "pubsub.googleapis.com/subscription/push_request_count" 6 | }, 7 | "resource": { 8 | "type": "pubsub_subscription", 9 | "labels": { 10 | "project_id": "sage-facet-201016" 11 | } 12 | }, 13 | "metricKind": "DELTA", 14 | "valueType": "INT64", 15 | "points": [ 16 | { 17 | "interval": { 18 | "startTime": "2019-02-08T14:00:00.311635Z", 19 | "endTime": "2019-02-08T15:00:00.311635Z" 20 | }, 21 | "value": { 22 | "int64Value": "788" 23 | } 24 | } 25 | ] 26 | } 27 | ] 28 | } 29 | -------------------------------------------------------------------------------- /test/.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | -------------------------------------------------------------------------------- /test/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | BATCH_ID="R8BK5S99QU4ZZOGCR1UDPWVH6LPKI5QU" 17 | PROJECT_ID="YOUR_PROJECT_ID" -------------------------------------------------------------------------------- /test/end_to_end_test_run.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | import json 19 | from googleapiclient.discovery import build 20 | import unittest 21 | import config 22 | 23 | class AppTest(unittest.TestCase): 24 | 25 | def setUp(self): 26 | """ Set-up the app 27 | """ 28 | self.batch_id = config.BATCH_ID 29 | self.project_id = config.PROJECT_ID 30 | 31 | def get_list_metrics_output_stats(self, batch_id): 32 | query = "SELECT count(message_id)" \ 33 | "FROM " \ 34 | "`metric_export.sd_metrics_stats` " \ 35 | "WHERE " \ 36 | "batch_id=\"{}\" AND app_name=\"list_metrics\" AND msg_written_cnt=1".format(batch_id) 37 | 38 | response = self.query_bigquery(query) 39 | job_ref = response["jobReference"] 40 | results = self.get_query_results_bigquery(job_ref) 41 | return results 42 | 43 | def get_list_metrics_num_full_recs(self, batch_id): 44 | query = "SELECT count(message_id)" \ 45 | "FROM " \ 46 | "`metric_export.sd_metrics_stats` " \ 47 | "WHERE " \ 48 | "batch_id=\"{}\" AND app_name=\"list_metrics\"".format(batch_id) 49 | 50 | response = self.query_bigquery(query) 51 | job_ref = response["jobReference"] 52 | results = self.get_query_results_bigquery(job_ref) 53 | return results 54 | 55 | def get_get_timeseries_output_stats(self, batch_id): 56 | query = "SELECT count(t.message_id) "\ 57 | "FROM (SELECT message_id " \ 58 | "FROM " \ 59 | "`metric_export.sd_metrics_stats` " \ 60 | "WHERE " \ 61 | "batch_id=\"{}\" AND app_name=\"get_timeseries\"" \ 62 | "group by message_id) t ".format(batch_id) 63 | 64 | response = self.query_bigquery(query) 65 | job_ref = response["jobReference"] 66 | results = self.get_query_results_bigquery(job_ref) 67 | return results 68 | 69 | def get_stats(self, batch_id, app_name): 70 | query = "SELECT count(message_id)" \ 71 | "FROM " \ 72 | "`metric_export.sd_metrics_stats` " \ 73 | "WHERE " \ 74 | "batch_id=\"{}\" AND app_name=\"{}\" AND msg_written_cnt>=1".format(batch_id, app_name) 75 | 76 | response = self.query_bigquery(query) 77 | job_ref = response["jobReference"] 78 | results = self.get_query_results_bigquery(job_ref) 79 | return results 80 | 81 | def get_metrics_export_cnt(self, batch_id): 82 | query = "SELECT count(metric)" \ 83 | "FROM " \ 84 | "`metric_export.sd_metrics_export_fin` " \ 85 | "WHERE " \ 86 | "batch_id=\"{}\"".format(batch_id) 87 | 88 | response = self.query_bigquery(query) 89 | job_ref = response["jobReference"] 90 | results = self.get_query_results_bigquery(job_ref) 91 | return results 92 | 93 | def get_write_metrics_sum_recs_written(self, batch_id, app_name): 94 | query = "SELECT sum(msg_written_cnt)" \ 95 | "FROM " \ 96 | "`metric_export.sd_metrics_stats` " \ 97 | "WHERE " \ 98 | "batch_id=\"{}\" AND app_name=\"{}\"".format(batch_id, app_name) 99 | 100 | response = self.query_bigquery(query) 101 | job_ref = response["jobReference"] 102 | results = self.get_query_results_bigquery(job_ref) 103 | return results 104 | 105 | def query_bigquery(self, query): 106 | 107 | bigquery = build('bigquery', 'v2',cache_discovery=True) 108 | 109 | body = { 110 | "query": query, 111 | "useLegacySql": "false" 112 | } 113 | logging.debug('body: {}'.format(json.dumps(body,sort_keys=True, indent=4))) 114 | print('body: {}'.format(json.dumps(body,sort_keys=True, indent=4))) 115 | response = bigquery.jobs().query( 116 | projectId=self.project_id, 117 | body=body 118 | ).execute() 119 | 120 | logging.debug("BigQuery said... = {}".format(response)) 121 | print("BigQuery said... = {}".format(response)) 122 | 123 | return response 124 | 125 | def get_query_results_bigquery(self, job_ref): 126 | bigquery = build('bigquery', 'v2', cache_discovery=True) 127 | 128 | response = bigquery.jobs().getQueryResults( 129 | projectId=self.project_id, 130 | jobId=job_ref["jobId"] 131 | ).execute() 132 | 133 | logging.debug("BigQuery said... = {}".format(response)) 134 | print("BigQuery said... = {}".format(json.dumps(response,sort_keys=True, indent=4))) 135 | 136 | return response 137 | 138 | def get_metric_descriptors_cnt(self): 139 | metrics_count_from_api = 0 140 | 141 | next_page_token = "" 142 | while True: 143 | metric_list = self.get_metrics(next_page_token) 144 | metrics_count_from_api += len(metric_list['metricDescriptors']) 145 | 146 | if "nextPageToken" in metric_list: 147 | next_page_token = metric_list["nextPageToken"] 148 | else: 149 | break 150 | return metrics_count_from_api 151 | 152 | def get_metrics(self, next_page_token, filter_str=""): 153 | service = build('monitoring', 'v3', cache_discovery=True) 154 | project_name = 'projects/{project_id}'.format( 155 | project_id=self.project_id 156 | ) 157 | 158 | metrics = service.projects().metricDescriptors().list( 159 | name=project_name, 160 | pageToken=next_page_token, 161 | filter=filter_str 162 | ).execute() 163 | 164 | logging.debug("response is {}".format(json.dumps(metrics, sort_keys=True, indent=4))) 165 | return metrics 166 | 167 | def test_1(self): 168 | """ 169 | test 1. The number of messages written to pubsub from list_metrics 170 | should match the number of messages received by get_timeseries 171 | """ 172 | response = self.get_list_metrics_output_stats(self.batch_id) 173 | # print "response: {}".format(response) 174 | list_metric_row_cnt = response["rows"][0]["f"][0]["v"] 175 | print("list_metric_row_cnt: {}".format(list_metric_row_cnt)) 176 | 177 | response = self.get_get_timeseries_output_stats(self.batch_id) 178 | # print "response: {}".format(response) 179 | get_timeseries_row_cnt = response["rows"][0]["f"][0]["v"] 180 | print("get_timeseries_row_cnt: {}".format(get_timeseries_row_cnt)) 181 | 182 | assert list_metric_row_cnt == get_timeseries_row_cnt, \ 183 | "Failed #1: The # of records written from list_metrics doesn't " \ 184 | "match the # of records received by get_timeseries" 185 | 186 | def test_2(self): 187 | """ 188 | test 2. The number of messages written to pubsub from get_timeseries 189 | should match the number of messages received by write_metrics 190 | """ 191 | response = self.get_stats(self.batch_id,"get_timeseries") 192 | # print "response: {}".format(response) 193 | get_timeseries_row_cnt = response["rows"][0]["f"][0]["v"] 194 | print("get_timeseries_row_cnt: {}".format(get_timeseries_row_cnt)) 195 | 196 | response = self.get_stats(self.batch_id,"write_metrics") 197 | # print "response: {}".format(response) 198 | write_metrics_row_cnt = response["rows"][0]["f"][0]["v"] 199 | print("write_metrics_row_cnt: {}".format(write_metrics_row_cnt)) 200 | 201 | assert get_timeseries_row_cnt == write_metrics_row_cnt, \ 202 | "Failed #2: The # of records written from get_timeseries doesn't " \ 203 | "match the # of records received by write_metrics "\ 204 | "write_metrics_row_cnt:{}, get_timeseries_row_cnt:{}"\ 205 | .format(write_metrics_row_cnt, get_timeseries_row_cnt) 206 | 207 | def test_3(self): 208 | """ 209 | test 3. The number of messages written to BigQuery from write_metrics 210 | should match the actual number of messages in the BigQuery table 211 | """ 212 | response = self.get_write_metrics_sum_recs_written(self.batch_id,"write_metrics") 213 | # print "response: {}".format(response) 214 | write_metrics_row_cnt = response["rows"][0]["f"][0]["v"] 215 | if write_metrics_row_cnt is None: 216 | write_metrics_row_cnt = 0 217 | else: 218 | write_metrics_row_cnt = int(write_metrics_row_cnt) 219 | print("write_metrics_row_cnt: {}".format(write_metrics_row_cnt)) 220 | 221 | response = self.get_metrics_export_cnt(self.batch_id) 222 | metrics_bq_row_cnt = response["rows"][0]["f"][0]["v"] 223 | if metrics_bq_row_cnt is None: 224 | metrics_bq_row_cnt = 0 225 | else: 226 | metrics_bq_row_cnt = int(metrics_bq_row_cnt) 227 | print("metrics_bq_row_cnt: {}".format(metrics_bq_row_cnt)) 228 | 229 | assert write_metrics_row_cnt == metrics_bq_row_cnt, \ 230 | "Failed #3: The # of records written from write_metrics doesn't " \ 231 | "match the # of records received in BigQuery "\ 232 | "write_metrics_row_cnt: {}, metrics_bq_row_cnt:{}"\ 233 | .format(write_metrics_row_cnt, metrics_bq_row_cnt) 234 | 235 | def test_4(self): 236 | """ 237 | test 4. The number of messages written to BigQuery from list_metrics 238 | should match the actual number of messages from the Monitoring API 239 | """ 240 | metric_descriptors_cnt = self.get_metric_descriptors_cnt() 241 | response = self.get_list_metrics_num_full_recs(self.batch_id) 242 | bq_metrics_cnt = int(response["rows"][0]["f"][0]["v"]) 243 | print("bq_metrics_cnt: {}, metric_descriptors_cnt: {}".format(bq_metrics_cnt, metric_descriptors_cnt)) 244 | 245 | assert metric_descriptors_cnt == bq_metrics_cnt, \ 246 | "Failed #4: The # of metric descriptors written from list_metrics "\ 247 | "doesn'tmatch the # of records received from the Monitoring API call "\ 248 | "bq_metrics_cnt: {}, metric_descriptors_cnt: {}"\ 249 | .format(bq_metrics_cnt, metric_descriptors_cnt) 250 | 251 | if __name__ == '__main__': 252 | logging.getLogger('googleapiclient.discovery_cache').setLevel(logging.ERROR) 253 | unittest.main() -------------------------------------------------------------------------------- /test/requirements.txt: -------------------------------------------------------------------------------- 1 | google-api-python-client -------------------------------------------------------------------------------- /write_metrics/.gitignore: -------------------------------------------------------------------------------- 1 | lib/ 2 | -------------------------------------------------------------------------------- /write_metrics/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2019 Google Inc. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | service: write-metrics 16 | runtime: python310 17 | app_engine_apis: true 18 | entrypoint: gunicorn -b :$PORT -w 2 main:app 19 | 20 | handlers: 21 | - url: /.* 22 | script: auto -------------------------------------------------------------------------------- /write_metrics/config.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | PUBSUB_VERIFICATION_TOKEN = '16b2ecfb-7734-48b9-817d-4ac8bd623c87' 18 | PUBSUB_TOPIC="metrics_for_bigquery" 19 | BIGQUERY_DATASET='metric_export' 20 | BIGQUERY_TABLE='sd_metrics_export_fin' 21 | BIGQUERY_STATS_TABLE='sd_metrics_stats' 22 | WRITE_BQ_STATS_FLAG=True 23 | 24 | 25 | GAUGE="GAUGE" 26 | DELTA="DELTA" 27 | CUMULATIVE="CUMULATIVE" 28 | 29 | BOOL="BOOL" 30 | INT64="INT64" 31 | DOUBLE="DOUBLE" 32 | STRING="STRING" 33 | DISTRIBUTION="DISTRIBUTION" 34 | 35 | ALIGN_DELTA="ALIGN_DELTA" 36 | ALIGN_FRACTION_TRUE="ALIGN_FRACTION_TRUE" 37 | ALIGN_SUM="ALIGN_SUM" 38 | ALIGN_COUNT="ALIGN_COUNT" 39 | ALIGN_NONE="ALIGN_NONE" 40 | REDUCE_FRACTION_TRUE="REDUCE_FRACTION_TRUE" 41 | REDUCE_MEAN="REDUCE_MEAN" 42 | REDUCE_SUM="REDUCE_SUM" 43 | REDUCE_COUNT="REDUCE_COUNT" 44 | REDUCE_NONE="REDUCE_NONE" 45 | 46 | bigquery_value_map = { 47 | "INT64":"int64_value", 48 | "BOOL":"boolean_value", 49 | "DOUBLE":"double_value", 50 | "STRING":"string_value", 51 | "DISTRIBUTION":"distribution_value" 52 | } 53 | 54 | api_value_map = { 55 | "INT64":"int64Value", 56 | "BOOL":"booleanValue", 57 | "DOUBLE":"doubleValue", 58 | "STRING":"stringValue", 59 | "DISTRIBUTION":"distributionValue" 60 | } -------------------------------------------------------------------------------- /write_metrics/main.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import logging 18 | from flask import Flask, Response, request 19 | from google.appengine.api import wrap_wsgi_app 20 | from google.appengine.api import app_identity 21 | import json 22 | import base64 23 | import config 24 | from datetime import datetime 25 | from googleapiclient.discovery import build 26 | 27 | app = Flask(__name__) 28 | app.wsgi_app = wrap_wsgi_app(app.wsgi_app) 29 | 30 | # logging.basicConfig(level=logging.DEBUG) 31 | 32 | 33 | def build_rows(timeseries, metadata): 34 | """Build a list of JSON object rows to insert into BigQuery 35 | This function may fan out the input by writing 1 entry into BigQuery for every point, 36 | if there is more than 1 point in the timeseries 37 | """ 38 | logging.debug("build_row") 39 | rows = [] 40 | 41 | # handle >= 1 points, potentially > 1 returned from Monitoring API call 42 | for point_idx in range(len(timeseries["points"])): 43 | row = {"batch_id": metadata["batch_id"]} 44 | 45 | metric = {"type": timeseries["metric"]["type"]} 46 | metric_labels_list = get_labels(timeseries["metric"], "labels") 47 | 48 | if len(metric_labels_list) > 0: 49 | metric["labels"] = metric_labels_list 50 | row["metric"] = metric 51 | 52 | resource = {"type": timeseries["resource"]["type"]} 53 | resource_labels_list = get_labels(timeseries["resource"], "labels") 54 | if len(resource_labels_list) > 0: 55 | resource["labels"] = resource_labels_list 56 | 57 | row["resource"] = resource 58 | 59 | interval = { 60 | "start_time": timeseries["points"][point_idx]["interval"]["startTime"], 61 | "end_time": timeseries["points"][point_idx]["interval"]["endTime"], 62 | } 63 | 64 | # map the API value types to the BigQuery value types 65 | value_type = timeseries["valueType"] 66 | bigquery_value_type_index = config.bigquery_value_map[value_type] 67 | api_value_type_index = config.api_value_map[value_type] 68 | value_type_label = {} 69 | value = timeseries["points"][point_idx]["value"][api_value_type_index] 70 | if value_type == config.DISTRIBUTION: 71 | value_type_label[bigquery_value_type_index] = build_distribution_value( 72 | value 73 | ) 74 | else: 75 | value_type_label[bigquery_value_type_index] = value 76 | 77 | point = {"interval": interval, "value": value_type_label} 78 | row["point"] = point 79 | 80 | if "metadata" in timeseries: 81 | metric_metadata = {} 82 | if "userLabels" in timeseries["metadata"]: 83 | user_labels_list = get_labels(timeseries["metadata"], "userLabels") 84 | if len(user_labels_list) > 0: 85 | metric_metadata["user_labels"] = user_labels_list 86 | 87 | if "systemLabels" in timeseries["metadata"]: 88 | system_labels_list = get_system_labels( 89 | timeseries["metadata"], "systemLabels" 90 | ) 91 | if len(system_labels_list) > 0: 92 | metric_metadata["system_labels"] = system_labels_list 93 | 94 | row["metric_metadata"] = metric_metadata 95 | 96 | row["metric_kind"] = timeseries["metricKind"] 97 | row["value_type"] = timeseries["valueType"] 98 | row_to_insert = {} 99 | row_to_insert["json"] = row 100 | rows.append(row_to_insert) 101 | logging.debug("row: {}".format(json.dumps(row, sort_keys=True, indent=4))) 102 | 103 | return rows 104 | 105 | 106 | def get_labels(timeseries, label_name): 107 | """Build a list of metric labels based on the API spec below 108 | See https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries#MonitoredResourceMetadata 109 | """ 110 | metric_labels_list = [] 111 | if label_name in timeseries: 112 | for label in timeseries[label_name]: 113 | metric_label = {} 114 | metric_label["key"] = label 115 | metric_label["value"] = timeseries[label_name][label] 116 | metric_labels_list.append(metric_label) 117 | logging.debug( 118 | "get_labels: {}".format( 119 | json.dumps(metric_labels_list, sort_keys=True, indent=4) 120 | ) 121 | ) 122 | return metric_labels_list 123 | 124 | 125 | def get_system_labels(timeseries, label_name): 126 | """Build a list of system_labels based on the API spec below 127 | See https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries#MonitoredResourceMetadata 128 | """ 129 | system_labels_list = [] 130 | if label_name in timeseries: 131 | for label in timeseries[label_name]: 132 | metric_label = {} 133 | metric_label["key"] = label 134 | # The values can be bool, list or str 135 | # See https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries#MonitoredResourceMetadata 136 | if type(timeseries[label_name][label]) is bool: 137 | metric_label["value"] = str(timeseries[label_name][label]) 138 | system_labels_list.append(metric_label) 139 | elif type(timeseries[label_name][label]) is list: 140 | for system_label_name in timeseries[label_name][label]: 141 | metric_label = {} 142 | metric_label["key"] = label 143 | metric_label["value"] = system_label_name 144 | system_labels_list.append(metric_label) 145 | else: 146 | metric_label["value"] = timeseries[label_name][label] 147 | system_labels_list.append(metric_label) 148 | 149 | return system_labels_list 150 | 151 | 152 | def build_distribution_value(value_json): 153 | """Build a distribution value based on the API spec below 154 | See https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries#Distribution 155 | """ 156 | distribution_value = {} 157 | if "count" in value_json: 158 | distribution_value["count"] = int(value_json["count"]) 159 | if "mean" in value_json: 160 | distribution_value["mean"] = round(value_json["mean"], 2) 161 | if "sumOfSquaredDeviation" in value_json: 162 | distribution_value["sumOfSquaredDeviation"] = round( 163 | value_json["sumOfSquaredDeviation"], 2 164 | ) 165 | 166 | if "range" in value_json: 167 | distribution_value_range = {} 168 | distribution_value_range["min"] = value_json["range"]["min"] 169 | distribution_value_range["max"] = value_json["range"]["max"] 170 | distribution_value["range"] = distribution_value_range 171 | 172 | bucketOptions = {} 173 | if "linearBuckets" in value_json["bucketOptions"]: 174 | linearBuckets = { 175 | "numFiniteBuckets": value_json["bucketOptions"]["linearBuckets"][ 176 | "numFiniteBuckets" 177 | ], 178 | "width": value_json["bucketOptions"]["linearBuckets"]["width"], 179 | "offset": value_json["bucketOptions"]["linearBuckets"]["offset"], 180 | } 181 | bucketOptions["linearBuckets"] = linearBuckets 182 | elif "exponentialBuckets" in value_json["bucketOptions"]: 183 | exponentialBuckets = { 184 | "numFiniteBuckets": value_json["bucketOptions"]["exponentialBuckets"][ 185 | "numFiniteBuckets" 186 | ], 187 | "growthFactor": round( 188 | value_json["bucketOptions"]["exponentialBuckets"]["growthFactor"], 2 189 | ), 190 | "scale": value_json["bucketOptions"]["exponentialBuckets"]["scale"], 191 | } 192 | bucketOptions["exponentialBuckets"] = exponentialBuckets 193 | elif "explicitBuckets" in value_json["bucketOptions"]: 194 | explicitBuckets = { 195 | "bounds": { 196 | "value": value_json["bucketOptions"]["explicitBuckets"]["bounds"] 197 | } 198 | } 199 | bucketOptions["explicitBuckets"] = explicitBuckets 200 | if bucketOptions: 201 | distribution_value["bucketOptions"] = bucketOptions 202 | 203 | if "bucketCounts" in value_json: 204 | bucketCounts = {} 205 | bucket_count_list = [] 206 | for bucket_count_val in value_json["bucketCounts"]: 207 | bucket_count_list.append(int(bucket_count_val)) 208 | bucketCounts["value"] = bucket_count_list 209 | distribution_value["bucketCounts"] = bucketCounts 210 | 211 | if "exemplars" in value_json: 212 | exemplars_list = [] 213 | for exemplar in value_json["exemplars"]: 214 | exemplar = {"value": exemplar["value"], "timestamp": exemplar["timestamp"]} 215 | exemplars_list.append(exemplar) 216 | distribution_value["exemplars"] = exemplars_list 217 | 218 | logging.debug( 219 | "created the distribution_value: {}".format( 220 | json.dumps(distribution_value, sort_keys=True, indent=4) 221 | ) 222 | ) 223 | return distribution_value 224 | 225 | 226 | def build_bigquery_stats_message(metadata): 227 | processing_end_time = datetime.now() 228 | processing_end_time_str = processing_end_time.strftime("%Y-%m-%dT%H:%M:%S.%fZ") 229 | 230 | # Write the stats to the BigQuery stats tabledata 231 | bq_msg = { 232 | "app_name": "write_metrics", 233 | "batch_id": metadata["batch_id"], 234 | "message_id": metadata["message_id"], 235 | "src_message_id": metadata["src_message_id"], 236 | "metric_type": metadata["metric_type"], 237 | "error_msg_cnt": metadata["error_msg_cnt"], 238 | "msg_written_cnt": metadata["msg_written_cnt"], 239 | "msg_without_timeseries": metadata["msg_without_timeseries"], 240 | "payload": metadata["payload"], 241 | "batch_start_time": metadata["batch_start_time"], 242 | "processing_end_time": processing_end_time_str, 243 | } 244 | json_msg = {"json": bq_msg} 245 | logging.debug("json_msg {}".format(json.dumps(json_msg, sort_keys=True, indent=4))) 246 | return json_msg 247 | 248 | 249 | def write_stats_to_bigquery(json_row_list): 250 | """Write rows to the BigQuery stats table using the googleapiclient and the streaming insertAll method 251 | https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll 252 | """ 253 | logging.debug("write_stats_to_bigquery") 254 | 255 | bigquery = build("bigquery", "v2", cache_discovery=True) 256 | 257 | body = { 258 | "kind": "bigquery#tableDataInsertAllRequest", 259 | "skipInvalidRows": "false", 260 | "rows": json_row_list, 261 | } 262 | logging.debug("body: {}".format(body)) 263 | 264 | response = ( 265 | bigquery.tabledata() 266 | .insertAll( 267 | projectId=app_identity.get_application_id(), 268 | datasetId=config.BIGQUERY_DATASET, 269 | tableId=config.BIGQUERY_STATS_TABLE, 270 | body=body, 271 | ) 272 | .execute() 273 | ) 274 | logging.debug("BigQuery said... = {}".format(response)) 275 | 276 | bq_stats_msgs_with_errors = 0 277 | if "insertErrors" in response: 278 | if len(response["insertErrors"]) > 0: 279 | logging.error("Error: {}".format(response)) 280 | bq_stats_msgs_with_errors = len(response["insertErrors"]) 281 | else: 282 | logging.debug( 283 | "By amazing luck, there are no errors, response = {}".format(response) 284 | ) 285 | logging.debug("bq_stats_msgs_with_errors: {}".format(bq_stats_msgs_with_errors)) 286 | return response 287 | 288 | 289 | def write_to_bigquery(timeseries, metadata): 290 | """Write rows to BigQuery using the googleapiclient and the streaming insertAll method 291 | https://cloud.google.com/bigquery/docs/reference/rest/v2/tabledata/insertAll 292 | """ 293 | logging.debug("write_to_bigquery") 294 | 295 | response = {} 296 | json_msg_list = [] 297 | stats = {} 298 | if not timeseries or "points" not in timeseries: 299 | logging.debug( 300 | "No timeseries data to write to BigQuery for: {}".format(timeseries) 301 | ) 302 | msgs_written = 0 303 | metadata["msg_without_timeseries"] = 1 304 | error_msg_cnt = 0 305 | else: 306 | rows = build_rows(timeseries, metadata) 307 | bigquery = build("bigquery", "v2", cache_discovery=True) 308 | body = { 309 | "kind": "bigquery#tableDataInsertAllRequest", 310 | "skipInvalidRows": "false", 311 | "rows": rows, 312 | } 313 | logging.debug("body: {}".format(body, sort_keys=True, indent=4)) 314 | 315 | response = ( 316 | bigquery.tabledata() 317 | .insertAll( 318 | projectId=app_identity.get_application_id(), 319 | datasetId=config.BIGQUERY_DATASET, 320 | tableId=config.BIGQUERY_TABLE, 321 | body=body, 322 | ) 323 | .execute() 324 | ) 325 | 326 | logging.debug("BigQuery said... = {}".format(response)) 327 | 328 | msgs_written = len(rows) 329 | error_msg_cnt = 0 330 | 331 | if "insertErrors" in response: 332 | if len(response["insertErrors"]) > 0: 333 | logging.error("Error: {}".format(response)) 334 | error_msg_cnt = len(response["insertErrors"]) 335 | msgs_written = msgs_written - error_msg_cnt 336 | else: 337 | logging.debug( 338 | "By amazing luck, there are no errors, response = {}".format(response) 339 | ) 340 | 341 | metadata["msg_without_timeseries"] = 0 342 | 343 | # set the metadata to write to the BigQuery stats table 344 | if config.WRITE_BQ_STATS_FLAG: 345 | metadata["error_msg_cnt"] = error_msg_cnt 346 | metadata["msg_written_cnt"] = msgs_written 347 | metadata["payload"] = "{}".format(json.dumps(timeseries)) 348 | metadata["metric_type"] = timeseries["metric"]["type"] 349 | json_msg = build_bigquery_stats_message(metadata) 350 | json_msg_list.append(json_msg) 351 | 352 | # write the list of stats messages to BigQuery 353 | write_stats_to_bigquery(json_msg_list) 354 | 355 | stats["msgs_written"] = msgs_written 356 | stats["msgs_with_errors"] = error_msg_cnt 357 | logging.info("Stats are {}".format(json.dumps(stats))) 358 | return response 359 | 360 | 361 | @app.route("/push-handlers/receive_messages", methods=["POST"]) 362 | def root(): 363 | """Receive the Pub/Sub message via POST 364 | Validate the input and then process the message 365 | """ 366 | logging.debug("received message") 367 | 368 | try: 369 | if not request.data: 370 | raise ValueError("No request data received") 371 | envelope = json.loads(request.data.decode("utf-8")) 372 | logging.debug("Raw pub/sub message: {}".format(envelope)) 373 | 374 | if "message" not in envelope: 375 | raise ValueError("No message in envelope") 376 | 377 | if "messageId" in envelope["message"]: 378 | logging.debug( 379 | "messageId: {}".format(envelope["message"].get("messageId", "")) 380 | ) 381 | message_id = envelope["message"]["messageId"] 382 | 383 | if "attributes" not in envelope["message"]: 384 | raise ValueError( 385 | "Attributes such as token and batch_id missing from request" 386 | ) 387 | 388 | # if the pubsub PUBSUB_VERIFICATION_TOKEN isn't included or doesn't match, don't continue 389 | if "token" not in envelope["message"]["attributes"]: 390 | raise ValueError("token missing from request") 391 | if ( 392 | not envelope["message"]["attributes"]["token"] 393 | == config.PUBSUB_VERIFICATION_TOKEN 394 | ): 395 | raise ValueError( 396 | "token from request doesn't match, received: {}".format( 397 | envelope["message"]["attributes"]["token"] 398 | ) 399 | ) 400 | 401 | # if the batch_id isn't included, fail immediately 402 | if "batch_id" not in envelope["message"]["attributes"]: 403 | raise ValueError("batch_id missing from request") 404 | batch_id = envelope["message"]["attributes"]["batch_id"] 405 | logging.debug("batch_id: {} ".format(batch_id)) 406 | 407 | if "batch_start_time" in envelope["message"]["attributes"]: 408 | batch_start_time = envelope["message"]["attributes"]["batch_start_time"] 409 | else: 410 | batch_start_time = datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") 411 | 412 | if "src_message_id" in envelope["message"]["attributes"]: 413 | src_message_id = envelope["message"]["attributes"]["src_message_id"] 414 | else: 415 | src_message_id = "" 416 | 417 | if "data" not in envelope["message"]: 418 | raise ValueError("No data in message") 419 | payload = base64.b64decode(envelope["message"]["data"]) 420 | logging.debug("payload: {} ".format(payload)) 421 | 422 | metadata = { 423 | "batch_id": batch_id, 424 | "message_id": message_id, 425 | "src_message_id": src_message_id, 426 | "batch_start_time": batch_start_time, 427 | } 428 | 429 | data = json.loads(payload) 430 | logging.debug("data: {} ".format(data)) 431 | 432 | # Check the input parameters 433 | if not data: 434 | raise ValueError("No data in Pub/Sub Message to write to BigQuery") 435 | 436 | # See https://cloud.google.com/monitoring/api/ref_v3/rest/v3/TimeSeries 437 | write_to_bigquery(data, metadata) 438 | 439 | except Exception as e: 440 | logging.error("Error: {}".format(e)) 441 | return Response(f"{e}", status=500) 442 | 443 | return Response("Ok", status=200) 444 | -------------------------------------------------------------------------------- /write_metrics/main_test.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2019 Google Inc. 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | import pytest 18 | from main import app as flask_app 19 | 20 | import main 21 | import config 22 | import base64 23 | import json 24 | 25 | batch_id = "R1HIA55JB5DOQZM8R53OKMCWZ5BEQKUJ" 26 | 27 | 28 | @pytest.fixture 29 | def app(): 30 | yield flask_app 31 | 32 | 33 | @pytest.fixture 34 | def client(app): 35 | return app.test_client() 36 | 37 | 38 | def test_post_empty_data(app, client): 39 | """Test sending an empty message""" 40 | response = client.post("/push-handlers/receive_messages") 41 | assert response.status_code == 500 42 | assert response.get_data(as_text=True) == "No request data received" 43 | 44 | 45 | def test_incorrect_token_post(app, client): 46 | """Test sending an incorrect token""" 47 | mimetype = "application/json" 48 | headers = { 49 | "Content-Type": mimetype, 50 | "Accept": mimetype, 51 | } 52 | request = build_request(token="incorrect_token") 53 | response = client.post( 54 | "/push-handlers/receive_messages", data=json.dumps(request), headers=headers 55 | ) 56 | 57 | assert response.status_code == 500 58 | 59 | 60 | def test_correct_labels(): 61 | """Test whether the correct labels are extracted from the metric API responses""" 62 | timeseries = build_timeseries() 63 | 64 | metric_labels_list = main.get_labels(timeseries["metric"], "labels") 65 | expected_metric_labels_list = build_metric_labels() 66 | assert sorted(metric_labels_list) == sorted(expected_metric_labels_list) 67 | 68 | resource_labels_list = main.get_labels(timeseries["resource"], "labels") 69 | expected_resource_labels_list = build_resource_labels() 70 | assert sorted(resource_labels_list, key=lambda item: str(item)) == sorted( 71 | expected_resource_labels_list, key=lambda item: str(item) 72 | ) 73 | 74 | user_labels_list = main.get_labels(build_user_labels_request(), "userLabels") 75 | expected_user_labels_list = build_expected_user_labels_response() 76 | assert sorted(user_labels_list, key=lambda item: str(item)) == sorted( 77 | expected_user_labels_list, key=lambda item: str(item) 78 | ) 79 | 80 | system_labels_list = main.get_system_labels( 81 | build_user_labels_request(), "systemLabels" 82 | ) 83 | expected_system_labels_list = build_expected_system_labels_response() 84 | assert sorted(system_labels_list, key=lambda item: str(item)) == sorted( 85 | expected_system_labels_list, key=lambda item: str(item) 86 | ) 87 | 88 | 89 | def test_correct_build_distribution_values(): 90 | """Test whether the correct distribution values are built given a timeseries input""" 91 | timeseries_with_distribution_values = build_distribution_value() 92 | 93 | distribution_value = main.build_distribution_value( 94 | timeseries_with_distribution_values["points"][0]["value"]["distributionValue"] 95 | ) 96 | expected_distribution_value = build_expected_distribution_value() 97 | assert distribution_value == expected_distribution_value 98 | 99 | 100 | def test_correct_build_row(): 101 | """Test whether the correct JSON object is created for insert into BigQuery given a timeseries input""" 102 | timeseries = build_timeseries() 103 | bq_body = main.build_rows(timeseries, {"batch_id": batch_id}) 104 | 105 | bq_expected_response = build_expected_bq_response() 106 | assert bq_body == bq_expected_response 107 | 108 | 109 | def build_timeseries(): 110 | """Build a timeseries object to use as input""" 111 | timeseries = { 112 | "metricKind": "DELTA", 113 | "metric": { 114 | "labels": {"response_code": "0"}, 115 | "type": "agent.googleapis.com/agent/request_count", 116 | }, 117 | "points": [ 118 | { 119 | "interval": { 120 | "endTime": "2019-02-18T22:09:53.939194Z", 121 | "startTime": "2019-02-18T21:09:53.939194Z", 122 | }, 123 | "value": {"int64Value": "62"}, 124 | }, 125 | { 126 | "interval": { 127 | "endTime": "2019-02-18T21:09:53.939194Z", 128 | "startTime": "2019-02-18T20:09:53.939194Z", 129 | }, 130 | "value": {"int64Value": "61"}, 131 | }, 132 | ], 133 | "resource": { 134 | "labels": { 135 | "instance_id": "9113659852587170607", 136 | "project_id": "YOUR_PROJECT_ID", 137 | "zone": "us-east4-a", 138 | }, 139 | "type": "gce_instance", 140 | }, 141 | "valueType": "INT64", 142 | } 143 | 144 | return timeseries 145 | 146 | 147 | def build_expected_bq_response(): 148 | """Build the expected BigQuery insert JSON object""" 149 | response = [ 150 | { 151 | "json": { 152 | "batch_id": batch_id, 153 | "metric": { 154 | "labels": [{"key": "response_code", "value": "0"}], 155 | "type": "agent.googleapis.com/agent/request_count", 156 | }, 157 | "metric_kind": "DELTA", 158 | "point": { 159 | "interval": { 160 | "end_time": "2019-02-18T22:09:53.939194Z", 161 | "start_time": "2019-02-18T21:09:53.939194Z", 162 | }, 163 | "value": {"int64_value": "62"}, 164 | }, 165 | "resource": { 166 | "labels": [ 167 | {"key": "instance_id", "value": "9113659852587170607"}, 168 | {"key": "project_id", "value": "YOUR_PROJECT_ID"}, 169 | {"key": "zone", "value": "us-east4-a"}, 170 | ], 171 | "type": "gce_instance", 172 | }, 173 | "value_type": "INT64", 174 | } 175 | }, 176 | { 177 | "json": { 178 | "batch_id": batch_id, 179 | "metric": { 180 | "labels": [{"key": "response_code", "value": "0"}], 181 | "type": "agent.googleapis.com/agent/request_count", 182 | }, 183 | "metric_kind": "DELTA", 184 | "point": { 185 | "interval": { 186 | "end_time": "2019-02-18T21:09:53.939194Z", 187 | "start_time": "2019-02-18T20:09:53.939194Z", 188 | }, 189 | "value": {"int64_value": "61"}, 190 | }, 191 | "resource": { 192 | "labels": [ 193 | {"key": "instance_id", "value": "9113659852587170607"}, 194 | {"key": "project_id", "value": "YOUR_PROJECT_ID"}, 195 | {"key": "zone", "value": "us-east4-a"}, 196 | ], 197 | "type": "gce_instance", 198 | }, 199 | "value_type": "INT64", 200 | } 201 | }, 202 | ] 203 | return response 204 | 205 | 206 | def build_metric_labels(): 207 | """Build the expected metric labels list""" 208 | response = [{"key": "response_code", "value": "0"}] 209 | return response 210 | 211 | 212 | def build_resource_labels(): 213 | """Build the expected resource labels list""" 214 | response = [ 215 | {"key": "instance_id", "value": "9113659852587170607"}, 216 | {"key": "project_id", "value": "YOUR_PROJECT_ID"}, 217 | {"key": "zone", "value": "us-east4-a"}, 218 | ] 219 | return response 220 | 221 | 222 | def build_request(token=config.PUBSUB_VERIFICATION_TOKEN): 223 | """Build a Pub/Sub message as input""" 224 | payload = { 225 | "metricKind": "DELTA", 226 | "metric": { 227 | "labels": {"response_code": "0"}, 228 | "type": "agent.googleapis.com/agent/request_count", 229 | }, 230 | "points": [ 231 | { 232 | "interval": { 233 | "endTime": "2019-02-18T22:09:53.939194Z", 234 | "startTime": "2019-02-18T21:09:53.939194Z", 235 | }, 236 | "value": {"int64Value": "62"}, 237 | }, 238 | { 239 | "interval": { 240 | "endTime": "2019-02-18T21:09:53.939194Z", 241 | "startTime": "2019-02-18T20:09:53.939194Z", 242 | }, 243 | "value": {"int64Value": "61"}, 244 | }, 245 | ], 246 | "resource": { 247 | "labels": { 248 | "instance_id": "9113659852587170607", 249 | "project_id": "YOUR_PROJECT_ID", 250 | "zone": "us-east4-a", 251 | }, 252 | "type": "gce_instance", 253 | }, 254 | "valueType": "INT64", 255 | } 256 | request = { 257 | "message": { 258 | "attributes": {"batch_id": batch_id, "token": token}, 259 | "data": base64.b64encode(json.dumps(payload).encode("utf-8")).decode(), 260 | } 261 | } 262 | return request 263 | 264 | 265 | def build_user_labels_request(): 266 | """Build the JSON input for the userLabels and systemLabels""" 267 | request = { 268 | "systemLabels": { 269 | "name": "appName", 270 | "list_name": ["a", "b", "c"], 271 | "boolean_value": False, 272 | }, 273 | "userLabels": {"key1": "value1", "key2": "value2"}, 274 | } 275 | return request 276 | 277 | 278 | def build_expected_system_labels_response(): 279 | """Build the expected system labels list""" 280 | labels = [ 281 | {"key": "name", "value": "appName"}, 282 | {"key": "boolean_value", "value": "False"}, 283 | {"key": "list_name", "value": "a"}, 284 | {"key": "list_name", "value": "b"}, 285 | {"key": "list_name", "value": "c"}, 286 | ] 287 | return labels 288 | 289 | 290 | def build_expected_user_labels_response(): 291 | """Build the expected user labels list""" 292 | labels = [{"key": "key1", "value": "value1"}, {"key": "key2", "value": "value2"}] 293 | return labels 294 | 295 | 296 | def build_distribution_value(): 297 | """Build the expected JSON object input for the distribution values test""" 298 | timeseries = { 299 | "metricKind": "DELTA", 300 | "metric": {"type": "serviceruntime.googleapis.com/api/response_sizes"}, 301 | "points": [ 302 | { 303 | "interval": { 304 | "endTime": "2019-02-19T04:00:00.841487Z", 305 | "startTime": "2019-02-19T03:00:00.841487Z", 306 | }, 307 | "value": { 308 | "distributionValue": { 309 | "count": "56", 310 | "mean": 17, 311 | "sumOfSquaredDeviation": 1.296382457204002e-25, 312 | "bucketCounts": ["56"], 313 | "bucketOptions": { 314 | "exponentialBuckets": { 315 | "scale": 1, 316 | "growthFactor": 10, 317 | "numFiniteBuckets": 8, 318 | } 319 | }, 320 | } 321 | }, 322 | } 323 | ], 324 | "resource": { 325 | "labels": { 326 | "service": "monitoring.googleapis.com", 327 | "credential_id": "serviceaccount:106579349769273816070", 328 | "version": "v3", 329 | "location": "us-central1", 330 | "project_id": "ms-demo-app01", 331 | "method": "google.monitoring.v3.MetricService.ListMetricDescriptors", 332 | }, 333 | "type": "consumed_api", 334 | }, 335 | "valueType": "DISTRIBUTION", 336 | } 337 | return timeseries 338 | 339 | 340 | def build_expected_distribution_value(): 341 | """Build the expected JSON object for the distribution values test""" 342 | distribution_value = { 343 | "count": 56, 344 | "mean": 17.0, 345 | "sumOfSquaredDeviation": 0.0, 346 | "bucketOptions": { 347 | "exponentialBuckets": { 348 | "numFiniteBuckets": 8, 349 | "growthFactor": 10.0, 350 | "scale": 1, 351 | } 352 | }, 353 | "bucketCounts": {"value": [56]}, 354 | } 355 | return distribution_value 356 | -------------------------------------------------------------------------------- /write_metrics/requirements.txt: -------------------------------------------------------------------------------- 1 | Flask 2 | gunicorn 3 | google-cloud-storage 4 | appengine-python-standard>=1.0.0 5 | google-api-python-client -------------------------------------------------------------------------------- /write_metrics/sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "timeSeries": 3 | [ 4 | { 5 | "metricKind": "DELTA", 6 | "metric": {"type": "agent.googleapis.com/agent/api_request_count"}, 7 | "points": [ 8 | {"interval": {"endTime": "2018-12-14T19:58:17.140600Z", "startTime": "2018-12-14T18:58:17.140600Z"}, 9 | "value": {"int64Value": "358"}}], 10 | "resource": 11 | { 12 | "labels": 13 | { 14 | "project_id": "sage-facet-201016" 15 | }, 16 | "type": "gce_instance" 17 | }, 18 | "valueType": "INT64" 19 | } 20 | ] 21 | } --------------------------------------------------------------------------------