├── .github └── dependabot.yml ├── .gitignore ├── AUTHORS ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── dashboard ├── .gitignore ├── dashboard_screenshot.jpg ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── google │ │ └── cloud │ │ └── solutions │ │ └── realtimedash │ │ └── dashboard │ │ ├── DashboardApplication.java │ │ ├── JedisBeanConfiguration.java │ │ ├── OverlapMetric.java │ │ ├── TimeSeriesKeyBuilder.java │ │ ├── TimeSeriesMetric.java │ │ └── TimeseriesMetricsController.java │ └── resources │ ├── application.properties │ └── static │ ├── index.html │ └── metrics_ui.js ├── loggen ├── message_generator.py └── requirements.txt ├── processor ├── deploy_dataflow.sh ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── google │ └── cloud │ └── solutions │ └── realtimedash │ └── pipeline │ ├── LogEvent.java │ ├── MetricsCalculationPipeline.java │ ├── MetricsPipelineOptions.java │ └── ParseMessageAsLogElement.java ├── realtime-analytics-architecture.svg └── set_variables.sh /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "maven" 4 | directory: "/processor" 5 | schedule: 6 | interval: "monthly" 7 | groups: 8 | processor-production-dependencies: 9 | dependency-type: "production" 10 | update-types: 11 | - "major" 12 | - "minor" 13 | 14 | - package-ecosystem: "maven" 15 | directory: "/dashboard" 16 | schedule: 17 | interval: "monthly" 18 | groups: 19 | dashboard-production-dependencies: 20 | dependency-type: "production" 21 | update-types: 22 | - "major" 23 | - "minor" 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /processor/target/ 2 | .idea 3 | *.iml 4 | 5 | 6 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | # This is the list of tool authors for copyright purposes. 2 | # 3 | # This does not necessarily list everyone who has contributed code, since in 4 | # some cases, their employer may be the copyright holder. To see the full list 5 | # of contributors, see the revision history in source control. 6 | Anant Damle 7 | Varun Dhussa 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # How to Contribute 2 | 3 | We'd love to accept your patches and contributions to this project. There are 4 | just a few small guidelines you need to follow. 5 | 6 | ## Contributor License Agreement 7 | 8 | Contributions to this project must be accompanied by a Contributor License 9 | Agreement. You (or your employer) retain the copyright to your contribution; 10 | this simply gives us permission to use and redistribute your contributions as 11 | part of the project. Head over to to see 12 | your current agreements on file or to sign a new one. 13 | 14 | You generally only need to submit a CLA once, so if you've already submitted one 15 | (even if it was for a different project), you probably don't need to do it 16 | again. 17 | 18 | ## Code reviews 19 | 20 | All submissions, including submissions by project members, require review. We 21 | use GitHub pull requests for this purpose. Consult 22 | [GitHub Help](https://help.github.com/articles/about-pull-requests/) for more 23 | information on using pull requests. 24 | 25 | ## Community Guidelines 26 | 27 | This project follows 28 | [Google's Open Source Community Guidelines](https://opensource.google.com/conduct/). 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | Apache License 204 | Version 2.0, January 2004 205 | http://www.apache.org/licenses/ 206 | 207 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 208 | 209 | 1. Definitions. 210 | 211 | "License" shall mean the terms and conditions for use, reproduction, 212 | and distribution as defined by Sections 1 through 9 of this document. 213 | 214 | "Licensor" shall mean the copyright owner or entity authorized by 215 | the copyright owner that is granting the License. 216 | 217 | "Legal Entity" shall mean the union of the acting entity and all 218 | other entities that control, are controlled by, or are under common 219 | control with that entity. For the purposes of this definition, 220 | "control" means (i) the power, direct or indirect, to cause the 221 | direction or management of such entity, whether by contract or 222 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 223 | outstanding shares, or (iii) beneficial ownership of such entity. 224 | 225 | "You" (or "Your") shall mean an individual or Legal Entity 226 | exercising permissions granted by this License. 227 | 228 | "Source" form shall mean the preferred form for making modifications, 229 | including but not limited to software source code, documentation 230 | source, and configuration files. 231 | 232 | "Object" form shall mean any form resulting from mechanical 233 | transformation or translation of a Source form, including but 234 | not limited to compiled object code, generated documentation, 235 | and conversions to other media types. 236 | 237 | "Work" shall mean the work of authorship, whether in Source or 238 | Object form, made available under the License, as indicated by a 239 | copyright notice that is included in or attached to the work 240 | (an example is provided in the Appendix below). 241 | 242 | "Derivative Works" shall mean any work, whether in Source or Object 243 | form, that is based on (or derived from) the Work and for which the 244 | editorial revisions, annotations, elaborations, or other modifications 245 | represent, as a whole, an original work of authorship. For the purposes 246 | of this License, Derivative Works shall not include works that remain 247 | separable from, or merely link (or bind by name) to the interfaces of, 248 | the Work and Derivative Works thereof. 249 | 250 | "Contribution" shall mean any work of authorship, including 251 | the original version of the Work and any modifications or additions 252 | to that Work or Derivative Works thereof, that is intentionally 253 | submitted to Licensor for inclusion in the Work by the copyright owner 254 | or by an individual or Legal Entity authorized to submit on behalf of 255 | the copyright owner. For the purposes of this definition, "submitted" 256 | means any form of electronic, verbal, or written communication sent 257 | to the Licensor or its representatives, including but not limited to 258 | communication on electronic mailing lists, source code control systems, 259 | and issue tracking systems that are managed by, or on behalf of, the 260 | Licensor for the purpose of discussing and improving the Work, but 261 | excluding communication that is conspicuously marked or otherwise 262 | designated in writing by the copyright owner as "Not a Contribution." 263 | 264 | "Contributor" shall mean Licensor and any individual or Legal Entity 265 | on behalf of whom a Contribution has been received by Licensor and 266 | subsequently incorporated within the Work. 267 | 268 | 2. Grant of Copyright License. Subject to the terms and conditions of 269 | this License, each Contributor hereby grants to You a perpetual, 270 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 271 | copyright license to reproduce, prepare Derivative Works of, 272 | publicly display, publicly perform, sublicense, and distribute the 273 | Work and such Derivative Works in Source or Object form. 274 | 275 | 3. Grant of Patent License. Subject to the terms and conditions of 276 | this License, each Contributor hereby grants to You a perpetual, 277 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 278 | (except as stated in this section) patent license to make, have made, 279 | use, offer to sell, sell, import, and otherwise transfer the Work, 280 | where such license applies only to those patent claims licensable 281 | by such Contributor that are necessarily infringed by their 282 | Contribution(s) alone or by combination of their Contribution(s) 283 | with the Work to which such Contribution(s) was submitted. If You 284 | institute patent litigation against any entity (including a 285 | cross-claim or counterclaim in a lawsuit) alleging that the Work 286 | or a Contribution incorporated within the Work constitutes direct 287 | or contributory patent infringement, then any patent licenses 288 | granted to You under this License for that Work shall terminate 289 | as of the date such litigation is filed. 290 | 291 | 4. Redistribution. You may reproduce and distribute copies of the 292 | Work or Derivative Works thereof in any medium, with or without 293 | modifications, and in Source or Object form, provided that You 294 | meet the following conditions: 295 | 296 | (a) You must give any other recipients of the Work or 297 | Derivative Works a copy of this License; and 298 | 299 | (b) You must cause any modified files to carry prominent notices 300 | stating that You changed the files; and 301 | 302 | (c) You must retain, in the Source form of any Derivative Works 303 | that You distribute, all copyright, patent, trademark, and 304 | attribution notices from the Source form of the Work, 305 | excluding those notices that do not pertain to any part of 306 | the Derivative Works; and 307 | 308 | (d) If the Work includes a "NOTICE" text file as part of its 309 | distribution, then any Derivative Works that You distribute must 310 | include a readable copy of the attribution notices contained 311 | within such NOTICE file, excluding those notices that do not 312 | pertain to any part of the Derivative Works, in at least one 313 | of the following places: within a NOTICE text file distributed 314 | as part of the Derivative Works; within the Source form or 315 | documentation, if provided along with the Derivative Works; or, 316 | within a display generated by the Derivative Works, if and 317 | wherever such third-party notices normally appear. The contents 318 | of the NOTICE file are for informational purposes only and 319 | do not modify the License. You may add Your own attribution 320 | notices within Derivative Works that You distribute, alongside 321 | or as an addendum to the NOTICE text from the Work, provided 322 | that such additional attribution notices cannot be construed 323 | as modifying the License. 324 | 325 | You may add Your own copyright statement to Your modifications and 326 | may provide additional or different license terms and conditions 327 | for use, reproduction, or distribution of Your modifications, or 328 | for any such Derivative Works as a whole, provided Your use, 329 | reproduction, and distribution of the Work otherwise complies with 330 | the conditions stated in this License. 331 | 332 | 5. Submission of Contributions. Unless You explicitly state otherwise, 333 | any Contribution intentionally submitted for inclusion in the Work 334 | by You to the Licensor shall be under the terms and conditions of 335 | this License, without any additional terms or conditions. 336 | Notwithstanding the above, nothing herein shall supersede or modify 337 | the terms of any separate license agreement you may have executed 338 | with Licensor regarding such Contributions. 339 | 340 | 6. Trademarks. This License does not grant permission to use the trade 341 | names, trademarks, service marks, or product names of the Licensor, 342 | except as required for reasonable and customary use in describing the 343 | origin of the Work and reproducing the content of the NOTICE file. 344 | 345 | 7. Disclaimer of Warranty. Unless required by applicable law or 346 | agreed to in writing, Licensor provides the Work (and each 347 | Contributor provides its Contributions) on an "AS IS" BASIS, 348 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 349 | implied, including, without limitation, any warranties or conditions 350 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 351 | PARTICULAR PURPOSE. You are solely responsible for determining the 352 | appropriateness of using or redistributing the Work and assume any 353 | risks associated with Your exercise of permissions under this License. 354 | 355 | 8. Limitation of Liability. In no event and under no legal theory, 356 | whether in tort (including negligence), contract, or otherwise, 357 | unless required by applicable law (such as deliberate and grossly 358 | negligent acts) or agreed to in writing, shall any Contributor be 359 | liable to You for damages, including any direct, indirect, special, 360 | incidental, or consequential damages of any character arising as a 361 | result of this License or out of the use or inability to use the 362 | Work (including but not limited to damages for loss of goodwill, 363 | work stoppage, computer failure or malfunction, or any and all 364 | other commercial damages or losses), even if such Contributor 365 | has been advised of the possibility of such damages. 366 | 367 | 9. Accepting Warranty or Additional Liability. While redistributing 368 | the Work or Derivative Works thereof, You may choose to offer, 369 | and charge a fee for, acceptance of support, warranty, indemnity, 370 | or other liability obligations and/or rights consistent with this 371 | License. However, in accepting such obligations, You may act only 372 | on Your own behalf and on Your sole responsibility, not on behalf 373 | of any other Contributor, and only if You agree to indemnify, 374 | defend, and hold each Contributor harmless for any liability 375 | incurred by, or claims asserted against, such Contributor by reason 376 | of your accepting any such warranty or additional liability. 377 | 378 | END OF TERMS AND CONDITIONS 379 | 380 | APPENDIX: How to apply the Apache License to your work. 381 | 382 | To apply the Apache License to your work, attach the following 383 | boilerplate notice, with the fields enclosed by brackets "[]" 384 | replaced with your own identifying information. (Don't include 385 | the brackets!) The text should be enclosed in the appropriate 386 | comment syntax for the file format. We also recommend that a 387 | file or class name and description of purpose be included on the 388 | same "printed page" as the copyright notice for easier 389 | identification within third-party archives. 390 | 391 | Copyright [yyyy] [name of copyright owner] 392 | 393 | Licensed under the Apache License, Version 2.0 (the "License"); 394 | you may not use this file except in compliance with the License. 395 | You may obtain a copy of the License at 396 | 397 | http://www.apache.org/licenses/LICENSE-2.0 398 | 399 | Unless required by applicable law or agreed to in writing, software 400 | distributed under the License is distributed on an "AS IS" BASIS, 401 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 402 | See the License for the specific language governing permissions and 403 | limitations under the License. 404 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Realtime Analytics using Dataflow and Cloud Memorystore (Redis) 2 | 3 | In today’s fast-paced world, there is emphasis on getting instant insights. 4 | Typical use-cases involve SaaS operators providing real-time metrics for their KPIs 5 | or marketeers' need for quick insights on performance of their offers or experiments on the website. 6 | 7 | This solution will demonstrate how to build a real-time website analytics dashboard on GCP. 8 | ![architecture](realtime-analytics-architecture.svg) 9 | 10 | ### Components 11 | 12 | **User events / Message bus** provides system decoupling, [Pub/Sub](https://cloud.google.com/pubsub) 13 | is a fully managed message/event bus and provides an easy way to handle the fast click-stream 14 | generated by typical websites. The click-stream contains signals which can be processed to derive 15 | insights in real time. 16 | 17 | **Metrics processing pipeline** is required to process the click-stream from Pub/Sub into the 18 | metrics database. [Dataflow](https://cloud.google.com/dataflow) will be used, which is a 19 | serverless, fully managed processing service supporting real-time streaming jobs. 20 | 21 | **Metrics Database**, needs to be an in-memory database to support real-time use-cases. 22 | Some common web analytic metrics are unique visitors, number of active experiments, conversion rate 23 | of each experiment, etc. The common theme is to calculate uniques, i.e. Cardinality counting, 24 | although from a marketeer's standpoint a good estimation is sufficient, the 25 | [HyperLogLog](https://en.wikipedia.org/wiki/HyperLogLog) algorithm is an efficient solution to the 26 | count-unique problem by trading off some accuracy. 27 | 28 | [Cloud Memorystore (Redis)](https://cloud.google.com/memorystore/docs/redis/redis-overview) provides 29 | a slew of in-built functions for sets and cardinality measurement, alleviating the need to perform 30 | them in code. 31 | 32 | The analytics reporting and visualization makes the reports available to the marketeer easily. 33 | A **Spring dashboard application** is used for demo purposes only. The application uses 34 | [Jedis](https://github.com/xetorthio/jedis) client to access metrics from Redis using 35 | [`scard`](https://redis.io/commands/scard) and 36 | [`sinterstore`](https://redis.io/commands/sinterstore) commands for identifying user overlap and 37 | other cardinality values. It then uses Javascript based web-ui to render graphs using 38 | [Google Charts](https://developers.google.com/chart) library. 39 | 40 | ## Video Tutorial 41 | | Part 1 | Part 2 | 42 | | ------ | ------ | 43 | | [![Part-1](https://img.youtube.com/vi/7NvgleOy480/1.jpg)](https://www.youtube.com/watch?v=7NvgleOy480) | [![Part-2](https://img.youtube.com/vi/FyDNn7gZNi4/1.jpg)](https://www.youtube.com/watch?v=FyDNn7gZNi4) | 44 | 45 | ## Quick Start 46 | [![Open in Cloud Shell](http://gstatic.com/cloudssh/images/open-btn.svg)](https://console.cloud.google.com/cloudshell/editor?cloudshell_git_repo=https://github.com/GoogleCloudPlatform/redis-dataflow-realtime-analytics.git) 47 | 48 | ### Setup Environment 49 | 1. Clone this repository 50 | ```shell script 51 | git clone https://github.com/GoogleCloudPlatform/redis-dataflow-realtime-analytics.git 52 | cd redis-dataflow-realtime-analytics 53 | ``` 54 | 2. Update and activate all environment variables in `set_variables.sh` 55 | ```shell script 56 | source set_variables.sh 57 | ``` 58 | 3. Enable required Cloud products 59 | ```shell script 60 | gcloud services enable \ 61 | compute.googleapis.com \ 62 | pubsub.googleapis.com \ 63 | redis.googleapis.com \ 64 | dataflow.googleapis.com \ 65 | storage-component.googleapis.com 66 | ``` 67 | ### Create Pub/Sub Topic 68 | Pub/Sub is a global message bus enabling easy message consumption in a decoupled fashion. 69 | Create a Pub/Sub topic to receive application instrumentation messages 70 | ```shell script 71 | gcloud pubsub topics create $APP_EVENTS_TOPIC --project $PROJECT_ID 72 | ``` 73 | ### Create VPC network 74 | Protecting the Redis instance is important as it does not provide any protections from external entities. 75 | 76 | 1. Creating a sepate VPC network with external ingress blocked by a firewall provides basic security for the instance. 77 | ```shell script 78 | gcloud compute networks create $VPC_NETWORK_NAME \ 79 | --subnet-mode=auto \ 80 | --bgp-routing-mode=regional 81 | ``` 82 | 2. Create Firewall rule to enable SSH 83 | ```shell script 84 | gcloud compute firewall-rules create allow-internal-ssh \ 85 | --network $VPC_NETWORK_NAME \ 86 | --allow tcp:22,icmp 87 | ``` 88 | 89 | ### Configure Cloud Memorystore 90 | [Cloud Memorystore](https://cloud.google.com/memorystore) provides a fully managed [Redis](https://redis.io/) database. 91 | Redis is a NoSQL In-Memory database, which offers comprehensive in-built functions for 92 | [SETs](https://redis.io/commands#set) operations, 93 | including efficient HLL operations for cardinality measurement. 94 | 95 | 1. Create Redis instance in Memorystore. 96 | ```shell script 97 | gcloud redis instances create $REDIS_NAME \ 98 | --size=1 \ 99 | --region=$REGION_ID \ 100 | --zone="$ZONE_ID" \ 101 | --network=$VPC_NETWORK_NAME \ 102 | --tier=standard 103 | ``` 104 | > Be patient, this can take some time. 105 | 2. Capture instance's IP to configure the Dataflow and Visualization application 106 | ```shell script 107 | export REDIS_IP="$(gcloud redis instances describe $REDIS_NAME --region=$REGION_ID \ 108 | | grep host \ 109 | | sed 's/host: //')" 110 | ``` 111 | 112 | ### Start Analytics pipeline 113 | The analytic metrics pipeline will read click-stream messages from Pub/Sub and update metrics in the Redis database in real-time. The visualization application can then use the Redis database for the dashboard. 114 | 115 | 1. Create Cloud Storage bucket for temporary and staging area for the pipeline 116 | ```shell script 117 | gsutil mb -l $REGION_ID -p $PROJECT_ID gs://$TEMP_GCS_BUCKET 118 | ``` 119 | 2. Launch the pipeline using [Maven](https://apache.org/maven) 120 | ```shell script 121 | cd processor 122 | ``` 123 | ```shell script 124 | mvn clean compile exec:java \ 125 | -Dexec.mainClass=com.google.cloud.solutions.realtimedash.pipeline.MetricsCalculationPipeline \ 126 | -Dexec.cleanupDaemonThreads=false \ 127 | -Dmaven.test.skip=true \ 128 | -Dexec.args=" \ 129 | --streaming \ 130 | --project=$PROJECT_ID \ 131 | --runner=DataflowRunner \ 132 | --stagingLocation=gs://$TEMP_GCS_BUCKET/stage/ \ 133 | --tempLocation=gs://$TEMP_GCS_BUCKET/temp/ \ 134 | --inputTopic=projects/$PROJECT_ID/topics/$APP_EVENTS_TOPIC \ 135 | --workerMachineType=n1-standard-4 \ 136 | --region=$REGION_ID \ 137 | --subnetwork=regions/$REGION_ID/subnetworks/$VPC_NETWORK_NAME \ 138 | --redisHost=$REDIS_IP \ 139 | --redisPort=6379" 140 | ``` 141 | 142 | ### Start the dummy website events generator 143 | The dummy event generator is a Python executable, which needs to keep running, this can be achieved by launching the generator in a __separate shell session__. 144 | 145 | 1. Create and initialize a new python3 virtual environment (you need to have `pyhton3-venv` package) 146 | ```shell script 147 | python3 -m venv ~/generator-venv 148 | source ~/generator-venv/bin/activate 149 | pip install -r loggen/requirements.txt 150 | ``` 151 | 2. Run the logs generator 152 | ```shell script 153 | python loggen/message_generator.py \ 154 | --topic $APP_EVENTS_TOPIC \ 155 | --project-id $PROJECT_ID \ 156 | --enable-log true 157 | ``` 158 | 159 | ### Run the Visualization Engine 160 | Use the simple reporting application located in `dashboard/` folder, built using SpringBoot and simple HTML+JS based UI. 161 | 162 | The application reads the metrics from the Redis database and makes it available to the dashboard UI. 163 | The Application server needs to be on the same VPC network as the Redis server, to achieve this for demo purposes, 164 | we will use a Proxy VM to tunnel the ports to Cloud Shell VM, as its not on the same network. 165 | 166 | 1. Create a VM to act as proxy 167 | ```shell script 168 | gcloud compute instances create proxy-server \ 169 | --zone $ZONE_ID \ 170 | --image-family debian-10 \ 171 | --image-project debian-cloud \ 172 | --network $VPC_NETWORK_NAME 173 | ``` 174 | 2. Start SSH port forwarding 175 | ```shell script 176 | gcloud compute ssh proxy-server --zone $ZONE_ID -- -N -L 6379:$REDIS_IP:6379 -4 & 177 | ``` 178 | 3. Start the Visualization Spring boot application. 179 | ```shell script 180 | cd dashboard/ 181 | mvn clean compile package spring-boot:run 182 | ``` 183 | 4. Click on the ![web-preview](https://cloud.google.com/shell/docs/images/web_preview.svg) icon to open [web preview](https://cloud.google.com/shell/docs/using-web-preview), 184 | to access the application's web-ui in the browser. 185 | 186 | a. Click "Preview on port 8080" 187 | b. On the dashboard, click "Auto Update" which will keep the dashboard fresh. 188 | 189 | __Sample Dashbaord__ 190 | ![dashboard-screenshot](dashboard/dashboard_screenshot.jpg) 191 | -------------------------------------------------------------------------------- /dashboard/.gitignore: -------------------------------------------------------------------------------- 1 | HELP.md 2 | target/ 3 | !.mvn/wrapper/maven-wrapper.jar 4 | !**/src/main/** 5 | !**/src/test/** 6 | 7 | ### STS ### 8 | .apt_generated 9 | .classpath 10 | .factorypath 11 | .project 12 | .settings 13 | .springBeans 14 | .sts4-cache 15 | 16 | ### IntelliJ IDEA ### 17 | .idea 18 | *.iws 19 | *.iml 20 | *.ipr 21 | 22 | ### NetBeans ### 23 | /nbproject/private/ 24 | /nbbuild/ 25 | /dist/ 26 | /nbdist/ 27 | /.nb-gradle/ 28 | build/ 29 | 30 | ### VS Code ### 31 | .vscode/ 32 | -------------------------------------------------------------------------------- /dashboard/dashboard_screenshot.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/redis-dataflow-realtime-analytics/48fec36ba1630cdd5eb48ff0e91aac74eeb733b5/dashboard/dashboard_screenshot.jpg -------------------------------------------------------------------------------- /dashboard/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 20 | 4.0.0 21 | 22 | org.springframework.boot 23 | spring-boot-starter-parent 24 | 3.2.3 25 | 26 | 27 | com.google.cloud.example.realtimedash 28 | dashboard 29 | 0.0.1-SNAPSHOT 30 | dashboard 31 | realtime dashboard for web metrics from cloud memorystore 32 | 33 | 34 | 1.8 35 | 36 | 37 | 38 | 39 | org.springframework.boot 40 | spring-boot-starter-data-redis 41 | 42 | 43 | org.springframework.boot 44 | spring-boot-starter-web 45 | 46 | 47 | 48 | redis.clients 49 | jedis 50 | 5.1.1 51 | jar 52 | compile 53 | 54 | 55 | 56 | com.google.auto.value 57 | auto-value 58 | 1.10.4 59 | 60 | 61 | 62 | joda-time 63 | joda-time 64 | 2.12.7 65 | 66 | 67 | 68 | com.google.guava 69 | guava 70 | 33.0.0-jre 71 | 72 | 73 | 74 | com.fasterxml.jackson.datatype 75 | jackson-datatype-joda 76 | 2.16.1 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | org.springframework.boot 85 | spring-boot-maven-plugin 86 | 87 | com.google.cloud.solutions.realtimedash.dashboard.DashboardApplication 88 | 89 | ZIP 90 | 91 | 92 | 93 | 94 | repackage 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/google/cloud/solutions/realtimedash/dashboard/DashboardApplication.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.dashboard; 18 | 19 | import org.springframework.boot.SpringApplication; 20 | import org.springframework.boot.autoconfigure.SpringBootApplication; 21 | 22 | @SpringBootApplication 23 | public class DashboardApplication { 24 | 25 | public static void main(String[] args) { 26 | SpringApplication.run(DashboardApplication.class, args); 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/google/cloud/solutions/realtimedash/dashboard/JedisBeanConfiguration.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.dashboard; 18 | 19 | import org.springframework.beans.factory.annotation.Value; 20 | import org.springframework.context.annotation.Bean; 21 | import org.springframework.context.annotation.Configuration; 22 | import redis.clients.jedis.Jedis; 23 | 24 | @Configuration 25 | public class JedisBeanConfiguration { 26 | 27 | @Value("${spring.redis.host}") 28 | private String REDIS_HOST; 29 | 30 | @Value("${spring.redis.port}") 31 | private Integer REDIS_PORT; 32 | 33 | @Bean 34 | Jedis redisClient() { 35 | return new Jedis(REDIS_HOST, REDIS_PORT); 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/google/cloud/solutions/realtimedash/dashboard/OverlapMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.dashboard; 18 | 19 | import com.google.auto.value.AutoValue; 20 | import com.google.common.collect.ImmutableSet; 21 | 22 | @AutoValue 23 | public abstract class OverlapMetric { 24 | 25 | public abstract ImmutableSet getDimensions(); 26 | 27 | public abstract Double getMetric(); 28 | 29 | 30 | public static Builder builder() { 31 | return new AutoValue_OverlapMetric.Builder(); 32 | } 33 | 34 | @AutoValue.Builder 35 | public abstract static class Builder { 36 | 37 | public abstract Builder setMetric(Double newMetric); 38 | 39 | public Builder setMetric(float newMetric) { 40 | return setMetric((double) newMetric); 41 | } 42 | 43 | public Builder setMetric(int newMetric) { 44 | return setMetric((double) newMetric); 45 | } 46 | 47 | public Builder setMetric(long newMetric) { 48 | return setMetric((double) newMetric); 49 | } 50 | 51 | public abstract Builder setDimensions(ImmutableSet dimensions); 52 | 53 | public abstract OverlapMetric build(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/google/cloud/solutions/realtimedash/dashboard/TimeSeriesKeyBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.dashboard; 18 | 19 | import java.util.Arrays; 20 | import org.joda.time.DateTime; 21 | 22 | public class TimeSeriesKeyBuilder { 23 | 24 | private final String dateTimeFormatter; 25 | 26 | private TimeSeriesKeyBuilder(String prefix) { 27 | this.dateTimeFormatter = "'" + prefix + "'" + "_yyyy_MM_dd'T'HH_mm"; 28 | } 29 | 30 | public String buildTimeKey(DateTime dateTime) { 31 | return dateTime.toString(dateTimeFormatter); 32 | } 33 | 34 | public String[] buildTimeKeys(DateTime[] dateTimes) { 35 | return Arrays.stream(dateTimes) 36 | .map(this::buildTimeKey) 37 | .toArray(String[]::new); 38 | } 39 | 40 | public static TimeSeriesKeyBuilder forPrefix(String prefix) { 41 | return new TimeSeriesKeyBuilder(prefix); 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/google/cloud/solutions/realtimedash/dashboard/TimeSeriesMetric.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.dashboard; 18 | 19 | import com.google.auto.value.AutoValue; 20 | import org.joda.time.DateTime; 21 | 22 | @AutoValue 23 | public abstract class TimeSeriesMetric { 24 | 25 | public abstract DateTime getTimestamp(); 26 | 27 | public abstract Double getMetric(); 28 | 29 | public static Builder builder() { 30 | return new AutoValue_TimeSeriesMetric.Builder(); 31 | } 32 | 33 | 34 | @AutoValue.Builder 35 | public abstract static class Builder { 36 | 37 | public abstract Builder setTimestamp(DateTime newTimestamp); 38 | 39 | public abstract Builder setMetric(Double newMetric); 40 | 41 | public Builder setMetric(float newMetric) { 42 | return setMetric((double) newMetric); 43 | } 44 | 45 | public Builder setMetric(int newMetric) { 46 | return setMetric((double) newMetric); 47 | } 48 | 49 | public Builder setMetric(long newMetric) { 50 | return setMetric((double) newMetric); 51 | } 52 | 53 | public abstract TimeSeriesMetric build(); 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /dashboard/src/main/java/com/google/cloud/solutions/realtimedash/dashboard/TimeseriesMetricsController.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.dashboard; 18 | 19 | import static com.google.common.collect.ImmutableSet.toImmutableSet; 20 | 21 | import com.google.common.base.Joiner; 22 | import com.google.common.collect.ImmutableList; 23 | import com.google.common.collect.ImmutableSet; 24 | import com.google.common.collect.Sets; 25 | import java.util.Arrays; 26 | import java.util.List; 27 | import java.util.Set; 28 | import java.util.stream.IntStream; 29 | import org.joda.time.DateTime; 30 | import org.joda.time.DateTimeZone; 31 | import org.joda.time.Duration; 32 | import org.springframework.web.bind.annotation.GetMapping; 33 | import org.springframework.web.bind.annotation.RequestMapping; 34 | import org.springframework.web.bind.annotation.RequestParam; 35 | import org.springframework.web.bind.annotation.RestController; 36 | import redis.clients.jedis.Jedis; 37 | 38 | @RestController 39 | @RequestMapping("/metrics/timeseries") 40 | public class TimeseriesMetricsController { 41 | 42 | private static final String REDIS_NIL = "(nil)"; 43 | private static final String DEFAULT_REPORT_TIME_MINUTE = "10"; 44 | 45 | @GetMapping("/visits") 46 | public ImmutableList visits(Jedis redisClient, 47 | @RequestParam(required = false, defaultValue = DEFAULT_REPORT_TIME_MINUTE) int lastMinutes) { 48 | DateTime[] times = timePatternForLastMinutes(lastMinutes); 49 | 50 | List values = redisClient 51 | .mget(TimeSeriesKeyBuilder.forPrefix("visitCounter").buildTimeKeys(times)); 52 | 53 | ImmutableList.Builder visitsMetricBuilder = ImmutableList.builder(); 54 | 55 | for (int index = 0; index < times.length; index++) { 56 | int value = getIntValue(values.get(index)); 57 | 58 | visitsMetricBuilder 59 | .add(TimeSeriesMetric.builder().setTimestamp(times[index]).setMetric(value).build()); 60 | } 61 | 62 | return visitsMetricBuilder.build(); 63 | } 64 | 65 | private Integer getIntValue(String value) { 66 | return (value == null || value.equals(REDIS_NIL)) ? 0 : Integer.valueOf(value); 67 | } 68 | 69 | @GetMapping("/users") 70 | public ImmutableList users(Jedis redisClient, 71 | @RequestParam(required = false, defaultValue = DEFAULT_REPORT_TIME_MINUTE) int lastMinutes) { 72 | ImmutableList.Builder usersMetricBuilder = ImmutableList.builder(); 73 | 74 | TimeSeriesKeyBuilder keyBuilder = TimeSeriesKeyBuilder.forPrefix("hll_dthr"); 75 | 76 | for (DateTime time : timePatternForLastMinutes(lastMinutes)) { 77 | usersMetricBuilder.add( 78 | TimeSeriesMetric 79 | .builder() 80 | .setTimestamp(time) 81 | .setMetric(redisClient.pfcount(keyBuilder.buildTimeKey(time))) 82 | .build()); 83 | } 84 | 85 | return usersMetricBuilder.build(); 86 | } 87 | 88 | @GetMapping("/experiments") 89 | public ImmutableList recentExperiments(Jedis redisClient, 90 | @RequestParam(required = false, defaultValue = DEFAULT_REPORT_TIME_MINUTE) int lastMinutes) { 91 | TimeSeriesKeyBuilder keyBuilder = TimeSeriesKeyBuilder 92 | .forPrefix("set_experiments_experiments"); 93 | 94 | ImmutableList.Builder experimentsMetricBuilder = ImmutableList.builder(); 95 | 96 | for (DateTime time : timePatternForLastMinutes(lastMinutes)) { 97 | long value = redisClient.scard(keyBuilder.buildTimeKey(time)); 98 | 99 | experimentsMetricBuilder.add( 100 | TimeSeriesMetric.builder() 101 | .setTimestamp(time) 102 | .setMetric(value) 103 | .build()); 104 | } 105 | 106 | return experimentsMetricBuilder.build(); 107 | } 108 | 109 | @GetMapping("/variantsOverlap") 110 | public ImmutableSet variantOverlap(Jedis redisClient) { 111 | Set variants = redisClient.keys("set_var_*"); 112 | 113 | if (variants == null || variants.size() == 0) { 114 | return ImmutableSet.of(); 115 | } 116 | 117 | return Sets.combinations(variants, 2) 118 | .stream() 119 | .map(variantCombination -> variantCombination.toArray(new String[0])) 120 | .map(variantCombination -> { 121 | 122 | String sinterStoreKey = Joiner.on("-").join("overlap_", variantCombination); 123 | 124 | redisClient.sinterstore(sinterStoreKey, variantCombination); 125 | 126 | ImmutableSet variantCombinationSet 127 | = Arrays.stream(variantCombination).map(name -> name.replaceFirst("set_var_", "")) 128 | .collect(toImmutableSet()); 129 | 130 | return OverlapMetric.builder() 131 | .setDimensions(variantCombinationSet) 132 | .setMetric(redisClient.scard(sinterStoreKey)) 133 | .build(); 134 | }) 135 | .collect(toImmutableSet()); 136 | } 137 | 138 | @GetMapping("/times") 139 | public DateTime[] getTimeString( 140 | @RequestParam(required = false, defaultValue = "10") Integer lastMinutes) { 141 | return timePatternForLastMinutes(lastMinutes); 142 | } 143 | 144 | private static DateTime[] timePatternForLastMinutes(int pastMinutes) { 145 | DateTime startTime = DateTime.now(DateTimeZone.UTC).minuteOfHour().roundFloorCopy(); 146 | 147 | return IntStream.rangeClosed(1, pastMinutes) 148 | .boxed() 149 | .map(Duration::standardMinutes) 150 | .map(startTime::minus) 151 | .toArray(DateTime[]::new); 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /dashboard/src/main/resources/application.properties: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2020 Google LLC 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | # 16 | spring.redis.host=localhost 17 | spring.redis.port=6379 18 | -------------------------------------------------------------------------------- /dashboard/src/main/resources/static/index.html: -------------------------------------------------------------------------------- 1 | 16 | 17 | 18 | 19 | 20 | 21 | Realtime Dashboard 22 | 23 | 24 | 25 | 26 |

Shiny Realtime dashboard

27 | 28 | 29 | 30 | 31 | 35 | 36 | 37 | 40 | 43 | 44 | 45 | 48 | 49 |
32 |

Unique Users Overlap in variants

33 |
34 |
38 |
39 | 		41 |
42 |
46 |
47 |
50 | 51 | 52 | 53 | -------------------------------------------------------------------------------- /dashboard/src/main/resources/static/metrics_ui.js: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | const UPDATE_INTERVAL_MILLS = 1000; 18 | const CHARTS_CONFIG = [ 19 | { 20 | name: 'visits', 21 | endpoint: '/metrics/timeseries/visits', 22 | chartClass: 'google.visualization.BarChart', 23 | elementId: 'visits', 24 | metricName: 'Visits', 25 | title: 'Visits/min', 26 | chartOptions: {bars: 'vertical'}, 27 | chartDataTransform: transformTimeSeriesToGraphData 28 | }, 29 | { 30 | name: 'users', 31 | endpoint: '/metrics/timeseries/users', 32 | chartClass: 'google.visualization.BarChart', 33 | elementId: 'users', 34 | metricName: 'Unique Users', 35 | title: 'Unique Users/min', 36 | chartOptions: {bars: 'vertical'}, 37 | chartDataTransform: transformTimeSeriesToGraphData 38 | }, 39 | { 40 | name: 'experiements', 41 | endpoint: '/metrics/timeseries/experiments', 42 | chartClass: 'google.visualization.LineChart', 43 | elementId: 'experiments', 44 | metricName: 'Live Experiments', 45 | title: 'Live Experiments/min', 46 | chartDataTransform: transformTimeSeriesToGraphData 47 | }, 48 | { 49 | name: 'variant_overlap', 50 | endpoint: '/metrics/timeseries/variantsOverlap', 51 | chartClass: 'google.visualization.Table', 52 | elementId: 'variant-overlap', 53 | metricName: 'Variant', 54 | title: 'Variant User Overlap', 55 | chartOptions: {width: '100%', height: '100%'}, 56 | chartDataTransform: transformOverlapDataToGraphData 57 | } 58 | ]; 59 | 60 | const BASIC_CHART_OPTIONS = (chartConfig) => { 61 | return {title: chartConfig.title, animation: {duration: 500, easing: 'out'}} 62 | }; 63 | 64 | let chartObjects = {}; 65 | 66 | let intervalTimerId = null; 67 | 68 | function transformTimeSeriesToGraphData(chartMetricName) { 69 | return (timeSeriesData) => { 70 | const chartData = new google.visualization.DataTable(); 71 | chartData.addColumn('date', 'Time of Day'); 72 | chartData.addColumn('number', chartMetricName); 73 | chartData.addRows(timeSeriesData.map( 74 | item => [new Date(item['timestamp']), item.metric])); 75 | 76 | return chartData; 77 | }; 78 | } 79 | 80 | function transformOverlapDataToGraphData(chartMetricName) { 81 | return (data) => { 82 | const uniqueDimensions = [...new Set(data.flatMap(x => x.dimensions))]; 83 | const chartData = new google.visualization.DataTable(); 84 | chartData.addColumn('string', chartMetricName); 85 | uniqueDimensions.forEach( 86 | dimension => chartData.addColumn('number', dimension)); 87 | 88 | let overlapMap = {}; 89 | data.forEach(item => { 90 | if (!overlapMap[item.dimensions[0]]) { 91 | overlapMap[item.dimensions[0]] = {}; 92 | } 93 | overlapMap[item.dimensions[0]][item.dimensions[1]] = item.metric; 94 | }); 95 | 96 | chartData.addRows( 97 | Object.keys(overlapMap).map( 98 | dimension => [dimension].concat(uniqueDimensions.map( 99 | overlapDimension => overlapMap[dimension][overlapDimension]))) 100 | ); 101 | 102 | return chartData; 103 | }; 104 | } 105 | 106 | function drawChart(metricsJson, chartConfig) { 107 | const chartOptions = { 108 | ...BASIC_CHART_OPTIONS(chartConfig), ...((chartConfig.chartOptions) 109 | ? chartConfig.chartOptions : {}) 110 | }; 111 | 112 | const transformFunction = chartConfig.chartDataTransform( 113 | chartConfig.metricName); 114 | chartObjects[chartConfig.name] 115 | .draw(transformFunction(metricsJson), chartOptions); 116 | } 117 | 118 | function initAllCharts() { 119 | for (let chartConfig of CHARTS_CONFIG) { 120 | const classObj = eval(`${chartConfig.chartClass}`); 121 | chartObjects[chartConfig.name] = (new classObj( 122 | document.getElementById(chartConfig.elementId))); 123 | retrieveFromServerAndDraw(chartConfig) 124 | } 125 | } 126 | 127 | function retrieveAllConfigs() { 128 | CHARTS_CONFIG.forEach(config => retrieveFromServerAndDraw(config)); 129 | } 130 | 131 | function retrieveFromServerAndDraw(chartConfig) { 132 | fetch(chartConfig.endpoint) 133 | .then(results => results.json()) 134 | .then(results => drawChart(results, chartConfig)); 135 | } 136 | 137 | function toggleAutoUpdate() { 138 | const updateBtn = document.getElementById('btn_auto_update'); 139 | if (intervalTimerId) { 140 | clearInterval(intervalTimerId); 141 | intervalTimerId = null; 142 | updateBtn.style.background = ''; 143 | updateBtn.style.color = 'black'; 144 | } else { 145 | intervalTimerId = setInterval(retrieveAllConfigs, UPDATE_INTERVAL_MILLS); 146 | updateBtn.style.background = 'green'; 147 | updateBtn.style.color = 'white'; 148 | } 149 | } 150 | 151 | google.charts.load('current', 152 | {'packages': ['corechart', 'line', 'bar', 'table']}); 153 | google.charts.setOnLoadCallback(initAllCharts); -------------------------------------------------------------------------------- /loggen/message_generator.py: -------------------------------------------------------------------------------- 1 | # Copyright 2020 Google LLC 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Pushes defined number of random messages to the Pub/Sub.""" 16 | 17 | import argparse 18 | import datetime 19 | import json 20 | import random 21 | import signal 22 | import sys 23 | import time 24 | 25 | from google.cloud import pubsub_v1 26 | 27 | # Configure Command line parser for arguments 28 | cmd_flags_parser = argparse.ArgumentParser( 29 | description='Publish messages to Pub/Sub', 30 | prefix_chars='-') 31 | cmd_flags_parser.add_argument('--event_count', type=int, 32 | help='Number of log events to generate', 33 | default=-1) 34 | cmd_flags_parser.add_argument('--topic', type=str, 35 | help='Name of the Pub/Sub topic') 36 | cmd_flags_parser.add_argument('--project-id', type=str, 37 | help='GCP Project Id running the Pub/Sub') 38 | cmd_flags_parser.add_argument('--enable-log', type=bool, 39 | default=False, 40 | help='print logs') 41 | 42 | # Extract command-line arguments 43 | cmd_arguments = cmd_flags_parser.parse_args() 44 | 45 | # Define configuration 46 | _LOGGING_ENABLED = cmd_arguments.enable_log 47 | _EXPERIMENT_VARIANTS = ['default', '1', '2', '3'] 48 | _SEND_EVENTS_COUNT = cmd_arguments.event_count # Default send infinite messages 49 | _PUB_SUB_TOPIC = cmd_arguments.topic 50 | _GCP_PROJECT_ID = cmd_arguments.project_id 51 | _PUBLISHER = pubsub_v1.PublisherClient() 52 | _START_TIME = time.time() 53 | _TOPIC_PATH = _PUBLISHER.topic_path(_GCP_PROJECT_ID, _PUB_SUB_TOPIC) 54 | 55 | message_count = 0 56 | 57 | 58 | def build_user_id(): 59 | """ 60 | Generates random user ids with some overlap to simulate a real world 61 | user behaviour on an app or website. 62 | :return: A slowly changing random number. 63 | """ 64 | elapsed_tens_minutes = int((time.time() - _START_TIME) / 600) + 1 65 | present_millis = int(1000 * (time.time() - int(time.time()))) 66 | 67 | if present_millis == 0: 68 | present_millis = random.randint(1,1000) 69 | 70 | if _LOGGING_ENABLED: 71 | print( 72 | 'generating user_id: elapsed_tens_minute = {}, present_millis = {}'.format( 73 | elapsed_tens_minutes, present_millis)) 74 | 75 | return random.randint(elapsed_tens_minutes + present_millis, 76 | (10 + elapsed_tens_minutes) * present_millis) 77 | 78 | 79 | def build_message(): 80 | """ Generates an event message imitation 81 | :return: A random event message 82 | """ 83 | return dict( 84 | uid=build_user_id(), 85 | # change experiment ids based on date/time 86 | experiment_id=random.randint(1, 100), 87 | variant=_EXPERIMENT_VARIANTS[random.randint(0, 3)], 88 | timestamp=datetime.datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%SZ')) 89 | 90 | 91 | def send_to_pub_sub(message): 92 | """ Sends the provided payload as JSON to Pub/Sub. 93 | :param message: the Event information payload 94 | :return: the published message future. 95 | """ 96 | return _PUBLISHER.publish(_TOPIC_PATH, 97 | data=json.dumps(message).encode('utf-8')) 98 | 99 | 100 | def print_message_count_before_exit(sig, frame): 101 | """ Interrupt handler to print the count of messages sent to pub/sub before 102 | exiting python. 103 | :param sig: the interrupt signal. 104 | :param frame: the stackframe. 105 | """ 106 | print('\nSent {} messages.\nExiting'.format(message_count)) 107 | sys.exit(0) 108 | 109 | 110 | # Register message count printer 111 | signal.signal(signal.SIGINT, print_message_count_before_exit) 112 | 113 | print('ProjectId: {}\nPub/Sub Topic: {}'.format(_GCP_PROJECT_ID, _TOPIC_PATH)) 114 | print('Sending events in background.') 115 | print('Press Ctrl+C to exit/stop.') 116 | 117 | # Infinite loop to keep sending messages to pub/sub 118 | while _SEND_EVENTS_COUNT == -1 or message_count < _SEND_EVENTS_COUNT: 119 | event_message = build_message() 120 | if (_LOGGING_ENABLED): 121 | print('Sending Message {}\n{}'.format(message_count + 1, 122 | json.dumps(event_message))) 123 | 124 | message_count += 1 125 | pub_sub_message_unique_id = send_to_pub_sub(event_message) 126 | 127 | if (_LOGGING_ENABLED): 128 | print( 129 | 'pub_sub_message_id: {}'.format(pub_sub_message_unique_id.result())) 130 | 131 | _sleep_time = random.randint(10, 10000) # Random sleep time in milli-seconds. 132 | if (_LOGGING_ENABLED): 133 | print('Sleeping for {} ms'.format(_sleep_time)) 134 | time.sleep(_sleep_time / 1000) 135 | -------------------------------------------------------------------------------- /loggen/requirements.txt: -------------------------------------------------------------------------------- 1 | google 2 | google-cloud-pubsub 3 | -------------------------------------------------------------------------------- /processor/deploy_dataflow.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | mvn clean compile exec:java \ 19 | -Dexec.mainClass=com.google.cloud.solutions.realtimedash.pipeline.MetricsCalculationPipeline \ 20 | -Dexec.cleanupDaemonThreads=false \ 21 | -Dmaven.test.skip=true \ 22 | -Dexec.args=" \ 23 | --project=$PROJECT_ID \ 24 | --runner=DataflowRunner \ 25 | --stagingLocation=gs://$TEMP_GCS_BUCKET/stage/ \ 26 | --tempLocation=gs://$TEMP_GCS_BUCKET/temp/ \ 27 | --inputTopic=projects/$PROJECT_ID/topics/$APP_EVENTS_TOPIC \ 28 | --workerMachineType=n1-standard-4 \ 29 | --region=$REGION_ID \ 30 | --subnetwork=regions/$REGION_ID/subnetworks/$VPC_NETWORK_NAME \ 31 | --redisHost=$REDIS_IP \ 32 | --redisPort=6379 \ 33 | --streaming\ 34 | " 35 | -------------------------------------------------------------------------------- /processor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 17 | 18 | 21 | 4.0.0 22 | com.google.cloud.solutions.rtdash 23 | processor 24 | 1.0-SNAPSHOT 25 | processor 26 | 27 | 28 | 2.54.0 29 | 4.13.2 30 | 8 31 | 1.8 32 | 1.8 33 | 3.12.1 34 | 3.2.0 35 | 3.3.0 36 | 3.5.2 37 | 3.2.5 38 | UTF-8 39 | 2.0.12 40 | 41 | 42 | 43 | junit 44 | junit 45 | ${junit.version} 46 | test 47 | 48 | 49 | org.apache.commons 50 | commons-lang3 51 | 3.14.0 52 | 53 | 54 | 55 | 56 | org.apache.beam 57 | beam-sdks-java-core 58 | ${beam.version} 59 | 60 | 61 | 62 | 63 | org.apache.beam 64 | beam-runners-google-cloud-dataflow-java 65 | ${beam.version} 66 | runtime 67 | 68 | 69 | 70 | 71 | org.apache.beam 72 | beam-sdks-java-io-google-cloud-platform 73 | ${beam.version} 74 | 75 | 76 | org.apache.beam 77 | beam-sdks-java-io-redis 78 | ${beam.version} 79 | 80 | 81 | com.fasterxml.jackson.datatype 82 | jackson-datatype-joda 83 | 2.16.1 84 | 85 | 86 | com.google.auto.value 87 | auto-value 88 | 1.10.4 89 | 90 | 91 | com.google.flogger 92 | flogger 93 | 0.8 94 | 95 | 96 | com.google.flogger 97 | flogger-system-backend 98 | 0.8 99 | runtime 100 | 101 | 102 | org.slf4j 103 | slf4j-api 104 | ${slf4j.version} 105 | 106 | 107 | org.slf4j 108 | slf4j-jdk14 109 | ${slf4j.version} 110 | 111 | runtime 112 | 113 | 114 | 115 | 116 | 117 | 118 | com.google.guava 119 | guava 120 | 33.0.0-jre 121 | 122 | 123 | 124 | 125 | com.google.cloud 126 | libraries-bom 127 | 26.33.0 128 | pom 129 | import 130 | 131 | 132 | 133 | 134 | 135 | 136 | org.apache.maven.plugins 137 | maven-compiler-plugin 138 | ${maven-compiler-plugin.version} 139 | 140 | true 141 | -parameters 142 | ${maven.compiler.release} 143 | ${maven.compiler.source} 144 | ${maven.compiler.target} 145 | 146 | 147 | 148 | 149 | org.apache.maven.plugins 150 | maven-surefire-plugin 151 | ${maven-surefire-plugin.version} 152 | 153 | all 154 | 4 155 | true 156 | 157 | 158 | 159 | org.apache.maven.surefire 160 | surefire-junit47 161 | ${maven-surefire-plugin.version} 162 | 163 | 164 | junit 165 | junit 166 | ${junit.version} 167 | 168 | 169 | 170 | 171 | 173 | 174 | org.apache.maven.plugins 175 | maven-jar-plugin 176 | ${maven-jar-plugin.version} 177 | 178 | 179 | 183 | 184 | org.apache.maven.plugins 185 | maven-shade-plugin 186 | ${maven-shade-plugin.version} 187 | 188 | 189 | package 190 | 191 | shade 192 | 193 | 194 | ${project.artifactId}-bundled-${project.version} 195 | 196 | 197 | *:* 198 | 199 | META-INF/LICENSE 200 | META-INF/*.SF 201 | META-INF/*.DSA 202 | META-INF/*.RSA 203 | 204 | 205 | 206 | 207 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | org.codehaus.mojo 220 | exec-maven-plugin 221 | ${maven-exec-plugin.version} 222 | 223 | false 224 | 225 | 226 | 227 | 228 | 229 | 230 | -------------------------------------------------------------------------------- /processor/src/main/java/com/google/cloud/solutions/realtimedash/pipeline/LogEvent.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.pipeline; 18 | 19 | import com.fasterxml.jackson.databind.annotation.JsonDeserialize; 20 | import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; 21 | import com.google.auto.value.AutoValue; 22 | import java.io.Serializable; 23 | import org.joda.time.DateTime; 24 | 25 | /** 26 | * Data model represents the message sent to Pub/Sub Module. 27 | */ 28 | @JsonDeserialize(builder = AutoValue_LogEvent.Builder.class) 29 | @AutoValue 30 | public abstract class LogEvent implements Serializable { 31 | 32 | public static Builder builder() { 33 | return new AutoValue_LogEvent.Builder(); 34 | } 35 | 36 | public abstract String getUid(); 37 | 38 | public abstract String getExperimentId(); 39 | 40 | public abstract String getVariant(); 41 | 42 | public abstract DateTime getTimestamp(); 43 | 44 | @JsonPOJOBuilder(withPrefix = "set") 45 | @AutoValue.Builder 46 | public abstract static class Builder { 47 | 48 | public abstract Builder setUid(String newUid); 49 | 50 | public abstract Builder setExperimentId(String newExperimentId); 51 | 52 | public abstract Builder setVariant(String newVariant); 53 | 54 | public abstract Builder setTimestamp(DateTime newTimestamp); 55 | 56 | public abstract LogEvent build(); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /processor/src/main/java/com/google/cloud/solutions/realtimedash/pipeline/MetricsCalculationPipeline.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.pipeline; 18 | 19 | import com.google.common.flogger.FluentLogger; 20 | import org.apache.beam.sdk.Pipeline; 21 | import org.apache.beam.sdk.io.gcp.pubsub.PubsubIO; 22 | import org.apache.beam.sdk.io.redis.RedisIO; 23 | import org.apache.beam.sdk.io.redis.RedisIO.Write.Method; 24 | import org.apache.beam.sdk.options.PipelineOptionsFactory; 25 | import org.apache.beam.sdk.transforms.DoFn; 26 | import org.apache.beam.sdk.transforms.ParDo; 27 | import org.apache.beam.sdk.transforms.ParDo.SingleOutput; 28 | import org.apache.beam.sdk.values.KV; 29 | import org.apache.beam.sdk.values.PCollection; 30 | import org.apache.commons.lang3.tuple.Pair; 31 | 32 | /** 33 | * Realtime Dataflow pipeline to extract experiment metrics from Log Events published on Pub/Sub. 34 | */ 35 | public final class MetricsCalculationPipeline { 36 | 37 | private static final FluentLogger logger = FluentLogger.forEnclosingClass(); 38 | private static final long DEFAULT_WINDOW_DURATION = 1L; // 1 - second 39 | 40 | /** 41 | * Parses the command line arguments and runs the pipeline. 42 | */ 43 | public static void main(String[] args) { 44 | MetricsPipelineOptions options = extractPipelineOptions(args); 45 | Pipeline pipeline = Pipeline.create(options); 46 | 47 | PCollection parsedLoggedEvents = 48 | pipeline 49 | .apply("Read PubSub Events", 50 | PubsubIO.readStrings().fromTopic(options.getInputTopic())) 51 | .apply("Parse Message JSON", 52 | ParDo.of(new ParseMessageAsLogElement())); 53 | 54 | RedisIO.Write redisWriter = 55 | RedisIO.write() 56 | .withEndpoint( 57 | options.getRedisHost(), options.getRedisPort()); 58 | 59 | //visit counter 60 | parsedLoggedEvents 61 | .apply("Count visits per minute", ParDo.of( 62 | new DoFn>() { 63 | @ProcessElement 64 | public void countSession(ProcessContext context) { 65 | LogEvent event = context.element(); 66 | context.output( 67 | KV.of(event.getTimestamp().toString(timeBasedKeyBuilder("visitCounter")), "1")); 68 | } 69 | } 70 | )) 71 | .apply("Update Visit counter", redisWriter.withMethod(Method.INCRBY)); 72 | 73 | // Build user - experiment/variant metric 74 | parsedLoggedEvents 75 | .apply("extract user for experiment-variant per minute metric", 76 | ParDo.of(new DoFn>() { 77 | 78 | @ProcessElement 79 | public void extractExperimentVariantPerTime(ProcessContext context) { 80 | LogEvent event = context.element(); 81 | 82 | String key = event.getTimestamp().toString(timeBasedKeyBuilder( 83 | "evcounter_e_" + event.getExperimentId() + "_v_" + event.getVariant())); 84 | context.output(KV.of(key, event.getUid())); 85 | } 86 | })) 87 | .apply("Update EV Counters", redisWriter.withMethod(Method.PFADD)); 88 | 89 | // Variant based metrics 90 | PCollection> variants = 91 | parsedLoggedEvents 92 | .apply("Extract Users per Variant", 93 | ParDo.of( 94 | new DoFn>() { 95 | @ProcessElement 96 | public void extractVariantAndUser(ProcessContext context) { 97 | LogEvent elem = context.element(); 98 | context.output(Pair.of("" + elem.getVariant(), elem.getUid())); 99 | } 100 | })); 101 | variants 102 | .apply("Build HLL Keys (Variant)", hllKeyGenerator("var")) 103 | .apply("Add user to Variant HLL", redisWriter.withMethod(Method.PFADD)); 104 | 105 | variants 106 | .apply("Build Set Keys (Variant)", setKeyGenerator("var")) 107 | .apply("Add user to Variant set", redisWriter.withMethod(Method.SADD)); 108 | 109 | // Experiment based metrics 110 | PCollection> experiments = 111 | parsedLoggedEvents 112 | .apply("Extract Users per Experiment", 113 | ParDo.of( 114 | new DoFn>() { 115 | @ProcessElement 116 | public void extractExperimentKvPair(ProcessContext context) { 117 | LogEvent elem = context.element(); 118 | context.output(Pair.of(elem.getExperimentId(), elem.getUid())); 119 | } 120 | } 121 | )); 122 | 123 | experiments 124 | .apply("Build HLL Keys (Experiment)", hllKeyGenerator("exp")) 125 | .apply("Add User to Experiment HLL", redisWriter.withMethod(Method.PFADD)); 126 | 127 | experiments 128 | .apply("Build Set Keys (Experiment)", setKeyGenerator("exp")) 129 | .apply("Add user to Experiment set", redisWriter.withMethod(Method.SADD)); 130 | 131 | PCollection> activeExperiments = 132 | parsedLoggedEvents 133 | .apply("Build Time bound experiments", 134 | ParDo.of(new DoFn>() { 135 | 136 | @ProcessElement 137 | public void extractExperimentForTime(ProcessContext context) { 138 | LogEvent event = context.element(); 139 | 140 | context.output(Pair.of( 141 | event.getTimestamp().toString( 142 | timeBasedKeyBuilder("experiments")), 143 | event.getExperimentId())); 144 | 145 | } 146 | })); 147 | 148 | activeExperiments 149 | .apply("build experiments hll key", hllKeyGenerator("experiments")) 150 | .apply("Write Active experiments data", 151 | redisWriter.withMethod(Method.PFADD)); 152 | 153 | activeExperiments 154 | .apply("build experiments set key", setKeyGenerator("experiments")) 155 | .apply("Write Active experiments count", redisWriter.withMethod(Method.SADD)); 156 | 157 | PCollection> activeVariants = 158 | parsedLoggedEvents 159 | .apply("Build Time bound Variants", 160 | ParDo.of(new DoFn>() { 161 | 162 | @ProcessElement 163 | public void extractExperimentForTime(ProcessContext context) { 164 | LogEvent event = context.element(); 165 | 166 | context.output(Pair.of( 167 | event.getTimestamp().toString( 168 | timeBasedKeyBuilder("variants")), 169 | event.getVariant())); 170 | } 171 | })); 172 | 173 | activeVariants 174 | .apply("build variants hll key", hllKeyGenerator("variants")) 175 | .apply("Write Active experiments data", 176 | redisWriter.withMethod(Method.PFADD)); 177 | 178 | activeVariants 179 | .apply("build variants set key", setKeyGenerator("variants")) 180 | .apply("Write Active experiments count", redisWriter.withMethod(Method.SADD)); 181 | 182 | // Date_Hour_Minute based metrics 183 | PCollection> datesHoursBasedEvents = 184 | parsedLoggedEvents 185 | .apply("Extract Users per minute", extractUsersForDateTime()); 186 | 187 | datesHoursBasedEvents 188 | .apply("Build HLL Keys (Date_Time)", hllKeyGenerator("dthr")) 189 | .apply("Add User to DateTime HLL", redisWriter.withMethod(Method.PFADD)); 190 | 191 | datesHoursBasedEvents 192 | .apply("Build Set Keys (Date_Time)", setKeyGenerator("dthr")) 193 | .apply("Add User to DateTime set", redisWriter.withMethod(Method.SADD)); 194 | 195 | pipeline.run(); 196 | } 197 | 198 | private static SingleOutput, KV> 199 | hllKeyGenerator(String name) { 200 | return 201 | ParDo.of(new DoFn, KV>() { 202 | @ProcessElement 203 | public void buildHllKey(ProcessContext context) { 204 | Pair elem = context.element(); 205 | context.output( 206 | KV.of("hll_" + buildPrefix(name) + elem.getKey(), elem.getValue())); 207 | } 208 | }); 209 | } 210 | 211 | private static SingleOutput, KV> setKeyGenerator( 212 | String name) { 213 | return ParDo.of( 214 | new DoFn, KV>() { 215 | @ProcessElement 216 | public void buildSetKey(ProcessContext context) { 217 | Pair elem = context.element(); 218 | context.output( 219 | KV.of("set_" + buildPrefix(name) + elem.getKey(), elem.getValue())); 220 | } 221 | }); 222 | } 223 | 224 | private static SingleOutput> extractUsersForDateTime() { 225 | return ParDo.of( 226 | new DoFn>() { 227 | @ProcessElement 228 | public void extractDateTimeForUser(ProcessContext context) { 229 | LogEvent elem = context.element(); 230 | context.output( 231 | Pair.of(elem.getTimestamp().toString(timeBasedKeyBuilder(null)), 232 | elem.getUid())); 233 | } 234 | }); 235 | } 236 | 237 | private static String timeBasedKeyBuilder(String prefix) { 238 | return (prefix == null ? "" : ("'" + buildPrefix(prefix) + "'")) + "yyyy_MM_dd'T'HH_mm"; 239 | } 240 | 241 | private static String buildPrefix(String prefix) { 242 | return (prefix == null || prefix.equals("")) ? "" : prefix + "_"; 243 | } 244 | 245 | /** 246 | * Parse Pipeline options from command line arguments. 247 | */ 248 | private static MetricsPipelineOptions extractPipelineOptions(String[] args) { 249 | MetricsPipelineOptions options = PipelineOptionsFactory 250 | .fromArgs(args) 251 | .withValidation() 252 | .as(MetricsPipelineOptions.class); 253 | 254 | return options; 255 | } 256 | } 257 | -------------------------------------------------------------------------------- /processor/src/main/java/com/google/cloud/solutions/realtimedash/pipeline/MetricsPipelineOptions.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.pipeline; 18 | 19 | import org.apache.beam.sdk.options.Description; 20 | import org.apache.beam.sdk.options.PipelineOptions; 21 | import org.apache.beam.sdk.options.Validation; 22 | 23 | /** 24 | * A Streaming pipeline option for the Metrics calculation pipeline. 25 | */ 26 | public interface MetricsPipelineOptions extends PipelineOptions { 27 | 28 | @Description("The Cloud Pub/Sub topic to read from.") 29 | @Validation.Required 30 | String getInputTopic(); 31 | 32 | void setInputTopic(String value); 33 | 34 | @Description("Redis Host") 35 | @Validation.Required 36 | String getRedisHost(); 37 | 38 | void setRedisHost(String redisHost); 39 | 40 | @Description("Redis Port") 41 | @Validation.Required 42 | Integer getRedisPort(); 43 | 44 | void setRedisPort(Integer redisPort); 45 | } 46 | -------------------------------------------------------------------------------- /processor/src/main/java/com/google/cloud/solutions/realtimedash/pipeline/ParseMessageAsLogElement.java: -------------------------------------------------------------------------------- 1 | /* 2 | * Copyright 2020 Google LLC 3 | * 4 | * Licensed under the Apache License, Version 2.0 (the "License"); 5 | * you may not use this file except in compliance with the License. 6 | * You may obtain a copy of the License at 7 | * 8 | * http://www.apache.org/licenses/LICENSE-2.0 9 | * 10 | * Unless required by applicable law or agreed to in writing, software 11 | * distributed under the License is distributed on an "AS IS" BASIS, 12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | * See the License for the specific language governing permissions and 14 | * limitations under the License. 15 | */ 16 | 17 | package com.google.cloud.solutions.realtimedash.pipeline; 18 | 19 | import com.fasterxml.jackson.databind.ObjectMapper; 20 | import com.fasterxml.jackson.databind.ObjectReader; 21 | import com.fasterxml.jackson.databind.PropertyNamingStrategy; 22 | import com.fasterxml.jackson.datatype.joda.JodaModule; 23 | import com.google.common.flogger.FluentLogger; 24 | import java.io.IOException; 25 | import java.util.concurrent.TimeUnit; 26 | import org.apache.beam.sdk.transforms.DoFn; 27 | 28 | /** 29 | * Pipeline operation to parse the Pub/Sub message as JSON into a POJO. 30 | */ 31 | public final class ParseMessageAsLogElement extends DoFn { 32 | 33 | private static final FluentLogger logger = FluentLogger.forEnclosingClass(); 34 | 35 | @ProcessElement 36 | public void parseStringToLogElement(ProcessContext context) { 37 | try { 38 | context.output(buildReader().readValue(context.element())); 39 | } catch (IOException ioexp) { 40 | logger.atWarning().atMostEvery(10, TimeUnit.SECONDS).withCause(ioexp).log(); 41 | } 42 | } 43 | 44 | private ObjectReader buildReader() { 45 | return new ObjectMapper() 46 | .registerModule(new JodaModule()) 47 | .setPropertyNamingStrategy(PropertyNamingStrategy.SNAKE_CASE) 48 | .readerFor(LogEvent.class); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /set_variables.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | # 3 | # Copyright 2020 Google LLC 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | # 17 | 18 | export PROJECT_ID="$(gcloud config get-value project)" 19 | export REGION_ID="us-central1" 20 | export ZONE_ID="$REGION_ID-a" 21 | export APP_EVENTS_TOPIC="web-events" 22 | export VPC_NETWORK_NAME="analytics-network" 23 | export REDIS_NAME="analytics-redis" 24 | export TEMP_GCS_BUCKET="$(whoami)-dataflow-temp" 25 | --------------------------------------------------------------------------------