├── .gitignore ├── .vscode ├── launch.json └── settings.json ├── LICENSE ├── README.md ├── data ├── images │ ├── data-intensive-app.png │ ├── grafana_dashboard.png │ ├── singlestore-easy.png │ └── what-are-we-building.png └── metrics │ ├── dashboards │ └── user_analytics.json │ ├── grafana_dashboards.yaml │ ├── grafana_datasources.yaml │ └── prometheus.yaml ├── docker-compose.yml ├── schema.sql ├── schema_finished.sql ├── src ├── Dockerfile ├── api.go ├── api_base.go ├── api_finished.go ├── bin │ ├── api │ │ └── main.go │ └── simulator │ │ └── main.go ├── config-api.toml ├── config-sim.toml ├── config.go ├── go.mod ├── go.sum ├── producer.go ├── random.go ├── simulator.go ├── simulator_base.go ├── simulator_finished.go ├── singlestore.go └── sitemap.go └── tasks /.gitignore: -------------------------------------------------------------------------------- 1 | /data/tripdata 2 | /.env 3 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "0.2.0", 3 | "configurations": [ 4 | { 5 | "name": "Run simulator", 6 | "type": "go", 7 | "request": "launch", 8 | "mode": "auto", 9 | "program": "${workspaceFolder}/src/bin/simulator", 10 | "args": [ 11 | "--config", 12 | "${workspaceFolder}/src/config-sim.toml", 13 | ] 14 | }, 15 | { 16 | "name": "Run api", 17 | "type": "go", 18 | "request": "launch", 19 | "mode": "auto", 20 | "program": "${workspaceFolder}/src/bin/api", 21 | "args": [ 22 | "--config", 23 | "${workspaceFolder}/src/config-api.toml", 24 | ] 25 | }, 26 | ] 27 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "sqltools.connections": [ 3 | { 4 | "mysqlOptions": { 5 | "authProtocol": "default" 6 | }, 7 | "previewLimit": 50, 8 | "server": "localhost", 9 | "port": 3306, 10 | "driver": "MariaDB", 11 | "name": "SingleStore", 12 | "username": "root", 13 | "database": "app", 14 | "password": "root" 15 | } 16 | ], 17 | "go.buildTags": "active_file", 18 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright 2021 SingleStore 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Workshop: Building data-intensive apps with SingleStore, Redpanda, and Golang 2 | 3 | **Attention**: The code in this repository is intended for experimental use only and is not fully tested, documented, or supported by SingleStore. Visit the [SingleStore Forums](https://www.singlestore.com/forum/) to ask questions about this repository. 4 | 5 | > 👋 Hello! I'm [@carlsverre][gh-carlsverre], and I'll be helping you out today 6 | > while we build a data-intensive application. Since everyone tends to work at 7 | > different speeds, this workshop is designed to be self-guided with assistance 8 | > as needed. If you are taking this workshop on your own and get stuck, feel 9 | > free to ask for help in the [SingleStore forums][s2-forums] via a 10 | > [Github Issue][gh-issue] 11 | 12 | This repo provides a starting point for building applications using SingleStore, 13 | Redpanda (by [Vectorized][vectorized]), and the Go language. SingleStore is a 14 | scale-out relational database built for data-intensive workloads. Redpanda is a 15 | Kafka API compatible streaming platform for mission-critical workloads created 16 | by the team at Vectorized. 17 | 18 | When you finish this workshop, you will have built a simple data-intensive 19 | application. You might be wondering, what is a data-intensive application. **An 20 | application is data-intensive when data defines its constraints.** This can 21 | manifest itself in many ways: 22 | 23 | - your application runs a complex query workload against multiple systems 24 | - you depend on data from many sources 25 | - you serve complex analytics to customers 26 | - as your customer base grows your data increases exponentially in volume or 27 | velocity 28 | 29 | Because of the inherent complexity of building a data-intensive application, 30 | they tend to grow out of control into sprawling systems which look something 31 | like this: 32 | 33 | ![data-intensive-app](data/images/data-intensive-app.png) 34 | 35 | SingleStore has always been focused on simplifying the diagram above which is 36 | why our customer's tend to have systems that look a bit more like this: 37 | 38 | ![singlestore-easy](data/images/singlestore-easy.png) 39 | 40 | By following the workshop documented in this file, you will build a 41 | data-intensive application. During the tutorial, you will accomplish the 42 | following tasks: 43 | 44 | 1. Prepare your environment 45 | 2. Write a digital-twin 46 | 3. Define a schema and load the data using Pipelines 47 | 4. Expose business logic via an HTTP API 48 | 5. Visualize your data 49 | 50 | Once complete, you will have an application architecture which looks something 51 | like this: 52 | 53 | ![workshop-result](data/images/what-are-we-building.png) 54 | 55 | Let's get started! 56 | 57 | ## 1. Prepare your environment 58 | 59 | Before we can start writing code, we need to make sure that your environment is 60 | setup and ready to go. 61 | 62 | 1. Make sure you clone this git repository to your machine 63 | 64 | ```bash 65 | git clone https://github.com/singlestore-labs/singlestore-workshop-data-intensive-app.git 66 | ``` 67 | 68 | 2. Make sure you have docker & docker-compose installed on your machine 69 | - [docker][docker] 70 | - [docker-compose][docker-compose] 71 | 72 | > ❗ **Note:** If you are following this tutorial on Windows or Mac OSX you may 73 | > need to increase the amount of RAM and CPU made available to docker. You can 74 | > do this from the docker configuration on your machine. More information: 75 | > [Mac OSX documentation][docker-ramcpu-osx], 76 | > [Windows documentation][docker-ramcpu-win] 77 | 78 | > ❗ **Note 2:** If you are following this tutorial on a Mac with the new M1 79 | > chipset, it is unlikely to work. SingleStore does not yet support running on 80 | > M1. We are actively fixing this, but in the meantime please follow along using 81 | > a different machine. 82 | 83 | 3. A code editor that you are comfortable using, if you aren't sure pick one of 84 | the options below: 85 | - [Visual Studio Code][vscode] 86 | 87 | > _Recommended:_ this repository is setup with VSCode launch configurations 88 | > as well as [SQL Tools][sqltools] support. 89 | 90 | - [Jetbrains Goland][jetbrains-go] 91 | 92 | 4. [Sign up][singlestore-signup] for a free SingleStore license. This allows you 93 | to run up to 4 nodes up to 32 gigs each for free. Grab your license key from 94 | [SingleStore portal][singlestore-portal] and set it as an environment 95 | variable. 96 | 97 | ```bash 98 | export SINGLESTORE_LICENSE="singlestore license" 99 | ``` 100 | 101 | ### Test that your environment is working 102 | 103 | > ℹ️ **Note:** This repository uses a bash script `./tasks` to run common tasks. 104 | > If you run the script with no arguments it will print out some information 105 | > about how to use it. 106 | 107 | Before proceeding, please execute `./tasks up` in the root of this repository to 108 | boot up all of the services we need and check that your environment is working 109 | as expected. 110 | 111 | ```bash 112 | $ ./tasks up 113 | (... lots of docker output here ...) 114 | Name Command State Ports 115 | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 116 | grafana /run.sh Up 0.0.0.0:3000->3000/tcp,:::3000->3000/tcp 117 | prometheus /bin/prometheus --config.f ... Up 0.0.0.0:9090->9090/tcp,:::9090->9090/tcp 118 | redpanda /bin/bash -c rpk config se ... Up 0.0.0.0:29092->29092/tcp,:::29092->29092/tcp, 0.0.0.0:9092->9092/tcp,:::9092->9092/tcp, 0.0.0.0:9093->9093/tcp,:::9093->9093/tcp, 9644/tcp 119 | simulator ./simulator --config confi ... Up 120 | singlestore /startup Up 0.0.0.0:3306->3306/tcp,:::3306->3306/tcp, 3307/tcp, 0.0.0.0:8080->8080/tcp,:::8080->8080/tcp 121 | ``` 122 | 123 | If the command fails or doesn't end with the above status report, you may need 124 | to start debugging your environment. If you are completely stumped please file 125 | an issue against this repository and someone will try to help you out. 126 | 127 | If the command succeeds, then you are good to go! Lets check out the various 128 | services before we continue: 129 | 130 | | service | url | port | username | password | 131 | | ------------------ | --------------------- | ---- | -------- | -------- | 132 | | SingleStore Studio | http://localhost:8080 | 8080 | root | root | 133 | | Grafana Dashboards | http://localhost:3000 | 3000 | root | root | 134 | | Prometheus | http://localhost:9090 | 9090 | | | 135 | 136 | You can open the three urls directly in your browser and login using the 137 | provided username & password. 138 | 139 | In addition, you can also connect directly to SingleStore DB using any MySQL 140 | compatible client or [VSCode SQL Tools][sqltools]. For example: 141 | 142 | ``` 143 | $ mariadb -u root -h 127.0.0.1 -proot 144 | Welcome to the MariaDB monitor. Commands end with ; or \g. 145 | Your MySQL connection id is 28 146 | Server version: 5.5.58 MemSQL source distribution (compatible; MySQL Enterprise & MySQL Commercial) 147 | 148 | Copyright (c) 2000, 2018, Oracle, MariaDB Corporation Ab and others. 149 | 150 | Type 'help;' or '\h' for help. Type '\c' to clear the current input statement. 151 | 152 | MySQL [(none)]> select "hello world"; 153 | +-------------+ 154 | | hello world | 155 | +-------------+ 156 | | hello world | 157 | +-------------+ 158 | 1 row in set (0.001 sec) 159 | ``` 160 | 161 | To make sure that Redpanda is receiving data use the following command to read 162 | from a test topic and see what the hello simulator is saying: 163 | 164 | ```bash 165 | $ ./tasks rpk topic consume --offset latest test 166 | { 167 | "message": "{\"message\":\"hello world\",\"time\":\"2021-07-21T21:25:50.708768659Z\",\"worker\":\"sim-0\"}\n", 168 | "partition": 3, 169 | "offset": 2, 170 | "timestamp": "2021-07-21T21:25:50.708Z" 171 | } 172 | { 173 | "message": "{\"message\":\"hello world\",\"time\":\"2021-07-21T21:25:51.70910547Z\",\"worker\":\"sim-0\"}\n", 174 | "partition": 4, 175 | "offset": 2, 176 | "timestamp": "2021-07-21T21:25:51.709Z" 177 | } 178 | ``` 179 | 180 | ## 2. Write a digital-twin 181 | 182 | Now that we have our environment setup it's time to start writing some code. Our 183 | first task is to build a digital-twin, which is basically a data simulator. We 184 | will use this digital-twin to generate large volumes of real-time data we need 185 | to make our application truly _data-intensive_. 186 | 187 | In this workshop, our digital-twin will be simulating simple page events for the 188 | SingleStore website. Lets define it's behavior: 189 | 190 | - The simulation will maintain a set of users browsing the website 191 | - New users will be randomly created 192 | - Users will "enter" the website at a random point in the sitemap 193 | - Users will navigate through the tree via adjacent nodes (up, down, sibling) at 194 | random 195 | - Users will spend a random amount of time on each page 196 | - While users remain on the page we will periodically record how long they have 197 | been there 198 | - No event will be generated when a user decides to leave the website 199 | 200 | > _Note:_ This simulation is far from realistic and will result in extremely 201 | > noisy data. A more sophisticated solution would be to take real page view data 202 | > and use it to generate more accurate distributions (or build an AI replicating 203 | > user behavior, left as an exercise to the reader). But for the purpose of this 204 | > workshop, we won't worry too much about the quality of the data. 205 | 206 | > _Note 2:_ I have already provided a **ton** of code and helpers to make this 207 | > workshop run smoothly, so if it feels like something is a bit magic - I 208 | > probably did that on purpose to keep the tutorial running smoothly. Please 209 | > take a look through some of the provided helpers if you want to learn more! 210 | 211 | ### Define our data structures 212 | 213 | We will be working in [simulator.go](src/simulator.go) for this section of the 214 | workshop, so please open that file. It should look like this: 215 | 216 | ```golang 217 | // +build active_file 218 | 219 | package src 220 | 221 | import ( 222 | "time" 223 | ) 224 | 225 | func (s *Simulator) Run() error { 226 | testTopic := s.producer.TopicEncoder("test") 227 | 228 | for s.Running() { 229 | time.Sleep(JitterDuration(time.Second, 200*time.Millisecond)) 230 | 231 | err := testTopic.Encode(map[string]interface{}{ 232 | "message": "hello world", 233 | "time": time.Now(), 234 | "worker": s.id, 235 | }) 236 | if err != nil { 237 | return err 238 | } 239 | } 240 | 241 | return nil 242 | } 243 | ``` 244 | 245 | > **Note:** A finished version of this file is also included at 246 | > [simulator_finished.go](src/simulator_finished.go) in case you get stuck. You 247 | > can switch between which one is used by changing the comment at the top of 248 | > both files. Go will build the one with the comment `// +build active_file` and 249 | > will ignore the one with the comment `// +build !active_file`. We will use 250 | > this technique later on in this workshop as well. 251 | 252 | First, we need to define an object to track our "users". Modify 253 | [simulator.go](src/simulator.go) as you follow along. 254 | 255 | ```golang 256 | type User struct { 257 | UserID string 258 | CurrentPage *Page 259 | LastChange time.Time 260 | } 261 | ``` 262 | 263 | We will also need an object which will help us write JSON to the events topic in 264 | Redpanda. Go provides a feature called `struct tags` to help configure what the 265 | resulting JSON object should look like. 266 | 267 | ```golang 268 | type Event struct { 269 | Timestamp int64 `json:"unix_timestamp"` 270 | PageTime float64 `json:"page_time_seconds,omitempty"` 271 | Referrer string `json:"referrer,omitempty"` 272 | UserID string `json:"user_id"` 273 | Path string `json:"path"` 274 | } 275 | ``` 276 | 277 | ### Data Structures 278 | 279 | Time to modify the `Run()` method which you will also find in 280 | [simulator.go](src/simulator.go). To start, we need a place to store our users. 281 | We will use a linked list data structure since we will be adding and removing 282 | users often. 283 | 284 | ```golang 285 | func (s *Simulator) Run() error { 286 | users := list.New() 287 | ``` 288 | 289 | We also need a way to write events to a Redpanda topic. I have already hooked up 290 | the code to Redpanda and provided a way to create what I call `TopicEncoders`. A 291 | TopicEncoder is a simple wrapper around a Redpanda producer which encodes each 292 | message using JSON. 293 | 294 | As you can see in the code, there is already a `TopicEncoder` for the `test` 295 | topic. Let's modify that line to instead target the events topic: 296 | 297 | ```golang 298 | events := s.producer.TopicEncoder("events") 299 | ``` 300 | 301 | ### Creating users 302 | 303 | The `Run()` method has a main loop which starts with the line `for s.Running()`. 304 | `s.Running()` will return `true` until the program starts to exit at which point 305 | it will return `false`. 306 | 307 | Within this loop the code "ticks" roughly once per second. Each time the loop 308 | ticks we need to run a bit of simulation code. 309 | 310 | The first thing we should do is create some users. Modify the simulation loop to 311 | look something like this: 312 | 313 | ```golang 314 | for s.Running() { 315 | time.Sleep(JitterDuration(time.Second, 200*time.Millisecond)) 316 | unixNow := time.Now().Unix() 317 | 318 | // create a random number of new users 319 | if users.Len() < s.config.MaxUsersPerThread { 320 | 321 | // figure out the max number of users we can create 322 | maxNewUsers := s.config.MaxUsersPerThread - users.Len() 323 | 324 | // calculate a random number between 1 and maxNewUsers 325 | numNewUsers := RandomIntInRange(0, maxNewUsers) 326 | 327 | // create the new users 328 | for i := 0; i < numNewUsers; i++ { 329 | // define a new user 330 | user := &User{ 331 | UserID: NextUserId(), 332 | CurrentPage: s.sitemap.RandomLeaf(), 333 | LastChange: time.Now(), 334 | } 335 | 336 | // add the user to the list 337 | users.PushBack(user) 338 | 339 | // write an event to the topic 340 | err := events.Encode(Event{ 341 | Timestamp: unixNow, 342 | UserID: user.UserID, 343 | Path: user.CurrentPage.Path, 344 | 345 | // pick a random referrer to use 346 | Referrer: RandomReferrer(), 347 | }) 348 | 349 | if err != nil { 350 | return err 351 | } 352 | } 353 | } 354 | } 355 | ``` 356 | 357 | Then update imports at the top of the file 358 | 359 | ```golang 360 | import ( 361 | "container/list" 362 | "time" 363 | ) 364 | ``` 365 | 366 | > ℹ️ **Note:** You can test your code as you go by running `./tasks simulator`. 367 | > This command will recompile the code and run the simulator. If you have any 368 | > errors they will show up in the output. 369 | 370 | ### Simulating browsing activity 371 | 372 | Now that we have users, let's simulate them browsing the site. The basic idea is 373 | that each time the simulation loop ticks, all of the users will decide to either 374 | stay on the current page, leave the site, or go to another page. Once again, we 375 | will roll virtual dice to make this happen. 376 | 377 | Add the following code within the `for s.Running() {` loop right after the code 378 | you added in the last section. 379 | 380 | ```golang 381 | // we will be removing elements from the list while we iterate, so we 382 | // need to keep track of next outside of the loop 383 | var next *list.Element 384 | 385 | // iterate through the users list and simulate each users behavior 386 | for el := users.Front(); el != nil; el = next { 387 | // loop bookkeeping 388 | next = el.Next() 389 | user := el.Value.(*User) 390 | pageTime := time.Since(user.LastChange) 391 | 392 | // users only consider leaving a page after at least 5 seconds 393 | if pageTime > time.Second*5 { 394 | 395 | // eventProb is a random value from 0 to 1 but is weighted 396 | // to be closer to 0 most of the time 397 | eventProb := math.Pow(rand.Float64(), 2) 398 | 399 | if eventProb > 0.98 { 400 | // user has left the site 401 | users.Remove(el) 402 | continue 403 | } else if eventProb > 0.9 { 404 | // user jumps to a random page 405 | user.CurrentPage = s.sitemap.RandomLeaf() 406 | user.LastChange = time.Now() 407 | } else if eventProb > 0.8 { 408 | // user goes to the "next" page 409 | user.CurrentPage = user.CurrentPage.RandomNext() 410 | user.LastChange = time.Now() 411 | } 412 | } 413 | 414 | // write an event to the topic recording the time on the current page 415 | // note that if the user has changed pages above, that fact will be reflected here 416 | err := events.Encode(Event{ 417 | Timestamp: unixNow, 418 | UserID: user.UserID, 419 | Path: user.CurrentPage.Path, 420 | PageTime: pageTime.Seconds(), 421 | }) 422 | if err != nil { 423 | return err 424 | } 425 | } 426 | ``` 427 | 428 | Then update imports at the top of the file: 429 | 430 | ```golang 431 | import ( 432 | "container/list" 433 | "math" 434 | "math/rand" 435 | "time" 436 | ) 437 | ``` 438 | 439 | Sweet! You have built your first digital twin! You can test it by running the 440 | following commands: 441 | 442 | ```bash 443 | $ ./tasks simulator 444 | ...output of building and running the simulator... 445 | 446 | $ ./tasks rpk topic consume --offset latest events 447 | { 448 | "message": "{\"unix_timestamp\":1626925973,\"page_time_seconds\":8.305101859,\"user_id\":\"a1e684e0-2a8f-48f3-8af3-26e55aadb86b\",\"path\":\"/blog/case-study-true-digital-group-helps-to-flatten-the-curve-with-memsql\"}", 449 | "partition": 1, 450 | "offset": 3290169, 451 | "timestamp": "2021-07-22T03:52:53.9Z" 452 | } 453 | { 454 | "message": "{\"unix_timestamp\":1626925973,\"page_time_seconds\":1.023932243,\"user_id\":\"a2e684e0-2a8f-48f3-8af3-26e55aadb86b\",\"path\":\"/media-hub/releases/memsql67\"}", 455 | "partition": 1, 456 | "offset": 3290170, 457 | "timestamp": "2021-07-22T03:52:53.9Z" 458 | } 459 | ...tons of messages, ctrl-C to cancel... 460 | ``` 461 | 462 | ## 3. Define a schema and load the data using Pipelines 463 | 464 | Next on our TODO list is getting the data into SingleStore! You can put away 465 | your Go skills for a moment, this step is all about writing SQL. 466 | 467 | We will be using SingleStore Studio to work on our schema and then saving the 468 | final result in [schema.sql](schema.sql). Open http://localhost:8080 and login 469 | with username: `root`, password: `root`. Once you are inside Studio, click **SQL 470 | Editor** on the left side. 471 | 472 | To start, we need a table for our events. Something like this should do the 473 | trick: 474 | 475 | ```sql 476 | DROP DATABASE IF EXISTS app; 477 | CREATE DATABASE app; 478 | USE app; 479 | 480 | CREATE TABLE events ( 481 | ts DATETIME NOT NULL, 482 | path TEXT NOT NULL COLLATE "utf8_bin", 483 | user_id TEXT NOT NULL COLLATE "utf8_bin", 484 | 485 | referrer TEXT, 486 | page_time_s DOUBLE NOT NULL DEFAULT 0, 487 | 488 | SORT KEY (ts), 489 | SHARD KEY (user_id) 490 | ); 491 | ``` 492 | 493 | At the top of the code you will see `DROP DATABASE...`. This makes it easy to 494 | iterate, just select-all (`ctrl/cmd-a`) and run (`ctrl/cmd-enter`) to rebuild 495 | the whole schema in SingleStore. Obviously, don't use this technique in 496 | production 😉. 497 | 498 | We will use SingleStore Pipelines to consume the events topic we created 499 | earlier. This is surprisingly easy, check it out: 500 | 501 | ```sql 502 | CREATE PIPELINE events 503 | AS LOAD DATA KAFKA 'redpanda/events' 504 | SKIP DUPLICATE KEY ERRORS 505 | INTO TABLE events 506 | FORMAT JSON ( 507 | @unix_timestamp <- unix_timestamp, 508 | path <- path, 509 | user_id <- user_id, 510 | referrer <- referrer DEFAULT NULL, 511 | page_time_s <- page_time_seconds DEFAULT 0 512 | ) 513 | SET ts = FROM_UNIXTIME(@unix_timestamp); 514 | 515 | START PIPELINE events; 516 | ``` 517 | 518 | The pipeline defined above connects to Redpanda and starts loading the events 519 | topic in batches. Each message is processed by the `FORMAT JSON` clause which 520 | maps the event's fields to the columns in the `events` table. You can read more 521 | about the powerful `CREATE PIPELINE` command [in our docs][create-pipeline]. 522 | 523 | Once `START PIPELINE` completes, SingleStore will start loading events from the 524 | simulator. Within a couple seconds you should start seeing rows show up in the 525 | events table. Try running `SELECT COUNT(*) FROM events`. 526 | 527 | If you don't see any rows check `SHOW PIPELINES` to see if your pipeline is 528 | running or has an error. You can also look for error messages using the query 529 | `SELECT * FROM INFORMATION_SCHEMA.PIPELINES_ERRORS`. 530 | 531 | > **Remember!** Once your schema works, copy the DDL statements 532 | > (`CREATE TABLE, CREATE PIPELINE, START PIPELINE`) into 533 | > [schema.sql](schema.sql) to make sure you don't loose it. 534 | 535 | You can find a finished version of [schema.sql](schema.sql) in 536 | [schema_finished.sql](schema_finished.sql). 537 | 538 | ## 4. Expose business logic via an HTTP API 539 | 540 | Now that we have successfully generated and loaded data into SingleStore, we can 541 | easily expose that data via a simple HTTP API. We will be working in 542 | [api.go](src/api.go) for most of this section. 543 | 544 | The first query I suggest writing is a simple leaderboard. It looks something 545 | like this: 546 | 547 | ```sql 548 | SELECT path, COUNT(DISTINCT user_id) AS count 549 | FROM events 550 | GROUP BY 1 551 | ORDER BY 2 DESC 552 | LIMIT 10 553 | ``` 554 | 555 | This query counts the number of distinct users per page and returns the top 10. 556 | 557 | We can expose the results of this query via the following function added to 558 | [api.go](src/api.go): 559 | 560 | ```golang 561 | func (a *Api) Leaderboard(c *gin.Context) { 562 | limit, err := strconv.Atoi(c.DefaultQuery("limit", "10")) 563 | if err != nil { 564 | c.JSON(http.StatusBadRequest, gin.H{"error": "limit must be an int"}) 565 | return 566 | } 567 | 568 | out := []struct { 569 | Path string `json:"path"` 570 | Count int `json:"count"` 571 | }{} 572 | 573 | err = a.db.SelectContext(c.Request.Context(), &out, ` 574 | SELECT path, COUNT(DISTINCT user_id) AS count 575 | FROM events 576 | GROUP BY 1 577 | ORDER BY 2 DESC 578 | LIMIT ? 579 | `, limit) 580 | if err != nil { 581 | c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 582 | return 583 | } 584 | 585 | c.JSON(http.StatusOK, out) 586 | } 587 | ``` 588 | 589 | Make sure you update the imports at the top of the file to: 590 | 591 | ```golang 592 | import ( 593 | "net/http" 594 | "strconv" 595 | 596 | "github.com/gin-gonic/gin" 597 | ) 598 | ``` 599 | 600 | Then register the function in the `RegisterRoutes` function like so: 601 | 602 | ```golang 603 | func (a *Api) RegisterRoutes(r *gin.Engine) { 604 | r.GET("/ping", a.Ping) 605 | r.GET("/leaderboard", a.Leaderboard) 606 | } 607 | ``` 608 | 609 | You can run the api and test your new endpoint using the following commands: 610 | 611 | ```bash 612 | $ ./tasks api 613 | ...output of building and running the api... 614 | 615 | $ ./tasks logs api 616 | Attaching to api 617 | api | [GIN-debug] [WARNING] Creating an Engine instance with the Logger and Recovery middleware already attached. 618 | api | 619 | api | [GIN-debug] [WARNING] Running in "debug" mode. Switch to "release" mode in production. 620 | api | - using env: export GIN_MODE=release 621 | api | - using code: gin.SetMode(gin.ReleaseMode) 622 | api | 623 | api | [GIN-debug] GET /ping --> src.(*Api).Ping-fm (3 handlers) 624 | api | [GIN-debug] GET /leaderboard --> src.(*Api).Leaderboard-fm (3 handlers) 625 | api | [GIN-debug] Listening and serving HTTP on :8000 626 | 627 | $ curl -s "localhost:8000/leaderboard?limit=2" | jq 628 | [ 629 | { 630 | "path": "/blog/memsql-spark-connector", 631 | "count": 524 632 | }, 633 | { 634 | "path": "/blog", 635 | "count": 507 636 | } 637 | ] 638 | ``` 639 | 640 | Before moving forward consider creating one or two additional endpoints or 641 | modifying the leaderboard. Here are some ideas: 642 | 643 | - (_easy_) change the leaderboard to accept a filter for a specific path prefix; 644 | i.e. `/leaderboard?prefix=/blog` 645 | - (_medium_) add a new endpoint which returns a referrer leaderboard (you only 646 | care about rows where referrer is `NOT NULL`) 647 | - (_hard_) add a new endpoint which returns the number of page loads over time 648 | bucketed by minute - keep in mind that each row in the events table represents 649 | a user viewing a page for 1 second 650 | 651 | ## 5. Visualize your data 652 | 653 | Whew! We are almost to the end of the workshop! As a final piece of the puzzle, 654 | we will visualize our data using Grafana. I have already setup grafana for you, 655 | so just head over to http://localhost:3000 to get started. Username and password 656 | are both `root`. 657 | 658 | Assuming your simulation and schema matches what is in this README, you can 659 | check out the pre-created dashboard I have already created here: 660 | http://localhost:3000/d/_TsB4vZ7k/user-analytics?orgId=1&refresh=5s 661 | 662 | ![grafana-dashboard](data/images/grafana_dashboard.png) 663 | 664 | I highly recommend playing around with Grafana and experimenting with its many 665 | features. I have setup SingleStore as a datasource called "SingleStore" so it 666 | should be pretty easy to create new dashboards and panels. 667 | 668 | For further Grafana education, I recommend checking out their docs 669 | [starting here][grafana-getting-started]. Good luck! 670 | 671 | # Wrapping up 672 | 673 | Congrats! You have now learned the basics of building a data-intensive app with 674 | SingleStore! As a quick recap, during this workshop you have: 675 | 676 | - created a user simulator to generate random browsing data 677 | - ingested that data into SingleStore using Pipelines 678 | - exposed business analytics via an HTTP API 679 | - created dashboards to help you run your business 680 | 681 | With these skills, you have learned the foundation of building data-intensive 682 | applications. For example, using this exact structure I built a 683 | [logistics simulator][logistics-sim] which simulated global package logistics at 684 | massive scale. You can read more about that project on the 685 | [SingleStore Blog][logistics-blog]. 686 | 687 | I hope you enjoyed the workshop! Please feel free to provide feedback in the 688 | [SingleStore forums][s2-forums] or via a [Github Issue][gh-issue]. 689 | 690 | Cheers! 691 | 692 | [@carlsverre][gh-carlsverre] 693 | 694 | # Contributors 695 | 696 | This project wouldn't be complete without saying thank you to these amazing 697 | contributors! 698 | 699 | - [Bailey Hayes (@ricochet)](https://github.com/ricochet) 700 | 701 | 702 | 703 | [docker-compose]: https://docs.docker.com/compose/install/ 704 | [docker-ramcpu-osx]: https://docs.docker.com/docker-for-mac/#resources 705 | [docker-ramcpu-win]: https://docs.docker.com/docker-for-windows/#resources 706 | [docker]: https://docs.docker.com/get-docker/ 707 | [jetbrains-go]: https://www.jetbrains.com/go/ 708 | [sqltools]: https://marketplace.visualstudio.com/items?itemName=mtxr.sqltools 709 | [vscode]: https://code.visualstudio.com/ 710 | [singlestore-signup]: https://www.singlestore.com/try-free/ 711 | [singlestore-portal]: https://portal.singlestore.com/?utm_medium=osm&utm_source=github 712 | [create-pipeline]: https://docs.singlestore.com/db/v7.3/en/reference/sql-reference/pipelines-commands/create-pipeline.html 713 | [grafana-getting-started]: https://grafana.com/docs/grafana/latest/getting-started/getting-started/ 714 | [s2-forums]: https://www.singlestore.com/forum 715 | [gh-issue]: https://github.com/singlestore-labs/singlestore-workshop-data-intensive-app/issues/new 716 | [logistics-sim]: https://github.com/singlestore-labs/singlestore-logistics-sim 717 | [logistics-blog]: https://www.singlestore.com/blog/scaling-worldwide-parcel-logistics-with-singlestore-and-vectorized/ 718 | [vectorized]: https://vectorized.io/ 719 | [gh-carlsverre]: https://github.com/carlsverre 720 | -------------------------------------------------------------------------------- /data/images/data-intensive-app.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/singlestore-labs/singlestore-workshop-data-intensive-app/b6bca41852952b1859a93c99acdbf984108bd33f/data/images/data-intensive-app.png -------------------------------------------------------------------------------- /data/images/grafana_dashboard.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/singlestore-labs/singlestore-workshop-data-intensive-app/b6bca41852952b1859a93c99acdbf984108bd33f/data/images/grafana_dashboard.png -------------------------------------------------------------------------------- /data/images/singlestore-easy.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/singlestore-labs/singlestore-workshop-data-intensive-app/b6bca41852952b1859a93c99acdbf984108bd33f/data/images/singlestore-easy.png -------------------------------------------------------------------------------- /data/images/what-are-we-building.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/singlestore-labs/singlestore-workshop-data-intensive-app/b6bca41852952b1859a93c99acdbf984108bd33f/data/images/what-are-we-building.png -------------------------------------------------------------------------------- /data/metrics/dashboards/user_analytics.json: -------------------------------------------------------------------------------- 1 | { 2 | "annotations": { 3 | "list": [ 4 | { 5 | "builtIn": 1, 6 | "datasource": "-- Grafana --", 7 | "enable": true, 8 | "hide": true, 9 | "iconColor": "rgba(0, 211, 255, 1)", 10 | "name": "Annotations & Alerts", 11 | "type": "dashboard" 12 | } 13 | ] 14 | }, 15 | "editable": true, 16 | "gnetId": null, 17 | "graphTooltip": 0, 18 | "links": [], 19 | "panels": [ 20 | { 21 | "datasource": "singlestore", 22 | "fieldConfig": { 23 | "defaults": { 24 | "color": { 25 | "mode": "palette-classic" 26 | }, 27 | "custom": { 28 | "axisLabel": "", 29 | "axisPlacement": "auto", 30 | "barAlignment": 0, 31 | "drawStyle": "line", 32 | "fillOpacity": 0, 33 | "gradientMode": "none", 34 | "hideFrom": { 35 | "legend": false, 36 | "tooltip": false, 37 | "viz": false 38 | }, 39 | "lineInterpolation": "linear", 40 | "lineWidth": 1, 41 | "pointSize": 5, 42 | "scaleDistribution": { 43 | "type": "linear" 44 | }, 45 | "showPoints": "auto", 46 | "spanNulls": false, 47 | "stacking": { 48 | "group": "A", 49 | "mode": "none" 50 | }, 51 | "thresholdsStyle": { 52 | "mode": "off" 53 | } 54 | }, 55 | "mappings": [], 56 | "thresholds": { 57 | "mode": "absolute", 58 | "steps": [ 59 | { 60 | "color": "green", 61 | "value": null 62 | }, 63 | { 64 | "color": "red", 65 | "value": 80 66 | } 67 | ] 68 | } 69 | }, 70 | "overrides": [] 71 | }, 72 | "gridPos": { 73 | "h": 8, 74 | "w": 19, 75 | "x": 0, 76 | "y": 0 77 | }, 78 | "id": 2, 79 | "options": { 80 | "legend": { 81 | "calcs": [], 82 | "displayMode": "list", 83 | "placement": "bottom" 84 | }, 85 | "tooltip": { 86 | "mode": "multi" 87 | } 88 | }, 89 | "targets": [ 90 | { 91 | "format": "time_series", 92 | "group": [], 93 | "hide": false, 94 | "metricColumn": "none", 95 | "rawQuery": true, 96 | "rawSql": "SELECT\n $__timeGroup(ts, \"$__interval\") as time,\n count(distinct user_id) as value\nFROM events\nWHERE $__timeFilter(ts)\nGROUP BY 1\nORDER BY ts ASC", 97 | "refId": "A", 98 | "select": [ 99 | [ 100 | { 101 | "params": [ 102 | "value" 103 | ], 104 | "type": "column" 105 | } 106 | ] 107 | ], 108 | "timeColumn": "time", 109 | "where": [ 110 | { 111 | "name": "$__timeFilter", 112 | "params": [], 113 | "type": "macro" 114 | } 115 | ] 116 | } 117 | ], 118 | "title": "Live users over time", 119 | "type": "timeseries" 120 | }, 121 | { 122 | "datasource": "singlestore", 123 | "fieldConfig": { 124 | "defaults": { 125 | "color": { 126 | "mode": "thresholds" 127 | }, 128 | "mappings": [], 129 | "thresholds": { 130 | "mode": "absolute", 131 | "steps": [ 132 | { 133 | "color": "green", 134 | "value": null 135 | }, 136 | { 137 | "color": "red", 138 | "value": 80 139 | } 140 | ] 141 | } 142 | }, 143 | "overrides": [] 144 | }, 145 | "gridPos": { 146 | "h": 8, 147 | "w": 5, 148 | "x": 19, 149 | "y": 0 150 | }, 151 | "id": 4, 152 | "options": { 153 | "reduceOptions": { 154 | "calcs": [ 155 | "lastNotNull" 156 | ], 157 | "fields": "", 158 | "values": false 159 | }, 160 | "showThresholdLabels": false, 161 | "showThresholdMarkers": true, 162 | "text": {} 163 | }, 164 | "pluginVersion": "8.0.6", 165 | "targets": [ 166 | { 167 | "format": "table", 168 | "group": [], 169 | "metricColumn": "none", 170 | "rawQuery": true, 171 | "rawSql": "select count(distinct user_id) from events\nwhere\n ts > date_sub($__timeTo(), interval 5 SECOND)", 172 | "refId": "A", 173 | "select": [ 174 | [ 175 | { 176 | "params": [ 177 | "value" 178 | ], 179 | "type": "column" 180 | } 181 | ] 182 | ], 183 | "timeColumn": "time", 184 | "where": [ 185 | { 186 | "name": "$__timeFilter", 187 | "params": [], 188 | "type": "macro" 189 | } 190 | ] 191 | } 192 | ], 193 | "title": "Live Users", 194 | "type": "gauge" 195 | }, 196 | { 197 | "datasource": "singlestore", 198 | "fieldConfig": { 199 | "defaults": { 200 | "color": { 201 | "mode": "thresholds" 202 | }, 203 | "custom": { 204 | "align": "auto", 205 | "displayMode": "auto" 206 | }, 207 | "mappings": [], 208 | "thresholds": { 209 | "mode": "absolute", 210 | "steps": [ 211 | { 212 | "color": "green", 213 | "value": null 214 | }, 215 | { 216 | "color": "red", 217 | "value": 80 218 | } 219 | ] 220 | } 221 | }, 222 | "overrides": [] 223 | }, 224 | "gridPos": { 225 | "h": 19, 226 | "w": 24, 227 | "x": 0, 228 | "y": 8 229 | }, 230 | "id": 6, 231 | "options": { 232 | "showHeader": true 233 | }, 234 | "pluginVersion": "8.0.6", 235 | "targets": [ 236 | { 237 | "format": "table", 238 | "group": [], 239 | "metricColumn": "none", 240 | "rawQuery": true, 241 | "rawSql": "SELECT path, count(distinct user_id)\nFROM events\nWHERE $__timeFilter(ts)\nGROUP BY 1\nORDER BY 2 DESC\nLIMIT 10;", 242 | "refId": "A", 243 | "select": [ 244 | [ 245 | { 246 | "params": [ 247 | "value" 248 | ], 249 | "type": "column" 250 | } 251 | ] 252 | ], 253 | "timeColumn": "time", 254 | "where": [ 255 | { 256 | "name": "$__timeFilter", 257 | "params": [], 258 | "type": "macro" 259 | } 260 | ] 261 | } 262 | ], 263 | "title": "Top 15 Leaderboard", 264 | "type": "table" 265 | } 266 | ], 267 | "refresh": "5s", 268 | "schemaVersion": 30, 269 | "style": "dark", 270 | "tags": [], 271 | "templating": { 272 | "list": [] 273 | }, 274 | "time": { 275 | "from": "now-5m", 276 | "to": "now" 277 | }, 278 | "timepicker": {}, 279 | "timezone": "", 280 | "title": "User Analytics", 281 | "uid": "_TsB4vZ7k", 282 | "version": 1 283 | } -------------------------------------------------------------------------------- /data/metrics/grafana_dashboards.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | providers: 4 | - name: singlestore 5 | orgId: 1 6 | disableDeletion: false 7 | updateIntervalSeconds: 10 8 | # to update a dashboard use "Save JSON to file" and wait for this provider to sync 9 | allowUiUpdates: false 10 | options: 11 | path: /dashboards 12 | foldersFromFilesStructure: true -------------------------------------------------------------------------------- /data/metrics/grafana_datasources.yaml: -------------------------------------------------------------------------------- 1 | apiVersion: 1 2 | 3 | datasources: 4 | - name: prometheus 5 | access: proxy 6 | editable: false 7 | isDefault: true 8 | type: prometheus 9 | url: 'http://prometheus:9090' 10 | version: 1 11 | jsonData: 12 | timeInterval: 5s 13 | - name: singlestore 14 | editable: false 15 | type: mysql 16 | url: singlestore:3306 17 | database: app 18 | user: root 19 | version: 1 20 | jsonData: 21 | timeInterval: 5s 22 | secureJsonData: 23 | password: root 24 | -------------------------------------------------------------------------------- /data/metrics/prometheus.yaml: -------------------------------------------------------------------------------- 1 | global: 2 | scrape_interval: '10s' 3 | evaluation_interval: '10s' 4 | 5 | scrape_configs: 6 | - job_name: 'redpanda' 7 | static_configs: 8 | - targets: ['redpanda:9644'] -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | services: 3 | redpanda: 4 | image: vectorized/redpanda:v21.7.1 5 | container_name: redpanda 6 | entrypoint: /bin/bash -c 7 | command: 8 | - | 9 | rpk config set redpanda.default_topic_partitions 8 10 | rpk config set redpanda.enable_idempotence true 11 | rpk redpanda start \ 12 | --smp=2 --memory=4G --overprovisioned --reserve-memory=0M --check=false \ 13 | --default-log-level=info --node-id=0 \ 14 | --kafka-addr PLAINTEXT://0.0.0.0:29092,DOCKER://redpanda:9092,OUTSIDE://redpanda:9093 \ 15 | --advertise-kafka-addr PLAINTEXT://redpanda:29092,DOCKER://redpanda:9092,OUTSIDE://127.0.0.1:9093 16 | ports: 17 | - 9092:9092 18 | - 9093:9093 19 | - 29092:29092 20 | volumes: 21 | - /var/lib/redpanda/data 22 | singlestore: 23 | image: singlestore/cluster-in-a-box:centos-7.5.6-9e799dbf4f-3.2.11-1.11.9 24 | container_name: singlestore 25 | volumes: 26 | - /var/lib/memsql 27 | - ./schema.sql:/init.sql:ro 28 | - ./data:/data:ro 29 | ports: 30 | - 3306:3306 31 | - 8080:8080 32 | environment: 33 | - ROOT_PASSWORD=root 34 | - LICENSE_KEY=${SINGLESTORE_LICENSE} 35 | - START_AFTER_INIT=Y 36 | prometheus: 37 | image: prom/prometheus:v2.28.1 38 | container_name: prometheus 39 | volumes: 40 | - ./data/metrics/prometheus.yaml:/etc/prometheus/prometheus.yml:ro 41 | ports: 42 | - 9090:9090 43 | grafana: 44 | image: grafana/grafana:8.0.6 45 | container_name: grafana 46 | environment: 47 | - GF_USERS_DEFAULT_THEME=light 48 | - GF_SECURITY_ADMIN_USER=root 49 | - GF_SECURITY_ADMIN_PASSWORD=root 50 | - "GF_INSTALL_PLUGINS=grafana-worldmap-panel,https://github.com/WilliamVenner/grafana-timepicker-buttons/releases/download/v4.1.1/williamvenner-timepickerbuttons-panel-4.1.1.zip;grafana-timepicker-buttons" 51 | volumes: 52 | - /var/lib/grafana 53 | - ./data/metrics/dashboards:/dashboards:ro 54 | - ./data/metrics/grafana_dashboards.yaml:/etc/grafana/provisioning/dashboards/all.yaml:ro 55 | - ./data/metrics/grafana_datasources.yaml:/etc/grafana/provisioning/datasources/all.yaml:ro 56 | ports: 57 | - 3000:3000 58 | simulator: 59 | build: 60 | context: src 61 | target: simulator 62 | image: singlestore-simulator:simulator 63 | container_name: simulator 64 | api: 65 | build: 66 | context: src 67 | target: api 68 | image: singlestore-simulator:api 69 | container_name: api 70 | ports: 71 | - 8000:8000 -------------------------------------------------------------------------------- /schema.sql: -------------------------------------------------------------------------------- 1 | DROP DATABASE IF EXISTS app; 2 | CREATE DATABASE app; 3 | USE app; -------------------------------------------------------------------------------- /schema_finished.sql: -------------------------------------------------------------------------------- 1 | DROP DATABASE IF EXISTS app; 2 | CREATE DATABASE app; 3 | USE app; 4 | 5 | CREATE TABLE events ( 6 | ts DATETIME NOT NULL, 7 | path TEXT NOT NULL COLLATE "utf8_bin", 8 | user_id TEXT NOT NULL COLLATE "utf8_bin", 9 | 10 | referrer TEXT, 11 | page_time_s DOUBLE NOT NULL DEFAULT 0, 12 | 13 | SORT KEY (ts), 14 | SHARD KEY (user_id) 15 | ); 16 | 17 | CREATE PIPELINE events 18 | AS LOAD DATA KAFKA 'redpanda/events' 19 | SKIP DUPLICATE KEY ERRORS 20 | INTO TABLE events 21 | FORMAT JSON ( 22 | @unix_timestamp <- unix_timestamp, 23 | path <- path, 24 | user_id <- user_id, 25 | referrer <- referrer DEFAULT NULL, 26 | page_time_s <- page_time_seconds DEFAULT 0 27 | ) 28 | SET ts = FROM_UNIXTIME(@unix_timestamp); 29 | 30 | START PIPELINE events; -------------------------------------------------------------------------------- /src/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM golang:1.15.14-buster as builder 2 | 3 | WORKDIR /go/src/app 4 | 5 | # cache dependencies to accelerate most rebuilds 6 | COPY go.mod go.sum ./ 7 | RUN go mod download all 8 | 9 | COPY . . 10 | 11 | RUN go fmt ./... 12 | RUN go build --tags active_file -o /simulator bin/simulator/main.go 13 | RUN go build --tags active_file -o /api bin/api/main.go 14 | 15 | # simulator image 16 | FROM debian:buster-slim as simulator 17 | ADD https://curl.haxx.se/ca/cacert.pem /etc/pki/tls/certs/ca-bundle.crt 18 | 19 | CMD ["./simulator", "--config", "config.toml"] 20 | COPY --from=builder /simulator . 21 | COPY config-sim.toml /config.toml 22 | 23 | # api image 24 | FROM debian:buster-slim as api 25 | ADD https://curl.haxx.se/ca/cacert.pem /etc/pki/tls/certs/ca-bundle.crt 26 | 27 | CMD ["./api", "--config", "config.toml"] 28 | COPY --from=builder /api . 29 | COPY config-api.toml /config.toml -------------------------------------------------------------------------------- /src/api.go: -------------------------------------------------------------------------------- 1 | // +build active_file 2 | 3 | package src 4 | 5 | import ( 6 | "net/http" 7 | 8 | "github.com/gin-gonic/gin" 9 | ) 10 | 11 | func (a *Api) RegisterRoutes(r *gin.Engine) { 12 | r.GET("/ping", a.Ping) 13 | } 14 | 15 | func (a *Api) Ping(c *gin.Context) { 16 | c.String(http.StatusOK, "pong") 17 | } 18 | -------------------------------------------------------------------------------- /src/api_base.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | type Api struct { 4 | db *SingleStore 5 | } 6 | 7 | func NewApi(db *SingleStore) *Api { 8 | return &Api{db: db} 9 | } 10 | -------------------------------------------------------------------------------- /src/api_finished.go: -------------------------------------------------------------------------------- 1 | // +build !active_file 2 | 3 | package src 4 | 5 | import ( 6 | "net/http" 7 | "strconv" 8 | 9 | "github.com/gin-gonic/gin" 10 | ) 11 | 12 | func (a *Api) RegisterRoutes(r *gin.Engine) { 13 | r.GET("/ping", a.Ping) 14 | r.GET("/leaderboard", a.Leaderboard) 15 | } 16 | 17 | func (a *Api) Ping(c *gin.Context) { 18 | c.String(http.StatusOK, "pong") 19 | } 20 | 21 | func (a *Api) Leaderboard(c *gin.Context) { 22 | limit, err := strconv.Atoi(c.DefaultQuery("limit", "10")) 23 | if err != nil { 24 | c.JSON(http.StatusBadRequest, gin.H{"error": "limit must be an int"}) 25 | return 26 | } 27 | 28 | out := []struct { 29 | Path string `json:"path"` 30 | Count int `json:"count"` 31 | }{} 32 | 33 | err = a.db.SelectContext(c.Request.Context(), &out, ` 34 | SELECT path, COUNT(DISTINCT user_id) AS count 35 | FROM events 36 | GROUP BY 1 37 | ORDER BY 2 DESC 38 | LIMIT ? 39 | `, limit) 40 | if err != nil { 41 | c.JSON(http.StatusInternalServerError, gin.H{"error": err.Error()}) 42 | return 43 | } 44 | 45 | c.JSON(http.StatusOK, out) 46 | } 47 | -------------------------------------------------------------------------------- /src/bin/api/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "log" 6 | "time" 7 | 8 | "src" 9 | 10 | "github.com/gin-gonic/gin" 11 | ) 12 | 13 | func main() { 14 | // global configuration 15 | log.SetFlags(log.Ldate | log.Ltime) 16 | 17 | // handle command line flags 18 | var configPath string 19 | flag.StringVar(&configPath, "config", "", "path to an optional config file") 20 | flag.Parse() 21 | 22 | // load configuration file if it exists 23 | var config *src.ApiConfig 24 | if configPath != "" { 25 | conf, err := src.NewApiConfigFromFile(configPath) 26 | if err != nil { 27 | log.Fatal(err) 28 | } 29 | config = conf 30 | } 31 | 32 | // connect to SingleStore 33 | var db *src.SingleStore 34 | var err error 35 | for { 36 | db, err = src.NewSingleStore(config.SingleStore) 37 | if err != nil { 38 | log.Printf("unable to connect to SingleStore: %s; retrying...", err) 39 | time.Sleep(time.Second) 40 | continue 41 | } 42 | break 43 | } 44 | 45 | // we will use gin as our http server and router 46 | router := gin.Default() 47 | 48 | // register the api 49 | api := src.NewApi(db) 50 | api.RegisterRoutes(router) 51 | 52 | router.Run(":8000") 53 | } 54 | -------------------------------------------------------------------------------- /src/bin/simulator/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "log" 7 | "math/rand" 8 | "os" 9 | "os/signal" 10 | "runtime" 11 | "sync" 12 | "syscall" 13 | "time" 14 | 15 | "src" 16 | ) 17 | 18 | func main() { 19 | // global configuration 20 | rand.Seed(time.Now().UnixNano()) 21 | log.SetFlags(log.Ldate | log.Ltime) 22 | 23 | // handle command line flags 24 | var configPath string 25 | flag.StringVar(&configPath, "config", "", "path to an optional config file") 26 | flag.Parse() 27 | 28 | // load configuration file if it exists 29 | var config *src.SimConfig 30 | if configPath != "" { 31 | conf, err := src.NewSimConfigFromFile(configPath) 32 | if err != nil { 33 | log.Fatal(err) 34 | } 35 | config = conf 36 | } 37 | 38 | sitemap, err := src.LoadSitemap(config.SitemapURL) 39 | if err != nil { 40 | log.Fatal(err) 41 | } 42 | 43 | simulators := make([]*src.Simulator, 0) 44 | stopAll := func() { 45 | for _, sim := range simulators { 46 | sim.Stop() 47 | } 48 | } 49 | 50 | // Trap SIGINT to trigger a clean shutdown. 51 | signals := make(chan os.Signal, 1) 52 | signal.Notify(signals, syscall.SIGINT, syscall.SIGTERM) 53 | go func() { 54 | sig := <-signals 55 | log.Printf("received shutdown signal: %s", sig) 56 | stopAll() 57 | }() 58 | 59 | numWorkers := runtime.NumCPU() 60 | if config.NumWorkers != 0 { 61 | numWorkers = config.NumWorkers 62 | } 63 | 64 | log.Printf("starting simulation with %d workers", numWorkers) 65 | 66 | wg := sync.WaitGroup{} 67 | for i := 0; i < numWorkers; i++ { 68 | wg.Add(1) 69 | 70 | simulator, err := src.NewSimulator(fmt.Sprintf("sim-%d", i), config, sitemap) 71 | if err != nil { 72 | log.Fatal(err) 73 | } 74 | 75 | simulators = append(simulators, simulator) 76 | 77 | go func(i int) { 78 | defer wg.Done() 79 | err := simulator.Run() 80 | if err != nil { 81 | log.Fatalf("sim-%d failed: %s", i, err) 82 | } 83 | }(i) 84 | } 85 | 86 | wg.Wait() 87 | } 88 | -------------------------------------------------------------------------------- /src/config-api.toml: -------------------------------------------------------------------------------- 1 | [singlestore] 2 | host = "singlestore" 3 | port = 3306 4 | username = "root" 5 | password = "root" 6 | database = "app" -------------------------------------------------------------------------------- /src/config-sim.toml: -------------------------------------------------------------------------------- 1 | numWorkers = 2 2 | maxUsersPerTick = 1000 3 | maxUsersPerThread = 10000 4 | 5 | sitemapURL = "https://www.singlestore.com/sitemap.xml" 6 | 7 | [producer] 8 | brokers = [ "redpanda:9092", "localhost:9093" ] -------------------------------------------------------------------------------- /src/config.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "github.com/BurntSushi/toml" 5 | ) 6 | 7 | type SimConfig struct { 8 | NumWorkers int 9 | MaxUsersPerTick int 10 | MaxUsersPerThread int 11 | SitemapURL string 12 | 13 | Producer ProducerConfig 14 | } 15 | 16 | func NewSimConfigFromFile(file string) (*SimConfig, error) { 17 | var config SimConfig 18 | if _, err := toml.DecodeFile(file, &config); err != nil { 19 | return nil, err 20 | } 21 | return &config, nil 22 | } 23 | 24 | type ApiConfig struct { 25 | SingleStore SingleStoreConfig 26 | } 27 | 28 | func NewApiConfigFromFile(file string) (*ApiConfig, error) { 29 | var config ApiConfig 30 | if _, err := toml.DecodeFile(file, &config); err != nil { 31 | return nil, err 32 | } 33 | return &config, nil 34 | } 35 | -------------------------------------------------------------------------------- /src/go.mod: -------------------------------------------------------------------------------- 1 | module src 2 | 3 | go 1.15 4 | 5 | require ( 6 | github.com/BurntSushi/toml v0.3.1 7 | github.com/gin-gonic/gin v1.7.2 8 | github.com/go-sql-driver/mysql v1.6.0 9 | github.com/golang/protobuf v1.4.3 // indirect 10 | github.com/golang/snappy v0.0.4 // indirect 11 | github.com/jmoiron/sqlx v1.3.4 12 | github.com/json-iterator/go v1.1.11 // indirect 13 | github.com/kr/text v0.2.0 // indirect 14 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect 15 | github.com/modern-go/reflect2 v1.0.1 // indirect 16 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect 17 | github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 18 | github.com/pkg/errors v0.9.1 19 | github.com/rogpeppe/fastuuid v1.2.0 20 | github.com/stretchr/testify v1.7.0 // indirect 21 | github.com/twmb/franz-go v0.8.7 22 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 // indirect 23 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect 24 | google.golang.org/protobuf v1.26.0-rc.1 // indirect 25 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f // indirect 26 | gopkg.in/yaml.v2 v2.4.0 // indirect 27 | ) 28 | -------------------------------------------------------------------------------- /src/go.sum: -------------------------------------------------------------------------------- 1 | github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= 2 | github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= 3 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 4 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 6 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 7 | github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= 8 | github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= 9 | github.com/gin-gonic/gin v1.7.2 h1:Tg03T9yM2xa8j6I3Z3oqLaQRSmKvxPd6g/2HJ6zICFA= 10 | github.com/gin-gonic/gin v1.7.2/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= 11 | github.com/go-playground/assert/v2 v2.0.1 h1:MsBgLAaY856+nPRTKrp3/OZK38U/wa0CcBYNjji3q3A= 12 | github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= 13 | github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= 14 | github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= 15 | github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= 16 | github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= 17 | github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= 18 | github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= 19 | github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 20 | github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= 21 | github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= 22 | github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= 23 | github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= 24 | github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= 25 | github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= 26 | github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= 27 | github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= 28 | github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= 29 | github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= 30 | github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= 31 | github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 32 | github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= 33 | github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= 34 | github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 35 | github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= 36 | github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 37 | github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= 38 | github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= 39 | github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= 40 | github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= 41 | github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= 42 | github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= 43 | github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= 44 | github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= 45 | github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o= 46 | github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= 47 | github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= 48 | github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= 49 | github.com/jmoiron/sqlx v1.3.4 h1:wv+0IJZfL5z0uZoUjlpKgHkgaFSYD+r9CfrXjEXsO7w= 50 | github.com/jmoiron/sqlx v1.3.4/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXLBMCQ= 51 | github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 52 | github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= 53 | github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= 54 | github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs= 55 | github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= 56 | github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= 57 | github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= 58 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 59 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 60 | github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= 61 | github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= 62 | github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= 63 | github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= 64 | github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= 65 | github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= 66 | github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= 67 | github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= 68 | github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 69 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= 70 | github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= 71 | github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 72 | github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= 73 | github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= 74 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= 75 | github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= 76 | github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4 h1:2vmb32OdDhjZf2ETGDlr9n8RYXx7c+jXPxMiPbwnA+8= 77 | github.com/oxffaa/gopher-parse-sitemap v0.0.0-20191021113419-005d2eb1def4/go.mod h1:2JQx4jDHmWrbABvpOayg/+OTU6ehN0IyK2EHzceXpJo= 78 | github.com/pierrec/lz4/v4 v4.1.7 h1:UDV9geJWhFIufAliH7HQlz9wP3JA0t748w+RwbWMLow= 79 | github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= 80 | github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= 81 | github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= 82 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 83 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 84 | github.com/rogpeppe/fastuuid v1.2.0 h1:Ppwyp6VYCF1nvBTXL3trRso7mXMlRrw9ooo375wvi2s= 85 | github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= 86 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 87 | github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= 88 | github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= 89 | github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 90 | github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= 91 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 92 | github.com/twmb/franz-go v0.8.7 h1:bc0Rch8qspqqBiqir5YsyzI6bhUJ6HDX9ODdV06u/04= 93 | github.com/twmb/franz-go v0.8.7/go.mod h1:v6QnB3abhlVAzlIEIO5L/1Emu8NlkreCI2HSps9utH0= 94 | github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg= 95 | github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc= 96 | github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= 97 | github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= 98 | github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= 99 | github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= 100 | golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= 101 | golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 102 | golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= 103 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a h1:kr2P4QFmQr29mSLA43kwrOcgcReGTfbE9N577tCTuBc= 104 | golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= 105 | golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= 106 | golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= 107 | golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= 108 | golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= 109 | golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= 110 | golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 111 | golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 112 | golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 113 | golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= 114 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40 h1:JWgyZ1qgdTaF3N3oxC+MdTV7qvEEgHo3otj+HB5CM7Q= 115 | golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 116 | golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= 117 | golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= 118 | golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= 119 | golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 120 | golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= 121 | golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= 122 | golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 123 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= 124 | golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= 125 | google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= 126 | google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= 127 | google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= 128 | google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= 129 | google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= 130 | google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= 131 | google.golang.org/protobuf v1.26.0-rc.1 h1:7QnIQpGRHE5RnLKnESfDoxm2dTapTZua5a0kS0A+VXQ= 132 | google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= 133 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 134 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= 135 | gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 136 | gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 137 | gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= 138 | gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= 139 | gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= 140 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= 141 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 142 | -------------------------------------------------------------------------------- /src/producer.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "context" 5 | "encoding/json" 6 | "log" 7 | "sync" 8 | "sync/atomic" 9 | "syscall" 10 | 11 | "github.com/pkg/errors" 12 | "github.com/twmb/franz-go/pkg/kgo" 13 | ) 14 | 15 | type TopicEncoder interface { 16 | Encode(v interface{}) error 17 | } 18 | 19 | type Producer interface { 20 | TopicEncoder(topic string) TopicEncoder 21 | Close() error 22 | } 23 | 24 | type FranzProducer struct { 25 | client *kgo.Client 26 | closed int32 // nonzero if the producer has started closing. accessed via atomics 27 | pendingWrites sync.WaitGroup 28 | } 29 | 30 | type ProducerConfig struct { 31 | Brokers []string 32 | } 33 | 34 | func NewFranzProducer(config ProducerConfig) (Producer, error) { 35 | opts := []kgo.Opt{ 36 | kgo.SeedBrokers(config.Brokers...), 37 | kgo.MaxBufferedRecords(1e7), 38 | } 39 | 40 | client, err := kgo.NewClient(opts...) 41 | if err != nil { 42 | return nil, err 43 | } 44 | 45 | return &FranzProducer{ 46 | client: client, 47 | }, nil 48 | } 49 | 50 | func (p *FranzProducer) TopicEncoder(topic string) TopicEncoder { 51 | return &FranzWriter{p: p, topic: topic} 52 | } 53 | 54 | func (p *FranzProducer) Closed() bool { 55 | return atomic.LoadInt32(&p.closed) != 0 56 | } 57 | 58 | func (p *FranzProducer) Close() error { 59 | if p.Closed() { 60 | return errors.New("already closed") 61 | } 62 | atomic.StoreInt32(&p.closed, 1) 63 | p.pendingWrites.Wait() 64 | p.client.Close() 65 | return nil 66 | } 67 | 68 | type FranzWriter struct { 69 | p *FranzProducer 70 | topic string 71 | } 72 | 73 | func (w *FranzWriter) Encode(i interface{}) error { 74 | out, err := json.Marshal(i) 75 | if err != nil { 76 | return err 77 | } 78 | n, err := w.Write(out) 79 | if err != nil { 80 | return err 81 | } 82 | if n != len(out) { 83 | return errors.New("short write") 84 | } 85 | return nil 86 | } 87 | 88 | func (w *FranzWriter) Write(d []byte) (int, error) { 89 | if w.p.Closed() { 90 | return 0, syscall.EINVAL 91 | } 92 | 93 | r := kgo.SliceRecord(d) 94 | r.Topic = w.topic 95 | 96 | w.p.pendingWrites.Add(1) 97 | w.p.client.Produce(context.Background(), r, func(r *kgo.Record, err error) { 98 | defer w.p.pendingWrites.Done() 99 | if err != nil && err != kgo.ErrClientClosed { 100 | log.Printf("FranzProducer error on topic %s: %+v", w.topic, err) 101 | } 102 | }) 103 | 104 | return len(d), nil 105 | } 106 | -------------------------------------------------------------------------------- /src/random.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "math/rand" 5 | "time" 6 | 7 | "github.com/rogpeppe/fastuuid" 8 | ) 9 | 10 | var UUIDGen = fastuuid.MustNewGenerator() 11 | 12 | var SampleReferrers = []string{ 13 | // List of search engines 14 | "http://www.google.com/", 15 | "http://www.bing.com/", 16 | "http://www.yahoo.com/", 17 | "http://www.baidu.com/", 18 | "http://www.aol.com/", 19 | "http://www.ask.com/", 20 | "http://www.altavista.com/", 21 | "http://www.live.com/", 22 | "http://www.msn.com/", 23 | 24 | // List of social networks 25 | "http://www.facebook.com/", 26 | "http://www.twitter.com/", 27 | "http://www.linkedin.com/", 28 | "http://www.pinterest.com/", 29 | "http://www.instagram.com/", 30 | "http://www.youtube.com/", 31 | 32 | // List of news sites 33 | "http://www.cnn.com/", 34 | "http://www.bbc.co.uk/", 35 | "http://www.nytimes.com/", 36 | "http://www.washingtonpost.com/", 37 | "http://www.reddit.com/", 38 | "http://www.huffingtonpost.com/", 39 | "http://www.theguardian.com/", 40 | "http://www.theverge.com/", 41 | } 42 | 43 | // return a random referrer 44 | func RandomReferrer() string { 45 | return SampleReferrers[rand.Intn(len(SampleReferrers))] 46 | } 47 | 48 | func NextUserId() string { 49 | return UUIDGen.Hex128() 50 | } 51 | 52 | func RandomIntInRange(min, max int) int { 53 | return rand.Intn(max-min) + min 54 | } 55 | 56 | // JitterDuration will add or subtract a random portion of the provided jitter duration to the base duration 57 | // ex: JitterDuration(time.Second, 100*time.Millisecond) could return 1.04777941s or 982.153551ms 58 | func JitterDuration(d time.Duration, jitter time.Duration) time.Duration { 59 | return (d - jitter) + time.Duration(rand.Int63n(int64(2*jitter))) 60 | } 61 | -------------------------------------------------------------------------------- /src/simulator.go: -------------------------------------------------------------------------------- 1 | // +build active_file 2 | 3 | package src 4 | 5 | import ( 6 | "time" 7 | ) 8 | 9 | func (s *Simulator) Run() error { 10 | testTopic := s.producer.TopicEncoder("test") 11 | 12 | for s.Running() { 13 | time.Sleep(JitterDuration(time.Second, 200*time.Millisecond)) 14 | 15 | err := testTopic.Encode(map[string]interface{}{ 16 | "message": "hello world", 17 | "time": time.Now(), 18 | "worker": s.id, 19 | }) 20 | if err != nil { 21 | return err 22 | } 23 | } 24 | 25 | return nil 26 | } 27 | -------------------------------------------------------------------------------- /src/simulator_base.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "fmt" 5 | "log" 6 | ) 7 | 8 | type Simulator struct { 9 | id string 10 | config *SimConfig 11 | closeCh chan struct{} 12 | stopped bool 13 | 14 | producer Producer 15 | sitemap *Page 16 | } 17 | 18 | func NewSimulator(id string, config *SimConfig, sitemap *Page) (*Simulator, error) { 19 | p, err := NewFranzProducer(config.Producer) 20 | if err != nil { 21 | return nil, err 22 | } 23 | 24 | return &Simulator{ 25 | id: id, 26 | config: config, 27 | closeCh: make(chan struct{}), 28 | producer: p, 29 | sitemap: sitemap, 30 | }, nil 31 | } 32 | 33 | func (s *Simulator) Running() bool { 34 | if !s.stopped { 35 | // If the closeCh is closed, we have been stopped. 36 | select { 37 | case <-s.closeCh: 38 | s.stopped = true 39 | default: 40 | } 41 | } 42 | return !s.stopped 43 | } 44 | 45 | func (s *Simulator) Stop() { 46 | s.logf("stopping") 47 | close(s.closeCh) 48 | } 49 | 50 | func (s *Simulator) logf(msg string, args ...interface{}) { 51 | log.Printf(fmt.Sprintf("[%s] %s", s.id, msg), args...) 52 | } 53 | -------------------------------------------------------------------------------- /src/simulator_finished.go: -------------------------------------------------------------------------------- 1 | // +build !active_file 2 | 3 | package src 4 | 5 | import ( 6 | "container/list" 7 | "math" 8 | "math/rand" 9 | "time" 10 | ) 11 | 12 | type User struct { 13 | UserID string 14 | CurrentPage *Page 15 | LastChange time.Time 16 | } 17 | 18 | type Event struct { 19 | Timestamp int64 `json:"unix_timestamp"` 20 | PageTime float64 `json:"page_time_seconds,omitempty"` 21 | Referrer string `json:"referrer,omitempty"` 22 | UserID string `json:"user_id"` 23 | Path string `json:"path"` 24 | } 25 | 26 | func (s *Simulator) Run() error { 27 | users := list.New() 28 | 29 | events := s.producer.TopicEncoder("events") 30 | 31 | for s.Running() { 32 | time.Sleep(JitterDuration(time.Second, 200*time.Millisecond)) 33 | unixNow := time.Now().Unix() 34 | 35 | // create a random number of new users 36 | if users.Len() < s.config.MaxUsersPerThread { 37 | 38 | // figure out the max number of users we can create 39 | maxNewUsers := s.config.MaxUsersPerThread - users.Len() 40 | 41 | // calculate a random number between 1 and maxNewUsers 42 | numNewUsers := RandomIntInRange(1, maxNewUsers) 43 | 44 | // create the new users 45 | for i := 0; i < numNewUsers; i++ { 46 | // define a new user 47 | user := &User{ 48 | UserID: NextUserId(), 49 | CurrentPage: s.sitemap.RandomLeaf(), 50 | LastChange: time.Now(), 51 | } 52 | 53 | // add the user to the list 54 | users.PushBack(user) 55 | 56 | // write an event to the topic 57 | err := events.Encode(Event{ 58 | Timestamp: unixNow, 59 | UserID: user.UserID, 60 | Path: user.CurrentPage.Path, 61 | 62 | // we provide a fake referrer here 63 | Referrer: RandomReferrer(), 64 | }) 65 | if err != nil { 66 | return err 67 | } 68 | } 69 | } 70 | 71 | // we will be removing elements from the list while we iterate, so we 72 | // need to keep track of next outside of the loop 73 | var next *list.Element 74 | 75 | // iterate through the users list and simulate each users behavior 76 | for el := users.Front(); el != nil; el = next { 77 | next = el.Next() 78 | user := el.Value.(*User) 79 | pageTime := time.Since(user.LastChange) 80 | 81 | // users only consider leaving a page after at least 5 seconds 82 | if pageTime > time.Second*5 { 83 | 84 | // eventProb is a random value from 0 to 1 but is weighted 85 | // to be closer to 0 most of the time 86 | eventProb := math.Pow(rand.Float64(), 2) 87 | 88 | if eventProb > 0.98 { 89 | // user has left the site 90 | users.Remove(el) 91 | continue 92 | } else if eventProb > 0.9 { 93 | // user jumps to a random page 94 | user.CurrentPage = s.sitemap.RandomLeaf() 95 | user.LastChange = time.Now() 96 | } else if eventProb > 0.8 { 97 | // user goes to the "next" page 98 | user.CurrentPage = user.CurrentPage.RandomNext() 99 | user.LastChange = time.Now() 100 | } 101 | } 102 | 103 | err := events.Encode(Event{ 104 | Timestamp: unixNow, 105 | UserID: user.UserID, 106 | Path: user.CurrentPage.Path, 107 | PageTime: pageTime.Seconds(), 108 | }) 109 | if err != nil { 110 | return err 111 | } 112 | } 113 | } 114 | 115 | return nil 116 | } 117 | -------------------------------------------------------------------------------- /src/singlestore.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "database/sql" 5 | "fmt" 6 | "time" 7 | 8 | "github.com/go-sql-driver/mysql" 9 | "github.com/jmoiron/sqlx" 10 | ) 11 | 12 | type SingleStore struct { 13 | *sqlx.DB 14 | } 15 | 16 | type SingleStoreConfig struct { 17 | Host string 18 | Port int 19 | Username string 20 | Password string 21 | Database string 22 | } 23 | 24 | // NewSingleStore connects to the specified SingleStore database and returns a 25 | // SingleStore object ready to serve queries 26 | func NewSingleStore(config SingleStoreConfig) (*SingleStore, error) { 27 | mysqlConf := mysql.NewConfig() 28 | mysqlConf.User = config.Username 29 | mysqlConf.Passwd = config.Password 30 | mysqlConf.DBName = config.Database 31 | mysqlConf.Addr = fmt.Sprintf("%s:%d", config.Host, config.Port) 32 | mysqlConf.ParseTime = true 33 | mysqlConf.Timeout = 10 * time.Second 34 | mysqlConf.InterpolateParams = true 35 | mysqlConf.AllowNativePasswords = true 36 | mysqlConf.MultiStatements = false 37 | 38 | mysqlConf.Params = map[string]string{ 39 | "collation_server": "utf8_general_ci", 40 | "sql_select_limit": "18446744073709551615", 41 | "compile_only": "false", 42 | "enable_auto_profile": "false", 43 | "sql_mode": "'STRICT_ALL_TABLES'", 44 | } 45 | 46 | connector, err := mysql.NewConnector(mysqlConf) 47 | if err != nil { 48 | return nil, err 49 | } 50 | 51 | db := sql.OpenDB(connector) 52 | 53 | err = db.Ping() 54 | if err != nil { 55 | db.Close() 56 | return nil, err 57 | } 58 | 59 | db.SetConnMaxLifetime(time.Hour) 60 | db.SetMaxIdleConns(20) 61 | 62 | return &SingleStore{sqlx.NewDb(db, "mysql")}, nil 63 | } 64 | -------------------------------------------------------------------------------- /src/sitemap.go: -------------------------------------------------------------------------------- 1 | package src 2 | 3 | import ( 4 | "math/rand" 5 | "net/url" 6 | "strings" 7 | 8 | sitemap "github.com/oxffaa/gopher-parse-sitemap" 9 | ) 10 | 11 | type Page struct { 12 | Path string 13 | Parent *Page 14 | Children []*Page 15 | } 16 | 17 | func (p *Page) NewChild(path string) *Page { 18 | child := &Page{Path: path, Parent: p} 19 | p.Children = append(p.Children, child) 20 | return child 21 | } 22 | 23 | func (p *Page) AddChildRecursive(path string) { 24 | ptr := p 25 | totalPath := "" 26 | path = strings.TrimLeft(path, "/") 27 | for _, part := range strings.Split(path, "/") { 28 | totalPath += strings.TrimRight("/"+part, "/") 29 | ptr = ptr.NewChild(totalPath) 30 | } 31 | } 32 | 33 | // Get a random leaf node starting at this point on the tree 34 | func (p *Page) RandomLeaf() *Page { 35 | if len(p.Children) == 0 { 36 | return p 37 | } 38 | 39 | return p.RandomChild().RandomLeaf() 40 | } 41 | 42 | // Get a random child node of this page, or return this page if it has no children 43 | func (p *Page) RandomChild() *Page { 44 | if len(p.Children) == 0 { 45 | return p 46 | } 47 | 48 | return p.Children[rand.Intn(len(p.Children))] 49 | } 50 | 51 | // Get a random child or this pages parent, prefers to traverse down the tree 52 | func (p *Page) RandomNext() *Page { 53 | if len(p.Children) > 0 { 54 | return p.RandomChild() 55 | } 56 | return p.Parent 57 | } 58 | 59 | func LoadSitemap(sitemapURL string) (*Page, error) { 60 | root := &Page{} 61 | 62 | return root, sitemap.ParseFromSite(sitemapURL, func(e sitemap.Entry) error { 63 | loc, err := url.Parse(e.GetLocation()) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | root.AddChildRecursive(loc.Path) 69 | 70 | return nil 71 | }) 72 | } 73 | -------------------------------------------------------------------------------- /tasks: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ##################################################################### 4 | # 5 | # Run ./tasks help for a list of available tasks 6 | # 7 | ##################################################################### 8 | 9 | # Enable shell strict mode 10 | set -euo pipefail 11 | 12 | # source environment variables from local env file if it exists 13 | ENV_FILE=".env" 14 | if [[ -f "${ENV_FILE}" ]]; then 15 | source "${ENV_FILE}" 16 | fi 17 | 18 | SINGLESTORE_LICENSE="${SINGLESTORE_LICENSE:-}" 19 | 20 | # prompt the user for license key if it's not provided in a environment variable 21 | if [[ -z "${SINGLESTORE_LICENSE}" ]]; then 22 | # prompt the user until they enter a valid license key 23 | while [[ -z "${SINGLESTORE_LICENSE}" ]]; do 24 | echo "SINGLESTORE_LICENSE environment variable not found" 25 | echo "You can get a free SingleStore license key from https://portal.singlestore.com/" 26 | echo "Please enter your SingleStore license key:" 27 | read SINGLESTORE_LICENSE 28 | done 29 | 30 | # save the license key to the environment file for future use 31 | echo "export SINGLESTORE_LICENSE='${SINGLESTORE_LICENSE}'" >>"${ENV_FILE}" 32 | fi 33 | 34 | help() { 35 | cat < [] 37 | 38 | tasks: 39 | up: (re)start all containers 40 | down: stop all containers 41 | status: show status of containers 42 | logs [SERVICE]: show logs of a service 43 | rpk: run a Redpanda command (run ./tasks rpk for help) 44 | monitoring: (re)start prometheus and grafana 45 | storage: (re)start singlestore and redpanda 46 | simulator: (re)start simulator 47 | stop-simulator: stop simulator 48 | api: (re)start api 49 | stop-api: stop api 50 | EOF 51 | } 52 | 53 | status() { 54 | docker-compose ps 55 | } 56 | 57 | logs() { 58 | local service=${1:-} 59 | docker-compose logs --tail 100 -f ${service} 60 | } 61 | 62 | rpk() { 63 | docker exec -it -e REDPANDA_BROKERS=redpanda:9092 redpanda rpk "${@}" 64 | } 65 | 66 | monitoring() { 67 | docker-compose rm -fsv prometheus grafana 68 | docker-compose up -d prometheus grafana 69 | } 70 | 71 | storage() { 72 | docker-compose rm -fsv singlestore redpanda 73 | 74 | docker-compose up -d redpanda 75 | sleep 1 76 | rpk topic create test --partitions 8 77 | rpk topic create events --partitions 8 78 | 79 | docker-compose up -d singlestore 80 | } 81 | 82 | simulator() { 83 | docker-compose rm -fsv simulator 84 | docker-compose up --build -d simulator 85 | docker-compose logs --tail=10 simulator 86 | } 87 | 88 | stop-simulator() { 89 | docker-compose rm -fsv simulator 90 | } 91 | 92 | api() { 93 | docker-compose rm -fsv api 94 | docker-compose up --build -d api 95 | docker-compose logs --tail=10 api 96 | } 97 | 98 | stop-api() { 99 | docker-compose rm -fsv api 100 | } 101 | 102 | down() { 103 | docker-compose down -v 104 | } 105 | 106 | up() { 107 | down 108 | monitoring 109 | storage 110 | simulator 111 | api 112 | status 113 | } 114 | 115 | type ${1:-help} >/dev/null 2>&1 || { 116 | echo "Unknown command: ${1:-help}" 117 | echo 118 | help 119 | exit 1 120 | } 121 | 122 | "${@:-help}" 123 | --------------------------------------------------------------------------------