├── .gitignore ├── Capfile ├── Gemfile ├── Gemfile.lock ├── Guardfile ├── LICENSE.txt ├── README.md ├── config ├── deploy.rb ├── system.config ├── voncount.config └── voncount.nginx.conf ├── lib ├── log_player.lua ├── nginx │ ├── get.lua │ ├── init.lua │ ├── ping.lua │ ├── request_metadata_parameters_plugins │ │ ├── country.lua │ │ ├── date_time.lua │ │ └── registered_plugins.lua │ ├── utils.lua │ └── voncount.lua ├── redis │ └── voncount.lua └── scripts │ ├── backup-upload.sh │ ├── backup.sh │ ├── monitors │ └── s3-monitor.sh │ ├── reload.sh │ ├── setup.sh │ └── update-geoip.sh └── spec ├── config ├── spec_config.yml └── voncount.config ├── integration └── log_player_integrator.rb ├── lib ├── counting_spec.rb ├── get_spec.rb └── utils_spec.rb ├── script_loader.rb ├── spec_helper.rb └── support └── redis_object_factory.rb /.gitignore: -------------------------------------------------------------------------------- 1 | config/personal.yml 2 | -------------------------------------------------------------------------------- /Capfile: -------------------------------------------------------------------------------- 1 | load 'deploy' 2 | # Uncomment if you are using Rails' asset pipeline 3 | # load 'deploy/assets' 4 | load 'config/deploy' # remove this line to skip loading any of the default tasks -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | group :development, :test do 4 | gem 'guard' 5 | gem 'guard-rspec' 6 | gem 'spork' 7 | gem 'rspec' 8 | gem 'redis' 9 | gem 'redis-rails' 10 | gem 'capistrano' 11 | gem 'json' 12 | gem 'debugger' 13 | end 14 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | actionpack (3.2.11) 5 | activemodel (= 3.2.11) 6 | activesupport (= 3.2.11) 7 | builder (~> 3.0.0) 8 | erubis (~> 2.7.0) 9 | journey (~> 1.0.4) 10 | rack (~> 1.4.0) 11 | rack-cache (~> 1.2) 12 | rack-test (~> 0.6.1) 13 | sprockets (~> 2.2.1) 14 | activemodel (3.2.11) 15 | activesupport (= 3.2.11) 16 | builder (~> 3.0.0) 17 | activesupport (3.2.11) 18 | i18n (~> 0.6) 19 | multi_json (~> 1.0) 20 | builder (3.0.4) 21 | capistrano (2.15.5) 22 | highline 23 | net-scp (>= 1.0.0) 24 | net-sftp (>= 2.0.0) 25 | net-ssh (>= 2.0.14) 26 | net-ssh-gateway (>= 1.1.0) 27 | coderay (1.0.9) 28 | columnize (0.3.6) 29 | debugger (1.6.2) 30 | columnize (>= 0.3.1) 31 | debugger-linecache (~> 1.2.0) 32 | debugger-ruby_core_source (~> 1.2.3) 33 | debugger-linecache (1.2.0) 34 | debugger-ruby_core_source (1.2.3) 35 | diff-lcs (1.2.4) 36 | erubis (2.7.0) 37 | ffi (1.9.0) 38 | formatador (0.2.4) 39 | guard (1.8.3) 40 | formatador (>= 0.2.4) 41 | listen (~> 1.3) 42 | lumberjack (>= 1.0.2) 43 | pry (>= 0.9.10) 44 | thor (>= 0.14.6) 45 | guard-rspec (2.5.0) 46 | guard (>= 1.1) 47 | rspec (~> 2.11) 48 | highline (1.6.19) 49 | hike (1.2.1) 50 | i18n (0.6.4) 51 | journey (1.0.4) 52 | json (1.8.0) 53 | listen (1.3.1) 54 | rb-fsevent (>= 0.9.3) 55 | rb-inotify (>= 0.9) 56 | rb-kqueue (>= 0.2) 57 | lumberjack (1.0.4) 58 | method_source (0.8.2) 59 | multi_json (1.6.1) 60 | net-scp (1.1.2) 61 | net-ssh (>= 2.6.5) 62 | net-sftp (2.1.2) 63 | net-ssh (>= 2.6.5) 64 | net-ssh (2.7.0) 65 | net-ssh-gateway (1.2.0) 66 | net-ssh (>= 2.6.5) 67 | pry (0.9.12.2) 68 | coderay (~> 1.0.5) 69 | method_source (~> 0.8) 70 | slop (~> 3.4) 71 | rack (1.4.5) 72 | rack-cache (1.2) 73 | rack (>= 0.4) 74 | rack-test (0.6.2) 75 | rack (>= 1.0) 76 | rb-fsevent (0.9.3) 77 | rb-inotify (0.9.2) 78 | ffi (>= 0.5.0) 79 | rb-kqueue (0.2.0) 80 | ffi (>= 0.5.0) 81 | redis (3.0.3) 82 | redis-actionpack (3.2.3) 83 | actionpack (~> 3.2.3) 84 | redis-rack (~> 1.4.0) 85 | redis-store (~> 1.1.0) 86 | redis-activesupport (3.2.3) 87 | activesupport (~> 3.2.3) 88 | redis-store (~> 1.1.0) 89 | redis-rack (1.4.2) 90 | rack (~> 1.4.1) 91 | redis-store (~> 1.1.0) 92 | redis-rails (3.2.3) 93 | redis-actionpack (~> 3.2.3) 94 | redis-activesupport (~> 3.2.3) 95 | redis-store (~> 1.1.0) 96 | redis-store (1.1.3) 97 | redis (>= 2.2.0) 98 | rspec (2.14.1) 99 | rspec-core (~> 2.14.0) 100 | rspec-expectations (~> 2.14.0) 101 | rspec-mocks (~> 2.14.0) 102 | rspec-core (2.14.5) 103 | rspec-expectations (2.14.3) 104 | diff-lcs (>= 1.1.3, < 2.0) 105 | rspec-mocks (2.14.3) 106 | slop (3.4.6) 107 | spork (1.0.0rc3) 108 | sprockets (2.2.2) 109 | hike (~> 1.2) 110 | multi_json (~> 1.0) 111 | rack (~> 1.0) 112 | tilt (~> 1.1, != 1.3.0) 113 | thor (0.18.1) 114 | tilt (1.3.5) 115 | 116 | PLATFORMS 117 | ruby 118 | 119 | DEPENDENCIES 120 | capistrano 121 | debugger 122 | guard 123 | guard-rspec 124 | json 125 | redis 126 | redis-rails 127 | rspec 128 | spork 129 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | guard :rspec, all_on_start:false, notification:false, cli: "--color --format nested --fail-fast --drb" do 2 | watch(%r{^spec/.+_spec\.rb$}) 3 | watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" } 4 | watch('spec/spec_helper.rb') { "spec" } 5 | end 6 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 FTBpro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Count Von Count 2 | 3 | ![alt tag](http://1.bp.blogspot.com/_zCGbA5Pv0PI/TGj5YnGEDDI/AAAAAAAADD8/ipYKIgc7Jg0/s400/CountVonCount.jpg) 4 | 5 | Count-von-Count is an open source project that was developed at FTBpro for a gamification project and it turned into something that can count any kind of action. It is based on Nginx and Redis, and leverages them to create a live, scalable and easy to use counting solution for a wide range of scenarios. 6 | 7 | 8 | # Table of Contents 9 | 10 | * [What It's All About](#what) 11 | * [General Overview](#general-overview) 12 | * [Getting Started - Counting, It's easy as 1,2,3](#getting-started) 13 | * [Clients](#clients) 14 | * [Server Configuration](#server-config) 15 | * [Counting Configuration](#counting-configuration) 16 | * [Counting Options](#counting-options) 17 | * [Simple Count](#simple-count) 18 | * [Multiple Objects of The Same Type](#multiple-objects-of-the-same-type) 19 | * [Multiple Objects of Different Type](#multiple-objects-of-different-types) 20 | * [Object With Multiple Ids](#object-with-multiple-ids) 21 | * [Dynamic Count - Parameter as Counter Name](#dynamic-count) 22 | * [Counting Several Actions](#counting-several) 23 | * [Ordered Sets (Leaderboard) - Save The Counters Data in Order](#ordered) 24 | * [Advanced: Custom Functions - Writing Your Own Logic](#advance) 25 | * [Variable Count - Not Just Increase by 1](#variable-count) 26 | * [Expire - Setting TTL on Redis Data](#expire) 27 | * [Request Metadata Parameters Plugins](#request-metadata) 28 | * [DateTime Plugin](#datatime-plugin) 29 | * [Customization](#customization) 30 | * [Country Plugin](#country-plugin) 31 | * [Prerequisites](#prerequisites) 32 | * [Customization](#customization) 33 | * [Retrieving Data](#retrieving-data) 34 | * [Advanced](#advanced) 35 | * [Architecture](#architecture) 36 | * [Log Player](#log-player) 37 | * [Deploy using Ruby and Capistrano](#deploy-using-ruby) 38 | * [Contributing](#contributing) 39 | * [Contact Us](#contact-us) 40 | 41 | # What It's All About 42 | 43 | Count-von-Count can help you whenever you need to store counters data. Its advantage is that it can process thousands of requests per second and update the numbers in real-time, no caching/background processes needed. 44 | 45 | [Here is a short video about this project](https://www.youtube.com/watch?v=VG8iOVNS_xg) 46 | 47 | With Count-von-Count you can: 48 | 49 | 1. Count number of visitors on a site/page. 50 | 2. Track number of clicks hourly/daily/weekly/yearly etc... 51 | 3. Measure load time in any client and quickly see the slowest clients. 52 | 4. Store any kind of Leaderboard, such as top writers, top readers, top countries your visitors come from. 53 | 5. Anything that can be counted! 54 | 55 | Here are some ways that we use it in [FTBPro.com](https://www.ftbpro.com) 56 | 57 | ![Some Counters](http://media.tumblr.com/c0508089f2e631613bff664a94599d10/tumblr_inline_mwtdlnw5d21s9eocl.png) 58 | 59 | # General Overview 60 | 61 | ![General Overview](https://s3-us-west-2.amazonaws.com/action-counter-logs/cvc-general.png) 62 | 63 | Count-von-Count is a web server that uses Redis as a database. When a client wants to report an action which should be counted, he initiates a request to the counting server: `/?`. The response is a 1X1 empty pixel, for reducing the overhead in calling it from JavaScript clients. 64 | A configuration file, [von_vount.config](#counting-configuration), which is defined in the server, sets the rules of the counting - what to update for each action. No coding is needed! The updates are synchronously committed to the database. 65 | The sever also has an API for [retrieving](#retrieving-data) the data. 66 | 67 | # Installation 68 | 69 | 1. Install redis-server (apt-get install redis-server). You can also use one of your previously installed servers. 70 | 2. Download and install [OpenResty](http://openresty.org/#install). Use default settings and directory structure! 71 | 3. Clone count-von-count. 72 | 4. If you are not using Redis with its default settings (localhost, port 6379), update `config/system.config` file with the Redis server IP and port. 73 | 5. Edit openresty's nginx.conf file (by default, it's in /usr/local/openresty/nginx/conf) 74 | * add `worker_rlimit_nofile 30000;` at the top level 75 | * add `include /usr/local/openresty/nginx/conf/include/*;` under the 'http' section 76 | 77 | ```conf 78 | #nginx.conf 79 | worker_rlimit_nofile 30000; 80 | 81 | http { 82 | include /usr/local/openresty/nginx/conf/include/*; 83 | . 84 | . 85 | . 86 | ``` 87 | 6. After installation is complete, you need to set up the server. 88 | 89 | If you are familiar with Ruby and Capistrano, you can skip this section and follow this - [Deploy using Ruby & Capistrano](#deploy-using-ruby-and-capistrano) 90 | 91 | Run `sudo ./lib/scripts/setup.sh`. If the last 2 output lines are 92 | 93 | ~~~ 94 | >>> nginx is running 95 | >>> redis-server is running 96 | ~~~ 97 | 98 | then you should be good to go. 99 | 100 | 7. For the first time and every time you modify the `voncount.config` or the code - run `sudo ./lib/scripts/reload.sh` 101 | 102 | # Getting Started - Counting, It's easy as 1,2,3 103 | 104 | ## Clients 105 | 106 | Each request to the server should be in the following fromat: `/?`. 107 | 108 | ### Browsers 109 | 110 | Add an img element to your html document `/?>`. You can also load it using JavaScript. 111 | 112 | Some browsers perform asset caching on images, so it's important to add to the request a parameter that changes from request to request, e.g, current time: `/?&&ts=` 113 | 114 | ## Server Configuration 115 | 116 | Now, after you've understood the general overview of how the system works, and you've installed & set up your server, you are ready to get your hands really dirty! :-) 117 | 118 | The `config/voncount.config` file is the heart of the system, and determines what gets counted and how. 119 | 120 | The file is written in standard JSON format, and for most use-cases changing it and customizing it to your needs is enough. **You won't even need to write code!** 121 | 122 | We'll show here some examples that cover all the different options the system supports. Most of them are real life examples taken from our production environment. 123 | 124 | But let's start with a simple example: Say that you have a blog site and you want to count the number of reads that each blog post gets. 125 | You need to take care of 2 things: 126 | 127 | 1. In each post page, make a server call (via img src=... or javascript) - http://my-von-count.com/reads?post=3144 (3144 is a unique identifier of the current post). 128 | 2. Set the following configration: 129 | 130 | ```JSON 131 | { 132 | "reads": { 133 | "Post": [ 134 | { 135 | "id": "post", 136 | "count": "num_reads" 137 | } 138 | ] 139 | } 140 | } 141 | ```` 142 | ** Don't forget to run `reload.sh` script after you change the configuration! ** 143 | 144 | That's it! For each post that gets read, you'll have data in the Redis DB in the format of 145 | >Post_3144: { num_reads: 5772 } 146 | 147 | To get the number of reads for post 3144 you can run `redis-cli hget Post_3144 num_reads` 148 | 149 | Now let's see a general example for the most basic use case: 150 | 151 | ```JSON 152 | { 153 | "": { 154 | "": [ 155 | { 156 | "id": "", 157 | "count": "" 158 | } 159 | ] 160 | } 161 | } 162 | ```` 163 | With this configuration, we can make a call to ```http://my-von-count.com/?=1234```, which will result in having a `_` key in our Redis DB, with value `{ : 1 }` 164 | 165 | Making the same call again will result in incrementing the value of `_` to `{ : 2 }` 166 | 167 | # Counting Configuration 168 | 169 | As mentioned earlier,`config/voncount.config` file is the heart of the system, and determines what gets counted and how. Let's go over the different configuration options. 170 | 171 | ## Counting Options 172 | 173 | To get you in context, here is a short description of our domain - 174 | 175 | At [FTBpro](https://www.ftbpro.com) we have `posts`, `users`, and `teams`. 176 | Each `post` is written by a `user` who is the author, and the post "belongs" to a `team`. 177 | 178 | ### Simple Count 179 | When a `post` gets read, we want to increase the number of reads the `author` has received. 180 | ```JSON 181 | { 182 | "reads": { 183 | "User": [ 184 | { 185 | "id": "author", 186 | "count": "reads_got" 187 | } 188 | ] 189 | } 190 | } 191 | ``` 192 | 193 | The top-most level of the configuration JSON keys is the action type that we want to count - `reads`. 194 | 195 | `User` is the object for which we want to count the `reads` action. 196 | 197 | `id` is the object (e.g. user's) id, and it should be defined in the query string parameters when making a call to the count-von-count server. 198 | 199 | `count` is what we count/increase. 200 | 201 | So, with the above configuration, when we make a call to http://my-von-count.com/reads?author=1234, then our Redis DB will have the following key: 202 | >User_1234: { reads_got: 1 } 203 | 204 | Compared to the general example: 205 | * `` = `reads` 206 | * `` = `User` 207 | * `` = `author` 208 | * `` = `reads_got` 209 | 210 | So given a `reads` ``, the `reads_got` `` of the `User` `` with `` equals to `author`'s value, will be increased by one. 211 | 212 | **Notice** - in voncount.config, the value of `` (`reads`) is a hash, and the value of the `` (`User`) is an array of hashes. 213 | 214 | ### Multiple Objects of The Same Type 215 | Now let's add another counter: the number of posts a user has read. 216 | 217 | That means that when a post gets read, we want to increase a counter for the author, like in previous example, and also increase a counter for the user who is now reading the post in order to know how many posts each user has read. 218 | 219 | ```JSON 220 | { 221 | "reads": { 222 | "User": [ 223 | { 224 | "id": "author", 225 | "count": "reads_got" 226 | }, 227 | { 228 | "id": "user", 229 | "count": "reads" 230 | } 231 | ] 232 | } 233 | } 234 | ``` 235 | 236 | The one who is reading the post now is also a `User`, so we define it under the already existing `User` object. 237 | 238 | The user's id is defined in the query string params as the `user` parameter and for him we count `reads`. 239 | After a call to http://my-von-count.com/reads?author=1234&user=5678 our Redis DB will have the following keys: 240 | 241 | >User_1234: { reads_got: 2 } 242 | > 243 | >User_5678: { reads: 1 } 244 | 245 | ### Multiple Objects of Different Types 246 | We also want to know how many `reads` each `Post` has received. 247 | 248 | To do this, we add the following configuration for `Post` object under the `reads` action, and we add a 'post' id to the query string parameters. 249 | 250 | ```JSON 251 | { 252 | "reads": { 253 | "User": [ 254 | { 255 | "id": "author", 256 | "count": "reads_got" 257 | }, 258 | { 259 | "id": "user", 260 | "count": "reads" 261 | } 262 | ], 263 | "Post": [ 264 | { 265 | "id": "post", 266 | "count": "reads" 267 | } 268 | ] 269 | } 270 | } 271 | ``` 272 | 273 | After a call to http://my-von-count.com/reads?author=1234&user=5678&post=888, our Redis db will have the following keys: 274 | 275 | >User_1234: { reads_got: 3 } 276 | > 277 | >User_5678: { reads: 2 } 278 | > 279 | >Post_888: { reads: 1 } 280 | 281 | 282 | ### Object With Multiple Ids 283 | On [FTBpro](https://www.ftbpro.com) we are doing daily analytics. 284 | 285 | For each `user` we want to know how many posts he has read each day. 286 | 287 | We'll define a "UserDaily" object. Its id will be the user's id and the current date. 288 | 289 | ```JSON 290 | { 291 | "reads": { 292 | . 293 | . 294 | . 295 | "UserDaily": [ 296 | { 297 | "id": [ 298 | "user", 299 | "day", 300 | "month", 301 | "year" 302 | ], 303 | "count": "reads", 304 | "expire": 1209600 305 | } 306 | ], 307 | ``` 308 | 309 | Notice the `` of the `UserDaily` object is an **array** composed of 4 parameters, so after a call to 310 | http://my-von-count.com/reads?user=5678, Redis DB will have the following keys: 311 | >UserDaily_5678_28_11_2013: { reads: 1 } 312 | 313 | 314 | **WAIT A SECOND!** the query string contains only a `user` parameter. Where do the other 3 parameters (`day`, `month`, `year`) come from?!? Read more about it on [Request Metadata Parameters Plugins](#request-metadata-parameters-plugins). 315 | 316 | 317 | ### Dynamic Count - Parameter as Name 318 | We can use parameters to determine the `` name. In that way we can dynamically determine what gets counted. 319 | 320 | In this example, we count the number of reads an author had from each country (every week). 321 | 322 | ```JSON 323 | { 324 | "reads": { 325 | . 326 | . 327 | . 328 | "UserWeeklyDemographics": [ 329 | { 330 | "id": [ 331 | "author", 332 | "week", 333 | "year" 334 | ], 335 | "count": "{country}" 336 | } 337 | ] 338 | ``` 339 | You can see we are using the `week` and `year` parameters for the `` as in the example above, and also the `country` parameter as the `` name. 340 | 341 | The `country` parameter is explained under [Request Metadata Parameters Plugins](#request-metadata-parameters-plugins). 342 | 343 | The data will be something like - 344 | >UserWeeklyDemographics_5678_42_2013: { US: 5, UK: 8, FR: 1 } 345 | 346 | \*It's possible to use a parameter name that is passed in the request query string (e.g. `author`, `post`, etc...), and not only the Metadata Parameters! 347 | 348 | \* `` names can be be more complex. With we have a `registered` parameter in the request query string, in the config file we can define - `"count": "from_{country}_{registered}"` 349 | 350 | 351 | ### Counting Several Actions 352 | So far we've seen examples related to post reads, but users can also comment on posts. 353 | Very similarly to the `reads` action, we also want to count: 354 | * for each `author` - how many comments he received on his posts 355 | * for each `user` - how many comments he wrote 356 | * for each `post` - how many comments it received 357 | 358 | On the `voncount.config` file, at the JSON's top level (**not** nested under the `reads` action!) we'll add - 359 | 360 | ```JSON 361 | { 362 | "reads": { 363 | . 364 | . 365 | . 366 | }, 367 | 368 | "comments": { 369 | "User": [ 370 | { 371 | "id": "user", 372 | "count": "comments" 373 | }, 374 | { 375 | "id": "author", 376 | "count": "comments_got" 377 | } 378 | ], 379 | 380 | "Post": [ 381 | { 382 | "id": "post", 383 | "count": "comments" 384 | } 385 | ] 386 | } 387 | } 388 | ``` 389 | 390 | When making a call to count.com/comments?author=1234&user=5678&post=888, our redis DB will have the following keys: 391 | >User_1234: { comments_got: 1 } 392 | > 393 | >User_5678: { comments: 1 } 394 | > 395 | >Post_888: { comments: 1 } 396 | 397 | ### Ordered Sets (Leaderboard) - Save The Counters Data in Order 398 | In all previous examples, the data was saved as a Redis Hash type. 399 | 400 | It is possible to save the data as an order set as well. In that way, your data can be sorted automatically. This is very useful for storing leaderboards. 401 | 402 | For example, we want to know the "top 3 posts" (posts that got the most reads) in each day - 403 | 404 | ```JSON 405 | { 406 | "reads": { 407 | . 408 | . 409 | . 410 | "PostDaily": [ 411 | { 412 | "type": "set", 413 | "id": [ 414 | "day", 415 | "month", 416 | "year" 417 | ], 418 | "count": "post_{post}", 419 | "expire": 604800 420 | } 421 | ] 422 | ``` 423 | You can see we defined a `type` of "set". You can also define `type` as "hash", but this is the default option, so you can just skip this definition as in previous examples. 424 | 425 | \* We are using the `post` id as part of the `` name, similar to [Dynamic Count - Parameter as Counter Name](#dynamic-counter) 426 | 427 | The data for this example will look like - 428 | 429 | >PostDaily_28_11_2013: { 430 | > 431 | > post_888: 2, 432 | > 433 | > post_53: 15, 434 | > 435 | > post_932: 26, 436 | > 437 | > ... 438 | >} 439 | 440 | The data is ordered, and you can retrieve it using Redis `zrange` and `zrevrange` commands, and - for the sake of our example - get the "top 3 posts" without fetching the entire set and doing your own sort on the values, but simply by - `zrevrange PostDaily_28_11_2013 0 2` 441 | 442 | Later when we talk about [retrieving data](#retrieving-data), we'll show how to retrieve data through the server, and without accessing Redis directly. 443 | 444 | ### Advanced: Custom Functions - Writing Your Own Logic 445 | You have the option to go crazy and implement more complex logics for your counters. To do that, you'll need to get your hands a bit dirty and write some Lua code. 446 | 447 | In this example, we'll implement *conditional count*: 448 | 449 | When a new `post` is created, we report a `post_create` action - http://my-von-count.com/post_create?user=1234, which increases the `post_created` counter for the `user` who wrote the post - 450 | ```JSON 451 | "post_create": { 452 | "User": [ 453 | { 454 | "id": "user", 455 | "count": "post_created" 456 | } 457 | ] 458 | ``` 459 | 460 | Reminder - a `post` "belongs" to a `team`, and we want to know for each `team` how many `posts` it has, 461 | 462 | **BUT** we want to count only posts that have at least 200 words. 463 | 464 | When reporting the `post_create` action, we'll add the `team` parameter to the query string, and let's also add another parameter - `long_post` that will get "true" or "false" as values. 465 | 466 | our call will look like - http://my-von-count.com/post_create?user=1234&team=10&long_post=true 467 | 468 | the config file - 469 | ```JSON 470 | "post_create": { 471 | "User": [ 472 | { 473 | "id": "user", 474 | "count": "post_created" 475 | } 476 | ], 477 | 478 | "TeamCounters": [ 479 | { 480 | "id": "team", 481 | "custom_functions": [ 482 | { 483 | "name": "conditionalCount", 484 | "args": [ 485 | "{long_post}", 486 | "posts" 487 | ] 488 | } 489 | ] 490 | } 491 | ] 492 | ``` 493 | 494 | Now let's write the Lua code for this custom function in `lib/redis/voncount.lua`: 495 | ```lua 496 | ----------------- Custom Methods ------------------------- 497 | 498 | function Base:conditionalCount(should_count, key) 499 | if should_count == "true" then 500 | self:count(key, 1) 501 | end 502 | end 503 | 504 | ``` 505 | 506 | What do we have here: 507 | - `conditionalCount` is the name of our new custom function, and it must be equal to the "name" in the config. 508 | - Our custom function received 2 arguments, which are defined by the "args" in the config: 509 | * `should_count` will receive the value of the `long_post` parameter provided in the query string. 510 | * `key` will always receive the same value - the string "posts" 511 | - `self:count(key, 1)` - is a call to a different function - `count` - which is the basic count functionality and is already defined in the `lib/redis/voncount.lua` file. (`self` is the current instance of the `Base` class that defines the `count` function and our new `conditionalCount` custom function) 512 | 513 | \* **Notice** that the `post_create` count for the `User` object is not affected, and it always gets counted, even if the `long_post` parameter is "false". 514 | 515 | The data will look like - 516 | >TeamCounters_7: { posts: 324 } 517 | 518 | ### Variable Count - Not Just Increase by 1 519 | In all previous examples we always increased the `` by 1 with each call, but this doesn't have to be the case. 520 | 521 | The system lets you decide the increment number using the `change` definition in the config file. 522 | 523 | In example #8 we showed how we count for a `user` how many posts he created. When a post is deleted we want to decrease this counter. We'll define a `post_remove` action that will be resposible for this decrement: 524 | 525 | ```JSON 526 | "post_remove": { 527 | "User": [ 528 | { 529 | "id": "user", 530 | "count": "post_created", 531 | "change": -1 532 | } 533 | ] 534 | } 535 | ``` 536 | Notice we decrease the value of the "post_created" `` 537 | 538 | So if in the DB we have: 539 | >User_1234: { post_created: 5 } 540 | Then after a call to http://my-von-count.com/post_remove?user=1234, our data will be: 541 | >User_1234: { post_created: 4 } 542 | 543 | \* `change` can have any integer value, not just 1 or -1! 544 | 545 | ## Expire - setting TTL on Redis data 546 | Those of you with a keen eye might have noticed that in examples #4 and #7 the configuration includes an `expire` definition. 547 | 548 | You can put it on any `` in the config, and it sets its TTL (time to live) in Redis. The value is in seconds (e.g. 1209600 = 2 weeks). 549 | 550 | This is useful in order to keep the amount of data in the DB at a sane size, and also helps keep it clean from old irrelevant data that we may never need again. 551 | 552 | The TTL is set on the first time the data is created in our Redis DB, and does **not** get extended when the data is updated! 553 | 554 | Take a look at example #4 again. Let's assume the DB is empty. 555 | 556 | After a call to http://my-von-count.com/reads?user=5678, our DB will have the following data - 557 | >UserDaily_5678_28_11_2013: { reads: 1 } 558 | 559 | This data will have a TTL of 2 weeks. 560 | 561 | If after 4 days we make another call to http://my-von-count.com/reads?user=5678, then our data will be 562 | >UserDaily_5678_28_11_2013: { reads: 2 } 563 | 564 | but its TTL will be equal to only the remaining 10 days! 565 | 566 | If we wait another 10 days, this data will be gone! 567 | 568 | 569 | # Request Metadata Parameters Plugins 570 | 571 | Sometimes there is a need for the key names to consist of data that is not part of the request arguments, but is based on the request metadata. Currently, we support 2 types of this case: date_time parameters and the country parameter. 572 | The Request Metadata Parameters works as a plugin mechanism. Enabling/disabling plugins can be done by adding/removing the plugin name in `lib/nginx/request_metadata_parameters_plugins/registered_plugins.lua'. Let's discuss the default plugins which come out of the box. 573 | 574 | ## DateTime Plugin 575 | 576 | | Parameter Name | Description | 577 | |----------------|----------------------------------------------------------------------| 578 | | *day* | current day of the month | 579 | | *yday* | day index of the year (out of 365) | 580 | | *week* | week index of the year (out of 52). Weeks start and end on Mondays. | 581 | | *month* | month index of the year (out of 12) | 582 | | *year* | in 4-digit format | 583 | 584 | ### Customization 585 | 586 | The plugin comes with the arguments that we think are needed. If you want to add your own parameters, just update `lib/nginx/request_metadata_parameters_plugins/date_time.lua' with the relevant time format. 587 | 588 | ## Country Plugin 589 | 590 | This plugin is disabled by default. To enable it, uncomment 'country' in `lib/nginx/request_metadata_parameters_plugins/registered_plugins.lua' file. 591 | 592 | | Parameter Name | Description | 593 | |----------------|----------------------------------------------------------------------| 594 | | *country* | 2-letter country code according to the IP of the call | 595 | 596 | 597 | ### Prerequisites 598 | 599 | This plugin uses [lua-geoip](https://github.com/agladysh/lua-geoip) and thus the following steps are necessary for this plugin: 600 | 601 | 1. Install LuaRocks (apt-get install luarocks) 602 | 2. Install lua-geoip (luarocks install lua-geoip) 603 | 3. Install Geoip (sudo apt-get install geoip-bin geoip-database libgeoip-dev) 604 | 4. Make sure to update the geoip.dat. You can use the provided `lib\scripts\update-geoip.sh' script for it. Just add it to your crontab and schedule it to execute during off-hours. 605 | 606 | ### Customization 607 | 608 | You make your changes to `lib/nginx/request_metadata_parameters_plugins/date_time.lua`. For example, if you want to save the name of the country insted of its code, you can use `geoip.name_by_id(id)` method instad of `geoip.code_by_id(id)` 609 | 610 | # Retrieving Data 611 | 612 | Retrieving data is possible through the count-von-count API or direct access to the Redis instance. 613 | 614 | ## Get Key API 615 | 616 | ### Simple Get 617 | 618 | Accessing /get?key= will return JSON containing all the fields of the given key. 619 | This works for both Redis Hashes and Sets. 620 | For example, Getting all counters for Post_1: 621 | 622 | http://my-von-count.com/get?key=Post_1 623 | 624 | ```JSON 625 | { 626 | reads: "10", 627 | likes: "3", 628 | shares: "1" 629 | } 630 | ``` 631 | 632 | ***Notice*** - you will not get zero values. For example, if you count a `share` action, but nobody has shared your post, then the result JSON will simply not have the "shares" key. 633 | 634 | ### Attributes Get 635 | 636 | Instead of getting all the existing key-value pairs of an object, you can query for specific attributes by passing an `attr[]` parameter in the request query string. 637 | 638 | For example, if you want to get only the `likes`, `shares` and `tweets` count for Post_1 (without the `reads`): 639 | 640 | http://my-von-count.com/get?key=Post_1&attr[]=likes&attr[]=shares&attr[]=tweets 641 | 642 | ```JSON 643 | { 644 | likes: "3", 645 | shares: "1", 646 | tweets: null 647 | } 648 | ``` 649 | 650 | ***Notice*** - 651 | * If you ask for an attribute that doesn't exist (or simply didn't get counts yet), the result will contain its key-value pair, as opposed to before, but with a `null` value, like for the `tweets` counter above. 652 | * You can ask for a single attribute, without using the array syntax - http://counter.ftbpro.com/get?key=Post_1&attr=likes - and the result will be a simple number (or null) and not JSON! 653 | 654 | ### Ordered Set - Get Range 655 | 656 | When requesting the value for an ordered set, you can get all the results by providing only the key, like in the first example in this section, but you can also request a specific range. Here are some examples: 657 | 658 | * top 5 (highest values) - http://my-von-count.com/get?key=PostDaily_28_11_2013&from=0&to=5 659 | * last 5 (lowest values) - http://my-von-count.com/get?key=PostDaily_28_11_2013&from=-5&to=-1 660 | * all values - http://my-von-count.com/get?key=PostDaily_28_11_2013&from=0&to=-1 661 | * places 10 to 30 - http://my-von-count.com/get?key=PostDaily_28_11_2013&from=10&to=20 662 | ***Notice*** the `to` parameter in the query string has a bad name, it should actually be called something like `amount`. In this example we ask for results from place 10, and we ask for 20 results! Giving a -1 value to `to` is actually asking for the results from `from` until the end. 663 | 664 | Results may look something like: 665 | ```JSON 666 | { 667 | post_471049: "13787", 668 | post_473365: "8857", 669 | post_473813: "8181", 670 | post_472293: "14016", 671 | post_476298: "4127", 672 | post_464297: "9228" 673 | } 674 | ``` 675 | 676 | ***Notice*** - like stated in the 1st example, the result JSON is NOT ordered, since JSON has no guarantee about order of key-value pairs. 677 | 678 | 679 | 680 | # Advanced 681 | 682 | ## Architecture 683 | 684 | ![Architecture](https://s3-us-west-2.amazonaws.com/action-counter-logs/cvc2.png) 685 | 686 | Count-von-Count uses [OpenResty](http://openresty.org/) as a web server. It's basically a Nginx server, bundled with 3rd party modules. One of them is [lua-nginx-module](https://github.com/chaoslawful/lua-nginx-module) which adds the ability to execute Lua scripts in the context of Nginx. Another useful module is [lua-resty-redis](https://github.com/agentzh/lua-resty-redis) which we use to comminicate with Redis, the place where we store the data. 687 | 688 | The flow of a counting request is: 689 | 690 | 1. A client sends a request in the format of count_von_count_server/?params. When Nginx receives the request, a Lua script is executed. After the script finishes, an empty 1X1 pixel is returned to the client. 691 | 2. This Lua script parses the query params from the request, adds additional params using the Request Metadata Parameters Plugins and calls Redis to evaluate a preloaded Lua script. 692 | 3. The Redis script updates all the relevant keys according to the von_count.config configuration file. 693 | 4. In case of a disaster, recovery is available through the Log Player. 694 | 695 | ## Log Player 696 | 697 | Count-von-count comes with a log player. It is very useful in cases of recovery after system failure or running on old logs with new logic. Its input is an access log file (a log file where the Nginx logs each incoming request). It updates Redis based on the voncount.config. 698 | 699 | # Deploy using Ruby and [Capistrano](https://github.com/capistrano/capistrano) 700 | 701 | Edit `deploy.rb` file and set the correct deploy user and your servers' IPs in the `deploy` and `env_servers` variables. 702 | 703 | **For the first time** run `cap deploy:setup` to bootstrap the server. 704 | 705 | Use `cap deploy` to deploy master branch to production. 706 | 707 | Use `cap deploy -S env=qa -S branch=bigbird` if you want to deploy to a different environment and/or a different branch. 708 | 709 | ## Contributing 710 | 711 | 1. Fork it 712 | 2. Create your feature branch (git checkout -b my-new-feature) 713 | 3. Commit your changes (git commit -am 'Added some feature') 714 | 4. Push to the branch (git push origin my-new-feature) 715 | 5. Create new Pull Request 716 | 717 | ## Contact Us 718 | 719 | For any questions, suggestions or feedback, feel free to mail us at: 720 | 721 | Ron Schwartz - ron@ftbpro.com 722 | 723 | Shai Kerer - shai@ftbpro.com 724 | -------------------------------------------------------------------------------- /config/deploy.rb: -------------------------------------------------------------------------------- 1 | # ------------------------------------------- 2 | # Git and deploytment details 3 | # ------------------------------------------- 4 | set :application, "count-von-count" 5 | set :repository, "git://github.com/FTBpro/count-von-count.git" 6 | set :scm, :git 7 | set :deploy_to, '/home/deploy/count-von-count' 8 | set :user, "deploy" 9 | set :use_sudo, false 10 | set :nginx_dir, "/usr/local/openresty/nginx" 11 | set :branch, fetch(:branch, "master") 12 | set :env, fetch(:env, "production") 13 | 14 | env_servers = { production: "***.***.***.***", qa: "***.***.***.***" } 15 | server env_servers[env.to_sym], :app, :web, :db 16 | 17 | after 'deploy:setup', 'nginx:folder_permissions', 'symlink:app', 'symlink:conf', 'redis:start', 'nginx:start' 18 | before 'deploy:restart', 'deploy:load_redis_lua' 19 | 20 | namespace :symlink do 21 | desc "Symlink nginx to application folder" 22 | task :app do 23 | run "sudo ln -sf #{deploy_to}/current/ #{nginx_dir}/count-von-count" 24 | end 25 | 26 | desc "Symlink to voncount.nginx.conf" 27 | task :conf do 28 | run "sudo mkdir -p #{nginx_dir}/conf/include" 29 | run "sudo ln -sf #{deploy_to}/current/config/voncount.nginx.conf #{nginx_dir}/conf/include/voncount.conf" 30 | end 31 | end 32 | 33 | 34 | namespace :nginx do 35 | task :start do 36 | run "sudo nginx" 37 | end 38 | 39 | task :stop do 40 | run "sudo nginx -s stop" 41 | end 42 | 43 | desc "Reload nginx with current configuration" 44 | task :reload do 45 | run "sudo nginx -s reload" 46 | end 47 | 48 | task :folder_permissions do 49 | run "sudo chown -R #{user}:#{user} #{nginx_dir}" 50 | end 51 | end 52 | 53 | namespace :redis do 54 | task :start, roles: :db do 55 | run "sudo service redis-server start" 56 | end 57 | 58 | task :stop, roles: :db do 59 | run "sudo service redis-server start" 60 | end 61 | 62 | task :restart, roles: :db do 63 | run "sudo service redis-server restart" 64 | end 65 | end 66 | 67 | def system_config 68 | @system_config ||= YAML.load_file('config/system.config') rescue {} 69 | end 70 | 71 | namespace :deploy do 72 | desc "Load the lua script to redis and saving the SHA in a file for nginx to use" 73 | task :load_redis_lua do 74 | run "sudo rm -f #{nginx_dir}/conf/include/vars.conf" 75 | run "sudo echo 'set \$redis_counter_hash '$(redis-cli -h #{system_config["redis_host"]} -p #{system_config["redis_port"]} SCRIPT LOAD \"$(cat '#{deploy_to}/current/lib/redis/voncount.lua')\")';' > #{nginx_dir}/conf/vars.conf" 76 | run "sudo echo 'set \$redis_mobile_hash '$(redis-cli SCRIPT LOAD \"$(cat '#{deploy_to}/current/lib/redis/mobile.lua')\")';' >> #{nginx_dir}/conf/vars.conf" 77 | run "sudo redis-cli set von_count_config_live \"$(cat '#{deploy_to}/current/config/voncount.config' | tr -d '\n' | tr -d ' ')\"" 78 | end 79 | 80 | task :restart do 81 | nginx.reload 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /config/system.config: -------------------------------------------------------------------------------- 1 | redis_host: 127.0.0.1 2 | redis_port: 6379 3 | -------------------------------------------------------------------------------- /config/voncount.config: -------------------------------------------------------------------------------- 1 | { 2 | "reads": { 3 | "User": [ 4 | { 5 | "id": "user", 6 | "count": "reads" 7 | }, 8 | { 9 | "id": "author", 10 | "count": "reads_got" 11 | } 12 | ], 13 | "Post": [ 14 | { 15 | "id": "post", 16 | "count": "reads" 17 | } 18 | ], 19 | 20 | "UserDaily": [ 21 | { 22 | "id": [ 23 | "user", 24 | "day", 25 | "month", 26 | "year" 27 | ], 28 | "count": "reads", 29 | "expire": 1209600 30 | } 31 | ] 32 | }, 33 | 34 | 35 | "comments": { 36 | "User": [ 37 | { 38 | "id": "user", 39 | "count": "comments" 40 | }, 41 | { 42 | "id": "author", 43 | "count": "comments_got" 44 | } 45 | ], 46 | 47 | "Post": [ 48 | { 49 | "id": "post", 50 | "count": "comments" 51 | } 52 | ] 53 | }, 54 | 55 | 56 | "post_create": { 57 | "User": [ 58 | { 59 | "id": "user", 60 | "count": "post_create" 61 | } 62 | ], 63 | 64 | 65 | "TeamCounters": [ 66 | { 67 | "id": "team", 68 | "custom_functions": [ 69 | { 70 | "name": "conditionalCount", 71 | "args": [ 72 | "{league_count}", 73 | "posts" 74 | ] 75 | } 76 | ] 77 | } 78 | ], 79 | 80 | "TeamWriters": [ 81 | { 82 | "type": "set", 83 | "id": "team", 84 | "custom_functions": [ 85 | { 86 | "name": "countAndSetIf", 87 | "args": [ 88 | "{writers_count}", 89 | "user_{user}", 90 | "TeamCounters_{team}", 91 | "writers" 92 | ] 93 | } 94 | ] 95 | } 96 | ] 97 | }, 98 | 99 | 100 | "post_remove": { 101 | "User": [ 102 | { 103 | "id": "user", 104 | "count": "post_create", 105 | "change": -1 106 | } 107 | ] 108 | } 109 | } 110 | -------------------------------------------------------------------------------- /config/voncount.nginx.conf: -------------------------------------------------------------------------------- 1 | lua_package_path "/usr/local/openresty/nginx/count-von-count/lib/nginx/?.lua;./lib/nginx/?.lua;./count-von-count/lib/nginx/?.lua;/usr/local/openresty/nginx/count-von-count/lib/nginx/request_metadata_parameters_plugins/?.lua;;"; 2 | init_by_lua_file "count-von-count/lib/nginx/init.lua"; 3 | server { 4 | listen 80; 5 | location ~/ping { 6 | default_type text/html; 7 | content_by_lua_file "count-von-count/lib/nginx/ping.lua"; 8 | } 9 | 10 | include vars.conf; 11 | 12 | location = /favicon.ico { 13 | empty_gif; 14 | } 15 | 16 | location = /robots.txt { 17 | empty_gif; 18 | } 19 | 20 | location ~ /_.gif { 21 | empty_gif; 22 | } 23 | 24 | location ~ /get { 25 | default_type application/json; 26 | add_header Access-Control-Allow-Origin *; 27 | content_by_lua_file "count-von-count/lib/nginx/get.lua"; 28 | } 29 | 30 | location ~ /(.*) { 31 | set $action $1; 32 | content_by_lua_file "count-von-count/lib/nginx/voncount.lua"; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /lib/log_player.lua: -------------------------------------------------------------------------------- 1 | package.cpath = "/usr/local/openresty/lualib/?.so;" .. package.cpath 2 | package.path = "/usr/local/openresty/nginx/Count-von-Count/lib/nginx/request_metadata_parameters_plugins/?.lua;" .. package.path 3 | local logFilePath = arg[1] 4 | 5 | local cjson = require "cjson" 6 | 7 | function setConfig() 8 | config_file = "/usr/local/openresty/nginx/Count-von-Count/config/voncount.config" 9 | f = io.popen("cat " .. config_file .. " | tr -d '\n' | tr -d ' '") 10 | content = f:read('*a') 11 | conf = cjson.encode(content) 12 | os.execute(redisCli() .. " set von_count_config_record " .. conf) 13 | end 14 | 15 | function redisCli() 16 | return "redis-cli -h " .. REDIS_CONFIG["host"] .. " -p " .. REDIS_CONFIG["port"] .. " -n " .. REDIS_CONFIG["db"] 17 | end 18 | 19 | function getRedisCountingHash() 20 | vars_conf_path = "/usr/local/openresty/nginx/conf/vars.conf" 21 | for line in io.lines(vars_conf_path) do 22 | if line:match("$redis_counter_hash") then 23 | for i in (line:gmatch("%S+")) do 24 | redisScriptHash = i 25 | end 26 | end 27 | end 28 | redisScriptHash = redisScriptHash:sub(0, -2) 29 | end 30 | 31 | function parseQueryArgs(queryArgs) 32 | params = {} 33 | for k,v in queryArgs:gmatch("([%w_]+%[?%]?)=([^&]+)") do 34 | if k:match("[[]]") then 35 | local key = k:gsub("[[]]", "") 36 | --key = k:gsub("amp;", "") 37 | if params[key] then 38 | table.insert(params[key], v) 39 | else 40 | params[key] = {v} 41 | end 42 | else 43 | params[k] = v 44 | end 45 | end 46 | return params 47 | end 48 | 49 | function parseArgs(line) 50 | ip, request_time, query_args = line:match("^([^%s]+).*%[(.*)].*GET%s*(.*)%s* HTTP") 51 | args = parseQueryArgs(query_args) 52 | args["action"] = query_args:match("%/(.*)%?") 53 | 54 | for i = 1, #request_metadata_parameters_plugins do 55 | _plugin = require (request_metadata_parameters_plugins[i]) 56 | _plugin:AddToArgsFromLogPlayer(args, line) 57 | end 58 | return args 59 | end 60 | 61 | function initAdditionalArgsPlugins() 62 | request_metadata_parameters_plugins = require "registered_plugins" 63 | for i = 1, #request_metadata_parameters_plugins do 64 | _plugin = require (request_metadata_parameters_plugins[i]) 65 | _plugin:init() 66 | end 67 | end 68 | 69 | function playLine(line) 70 | args = parseArgs(line) 71 | args_json = "'" .. cjson.encode(args) .. "'" 72 | os.execute(redisCli() .. " evalsha " .. redisScriptHash .. " 2 args mode " .. args_json .. " record") 73 | end 74 | 75 | function loadSystemConfig() 76 | config_path = "/usr/local/openresty/nginx/Count-von-Count/config/system.config" 77 | SYSTEM_CONFIG = {} 78 | for line in io.lines(config_path) do 79 | for i,j in line:gmatch("(%S+):%s*(%S+)") do 80 | SYSTEM_CONFIG[i] = j 81 | end 82 | end 83 | REDIS_CONFIG = {} 84 | REDIS_CONFIG["host"] = arg[2] or SYSTEM_CONFIG["redis_host"] 85 | REDIS_CONFIG["port"] = arg[3] or SYSTEM_CONFIG["redis_port"] 86 | REDIS_CONFIG["db"] = arg[4] or 0 87 | end 88 | 89 | ------------------------------------ 90 | 91 | loadSystemConfig() 92 | setConfig() 93 | getRedisCountingHash() 94 | 95 | initAdditionalArgsPlugins() 96 | 97 | 98 | for line in io.lines(logFilePath) do 99 | print("Playing line: " .. line) 100 | local status, err = pcall(playLine, line) 101 | if not status then 102 | print("Error!!! " .. err) 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/nginx/get.lua: -------------------------------------------------------------------------------- 1 | local utils = require "utils" 2 | local cjson = require "cjson" 3 | 4 | local args = utils:normalizeKeys( ngx.req.get_query_args() ) 5 | local red = utils:initRedis() 6 | local key = args["key"] 7 | local attributes = args["attr"] 8 | local from = args["from"] or 0 9 | local to = args["to"] or -1 10 | 11 | 12 | function getValues(key, attributes) 13 | local value 14 | if red:type(key) == "hash" then 15 | if attributes then 16 | if type(attributes) == "table" then 17 | values = red:hmget(key, unpack(attributes)) 18 | value = {} 19 | for i = 1, #attributes, 1 do 20 | table.insert(value, attributes[i]) 21 | table.insert(value, values[i]) 22 | end 23 | else 24 | value = red:hget(key, attributes) 25 | end 26 | else 27 | value = red:hgetall(key) 28 | end 29 | else 30 | value = red:zrevrange(key, from, to, "withscores") 31 | end 32 | 33 | if type(value) == "table" then 34 | value = red:array_to_hash(value) 35 | end 36 | return value 37 | end 38 | 39 | local response 40 | if type(key) == "table" then 41 | response = {} 42 | for i, curKey in ipairs(key) do 43 | response[curKey] = getValues(curKey, attributes) 44 | end 45 | else 46 | response = getValues(key, attributes) 47 | end 48 | 49 | if type(response) == "table" then 50 | response = cjson.encode(response) 51 | end 52 | ngx.say(response) 53 | -------------------------------------------------------------------------------- /lib/nginx/init.lua: -------------------------------------------------------------------------------- 1 | request_metadata_parameters_plugins = require "registered_plugins" 2 | for i = 1, #request_metadata_parameters_plugins do 3 | _plugin = require (request_metadata_parameters_plugins[i]) 4 | _plugin:init() 5 | end 6 | 7 | local utils = require "utils" 8 | utils:loadSystemConfig() 9 | -------------------------------------------------------------------------------- /lib/nginx/ping.lua: -------------------------------------------------------------------------------- 1 | local utils = require "utils" 2 | local red = utils:initRedis() 3 | ok, err = red:ping() 4 | if not ok then 5 | ngx.say("Failed to connect to Redis") 6 | else 7 | ngx.say(ok) 8 | end 9 | -------------------------------------------------------------------------------- /lib/nginx/request_metadata_parameters_plugins/country.lua: -------------------------------------------------------------------------------- 1 | local country = {} 2 | 3 | function country:init(from_nginx) 4 | local geoip = require "geoip"; 5 | local geoip_country = require "geoip.country"; 6 | local geoip_file = "/usr/share/GeoIP/GeoIP.dat" 7 | local geoip_country_filename = geoip_file 8 | geodb = geoip_country.open(geoip_country_filename) 9 | end 10 | 11 | function country:AddtoArgsFromNginx(args) 12 | country:fromString(args, ngx.var.remote_addr) 13 | end 14 | 15 | function country:AddToArgsFromLogPlayer(args, line) 16 | ip, request_time, query_args = line:match("^([^%s]+).*%[(.*)].*GET%s*(.*)%s* HTTP") 17 | country:fromString(args,ip) 18 | end 19 | 20 | function country:fromString(args, ip) 21 | local country = geodb:query_by_addr(ip, "id") 22 | args["country"] = geoip.code_by_id(country) or "--" 23 | end 24 | 25 | return country 26 | -------------------------------------------------------------------------------- /lib/nginx/request_metadata_parameters_plugins/date_time.lua: -------------------------------------------------------------------------------- 1 | local date_time = {} 2 | local MON={Jan=1,Feb=2,Mar=3,Apr=4,May=5,Jun=6,Jul=7,Aug=8,Sep=9,Oct=10,Nov=11,Dec=12} 3 | 4 | function date_time:init() 5 | end 6 | 7 | 8 | function date_time:AddtoArgsFromNginx(args) 9 | date_time:fromString(args, ngx.req.start_time()) 10 | end 11 | 12 | function date_time:AddToArgsFromLogPlayer(args, line) 13 | ip, request_time, query_args = line:match("^([^%s]+).*%[(.*)].*GET%s*(.*)%s* HTTP") 14 | date_time:fromString(args, 15 | date_time:parseDateFromString(request_time)) 16 | end 17 | 18 | function date_time:fromString(args, str) 19 | args["day"] = os.date("%d", str) 20 | args["yday"] = os.date("%j", str) 21 | args["week"] = os.date("%W", str) 22 | args["month"] = os.date("%m", str) 23 | args["year"] = os.date("%Y", str) 24 | args["week_year"] = args["week"] .. "_" .. args["year"] 25 | 26 | if args["week"] == "00" then 27 | local last_year = tostring( tonumber(args["year"]) - 1 ) 28 | args["week_year"] = "52_" .. last_year 29 | end 30 | end 31 | 32 | function date_time:parseDateFromString(date) 33 | -- 28/Oct/2013:10:41:15 +0000 34 | print(date) 35 | local p ="(%d+)/(%a+)/(%d+):(%d+):(%d+):(%d+)" 36 | local day,month,year,hour,min,sec = date:match(p) 37 | month=MON[month] 38 | return os.time({day=day,month=month,year=year,hour=hour,min=min,sec=sec}) 39 | end 40 | 41 | return date_time 42 | -------------------------------------------------------------------------------- /lib/nginx/request_metadata_parameters_plugins/registered_plugins.lua: -------------------------------------------------------------------------------- 1 | local registered_plugins = 2 | { 3 | "date_time" 4 | --"country" 5 | } 6 | return registered_plugins 7 | -------------------------------------------------------------------------------- /lib/nginx/utils.lua: -------------------------------------------------------------------------------- 1 | local redis = require "resty.redis" 2 | local utils = {} 3 | 4 | function utils:normalizeKeys(tbl) 5 | local normalized = {} 6 | for k, v in pairs(tbl) do 7 | local key = k:gsub("amp;", "") 8 | local value = v 9 | if key:match("[[]]") then 10 | key = key:gsub("[[]]", "") 11 | if type(value) ~= "table" then 12 | value = { v } 13 | end 14 | end 15 | normalized[key] = value 16 | end 17 | return normalized 18 | end 19 | 20 | function utils:initRedis() 21 | local red = redis:new() 22 | red:set_timeout(3000) -- 3 sec 23 | local ok, err = red:connect(SYSTEM_CONFIG["redis_host"], SYSTEM_CONFIG["redis_port"]) 24 | if not ok then utils:logErrorAndExit("Error connecting to redis: ".. err) end 25 | return red 26 | end 27 | 28 | function utils:loadSystemConfig() 29 | config_path = "/usr/local/openresty/nginx/count-von-count/config/system.config" 30 | SYSTEM_CONFIG = {} 31 | for line in io.lines(config_path) do 32 | for i,j in line:gmatch("(%S+):%s*(%S+)") do 33 | SYSTEM_CONFIG[i] = j 34 | end 35 | end 36 | end 37 | 38 | function utils:logErrorAndExit(err) 39 | ngx.log(ngx.ERR, err) 40 | utils:emptyGif() 41 | end 42 | 43 | function utils:emptyGif() 44 | ngx.exec('/_.gif') 45 | end 46 | 47 | return utils 48 | -------------------------------------------------------------------------------- /lib/nginx/voncount.lua: -------------------------------------------------------------------------------- 1 | local utils = require "utils" 2 | 3 | ngx.header["Cache-Control"] = "no-cache" 4 | local args = ngx.req.get_query_args() 5 | args = utils:normalizeKeys(args) 6 | args["action"] = ngx.var.action 7 | 8 | for i = 1, #request_metadata_parameters_plugins do 9 | _plugin = require (request_metadata_parameters_plugins[i]) 10 | _plugin:AddtoArgsFromNginx(args) 11 | end 12 | 13 | local cjson = require "cjson" 14 | local args_json = cjson.encode(args) 15 | local red = utils:initRedis() 16 | 17 | ok, err = red:evalsha(ngx.var.redis_counter_hash, 1, "args", args_json) 18 | if not ok then utils:logErrorAndExit("Error evaluating redis script: ".. err) end 19 | 20 | ok, err = red:set_keepalive(10000, 1000) 21 | if not ok then utils:logErrorAndExit("Error setting redis keep alive ".. err) end 22 | utils:emptyGif() 23 | -------------------------------------------------------------------------------- /lib/redis/voncount.lua: -------------------------------------------------------------------------------- 1 | -------------- Function to simulate inheritance ------------------------------------- 2 | -- local function inheritsFrom( baseClass ) 3 | -- local new_class = {} 4 | -- local class_mt = { __index = new_class } 5 | 6 | -- if baseClass then 7 | -- setmetatable( new_class, { __index = baseClass } ) 8 | -- end 9 | 10 | -- return new_class 11 | -- end 12 | 13 | ---------- Array methods --------------------------- 14 | 15 | local function concatToArray(a1, a2) 16 | for i = 1, #a2 do 17 | a1[#a1 + 1] = a2[i] 18 | end 19 | return a1 20 | end 21 | 22 | local function flattenArray(arr) 23 | local flat = {} 24 | for i = 1, #arr do 25 | if type(arr[i]) == "table" then 26 | local inner_flatten = flattenArray(arr[i]) 27 | concatToArray(flat, inner_flatten) 28 | else 29 | flat[#flat + 1] = arr[i] 30 | end 31 | end 32 | return flat 33 | end 34 | 35 | local function dupArray(arr) 36 | local dup = {} 37 | for i = 1, #arr do 38 | dup[i] = arr[i] 39 | end 40 | return dup 41 | end 42 | 43 | ----------------------------------------------------- 44 | 45 | -------------- Base Class --------------------------------------------------------- 46 | 47 | local Base = {} 48 | 49 | function Base:new(_obj_type, ids, _type) 50 | local redis_key = _obj_type 51 | for k, id in ipairs(ids) do 52 | redis_key = redis_key .. "_" .. id 53 | end 54 | local baseObj = { redis_key = redis_key, _type = _type, _ids = ids, _obj_type = _obj_type } 55 | self.__index = self 56 | return setmetatable(baseObj, self) 57 | end 58 | 59 | 60 | function Base:count(key, num) 61 | local allKeys = flattenArray({ key }) 62 | for i, curKey in ipairs(allKeys) do 63 | if self._type == "set" then 64 | redis.call("ZINCRBY", self.redis_key, num, curKey) 65 | else 66 | redis.call("HINCRBY", self.redis_key, curKey, num) 67 | end 68 | end 69 | end 70 | 71 | function Base:expire(ttl) 72 | if redis.call("TTL", self.redis_key) == -1 then 73 | redis.call("EXPIRE", self.redis_key, ttl) 74 | end 75 | end 76 | ----------------- Custom Methods ------------------------- 77 | 78 | function Base:conditionalCount(should_count, key) 79 | if should_count ~= "0" and should_count ~= "false" then 80 | self:count(key, 1) 81 | end 82 | end 83 | 84 | function Base:countIfExist(value, should_count, key) 85 | if value and value ~= "" and value ~= "null" and value ~= "nil" then 86 | self:conditionalCount(should_count, key) 87 | end 88 | end 89 | 90 | function Base:sevenDaysCount(should_count, key) 91 | if should_count ~= "0" and should_count ~= "false" then 92 | local first_day = tonumber(self._ids[3]) 93 | for day = 0, 6, 1 do 94 | local curDayObjIds = dupArray(self._ids) 95 | if (first_day + day) > 365 then 96 | curDayObjIds[4] = string.format("%03d", (tonumber(curDayObjIds[4]) + 1) ) 97 | end 98 | local curDayObj = Base:new(self._obj_type, curDayObjIds, self._type) 99 | curDayObj:count(key, 1) 100 | curDayObj:expire(1209600) -- expire in 2 weeks 101 | end 102 | end 103 | end 104 | 105 | function Base:countAndSetIf(should_count, countKey, redisKey, setKey) 106 | if should_count ~= "0" and should_count ~= "false" then 107 | self:count(countKey, 1) 108 | local setCount = redis.call("ZCOUNT", self.redis_key, "-inf", "+inf") 109 | redis.call("HSET", redisKey, setKey, setCount) 110 | end 111 | end 112 | ---------------------------------------------------------- 113 | 114 | 115 | ------------- Helper Methods ------------------------ 116 | 117 | -- return an array with all the values in tbl that match the given keys array 118 | local function getValueByKeys(tbl, keys) 119 | local values = {} 120 | if type(keys) == "table" then 121 | for i, key in ipairs(keys) do 122 | table.insert(values, tbl[key]) 123 | end 124 | else 125 | table.insert(values, tbl[keys]) 126 | end 127 | return values 128 | end 129 | 130 | 131 | -- parse key and replace "place holders" with their value from tbl. 132 | -- matching replace values in tbl can be arrays, in such case an array will be returned with all the possible keys combinations 133 | local function addValuesToKey(tbl, key) 134 | local rslt = { key } 135 | local match = key:match("{[%w_]*}") 136 | 137 | while match do 138 | local subStrings = flattenArray({ tbl[match:sub(2, -2)] }) 139 | local tempResult = {} 140 | for i, subStr in ipairs(subStrings) do 141 | local dup = dupArray(rslt) 142 | for j, existingKey in ipairs(dup) do 143 | local curKey = existingKey:gsub(match, subStr) 144 | dup[j] = curKey 145 | end 146 | concatToArray(tempResult, dup) 147 | end 148 | rslt = tempResult 149 | if #rslt > 0 then 150 | match = rslt[1]:match("{[%w_]*}") 151 | else 152 | match = nil 153 | end 154 | end 155 | 156 | if #rslt == 1 then 157 | return rslt[1] 158 | else 159 | return rslt 160 | end 161 | end 162 | 163 | 164 | -------------------------------------------------- 165 | local mode = ARGV[2] or "live" 166 | local arg = ARGV[1] 167 | local params = cjson.decode(arg) 168 | local config = cjson.decode(redis.call("get", "von_count_config_".. mode)) 169 | local action = params["action"] 170 | local defaultMethod = { change = 1, custom_functions = {} } 171 | local action_config = config[action] 172 | 173 | 174 | if action_config then 175 | for obj_type, methods in pairs(action_config) do 176 | for i, defs in ipairs(methods) do 177 | setmetatable(defs, { __index = defaultMethod }) 178 | 179 | local ids = getValueByKeys(params, defs["id"]) 180 | local _type = defs["type"] or "hash" 181 | 182 | local obj = Base:new(obj_type, ids, _type) 183 | 184 | if defs["count"] then 185 | local key = addValuesToKey(params, defs["count"]) 186 | local change = defs["change"] 187 | obj:count(key, change) 188 | end 189 | 190 | for j, custom_function in ipairs(defs["custom_functions"]) do 191 | local function_name = custom_function["name"] 192 | local args = {} 193 | for z, arg in ipairs(custom_function["args"]) do 194 | local arg_value = addValuesToKey(params, arg) 195 | table.insert(args, arg_value) 196 | end 197 | obj[function_name](obj, unpack(args)) 198 | end 199 | 200 | if defs["expire"] then 201 | obj:expire(defs["expire"]) 202 | end 203 | end 204 | end 205 | end 206 | 207 | -------------------------------------------------------------------------------- /lib/scripts/backup-upload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BACKUP_FOLDER=$1 3 | BUCKET_NAME=$2 4 | FOLDER_TO_SYNC=$(date +%Y/%m/%d -d "yesterday") 5 | DIRECTORY=$BACKUP_FOLDER/$FOLDER_TO_SYNC 6 | if [ -d "$DIRECTORY" ]; then 7 | aws s3 cp $DIRECTORY s3://$BUCKET_NAME/$FOLDER_TO_SYNC --recursive 8 | else 9 | echo "$DIRECTORY doesn't exist" 10 | fi 11 | ACCESS_LOGS_FOLDER_TO_SYNC=$BACKUP_FOLDER/access_logs/$FOLDER_TO_SYNC 12 | if [ -d "$ACCESS_LOGS_FOLDER_TO_SYNC" ]; then 13 | aws s3 cp $ACCESS_LOGS_FOLDER_TO_SYNC s3://$BUCKET_NAME/access_logs/$FOLDER_TO_SYNC --recursive 14 | else 15 | echo "$ACCESS_LOGS_FOLDER_TO_SYNC doesn't exist" 16 | fi 17 | -------------------------------------------------------------------------------- /lib/scripts/backup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BACKUP_FOLDER=$1 3 | RDB_FILEPATH=$2 4 | NGINX_LOGS_DIR=$3 5 | TIMESTAMP=$(redis-cli lastsave) 6 | 7 | copyLogsFile() 8 | { 9 | for file in $NGINX_LOGS_DIR/access.log*gz 10 | do 11 | FILENAME=$(basename $file) 12 | FILENAME="${FILENAME%.*}" 13 | REGEX='[0-9]{10}' 14 | if [[ $FILENAME =~ $REGEX ]] 15 | then 16 | DAY_FOLDER=$(date --date @${BASH_REMATCH[0]} +%Y/%m/%d) 17 | DEST_FOLDER=$BACKUP_FOLDER//access_logs/$DAY_FOLDER 18 | mkdir -p $DEST_FOLDER 19 | EXT=$(date --date @${BASH_REMATCH[0]} +%H-%M-%S) 20 | cp -n $file $DEST_FOLDER/$FILENAME-$EXT.gz 21 | fi 22 | done 23 | } 24 | 25 | copyRDBFile() 26 | { 27 | TIMESTAMP=$(redis-cli lastsave) 28 | DAY_FOLDER=$(date --date @$TIMESTAMP +%Y/%m/%d) 29 | mkdir -p $BACKUP_FOLDER/$DAY_FOLDER 30 | FILE_NAME=$(date --date @$TIMESTAMP +%H-%M-%S) 31 | DEST_RDB_FILE_PATH=$BACKUP_FOLDER/$DAY_FOLDER/$FILE_NAME.rdb 32 | cp $RDB_FILEPATH $DEST_RDB_FILE_PATH 33 | gzip $DEST_RDB_FILE_PATH 34 | } 35 | 36 | monthlyBackup() 37 | { 38 | MONTH=$(date +%Y/%m) 39 | MONTH_FOLDER=$BACKUP_FOLDER/$MONTH/monthly_backup 40 | mkdir -p $MONTH_FOLDER 41 | rm -rfv $MONTH_FOLDER/* 42 | cp $DEST_RDB_FILE_PATH.gz $MONTH_FOLDER/$(basename $DEST_RDB_FILE_PATH).gz 43 | } 44 | 45 | weeklyBackup() 46 | { 47 | MONTH=$(date +%Y/%m) 48 | WEEK_BACKUPS_FOLDER=$BACKUP_FOLDER/$MONTH/weekly_backup 49 | mkdir -p $WEEK_BACKUPS_FOLDER 50 | CURRENT_WEEK_BACKUPS_FOLDER=$WEEK_BACKUPS_FOLDER/$(date +%d) 51 | mkdir -p $CURRENT_WEEK_BACKUPS_FOLDER 52 | rm -rfv $CURRENT_WEEK_BACKUPS_FOLDER/* 53 | cp $DEST_RDB_FILE_PATH.gz $CURRENT_WEEK_BACKUPS_FOLDER/$(basename $DEST_RDB_FILE_PATH).gz 54 | } 55 | 56 | copyLogsFile 57 | copyRDBFile 58 | if [ $(date +%d) == "01" ] 59 | then 60 | monthlyBackup 61 | fi 62 | 63 | #Sunday's 64 | if [ $(date +%u) == "7" ] 65 | then 66 | weeklyBackup 67 | fi 68 | -------------------------------------------------------------------------------- /lib/scripts/monitors/s3-monitor.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | BUCKET_NAME=$1 3 | FOLDER_TO_CHECK=$(date +%Y/%m/%d -d "yesterday") 4 | RESPONSE=$(aws s3 ls s3://$BUCKET_NAME/$FOLDER_TO_CHECK) 5 | 6 | if echo "$RESPONSE" | grep ".gz"; then 7 | echo "matched" 8 | else 9 | echo "No S3 backups for $FOLDER_TO_CHECK." | mail -s "Action Counter Monitor" ron@ftbpro.com, dor@ftbpro.com, shai@ftbpro.com 10 | fi 11 | -------------------------------------------------------------------------------- /lib/scripts/reload.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DEPLOY_TO="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../.." 3 | NGINX_DIR="/usr/local/openresty/nginx" 4 | 5 | rm -f $NGINX_DIR/conf/include/vars.conf 6 | echo 'set $redis_counter_hash '$(redis-cli SCRIPT LOAD "$(cat $DEPLOY_TO'/lib/redis/voncount.lua')")';' > $NGINX_DIR/conf/vars.conf 7 | redis-cli set von_count_config_live $(cat $DEPLOY_TO'/config/voncount.config' | tr -d '\n' | tr -d ' ') 8 | $NGINX_DIR/sbin/nginx -s reload 9 | -------------------------------------------------------------------------------- /lib/scripts/setup.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | DEPLOY_TO="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )/../.." 3 | NGINX_DIR="/usr/local/openresty/nginx" 4 | USER=$(whoami) 5 | 6 | chown -R $USER:$USER $NGINX_DIR 7 | ln -sf $DEPLOY_TO/ $NGINX_DIR/count-von-count 8 | mkdir -p $NGINX_DIR/conf/include 9 | ln -sf $DEPLOY_TO/config/voncount.nginx.conf $NGINX_DIR/conf/include/voncount.conf 10 | service redis-server start 11 | $DEPLOY_TO/lib/scripts/reload.sh 12 | $NGINX_DIR/sbin/nginx 13 | 14 | if ps aux | grep nginx | grep master > /dev/null ; then 15 | echo ">>> nginx is running" 16 | else 17 | echo "ERROR: nginx is not running" 18 | fi 19 | 20 | if ps aux | grep redis-server | grep -v 'grep' > /dev/null ; then 21 | echo ">>> redis-server is running" 22 | else 23 | echo "ERROR: redis-server is not running" 24 | fi 25 | -------------------------------------------------------------------------------- /lib/scripts/update-geoip.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | wget -N -q http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz 3 | gunzip -f GeoIP.dat.gz 4 | mv GeoIP.dat /usr/share/GeoIP/GeoIP.dat 5 | nginx -s reload 6 | -------------------------------------------------------------------------------- /spec/config/spec_config.yml: -------------------------------------------------------------------------------- 1 | redis_host: "127.0.0.1" 2 | redis_port: 6379 3 | redis_db: 0 4 | log_player_integration: true 5 | log_player_redis_db: 1 6 | -------------------------------------------------------------------------------- /spec/config/voncount.config: -------------------------------------------------------------------------------- 1 | { 2 | "reads": { 3 | 4 | "User": [ 5 | { 6 | "id": "user", 7 | "count": "reads" 8 | }, 9 | { 10 | "id": "author", 11 | "count": "reads_got" 12 | } 13 | ], 14 | 15 | "Post": [ 16 | { 17 | "id": "post", 18 | "count": "reads" 19 | } 20 | ], 21 | 22 | "UserDaily": [ 23 | { 24 | "id": [ 25 | "user", 26 | "day", 27 | "month", 28 | "year" 29 | ], 30 | "count": "reads", 31 | "expire": 1209600 32 | } 33 | ], 34 | 35 | "UserWeeklyDemographics": [ 36 | { 37 | "id": [ 38 | "author", 39 | "week", 40 | "year" 41 | ], 42 | "count": "{country}" 43 | } 44 | ] 45 | }, 46 | 47 | "comments": { 48 | "User": [ 49 | { 50 | "id": "user", 51 | "count": "comments" 52 | }, 53 | { 54 | "id": "author", 55 | "count": "comments_got" 56 | } 57 | ], 58 | 59 | "Post": [ 60 | { 61 | "id": "post", 62 | "count": "comments" 63 | } 64 | ] 65 | }, 66 | 67 | 68 | "post_create": { 69 | "User": [ 70 | { 71 | "id": "user", 72 | "count": "post_create" 73 | } 74 | ] 75 | }, 76 | 77 | "post_remove": { 78 | "User": [ 79 | { 80 | "id": "user", 81 | "count": "post_create", 82 | "change": -1 83 | } 84 | ] 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /spec/integration/log_player_integrator.rb: -------------------------------------------------------------------------------- 1 | require 'ruby-debug' 2 | require 'spec_helper' 3 | 4 | def self.spec_config 5 | @@spec_config ||= YAML.load_file('spec/config/spec_config.yml') rescue {} 6 | end 7 | 8 | Spec::Runner.configure do |config| 9 | if self.spec_config["log_player_integration"] 10 | config.before(:all) do 11 | flush_keys 12 | ScriptLoader.clean_access_log 13 | ScriptLoader.restart_nginx 14 | end 15 | config.after(:all) do 16 | compare_log_player_values_to_real_time_values 17 | end 18 | end 19 | 20 | def compare_log_player_values_to_real_time_values 21 | run_log_player 22 | $redis.keys.each do |key| 23 | if !unrelevant_keys.include?(key) 24 | if !compare_value(key) 25 | raise RSpec::Expectations::ExpectationNotMetError, "Log Player Intregration: difference in #{key}" 26 | end 27 | end 28 | end 29 | end 30 | 31 | def flush_keys 32 | cache_keys = $redis.keys "*" 33 | cache_keys = cache_keys.reject { |key| key.match(/^von_count_config/)} 34 | $redis.del(cache_keys) if cache_keys.any? 35 | cache_keys = $log_player_redis.keys "*" 36 | cache_keys = cache_keys.reject { |key| key.match(/^von_count_config/)} 37 | $log_player_redis.del(cache_keys.reject { |key| key.match(/^von_count_config/)}) if cache_keys.any? 38 | end 39 | 40 | def unrelevant_keys 41 | ["von_count_config_live"] 42 | end 43 | 44 | def compare_value(key) 45 | if $redis.type(key) == "hash" 46 | return $redis.hgetall(key) == $log_player_redis.hgetall(key) 47 | elsif $redis.type(key) == "zset" 48 | return $redis.zrevrange(key, 0, -1, withscores:true) == $log_player_redis.zrevrange(key, 0, -1, withscores:true) 49 | elsif $redis.type(key) == "string" 50 | return $redis.get(key) == $log_player_redis.get(key) 51 | end 52 | false 53 | end 54 | 55 | def run_log_player 56 | `lua \ 57 | /usr/local/openresty/nginx/count-von-count/lib/log_player.lua \ 58 | /usr/local/openresty/nginx/logs/access.log \ 59 | #{spec_config["redis_host"]} \ 60 | #{spec_config["redis_port"]} \ 61 | #{spec_config["log_player_redis_db"]} \ 62 | ` 63 | end 64 | 65 | 66 | end 67 | -------------------------------------------------------------------------------- /spec/lib/counting_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'ruby-debug' 3 | describe "Counting" do 4 | before :each do 5 | @user = create :User, id: 10 6 | @author = create :User, id: 30 7 | @post = create :Post, id: 100 8 | end 9 | 10 | describe "reads action" do 11 | 12 | describe "User Hash" do 13 | before do 14 | open("http://#{HOST}/reads?post=#{@post.id}&user=#{@user.id}&author=#{@author.id}") 15 | end 16 | it "should increase the reads of the user by one" do 17 | @user.should_plus_1("reads") 18 | end 19 | 20 | describe "Author" do 21 | it "should increase the post's author num of time he's been read" do 22 | @author.should_plus_1("reads_got") 23 | end 24 | end 25 | end 26 | 27 | describe "Post Hash" do 28 | before do 29 | open("http://#{HOST}/reads?post=#{@post.id}") 30 | end 31 | it "should increase the post reads counter" do 32 | @post.should_plus_1("reads") 33 | end 34 | end 35 | 36 | describe "UserDaily Hash" do 37 | before do 38 | @user_daily = create :UserDaily, { user: @user.id, day: Time.now.strftime("%d"), month: Time.now.strftime("%m"), year: Time.now.strftime("%Y") } 39 | open("http://#{HOST}/reads?user=#{@user.id}") 40 | end 41 | it "should increase the daily reads of the user by one" do 42 | @user_daily.should_plus_1("reads") 43 | end 44 | 45 | it "should set a TTL for the objects" do 46 | $redis.ttl(@user_daily.key).should > 0 47 | end 48 | end 49 | 50 | # describe "AuthorWeeklyDemographics Hash" do 51 | # before do 52 | # @author_weekly_demographics = create :UserWeeklyDemographics, {user: @author.id, week: Time.now.strftime("%W"), year: Time.now.strftime("%Y")} 53 | # open("http://#{HOST}/reads?author=#{@author.id}") 54 | # end 55 | # it "should increase reads counter" do 56 | # @author_weekly_demographics.should_plus_1("--") 57 | # end 58 | # end 59 | end 60 | 61 | describe "comments action" do 62 | before :each do 63 | open("http://#{HOST}/comments?user=#{@user.id}&author=#{@author.id}") 64 | end 65 | 66 | describe "User Hash" do 67 | it "should increase the user's num of comments by 1" do 68 | @user.should_plus_1("comments") 69 | end 70 | 71 | describe "Author" do 72 | it "should increase the post's author num of comments he got" do 73 | @author.should_plus_1("comments_got") 74 | end 75 | end 76 | end 77 | 78 | describe "Post Hash" do 79 | before :each do 80 | open("http://#{HOST}/comments?post=#{@post.id}") 81 | end 82 | it "should increase the post's num of comments" do 83 | @post.should_plus_1("comments") 84 | end 85 | end 86 | end 87 | 88 | describe "post_create action" do 89 | # before do 90 | # open("http://#{HOST}/post_create?user=#{@user.id}") 91 | # end 92 | # it "should increase the user's num of posts created by 1" do 93 | # @user.should_plus_1("post_create") 94 | # end 95 | 96 | # describe "TeamCounters" do 97 | # #TODO: Add spec 98 | # end 99 | 100 | # describe "TeamWriters" do 101 | # #TODO: Add spec 102 | # end 103 | end 104 | 105 | describe "post_remove action" do 106 | before do 107 | open("http://#{HOST}/post_remove?user=#{@user.id}") 108 | end 109 | it "should decrease the user's number of post_create" do 110 | @user.should_minus_1("post_create") 111 | end 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /spec/lib/get_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | describe "Get" do 3 | before do 4 | @user = create :User 5 | @author= create :User 6 | open("http://#{HOST}/reads?user=#{@user.id}&author=#{@author.id}") 7 | end 8 | 9 | def get(query_string) 10 | JSON.parse(open("http://#{HOST}/get?#{query_string}").read.gsub("\n", "") ) 11 | end 12 | 13 | describe "Header" do 14 | it "should have a 'Access-Control-Allow-Origin *' in the response header" do 15 | response = open("http://#{HOST}/get?key=User_#{@user.id}") 16 | response.meta.keys.should include "access-control-allow-origin" 17 | response.meta["access-control-allow-origin"].should == "*" 18 | end 19 | end 20 | 21 | describe "Hash key" do 22 | describe "single key" do 23 | it "should return a json with all the attributes of the key when attr params are not defined" do 24 | hash = get("key=User_#{@user.id}") 25 | hash.keys.sort.should match_array(["reads"]) 26 | end 27 | 28 | it "should return a json containing only the values defined by the attr array parameter in the query string" do 29 | hash = get("key=User_#{@user.id}&attr[]=reads&attr[]=logins") 30 | hash.keys.sort.should match_array(["logins", "reads"]) 31 | end 32 | 33 | context "when attribute doesnt exist" do 34 | it "should return nil" do 35 | hash = get("key=User_#{@user.id}&attr[]=reads&attr[]=logins") 36 | hash["reads"].to_i.should eq 1 37 | hash.keys.should match_array(["logins", "reads"]) 38 | hash["logins"].should eq nil 39 | end 40 | end 41 | 42 | context "when only attribute defined" do 43 | it "should return only the value of the desired attribute" do 44 | rslt = open("http://#{HOST}/get?key=User_#{@user.id}&attr=reads").read.gsub("\n", "") 45 | rslt.should == "1" 46 | end 47 | end 48 | 49 | context "as array" do 50 | it "should return a json with a single key equal to the requested key and its value as hash" do 51 | hash = get("key[]=User_#{@user.id}") 52 | hash.keys.should match_array([@user.key]) 53 | hash[@user.key].keys.should match_array(["reads"]) 54 | end 55 | end 56 | end 57 | end 58 | 59 | describe "multiple keys" do 60 | describe "without attributes" do 61 | it "should return a json with the given keys as keys and for each key a hash with all of its attributes " do 62 | hash = get("key[]=#{@user.key}&key[]=#{@author.key}") 63 | hash.keys.should match_array([@user.key, @author.key]) 64 | hash[@user.key].keys.should match_array(["reads"]) 65 | hash[@author.key].keys.should match_array(["reads_got"]) 66 | end 67 | end 68 | 69 | describe "with attributes" do 70 | it "should return a json with the given keys as keys and for each key a hash with all the given attributes" do 71 | hash = get("key[]=#{@user.key}&key[]=#{@author.key}&attr[]=reads&attr[]=logins") 72 | hash.keys.should match_array([@user.key, @author.key]) 73 | hash[@user.key].keys.should match_array(["logins", "reads"]) 74 | hash[@author.key].keys.should match_array(["logins", "reads"]) 75 | end 76 | 77 | context "multiple attributes" do 78 | before do 79 | open("http://#{HOST}/reads?user=#{@author.id}") 80 | end 81 | it "should return a json with the given keys as keys and for each key a value for the given attribute (single attribute)" do 82 | hash = get("key[]=#{@user.key}&key[]=#{@author.key}&attr=reads") 83 | hash.keys.should match_array([@user.key, @author.key]) 84 | hash[@user.key].should eq "1" 85 | hash[@author.key].should eq "1" 86 | end 87 | end 88 | 89 | it "should return a json with the given keys as keys and for each key a hash with the given attribute (single attribute as array)" do 90 | hash = get("key[]=#{@user.key}&key[]=#{@author.key}&attr[]=reads") 91 | hash.keys.should match_array([@user.key, @author.key]) 92 | hash[@user.key].keys.should match_array(["reads"]) 93 | hash[@author.key].keys.should match_array(["reads"]) 94 | end 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/lib/utils_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | describe "Utils" do 3 | describe "Android bad Query String with amp;" do 4 | 5 | before :each do 6 | @user = create :User 7 | @author = create :User 8 | open("http://#{HOST}/reads?user=#{@user.id}&author=#{@author.id}") 9 | end 10 | 11 | context "it should replace & with &" do 12 | it "should have a correct redis key for user" do 13 | $redis.keys("*").include?(@user.key).should be_true 14 | end 15 | 16 | 17 | it "should have a correct redis key for other user" do 18 | $redis.keys("*").include?(@author.key).should be_true 19 | end 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /spec/script_loader.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'json' 3 | require 'ruby-debug' 4 | class ScriptLoader 5 | class << self 6 | attr_accessor :log_player_reads_hash 7 | end 8 | def self.load 9 | set_config 10 | load_scripts_to_log_player_test_db if self.spec_config["log_player_integration"] 11 | File.open("/usr/local/openresty/nginx/conf/vars.conf", 'w') { |f| f.write(<<-VARS 12 | set $redis_counter_hash #{von_count_script_hash}; 13 | VARS 14 | ) } 15 | restart_nginx 16 | end 17 | 18 | def self.von_count_script_hash 19 | @von_count_script_hash ||= `redis-cli SCRIPT LOAD "$(cat "lib/redis/voncount.lua")"`.strip 20 | end 21 | 22 | 23 | def self.load_scripts_to_log_player_test_db 24 | @log_player_reads_hash ||= `redis-cli -n #{self.spec_config["log_player_redis_db"]} SCRIPT LOAD "$(cat "lib/redis/voncount.lua")"`.strip 25 | end 26 | 27 | def self.set_config 28 | redis = Redis.new(host: HOST, port: "6379") 29 | config = `cat spec/config/voncount.config | tr -d '\n' | tr -d ' '` 30 | redis.set("von_count_config_live", config) 31 | end 32 | 33 | def self.restart_nginx 34 | `echo "#{personal_settings['sudo_password']}" | sudo -S nginx -s reload` 35 | sleep 1 36 | end 37 | 38 | def self.clean_access_log 39 | `rm -f /usr/local/openresty/nginx/logs/access.log` 40 | end 41 | 42 | def self.personal_settings 43 | @@settings ||= YAML.load_file("config/personal.yml") rescue {} 44 | end 45 | 46 | def self.spec_config 47 | @@spec_config ||= YAML.load_file('spec/config/spec_config.yml') rescue {} 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'open-uri' 2 | require 'script_loader' 3 | require 'rubygems' 4 | require 'spork' 5 | require 'redis' 6 | require 'json' 7 | require 'support/redis_object_factory' 8 | require "integration/log_player_integrator" 9 | #uncomment the following line to use spork with the debugger 10 | #require 'spork/ext/ruby-debug' 11 | 12 | HOST = "127.0.0.1" 13 | 14 | Spork.prefork do 15 | # Loading more in this block will cause your tests to run faster. However, 16 | # if you change any configuration or code from libraries loaded here, you'll 17 | # need to restart spork for it take effect. 18 | 19 | end 20 | 21 | Spork.each_run do 22 | # This code will be run each time you run your specs. 23 | end 24 | 25 | # --- Instructions --- 26 | # Sort the contents of this file into a Spork.prefork and a Spork.each_run 27 | # block. 28 | # 29 | # The Spork.prefork block is run only once when the spork server is started. 30 | # You typically want to place most of your (slow) initializer code in here, in 31 | # particular, require'ing any 3rd-party gems that you don't normally modify 32 | # during development. 33 | # 34 | # The Spork.each_run block is run each time you run your specs. In case you 35 | # need to load files that tend to change during development, require them here. 36 | # With Rails, your application modules are loaded automatically, so sometimes 37 | # this block can remain empty. 38 | # 39 | # Note: You can modify files loaded *from* the Spork.each_run block without 40 | # restarting the spork server. However, this file itself will not be reloaded, 41 | # so if you change any of the code inside the each_run block, you still need to 42 | # restart the server. In general, if you have non-trivial code in this file, 43 | # it's advisable to move it into a separate file so you can easily edit it 44 | # without restarting spork. (For example, with RSpec, you could move 45 | # non-trivial code into a file spec/support/my_helper.rb, making sure that the 46 | # spec/support/* files are require'd from inside the each_run block.) 47 | # 48 | # Any code that is left outside the two blocks will be run during preforking 49 | # *and* during each_run -- that's probably not what you want. 50 | # 51 | # These instructions should self-destruct in 10 seconds. If they don't, feel 52 | # free to delete them. 53 | 54 | def spec_config 55 | @spec_config ||= YAML.load_file('spec/config/spec_config.yml') rescue {} 56 | end 57 | 58 | spec_config["redis_port"] 59 | lib = File.expand_path('../../lib', __FILE__) 60 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 61 | 62 | $redis = Redis.new(host: spec_config["redis_host"], port: spec_config["redis_port"], db: spec_config["redis_db"]) 63 | $log_player_redis = Redis.new(host: HOST, port: spec_config["redis_port"] , db: spec_config["log_player_redis_db"]) 64 | ScriptLoader.load 65 | RedisObjectFactory.redis = $redis 66 | 67 | def create(type, ids = nil) 68 | ids ||= { id: rand(1000000) } 69 | RedisObjectFactory.new(type, ids) 70 | end 71 | 72 | -------------------------------------------------------------------------------- /spec/support/redis_object_factory.rb: -------------------------------------------------------------------------------- 1 | class RedisObjectFactory 2 | attr_reader :type, :key, :ids, :id 3 | 4 | class << self 5 | attr_accessor :redis 6 | end 7 | 8 | def initialize(type_, ids_) 9 | @type = type_ 10 | @ids = ids_ 11 | @id = @ids.values.join("_") 12 | @key = "#{@type}_#{@id}" 13 | key_type = self.class.redis.type(@key) 14 | initial_data if key_type == "hash" || key_type == "none" 15 | initial_set if key_type == "zset" || key_type == "none" 16 | end 17 | 18 | def data 19 | data = Hash.new(0) 20 | data.merge(self.class.redis.hgetall @key) 21 | end 22 | 23 | def initial_data 24 | @initial_data ||= data 25 | end 26 | 27 | def initial_set 28 | @initial_set ||= set 29 | end 30 | 31 | def set 32 | set = Hash.new(0) 33 | set.merge(Hash[*$redis.zrange(key, 0, -1, withscores: true).flatten]) 34 | end 35 | 36 | def should_plus_1(key) 37 | data[key].to_i.should == initial_data[key].to_i + 1 38 | end 39 | 40 | def should_minus_1(key) 41 | data[key].to_i.should == initial_data[key].to_i - 1 42 | end 43 | end 44 | --------------------------------------------------------------------------------