├── .github └── workflows │ └── tests.yml ├── .gitignore ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── lib ├── redlics.rb └── redlics │ ├── config.rb │ ├── connection.rb │ ├── counter.rb │ ├── exception.rb │ ├── granularity.rb │ ├── key.rb │ ├── lua │ └── script.lua │ ├── operators.rb │ ├── query.rb │ ├── query │ └── operation.rb │ ├── time_frame.rb │ ├── tracker.rb │ └── version.rb ├── redlics.gemspec └── test ├── redlics_test.rb └── test_helper.rb /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test: 7 | runs-on: ubuntu-latest 8 | 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | ruby-version: 13 | - head 14 | - '3.2' 15 | - '3.1' 16 | - '3.0' 17 | - '2.7' 18 | 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up Ruby ${{ matrix.ruby-version }} 22 | uses: ruby/setup-ruby@v1 23 | with: 24 | ruby-version: ${{ matrix.ruby-version }} 25 | bundler-cache: true # 'bundle install' and cache 26 | - name: Run tests 27 | run: bundle exec rake 28 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.lock 2 | *.gem 3 | *.swp 4 | .project 5 | .ruby-version 6 | .ruby-gemset 7 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source 'https://rubygems.org' 4 | 5 | # Specify your gem's dependencies in redlics.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 Phlegx Systems OG 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Redlics 2 | 3 | [![Gem Version](https://badge.fury.io/rb/redlics.svg)](https://rubygems.org/gems/redlics) 4 | [![Gem](https://img.shields.io/gem/dt/redlics.svg?maxAge=2592000)](https://rubygems.org/gems/redlics) 5 | [![Code Climate](https://codeclimate.com/github/phlegx/redlics.svg)](https://codeclimate.com/github/phlegx/redlics) 6 | [![Inline Docs](https://inch-ci.org/github/phlegx/redlics.svg?branch=master)](https://inch-ci.org/github/phlegx/redlics) 7 | [![License](https://img.shields.io/github/license/phlegx/redlics.svg)](https://opensource.org/licenses/MIT) 8 | 9 | Record millions of tracks and counts consuming low memory! Redlics is a gem for Redis analytics with tracks (using bitmaps) and counts (using buckets) encoding numbers in Redis keys and values. 10 | 11 | ## Features 12 | 13 | * Tracking with bitmaps 14 | * Counting with buckets 15 | * High configurable 16 | * Encode/decode numbers in Redis keys and values 17 | * Very less memory consumption in Redis 18 | * Support of time frames 19 | * Uses Lua script for better performance 20 | * Plot option for tracks and counts 21 | * Keeps Redis clean 22 | * and many more, see the [documentation](http://www.rubydoc.info/gems/redlics) 23 | 24 | ## Installation 25 | 26 | **System Requirements:** Redis >= v3.x is recommended! 27 | 28 | Add this line to your application's Gemfile: 29 | 30 | ```ruby 31 | gem 'redlics' 32 | ``` 33 | 34 | And then execute: 35 | 36 | $ bundle 37 | 38 | Or install it yourself as: 39 | 40 | $ gem install redlics 41 | 42 | ## Usage 43 | 44 | ### Configuration 45 | 46 | The following configuration is the default configuration of Redlics. Store the configration code and load it at the beginning of Redlics use. 47 | Rails users can create a file `redlics.rb` in `config/initializers` to load the own Redlics configuration. 48 | 49 | ```ruby 50 | Redlics.configure do |config| 51 | config.pool_size = 5 # Default connection pool size is 5 52 | config.pool_timeout = 5 # Default connection pool timeout is 5 53 | config.namespace = 'rl' # Default Redis namespace is 'rl', short name saves memory 54 | config.redis = { url: 'redis://127.0.0.1:6379' } # Default Redis configuration or Redis object, see: https://github.com/redis/redis-rb/blob/master/lib/redis.rb 55 | config.silent = false # Silent Redis errors, default is false 56 | config.separator = ':' # Default Redis namespace separator, default is ':' 57 | config.bucket = true # Bucketize counter object ids, default is true 58 | config.bucket_size = 1000 # Bucket size, best performance with bucket size 1000. See hash-max-ziplist-entries 59 | config.auto_clean = true # Auto remove operation keys from Redis 60 | config.encode = { # Encode event ids or object ids 61 | events: true, 62 | ids: true 63 | } 64 | config.granularities = { 65 | minutely: { step: 1.minute, pattern: '%Y%m%d%H%M' }, 66 | hourly: { step: 1.hour, pattern: '%Y%m%d%H' }, 67 | daily: { step: 1.day, pattern: '%Y%m%d' }, 68 | weekly: { step: 1.week, pattern: '%GW%V' }, 69 | monthly: { step: 1.month, pattern: '%Y%m' }, 70 | yearly: { step: 1.year, pattern: '%Y' } 71 | } 72 | config.counter_expirations = { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year } 73 | config.counter_granularity = :daily..:yearly 74 | config.tracker_expirations = { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year }, 75 | config.tracker_granularity = :daily..:yearly 76 | config.operation_expiration = 1.day 77 | end 78 | ``` 79 | 80 | #### Buckets 81 | 82 | If Redlics is configured to use buckets, please configure Redis to allow an ideal size of list entries. 83 | 84 | ```ruby 85 | # Redlics config 86 | config.bucket = true 87 | config.bucket_size = 1000 88 | ``` 89 | 90 | The Redis configuration can be found in file `redis.conf`. The default bucket size is 1000 and is an ideal size. Any higher size and 91 | the HSET commands would cause noticeable CPU activity. The Redis setting `hash-max-ziplist-entries` configures the maximum number 92 | of entries a hash can have while still being encoded efficiently. 93 | 94 | ``` 95 | # /etc/redis/redis.conf 96 | hash-max-ziplist-entries 1024 97 | hash-max-ziplist-value 64 98 | ``` 99 | 100 | Read more: 101 | * [Special encoding of small aggregate data types](http://redis.io/topics/memory-optimization) 102 | * [Storing hundreds of millions of simple key-value pairs in Redis](http://instagram-engineering.tumblr.com/post/12202313862/storing-hundreds-of-millions-of-simple-key-value) 103 | 104 | ##### Example 105 | 106 | * **Id:** 1234 107 | * **Bucket size**: 1000 108 | 109 | results in: 110 | 111 | * **Bucket nr.:** 1 (part of Redis key) 112 | * **Bucket entry nr.:** 234 (part of Redis value as hash key) 113 | 114 | #### Encoding 115 | 116 | If Redlics is configured to encode events and object ids, all numbers are encoded to save memory. 117 | 118 | ```ruby 119 | config.encode = { 120 | events: true, 121 | ids: true 122 | } 123 | ``` 124 | 125 | ##### Examples 126 | 127 | Byte size reduction of id `1234` from 4 bytes to 2 bytes. 128 | 129 | * Ids encoding 130 | 131 | ```ruby 132 | Redlics::Key.encode(1234) 133 | # => "2+" 134 | ``` 135 | 136 | * Event encoding 137 | 138 | Encodes numbers in event names separated by the defined separator in the configuration. 139 | **Event name:** `products:1234`, **encoded event:** `products:!k`. 140 | 141 | ### Counting 142 | 143 | Counting an event can be done by call count with **arguments**, **hash parameters** or a **block**. 144 | 145 | ```ruby 146 | # By arguments 147 | Redlics.count('products:list') 148 | 149 | # By hash parameters 150 | Redlics.count(event: 'products:list', id: 1234) 151 | 152 | # By block 153 | Redlics.count do |c| 154 | c.event = 'products:list' 155 | c.id = 1234 156 | 157 | # Count this event in the past 158 | c.past = 3.days.ago 159 | 160 | # Count granularity for this event: Symbol, String, Array or Range 161 | c.granularity = :daily..:monthly 162 | # c.granularity = :daily 163 | # c.granularity = [:daily, :monthly] 164 | 165 | # Expire (delete) count for this event for specific granularities after defined period. 166 | c.expiration_for = { daily: 6.days, monthly: 2.months } 167 | end 168 | ``` 169 | 170 | **Parameters** 171 | 172 | * **event:** event name **(required)**. 173 | * **id:** object id (optional), e.g. user id 174 | * **past:** time object (optional), if not set `Time.now` is used. 175 | * **granularity:** granularities defined in configuration (optional), if not set `config.counter_granularity` is used. 176 | * **expiration_for:** expire count for given granularities (optional), if not set `config.counter_expirations` is used. 177 | 178 | ### Tracking 179 | 180 | Tracking an event can be done by call track with **arguments**, **hash parameters** or a **block**. 181 | 182 | ```ruby 183 | # By arguments 184 | Redlics.track('products:list', 1234) 185 | 186 | # By hash parameters 187 | Redlics.track(event: 'products:list', id: 1234) 188 | 189 | # By block 190 | Redlics.track do |t| 191 | t.event = 'products:list' 192 | t.id = 1234 193 | 194 | # Track this event in the past 195 | t.past = 3.days.ago 196 | 197 | # Track granularity for this event: Symbol, String, Array or Range 198 | t.granularity = :daily..:monthly 199 | # t.granularity = :daily 200 | # t.granularity = [:daily, :monthly] 201 | 202 | # Expire (delete) tracking for this event for specific granularities after defined period. 203 | t.expiration_for = { daily: 6.days, monthly: 2.months } 204 | end 205 | ``` 206 | 207 | **Parameters** 208 | 209 | * **event:** event name **(required)**. 210 | * **id:** object id **(required)**, e.g. user id 211 | * **past:** time object (optional), if not set `Time.now` is used. 212 | * **granularity:** granularities defined in configuration (optional), if not set `config.counter_granularity` is used. 213 | * **expiration_for:** expire track for given granularities (optional), if not set `config.counter_expirations` is used. 214 | 215 | ### Analyze 216 | 217 | To analyze recorded data an analyzable query object must be defined first. 218 | 219 | ```ruby 220 | a1 = Redlics.analyze('products:list', :today) 221 | 222 | # Examples 223 | a2 = Redlics.analyze('products:list', :today, granularity: :minutely) 224 | a3 = Redlics.analyze('products:list', :today, id: 1234) 225 | ``` 226 | 227 | **Parameters** 228 | 229 | * **event:** event name **(required)**. 230 | * **time:** time object **(required)**, can be: 231 | * **a symbol:** predefined in Redlics::TimeFrame.init_with_symbol 232 | * e.g. *:hour, :day, :week, :month, :year, :today, :yesterday, :this_week, :last_week, :this_month, :last_month, :this_year, :last_year* 233 | * **a hash:** with keys `from` and `to` 234 | * e.g. `{ from: 30.days.ago, to: Time.now}` 235 | * **a range:** defined as a range 236 | * e.g. `30.days.ago..Time.now` 237 | * **a time:** simple time object 238 | * e.g. `Time.new(2016, 1, 12)` or `1.day.ago.to_time` 239 | * **Options:** 240 | * **id:** object id, e.g. user id 241 | * **granularity:** one granularitiy defined in configuration (optional), if not set first element of `config.counter_granularity` is used. 242 | 243 | Analyzable query objects can be used to analyze **counts** and **tracks**. 244 | Queries are not *"realized"* until an action is performed: 245 | 246 | #### Counts 247 | 248 | ```ruby 249 | # Check how many counts has been recorded. 250 | a1.counts 251 | 252 | # Use this method to get plot-friendly data for graphs. 253 | a1.plot_counts 254 | 255 | # See what's under the hood. No Redis access. 256 | a1.realize_counts! 257 | ``` 258 | 259 | #### Tracks 260 | 261 | ```ruby 262 | # Check how many unique tracks has been recorded. 263 | a1.tracks 264 | 265 | # Check if given id exists in the tracks result. 266 | a1.exists? 267 | 268 | # Use this method to get plot-friendly data for graphs. 269 | a1.plot_tracks 270 | 271 | # See what's under the hood. No Redis access. 272 | a1.realize_tracks! 273 | ``` 274 | 275 | #### Reset 276 | 277 | Reset is required to keep clean redis operation results. To calculate counts and tracks operations are stored in Redis. 278 | It is possible to delete this operation result keys in Redis manually or let the Ruby garbage collector clean redis before the 279 | analyzable query objects are destructed (configuration `config.auto_clean`). The third way is hard coded and uses an expiration 280 | time in Redis for that given operation result keys. The expiration time for operations can be configured with `config.operation_expiration`.` 281 | 282 | ```ruby 283 | a1.reset! 284 | ``` 285 | 286 | Partial resets are also possible by pass a `space` argument as symbol: 287 | 288 | ```ruby 289 | # :counter, :tracker, :counts, :tracks, :exists, 290 | # :plot_counts, :plot_tracks, :realize_counts, :realize_tracks 291 | a1.reset!(:counter) 292 | a1.reset!(:tracker) 293 | ``` 294 | 295 | **It is recommended to do a reset if the analyzable query object is no more needed!** 296 | 297 | The analyzable query objects can also be created and used in a block. 298 | 299 | ```ruby 300 | Redlics.analyze('products:list', :today) do |a| 301 | a.tracks 302 | # ... 303 | a.reset! 304 | end 305 | ``` 306 | 307 | ### Operators 308 | 309 | Analyzable query objects can be calculated also using operators (for tracking data). The following operators are available: 310 | 311 | * **AND** (`&`) 312 | * **OR** (`|`), 313 | * **NOT** (`~`, `-`) 314 | * **XOR** (`^`) 315 | * **PLUS** (`+`) 316 | * **MINUS** (`-`) 317 | 318 | Assuming users has been tracked for the actions `products:list, products:featured, logged_in`, then it is 319 | possible to use operators to check users that: 320 | 321 | * has viewed the products list 322 | * and the featured products list 323 | * but not logged in today 324 | 325 | ```ruby 326 | # Create analyzable query objects 327 | a1 = Redlics.analyze('products:list', :today) 328 | a2 = Redlics.analyze('products:featured', :today) 329 | a3 = Redlics.analyze('logged_in', :today) 330 | 331 | # The operation 332 | o = (( a1 & a2) - a3) 333 | 334 | # To check how many users are in this result set. 335 | o.tracks 336 | 337 | # To check if a user is in this result set. 338 | o.exists?(1234) 339 | 340 | # Clean up complete operation results. 341 | o.reset!(:tree) 342 | ``` 343 | 344 | ### Tips and hints 345 | 346 | #### Granularities 347 | 348 | * You should be aware that there is a close relation between counting, tracking and querying in regards to granularities. 349 | * When querying, make sure to tracking in the same granularity. 350 | * If you are tracking in the range of `:daily..:monthly` then you can only query in that range (or you will get wrong results). 351 | * Another possible error you should be aware of is when querying for a time frame that is not correlated with the granularity. 352 | 353 | #### Buckets 354 | 355 | * Use buckets if you have many counters to save memory. 356 | * 1000 is the ideal bucket size. 357 | 358 | #### Encoding 359 | 360 | * Use event and ids encoding if you have many counters to save memory. 361 | 362 | #### Namespaces 363 | 364 | Keys in Redis look like this: 365 | 366 | ```ruby 367 | # Tracker 368 | 'rl:t:products:list:2016' 369 | 370 | # Counter without buckets (unencoded) 371 | 'rl:c:products:list:2016:1234' 372 | 373 | # Counter without buckets (encoded) 374 | 'rl:c:products:list:2016:!k' 375 | 376 | # Counter with buckets (unencoded, 234 is value of key) 377 | 'rl:c:products:list:2016:1' => '234' 378 | 379 | # Counter with buckets (encoded, 3k is value of key) 380 | 'rl:c:products:list:2016:2' => '3k' 381 | 382 | # Operation 383 | 'rl:o:f56fa42d-1e85-4e2f-b8c8-a0f9b5bee5d0' 384 | ``` 385 | 386 | ## Contributors 387 | 388 | * Inspired by Minuteman [github.com/elcuervo/minuteman](https://github.com/elcuervo/minuteman). 389 | * Inspired by Btrack [github.com/chenfisher/Btrack](https://github.com/chenfisher/Btrack). 390 | * Inspired by Counterman [github.com/maccman/counterman](https://github.com/maccman/counterman). 391 | 392 | ## Contributing 393 | 394 | 1. Fork it ( https://github.com/[your-username]/redlics/fork ) 395 | 2. Create your feature branch (`git checkout -b my-new-feature`) 396 | 3. Commit your changes (`git commit -am 'Add some feature'`) 397 | 4. Push to the branch (`git push origin my-new-feature`) 398 | 5. Create a new Pull Request 399 | 400 | ## License 401 | 402 | The MIT License 403 | 404 | Copyright (c) 2023 Phlegx Systems Technologies GmbH 405 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'rake/testtask' 4 | 5 | Rake::TestTask.new do |t| 6 | t.pattern = 'test/**/*_test.rb' 7 | end 8 | 9 | task default: :test 10 | -------------------------------------------------------------------------------- /lib/redlics.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'active_support/core_ext/module/delegation' 4 | require 'active_support/time' 5 | require 'msgpack' 6 | require 'ostruct' 7 | require 'redlics/version' 8 | require 'redlics/config' 9 | require 'redlics/exception' 10 | require 'redlics/connection' 11 | require 'redlics/granularity' 12 | require 'redlics/key' 13 | require 'redlics/time_frame' 14 | require 'redlics/counter' 15 | require 'redlics/tracker' 16 | require 'redlics/operators' 17 | require 'redlics/query' 18 | require 'redlics/query/operation' 19 | 20 | # Redlics namespace 21 | module Redlics 22 | extend self 23 | 24 | # Delegate methods to right objects. 25 | delegate :count, to: Counter 26 | delegate :track, to: Tracker 27 | delegate :analyze, to: Query 28 | 29 | # Get or initialize the Redis connection. 30 | # @return [Object] redis connection 31 | def redis 32 | raise ArgumentError, 'requires a block' unless block_given? 33 | redis_pool.with do |conn| 34 | retryable = true 35 | begin 36 | yield conn 37 | rescue Redis::BaseError => e 38 | raise e unless config.silent 39 | rescue Redis::CommandError => ex 40 | (conn.disconnect!; retryable = false; retry) if retryable && ex.message =~ /READONLY/ 41 | raise unless config.silent 42 | end 43 | end 44 | end 45 | 46 | # Load Lua script file and arguments in Redis. 47 | # 48 | # @param file [String] absolute path to the Lua script file 49 | # @param *args [Array] list of arguments for Redis evalsha 50 | # @return [String] Lua script result 51 | def script(file, *args) 52 | begin 53 | cache = LUA_CACHE[redis { |r| r.client.options[:url] }] 54 | if cache.key?(file) 55 | sha = cache[file] 56 | else 57 | src = File.read(file) 58 | sha = redis { |r| r.redis.script(:load, src) } 59 | cache[file] = sha 60 | end 61 | redis { |r| r.evalsha(sha, *args) } 62 | rescue RuntimeError 63 | case $!.message 64 | when Exception::ErrorPatterns::NOSCRIPT 65 | LUA_CACHE[redis { |r| r.client.options[:url] }].clear 66 | retry 67 | else 68 | raise $! unless config.silent 69 | end 70 | end 71 | end 72 | 73 | # Get or initialize Redlics config. 74 | # @return [OpenStruct] Redlics configuration 75 | def config 76 | @config ||= Redlics::Config.new 77 | end 78 | 79 | # Set configuration of Redlics in a block. 80 | # @return [OpenStruct] Redlics configuration 81 | def configure 82 | yield config if block_given? 83 | end 84 | 85 | private 86 | 87 | # Get or initialize the Redis connection pool. 88 | # @return [ConnectionPool] redis connection pool 89 | def redis_pool 90 | @redis ||= Redlics::Connection.create(config.to_h) 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/redlics/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Redlics constants. 5 | LUA_CACHE = Hash.new { |h, k| h[k] = Hash.new } 6 | LUA_SCRIPT = File.expand_path('../lua/script.lua', __FILE__).freeze 7 | CONTEXTS = { counter: { short: :c, long: :counter }, tracker: { short: :t, long: :tracker }, operation: { short: :o, long: :operation } }.freeze 8 | 9 | # Configuration class 10 | class Config 11 | # Initialization with default configuration. 12 | # 13 | # Configure Redis: 14 | # /etc/redis/redis.conf 15 | # hash-max-ziplist-entries 1024 16 | # hash-max-ziplist-value 64 17 | # 18 | # @return [OpenStruct] default configuration 19 | def initialize 20 | @config = OpenStruct.new( 21 | pool_size: 5, # Default connection pool size is 5 22 | pool_timeout: 5, # Default connection pool timeout is 5 23 | namespace: 'rl', # Default Redis namespace is 'rl', short name saves memory 24 | redis: { url: 'redis://127.0.0.1:6379' }, # Default Redis configuration 25 | silent: false, # Silent Redis errors, default is false 26 | separator: ':', # Default Redis namespace separator, default is ':' 27 | bucket: true, # Bucketize counter object ids, default is true 28 | bucket_size: 1000, # Bucket size, best performance with bucket size 1000. See hash-max-ziplist-entries 29 | auto_clean: true, # Auto remove operation keys from Redis 30 | encode: { # Encode event ids or object ids 31 | events: true, 32 | ids: true 33 | }, 34 | granularities: { 35 | minutely: { step: 1.minute, pattern: '%Y%m%d%H%M' }, 36 | hourly: { step: 1.hour, pattern: '%Y%m%d%H' }, 37 | daily: { step: 1.day, pattern: '%Y%m%d' }, 38 | weekly: { step: 1.week, pattern: '%GW%V' }, 39 | monthly: { step: 1.month, pattern: '%Y%m' }, 40 | yearly: { step: 1.year, pattern: '%Y' } 41 | }, 42 | counter_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year }, 43 | counter_granularity: :daily..:yearly, 44 | tracker_expirations: { minutely: 1.day, hourly: 1.week, daily: 3.months, weekly: 1.year, monthly: 1.year, yearly: 1.year }, 45 | tracker_granularity: :daily..:yearly, 46 | operation_expiration: 1.day 47 | ) 48 | end 49 | 50 | # Send missing methods to the OpenStruct configuration. 51 | # 52 | # @param method [String] the missing method name 53 | # @param *args [Array] list of arguments of the missing method 54 | # @return [Object] a configuration parameter 55 | def method_missing(method, *args, &block) 56 | @config.send(method, *args, &block) 57 | end 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/redlics/connection.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'connection_pool' 4 | require 'redis' 5 | require 'redis/namespace' 6 | 7 | module Redlics 8 | # Connection namespace 9 | module Connection 10 | extend self 11 | 12 | # Create a new connection pool for Redis connection. 13 | # 14 | # @param options [Hash] configuration options 15 | # @return [ConnectionPool] Redlics connection pool 16 | def create(options = {}) 17 | ConnectionPool.new(pool_options(options)) do 18 | build_connection(options) 19 | end 20 | end 21 | 22 | private 23 | 24 | # Set connection pool options. 25 | # 26 | # @param options [Hash] configuration options 27 | # @return [Hash] connection pool options 28 | def pool_options(options) 29 | { size: options[:pool_size], 30 | timeout: options[:pool_timeout] } 31 | end 32 | 33 | # Build Redis connection with options. 34 | # 35 | # @param options [Hash] configuration options 36 | # @return [Redis] Redis connection 37 | # @return [Redis::Namespace] Redis namespaced connection 38 | def build_connection(options) 39 | namespace = options[:namespace] 40 | connection = options[:redis].is_a?(Redis) ? options[:redis] : Redis.new(redis_opts(options)) 41 | if namespace 42 | Redis::Namespace.new(namespace, redis: connection) 43 | else 44 | connection 45 | end 46 | end 47 | 48 | # Client options provided by redis-rb 49 | # @see https://github.com/redis/redis-rb/blob/master/lib/redis.rb 50 | # 51 | # @param options [Hash] options 52 | # @option options [String] :url (value of the environment variable REDIS_URL) a Redis URL, for a TCP connection: `redis://:[password]@[hostname]:[port]/[db]` (password, port and database are optional), for a unix socket connection: `unix://[path to Redis socket]`. This overrides all other options. 53 | # @option options [String] :host ("127.0.0.1") server hostname 54 | # @option options [Fixnum] :port (6379) server port 55 | # @option options [String] :path path to server socket (overrides host and port) 56 | # @option options [Float] :timeout (5.0) timeout in seconds 57 | # @option options [Float] :connect_timeout (same as timeout) timeout for initial connect in seconds 58 | # @option options [String] :password Password to authenticate against server 59 | # @option options [Fixnum] :db (0) Database to select after initial connect 60 | # @option options [Symbol] :driver Driver to use, currently supported: `:ruby`, `:hiredis`, `:synchrony` 61 | # @option options [String] :id ID for the client connection, assigns name to current connection by sending `CLIENT SETNAME` 62 | # @option options [Hash, Fixnum] :tcp_keepalive Keepalive values, if Fixnum `intvl` and `probe` are calculated based on the value, if Hash `time`, `intvl` and `probes` can be specified as a Fixnum 63 | # @option options [Fixnum] :reconnect_attempts Number of attempts trying to connect 64 | # @option options [Boolean] :inherit_socket (false) Whether to use socket in forked process or not 65 | # @option options [Array] :sentinels List of sentinels to contact 66 | # @option options [Symbol] :role (:master) Role to fetch via Sentinel, either `:master` or `:slave` 67 | def redis_opts(options) 68 | opts = options[:redis] 69 | opts[:driver] ||= 'ruby' 70 | opts 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/redlics/counter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Counter class 5 | module Counter 6 | # Context constant for given class. 7 | CONTEXT = Redlics::CONTEXTS[:counter].freeze 8 | 9 | extend self 10 | 11 | # Count for a given event and object id with options. 12 | # 13 | # @param *args [Array] list of arguments for count 14 | # @return [Array] list of counted granularities 15 | def count(*args, &block) 16 | return count_with_block(&block) if block_given? 17 | return count_with_hash(args.first) if args.first.is_a?(Hash) 18 | count_with_args(*args) 19 | end 20 | 21 | private 22 | 23 | # Count with hash. 24 | # 25 | # @param options [Hash] configuration options 26 | # @return [Array] list of counted granularities 27 | def count_with_hash(options) 28 | options[:id] = options[:id].to_i unless options[:id].nil? 29 | Granularity.validate(CONTEXT, options[:granularity]).each do |granularity| 30 | opt = options.clone.merge(granularity: granularity) 31 | if Redlics.config.bucket && opt[:id] 32 | count_by_hash(opt) 33 | else 34 | count_by_key(opt) 35 | end 36 | end 37 | end 38 | 39 | # Count with hash. 40 | # 41 | # @param [&Block] a block with configuration options 42 | # @return [Array] list of counted granularities 43 | def count_with_block 44 | yield options = OpenStruct.new 45 | count_with_hash(options.to_h) 46 | end 47 | 48 | # Count with hash. 49 | # 50 | # @param *args [Array] list of arguments for count 51 | # @return [Array] list of counted granularities 52 | def count_with_args(*args) 53 | options = args.last.instance_of?(Hash) ? args.pop : {} 54 | options.merge!(event: args[0]) 55 | count_with_hash(options) 56 | end 57 | 58 | # Count by hash. 59 | # 60 | # @param options [Hash] configuration options 61 | # @return [Array] result of pipelined redis commands 62 | def count_by_hash(options) 63 | granularity = options[:granularity] 64 | key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: true }) 65 | Redlics.redis do |conn| 66 | conn.pipelined do |redis| 67 | redis.hincrby(key[0], key[1], 1) 68 | redis.expire(key[0], (options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity]).to_i) 69 | end 70 | end 71 | end 72 | 73 | # Count by key. 74 | # 75 | # @param options [Hash] configuration options 76 | # @return [Array] result of pipelined redis commands 77 | def count_by_key(options) 78 | granularity = options[:granularity] 79 | key = Key.name(CONTEXT, options[:event], granularity, options[:past], { id: options[:id], bucketized: false }) 80 | Redlics.redis do |conn| 81 | conn.pipelined do |redis| 82 | redis.incr(key) 83 | redis.expire(key, (options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.counter_expirations[granularity]).to_i) 84 | end 85 | end 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/redlics/exception.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Exception namespace 5 | module Exception 6 | # Error Pattern namespace 7 | module ErrorPatterns 8 | NOSCRIPT = /^NOSCRIPT/.freeze 9 | end 10 | 11 | # Lua Range Error class 12 | # 13 | # Maximal Lua stack size for the method `unpack` is by default 8000. 14 | # To change this parameter in Redis an own make and build of Redis is needed. 15 | # @see https://github.com/antirez/redis/blob/3.2/deps/lua/src/luaconf.h 16 | class LuaRangeError < StandardError; 17 | # Initialization with default error message. 18 | # 19 | # @param msg [String] the error message 20 | # @return [Redlics::Exception::LuaRangeError] error message 21 | def initialize(msg = 'Too many keys (max. 8000 keys defined by LUAI_MAXCSTACK)') 22 | super(msg) 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/redlics/granularity.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Granularity namespace 5 | module Granularity 6 | extend self 7 | 8 | # Validate granularities by given context. 9 | # 10 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 11 | # @param granularities [Range] granularity range 12 | # @param granularities [String] single granularity 13 | # @param granularities [Array] granularity array 14 | # @return [Array] includes all valid granularities 15 | def validate(context, granularities) 16 | check(granularities) || default(context) 17 | end 18 | 19 | # Get default granularities by given context. 20 | # 21 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 22 | # @return [Array] includes all valid default granularities 23 | def default(context) 24 | check(Redlics.config["#{context[:long]}_granularity"]) || [Redlics.config.granularities.keys.first] 25 | end 26 | 27 | private 28 | 29 | # Check if granularities are defined in the configuration. 30 | # 31 | # @param granularities [Range] granularity range 32 | # @param granularities [String] single granularity 33 | # @param granularities [Array] granularity array 34 | # @return [Array] includes all valid granularities 35 | def check(granularities) 36 | keys = Redlics.config.granularities.keys 37 | checked = case granularities 38 | when Range 39 | keys[keys.index(granularities.first.to_sym)..keys.index(granularities.last.to_sym)] 40 | when Array 41 | [granularities.map(&:to_sym)].flatten & keys 42 | else 43 | [granularities && granularities.to_sym].flatten & keys 44 | end 45 | checked.any? ? checked : nil 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/redlics/key.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Key namespace 5 | module Key 6 | extend self 7 | 8 | # Construct the key name with given parameters. 9 | # 10 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 11 | # @param event [String] event name with eventual Redis namespace separator 12 | # @param granularity [Symbol] existing granularity 13 | # @param past [Time] a time object 14 | # @param options [Hash] configuration options 15 | # @return [String] unbucketized key name 16 | # @return [Array] bucketized key name 17 | def name(context, event, granularity, past, options = {}) 18 | past ||= Time.now 19 | event ||= 'nil' 20 | granularity = Granularity.validate(context, granularity).first 21 | event = encode_event(event) if Redlics.config.encode[:events] 22 | key = "#{context[:short]}#{Redlics.config.separator}#{event}#{Redlics.config.separator}#{time_format(granularity, past)}" 23 | key = with_namespace(key) if options[:namespaced] 24 | return bucketize(key, options[:id]) if bucketize?(context, options) 25 | return unbucketize(key, options[:id]) if context[:long] == :counter && !options[:id].nil? 26 | key 27 | end 28 | 29 | # Construct an array with all keys of a time frame in a given granularity. 30 | # 31 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 32 | # @param event [String] event name with eventual Redis namespace separator 33 | # @param time_object [Symbol] time object predefined in Redlics::TimeFrame.init_with_symbol 34 | # @param time_object [Hash] time object with keys `from` and `to` 35 | # @param time_object [Range] time object as range 36 | # @param time_object [Time] time object 37 | # @param options [Hash] configuration options 38 | # @return [Array] array with all keys of a time frame in a given granularity 39 | def timeframed(context, event, time_object, options = {}) 40 | options = { namespaced: true }.merge(options) 41 | timeframe = TimeFrame.new(context, time_object, options) 42 | timeframe.splat do |time| 43 | name(context, event, timeframe.granularity, time, options) 44 | end 45 | end 46 | 47 | # Prepend namespace to a key. 48 | # 49 | # @param key [String] the key name 50 | # @return [String] the key name with prepended namespace 51 | def with_namespace(key) 52 | return key unless Redlics.config.namespace.length > 0 53 | return key if key.split(Redlics.config.separator).first == Redlics.config.namespace.to_s 54 | "#{Redlics.config.namespace}#{Redlics.config.separator}#{key}" 55 | end 56 | 57 | # Encode a number with a mapping table. 58 | # 59 | # @param number [Integer] the number to encode 60 | # @return [String] the encoded number as string 61 | def encode(number) 62 | encoded = '' 63 | number = number.to_s 64 | number = (number.size % 2) != 0 ? "0#{number}" : number 65 | token = 0 66 | while token <= number.size - 1 67 | encoded += encode_map[number[token..token+1].to_i.to_s].to_s 68 | token += 2 69 | end 70 | encoded 71 | end 72 | 73 | # Decode a number with a mapping table. 74 | # 75 | # @param string [String] the string to encode 76 | # @return [Integer] the decoded string as integer 77 | def decode(string) 78 | decoded = '' 79 | string = string.to_s 80 | token = 0 81 | while token <= string.size - 1 82 | number = decode_map[string[token]].to_s 83 | decoded += number.size == 1 ? "0#{number}" : number 84 | token += 1 85 | end 86 | decoded.to_i 87 | end 88 | 89 | # Check if a key exists in Redis. 90 | # 91 | # @param string [String] the key name to check 92 | # @return [Boolean] true id key exists, false if not 93 | def exists?(key) 94 | Redlics.redis { |r| r.exists(key) } 95 | end 96 | 97 | # Check if Redlics can bucketize. 98 | # 99 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 100 | # @param options [Hash] configuration options 101 | # @return [Boolean] true if can bucketize, false if not 102 | def bucketize?(context, options = {}) 103 | context[:long] == :counter && Redlics.config.bucket && !options[:id].nil? 104 | end 105 | 106 | # Create a unique operation key in Redis. 107 | # @return [String] the created unique operation key 108 | def unique_namespace 109 | loop do 110 | ns = operation 111 | unless exists?(ns) 112 | Redlics.redis do |conn| 113 | conn.pipelined do |redis| 114 | redis.set(ns, 0) 115 | redis.expire(ns, Redlics.config.operation_expiration) 116 | end 117 | end 118 | break ns 119 | end 120 | end 121 | end 122 | 123 | private 124 | 125 | # Create a operation key. 126 | # @return [String] the created operation key 127 | def operation 128 | "#{Redlics::CONTEXTS[:operation][:short]}#{Redlics.config.separator}#{SecureRandom.uuid}" 129 | end 130 | 131 | # Get the time format pattern of a granularity. 132 | # 133 | # @param granularity [Symbol] existing granularity 134 | # @param past [Time] a time object 135 | # @return [String] pattern of defined granularity 136 | def time_format(granularity, past) 137 | past.strftime(Redlics.config.granularities[granularity][:pattern]) 138 | end 139 | 140 | # Encode ids in event names. 141 | # 142 | # @param event [String] event name with eventual Redis namespace separator 143 | # @return [String] event name with encoded ids 144 | def encode_event(event) 145 | event.to_s.split(Redlics.config.separator).map { |v| v.match(/\A\d+\z/) ? encode(v) : v }.join(Redlics.config.separator) 146 | end 147 | 148 | # Bucketize key name with id. 149 | # 150 | # @param key [String] key name 151 | # @param id [Integer] object id 152 | # @return [Array] bucketized key name and value 153 | def bucketize(key, id) 154 | bucket = id.to_i / Redlics.config.bucket_size.to_i 155 | value = id.to_i % Redlics.config.bucket_size.to_i 156 | if Redlics.config.encode[:ids] 157 | bucket = encode(bucket) 158 | value = encode(value) 159 | end 160 | ["#{key}#{Redlics.config.separator}#{bucket}", value] 161 | end 162 | 163 | # Unbucketize key name with id. Encode the id if configured to encode. 164 | # 165 | # @param key [String] key name 166 | # @param id [Integer] object id 167 | # @return [String] unbucketized key name with eventual encoded object id 168 | def unbucketize(key, id) 169 | id = encode(id) if Redlics.config.encode[:ids] 170 | "#{key}#{Redlics.config.separator}#{id}" 171 | end 172 | 173 | # Defined encode map. 174 | # @return [Hash] the encode map with numbers as keys 175 | def encode_map 176 | @encode_map ||= replace_separator_encode({ 177 | '0' => '1', '1' => '2', '2' => '3', '3' => '4', '4' => '5', '5' => '6', '6' => '7', '7' => '8', '8' => '9', '9' => '0', 178 | '10' => '-', '11' => '=', '12' => '!', '13' => '@', '14' => '#', '15' => '$', '16' => '%', '17' => '^', '18' => '&', '19' => '*', 179 | '20' => '(', '21' => ')', '22' => '_', '23' => '+', '24' => 'a', '25' => 'b', '26' => 'c', '27' => 'd', '28' => 'e', '29' => 'f', 180 | '30' => 'g', '31' => 'h', '32' => 'i', '33' => 'j', '34' => 'k', '35' => 'l', '36' => 'm', '37' => 'n', '38' => 'o', '39' => 'p', 181 | '40' => 'q', '41' => 'r', '42' => 's', '43' => 't', '44' => 'u', '45' => 'v', '46' => 'w', '47' => 'x', '48' => 'y', '49' => 'z', 182 | '50' => 'A', '51' => 'B', '52' => 'C', '53' => 'D', '54' => 'E', '55' => 'F', '56' => 'G', '57' => 'H', '58' => 'I', '59' => 'J', 183 | '60' => 'K', '61' => 'L', '62' => 'M', '63' => 'N', '64' => 'O', '65' => 'P', '66' => 'Q', '67' => 'R', '68' => 'S', '69' => 'T', 184 | '70' => 'U', '71' => 'V', '72' => 'W', '73' => 'X', '74' => 'Y', '75' => 'Z', '76' => '[', '77' => ']', '78' => '\\', '79' => ';', 185 | '80' => ',', '81' => '.', '82' => '/', '83' => '{', '84' => '}', '85' => '|', '86' => '§', '87' => '<', '88' => '>', '89' => '?', 186 | '90' => '`', '91' => '~', '92' => 'ä', '93' => 'Ä', '94' => 'ü', '95' => 'Ü', '96' => 'ö', '97' => 'Ö', '98' => 'é', '99' => 'É' }).freeze 187 | end 188 | 189 | # Defined decode map. 190 | # @return [Hash] the decode map with numbers as values 191 | def decode_map 192 | @decode_map ||= replace_separator_decode({ 193 | '1' => '0', '2' => '1', '3' => '2', '4' => '3', '5' => '4', '6' => '5', '7' => '6', '8' => '7', '9' => '8', '0' => '9', 194 | '-' => '10', '=' => '11', '!' => '12', '@' => '13', '#' => '14', '$' => '15', '%' => '16', '^' => '17', '&' => '18', '*' => '19', 195 | '(' => '20', ')' => '21', '_' => '22', '+' => '23', 'a' => '24', 'b' => '25', 'c' => '26', 'd' => '27', 'e' => '28', 'f' => '29', 196 | 'g' => '30', 'h' => '31', 'i' => '32', 'j' => '33', 'k' => '34', 'l' => '35', 'm' => '36', 'n' => '37', 'o' => '38', 'p' => '39', 197 | 'q' => '40', 'r' => '41', 's' => '42', 't' => '43', 'u' => '44', 'v' => '45', 'w' => '46', 'x' => '47', 'y' => '48', 'z' => '49', 198 | 'A' => '50', 'B' => '51', 'C' => '52', 'D' => '53', 'E' => '54', 'F' => '55', 'G' => '56', 'H' => '57', 'I' => '58', 'J' => '59', 199 | 'K' => '60', 'L' => '61', 'M' => '62', 'N' => '63', 'O' => '64', 'P' => '65', 'Q' => '66', 'R' => '67', 'S' => '68', 'T' => '69', 200 | 'U' => '70', 'V' => '71', 'W' => '72', 'X' => '73', 'Y' => '74', 'Z' => '75', '[' => '76', ']' => '77', '\\' => '78', ';' => '79', 201 | ',' => '80', '.' => '81', '/' => '82', '{' => '83', '}' => '84', '|' => '85', '§' => '86', '<' => '87', '>' => '88', '?' => '89', 202 | '`' => '90', '~' => '91', 'ä' => '92', 'Ä' => '93', 'ü' => '94', 'Ü' => '95', 'ö' => '96', 'Ö' => '97', 'é' => '98', 'É' => '99' }).freeze 203 | end 204 | 205 | # Replace defined separator in configuration from the encode map. 206 | # 207 | # @param map [Hash] encode map hash 208 | # @return [Hash] encode map hash without defined separator in configuration. 209 | def replace_separator_encode(map) 210 | unless Redlics.config.separator == ':' 211 | key = map.key(Redlics.config.separator) 212 | map[key] = ':' if key 213 | end 214 | map 215 | end 216 | 217 | # Replace defined separator in configuration from the decode map. 218 | # 219 | # @param map [Hash] decode map hash 220 | # @return [Hash] decode map hash without defined separator in configuration. 221 | def replace_separator_decode(map) 222 | unless Redlics.config.separator == ':' 223 | key = Redlics.config.separator.to_s.to_sym 224 | map[':'.to_sym] = map.delete(key) if map.key?(key) 225 | end 226 | map 227 | end 228 | end 229 | end 230 | -------------------------------------------------------------------------------- /lib/redlics/lua/script.lua: -------------------------------------------------------------------------------- 1 | --- 2 | redis.log(redis.LOG_NOTICE, 'Redlics') 3 | 4 | local func = cmsgpack.unpack(ARGV[1]) 5 | local keys = cmsgpack.unpack(ARGV[2]) 6 | local options = cmsgpack.unpack(ARGV[3]) 7 | 8 | 9 | local function operate(operator, keys) 10 | redis.call('BITOP', operator, options['dest'], unpack(keys)) 11 | return options['dest'] 12 | end 13 | 14 | 15 | local function AND(keys) return operate('AND', keys) end 16 | local function OR(keys) return operate('OR', keys) end 17 | local function XOR(keys) return operate('XOR', keys) end 18 | local function NOT(keys) return operate('NOT', keys) end 19 | local function MINUS(keys) 20 | local items = keys 21 | local src = table.remove(items, 1) 22 | local and_op = AND(keys) 23 | return XOR({ src, and_op }) 24 | end 25 | 26 | 27 | local function operation(keys, options) 28 | if options['operator'] == 'MINUS' then 29 | return MINUS(keys) 30 | else 31 | return operate(options['operator'], keys) 32 | end 33 | end 34 | 35 | 36 | local function counts(keys, options) 37 | local result 38 | if options['bucketized'] then 39 | result = 0 40 | for i,v in ipairs(keys) do 41 | result = result + (redis.call('HGET', v[1], v[2]) or 0) 42 | end 43 | else 44 | result = redis.call('MGET', unpack(keys)) 45 | end 46 | return result 47 | end 48 | 49 | 50 | local function plot_counts(keys, options) 51 | local plot = {} 52 | if options['bucketized'] then 53 | for i,v in ipairs(keys) do 54 | plot[v[1]..v[2]] = (redis.call('HGET', v[1], v[2]) or 0) 55 | end 56 | else 57 | local values = redis.call('MGET', unpack(keys)) 58 | for i,v in ipairs(keys) do 59 | plot[v] = values[i] 60 | end 61 | end 62 | return cjson.encode(plot) 63 | end 64 | 65 | 66 | local function plot_tracks(keys, options) 67 | local plot = {} 68 | for i,v in ipairs(keys) do 69 | plot[v] = redis.call('bitcount', keys[i]) 70 | end 71 | return cjson.encode(plot) 72 | end 73 | 74 | 75 | local exportFuncs = { 76 | operation = operation, 77 | counts = counts, 78 | plot_counts = plot_counts, 79 | plot_tracks = plot_tracks 80 | } 81 | 82 | return exportFuncs[func](keys, options) 83 | -------------------------------------------------------------------------------- /lib/redlics/operators.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Operators namespace 5 | module Operators 6 | # AND (&) operator. 7 | # 8 | # @param query [Redlics::Query] Redlics query object 9 | # @return [Redlics::Query::Operation] a Redlics query operation object 10 | def &(query) 11 | Query::Operation.new('AND', [self, query]) 12 | end 13 | 14 | # OR (|) operator. 15 | # 16 | # @param query [Redlics::Query] Redlics query object 17 | # @return [Redlics::Query::Operation] a Redlics query operation object 18 | def |(query) 19 | Query::Operation.new('OR', [self, query]) 20 | end 21 | alias_method :+, :| 22 | 23 | # XOR (^) operator. 24 | # 25 | # @param query [Redlics::Query] Redlics query object 26 | # @return [Redlics::Query::Operation] a Redlics query operation object 27 | def ^(query) 28 | Query::Operation.new('XOR', [self, query]) 29 | end 30 | 31 | # NOT (-, ~) operator. 32 | # @return [Redlics::Query::Operation] a Redlics query operation object 33 | def -@() 34 | Query::Operation.new('NOT', [self]) 35 | end 36 | alias_method :~@, :-@ 37 | 38 | # MINUS (-) operator. 39 | # 40 | # @param query [Redlics::Query] Redlics query object 41 | # @return [Redlics::Query::Operation] a Redlics query operation object 42 | def -(query) 43 | Query::Operation.new('MINUS', [self, query]) 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/redlics/query.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Query class 5 | class Query 6 | # Include Redlics operators. 7 | include Redlics::Operators 8 | 9 | # Gives read access to the listed instance variables. 10 | attr_reader :namespaces 11 | 12 | # Initialization of a query object 13 | # 14 | # @param event [String] event name with eventual Redis namespace separator 15 | # @param time_object [Symbol] time object predefined in Redlics::TimeFrame.init_with_symbol 16 | # @param time_object [Hash] time object with keys `from` and `to` 17 | # @param time_object [Range] time object as range 18 | # @param time_object [Time] time object 19 | # @param options [Hash] configuration options 20 | # @return [Redlics::Query] query object 21 | def initialize(event, time_object, options = {}) 22 | @event = event.freeze 23 | @time_object = time_object.freeze 24 | @options = options 25 | @namespaces = [] 26 | ObjectSpace.define_finalizer(self, self.class.finalize(namespaces)) if Redlics.config.auto_clean 27 | end 28 | 29 | # Get or process counts on Redis. 30 | # @return [Integer] count result of given query 31 | def counts 32 | @counts ||= ( 33 | result = Redlics.script(Redlics::LUA_SCRIPT, [], ['counts'.to_msgpack, realize_counts!.to_msgpack, 34 | { bucketized: Redlics.config.bucket }.to_msgpack]) 35 | result.is_a?(Array) ? result.map(&:to_i).reduce(0, :+) : result.to_i 36 | ) 37 | end 38 | 39 | # Get or process tracks on Redis. 40 | # @return [Integer] tracks result of given query 41 | def tracks 42 | @tracks ||= Redlics.redis { |r| r.bitcount(track_bits) } 43 | end 44 | 45 | # Get or process track bits on Redis. 46 | # @return [String] key of track bits result 47 | def track_bits 48 | @track_bits ||= ( 49 | @track_bits_namespace = Key.unique_namespace 50 | @namespaces << @track_bits_namespace 51 | Redlics.script(Redlics::LUA_SCRIPT, [], ['operation'.to_msgpack, realize_tracks!.to_msgpack, 52 | { operator: 'OR', dest: Key.with_namespace(@track_bits_namespace) }.to_msgpack]) 53 | @track_bits_namespace 54 | ) 55 | end 56 | 57 | # Check if object id exists in track bits. 58 | # @return [Boolean] true if exists, false if not 59 | # @return [NilClass] nil if no object id is given 60 | def exists? 61 | @exists ||= @options[:id] ? Redlics.redis { |r| r.getbit(track_bits, @options[:id]) } == 1 : nil 62 | end 63 | 64 | # Get or process counts and plot. 65 | # 66 | # @return [Hash] with date times and counts 67 | # @return [NilClass] nil if result has errors 68 | def plot_counts 69 | @plot_counts ||= ( 70 | result = JSON.parse( 71 | Redlics.script(Redlics::LUA_SCRIPT, [], ['plot_counts'.to_msgpack, realize_counts!.to_msgpack, 72 | { bucketized: Redlics.config.bucket }.to_msgpack]) 73 | ) 74 | format_plot(Redlics::CONTEXTS[:counter], result) 75 | ) 76 | rescue JSON::ParserError 77 | nil 78 | end 79 | 80 | # Get or process tracks and plot. 81 | # 82 | # @return [Hash] with date times and counts 83 | # @return [NilClass] nil if result has errors 84 | def plot_tracks 85 | @plot_tracks ||= ( 86 | result = JSON.parse( 87 | Redlics.script(Redlics::LUA_SCRIPT, [], ['plot_tracks'.to_msgpack, realize_tracks!.to_msgpack, 88 | {}.to_msgpack]) 89 | ) 90 | format_plot(Redlics::CONTEXTS[:tracker], result) 91 | ) 92 | rescue JSON::ParserError 93 | nil 94 | end 95 | 96 | # Get or process counts and show keys to analyze. 97 | # @return [Array] list of keys to analyze 98 | def realize_counts! 99 | @realize_counts ||= ( 100 | keys = Key.timeframed(Redlics::CONTEXTS[:counter], @event, @time_object, @options) 101 | raise Exception::LuaRangeError if keys.length > 8000 102 | keys 103 | ) 104 | end 105 | 106 | # Get or process tracks and show keys to analyze. 107 | # @return [Array] list of keys to analyze 108 | def realize_tracks! 109 | @realize_tracks ||= ( 110 | keys = Key.timeframed(Redlics::CONTEXTS[:tracker], @event, @time_object, @options) 111 | raise Exception::LuaRangeError if keys.length > 8000 112 | keys 113 | ) 114 | end 115 | 116 | # Reset processed data (also operation keys on Redis). 117 | # 118 | # @param space [Symbol] define space to reset 119 | # @param space [String] define space to reset 120 | # @return [Boolean] true 121 | def reset!(space = nil) 122 | space = space.to_sym if space 123 | case space 124 | when :counts, :plot_counts, :plot_tracks, :realize_counts, :realize_tracks 125 | instance_variable_set("@#{space}", nil) 126 | when :tracks, :exists 127 | instance_variable_set("@#{space}", nil) 128 | reset_track_bits 129 | when :counter 130 | @counts, @plot_counts, @realize_counts = [nil] * 3 131 | when :tracker 132 | @tracks, @exists, @plot_tracks, @realize_tracks = [nil] * 4 133 | reset_track_bits 134 | else 135 | @counts, @tracks, @exists, @plot_counts, @plot_tracks, @realize_counts, @realize_tracks = [nil] * 7 136 | reset_track_bits 137 | self.class.reset_redis_namespaces(@namespaces) 138 | @namespaces = [] 139 | end 140 | return true 141 | end 142 | 143 | # Check if query is a leaf. A query is always a leaf. 144 | # This method is required for query operations. 145 | # @return [Boolean] true 146 | def is_leaf? 147 | true 148 | end 149 | 150 | # Singleton class 151 | class << self 152 | # Short query access to analyze data. 153 | # 154 | # @param *args [Array] list of arguments of the query 155 | # @return [Redlics::Query] instantiated query object 156 | def analyze(*args) 157 | options = args.last.instance_of?(Hash) ? args.pop : {} 158 | query = case args.size 159 | when 2 160 | Query.new(args[0], args[1], options) 161 | when 3 162 | Query.new(args[0], args[1], options.merge!({ id: args[2].to_i })) 163 | end 164 | return yield query if block_given? 165 | query 166 | end 167 | 168 | # Finalize query called from garbage collector. 169 | # 170 | # @param namespaces [Array] list of created operation keys in Redis 171 | # @return [Integer] result of Redis delete keys 172 | # @return [NilClass] nil if namespaces are empty 173 | def finalize(namespaces) 174 | proc { reset_redis_namespaces(namespaces) } 175 | end 176 | 177 | # Reset Redis created namespace keys. 178 | # 179 | # @param namespaces [Array] list of created operation keys in Redis 180 | # @return [Integer] result of Redis delete keys 181 | # @return [NilClass] nil if namespaces are empty 182 | def reset_redis_namespaces(namespaces) 183 | Redlics.redis { |r| r.del(namespaces) } if namespaces.any? 184 | end 185 | end 186 | 187 | private 188 | 189 | # Format plot result with time objects as keys. 190 | # 191 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 192 | # @param result [Hash] the result hash with Redis keys as hash keys 193 | # @return [Hash] the formatted result hash 194 | def format_plot(context, result) 195 | granularity = Granularity.validate(context, @options[:granularity]).first 196 | pattern = Redlics.config.granularities[granularity][:pattern] 197 | el = Key.bucketize?(context, @options) ? -2 : -1 198 | result.keys.each { |k| 199 | result[Time.strptime(k.split(Redlics.config.separator)[el], pattern)] = result.delete(k) 200 | } 201 | result 202 | end 203 | 204 | # Reset track bits (also operation key on Redis). 205 | # @return [NilClass] nil 206 | def reset_track_bits 207 | self.class.reset_redis_namespaces([@track_bits_namespace]) 208 | @namespaces.delete(@track_bits_namespace) 209 | @track_bits, @track_bits_namespace = nil, nil 210 | end 211 | end 212 | end 213 | -------------------------------------------------------------------------------- /lib/redlics/query/operation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | class Query 5 | # Operation class 6 | class Operation 7 | # Include Redlics operators. 8 | include Redlics::Operators 9 | 10 | # Gives read access to the listed instance variables. 11 | attr_reader :namespaces 12 | 13 | # Initialization of a query operation object. 14 | # 15 | # @param operator [String] operator to calculate 16 | # @param queries [Array] queries to calculate with the given operator 17 | # @return [Redlics::Query::Operation] query operation object 18 | def initialize(operator, queries) 19 | @operator = operator.upcase.freeze 20 | @queries = queries.freeze 21 | @track_bits = nil 22 | @namespaces = [] 23 | ObjectSpace.define_finalizer(self, self.class.finalize(namespaces)) if Redlics.config.auto_clean 24 | end 25 | 26 | # Get or process tracks on Redis. 27 | # @return [Integer] tracks result of given query operation 28 | def tracks 29 | @tracks ||= ( 30 | Redlics.redis { |r| r.bitcount(@track_bits || traverse) } 31 | ) 32 | end 33 | 34 | # Get or process track bits on Redis. 35 | # @return [String] key of track bits result 36 | def track_bits 37 | @track_bits ||= ( 38 | keys = [] 39 | track_bits_namespace = Key.unique_namespace 40 | @namespaces << track_bits_namespace 41 | if @operator == 'NOT' 42 | keys << Key.with_namespace(@queries[0].track_bits) 43 | else 44 | @queries.each { |q| keys << Key.with_namespace(q.track_bits) } 45 | end 46 | Redlics.script(Redlics::LUA_SCRIPT, [], ['operation'.to_msgpack, keys.to_msgpack, 47 | { operator: @operator, dest: Key.with_namespace(track_bits_namespace) }.to_msgpack]) 48 | track_bits_namespace 49 | ) 50 | end 51 | 52 | # Check if object id exists in track bits. 53 | # 54 | # @param [Integer] the object id to check 55 | # @return [Boolean] true if exists, false if not 56 | def exists?(id) 57 | Redlics.redis { |r| r.getbit(@track_bits || traverse, id.to_i) } == 1 58 | end 59 | 60 | # Reset processed data (also operation keys on Redis). 61 | # 62 | # @param space [Symbol] define space to reset 63 | # @param space [String] define space to reset 64 | # @return [Boolean] true 65 | def reset!(space = nil) 66 | space = space.to_sym if space 67 | case space 68 | when :tree 69 | @queries.each { |q| q.reset!(:tree) } 70 | reset! 71 | else 72 | @tracks, @track_bits = nil, nil 73 | self.class.reset_redis_namespaces(@namespaces) 74 | @namespaces = [] 75 | end 76 | return true 77 | end 78 | 79 | # Check if query operation is a leaf in the binary tree. 80 | # @return [Boolean] true if a leaf, false if not 81 | def is_leaf? 82 | is_a?(Redlics::Query::Operation) && @track_bits.nil? 83 | end 84 | 85 | # Singleton class 86 | class << self 87 | # Finalize query operation called from garbage collector. 88 | # 89 | # @param namespaces [Array] list of created operation keys in Redis 90 | # @return [Integer] result of Redis delete keys 91 | # @return [NilClass] nil if namespaces are empty 92 | def finalize(namespaces) 93 | proc { reset_redis_namespaces(namespaces) } 94 | end 95 | 96 | # Reset Redis created namespace keys. 97 | # 98 | # @param namespaces [Array] list of created operation keys in Redis 99 | # @return [Integer] result of Redis delete keys 100 | # @return [NilClass] nil if namespaces are empty 101 | def reset_redis_namespaces(namespaces) 102 | Redlics.redis { |r| r.del(namespaces) } if namespaces.any? 103 | end 104 | end 105 | 106 | private 107 | 108 | # Traverse query operation binary tree and calculate operation leafs. 109 | # @return [String] result operation key in Redis 110 | def traverse 111 | if @operator == 'NOT' 112 | @queries[0].traverse unless @queries[0].is_leaf? 113 | track_bits 114 | else 115 | @queries.each { |q| q.traverse unless q.is_leaf? } 116 | track_bits 117 | end 118 | end 119 | end 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /lib/redlics/time_frame.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Time Frame class 5 | class TimeFrame 6 | # Gives read access to the listed instance variables. 7 | attr_reader :from, :to, :granularity 8 | 9 | # Initialization of a time frame object. 10 | # 11 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 12 | # @param time_object [Symbol] time object predefined in Redlics::TimeFrame.init_with_symbol 13 | # @param time_object [Hash] time object with keys `from` and `to` 14 | # @param time_object [Range] time object as range 15 | # @param time_object [Time] time object 16 | # @param options [Hash] configuration options 17 | # @return [Redlics::TimeFrame] time frame object 18 | def initialize(context, time_object, options = {}) 19 | raise ArgumentError, 'TimeFrame should be initialized with Symbol, Hash, Range or Time' unless [Symbol, Hash, Range, Time].include?(time_object.class) 20 | @from, @to = self.send("init_with_#{time_object.class.name.demodulize.underscore}", time_object, context) 21 | @granularity = Granularity.validate(context, options[:granularity]).first 22 | end 23 | 24 | # Construct keys by time frame steps. 25 | # @return [Array] keys 26 | def splat 27 | [].tap do |keys| 28 | (from.to_i .. to.to_i).step(Redlics.config.granularities[@granularity][:step]) do |t| 29 | keys << (block_given? ? (yield Time.at(t)) : Time.at(t)) 30 | end 31 | end 32 | end 33 | 34 | private 35 | 36 | # Initialize time frames `from` and `to` by time. 37 | # 38 | # @param time [Time] a time 39 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 40 | # @return [Array] with `from` and `to` time 41 | def init_with_time(time, context) 42 | [time.beginning_of_day, time.end_of_day] 43 | end 44 | 45 | # Initialize time frames `from` and `to` by symbol. 46 | # 47 | # @param symbol [Symbol] a time span 48 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 49 | # @return [Array] with `from` and `to` time 50 | def init_with_symbol(symbol, context) 51 | case symbol 52 | when :hour, :day, :week, :month, :year 53 | return 1.send(symbol).ago, Time.now 54 | when :today 55 | return Time.now.beginning_of_day, Time.now 56 | when :yesterday 57 | return 1.day.ago.beginning_of_day, 1.day.ago.end_of_day 58 | when :this_week 59 | return Time.now.beginning_of_week, Time.now 60 | when :last_week 61 | return 1.week.ago.beginning_of_week, 1.week.ago.end_of_week 62 | when :this_month 63 | return Time.now.beginning_of_month, Time.now 64 | when :last_month 65 | return 1.month.ago.beginning_of_month, 1.month.ago.end_of_month 66 | when :this_year 67 | return Time.now.beginning_of_year, Time.now 68 | when :last_year 69 | return 1.year.ago.beginning_of_year, 1.year.ago.end_of_year 70 | else 71 | return default(context), Time.now 72 | end 73 | end 74 | 75 | # Initialize time frames `from` and `to` by hash. 76 | # 77 | # @param hash [Hash] a time hash with keys `from` and `to` 78 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 79 | # @return [Array] with `from` and `to` time 80 | def init_with_hash(hash, context) 81 | [ hash[:from] && hash[:from].is_a?(String) && Time.parse(hash[:from]) || hash[:from] || default(context), 82 | hash[:to] && hash[:to].is_a?(String) && Time.parse(hash[:to]) || hash[:to] || Time.now ] 83 | end 84 | 85 | # Initialize time frames `from` and `to` by hash. 86 | # 87 | # @param range [Range] a time range 88 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 89 | # @return [Array] with `from` and `to` time 90 | def init_with_range(range, context) 91 | init_with_hash({ from: range.first, to: range.last }, context) 92 | end 93 | 94 | # Get default granularity by given context. 95 | # 96 | # @param context [Hash] the hash of a context defined in Redlics::CONTEXTS 97 | # @return [ActiveSupport::TimeWithZone] a time 98 | def default(context) 99 | Redlics.config.granularities[Granularity.default(context).first][:step].ago 100 | end 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/redlics/tracker.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Redlics 4 | # Tracker class 5 | module Tracker 6 | # Context constant for given class. 7 | CONTEXT = Redlics::CONTEXTS[:tracker].freeze 8 | 9 | extend self 10 | 11 | # Track for a given event and object id with options. 12 | # 13 | # @param *args [Array] list of arguments for track 14 | # @return [Array] list of tracked granularities 15 | def track(*args, &block) 16 | return track_with_block(&block) if block_given? 17 | return track_with_hash(args.first) if args.first.is_a?(Hash) 18 | track_with_args(*args) 19 | end 20 | 21 | private 22 | 23 | # Track with hash. 24 | # 25 | # @param options [Hash] configuration options 26 | # @return [Array] list of tracked granularities 27 | def track_with_hash(options) 28 | Granularity.validate(CONTEXT, options[:granularity]).each do |granularity| 29 | key = Key.name(CONTEXT, options[:event], granularity, options[:past]) 30 | Redlics.redis do |conn| 31 | conn.pipelined do |redis| 32 | redis.setbit(key, options[:id].to_i, 1) 33 | redis.expire(key, (options[:expiration_for] && options[:expiration_for][granularity] || Redlics.config.tracker_expirations[granularity]).to_i) 34 | end 35 | end 36 | end 37 | end 38 | 39 | # Track with hash. 40 | # 41 | # @param [&Block] a block with configuration options 42 | # @return [Array] list of tracked granularities 43 | def track_with_block 44 | yield options = OpenStruct.new 45 | track_with_hash(options.to_h) 46 | end 47 | 48 | # Track with hash. 49 | # 50 | # @param *args [Array] list of arguments for track 51 | # @return [Array] list of tracked granularities 52 | def track_with_args(*args) 53 | options = args.last.instance_of?(Hash) ? args.pop : {} 54 | options.merge!({ 55 | event: args[0], 56 | id: args[1] 57 | }) 58 | track_with_hash(options) 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /lib/redlics/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Redlics version. 4 | module Redlics 5 | VERSION = '0.2.2' 6 | end 7 | -------------------------------------------------------------------------------- /redlics.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path('lib', __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require 'redlics/version' 6 | 7 | Gem::Specification.new do |spec| 8 | spec.name = 'redlics' 9 | spec.version = Redlics::VERSION 10 | spec.authors = ['Egon Zemmer'] 11 | spec.email = ['office@phlegx.com'] 12 | spec.date = Time.now.utc.strftime('%Y-%m-%d') 13 | spec.homepage = "http://github.com/phlegx/#{spec.name}" 14 | 15 | spec.summary = 'Redis analytics with tracks and counts.' 16 | spec.description = 'Redis analytics with tracks (using bitmaps) and counts (using buckets)' \ 17 | 'encoding numbers in Redis keys and values.' 18 | spec.license = 'MIT' 19 | 20 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 21 | spec.test_files = Dir.glob('test/*_test.rb') 22 | spec.require_paths = ['lib'] 23 | 24 | spec.required_ruby_version = '>= 2.3' 25 | spec.add_dependency 'activesupport', '>= 5.2.8' 26 | spec.add_dependency 'connection_pool', '~> 2.2' 27 | spec.add_dependency 'msgpack', '>= 0.7.2' 28 | spec.add_dependency 'redis', '>= 4.2' 29 | spec.add_dependency 'redis-namespace', '~> 1.5' 30 | 31 | spec.add_development_dependency 'minitest', '~> 5.8' 32 | spec.add_development_dependency 'rake', '~> 12.0' 33 | end 34 | -------------------------------------------------------------------------------- /test/redlics_test.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path(File.join(File.dirname(__FILE__), 'test_helper')) 4 | 5 | describe Redlics do 6 | subject { Redlics } 7 | 8 | it 'must respond positively' do 9 | subject.redis { |r| r.namespace }.must_equal Redlics.config.namespace 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'minitest/autorun' 4 | require 'active_support/core_ext/module/delegation' 5 | require 'active_support/time' 6 | require 'redlics' 7 | --------------------------------------------------------------------------------