├── README.md ├── Rakefile ├── VERSION ├── examples └── simple.rb ├── lib └── rrrdtool.rb └── spec └── rrrdtool_spec.rb /README.md: -------------------------------------------------------------------------------- 1 | # RRRDTool 2 | 3 | Implements a [round-robin database](http://en.wikipedia.org/wiki/RRDtool) (circular buffer) pattern on top of Redis sorted sets. Ideal for answering top/last N queries in (almost) fixed (memory) space - actual footprint depends on the number of unique keys you are tracking. Specify the period and precision (step) of each collection bucket, and RRRDTool will do the rest. 4 | 5 | Memory footprint will be limited to number of buckets * number of keys in each. New samples will be automatically placed into correct epoch/bucket. 6 | 7 | ## Store up to 5s worth of samples, in 1s buckets: 8 | rr = RRRDTool.new(:step => 1, :buckets => 5) 9 | 10 | rr.set("namespace", "key", 1) 11 | rr.incr("namespace", "key", 5) 12 | p rr.score("namespace", "key") # => 6 13 | 14 | sleep (1) 15 | 16 | rr.incr("namespace", "key") 17 | p rr.score("namespace", "key") # => 7 18 | p rr.first("namespace", 1, :with_scores => true) # => {"key"=>"7"} 19 | 20 | sleep(4) 21 | 22 | p rr.score("namespace", "key") # => 1 23 | p rr.stats("namespace") # => {:buckets=>5, :unique_keys=>1, :key_count=>{0=>0, 1=>0, 2=>0, 3=>1, 4=>0}} 24 | 25 | # find out high-to-low rank of a key across all epochs 26 | p rr.rank("namespace", "key") # => 0 27 | 28 | # License 29 | 30 | (The MIT License) 31 | 32 | Copyright (c) 2010 Ilya Grigorik 33 | 34 | Permission is hereby granted, free of charge, to any person obtaining 35 | a copy of this software and associated documentation files (the 36 | 'Software'), to deal in the Software without restriction, including 37 | without limitation the rights to use, copy, modify, merge, publish, 38 | distribute, sublicense, and/or sell copies of the Software, and to 39 | permit persons to whom the Software is furnished to do so, subject to 40 | the following conditions: 41 | 42 | The above copyright notice and this permission notice shall be 43 | included in all copies or substantial portions of the Software. 44 | 45 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 46 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 47 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 48 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 49 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 50 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 51 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "rake" 2 | 3 | begin 4 | require "jeweler" 5 | Jeweler::Tasks.new do |gemspec| 6 | gemspec.name = "rrrdtool" 7 | gemspec.summary = "Round robin database pattern via Redis sorted sets" 8 | gemspec.description = gemspec.summary 9 | gemspec.email = "ilya@igvita.com" 10 | gemspec.homepage = "http://github.com/igrigorik/rrrdtool" 11 | gemspec.authors = ["Ilya Grigorik"] 12 | gemspec.required_ruby_version = ">= 1.8" 13 | gemspec.add_dependency("redis", ">= 1.9") 14 | gemspec.rubyforge_project = "rrrdtool" 15 | end 16 | 17 | Jeweler::GemcutterTasks.new 18 | rescue LoadError 19 | puts "Jeweler not available: gem install jeweler" 20 | end -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 0.1.1 2 | -------------------------------------------------------------------------------- /examples/simple.rb: -------------------------------------------------------------------------------- 1 | require 'lib/rrrdtool' 2 | 3 | rr = RRRDTool.new(:step => 1, :buckets => 5) 4 | rr.flushdb 5 | 6 | rr.incr("namespace", "key") 7 | rr.incr("namespace", "key", 5) 8 | p rr.score("namespace", "key") # => 6 9 | 10 | sleep (1) 11 | 12 | rr.incr("namespace", "key") 13 | p rr.score("namespace", "key") # => 7 14 | p rr.first("namespace", 1, :with_scores => true) # => {"key"=>"7"} 15 | 16 | sleep(4) 17 | 18 | p rr.score("namespace", "key") # => 1 19 | p rr.stats("namespace") # => {:buckets=>5, :unique_keys=>1, :key_count=>{0=>0, 1=>0, 2=>0, 3=>1, 4=>0}} -------------------------------------------------------------------------------- /lib/rrrdtool.rb: -------------------------------------------------------------------------------- 1 | require 'redis' 2 | 3 | class RRRDTool 4 | def initialize(opts) 5 | @buckets = opts[:buckets] 6 | @step = opts[:step] 7 | @debug = opts[:debug] || false 8 | 9 | @current = nil 10 | @db = Redis.new 11 | end 12 | 13 | def time_epoch; (Time.now.to_i / @step) % @buckets; end 14 | def epochs_ago(set, num) 15 | b = time_epoch-num 16 | b = (b < 0) ? @buckets + b : b 17 | 18 | "#{set}:#{b}" 19 | end 20 | 21 | def buckets(set) 22 | (0...@buckets).inject([]) {|a,v| a.push epochs_ago(set, v) } 23 | end 24 | 25 | def epoch(set) 26 | current_epoch = time_epoch 27 | last_epoch = @db.get("#{set}:epoch").to_i 28 | now = set + ":" + current_epoch.to_s 29 | 30 | if now != @current and current_epoch != last_epoch 31 | debug [:new_epoch, current_epoch] 32 | 33 | [(Time.now.to_i / @step - last_epoch).abs, @buckets].min.times do |n| 34 | clear_bucket(epochs_ago(set, n)) 35 | end 36 | 37 | @current = now 38 | @db.set("#{set}:epoch", Time.now.to_i / @step) 39 | end 40 | 41 | @current 42 | end 43 | alias :check_epoch :epoch 44 | 45 | def union_epochs(set) 46 | check_epoch(set) 47 | 48 | debug [:union_epochs, buckets(set)] 49 | @db.zunion("#{set}:union", buckets(set)) 50 | end 51 | 52 | def score(set, key) 53 | union_epochs(set) 54 | 55 | buckets(set).each {|b| debug [b, @db.zscore(b, key)]} 56 | @db.zscore("#{set}:union", key).to_i 57 | end 58 | 59 | def incr(set, key, val=1) 60 | debug [:zincrby, epoch(set), val, key] 61 | @db.zincrby(epoch(set), val, key).to_i 62 | end 63 | 64 | def set(set, key, val) 65 | debug [:zadd, epoch(set), val, key] 66 | @db.zadd(epoch(set), val, key) 67 | end 68 | 69 | def first(set, num, options = {}) 70 | union_epochs(set) 71 | e = @db.zrevrange("#{set}:union", 0, num, options) 72 | options.key?(:with_scores) ? Hash[*e] : e 73 | end 74 | 75 | def last(set, num, options = {}) 76 | union_epochs(set) 77 | e = @db.zrange("#{set}:union", 0, num, options) 78 | options.key?(:with_scores) ? Hash[*e] : e 79 | end 80 | 81 | def rank(set, key) 82 | union_epochs(set) 83 | @db.zrevrank("#{set}:union", key) 84 | end 85 | 86 | def stats(set) 87 | stats = {} 88 | 89 | union_epochs(set) 90 | stats[:buckets] = @buckets 91 | stats[:unique_keys] = @db.zcard("#{set}:union") 92 | stats[:key_count] = (0...@buckets).inject({}) do |h,v| 93 | h[v] = @db.zcard("#{set}:#{v}") 94 | h 95 | end 96 | 97 | stats 98 | end 99 | 100 | def delete(set, key) 101 | buckets(set).each do |b| 102 | @db.zrem(b, key) 103 | end 104 | end 105 | 106 | def clear(set) 107 | buckets(set).each do |b| 108 | clear_bucket(b) 109 | end 110 | end 111 | 112 | def flushdb; @db.flushdb; end 113 | 114 | private 115 | 116 | def clear_bucket(b) 117 | debug [:clearing_epoch, b] 118 | @db.zremrangebyrank(b, 0, 2**32) 119 | end 120 | 121 | def debug(msg); p msg if @debug; end 122 | end 123 | -------------------------------------------------------------------------------- /spec/rrrdtool_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec' 2 | require 'delorean' 3 | require 'time' 4 | 5 | require 'lib/rrrdtool' 6 | 7 | describe RRRDTool do 8 | include Delorean 9 | 10 | let(:rr) { RRRDTool.new(:step => 10, :buckets => 6) } 11 | 12 | before(:each) { time_travel_to("Jan 1 2010") } 13 | before(:each) { rr.clear("test") } 14 | 15 | it "should initialize db" do 16 | lambda { 17 | RRRDTool.new(:step => 10, :buckets => 6) 18 | }.should_not raise_error 19 | end 20 | 21 | it "should return score of item" do 22 | rr.score("test", "random_key").should == 0 23 | end 24 | 25 | it "should (re)set score within correct epoch" do 26 | rr.score("test", "key").should == 0 27 | rr.set("test", "key", 5) 28 | rr.score("test", "key").should == 5 29 | 30 | rr.incr("test", "key").should == 6 31 | rr.score("test", "key").should == 6 32 | 33 | rr.set("test", "key", 5) 34 | rr.score("test", "key").should == 5 35 | end 36 | 37 | it "should increment buckets within correct epoch" do 38 | rr.epoch("test").should match(/test:0/) 39 | 40 | rr.incr("test", "key") 41 | rr.score("test", "key").should == 1 42 | 43 | rr.incr("test", "key", 2) 44 | rr.score("test", "key").should == 3 45 | 46 | # advance to next epoch 47 | time_travel_to(Time.now + 10) do 48 | rr.epoch("test").should match(/test:1/) 49 | 50 | rr.incr("test", "key") 51 | rr.score("test", "key").should == 4 52 | end 53 | 54 | # advance 5 epochs, to scroll original incr's off the list 55 | time_travel_to(Time.now + 60) do 56 | rr.epoch("test").should match(/test:0/) 57 | 58 | rr.incr("test", "key") 59 | rr.score("test", "key").should == 2 60 | end 61 | end 62 | 63 | it "should wipe skipped buckets" do 64 | rr.incr("test", "key", 1) 65 | 66 | time_travel_to(Time.now + 20) do 67 | rr.incr("test", "key", 1) 68 | 69 | # fast forward & skip to an epoch with data 70 | time_travel_to(Time.now + 50) do 71 | rr.epoch("test").should match(/test:1/) 72 | rr.score("test", "key").should == 1 73 | end 74 | 75 | # fast forward & skip to an epoch with no data 76 | time_travel_to(Time.now + 150) do 77 | rr.score("test", "key").should == 0 78 | end 79 | end 80 | end 81 | 82 | it "should return top N items from all epochs" do 83 | rr.incr("test", "key1", 1) 84 | rr.incr("test", "key2", 3) 85 | 86 | # advance to next epoch 87 | time_travel_to(Time.now + 10) do 88 | rr.epoch("test").should match(/test:1/) 89 | rr.incr("test", "key3", 5) 90 | 91 | rr.first("test", 3).should == ["key3", "key2", "key1"] 92 | rr.first("test", 3, :with_scores => true).should == {"key3"=>"5", "key2"=>"3", "key1"=>"1"} 93 | end 94 | end 95 | 96 | it "should return last N items from all epochs" do 97 | rr.incr("test", "key1", 1) 98 | rr.incr("test", "key2", 3) 99 | 100 | # advance to next epoch 101 | time_travel_to(Time.now + 10) do 102 | rr.epoch("test").should match(/test:1/) 103 | rr.incr("test", "key3", 5) 104 | 105 | rr.last("test", 3).should == ["key1", "key2", "key3"] 106 | rr.last("test", 3, :with_scores => true).should == {"key1"=>"1", "key2"=>"3", "key3"=>"5"} 107 | end 108 | end 109 | 110 | it "should erase key from all epochs" do 111 | rr.incr("test", "key", 1) 112 | rr.score("test", "key").should == 1 113 | 114 | # advance to next epoch 115 | time_travel_to(Time.now + 10) do 116 | rr.epoch("test").should match(/test:1/) 117 | rr.incr("test", "key") 118 | rr.score("test", "key").should == 2 119 | 120 | rr.delete("test", "key") 121 | rr.score("test", "key").should == 0 122 | end 123 | end 124 | 125 | it "should return rank of key across all epochs" do 126 | rr.set("test", "key1", 2) 127 | rr.set("test", "key2", 1) 128 | 129 | time_travel_to(Time.now + 10) do 130 | rr.epoch("test").should match(/test:1/) 131 | 132 | rr.set("test", "key1", 1) 133 | rr.set("test", "key2", 3) 134 | 135 | rr.rank("test", "key2").should == 0 136 | rr.rank("test", "key1").should == 1 137 | end 138 | end 139 | 140 | it "should footprint stats" do 141 | rr.incr("test", "key") 142 | rr.incr("test", "key2") 143 | 144 | time_travel_to(Time.now + 10) do 145 | rr.incr("test", "key") 146 | 147 | rr.stats("test").should == { 148 | :buckets => 6, 149 | :unique_keys => 2, 150 | :key_count => { 0 => 2, 1 => 1, 2 => 0, 3 => 0, 4 => 0, 5 => 0 } 151 | } 152 | end 153 | 154 | time_travel_to(Time.now + 60) do 155 | rr.stats("test").should == { 156 | :buckets => 6, 157 | :unique_keys => 1, 158 | :key_count => { 0 => 0, 1 => 1, 2 => 0, 3 => 0, 4 => 0, 5 => 0 } 159 | } 160 | end 161 | end 162 | 163 | end 164 | --------------------------------------------------------------------------------