├── .document ├── .gitignore ├── .rspec ├── Gemfile ├── Gemfile.lock ├── MIT-LICENSE ├── README.md ├── analytics.rb ├── lib └── redis │ ├── bitops.rb │ └── bitops │ ├── bitmap.rb │ ├── configuration.rb │ ├── queries │ ├── binary_operator.rb │ ├── lazy_evaluation.rb │ ├── materialization_helpers.rb │ ├── tree_building_helpers.rb │ └── unary_operator.rb │ └── sparse_bitmap.rb ├── redis-bitops.gemspec └── spec ├── redis └── bitops │ ├── bitmap_spec.rb │ ├── queries │ ├── binary_operator_spec.rb │ └── unary_operator_spec.rb │ └── sparse_bitmap_spec.rb ├── spec_helper.rb └── support └── bitmap_examples.rb /.document: -------------------------------------------------------------------------------- 1 | lib 2 | README.md 3 | MIT-LICENSE -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | doc/ -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | # Declare your gem's dependencies in motivoo.gemspec. 4 | # Bundler will treat runtime dependencies like base dependencies, and 5 | # development dependencies will be added by default to the :development group. 6 | gemspec 7 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | redis-bitops (0.2) 5 | hiredis 6 | redis 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | awesome_print (1.2.0) 12 | byebug (2.7.0) 13 | columnize (~> 0.3) 14 | debugger-linecache (~> 1.2) 15 | celluloid (0.15.2) 16 | timers (~> 1.1.0) 17 | coderay (1.1.0) 18 | columnize (0.8.9) 19 | debugger-linecache (1.2.0) 20 | diff-lcs (1.2.5) 21 | ffi (1.9.3) 22 | formatador (0.2.4) 23 | guard (2.6.1) 24 | formatador (>= 0.2.4) 25 | listen (~> 2.7) 26 | lumberjack (~> 1.0) 27 | pry (>= 0.9.12) 28 | thor (>= 0.18.1) 29 | guard-rspec (4.2.9) 30 | guard (~> 2.1) 31 | rspec (>= 2.14, < 4.0) 32 | hiredis (0.5.2) 33 | listen (2.7.5) 34 | celluloid (>= 0.15.2) 35 | rb-fsevent (>= 0.9.3) 36 | rb-inotify (>= 0.9) 37 | lumberjack (1.0.5) 38 | method_source (0.8.2) 39 | pry (0.9.12.6) 40 | coderay (~> 1.0) 41 | method_source (~> 0.8) 42 | slop (~> 3.4) 43 | pry-byebug (1.3.2) 44 | byebug (~> 2.7) 45 | pry (~> 0.9.12) 46 | rb-fsevent (0.9.4) 47 | rb-inotify (0.9.4) 48 | ffi (>= 0.5.0) 49 | redis (3.0.7) 50 | rspec (2.14.1) 51 | rspec-core (~> 2.14.0) 52 | rspec-expectations (~> 2.14.0) 53 | rspec-mocks (~> 2.14.0) 54 | rspec-core (2.14.8) 55 | rspec-expectations (2.14.5) 56 | diff-lcs (>= 1.1.3, < 2.0) 57 | rspec-mocks (2.14.6) 58 | slop (3.5.0) 59 | thor (0.19.1) 60 | timers (1.1.0) 61 | 62 | PLATFORMS 63 | ruby 64 | 65 | DEPENDENCIES 66 | awesome_print 67 | byebug 68 | guard-rspec 69 | pry 70 | pry-byebug 71 | redis-bitops! 72 | rspec 73 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2014 Martin Bilski 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction 2 | 3 | This gem makes it easier to do bit-wise operations on large Redis bitsets, usually called bitmaps, with a natural expression syntax. It also supports huge **sparse bitmaps** by storing data in multiple keys, called chunks, per bitmap. 4 | 5 | The typical use is real-time web analytics where each bit in a bitmap/bitset corresponds to a user ([introductory article here](http://blog.getspool.com/2011/11/29/fast-easy-realtime-metrics-using-redis-bitmaps/)). This library isn't an analytic package though, it's more low level than that and you can use it for anything. 6 | 7 | **This library is under development and its interface might change.** 8 | 9 | 10 | # Quick start/pitch 11 | 12 | Why use the library? 13 | 14 | ```ruby 15 | require 'redis/bitops' 16 | redis = Redis.new 17 | 18 | b1 = redis.sparse_bitmap("b1") 19 | b1[128_000_000] = true 20 | b2 = redis.sparse_bitmap("b2") 21 | b2[128_000_000] = true 22 | result = b1 & b2 23 | ``` 24 | 25 | Memory usage: about 20kb because it uses a sparse bitmap implementation using chunks of data. 26 | 27 | Let's go crazy with super-complicated expressions: 28 | 29 | ```ruby 30 | ... 31 | result = (b1 & ~b2) | b3 (b4 | (b5 & b6 & b7 & ~b8)) 32 | ``` 33 | 34 | Imagine writing this expression using Redis#bitop! 35 | 36 | 37 | # Installation 38 | 39 | To install the gem: 40 | 41 | gem install redis-bitops 42 | 43 | To use it in your code: 44 | 45 | require 'redis/bitops' 46 | 47 | # Usage 48 | 49 | Reference: [here](http://rdoc.info/github/bilus/redis-bitops/master/frames) 50 | 51 | ## Basic example 52 | 53 | An example is often better than theory so here's one. Let's create a few bitmaps and set their individual bits; we'll use those bitmaps in the examples below: 54 | 55 | ```ruby 56 | redis = Redis.new 57 | 58 | a = redis.bitmap("a") 59 | b = redis.bitmap("b") 60 | result = redis.bitmap("result") 61 | 62 | b[0] = true; b[2] = true; b[7] = true # 10100001 63 | a[0] = true; a[1] = true; a[7] = true # 11000001 64 | ``` 65 | 66 | So, now here's a very simple expression: 67 | 68 | ```ruby 69 | c = a & b 70 | ``` 71 | 72 | You may be surprised but the above statement does not query Redis at all! The expression is lazy-evaluated when you access the result: 73 | 74 | ```ruby 75 | puts c.bitcount # => 2 76 | puts c[0] # => true 77 | puts c[1] # => false 78 | puts c[2] # => false 79 | puts c[7] # => false 80 | ``` 81 | 82 | So, in the above example, the call to `c.bitcount` happens to be the first moment when Redis is queried. The result is stored under a temporary unique key. 83 | 84 | ```ruby 85 | puts c.root_key # => "redis:bitops:8eef38u9o09334" 86 | ``` 87 | 88 | Let's delete the temporary result: 89 | 90 | ```ruby 91 | c.delete! 92 | ``` 93 | 94 | If you want to store the result directly under a specific key: 95 | 96 | ```ruby 97 | result << c 98 | ``` 99 | 100 | Or, more adventurously, we can use the following more complex one-liner: 101 | 102 | ```ruby 103 | result << (~c & (a | b)) 104 | ``` 105 | 106 | **Note: ** expressions are optimized by reducing the number of Redis commands and using as few temporary keys to hold intermediate values as possible. See below for details. 107 | 108 | 109 | ## Sparse bitmaps 110 | 111 | ### Usage 112 | 113 | You don't have to do anything special, simply use `Redis#sparse_bitmap` instead of `Redis#bitmap`: 114 | 115 | ```ruby 116 | a = redis.sparse_bitmap("a") 117 | b = redis.sparse_bitmap("b") 118 | result = redis.sparse_bitmap("result") 119 | 120 | b[0] = true; b[2] = true; b[7] = true # 10100001 121 | a[0] = true; a[1] = true; a[7] = true # 11000001 122 | 123 | c = a & b 124 | 125 | result << c 126 | ``` 127 | 128 | or just: 129 | 130 | ```ruby 131 | result << (a & b) 132 | ``` 133 | 134 | You can specify the chunk size (in bytes). 135 | 136 | Use the size consistently. Note that it cannot be re-adjusted for data already saved to Redis: 137 | 138 | ```ruby 139 | x = redis.sparse_bitmap("x", 1024 * 1024) # 1 MB per chunk. 140 | x[0] = true 141 | x[1000] = true 142 | ``` 143 | 144 | **Important:** Do not mix sparse bitmaps with regular ones and never mix sparse bitmaps with different chunk sizes in the same expressions. 145 | 146 | ### Rationale 147 | 148 | If you want to store a lot of huge but sparse bitsets, with not many bits set, using regular Redis bitmaps doesn't work very well. It wastes a lot of space. In analytics, it's a reasonable requirement, to be able to store data about several million users. A bitmap for 10 million users weights over 1MB! Imagine storing hourly statistics and using up memory at a rate of 720MB per month. 149 | 150 | For, say, 100 million users it becomes outright prohibitive! 151 | 152 | But even with a fairly popular websites, I dare say, you don't often have one million users per hour :) This means that the majority of those bits is never sets and a lot of space goes wasted. 153 | 154 | Enter sparse bitmaps. They divide each bitmap into chunks thus minimizing memory use (chunks' size can be configured, see Configuration below). 155 | 156 | Creating and using sparse bitmaps is identical to using regular bitmaps: 157 | 158 | ```ruby 159 | huge = redis.sparse_bitmap("huge_bitmap") 160 | huge[128_000_000] = true 161 | ``` 162 | 163 | The only difference in the above example is that it will allocate two 32kb chunks as opposed to 1MB that would be allocated if we used a regular bitmap (Redis#bitmap). In addition, setting the bit is nearly instantaneous. 164 | 165 | Compare: 166 | 167 | ```ruby 168 | puts Benchmark.measure { 169 | sparse = redis.sparse_bitmap("huge_sparse_bitmap") 170 | sparse[500_000_000] = true 171 | } 172 | ``` 173 | 174 | which on my machine this generates: 175 | 176 | 0.000000 0.000000 0.000000 ( 0.000366) 177 | 178 | It uses just 23kb memory as opposed to 120MB (megabytes!) to store the bit using a regular Redis bitmap: 179 | 180 | ```ruby 181 | regular = redis.bitmap("huge_regular_bitmap") 182 | regular[500_000_000] = true 183 | ``` 184 | 185 | ## Configuration 186 | 187 | Here's how to configure the gem: 188 | 189 | ```ruby 190 | Redis::Bitops.configure do |config| 191 | config.default_bytes_per_chunk = 8096 # Eight kilobytes. 192 | config.transaction_level = :bitmap # allowed values: :bitmap or :none. 193 | end 194 | ``` 195 | 196 | # Implementation & efficiency 197 | 198 | ## Optimization phase 199 | 200 | Prior to evaluation, the expression is optimized by combining operators into single BITOP commands and reusing temporary keys (required to store intermediate results) as much as possible. 201 | 202 | This silly example: 203 | 204 | ```ruby 205 | result << (a & b & c | a | b) 206 | ``` 207 | 208 | translates into simply: 209 | 210 | BITOP AND result a b c 211 | BITOP OR result result a b 212 | 213 | and doesn't create any temporary keys at all! 214 | 215 | ## Materialization phase 216 | 217 | At this point, the calculations are carried out and the result is saved under the destination key. Note that, for sparse bitmaps, multiple keys may be created. 218 | 219 | 220 | ## Transaction levels 221 | 222 | TBD 223 | 224 | 225 | ## Contributing/feedback 226 | 227 | Please send in your suggestions to [gyamtso@gmail.com](mailto:gyamtso@gmail.com). Pull requests, issues, comments are more than welcome. 228 | -------------------------------------------------------------------------------- /analytics.rb: -------------------------------------------------------------------------------- 1 | require 'redis' 2 | require 'redis/bitops' 3 | 4 | # #track needs user_id. 5 | 6 | Connection.new 7 | 8 | # Connection to a MongoDB database storing Motivoo data. 9 | # 10 | class Connection 11 | include Singleton 12 | 13 | # Creates a new connection. 14 | # 15 | def initialize 16 | # config = Motivoo.configuration 17 | # db = Mongo::Connection.new(config.mongo_host, config.mongo_port)[config.mongo_db] 18 | @db = Redis.new 19 | end 20 | 21 | # Event tracking 22 | 23 | # Tracks an event. 24 | # 25 | def track(category, event, cohort_category, cohort, date_override = nil) 26 | 27 | # ap(track: {category: category, event: event, cohort_category: cohort_category, cohort: cohort}) 28 | # When changing the query, revise Connection#create_indices. 29 | day = (date_override || Date.today).strftime("%Y-%m-%d") 30 | totals = @db.sparse_bitmap("#{category}:#{event}:#{cohort_category}:#{cohort}") 31 | totals[user_id] = true 32 | 33 | @tracking.update({category: category, event: event, cohort_category: cohort_category, cohort: cohort}, {"$inc" => {count: 1, "daily.#{day}" => 1}}, upsert: true) 34 | end 35 | 36 | # Finds an event. 37 | # 38 | def find(category, event, cohort_category, options = {}) 39 | # ap(find: {category: category, event: event, cohort_category: cohort_category}) 40 | # When changing the query, revise Connection#create_indices. It's less important than Connection#track because only Report uses it. 41 | 42 | source = options[:source] || "count" 43 | @tracking.find(category: category, event: event, cohort_category: cohort_category).to_a.inject({}) { |a, r| a.merge!(r["cohort"] => r[source.to_s]) } 44 | end 45 | # 46 | # # Lists events for a category. 47 | # # 48 | # def events_for(category) 49 | # @tracking.distinct(:event, category: category).map(&:to_sym) 50 | # end 51 | # 52 | # # Lists cohorts in a cohort category. 53 | # # 54 | # def cohorts_in(cohort_category) 55 | # @tracking.distinct(:cohort, cohort_category: cohort_category) 56 | # end 57 | # 58 | # # Lists all cohort categories. 59 | # # 60 | # def cohort_categories 61 | # @tracking.distinct(:cohort_category).map(&:to_sym) 62 | # end 63 | # 64 | # # User data 65 | # 66 | # # Finds user data given an internal user id and if it's not there, creates it. 67 | # # Returns a hash containing user data without the primary key itself (user id). 68 | # # 69 | # def find_or_create_user_data(user_id) 70 | # # ap(find_or_create_user_data: {user_id: user_id}) 71 | # # NOTE: The code below is designed to avoid a race condition. 72 | # q = {"_id" => BSON::ObjectId(user_id)} 73 | # user_data = @user_data.find_one(q) 74 | # if user_data 75 | # reject_record_id(user_data) 76 | # else 77 | # begin 78 | # # Regardless whether we or some other process did the actual insert, we return an empty hash, as the record is fresh. 79 | # @user_data.insert(q) 80 | # {} 81 | # rescue Mongo::OperationFailure => e 82 | # raise unless e.error_code == 11000 # duplicate key error index 83 | # # Another process/thread has just inserted the record. 84 | # {} 85 | # end 86 | # end 87 | # end 88 | # 89 | # # Finds user data by an external user id. 90 | # # Returns ar array consisting of a primary key (user id) and a hash containing user data or nil if record not found. 91 | # # TODO: Change it to find_user_data(query_hash), in this specific case find_user_data("ext_user_id" => ext_user_id) so UserData completely encapsulates the concept. 92 | # # 93 | # def find_user_data_by_ext_user_id(ext_user_id) 94 | # # ap(find_user_data_by_ext_user_id: {ext_user_id: ext_user_id}) 95 | # if existing_record = @user_data.find_one("ext_user_id" => ext_user_id) 96 | # [existing_record["_id"].to_s, reject_record_id(existing_record)] 97 | # end 98 | # end 99 | # 100 | # # Generates a new unique user id. 101 | # # It inserts a new record to make it possible to update it using assign_cohort, set_user_data without worrying about the record's existence but technically this isn't necessary. 102 | # # 103 | # def generate_user_id 104 | # # ap(generate_user_id: {}) 105 | # @user_data.insert({}).to_s 106 | # end 107 | # 108 | # # Inserts a user data object, making sure first_visit_at is set to the provided time. 109 | # # 110 | # def create_user_data_with_first_visit_at(first_visit_at) 111 | # # ap(create_user_data_with_first_visit_at: {first_visit_at: first_visit_at}) 112 | # user_id = BSON::ObjectId.from_time(first_visit_at).to_s 113 | # find_or_create_user_data(user_id) 114 | # user_id 115 | # end 116 | # 117 | # # Assigns a user to a cohort. 118 | # # 119 | # def assign_cohort(user_id, cohort_category, cohort) 120 | # # ap(assign_cohort: {user_id: user_id, cohort_category: cohort_category, cohort: cohort}) 121 | # @user_data.update({"_id" => BSON::ObjectId(user_id)}, "$set" => {"cohorts.#{cohort_category}" => cohort}) 122 | # end 123 | # 124 | # # Sets a user-defined user data field. 125 | # # 126 | # def set_user_data(user_id, hash) 127 | # # ap(set_user_data: {user_id: user_id, hash: hash}) 128 | # @user_data.update({"_id" => BSON::ObjectId(user_id)}, "$set" => hash) 129 | # end 130 | # 131 | # # Returns a user-defined user data field or nil if record not found. 132 | # # 133 | # def get_user_data(user_id, key) 134 | # # ap(get_user_data: {user_id: user_id, key: key}) 135 | # record = @user_data.find_one({"_id" => BSON::ObjectId(user_id)}, fields: {key => 1}) 136 | # if record 137 | # record[key] 138 | # else 139 | # nil 140 | # end 141 | # end 142 | # 143 | # # Sets user data fields if the key field is nil. 144 | # # 145 | # def set_user_data_unless_exists(user_id, key, hash) 146 | # doc = @user_data.find_and_modify(query: {"_id" => BSON::ObjectId(user_id), key => nil}, update: {"$set" => hash}) 147 | # # ap(doc: doc, user_id: user_id, key: key) 148 | # # binding.pry 149 | # !doc.nil? 150 | # end 151 | # 152 | # # Removes user data from the database. 153 | # # 154 | # def destroy_user_data(user_id) 155 | # # ap(destroy_user_data: {user_id: user_id}) 156 | # @user_data.remove("_id" => BSON::ObjectId(user_id)) 157 | # end 158 | # 159 | # # Time of the user record creation time. 160 | # # 161 | # def user_data_created_at(user_id) 162 | # BSON::ObjectId(user_id).generation_time 163 | # end 164 | # 165 | # 166 | # # Testing 167 | # 168 | # # Removes all tables and indices. 169 | # # 170 | # def clear! 171 | # @tracking.drop 172 | # @user_data.drop 173 | # create_indices 174 | # end 175 | end 176 | -------------------------------------------------------------------------------- /lib/redis/bitops.rb: -------------------------------------------------------------------------------- 1 | require 'redis' 2 | require 'redis/bitops/queries/materialization_helpers' 3 | require 'redis/bitops/queries/tree_building_helpers' 4 | require 'redis/bitops/queries/lazy_evaluation' 5 | require 'redis/bitops/queries/binary_operator' 6 | require 'redis/bitops/queries/unary_operator' 7 | require 'redis/bitops/bitmap' 8 | require 'redis/bitops/sparse_bitmap' 9 | 10 | require 'redis/bitops/configuration' 11 | 12 | 13 | class Redis 14 | 15 | # Creates a new bitmap. 16 | # 17 | def bitmap(key) 18 | Bitops::Bitmap.new(key, self) 19 | end 20 | 21 | # Creates a new sparse bitmap storing data in n chunks to conserve memory. 22 | # 23 | def sparse_bitmap(key, bytes_per_chunk = nil) 24 | Bitops::SparseBitmap.new(key, self, bytes_per_chunk) 25 | end 26 | end -------------------------------------------------------------------------------- /lib/redis/bitops/bitmap.rb: -------------------------------------------------------------------------------- 1 | class Redis 2 | module Bitops 3 | 4 | # A sparse bitmap using multiple key to store its data to save memory. 5 | # 6 | # Note: When adding new public methods, revise the LazyEvaluation module. 7 | # 8 | class Bitmap 9 | 10 | include Queries 11 | include TreeBuildingHelpers # See for a list of supported operators. 12 | 13 | # Creates a new regular Redis bitmap stored in 'redis' under 'root_key'. 14 | # 15 | def initialize(root_key, redis) 16 | @redis = redis 17 | @root_key = root_key 18 | end 19 | 20 | # Saves the result of the query in the bitmap. 21 | # 22 | def << (query) 23 | query.evaluate(self) 24 | end 25 | 26 | # Reads bit at position 'pos' returning a boolean. 27 | # 28 | def [] (pos) 29 | i2b(@redis.getbit(key(pos), offset(pos))) 30 | end 31 | 32 | # Sets bit at position 'pos' to 1 or 0 based on the boolean 'b'. 33 | # 34 | def []= (pos, b) 35 | @redis.setbit(key(pos), offset(pos), b2i(b)) 36 | end 37 | 38 | # Returns the number of set bits. 39 | # 40 | def bitcount 41 | @redis.bitcount(@root_key) 42 | end 43 | 44 | # Deletes the bitmap and all its keys. 45 | # 46 | def delete! 47 | @redis.del(@root_key) 48 | end 49 | 50 | # Redis BITOP operator 'op' (one of :and, :or, :xor or :not) on operands 51 | # (bitmaps). The result is stored in 'result'. 52 | # 53 | def bitop(op, *operands, result) 54 | @redis.bitop(op, result.root_key, self.root_key, *operands.map(&:root_key)) 55 | result 56 | end 57 | 58 | # The key the bitmap is stored under. 59 | # 60 | def root_key 61 | @root_key 62 | end 63 | 64 | # Returns lambda creating Bitmap objects using @redis as the connection. 65 | # 66 | def bitmap_factory 67 | lambda { |key| @redis.bitmap(key) } 68 | end 69 | 70 | # Copy this bitmap to 'dest' bitmap. 71 | # 72 | def copy_to(dest) 73 | copy(root_key, dest.root_key) 74 | end 75 | 76 | protected 77 | 78 | def key(pos) 79 | @root_key 80 | end 81 | 82 | def offset(pos) 83 | pos 84 | end 85 | 86 | def b2i(b) 87 | b ? 1 : 0 88 | end 89 | 90 | def i2b(i) 91 | i.to_i != 0 ? true : false 92 | end 93 | 94 | COPY_SCRIPT = 95 | <<-EOS 96 | redis.call("DEL", KEYS[2]) 97 | if redis.call("EXISTS", KEYS[1]) == 1 then 98 | local val = redis.call("DUMP", KEYS[1]) 99 | redis.call("RESTORE", KEYS[2], 0, val) 100 | end 101 | EOS 102 | def copy(source_key, dest_key) 103 | @redis.eval(COPY_SCRIPT, [source_key, dest_key]) 104 | end 105 | end 106 | end 107 | end 108 | -------------------------------------------------------------------------------- /lib/redis/bitops/configuration.rb: -------------------------------------------------------------------------------- 1 | class Redis 2 | module Bitops 3 | 4 | # Configurable settings. 5 | # 6 | class Configuration 7 | 8 | # Number of bytes per one sparse bitmap chunk. 9 | # 10 | attr_accessor :default_bytes_per_chunk 11 | 12 | # Granulatity of MULTI transactions. Currently supported values are :bitmap and nil. 13 | # 14 | attr_accessor :transaction_level 15 | 16 | def initialize 17 | reset! 18 | end 19 | 20 | def reset! 21 | @default_bytes_per_chunk = 32 * 1024 22 | @transaction_level = :bitmap 23 | end 24 | end 25 | 26 | extend self 27 | attr_accessor :configuration 28 | 29 | # Call this method to modify defaults in your initializers. 30 | # 31 | def configure 32 | self.configuration ||= Configuration.new 33 | yield(configuration) 34 | end 35 | end 36 | 37 | Bitops.configure {} 38 | end 39 | -------------------------------------------------------------------------------- /lib/redis/bitops/queries/binary_operator.rb: -------------------------------------------------------------------------------- 1 | require 'securerandom' 2 | 3 | class Redis 4 | module Bitops 5 | module Queries 6 | 7 | # Binary bitwise operator. 8 | # 9 | class BinaryOperator 10 | include MaterializationHelpers 11 | include TreeBuildingHelpers 12 | include LazyEvaluation 13 | 14 | # Creates a bitwise operator 'op' with left-hand operand, 'lhs', and right-hand operand, 'rhs'. 15 | # 16 | def initialize(op, lhs, rhs) 17 | @args = [lhs, rhs] 18 | @op = op 19 | end 20 | 21 | # Runs the expression tree against the redis database, saving the results 22 | # in bitmap 'dest'. 23 | # 24 | def materialize(dest) 25 | # Resolve lhs and rhs operand, using 'dest' to store intermediate result so 26 | # a maximum of one temporary Bitmap has to be created. 27 | # Then apply the bitwise operator storing the final result in 'dest'. 28 | 29 | intermediate = dest 30 | 31 | lhs, *other_args = @args 32 | temp_intermediates = [] 33 | 34 | # Side-effects: if a temp intermediate bitmap is created, it's added to 'temp_intermediates' 35 | # to be deleted in the "ensure" block. Marked with "<- SE". 36 | 37 | lhs_operand, intermediate = resolve_operand(lhs, intermediate, temp_intermediates) # <- SE 38 | other_operands, *_ = other_args.inject([[], intermediate]) do |(operands, intermediate), arg| 39 | operand, intermediate = resolve_operand(arg, intermediate, temp_intermediates) # <- SE 40 | [operands << operand, intermediate] 41 | end 42 | 43 | lhs_operand.bitop(@op, *other_operands, dest) 44 | ensure 45 | temp_intermediates.each(&:delete!) 46 | end 47 | 48 | # Recursively optimizes the expression tree by combining operands for neighboring identical 49 | # operators, so for instance a & b & c ultimately becomes BITOP :and dest a b c as opposed 50 | # to running two separate BITOP commands. 51 | # 52 | def optimize!(parent_op = nil) 53 | @args.map! { |arg| arg.respond_to?(:optimize!) ? arg.optimize!(@op) : arg }.flatten! 54 | if parent_op == @op 55 | @args 56 | else 57 | self 58 | end 59 | end 60 | 61 | # Finds the first bitmap factory in the expression tree. 62 | # Required by LazyEvaluation and MaterializationHelpers. 63 | # 64 | def bitmap_factory 65 | arg = @args.find { |arg| arg.bitmap_factory } or raise "Internal error. Cannot find a bitmap factory." 66 | arg.bitmap_factory 67 | end 68 | end 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/redis/bitops/queries/lazy_evaluation.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | 3 | class Redis 4 | module Bitops 5 | module Queries 6 | 7 | # Support for materializing expressions when one of the supported bitmap methods is called. 8 | # 9 | # Example 10 | # 11 | # a = $redis.sparse_bitmap("a") 12 | # b = $redis.sparse_bitmap("b") 13 | # a[0] = true 14 | # result = a | b 15 | # puts result[0] => true 16 | # 17 | module LazyEvaluation 18 | extend Forwardable 19 | 20 | def_delegators :dest, :bitcount, :[], :[]=, :<<, :delete!, :root_key 21 | 22 | def dest 23 | if @dest.nil? 24 | @dest = temp_bitmap 25 | do_evaluate(@dest) 26 | end 27 | @dest 28 | end 29 | 30 | # Optimizes the expression and materializes it into bitmap 'dest'. 31 | # 32 | def evaluate(dest_bitmap) 33 | if @dest 34 | @dest.copy_to(dest_bitmap) 35 | else 36 | do_evaluate(dest_bitmap) 37 | end 38 | end 39 | 40 | protected def do_evaluate(dest_bitmap) 41 | optimize! 42 | materialize(dest_bitmap) 43 | end 44 | end 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/redis/bitops/queries/materialization_helpers.rb: -------------------------------------------------------------------------------- 1 | class Redis 2 | module Bitops 3 | module Queries 4 | 5 | # Helpers for materialization, i.e. running a BITOP command or, possibly, another command 6 | # and saving its results to a Redis database in 'intermediate'. 7 | # 8 | module MaterializationHelpers 9 | 10 | # Materializes the operand 'o' saving the result in 'redis'. 11 | # If the operand can be materialized, it does so storing the result in 'intermediate' 12 | # unless the latter is nil. In that case, a temp intermediate bitmap is created to hold 13 | # the result (and the bitmap is added to 'temp_intermediates'). 14 | # 15 | def resolve_operand(o, intermediate, temp_intermediates) 16 | if o.respond_to?(:materialize) 17 | if intermediate.nil? 18 | new_intermediate = temp_bitmap 19 | temp_intermediates << new_intermediate 20 | end 21 | intermediate ||= new_intermediate 22 | o.materialize(intermediate) 23 | [intermediate, nil] 24 | else 25 | [o, intermediate] 26 | end 27 | end 28 | 29 | # Creates a temp bitmap. 30 | # 31 | def temp_bitmap 32 | bitmap = bitmap_factory.call(unique_key) 33 | bitmap 34 | end 35 | 36 | # Generates a random unique key. 37 | # 38 | # TODO: The key _should_ be unique and not repeat in the 39 | # database but this isn't guaranteed. Considering the intended usage though 40 | # (creation of temporary intermediate bitmaps while materializing 41 | # queries), it should be sufficient. 42 | # 43 | def unique_key 44 | "redis:bitops:#{SecureRandom.hex(20)}" 45 | end 46 | 47 | def bitmap_factory 48 | raise "Override in the class using the module to return the bitmap factory." 49 | end 50 | end 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/redis/bitops/queries/tree_building_helpers.rb: -------------------------------------------------------------------------------- 1 | class Redis 2 | module Bitops 3 | module Queries 4 | # Helpers for expression tree building. 5 | # 6 | # Add new logical operators here as/if they become supported by Redis' BITOP command. 7 | # 8 | module TreeBuildingHelpers 9 | 10 | # Logical AND operator. 11 | # 12 | def & (rhs) 13 | BinaryOperator.new(:and, self, rhs) 14 | end 15 | 16 | # Logical OR operator. 17 | # 18 | def | (rhs) 19 | BinaryOperator.new(:or, self, rhs) 20 | end 21 | 22 | # Logical XOR operator. 23 | # 24 | def ^ (rhs) 25 | BinaryOperator.new(:xor, self, rhs) 26 | end 27 | 28 | # Logical NOT operator. 29 | # 30 | # IMPORTANT: It inverts bits padding with zeroes till the nearest byte boundary. 31 | # It means that setting the left-most bit to 1 and inverting will result in 01111111 not 0. 32 | # 33 | # Corresponding Redis commands: 34 | # 35 | # SETBIT "a" 0 1 36 | # BITOP NOT "b" "a" 37 | # BITCOUNT "b" 38 | # => (integer) 7 39 | # 40 | def ~ 41 | UnaryOperator.new(:not, self) 42 | end 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/redis/bitops/queries/unary_operator.rb: -------------------------------------------------------------------------------- 1 | class Redis 2 | module Bitops 3 | module Queries 4 | 5 | # A unary bitwise operator. Currently only NOT is supported by redis. 6 | # 7 | class UnaryOperator 8 | include MaterializationHelpers 9 | include TreeBuildingHelpers 10 | include LazyEvaluation 11 | 12 | # Create a new bitwise operator 'op' with one argument 'arg'. 13 | # 14 | def initialize(op, arg) 15 | @op = op 16 | @arg = arg 17 | end 18 | 19 | # Runs the expression tree against the redis database, saving the results 20 | # in bitmap 'dest'. 21 | # 22 | def materialize(dest) 23 | temp_intermediates = [] 24 | result, = resolve_operand(@arg, dest, temp_intermediates) 25 | result.bitop(@op, dest) 26 | ensure 27 | temp_intermediates.each(&:delete!) 28 | end 29 | 30 | # Optimizes the expression tree by combining operands for neighboring identical operators. 31 | # Because NOT operator in Redis can only accept one operand, no optimization is made 32 | # for the operand but the children are optimized recursively. 33 | # 34 | def optimize!(parent_op = nil) 35 | @arg.optimize!(@op) if @arg.respond_to?(:optimize!) 36 | self 37 | end 38 | 39 | # Finds the first bitmap factory in the expression tree. 40 | # Required by LazyEvaluation and MaterializationHelpers. 41 | # 42 | def bitmap_factory 43 | @arg.bitmap_factory or raise "Internal error. Cannot get redis connection." 44 | end 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/redis/bitops/sparse_bitmap.rb: -------------------------------------------------------------------------------- 1 | require 'set' 2 | 3 | class Redis 4 | module Bitops 5 | 6 | # A sparse bitmap using multiple key to store its data to save memory. 7 | # 8 | # Note: When adding new public methods, revise the LazyEvaluation module. 9 | # 10 | class SparseBitmap < Bitmap 11 | 12 | # Creates a new sparse bitmap stored in 'redis' under 'root_key'. 13 | # 14 | def initialize(root_key, redis, bytes_per_chunk = nil) 15 | @bytes_per_chunk = bytes_per_chunk || Redis::Bitops.configuration.default_bytes_per_chunk 16 | super(root_key, redis) 17 | end 18 | 19 | # Returns the number of set bits. 20 | # 21 | def bitcount 22 | chunk_keys.map { |key| @redis.bitcount(key) }.reduce(:+) || 0 23 | end 24 | 25 | # Deletes the bitmap and all its keys. 26 | # 27 | def delete! 28 | chunk_keys.each do |key| 29 | @redis.del(key) 30 | end 31 | super 32 | end 33 | 34 | # Redis BITOP operator 'op' (one of :and, :or, :xor or :not) on operands 35 | # (bitmaps). The result is stored in 'result'. 36 | # 37 | def bitop(op, *operands, result) 38 | # TODO: Optimization is possible for AND. We can use an intersection of each operand 39 | # chunk numbers to minimize the number of database accesses. 40 | 41 | all_keys = self.chunk_keys + (operands.map(&:chunk_keys).flatten! || []) 42 | unique_chunk_numbers = Set.new(chunk_numbers(all_keys)) 43 | 44 | maybe_multi(level: :bitmap, watch: all_keys) do 45 | unique_chunk_numbers.each do |i| 46 | @redis.bitop(op, result.chunk_key(i), self.chunk_key(i), *operands.map { |o| o.chunk_key(i) }) 47 | end 48 | end 49 | result 50 | end 51 | 52 | def chunk_keys 53 | @redis.keys("#{@root_key}:chunk:*") 54 | end 55 | 56 | def chunk_key(i) 57 | "#{@root_key}:chunk:#{i}" 58 | end 59 | 60 | # Returns lambda creating SparseBitmap objects using @redis as the connection. 61 | # 62 | def bitmap_factory 63 | lambda { |key| @redis.sparse_bitmap(key, @bytes_per_chunk) } 64 | end 65 | 66 | # Copy this bitmap to 'dest' bitmap. 67 | # 68 | def copy_to(dest) 69 | 70 | # Copies all source chunks to destination chunks and deletes remaining destination chunk keys. 71 | 72 | source_keys = self.chunk_keys 73 | dest_keys = dest.chunk_keys 74 | 75 | maybe_multi(level: :bitmap, watch: source_keys + dest_keys) do 76 | source_chunks = Set.new(chunk_numbers(source_keys)) 77 | source_chunks.each do |i| 78 | copy(chunk_key(i), dest.chunk_key(i)) 79 | end 80 | dest_chunks = Set.new(chunk_numbers(dest_keys)) 81 | (dest_chunks - source_chunks).each do |i| 82 | @redis.del(dest.chunk_key(i)) 83 | end 84 | end 85 | end 86 | 87 | protected 88 | 89 | def bits_per_chunk 90 | @bytes_per_chunk * 8 91 | end 92 | 93 | def key(pos) 94 | chunk_key(chunk_number(pos)) 95 | end 96 | 97 | def offset(pos) 98 | pos.modulo bits_per_chunk 99 | end 100 | 101 | def chunk_number(pos) 102 | (pos / bits_per_chunk).to_i 103 | end 104 | 105 | def chunk_numbers(keys) 106 | keys.map { |key| key.split(":").last.to_i } 107 | end 108 | 109 | # Maybe pipeline/make atomic based on the configuration. 110 | # 111 | def maybe_multi(options = {}, &block) 112 | current_level = options[:level] or raise "Specify the current transaction level." 113 | 114 | if Redis::Bitops.configuration.transaction_level == current_level 115 | watched_keys = options[:watch] 116 | @redis.watch(watched_keys) if watched_keys && watched_keys != [] 117 | @redis.multi(&block) 118 | else 119 | block.call 120 | end 121 | end 122 | 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /redis-bitops.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = 'redis-bitops' 3 | s.version = '0.2.1' 4 | s.summary = "Bitmap and sparse bitmap operations for Redis." 5 | s.description = "Optimized operations on Redis bitmaps using built-in Ruby operators. Supports sparse bitmaps to preserve storage." 6 | s.authors = ["Martin Bilski"] 7 | s.email = 'gyamtso@gmail.com' 8 | s.homepage = 9 | 'http://github.com/bilus/redis-bitops' 10 | s.license = 'MIT' 11 | 12 | s.files = Dir['README.md', 'MIT-LICENSE', 'lib/**/*', 'spec/**/*'] 13 | 14 | s.add_dependency 'redis' 15 | 16 | s.add_development_dependency 'rspec' 17 | s.add_development_dependency 'awesome_print' 18 | s.add_development_dependency 'guard-rspec' 19 | s.add_development_dependency 'pry' 20 | s.add_development_dependency 'byebug' 21 | s.add_development_dependency 'pry-byebug' 22 | 23 | s.require_path = 'lib' 24 | end 25 | -------------------------------------------------------------------------------- /spec/redis/bitops/bitmap_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Redis::Bitops::Bitmap, redis_cleanup: true, redis_key_prefix: "rsb:" do 4 | it_should_behave_like "a bitmap", :bitmap 5 | end 6 | 7 | describe "Redis#bitmap" do 8 | it_should_behave_like "a bitmap factory method", :bitmap, Redis::Bitops::Bitmap 9 | end 10 | -------------------------------------------------------------------------------- /spec/redis/bitops/queries/binary_operator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Redis::Bitops::Queries::BinaryOperator do 4 | let(:a) { double("a") } 5 | let(:b) { double("b") } 6 | let(:c) { double("c") } 7 | let(:d) { double("d") } 8 | let(:e) { double("e") } 9 | 10 | let(:redis) { double("redis", del: nil) } 11 | let(:result) { double("output", redis: redis) } 12 | 13 | it "optimizes the expression tree" do 14 | a.should_receive(:bitop).with(:and, result, d, e, result) 15 | b.should_receive(:bitop).with(:or, c, result) 16 | expr = Redis::Bitops::Queries::BinaryOperator.new(:and, 17 | a, 18 | Redis::Bitops::Queries::BinaryOperator.new(:and, 19 | Redis::Bitops::Queries::BinaryOperator.new(:or, b, c), 20 | Redis::Bitops::Queries::BinaryOperator.new(:and, d, e))) 21 | expr.optimize! 22 | expr.materialize(result) 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /spec/redis/bitops/queries/unary_operator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Redis::Bitops::Queries::UnaryOperator do 4 | let(:a) { double("a") } 5 | let(:b) { double("b") } 6 | let(:c) { double("c") } 7 | let(:d) { double("d") } 8 | let(:e) { double("e") } 9 | 10 | let(:redis) { double("redis", del: nil) } 11 | let(:result) { double("result", redis: redis) } 12 | 13 | it "optimizes the expression tree" do 14 | a.should_receive(:bitop).with(:and, result, d, e, result) 15 | b.should_receive(:bitop).with(:or, c, result) 16 | result.should_receive(:bitop).with(:not, result) 17 | expr = 18 | Redis::Bitops::Queries::UnaryOperator.new(:not, 19 | Redis::Bitops::Queries::BinaryOperator.new(:and, 20 | a, 21 | Redis::Bitops::Queries::BinaryOperator.new(:and, 22 | Redis::Bitops::Queries::BinaryOperator.new(:or, b, c), 23 | Redis::Bitops::Queries::BinaryOperator.new(:and, d, e)))) 24 | expr.optimize! 25 | expr.materialize(result) 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /spec/redis/bitops/sparse_bitmap_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Redis::Bitops::SparseBitmap, redis_cleanup: true, redis_key_prefix: "rsb:" do 4 | it_should_behave_like "a bitmap", :sparse_bitmap 5 | 6 | let(:redis) { Redis.new } 7 | let(:bytes_per_chunk) { 4 } 8 | let(:bits_per_chunk) { bytes_per_chunk * 8 } 9 | let(:a) { redis.sparse_bitmap("rsb:a", bytes_per_chunk) } 10 | let(:b) { redis.sparse_bitmap("rsb:b", bytes_per_chunk) } 11 | let(:c) { redis.sparse_bitmap("rsb:c", bytes_per_chunk) } 12 | let(:empty) { redis.sparse_bitmap("rsb:empty", bytes_per_chunk) } 13 | let(:result) { redis.sparse_bitmap("rsb:output", bytes_per_chunk) } 14 | 15 | describe "edge cases" do 16 | before do 17 | a[0] = true 18 | a[bits_per_chunk - 1] = true 19 | a[bits_per_chunk] = true 20 | a[2 * bits_per_chunk - 1] = true 21 | a[2 * bits_per_chunk] = true 22 | 23 | b[0] = true 24 | b[bits_per_chunk] = true 25 | b[2 * bits_per_chunk - 1] = true 26 | end 27 | 28 | describe "#[]" do 29 | it "handles bits arround chunk boundaries" do 30 | a.chunk_keys.should have(3).items 31 | set_bits = 32 | (0..(3 * bits_per_chunk)).inject([]) { |acc, i| 33 | acc << i if a[i] 34 | acc 35 | } 36 | set_bits.should match_array([ 37 | 0, 38 | bits_per_chunk - 1, 39 | bits_per_chunk, 40 | 2 * bits_per_chunk - 1, 41 | 2 * bits_per_chunk]) 42 | end 43 | end 44 | 45 | describe "#bitcount" do 46 | it "handles bits around chunk boundaries" do 47 | a.bitcount.should == 5 48 | end 49 | end 50 | 51 | describe "#operator |" do 52 | it "handles bits around chunk boundaries" do 53 | result << (a | b) 54 | result.bitcount.should == 5 55 | end 56 | 57 | it "handles empty bitmaps" do 58 | result << (empty | empty) 59 | result.bitcount.should == 0 60 | end 61 | end 62 | 63 | describe "#operator &" do 64 | it "handles bits around chunk boundaries" do 65 | result << (a & b) 66 | result.bitcount.should == 3 67 | end 68 | 69 | it "handles empty bitmaps" do 70 | result << (empty & empty) 71 | result.bitcount.should == 0 72 | end 73 | end 74 | 75 | describe "#operator ~" do 76 | it "handles bits around chunk boundaries" do 77 | result << (~(a & b)) 78 | result[0].should be_false 79 | result[1].should be_true 80 | result[bits_per_chunk - 1].should be_true 81 | result[bits_per_chunk].should be_false 82 | result[bits_per_chunk + 1].should be_true 83 | result[2 * bits_per_chunk - 2].should be_true 84 | result[2 * bits_per_chunk - 1].should be_false 85 | result[2 * bits_per_chunk].should be_true 86 | end 87 | end 88 | 89 | describe "#operator ^" do 90 | it "handles bits around chunk boundaries" do 91 | result << (a ^ b) 92 | result.bitcount.should == 2 93 | end 94 | 95 | it "handles empty bitmaps" do 96 | result << (empty ^ empty) 97 | result.bitcount.should == 0 98 | end 99 | end 100 | 101 | describe "#copy_to" do 102 | it "overrides all chunks in the target bitmap" do 103 | # Fix expression with bits set using [] after evaluation doesn't materialize the newly set bits. 104 | result[4*bits_per_chunk + 1] = true 105 | a.copy_to(result) 106 | result.bitcount.should == a.bitcount 107 | result[4*bits_per_chunk + 1].should be_false 108 | end 109 | end 110 | end 111 | end 112 | 113 | describe "Redis#sparse_bitmap" do 114 | it_should_behave_like "a bitmap factory method", :sparse_bitmap, Redis::Bitops::SparseBitmap 115 | end 116 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $: << "../../lib" 2 | 3 | require 'redis/bitops' 4 | 5 | require_relative './support/bitmap_examples' 6 | 7 | require 'pry' 8 | require 'awesome_print' 9 | 10 | RSpec.configure do |config| 11 | config.after(:each, redis_cleanup: true) do 12 | key_prefix = example.metadata[:redis_key_prefix] or raise "Specify the key prefix using RSpec metadata (e.g. redis_key_prefix: 'rsb:')." 13 | keys = redis.keys("#{key_prefix}*") 14 | redis.del(*keys) unless keys.empty? 15 | end 16 | end -------------------------------------------------------------------------------- /spec/support/bitmap_examples.rb: -------------------------------------------------------------------------------- 1 | shared_examples_for "a bitmap" do |creation_method| 2 | let(:redis) { Redis.new } 3 | let(:a) { redis.send(creation_method, "rsb:a") } 4 | let(:b) { redis.send(creation_method, "rsb:b") } 5 | let(:c) { redis.send(creation_method, "rsb:c") } 6 | let(:result) { redis.send(creation_method, "rsb:output") } 7 | 8 | describe "#[]" do 9 | it "sets individual bits" do 10 | b[0] = true 11 | b[99] = true 12 | b[0].should be_true 13 | b[99].should be_true 14 | end 15 | 16 | it "resizes the bitmap as necessary" do 17 | expect { b[10_000_000] = 1 }.to_not raise_error 18 | b[10_000_000].should be_true 19 | end 20 | 21 | it "doesn't zeroes unset bits" do 22 | max_bit_pos = 5_000 23 | approx_percent_bits_set = 0.2 24 | 25 | # TODO: There are definitely faster ways to generate the data. 26 | Inf = 1.0/0.0 unless Kernel.const_defined? "Inf" 27 | set = Set.new((0..Inf).lazy.map {|i| (rand * max_bit_pos).to_i}.take(approx_percent_bits_set * max_bit_pos)) 28 | 29 | set.each do |pos| 30 | b[pos] = true 31 | end 32 | 33 | set.each do |pos| 34 | b[pos].should be_true 35 | end 36 | 37 | (0..Inf).lazy.take(max_bit_pos).each do |pos| 38 | b[pos].should eq set.include?(pos) 39 | end 40 | end 41 | end 42 | 43 | describe "#bitcount" do 44 | it "returns the number of set bits" do 45 | b.bitcount.should == 0 46 | b[1] = true 47 | b[2] = true 48 | b[100] = true 49 | b.bitcount.should == 3 50 | end 51 | end 52 | 53 | describe "#operator &" do 54 | it "returns a result query that can be materialized" do 55 | a[0] = true 56 | a[1] = true 57 | a[2] = true 58 | a[100] = true 59 | a[110] = true 60 | 61 | b[0] = true 62 | b[100] = true 63 | 64 | q = a & b 65 | 66 | result << q 67 | result.bitcount.should == 2 68 | end 69 | 70 | it "allows operator nesting" do 71 | a[0] = true 72 | a[1] = true 73 | a[2] = true 74 | a[100] = true 75 | a[110] = true 76 | 77 | b[0] = true 78 | b[100] = true 79 | 80 | c[0] = true 81 | 82 | q = a & b & (c & a) 83 | 84 | result << q 85 | result.bitcount.should == 1 86 | end 87 | end 88 | 89 | describe "#operator |" do 90 | it "returns a result query that can be materialized" do 91 | a[0] = true 92 | a[1] = true 93 | a[2] = true 94 | a[100] = true 95 | 96 | b[0] = true 97 | b[100] = true 98 | b[110] = true 99 | 100 | q = a | b 101 | 102 | result << q 103 | result.bitcount.should == 5 104 | end 105 | 106 | it "allows operator nesting" do 107 | a[0] = true 108 | a[1] = true 109 | a[2] = true 110 | a[100] = true 111 | a[110] = true 112 | 113 | b[0] = true 114 | b[100] = true 115 | 116 | c[0] = true 117 | 118 | q = a | b | (c | a) 119 | 120 | result << q 121 | result.bitcount.should == 5 122 | end 123 | end 124 | 125 | describe "#operator ~" do 126 | it "returns a result query that can be materialized" do 127 | a[0] = true 128 | a[1] = true 129 | a[2] = true 130 | a[100] = true 131 | 132 | q = ~a 133 | 134 | result << q 135 | result[0].should be_false 136 | result[1].should be_false 137 | result[2].should be_false 138 | result[3].should be_true 139 | result[99].should be_true 140 | result[100].should be_false 141 | end 142 | 143 | it "allows operator nesting" do 144 | a[0] = true 145 | a[1] = true 146 | a[2] = true 147 | a[100] = true 148 | a[110] = true 149 | 150 | b[0] = true 151 | b[100] = true 152 | 153 | c[0] = true 154 | 155 | q = ~a | b & c 156 | 157 | result << q 158 | result[0].should be_true 159 | result[1].should be_false 160 | result[2].should be_false 161 | result[3].should be_true 162 | result[99].should be_true 163 | result[100].should be_false 164 | result[109].should be_true 165 | result[110].should be_false 166 | result[111].should be_true 167 | end 168 | 169 | # Commented out because this is how redis BITOP NOT works: 170 | # it pads the results to the full byte thus messing up with 171 | # the operation. 172 | 173 | # it "returns result with the correct bitcount" do 174 | # pending 175 | # a[0] = true 176 | # a[1] = true 177 | # a[2] = true 178 | # a[100] = true 179 | # 180 | # q = ~a 181 | # 182 | # result << q 183 | # result.bitcount.should == 96 184 | # end 185 | end 186 | 187 | describe "#operator ^" do 188 | it "returns a result query that can be materialized" do 189 | a[0] = true 190 | a[1] = true 191 | a[2] = true 192 | a[100] = true 193 | 194 | b[0] = true 195 | b[100] = true 196 | b[110] = true 197 | 198 | q = a ^ b 199 | 200 | result << q 201 | result.bitcount.should == 3 202 | end 203 | 204 | it "allows operator nesting" do 205 | a[0] = true 206 | a[1] = true 207 | a[2] = true 208 | a[100] = true 209 | a[110] = true 210 | 211 | b[0] = true 212 | b[100] = true 213 | 214 | c[0] = true 215 | 216 | q = a ^ (b ^ c) 217 | 218 | result << q 219 | result.bitcount.should == 4 220 | end 221 | end 222 | 223 | describe "#delete!" do 224 | it "removes all bitmap's keys" do 225 | a[0] = true 226 | a[10_000] = true 227 | a.delete! 228 | redis.keys("rsb:*").should be_empty 229 | end 230 | 231 | it "effectively sets all bitmap's keys to zero" do 232 | a[0] = true 233 | a[10_000] = true 234 | a.delete! 235 | a[0].should be_false 236 | a[10_000].should be_false 237 | b[0] = true 238 | result << (a & b) 239 | result.bitcount.should == 0 240 | end 241 | end 242 | 243 | describe "#<<" do 244 | before do 245 | a[0] = true 246 | a[1] = true 247 | a[2] = true 248 | a[3] = true 249 | 250 | b[0] = true 251 | b[1] = true 252 | b[2] = true 253 | 254 | c[0] = true 255 | c[1] = true 256 | end 257 | 258 | after do 259 | @temp.delete! if @temp 260 | end 261 | 262 | it "materializes an arbitrarily-complicated expression" do 263 | result << (a & (a & b) | c & b & a) 264 | result.bitcount.should == 3 265 | end 266 | 267 | it "is lazy-invoked when expression is evaluated" do 268 | result = (a & (a & b) | c & b & a) 269 | result.should be_a Redis::Bitops::Queries::BinaryOperator 270 | @temp = result 271 | result.bitcount.should == 3 272 | end 273 | 274 | it "takes into account modifications made to the result" do 275 | output = (a & (a & b) | c & b & a) 276 | output[100] = true 277 | @temp = output 278 | result << output 279 | result.bitcount.should == 4 280 | end 281 | end 282 | 283 | describe "#copy_to" do 284 | it "overrides the target bitmap" do 285 | # Fix expression with bits set using [] after evaluation doesn't materialize the newly set bits. 286 | result[1000] = true 287 | a[0] = true 288 | a[1] = true 289 | a.copy_to(result) 290 | result.bitcount.should == a.bitcount 291 | result[1000].should be_false 292 | end 293 | end 294 | end 295 | 296 | shared_examples_for "a bitmap factory method" do |creation_method, bitmap_class| 297 | let(:redis) { Redis.new } 298 | 299 | after do 300 | @bitmap.delete! if @bitmap 301 | end 302 | 303 | it "creates a new bitmap" do 304 | @bitmap = redis.send(creation_method, "rsb:xxx") 305 | @bitmap.should be_a bitmap_class 306 | end 307 | 308 | it "doesn't add keys until the bitmap is modified" do 309 | @bitmap = redis.send(creation_method, "rsb:xxx") 310 | expect { @bitmap }.to_not change { redis.keys.size } 311 | expect { @bitmap[1] = true }.to change { redis.keys.size } 312 | end 313 | end 314 | --------------------------------------------------------------------------------