├── .github └── workflows │ └── build.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── blind_index.gemspec ├── docs └── Other-Algorithms.md ├── gemfiles ├── activerecord71.gemfile ├── activerecord72.gemfile ├── mongoid8.gemfile └── mongoid9.gemfile ├── lib ├── blind_index.rb └── blind_index │ ├── backfill.rb │ ├── extensions.rb │ ├── key_generator.rb │ ├── model.rb │ ├── mongoid.rb │ └── version.rb └── test ├── blind_index_test.rb ├── support ├── active_record.rb └── mongoid.rb └── test_helper.rb /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: [push, pull_request] 3 | jobs: 4 | build: 5 | strategy: 6 | fail-fast: false 7 | matrix: 8 | include: 9 | - ruby: 3.4 10 | gemfile: Gemfile 11 | - ruby: 3.3 12 | gemfile: gemfiles/activerecord72.gemfile 13 | - ruby: 3.2 14 | gemfile: gemfiles/activerecord71.gemfile 15 | - ruby: 3.3 16 | gemfile: gemfiles/mongoid9.gemfile 17 | mongodb: true 18 | - ruby: 3.2 19 | gemfile: gemfiles/mongoid8.gemfile 20 | mongodb: true 21 | runs-on: ubuntu-latest 22 | env: 23 | BUNDLE_GEMFILE: ${{ matrix.gemfile }} 24 | steps: 25 | - uses: actions/checkout@v4 26 | - uses: ruby/setup-ruby@v1 27 | with: 28 | ruby-version: ${{ matrix.ruby }} 29 | bundler-cache: true 30 | - if: ${{ matrix.mongodb }} 31 | uses: ankane/setup-mongodb@v1 32 | - run: bundle exec rake test 33 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | *.lock 10 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 2.7.0 (2025-05-04) 2 | 3 | - Dropped support for Ruby < 3.2 and Active Record < 7.1 4 | - Dropped support for Mongoid < 8 5 | 6 | ## 2.6.2 (2025-02-23) 7 | 8 | - Fixed querying with normalized attributes 9 | 10 | ## 2.6.1 (2024-11-01) 11 | 12 | - Fixed issue with `includes` and Active Record 7 13 | 14 | ## 2.6.0 (2024-10-07) 15 | 16 | - Removed dependency on `scrypt` gem for scrypt algorithm 17 | - Dropped support for Active Record < 7 18 | 19 | ## 2.5.0 (2024-06-03) 20 | 21 | - Added support for Mongoid 9 22 | - Dropped support for Ruby < 3.1 23 | 24 | ## 2.4.0 (2023-07-02) 25 | 26 | - Dropped support for Ruby < 3 and Rails < 6.1 27 | - Dropped support for Mongoid < 7 28 | 29 | ## 2.3.2 (2023-04-26) 30 | 31 | - Added `key_table` and `key_attribute` options 32 | 33 | ## 2.3.1 (2022-09-06) 34 | 35 | - Fixed error with `backfill` when `bidx_attribute` is a symbol 36 | 37 | ## 2.3.0 (2022-01-16) 38 | 39 | - Added blind indexes to `filter_attributes` 40 | - Dropped support for Ruby < 2.6 and Rails < 5.2 41 | 42 | ## 2.2.0 (2020-09-07) 43 | 44 | - Added support for `where` with table in Active Record 5.2+ 45 | 46 | ## 2.1.1 (2020-08-14) 47 | 48 | - Fixed `version` option 49 | 50 | ## 2.1.0 (2020-07-06) 51 | 52 | - Improved performance of uniqueness validations 53 | - Fixed deprecation warnings in Ruby 2.7 with Mongoid 54 | 55 | ## 2.0.2 (2020-06-01) 56 | 57 | - Improved error message for bad key length 58 | - Fixed `backfill` method with relations for Mongoid 59 | 60 | ## 2.0.1 (2020-02-14) 61 | 62 | - Added `BlindIndex.backfill` method 63 | 64 | ## 2.0.0 (2020-02-10) 65 | 66 | - Blind indexes are updated immediately instead of in a `before_validation` callback 67 | - Better Lockbox integration - no need to generate a separate key 68 | - The `argon2` gem has been replaced with `argon2-kdf` for less dependencies and Windows support 69 | - Removed deprecated `compute_email_bidx` 70 | 71 | ## 1.0.2 (2019-12-26) 72 | 73 | - Fixed `OpenSSL::KDF` error on some platforms 74 | - Fixed deprecation warnings in Ruby 2.7 75 | 76 | ## 1.0.1 (2019-08-16) 77 | 78 | - Added support for Mongoid 79 | 80 | ## 1.0.0 (2019-07-08) 81 | 82 | - Added support for master key 83 | - Added support for Argon2id 84 | - Fixed `generate_key` for JRuby 85 | - Dropped support for Rails 4.2 86 | 87 | Breaking changes 88 | 89 | - Made Argon2id the default algorithm 90 | - Removed `encrypted_` prefix from columns 91 | - Changed default encoding to Base64 strict 92 | 93 | ## 0.3.5 (2019-05-28) 94 | 95 | - Added support for hex keys 96 | - Added `generate_key` method 97 | - Fixed querying with array values 98 | 99 | ## 0.3.4 (2018-12-16) 100 | 101 | - Added `size` option 102 | - Added sanity checks for Argon2 cost parameters 103 | - Fixed Active Record callback issues introduced in 0.3.3 104 | 105 | ## 0.3.3 (2018-11-12) 106 | 107 | - Added support for string keys in finders 108 | 109 | ## 0.3.2 (2018-06-18) 110 | 111 | - Added support for dynamic finders 112 | - Added support for inherited models 113 | 114 | ## 0.3.1 (2018-06-04) 115 | 116 | - Added scrypt and Argon2 algorithms 117 | - Added `cost` option 118 | 119 | ## 0.3.0 (2018-06-03) 120 | 121 | - Enforce secure key generation 122 | - Added `encode` option 123 | - Added `default_options` method 124 | 125 | ## 0.2.1 (2018-05-26) 126 | 127 | - Added class method to compute blind index 128 | - Fixed issue with cached statements 129 | 130 | ## 0.2.0 (2018-05-11) 131 | 132 | - Added support for Active Record 4.2 133 | - Improved validation support when multiple blind indexes 134 | - Fixed `nil` handling 135 | 136 | ## 0.1.1 (2018-04-09) 137 | 138 | - Added support for Active Record 5.2 139 | - Added `callback` option 140 | - Added support for `key` proc 141 | - Fixed error inheritance 142 | 143 | ## 0.1.0 (2017-12-17) 144 | 145 | - First release 146 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "attr_encrypted" 8 | gem "activerecord", "~> 8.0.0" 9 | gem "benchmark-ips" 10 | gem "lockbox", ">= 1.4" 11 | gem "sqlite3", platform: :ruby 12 | gem "sqlite3-ffi", platform: :jruby 13 | 14 | # to test different adapters 15 | # gem "mysql2" 16 | # gem "pg" 17 | # gem "trilogy" 18 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2017-2025 Andrew Kane 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Blind Index 2 | 3 | Securely search encrypted database fields 4 | 5 | Works with [Lockbox](https://github.com/ankane/lockbox) ([full example](https://ankane.org/securing-user-emails-lockbox)) and [attr_encrypted](https://github.com/attr-encrypted/attr_encrypted) ([full example](https://ankane.org/securing-user-emails-in-rails)) 6 | 7 | Learn more about [securing sensitive data in Rails](https://ankane.org/sensitive-data-rails) 8 | 9 | [![Build Status](https://github.com/ankane/blind_index/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/blind_index/actions) 10 | 11 | ## How It Works 12 | 13 | We use [this approach](https://paragonie.com/blog/2017/05/building-searchable-encrypted-databases-with-php-and-sql) by Scott Arciszewski. To summarize, we compute a keyed hash of the sensitive data and store it in a column. To query, we apply the keyed hash function to the value we’re searching and then perform a database search. This results in performant queries for exact matches. Efficient `LIKE` queries are [not possible](#like-ilike-and-full-text-searching), but you can index expressions. 14 | 15 | ## Leakage 16 | 17 | An important consideration in searchable encryption is leakage, which is information an attacker can gain. Blind indexing leaks that rows have the same value. If you use this for a field like last name, an attacker can use frequency analysis to predict the values. In an active attack where an attacker can control the input values, they can learn which other values in the database match. 18 | 19 | Here’s a [great article](https://blog.cryptographyengineering.com/2019/02/11/attack-of-the-week-searchable-encryption-and-the-ever-expanding-leakage-function/) on leakage in searchable encryption. Blind indexing has the same leakage as [deterministic encryption](#alternatives). 20 | 21 | ## Installation 22 | 23 | Add this line to your application’s Gemfile: 24 | 25 | ```ruby 26 | gem "blind_index" 27 | ``` 28 | 29 | ## Prep 30 | 31 | Your model should already be set up with Lockbox or attr_encrypted. The examples are for a `User` model with `has_encrypted :email` or `attr_encrypted :email`. See the full examples for [Lockbox](https://ankane.org/securing-user-emails-lockbox) and [attr_encrypted](https://ankane.org/securing-user-emails-in-rails) if needed. 32 | 33 | Also, if you use attr_encrypted, [generate a key](#key-generation). 34 | 35 | ## Getting Started 36 | 37 | Create a migration to add a column for the blind index 38 | 39 | ```ruby 40 | add_column :users, :email_bidx, :string 41 | add_index :users, :email_bidx # unique: true if needed 42 | ``` 43 | 44 | Add to your model 45 | 46 | ```ruby 47 | class User < ApplicationRecord 48 | blind_index :email 49 | end 50 | ``` 51 | 52 | For more sensitive fields, use 53 | 54 | ```ruby 55 | class User < ApplicationRecord 56 | blind_index :email, slow: true 57 | end 58 | ``` 59 | 60 | Backfill existing records 61 | 62 | ```ruby 63 | BlindIndex.backfill(User) 64 | ``` 65 | 66 | And query away 67 | 68 | ```ruby 69 | User.where(email: "test@example.org") 70 | ``` 71 | 72 | ## Expressions 73 | 74 | You can apply expressions to attributes before indexing and searching. This gives you the the ability to perform case-insensitive searches and more. 75 | 76 | ```ruby 77 | class User < ApplicationRecord 78 | blind_index :email, expression: ->(v) { v.downcase } 79 | end 80 | ``` 81 | 82 | ## Validations 83 | 84 | You can use blind indexes for uniqueness validations. 85 | 86 | ```ruby 87 | class User < ApplicationRecord 88 | validates :email, uniqueness: true 89 | end 90 | ``` 91 | 92 | We recommend adding a unique index to the blind index column through a database migration. 93 | 94 | ```ruby 95 | add_index :users, :email_bidx, unique: true 96 | ``` 97 | 98 | For `allow_blank: true`, use: 99 | 100 | ```ruby 101 | class User < ApplicationRecord 102 | blind_index :email, expression: ->(v) { v.presence } 103 | validates :email, uniqueness: {allow_blank: true} 104 | end 105 | ``` 106 | 107 | For `case_sensitive: false`, use: 108 | 109 | ```ruby 110 | class User < ApplicationRecord 111 | blind_index :email, expression: ->(v) { v.downcase } 112 | validates :email, uniqueness: true # for best performance, leave out {case_sensitive: false} 113 | end 114 | ``` 115 | 116 | ## Multiple Indexes 117 | 118 | You may want multiple blind indexes for an attribute. To do this, add another column: 119 | 120 | ```ruby 121 | add_column :users, :email_ci_bidx, :string 122 | add_index :users, :email_ci_bidx 123 | ``` 124 | 125 | Update your model 126 | 127 | ```ruby 128 | class User < ApplicationRecord 129 | blind_index :email 130 | blind_index :email_ci, attribute: :email, expression: ->(v) { v.downcase } 131 | end 132 | ``` 133 | 134 | Backfill existing records 135 | 136 | ```ruby 137 | BlindIndex.backfill(User, columns: [:email_ci_bidx]) 138 | ``` 139 | 140 | And query away 141 | 142 | ```ruby 143 | User.where(email_ci: "test@example.org") 144 | ``` 145 | 146 | ## Index Only 147 | 148 | If you don’t need to store the original value (for instance, when just checking duplicates), use a virtual attribute: 149 | 150 | ```ruby 151 | class User < ApplicationRecord 152 | attribute :email, :string 153 | blind_index :email 154 | end 155 | ``` 156 | 157 | ## Multiple Columns 158 | 159 | You can also use virtual attributes to index data from multiple columns: 160 | 161 | ```ruby 162 | class User < ApplicationRecord 163 | attribute :initials, :string 164 | blind_index :initials 165 | 166 | before_validation :set_initials, if: -> { changes.key?(:first_name) || changes.key?(:last_name) } 167 | 168 | def set_initials 169 | self.initials = "#{first_name[0]}#{last_name[0]}" 170 | end 171 | end 172 | ``` 173 | 174 | ## Migrating Data 175 | 176 | If you’re encrypting a column and adding a blind index at the same time, use the `migrating` option. 177 | 178 | ```ruby 179 | class User < ApplicationRecord 180 | blind_index :email, migrating: true 181 | end 182 | ``` 183 | 184 | This allows you to backfill records while still querying the unencrypted field. 185 | 186 | ```ruby 187 | BlindIndex.backfill(User) 188 | ``` 189 | 190 | Once that completes, you can remove the `migrating` option. 191 | 192 | ## Key Rotation 193 | 194 | To rotate keys without downtime, add a new column: 195 | 196 | ```ruby 197 | add_column :users, :email_bidx_v2, :string 198 | add_index :users, :email_bidx_v2 199 | ``` 200 | 201 | And add to your model 202 | 203 | ```ruby 204 | class User < ApplicationRecord 205 | blind_index :email, rotate: {version: 2, master_key: ENV["BLIND_INDEX_MASTER_KEY_V2"]} 206 | end 207 | ``` 208 | 209 | This will keep the new column synced going forward. Next, backfill the data: 210 | 211 | ```ruby 212 | BlindIndex.backfill(User, columns: [:email_bidx_v2]) 213 | ``` 214 | 215 | Then update your model 216 | 217 | ```ruby 218 | class User < ApplicationRecord 219 | blind_index :email, version: 2, master_key: ENV["BLIND_INDEX_MASTER_KEY_V2"] 220 | end 221 | ``` 222 | 223 | Finally, drop the old column. 224 | 225 | ## Key Separation 226 | 227 | The master key is used to generate unique keys for each blind index. This technique comes from [CipherSweet](https://ciphersweet.paragonie.com/internals/key-hierarchy). The table name and blind index column name are both used in this process. 228 | 229 | You can get an individual key with: 230 | 231 | ```ruby 232 | BlindIndex.index_key(table: "users", bidx_attribute: "email_bidx") 233 | ``` 234 | 235 | To rename a table with blind indexes, use: 236 | 237 | ```ruby 238 | class User < ApplicationRecord 239 | blind_index :email, key_table: "original_table" 240 | end 241 | ``` 242 | 243 | To rename a blind index column, use: 244 | 245 | ```ruby 246 | class User < ApplicationRecord 247 | blind_index :email, key_attribute: "original_column" 248 | end 249 | ``` 250 | 251 | ## Algorithm 252 | 253 | Argon2id is used for best security. The default cost parameters are 3 iterations and 4 MB of memory. For `slow: true`, the cost parameters are 4 iterations and 32 MB of memory. 254 | 255 | A number of other algorithms are [also supported](docs/Other-Algorithms.md). Unless you have specific reasons to use them, go with Argon2id. 256 | 257 | ## Fixtures 258 | 259 | You can use blind indexes in fixtures with: 260 | 261 | ```yml 262 | test_user: 263 | email_bidx: <%= User.generate_email_bidx("test@example.org").inspect %> 264 | ``` 265 | 266 | Be sure to include the `inspect` at the end or it won’t be encoded properly in YAML. 267 | 268 | ## Mongoid 269 | 270 | For Mongoid, use: 271 | 272 | ```ruby 273 | class User 274 | field :email_bidx, type: String 275 | index({email_bidx: 1}) 276 | end 277 | ``` 278 | 279 | ## Key Generation 280 | 281 | This is optional for Lockbox, as its master key is used by default. 282 | 283 | Generate a key with: 284 | 285 | ```ruby 286 | BlindIndex.generate_key 287 | ``` 288 | 289 | Store the key with your other secrets. This is typically Rails credentials or an environment variable ([dotenv](https://github.com/bkeepers/dotenv) is great for this). Be sure to use different keys in development and production. Keys don’t need to be hex-encoded, but it’s often easier to store them this way. 290 | 291 | Set the following environment variable with your key (you can use this one in development) 292 | 293 | ```sh 294 | BLIND_INDEX_MASTER_KEY=ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff 295 | ``` 296 | 297 | or create `config/initializers/blind_index.rb` with something like 298 | 299 | ```ruby 300 | BlindIndex.master_key = Rails.application.credentials.blind_index_master_key 301 | ``` 302 | 303 | ## LIKE, ILIKE, and Full-Text Searching 304 | 305 | Unfortunately, blind indexes can’t be used for `LIKE`, `ILIKE`, or full-text searching. Instead, records must be loaded, decrypted, and searched in memory. 306 | 307 | For `LIKE`, use: 308 | 309 | ```ruby 310 | User.select { |u| u.email.include?("value") } 311 | ``` 312 | 313 | For `ILIKE`, use: 314 | 315 | ```ruby 316 | User.select { |u| u.email =~ /value/i } 317 | ``` 318 | 319 | For full-text or fuzzy searching, use a gem like [FuzzyMatch](https://github.com/seamusabshere/fuzzy_match): 320 | 321 | ```ruby 322 | FuzzyMatch.new(User.all, read: :email).find("value") 323 | ``` 324 | 325 | If the number of records is large, try to find a way to narrow it down. An [expression index](#expressions) is one way to do this, but leaks which records have the same value of the expression, so use it carefully. 326 | 327 | ## Reference 328 | 329 | Set default options in an initializer with: 330 | 331 | ```ruby 332 | BlindIndex.default_options = {algorithm: :pbkdf2_sha256} 333 | ``` 334 | 335 | By default, blind indexes are encoded in Base64. Set a different encoding with: 336 | 337 | ```ruby 338 | class User < ApplicationRecord 339 | blind_index :email, encode: ->(v) { [v].pack("H*") } 340 | end 341 | ``` 342 | 343 | By default, blind indexes are 32 bytes. Set a smaller size with: 344 | 345 | ```ruby 346 | class User < ApplicationRecord 347 | blind_index :email, size: 16 348 | end 349 | ``` 350 | 351 | Set a key directly for an index with: 352 | 353 | ```ruby 354 | class User < ApplicationRecord 355 | blind_index :email, key: ENV["USER_EMAIL_BLIND_INDEX_KEY"] 356 | end 357 | ``` 358 | 359 | ## Compatibility 360 | 361 | You can generate blind indexes from other languages as well. For Python, you can use [argon2-cffi](https://github.com/hynek/argon2-cffi). 362 | 363 | ```python 364 | from argon2.low_level import Type, hash_secret_raw 365 | from base64 import b64encode 366 | 367 | key = '289737bab72fa97b1f4b081cef00d7b7d75034bcf3183c363feaf3e6441777bc' 368 | value = 'test@example.org' 369 | 370 | bidx = b64encode(hash_secret_raw( 371 | secret=value.encode(), 372 | salt=bytes.fromhex(key), 373 | time_cost=3, 374 | memory_cost=2**12, 375 | parallelism=1, 376 | hash_len=32, 377 | type=Type.ID 378 | )) 379 | ``` 380 | 381 | ## Alternatives 382 | 383 | One alternative to blind indexing is to use a deterministic encryption scheme, like [AES-SIV](https://github.com/miscreant/miscreant). In this approach, the encrypted data will be the same for matches. We recommend blind indexing over deterministic encryption because: 384 | 385 | 1. You can keep encryption consistent for all fields (both searchable and non-searchable) 386 | 2. Blind indexing supports expressions 387 | 388 | ## History 389 | 390 | View the [changelog](https://github.com/ankane/blind_index/blob/master/CHANGELOG.md) 391 | 392 | ## Contributing 393 | 394 | Everyone is encouraged to help improve this project. Here are a few ways you can help: 395 | 396 | - [Report bugs](https://github.com/ankane/blind_index/issues) 397 | - Fix bugs and [submit pull requests](https://github.com/ankane/blind_index/pulls) 398 | - Write, clarify, or fix documentation 399 | - Suggest or add new features 400 | 401 | To get started with development and testing: 402 | 403 | ```sh 404 | git clone https://github.com/ankane/blind_index.git 405 | cd blind_index 406 | bundle install 407 | bundle exec rake test 408 | ``` 409 | 410 | For security issues, send an email to the address on [this page](https://github.com/ankane). 411 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rake/testtask" 3 | 4 | Rake::TestTask.new(:test) do |t| 5 | t.libs << "test" 6 | t.test_files = FileList["test/**/*_test.rb"] 7 | t.warning = false # for attr_encrypted 8 | end 9 | 10 | task default: :test 11 | 12 | namespace :benchmark do 13 | task :algorithms do 14 | require "benchmark/ips" 15 | require "blind_index" 16 | require "scrypt" 17 | require "argon2" 18 | 19 | key = BlindIndex.generate_key 20 | value = "secret" 21 | 22 | Benchmark.ips do |x| 23 | x.report("pbkdf2_sha256") { BlindIndex.generate_bidx(value, key: key, algorithm: :pbkdf2_sha256) } 24 | x.report("pbkdf2_sha256 slow") { BlindIndex.generate_bidx(value, key: key, algorithm: :pbkdf2_sha256, slow: true) } 25 | x.report("argon2id") { BlindIndex.generate_bidx(value, key: key, algorithm: :argon2id) } 26 | x.report("argon2id slow") { BlindIndex.generate_bidx(value, key: key, algorithm: :argon2id, slow: true) } 27 | # x.report("argon2i") { BlindIndex.generate_bidx(value, key: key, algorithm: :argon2i) } 28 | # x.report("scrypt") { BlindIndex.generate_bidx(value, key: key, algorithm: :scrypt) } 29 | end 30 | end 31 | 32 | task :queries do 33 | require "benchmark/ips" 34 | require "active_record" 35 | require "blind_index" 36 | 37 | ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:" 38 | 39 | ActiveRecord::Migration.create_table :users do |t| 40 | t.string :email_bidx 41 | end 42 | 43 | class User < ActiveRecord::Base 44 | blind_index :email, key: BlindIndex.generate_key, slow: true 45 | end 46 | 47 | Benchmark.ips do |x| 48 | x.report("no index") { User.find_by(id: 1) } 49 | x.report("index") { User.find_by(email: "test@example.org") } 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /blind_index.gemspec: -------------------------------------------------------------------------------- 1 | require_relative "lib/blind_index/version" 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "blind_index" 5 | spec.version = BlindIndex::VERSION 6 | spec.summary = "Securely search encrypted database fields" 7 | spec.homepage = "https://github.com/ankane/blind_index" 8 | spec.license = "MIT" 9 | 10 | spec.author = "Andrew Kane" 11 | spec.email = "andrew@ankane.org" 12 | 13 | spec.files = Dir["*.{md,txt}", "{lib}/**/*"] 14 | spec.require_path = "lib" 15 | 16 | spec.required_ruby_version = ">= 3.2" 17 | 18 | spec.add_dependency "activesupport", ">= 7.1" 19 | spec.add_dependency "argon2-kdf", ">= 0.2" 20 | end 21 | -------------------------------------------------------------------------------- /docs/Other-Algorithms.md: -------------------------------------------------------------------------------- 1 | # Other Algorithms 2 | 3 | ## PBKDF2-SHA256 4 | 5 | Use: 6 | 7 | ```ruby 8 | class User < ApplicationRecord 9 | blind_index :email, algorithm: :pbkdf2_sha256 10 | end 11 | ``` 12 | 13 | The default number of iterations is 10,000. For more sensitive fields, use: 14 | 15 | ```ruby 16 | class User < ApplicationRecord 17 | blind_index :email, algorithm: :pbkdf2_sha256, slow: true 18 | end 19 | ``` 20 | 21 | This uses 100,000 iterations. 22 | 23 | ## Argon2i 24 | 25 | Use: 26 | 27 | ```ruby 28 | class User < ApplicationRecord 29 | blind_index :email, algorithm: :argon2i 30 | end 31 | ``` 32 | 33 | Set the cost parameters with: 34 | 35 | ```ruby 36 | class User < ApplicationRecord 37 | blind_index :email, algorithm: :argon2i, cost: {t: 4, m: 15} 38 | end 39 | ``` 40 | 41 | ## scrypt 42 | 43 | Add [scrypt](https://github.com/pbhogan/scrypt) to your Gemfile and use: 44 | 45 | ```ruby 46 | class User < ApplicationRecord 47 | blind_index :email, algorithm: :scrypt 48 | end 49 | ``` 50 | 51 | Set the cost parameters with: 52 | 53 | ```ruby 54 | class User < ApplicationRecord 55 | blind_index :email, algorithm: :scrypt, cost: {n: 4096, r: 8, p: 1} 56 | end 57 | ``` 58 | -------------------------------------------------------------------------------- /gemfiles/activerecord71.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "attr_encrypted" 8 | gem "activerecord", "~> 7.1.0" 9 | gem "lockbox", ">= 1.4" 10 | gem "sqlite3", "< 2" 11 | -------------------------------------------------------------------------------- /gemfiles/activerecord72.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "attr_encrypted" 8 | gem "activerecord", "~> 7.2.0" 9 | gem "lockbox", ">= 1" 10 | gem "sqlite3" 11 | -------------------------------------------------------------------------------- /gemfiles/mongoid8.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "mongoid", "~> 8" 8 | gem "lockbox", ">= 1" 9 | -------------------------------------------------------------------------------- /gemfiles/mongoid9.gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec path: ".." 4 | 5 | gem "rake" 6 | gem "minitest", ">= 5" 7 | gem "mongoid", "~> 9" 8 | gem "lockbox", ">= 1" 9 | -------------------------------------------------------------------------------- /lib/blind_index.rb: -------------------------------------------------------------------------------- 1 | # dependencies 2 | require "active_support" 3 | require "argon2/kdf" 4 | 5 | # stdlib 6 | require "openssl" 7 | 8 | # modules 9 | require_relative "blind_index/backfill" 10 | require_relative "blind_index/key_generator" 11 | require_relative "blind_index/model" 12 | require_relative "blind_index/version" 13 | 14 | module BlindIndex 15 | class Error < StandardError; end 16 | 17 | class << self 18 | attr_accessor :default_options 19 | attr_writer :master_key 20 | end 21 | self.default_options = {} 22 | 23 | def self.master_key 24 | @master_key ||= ENV["BLIND_INDEX_MASTER_KEY"] || (defined?(Lockbox.master_key) && Lockbox.master_key) 25 | end 26 | 27 | def self.generate_bidx(value, key:, **options) 28 | options = { 29 | encode: true 30 | }.merge(default_options).merge(options) 31 | 32 | # apply expression 33 | value = options[:expression].call(value) if options[:expression] 34 | 35 | unless value.nil? 36 | algorithm = (options[:algorithm] || (options[:legacy] ? :pbkdf2_sha256 : :argon2id)).to_sym 37 | algorithm = :pbkdf2_sha256 if algorithm == :pbkdf2_hmac 38 | algorithm = :argon2i if algorithm == :argon2 39 | 40 | key = key.call if key.respond_to?(:call) 41 | raise BlindIndex::Error, "Missing key for blind index" unless key 42 | 43 | key = key.to_s 44 | unless options[:insecure_key] && algorithm == :pbkdf2_sha256 45 | key = decode_key(key) 46 | end 47 | 48 | # gist to compare algorithm results 49 | # https://gist.github.com/ankane/fe3ac63fbf1c4550ee12554c664d2b8c 50 | cost_options = options[:cost] || {} 51 | 52 | # check size 53 | size = (options[:size] || 32).to_i 54 | raise BlindIndex::Error, "Size must be between 1 and 32" unless (1..32).cover?(size) 55 | 56 | value = value.to_s 57 | 58 | value = 59 | case algorithm 60 | when :argon2id 61 | t = (cost_options[:t] || (options[:slow] ? 4 : 3)).to_i 62 | # use same bounds as rbnacl 63 | raise BlindIndex::Error, "t must be between 3 and 10" if t < 3 || t > 10 64 | 65 | # m is memory in kibibytes (1024 bytes) 66 | m = (cost_options[:m] || (options[:slow] ? 15 : 12)).to_i 67 | # use same bounds as rbnacl 68 | raise BlindIndex::Error, "m must be between 3 and 22" if m < 3 || m > 22 69 | 70 | Argon2::KDF.argon2id(value, salt: key, t: t, m: m, p: 1, length: size) 71 | when :pbkdf2_sha256 72 | iterations = cost_options[:iterations] || options[:iterations] || (options[:slow] ? 100000 : 10000) 73 | OpenSSL::KDF.pbkdf2_hmac(value, salt: key, iterations: iterations, length: size, hash: "sha256") 74 | when :argon2i 75 | t = (cost_options[:t] || 3).to_i 76 | # use same bounds as rbnacl 77 | raise BlindIndex::Error, "t must be between 3 and 10" if t < 3 || t > 10 78 | 79 | # m is memory in kibibytes (1024 bytes) 80 | m = (cost_options[:m] || 12).to_i 81 | # use same bounds as rbnacl 82 | raise BlindIndex::Error, "m must be between 3 and 22" if m < 3 || m > 22 83 | 84 | Argon2::KDF.argon2i(value, salt: key, t: t, m: m, p: 1, length: size) 85 | when :scrypt 86 | n = cost_options[:n] || 4096 87 | r = cost_options[:r] || 8 88 | cp = cost_options[:p] || 1 89 | OpenSSL::KDF.scrypt(value, salt: key, N: n, r: r, p: cp, length: size) 90 | else 91 | raise BlindIndex::Error, "Unknown algorithm" 92 | end 93 | 94 | encode = options[:encode] 95 | if encode 96 | if encode.respond_to?(:call) 97 | encode.call(value) 98 | else 99 | [value].pack(options[:legacy] ? "m" : "m0") 100 | end 101 | else 102 | value 103 | end 104 | end 105 | end 106 | 107 | def self.generate_key 108 | require "securerandom" 109 | # force encoding to make JRuby consistent with MRI 110 | SecureRandom.hex(32).force_encoding(Encoding::US_ASCII) 111 | end 112 | 113 | def self.index_key(table:, bidx_attribute:, master_key: nil, encode: true) 114 | master_key ||= BlindIndex.master_key 115 | raise BlindIndex::Error, "Missing master key" unless master_key 116 | 117 | key = BlindIndex::KeyGenerator.new(master_key).index_key(table: table, bidx_attribute: bidx_attribute) 118 | key = key.unpack("H*").first if encode 119 | key 120 | end 121 | 122 | def self.decode_key(key, name: "Key") 123 | # decode hex key 124 | if key.encoding != Encoding::BINARY && key =~ /\A[0-9a-f]{64}\z/i 125 | key = [key].pack("H*") 126 | end 127 | 128 | raise BlindIndex::Error, "#{name} must be 32 bytes (64 hex digits)" if key.bytesize != 32 129 | raise BlindIndex::Error, "#{name} must use binary encoding" if key.encoding != Encoding::BINARY 130 | 131 | key 132 | end 133 | 134 | def self.backfill(relation, columns: nil, batch_size: 1000) 135 | Backfill.new(relation, columns: columns, batch_size: batch_size).perform 136 | end 137 | end 138 | 139 | ActiveSupport.on_load(:active_record) do 140 | require_relative "blind_index/extensions" 141 | extend BlindIndex::Model 142 | 143 | ActiveRecord::TableMetadata.prepend(BlindIndex::Extensions::TableMetadata) 144 | ActiveRecord::DynamicMatchers::Method.prepend(BlindIndex::Extensions::DynamicMatchers) 145 | ActiveRecord::Validations::UniquenessValidator.prepend(BlindIndex::Extensions::UniquenessValidator) 146 | ActiveRecord::PredicateBuilder.prepend(BlindIndex::Extensions::PredicateBuilder) 147 | end 148 | 149 | ActiveSupport.on_load(:mongoid) do 150 | require_relative "blind_index/mongoid" 151 | Mongoid::Document::ClassMethods.include(BlindIndex::Model) 152 | Mongoid::Criteria.prepend(BlindIndex::Mongoid::Criteria) 153 | Mongoid::Validatable::UniquenessValidator.prepend(BlindIndex::Mongoid::UniquenessValidator) 154 | end 155 | -------------------------------------------------------------------------------- /lib/blind_index/backfill.rb: -------------------------------------------------------------------------------- 1 | module BlindIndex 2 | class Backfill 3 | attr_reader :blind_indexes 4 | 5 | def initialize(relation, batch_size:, columns:) 6 | @relation = relation 7 | @transaction = @relation.respond_to?(:transaction) && !mongoid_relation?(relation.all) 8 | @batch_size = batch_size 9 | @blind_indexes = @relation.blind_indexes 10 | filter_columns!(columns) if columns 11 | end 12 | 13 | def perform 14 | each_batch do |records| 15 | backfill_records(records) 16 | end 17 | end 18 | 19 | private 20 | 21 | # modify in-place 22 | def filter_columns!(columns) 23 | columns = Array(columns).map(&:to_s) 24 | blind_indexes.select! { |_, v| columns.include?(v[:bidx_attribute].to_s) } 25 | bad_columns = columns - blind_indexes.map { |_, v| v[:bidx_attribute].to_s } 26 | raise ArgumentError, "Bad column: #{bad_columns.first}" if bad_columns.any? 27 | end 28 | 29 | def build_relation 30 | # build relation 31 | relation = @relation 32 | 33 | if defined?(ActiveRecord::Base) && relation.is_a?(ActiveRecord::Base) 34 | relation = relation.unscoped 35 | end 36 | 37 | # convert from possible class to ActiveRecord::Relation or Mongoid::Criteria 38 | relation = relation.all 39 | 40 | attributes = blind_indexes.map { |_, v| v[:bidx_attribute] } 41 | 42 | if defined?(ActiveRecord::Relation) && relation.is_a?(ActiveRecord::Relation) 43 | base_relation = relation.unscoped 44 | or_relation = relation.unscoped 45 | 46 | attributes.each_with_index do |attribute, i| 47 | or_relation = 48 | if i == 0 49 | base_relation.where(attribute => nil) 50 | else 51 | or_relation.or(base_relation.where(attribute => nil)) 52 | end 53 | end 54 | 55 | relation.merge(or_relation) 56 | else 57 | relation.merge(relation.unscoped.or(attributes.map { |a| {a => nil} })) 58 | end 59 | end 60 | 61 | def each_batch 62 | relation = build_relation 63 | 64 | if relation.respond_to?(:find_in_batches) 65 | relation.find_in_batches(batch_size: @batch_size) do |records| 66 | yield records 67 | end 68 | else 69 | # https://github.com/karmi/tire/blob/master/lib/tire/model/import.rb 70 | # use cursor for Mongoid 71 | records = [] 72 | relation.all.each do |record| 73 | records << record 74 | if records.length == @batch_size 75 | yield records 76 | records = [] 77 | end 78 | end 79 | yield records if records.any? 80 | end 81 | end 82 | 83 | def backfill_records(records) 84 | # do expensive blind index computation outside of transaction 85 | records.each do |record| 86 | blind_indexes.each do |k, v| 87 | record.send("compute_#{k}_bidx") if !record.send(v[:bidx_attribute]) 88 | end 89 | end 90 | 91 | # don't need to save records that went from nil => nil 92 | records.select! { |r| r.changed? } 93 | 94 | if records.any? 95 | with_transaction do 96 | records.each do |record| 97 | record.save!(validate: false) 98 | end 99 | end 100 | end 101 | end 102 | 103 | def mongoid_relation?(relation) 104 | defined?(Mongoid::Criteria) && relation.is_a?(Mongoid::Criteria) 105 | end 106 | 107 | def with_transaction 108 | if @transaction 109 | @relation.transaction do 110 | yield 111 | end 112 | else 113 | yield 114 | end 115 | end 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /lib/blind_index/extensions.rb: -------------------------------------------------------------------------------- 1 | module BlindIndex 2 | module Extensions 3 | module TableMetadata 4 | # memoize for performance 5 | def has_blind_indexes? 6 | unless defined?(@has_blind_indexes) 7 | @has_blind_indexes = klass.respond_to?(:blind_indexes) 8 | end 9 | @has_blind_indexes 10 | end 11 | end 12 | 13 | module PredicateBuilder 14 | # https://github.com/rails/rails/commit/56f30962b84fc53b76001301fb830c1594fd377e 15 | def build(attribute, value, *args) 16 | if table.has_blind_indexes? && (bi = table.send(:klass).blind_indexes[attribute.name.to_sym]) && !value.is_a?(ActiveRecord::StatementCache::Substitute) 17 | model = table.send(:klass) 18 | attribute_name = attribute.name.to_sym 19 | cast = 20 | if model.respond_to?(:normalized_attributes) && model.normalized_attributes.include?(attribute_name) 21 | ->(v) { model.normalize_value_for(attribute_name, v) } 22 | else 23 | ->(v) { v } 24 | end 25 | attribute = attribute.relation[bi[:bidx_attribute]] 26 | value = 27 | if value.is_a?(Array) || (defined?(Set) && value.is_a?(Set)) 28 | value.map { |v| BlindIndex.generate_bidx(cast.call(v), **bi) } 29 | else 30 | BlindIndex.generate_bidx(cast.call(value), **bi) 31 | end 32 | end 33 | 34 | super(attribute, value, *args) 35 | end 36 | end 37 | 38 | module UniquenessValidator 39 | def validate_each(record, attribute, value) 40 | klass = record.class 41 | if klass.respond_to?(:blind_indexes) && (bi = klass.blind_indexes[attribute]) 42 | value = record.read_attribute_for_validation(bi[:bidx_attribute]) 43 | end 44 | super(record, attribute, value) 45 | end 46 | 47 | # change attribute name here instead of validate_each for better error message 48 | def build_relation(klass, attribute, value) 49 | if klass.respond_to?(:blind_indexes) && (bi = klass.blind_indexes[attribute]) 50 | attribute = bi[:bidx_attribute] 51 | end 52 | super(klass, attribute, value) 53 | end 54 | end 55 | 56 | module DynamicMatchers 57 | def valid? 58 | attribute_names.all? { |name| model.columns_hash[name] || model.reflect_on_aggregation(name.to_sym) || blind_index?(name.to_sym) } 59 | end 60 | 61 | def blind_index?(name) 62 | model.respond_to?(:blind_indexes) && model.blind_indexes[name] 63 | end 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/blind_index/key_generator.rb: -------------------------------------------------------------------------------- 1 | module BlindIndex 2 | class KeyGenerator 3 | def initialize(master_key) 4 | @master_key = master_key 5 | end 6 | 7 | # pattern ported from CipherSweet 8 | # https://ciphersweet.paragonie.com/internals/key-hierarchy 9 | def index_key(table:, bidx_attribute:) 10 | raise ArgumentError, "Missing table for key generation" if table.to_s.empty? 11 | raise ArgumentError, "Missing field for key generation" if bidx_attribute.to_s.empty? 12 | 13 | c = "\x7E"*32 14 | root_key = hkdf(BlindIndex.decode_key(@master_key, name: "Master key"), salt: table.to_s, info: "#{c}#{bidx_attribute}", length: 32, hash: "sha384") 15 | hash_hmac("sha256", pack([table, bidx_attribute, bidx_attribute]), root_key) 16 | end 17 | 18 | private 19 | 20 | def hash_hmac(hash, ikm, salt) 21 | OpenSSL::HMAC.digest(hash, salt, ikm) 22 | end 23 | 24 | def hkdf(ikm, salt:, info:, length:, hash:) 25 | if defined?(OpenSSL::KDF.hkdf) 26 | # OpenSSL 1.1.0+ 27 | return OpenSSL::KDF.hkdf(ikm, salt: salt, info: info, length: length, hash: hash) 28 | end 29 | 30 | prk = hash_hmac(hash, ikm, salt) 31 | 32 | # empty binary string 33 | t = String.new 34 | last_block = String.new 35 | block_index = 1 36 | while t.bytesize < length 37 | last_block = hash_hmac(hash, last_block + info + [block_index].pack("C"), prk) 38 | t << last_block 39 | block_index += 1 40 | end 41 | 42 | t[0, length] 43 | end 44 | 45 | def pack(pieces) 46 | output = String.new 47 | output << [pieces.size].pack("V") 48 | pieces.map(&:to_s).each do |piece| 49 | output << [piece.bytesize].pack("Q<") 50 | output << piece 51 | end 52 | output 53 | end 54 | end 55 | end 56 | -------------------------------------------------------------------------------- /lib/blind_index/model.rb: -------------------------------------------------------------------------------- 1 | module BlindIndex 2 | module Model 3 | def blind_index(*attributes, rotate: false, migrating: false, **opts) 4 | indexes = attributes.map { |a| [a, opts.dup] } 5 | indexes.concat(attributes.map { |a| [a, rotate.merge(rotate: true)] }) if rotate 6 | 7 | indexes.each do |name, options| 8 | rotate = options.delete(:rotate) 9 | 10 | # check here so we validate rotate options as well 11 | unknown_keywords = options.keys - [:algorithm, :attribute, :bidx_attribute, 12 | :callback, :cost, :encode, :expression, :insecure_key, :iterations, :key, 13 | :key_attribute, :key_table, :legacy, :master_key, :size, :slow, :version] 14 | raise ArgumentError, "unknown keywords: #{unknown_keywords.join(", ")}" if unknown_keywords.any? 15 | 16 | attribute = options[:attribute] || name 17 | version = (options[:version] || 1).to_i 18 | callback = options[:callback].nil? ? true : options[:callback] 19 | if options[:bidx_attribute] 20 | bidx_attribute = options[:bidx_attribute] 21 | else 22 | bidx_attribute = name 23 | bidx_attribute = "encrypted_#{bidx_attribute}" if options[:legacy] 24 | bidx_attribute = "#{bidx_attribute}_bidx" 25 | bidx_attribute = "#{bidx_attribute}_v#{version}" if version != 1 26 | end 27 | 28 | name = "migrated_#{name}" if migrating 29 | name = "rotated_#{name}" if rotate 30 | name = name.to_sym 31 | attribute = attribute.to_sym 32 | method_name = :"compute_#{name}_bidx" 33 | class_method_name = :"generate_#{name}_bidx" 34 | 35 | key = options[:key] 36 | key ||= -> { BlindIndex.index_key(table: options[:key_table] || try(:table_name) || collection_name.to_s, bidx_attribute: options[:key_attribute] || bidx_attribute, master_key: options[:master_key], encode: false) } 37 | 38 | class_eval do 39 | activerecord = defined?(ActiveRecord) && self < ActiveRecord::Base 40 | 41 | if activerecord 42 | # blind index value isn't really sensitive 43 | # but don't need to show it in the Rails console 44 | self.filter_attributes += [/\A#{Regexp.escape(bidx_attribute)}\z/] 45 | end 46 | 47 | @blind_indexes ||= {} 48 | 49 | unless respond_to?(:blind_indexes) 50 | def self.blind_indexes 51 | parent_indexes = 52 | if superclass.respond_to?(:blind_indexes) 53 | superclass.blind_indexes 54 | else 55 | {} 56 | end 57 | 58 | parent_indexes.merge(@blind_indexes || {}) 59 | end 60 | end 61 | 62 | raise BlindIndex::Error, "Duplicate blind index: #{name}" if blind_indexes[name] 63 | 64 | @blind_indexes[name] = options.merge( 65 | key: key, 66 | attribute: attribute, 67 | bidx_attribute: bidx_attribute, 68 | migrating: migrating 69 | ) 70 | 71 | define_singleton_method class_method_name do |value| 72 | BlindIndex.generate_bidx(value, **blind_indexes[name]) 73 | end 74 | 75 | define_method method_name do 76 | send("#{bidx_attribute}=", self.class.send(class_method_name, send(attribute))) 77 | end 78 | 79 | if callback 80 | # TODO reuse module 81 | m = Module.new do 82 | define_method "#{attribute}=" do |value| 83 | result = super(value) 84 | send(method_name) 85 | result 86 | end 87 | 88 | unless activerecord 89 | define_method "reset_#{attribute}!" do 90 | result = super() 91 | send(method_name) 92 | result 93 | end 94 | end 95 | end 96 | prepend m 97 | end 98 | 99 | # use include so user can override 100 | include InstanceMethods if blind_indexes.size == 1 101 | end 102 | end 103 | end 104 | 105 | module InstanceMethods 106 | def read_attribute_for_validation(key) 107 | if (bi = self.class.blind_indexes[key]) 108 | send(bi[:attribute]) 109 | else 110 | super 111 | end 112 | end 113 | end 114 | end 115 | end 116 | -------------------------------------------------------------------------------- /lib/blind_index/mongoid.rb: -------------------------------------------------------------------------------- 1 | module BlindIndex 2 | module Mongoid 3 | module Criteria 4 | private 5 | 6 | def expr_query(criterion) 7 | if criterion.is_a?(Hash) && klass.respond_to?(:blind_indexes) 8 | criterion.keys.each do |key| 9 | key_sym = (key.is_a?(::Mongoid::Criteria::Queryable::Key) ? key.name : key).to_sym 10 | 11 | if (bi = klass.blind_indexes[key_sym]) 12 | value = criterion.delete(key) 13 | 14 | bidx_key = 15 | if key.is_a?(::Mongoid::Criteria::Queryable::Key) 16 | ::Mongoid::Criteria::Queryable::Key.new( 17 | bi[:bidx_attribute], 18 | key.strategy, 19 | key.operator, 20 | key.expanded, 21 | &key.block 22 | ) 23 | else 24 | bi[:bidx_attribute] 25 | end 26 | 27 | criterion[bidx_key] = 28 | if value.is_a?(Array) 29 | value.map { |v| BlindIndex.generate_bidx(v, **bi) } 30 | else 31 | BlindIndex.generate_bidx(value, **bi) 32 | end 33 | end 34 | end 35 | end 36 | 37 | super(criterion) 38 | end 39 | end 40 | 41 | module UniquenessValidator 42 | def validate_each(record, attribute, value) 43 | klass = record.class 44 | if klass.respond_to?(:blind_indexes) && (bi = klass.blind_indexes[attribute]) 45 | value = record.read_attribute_for_validation(bi[:bidx_attribute]) 46 | end 47 | super(record, attribute, value) 48 | end 49 | 50 | # change attribute name here instead of validate_each for better error message 51 | def create_criteria(base, document, attribute, value) 52 | klass = document.class 53 | if klass.respond_to?(:blind_indexes) && (bi = klass.blind_indexes[attribute]) 54 | attribute = bi[:bidx_attribute] 55 | end 56 | super(base, document, attribute, value) 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/blind_index/version.rb: -------------------------------------------------------------------------------- 1 | module BlindIndex 2 | VERSION = "2.7.0" 3 | end 4 | -------------------------------------------------------------------------------- /test/blind_index_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class BlindIndexTest < Minitest::Test 4 | def setup 5 | User.delete_all 6 | end 7 | 8 | def test_find_by 9 | create_user 10 | assert User.find_by(email: "test@example.org") 11 | end 12 | 13 | def test_find_by_string_key 14 | create_user 15 | assert User.find_by({"email" => "test@example.org"}) 16 | end 17 | 18 | def test_find_or_create_by 19 | user = create_user 20 | assert_equal user, User.find_or_create_by(email: "test@example.org") 21 | end 22 | 23 | def test_delete_by 24 | skip if mongoid? 25 | 26 | create_user 27 | assert_equal 1, User.delete_by(email: "test@example.org") 28 | assert_equal 0, User.count 29 | end 30 | 31 | def test_destroy_by 32 | skip if mongoid? 33 | 34 | user = create_user 35 | assert_equal [user], User.destroy_by(email: "test@example.org") 36 | assert_equal 0, User.count 37 | end 38 | 39 | def test_dynamic_finders 40 | skip if mongoid? 41 | 42 | user = create_user 43 | assert User.find_by_email("test@example.org") 44 | assert User.find_by_id_and_email(user.id, "test@example.org") 45 | end 46 | 47 | def test_where 48 | create_user 49 | assert User.where(email: "test@example.org").first 50 | end 51 | 52 | def test_where_table 53 | skip if mongoid? 54 | 55 | create_user 56 | assert User.where(users: {email: "test@example.org"}).first 57 | end 58 | 59 | def test_where_array 60 | create_user 61 | create_user(email: "test2@example.org") 62 | key = mongoid? ? :email.in : :email 63 | assert_equal 2, User.where(key => ["test@example.org", "test2@example.org"]).count 64 | end 65 | 66 | def test_where_string_key 67 | create_user 68 | assert User.where({"email" => "test@example.org"}).first 69 | end 70 | 71 | def test_where_not 72 | skip if mongoid? 73 | 74 | create_user 75 | assert User.where.not(email: "test2@example.org").first 76 | end 77 | 78 | def test_where_not_empty 79 | skip if mongoid? 80 | 81 | create_user 82 | assert_nil User.where.not(email: "test@example.org").first 83 | end 84 | 85 | def test_expression 86 | create_user 87 | assert User.where(email_ci: "TEST@example.org").first 88 | end 89 | 90 | def test_expression_different_case 91 | create_user email: "TEST@example.org" 92 | assert User.where(email_ci: "test@example.org").first 93 | end 94 | 95 | def test_encode 96 | skip if mongoid? 97 | 98 | user = create_user 99 | assert User.find_by(email_binary: "test@example.org") 100 | assert_equal 32, user.email_binary_bidx.bytesize 101 | end 102 | 103 | def test_validation 104 | create_user 105 | user = User.new(email: "test@example.org") 106 | assert !user.valid? 107 | expected = mongoid? && Mongoid::VERSION.to_i < 8 ? "Email is already taken" : "Email has already been taken" 108 | assert_equal expected, user.errors.full_messages.first 109 | end 110 | 111 | def test_validation_case_insensitive 112 | create_user 113 | user = User.new(email: "TEST@example.org") 114 | assert !user.valid? 115 | expected = mongoid? && Mongoid::VERSION.to_i < 8 ? "Email ci is already taken" : "Email ci has already been taken" 116 | assert_equal expected, user.errors.full_messages.first 117 | end 118 | 119 | def test_validation_allow_blank 120 | User.create!(email: "") 121 | user = User.new(email: "") 122 | assert user.valid? 123 | assert_raises do 124 | user.save! # index prevents saving 125 | end 126 | end 127 | 128 | def test_validation_allow_blank_nil 129 | User.create!(email: nil) 130 | user = User.new(email: nil) 131 | assert user.valid? 132 | assert user.save! 133 | end 134 | 135 | def test_nil 136 | user = create_user(email: nil) 137 | assert_nil user.email_bidx 138 | assert User.where(email: nil).first 139 | assert_nil User.where(email: "").first 140 | end 141 | 142 | def test_empty_string 143 | user = create_user(email: "") 144 | assert user.email_bidx 145 | assert User.where(email: "").first 146 | assert_nil User.where(email: nil).first 147 | end 148 | 149 | def test_unset 150 | user = create_user 151 | user.email = nil 152 | user.save! 153 | assert_nil user.email_bidx 154 | assert User.where(email: nil).first 155 | end 156 | 157 | def test_class_method 158 | user = create_user 159 | assert_equal user.email_bidx, User.generate_email_bidx("test@example.org") 160 | end 161 | 162 | def test_key_bad_length 163 | error = assert_raises(BlindIndex::Error) do 164 | BlindIndex.generate_bidx("test@example.org", key: SecureRandom.hex(31)) 165 | end 166 | assert_equal "Key must be 32 bytes (64 hex digits)", error.message 167 | end 168 | 169 | def test_key_bad_encoding 170 | error = assert_raises(BlindIndex::Error) do 171 | BlindIndex.generate_bidx("test@example.org", key: SecureRandom.hex(16)) 172 | end 173 | assert_equal "Key must use binary encoding", error.message 174 | end 175 | 176 | def test_master_key_bad_length 177 | with_master_key(SecureRandom.hex(31)) do 178 | error = assert_raises(BlindIndex::Error) do 179 | BlindIndex.index_key(table: "users", bidx_attribute: "test") 180 | end 181 | assert_equal "Master key must be 32 bytes (64 hex digits)", error.message 182 | end 183 | end 184 | 185 | def test_master_key_bad_encoding 186 | with_master_key(SecureRandom.hex(16)) do 187 | error = assert_raises(BlindIndex::Error) do 188 | BlindIndex.index_key(table: "users", bidx_attribute: "test") 189 | end 190 | assert_equal "Master key must use binary encoding", error.message 191 | end 192 | end 193 | 194 | def test_inheritance 195 | assert_equal %i[email email_ci email_binary initials phone city rotated_city region], User.blind_indexes.keys 196 | assert_equal User.blind_indexes.keys + %i[child], ActiveUser.blind_indexes.keys 197 | end 198 | 199 | def test_initials 200 | create_user(first_name: "Test", last_name: "User") 201 | assert User.find_by(initials: "TU") 202 | 203 | user = User.last 204 | user.email = "test2@example.org" 205 | user.save! 206 | assert User.find_by(initials: "TU") 207 | end 208 | 209 | def test_size 210 | result = BlindIndex.generate_bidx("secret", key: random_key, size: 16, encode: false) 211 | assert_equal 16, result.bytesize 212 | end 213 | 214 | def test_invalid_size 215 | error = assert_raises(BlindIndex::Error) do 216 | BlindIndex.generate_bidx("secret", key: random_key, size: 0, encode: false) 217 | end 218 | assert_equal "Size must be between 1 and 32", error.message 219 | end 220 | 221 | def test_index_key 222 | index_key = BlindIndex.index_key(table: "users", bidx_attribute: "email_bidx", master_key: "0"*64) 223 | assert_equal "289737bab72fa97b1f4b081cef00d7b7d75034bcf3183c363feaf3e6441777bc", index_key 224 | end 225 | 226 | def test_default_algorithm 227 | create_user 228 | expected = BlindIndex.generate_bidx("test@example.org", algorithm: :argon2id, key: User.blind_indexes[:email][:key]) 229 | assert_equal expected, User.last.email_bidx 230 | end 231 | 232 | def test_attr_encrypted 233 | create_user(phone: "555-555-5555") 234 | assert User.find_by(phone: "555-555-5555") 235 | end 236 | 237 | def test_lockbox_restore 238 | user = User.new 239 | user.email = "test@example.org" 240 | assert user.email 241 | assert user.email_ciphertext 242 | assert user.email_bidx 243 | if mongoid? 244 | user.reset_email! 245 | else 246 | user.restore_email! 247 | end 248 | assert_nil user.email 249 | assert_nil user.email_ciphertext 250 | assert_nil user.email_bidx 251 | end 252 | 253 | def test_set 254 | user = User.new 255 | user.email = "test@example.org" 256 | assert_equal User.generate_email_bidx("test@example.org"), user.email_bidx 257 | end 258 | 259 | def test_update_attribute 260 | user = create_user 261 | user.update_attribute(:email, "new@example.org") 262 | assert_equal User.generate_email_bidx("new@example.org"), user.email_bidx 263 | end 264 | 265 | def test_validate_false 266 | user = User.new 267 | user.email = "test@example.org" 268 | user.save(validate: false) 269 | assert_equal User.generate_email_bidx("test@example.org"), user.email_bidx 270 | end 271 | 272 | def test_backfill 273 | 10.times do |i| 274 | User.create!(email: "test#{i}@example.org") 275 | end 276 | User.update_all(email_bidx: nil) 277 | 278 | assert_equal 0, User.where(email: "test9@example.org").count 279 | BlindIndex.backfill(User, columns: [:email_bidx], batch_size: 5) 280 | assert_equal 1, User.where(email: "test9@example.org").count 281 | end 282 | 283 | def test_backfill_relation 284 | 10.times do |i| 285 | User.create!(email: "test#{i}@example.org") 286 | end 287 | last_id = User.last.id 288 | User.update_all(email_bidx: nil) 289 | 290 | assert_equal 0, User.where(email: "test9@example.org").count 291 | BlindIndex.backfill(User.where(id: last_id)) 292 | assert_equal 0, User.where(email: "test8@example.org").count 293 | assert_equal 1, User.where(email: "test9@example.org").count 294 | end 295 | 296 | def test_backfill_bad_column 297 | error = assert_raises(ArgumentError) do 298 | BlindIndex.backfill(User, columns: [:bad]) 299 | end 300 | assert_equal "Bad column: bad", error.message 301 | end 302 | 303 | def test_manual_backfill 304 | create_user 305 | User.update_all(email_bidx: nil) 306 | user = User.last 307 | assert_nil user.email_bidx 308 | user.compute_email_bidx 309 | assert user.email_bidx 310 | end 311 | 312 | def test_version 313 | create_user(city: "Test") 314 | assert User.find_by(city: "Test") 315 | end 316 | 317 | def test_rotate 318 | user = create_user(city: "Test") 319 | assert user.city_bidx_v2 320 | assert user.city_bidx_v3 321 | refute_equal user.city_bidx_v2, user.city_bidx_v3 322 | 323 | assert User.find_by(city: "Test") 324 | 325 | # non-public 326 | assert User.find_by(rotated_city: "Test") 327 | end 328 | 329 | def test_association 330 | Group.delete_all 331 | group = Group.create! 332 | create_user(group: group) 333 | assert group.users.find_by(email: "test@example.org") 334 | end 335 | 336 | def test_joins 337 | skip if mongoid? 338 | 339 | Group.delete_all 340 | group = Group.create! 341 | create_user(group: group) 342 | assert Group.joins(:users).where(users: {email: "test@example.org"}).first 343 | end 344 | 345 | def test_inspect_filter_attributes 346 | skip if mongoid? 347 | 348 | previous_value = User.filter_attributes 349 | 350 | begin 351 | user = create_user 352 | assert_includes user.inspect, "email_bidx: [FILTERED]" 353 | 354 | # Active Record still shows nil for filtered attributes 355 | user = create_user(email: nil) 356 | assert_includes user.inspect, "name: nil" 357 | ensure 358 | User.filter_attributes = previous_value 359 | end 360 | end 361 | 362 | def test_normalizes 363 | skip if mongoid? 364 | 365 | User.create!(region: "Test") 366 | assert User.find_by(region: "test") 367 | assert User.find_by(region: "TEST") 368 | assert User.where(region: ["test"]).first 369 | assert User.where(region: ["TEST"]).first 370 | end 371 | 372 | private 373 | 374 | def random_key 375 | SecureRandom.random_bytes(32) 376 | end 377 | 378 | def with_master_key(key) 379 | BlindIndex.stub(:master_key, key) do 380 | yield 381 | end 382 | end 383 | 384 | def create_user(email: "test@example.org", **attributes) 385 | User.create!({email: email}.merge(attributes)) 386 | end 387 | 388 | def mongoid? 389 | defined?(Mongoid) 390 | end 391 | end 392 | -------------------------------------------------------------------------------- /test/support/active_record.rb: -------------------------------------------------------------------------------- 1 | ActiveRecord::Base.logger = $logger 2 | ActiveRecord::Migration.verbose = ENV["VERBOSE"] 3 | 4 | adapter = ENV["ADAPTER"] || "sqlite" 5 | case adapter 6 | when "sqlite" 7 | ActiveRecord::Base.establish_connection adapter: "sqlite3", database: ":memory:" 8 | when "postgresql" 9 | ActiveRecord::Base.establish_connection adapter: "postgresql", database: "blind_index_test" 10 | when "mysql" 11 | ActiveRecord::Base.establish_connection adapter: "mysql2", database: "blind_index_test" 12 | when "trilogy" 13 | ActiveRecord::Base.establish_connection adapter: "trilogy", database: "blind_index_test", host: "127.0.0.1" 14 | else 15 | raise "Unknown adapter: #{adapter}" 16 | end 17 | 18 | ActiveRecord::Schema.define do 19 | create_table :users, force: true do |t| 20 | t.string :email_ciphertext 21 | t.string :email_bidx, index: {unique: true} 22 | t.string :email_ci_bidx, index: {unique: true} 23 | t.binary :email_binary_bidx 24 | t.string :first_name_ciphertext 25 | t.string :last_name_ciphertext 26 | t.string :initials_bidx 27 | t.string :encrypted_phone 28 | t.string :encrypted_phone_iv 29 | t.string :phone_bidx 30 | t.string :city_ciphertext 31 | t.string :city_bidx_v2 32 | t.string :city_bidx_v3 33 | t.string :region_ciphertext 34 | t.string :region_bidx 35 | t.references :group 36 | end 37 | 38 | create_table :admins, force: true do |t| 39 | t.string :email_ciphertext 40 | t.string :email_bidx 41 | end 42 | 43 | create_table :groups, force: true do |t| 44 | end 45 | end 46 | 47 | class User < ActiveRecord::Base 48 | attribute :initials, :string 49 | 50 | has_encrypted :email, :first_name, :last_name, :city, :region 51 | 52 | normalizes :region, with: ->(v) { v&.downcase } 53 | 54 | attr_encrypted :phone, key: SecureRandom.random_bytes(32) 55 | 56 | # ensure custom method still works 57 | def read_attribute_for_validation(key) 58 | super 59 | end 60 | end 61 | 62 | class Admin < ActiveRecord::Base 63 | has_encrypted :email 64 | blind_index :email 65 | 66 | belongs_to :user, primary_key: "email", foreign_key: "email", optional: true 67 | end 68 | 69 | class Group < ActiveRecord::Base 70 | end 71 | -------------------------------------------------------------------------------- /test/support/mongoid.rb: -------------------------------------------------------------------------------- 1 | Mongoid.logger = $logger 2 | Mongo::Logger.logger = $logger if defined?(Mongo::Logger) 3 | 4 | Mongoid.configure do |config| 5 | config.connect_to "blind_index_test" 6 | end 7 | 8 | class User 9 | include Mongoid::Document 10 | 11 | field :email_ciphertext, type: String 12 | field :email_bidx, type: String 13 | field :email_ci_bidx, type: String 14 | field :email_binary_bidx, type: String 15 | field :first_name, type: String 16 | field :last_name, type: String 17 | field :initials, type: String # Mongoid doesn't have virtual attributes 18 | field :initials_bidx, type: String 19 | field :phone_ciphertext, type: String 20 | field :phone_bidx, type: String 21 | field :city_ciphertext, type: String 22 | field :city_bidx_v2, type: String 23 | field :city_bidx_v3, type: String 24 | field :region_ciphertext, type: String 25 | field :region_bidx, type: String 26 | 27 | has_encrypted :email, :phone, :city, :region 28 | 29 | index({email_bidx: 1}, {unique: true, partial_filter_expression: {email_bidx: {"$type" => "string"}}}) 30 | index({email_ci_bidx: 1}, {unique: true, partial_filter_expression: {email_ci_bidx: {"$type" => "string"}}}) 31 | end 32 | 33 | User.create_indexes 34 | 35 | class Group 36 | include Mongoid::Document 37 | end 38 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | 3 | begin 4 | require "active_record" 5 | rescue LoadError 6 | end 7 | 8 | begin 9 | require "mongoid" 10 | rescue LoadError 11 | end 12 | 13 | Bundler.require(:default) 14 | require "minitest/autorun" 15 | require "minitest/pride" 16 | 17 | BlindIndex.master_key = BlindIndex.generate_key 18 | 19 | Lockbox.master_key = Lockbox.generate_key 20 | 21 | $logger = ActiveSupport::Logger.new(ENV["VERBOSE"] ? STDOUT : nil) 22 | 23 | if defined?(Mongoid) 24 | require_relative "support/mongoid" 25 | else 26 | require_relative "support/active_record" 27 | end 28 | 29 | class User 30 | belongs_to :group, optional: true 31 | 32 | blind_index :email 33 | blind_index :email_ci, algorithm: (RUBY_ENGINE == "jruby" ? nil : :scrypt), attribute: :email, expression: ->(v) { v.try(:downcase) } 34 | blind_index :email_binary, algorithm: :argon2, key: BlindIndex.generate_key, attribute: :email, encode: defined?(Mongoid) # can't get binary working with Mongoid 35 | blind_index :initials, key: BlindIndex.generate_key, size: 16 36 | blind_index :phone, algorithm: :pbkdf2_sha256 37 | blind_index :city, version: 2, rotate: {version: 3, master_key: BlindIndex.generate_key} 38 | blind_index :region 39 | 40 | validates :email, uniqueness: {allow_blank: true} 41 | validates :email_ci, uniqueness: {allow_blank: true} 42 | 43 | before_validation :set_initials, if: -> { changes.key?(:first_name) || changes.key?(:last_name) } 44 | 45 | def set_initials 46 | self.initials = [first_name.first, last_name.first].join 47 | end 48 | end 49 | 50 | unless defined?(Mongoid) 51 | # ensure blind_index does not cause model schema to load 52 | raise "blind_index loading model schema early" if User.send(:schema_loaded?) 53 | end 54 | 55 | class ActiveUser < User 56 | blind_index :child, key: BlindIndex.generate_key 57 | end 58 | 59 | class Group 60 | has_many :users 61 | end 62 | --------------------------------------------------------------------------------