├── .travis.yml ├── Gemfile ├── .rspec ├── lib ├── shameless.rb └── shameless │ ├── version.rb │ ├── errors.rb │ ├── configuration.rb │ ├── index.rb │ ├── cell.rb │ ├── store.rb │ └── model.rb ├── .gitignore ├── Rakefile ├── spec ├── shameless_spec.rb ├── spec_helper.rb ├── shameless │ ├── index_spec.rb │ ├── store_spec.rb │ ├── cell_spec.rb │ └── model_spec.rb └── support │ └── build_store.rb ├── bin ├── setup └── console ├── LICENSE.txt ├── shameless.gemspec ├── CHANGELOG.md └── README.md /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.3.2 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /lib/shameless.rb: -------------------------------------------------------------------------------- 1 | require 'shameless/store' 2 | require 'shameless/version' 3 | -------------------------------------------------------------------------------- /lib/shameless/version.rb: -------------------------------------------------------------------------------- 1 | module Shameless 2 | VERSION = "0.7.0" 3 | end 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /Gemfile.lock 4 | /_yardoc/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | -------------------------------------------------------------------------------- /lib/shameless/errors.rb: -------------------------------------------------------------------------------- 1 | module Shameless 2 | Error = Class.new(StandardError) 3 | ReadonlyAttributeMutation = Class.new(Error) 4 | end 5 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task default: :spec 7 | -------------------------------------------------------------------------------- /spec/shameless_spec.rb: -------------------------------------------------------------------------------- 1 | describe Shameless do 2 | it 'has a version number' do 3 | expect(Shameless::VERSION).not_to be nil 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) 2 | require 'shameless' 3 | 4 | Dir[File.dirname(__FILE__) + '/support/**/*.rb'].each {|file| require file } 5 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "shameless" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start 15 | -------------------------------------------------------------------------------- /lib/shameless/configuration.rb: -------------------------------------------------------------------------------- 1 | module Shameless 2 | class Configuration 3 | attr_accessor :partition_urls, :shards_count, :connection_options, :database_extensions, 4 | :create_table_options 5 | 6 | # Needed to deal with our legacy schema that stores created_at as an integer timestamp 7 | # and does date conversions in Ruby-land, don't set to `true` for new projects 8 | attr_accessor :legacy_created_at_is_bigint 9 | 10 | def shards_per_partition_count 11 | shards_count / partitions_count 12 | end 13 | 14 | def partitions_count 15 | partition_urls.count 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 HotelTonight 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /shameless.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require 'shameless/version' 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "shameless" 7 | spec.version = Shameless::VERSION 8 | spec.authors = ["Olek Janiszewski", "Chas Lemley", "Marek Rosa", "Kai Rubarth"] 9 | spec.email = ["olek@hoteltonight.com", "chas@hoteltonight.com", "marek@hoteltonight.com", "kai.rubarth@hoteltonight.com"] 10 | 11 | spec.summary = %q{Scalable distributed append-only data store} 12 | spec.homepage = "https://github.com/hoteltonight/shameless" 13 | spec.license = "MIT" 14 | 15 | spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 16 | spec.bindir = "exe" 17 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 18 | spec.require_paths = ["lib"] 19 | 20 | spec.add_dependency "msgpack" 21 | spec.add_dependency "sequel", "~> 4.0" 22 | 23 | spec.add_development_dependency "bundler" 24 | spec.add_development_dependency "rake" 25 | spec.add_development_dependency "rspec" 26 | spec.add_development_dependency "sqlite3" 27 | end 28 | -------------------------------------------------------------------------------- /spec/shameless/index_spec.rb: -------------------------------------------------------------------------------- 1 | describe Shameless::Index do 2 | before(:context) do 3 | build_store do |store| 4 | Store = store 5 | 6 | class MyModel 7 | Store.attach(self) 8 | 9 | index do 10 | integer :primary_id 11 | shard_on :primary_id 12 | end 13 | 14 | index :foo do 15 | integer :my_id 16 | string :check_in_date 17 | shard_on :my_id 18 | end 19 | end 20 | end 21 | end 22 | 23 | after(:context) do 24 | Object.send(:remove_const, :MyModel) 25 | Object.send(:remove_const, :Store) 26 | end 27 | 28 | let(:today) { Date.today.to_s } 29 | let(:tomorrow) { (Date.today + 1).to_s } 30 | 31 | it 'allows querying by a named index' do 32 | MyModel.put(primary_id: 1, my_id: 1, foo: 'bar', check_in_date: today) 33 | MyModel.put(primary_id: 2, my_id: 1, foo: 'baz', check_in_date: tomorrow) 34 | 35 | results = MyModel.foo_index.where(my_id: 1) 36 | 37 | expect(results.size).to eq(2) 38 | expect(results.first[:foo]).to eq('bar') 39 | end 40 | 41 | it 'allows querying with block filters' do 42 | MyModel.put(primary_id: 1, my_id: 1, check_in_date: today) 43 | MyModel.put(primary_id: 2, my_id: 1, check_in_date: tomorrow) 44 | 45 | results = MyModel.foo_index.where(my_id: 1) { |o| o.check_in_date > today } 46 | 47 | expect(results.size).to eq(1) 48 | expect(results.first[:check_in_date]).to eq(tomorrow) 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /spec/support/build_store.rb: -------------------------------------------------------------------------------- 1 | RSpec.configure do |c| 2 | c.include(Module.new do 3 | def build_store(name: :store, partitions_count: 1, connection_options: nil, database_extensions: nil, 4 | legacy_created_at_is_bigint: nil, create_table_options: nil, &block) 5 | store = Shameless::Store.new(name) do |c| 6 | c.partition_urls = Array.new(partitions_count) { 'sqlite:/' } 7 | c.shards_count = 4 8 | c.connection_options = connection_options 9 | c.database_extensions = database_extensions 10 | c.legacy_created_at_is_bigint = legacy_created_at_is_bigint 11 | c.create_table_options = create_table_options 12 | end 13 | 14 | model = Class.new do 15 | store.attach(self, :rates) 16 | 17 | index do 18 | integer :hotel_id 19 | string :room_type 20 | string :check_in_date 21 | 22 | shard_on :hotel_id 23 | end 24 | end 25 | 26 | block.call(store) if block 27 | 28 | store.create_tables! 29 | 30 | [store, model] 31 | end 32 | 33 | def build_model_with_cell 34 | store, _ = build_store 35 | Class.new do 36 | store.attach(self, :rates) 37 | 38 | index do 39 | integer :hotel_id 40 | string :room_type 41 | string :check_in_date 42 | 43 | shard_on :hotel_id 44 | end 45 | 46 | cell :meta 47 | cell :ota_rate 48 | end 49 | end 50 | end) 51 | end 52 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ### Unreleased 2 | 3 | ### 0.7.0 (2019-04-23) 4 | 5 | * Add `Model#max_id_on_shard` 6 | 7 | ### 0.6.1 (2017-07-31) 8 | 9 | * Fix Sequel deprecation warning 10 | 11 | ### 0.6.0 (2017-07-26) 12 | 13 | * Require `securerandom` explicitly 14 | * Add support for queries using Sequel's virtual rows 15 | 16 | ### 0.5.2 (2016-12-02) 17 | 18 | * Eagerly initialize `Model#cells` 19 | 20 | ### 0.5.1 (2016-12-01) 21 | 22 | * Convert shardable value to integer for index tables 23 | 24 | ### 0.5.0 (2016-12-01) 25 | 26 | * Add `Model#as_json` and `Cell#as_json` 27 | * Remember cell ID after save 28 | 29 | ### 0.4.0 (2016-11-21) 30 | 31 | * `Model#reload` now reloads all cells 32 | * Add `Model#cells` 33 | * Add `Configuration#create_table_options` 34 | * Include shard in underlying index name 35 | 36 | ### 0.3.1 (2016-11-18) 37 | 38 | * Use more static names for underlying database indices 39 | * Use database-portable data types for index columns 40 | 41 | ### 0.3.0 (2016-11-18) 42 | 43 | * Add `Cell#uuid` 44 | * Add `Cell#id` 45 | * Add `Model.fetch_latest_cells` 46 | * Initialize `ref_key` with zero, not one 47 | * Add `Model#present?` and `Cell#present?` 48 | * Allow `Cell#save` to be called even without making any changes 49 | * Add `Model#fetch` and `Cell#fetch` 50 | * Add `Model#reload` and `Cell#reload` 51 | * Add `Model#previous` and `Cell#previous` 52 | * `Model.put` now correctly looks up and updates an existing instance 53 | * Add `Model#update` and `Cell#update` 54 | * Expose `Model#base` 55 | * Add `Configuration#legacy_created_at_is_bigint` 56 | * Keep a reference to only one model class per table name 57 | * Make `Store#find_shard` public 58 | * Make `Store#each_shard` public 59 | * Name index tables `*_:name_index_*` 60 | * Add `Configuration#database_extensions`, they're being passed to the Sequel adapter 61 | * Add `Configuration#connection_options`, they're being passed to the Sequel adapter 62 | * Don't prefix table names with underscore when store name is `nil` 63 | * Add `Store#each_partition` 64 | * Add `Store#disconnect` 65 | 66 | ### 0.2.0 (2016-11-14) 67 | 68 | * Add `Store#padded_shard` to get the formatted shard number for a shardable value 69 | 70 | ### 0.1.0 (2016-10-14) 71 | 72 | * Initial release 73 | -------------------------------------------------------------------------------- /lib/shameless/index.rb: -------------------------------------------------------------------------------- 1 | require 'shameless/errors' 2 | 3 | module Shameless 4 | class Index 5 | PRIMARY = :primary 6 | 7 | attr_reader :name 8 | 9 | def initialize(name, model, &block) 10 | @name = name || PRIMARY 11 | @model = model 12 | instance_eval(&block) 13 | end 14 | 15 | DataTypes = {integer: Integer, string: String} 16 | 17 | DataTypes.each do |name, type| 18 | define_method(name) do |column| 19 | self.column(column, type) 20 | end 21 | end 22 | 23 | def column(name, type) 24 | @columns ||= {} 25 | @columns[name] = type 26 | end 27 | 28 | def shard_on(shard_on) 29 | @shard_on = shard_on 30 | end 31 | 32 | def put(values) 33 | shardable_value = values.fetch(@shard_on).to_i 34 | index_values = index_values(values, true) 35 | 36 | @model.store.put(table_name, shardable_value, index_values) 37 | end 38 | 39 | def where(query, &block) 40 | shardable_value = query.fetch(@shard_on).to_i 41 | query = index_values(query, false) 42 | @model.store.where(table_name, shardable_value, query, &block).map {|r| @model.new(r[:uuid]) } 43 | end 44 | 45 | def table_name 46 | "#{@model.table_name}_#{full_name}" 47 | end 48 | 49 | def full_name 50 | "#{@name}_index" 51 | end 52 | 53 | def index_values(values, all_required) 54 | (@columns.keys + [:uuid]).each_with_object({}) do |column, o| 55 | if all_required 56 | o[column] = values.fetch(column) 57 | else 58 | o[column] = values[column] if values.key?(column) 59 | end 60 | end 61 | end 62 | 63 | def create_tables! 64 | @model.store.create_table!(table_name) do |t, sharded_table_name| 65 | @columns.each do |name, type| 66 | t.column name, type, null: false 67 | end 68 | 69 | t.varchar :uuid, size: 36 70 | 71 | t.index @columns.keys, name: "#{sharded_table_name}_index", unique: true 72 | end 73 | end 74 | 75 | def column?(key) 76 | @columns.keys.any? {|c| c.to_s == key.to_s } 77 | end 78 | 79 | def prevent_readonly_attribute_mutation!(key) 80 | if column?(key) 81 | raise ReadonlyAttributeMutation, "The attribute #{key} cannot be modified because it's part of the #{@name} index" 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/shameless/cell.rb: -------------------------------------------------------------------------------- 1 | require 'msgpack' 2 | 3 | module Shameless 4 | class Cell 5 | BASE = 'base' 6 | 7 | def self.base(model, body) 8 | serialized_body = serialize_body(body) 9 | new(model, BASE, body: serialized_body) 10 | end 11 | 12 | def self.serialize_body(body) 13 | MessagePack.pack(body) 14 | end 15 | 16 | attr_reader :model, :name, :id 17 | 18 | def initialize(model, name, values = nil) 19 | @model = model 20 | @name = name 21 | reload 22 | initialize_from_values(values) 23 | end 24 | 25 | def [](key) 26 | body[key.to_s] 27 | end 28 | 29 | def []=(key, value) 30 | @model.prevent_readonly_attribute_mutation!(key) 31 | body[key.to_s] = value 32 | end 33 | 34 | def save 35 | load 36 | @created_at = Time.now 37 | @created_at = (@created_at.to_f * 1000).to_i if @model.class.store.configuration.legacy_created_at_is_bigint 38 | @ref_key ||= -1 39 | @ref_key += 1 40 | @id = @model.put_cell(cell_values(true)) 41 | end 42 | 43 | def update(values) 44 | values.each do |key, value| 45 | self[key] = value 46 | end 47 | 48 | save 49 | end 50 | 51 | def ref_key 52 | load 53 | @ref_key 54 | end 55 | 56 | def created_at 57 | load 58 | @created_at 59 | end 60 | 61 | def body 62 | load 63 | @body 64 | end 65 | 66 | def previous 67 | if ref_key && previous_cell_values = @model.fetch_cell(@name, ref_key - 1) 68 | self.class.new(@model, @name, previous_cell_values) 69 | end 70 | end 71 | 72 | def reload 73 | @id = @body = @ref_key = @created_at = nil 74 | end 75 | 76 | def fetch(key, default) 77 | body.key?(key.to_s) ? self[key] : default 78 | end 79 | 80 | def present? 81 | load 82 | !@ref_key.nil? 83 | end 84 | 85 | def uuid 86 | @model.uuid 87 | end 88 | 89 | def as_json(*) 90 | cell_values(false).merge(id: id) 91 | end 92 | 93 | private 94 | 95 | def cell_values(serialize_body) 96 | { 97 | uuid: uuid, 98 | column_name: @name, 99 | ref_key: ref_key, 100 | created_at: created_at, 101 | body: serialize_body ? serialized_body : body 102 | } 103 | end 104 | 105 | def serialized_body 106 | self.class.serialize_body(body) 107 | end 108 | 109 | def deserialize_body(body) 110 | MessagePack.unpack(body) 111 | end 112 | 113 | private 114 | 115 | def load 116 | if @body.nil? 117 | values = @model.fetch_cell(@name) 118 | initialize_from_values(values) 119 | @body ||= {} 120 | end 121 | end 122 | 123 | def initialize_from_values(values) 124 | if values 125 | @id = values[:id] 126 | @body = deserialize_body(values[:body]) 127 | @ref_key = values[:ref_key] 128 | @created_at = values[:created_at] 129 | end 130 | end 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /lib/shameless/store.rb: -------------------------------------------------------------------------------- 1 | require 'sequel' 2 | require 'shameless/configuration' 3 | require 'shameless/model' 4 | 5 | module Shameless 6 | class Store 7 | attr_reader :name, :configuration 8 | 9 | def initialize(name, &block) 10 | @name = name 11 | @configuration = Configuration.new 12 | block.call(@configuration) 13 | end 14 | 15 | def attach(model_class, name = nil) 16 | model_class.extend(Model) 17 | model_class.attach_to(self, name) 18 | models_hash[name] = model_class 19 | end 20 | 21 | def put(table_name, shardable_value, values) 22 | find_table(table_name, shardable_value).insert(values) 23 | end 24 | 25 | def where(table_name, shardable_value, query, &block) 26 | find_table(table_name, shardable_value).where(query, &block) 27 | end 28 | 29 | def disconnect 30 | if instance_variable_defined?(:@partitions) 31 | partitions.each(&:disconnect) 32 | end 33 | end 34 | 35 | def each_partition(&block) 36 | partitions.each do |partition| 37 | block.call(partition, table_names_on_partition(partition)) 38 | end 39 | end 40 | 41 | def create_tables! 42 | models.each(&:create_tables!) 43 | end 44 | 45 | def create_table!(table_name, &block) 46 | each_shard do |shard| 47 | partition = find_partition_for_shard(shard) 48 | sharded_table_name = table_name_with_shard(table_name, shard) 49 | options = @configuration.create_table_options || {} 50 | partition.create_table(sharded_table_name, options) { block.call(self, sharded_table_name) } 51 | end 52 | end 53 | 54 | def padded_shard(shardable_value) 55 | shard = find_shard(shardable_value) 56 | format_shard(shard) 57 | end 58 | 59 | def each_shard(&block) 60 | 0.upto(@configuration.shards_count - 1, &block) 61 | end 62 | 63 | def find_shard(shardable_value) 64 | shardable_value % @configuration.shards_count 65 | end 66 | 67 | def find_table(table_name, shardable_value) 68 | shard = find_shard(shardable_value) 69 | partition = find_partition_for_shard(shard) 70 | table_name = table_name_with_shard(table_name, shard) 71 | partition.from(table_name) 72 | end 73 | 74 | private 75 | 76 | def models_hash 77 | @models_hash ||= {} 78 | end 79 | 80 | def models 81 | models_hash.values 82 | end 83 | 84 | def partitions 85 | @partitions ||= @configuration.partition_urls.map {|url| connect(url) } 86 | end 87 | 88 | def connect(url) 89 | Sequel.connect(url, @configuration.connection_options || Sequel::OPTS).tap do |db| 90 | db.extension(*@configuration.database_extensions) 91 | end 92 | end 93 | 94 | def table_names_on_partition(partition) 95 | partition_index = partitions.index(partition) 96 | first_shard = partition_index * @configuration.shards_per_partition_count 97 | last_shard = first_shard + @configuration.shards_per_partition_count - 1 98 | shards = first_shard..last_shard 99 | table_names = models.flat_map(&:table_names) 100 | 101 | table_names.flat_map {|t| shards.map {|s| table_name_with_shard(t, s) } } 102 | end 103 | 104 | def table_name_with_shard(table_name, shard) 105 | padded_shard = format_shard(shard) 106 | "#{table_name}_#{padded_shard}" 107 | end 108 | 109 | def format_shard(shard) 110 | shard.to_s.rjust(6, '0') 111 | end 112 | 113 | def find_partition_for_shard(shard) 114 | partition_index = shard / @configuration.shards_per_partition_count 115 | partitions[partition_index] 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/shameless/model.rb: -------------------------------------------------------------------------------- 1 | require 'securerandom' 2 | require 'shameless/index' 3 | require 'shameless/cell' 4 | 5 | module Shameless 6 | module Model 7 | attr_reader :store 8 | 9 | def attach_to(store, name) 10 | @store = store 11 | @name = name || self.name.downcase # TODO use activesupport? 12 | @cell_names = [] 13 | cell(Cell::BASE) 14 | 15 | include(InstanceMethods) 16 | end 17 | 18 | def index(name = nil, &block) 19 | @indices ||= [] 20 | index = Index.new(name, self, &block) 21 | @indices << index 22 | 23 | define_singleton_method(index.full_name) { index } 24 | end 25 | 26 | def cell(name) 27 | name = name.to_s 28 | @cell_names << name 29 | 30 | define_method(name) { @cells[name] } 31 | end 32 | 33 | def initialize_cells(instance, base_body) 34 | Hash[@cell_names.map do |name| 35 | cell = name == Cell::BASE ? Cell.base(instance, base_body) : Cell.new(instance, name) 36 | [name, cell] 37 | end] 38 | end 39 | 40 | def put(values) 41 | if model = where(values).first 42 | model_values = reject_index_values(values) 43 | model.update(model_values) 44 | model 45 | else 46 | uuid = SecureRandom.uuid 47 | 48 | new(uuid, values).tap do |m| 49 | m.save 50 | 51 | index_values = values.merge(uuid: uuid) 52 | @indices.each {|i| i.put(index_values) } 53 | end 54 | end 55 | end 56 | 57 | def put_cell(shardable_value, cell_values) 58 | @store.put(table_name, shardable_value, cell_values) 59 | end 60 | 61 | def fetch_cell(shardable_value, uuid, cell_name, ref_key) 62 | query = {uuid: uuid, column_name: cell_name} 63 | query[:ref_key] = ref_key if ref_key 64 | 65 | @store.where(table_name, shardable_value, query).order(:ref_key).last 66 | end 67 | 68 | def fetch_latest_cells(shard:, cursor:, limit:) 69 | query = Sequel.lit("id > ?", cursor) 70 | @store.where(table_name, shard, query).limit(limit).map do |cell_values| 71 | model = new(cell_values[:uuid]) 72 | name = cell_values[:column_name].to_sym 73 | Cell.new(model, name, cell_values) 74 | end 75 | end 76 | 77 | def max_id_on_shard(shard) 78 | @store.find_table(table_name, shard).max(:id) 79 | end 80 | 81 | def table_name 82 | [@store.name, @name].compact.join('_') 83 | end 84 | 85 | def table_names 86 | [table_name, *@indices.map(&:table_name)] 87 | end 88 | 89 | def create_tables! 90 | @store.create_table!(table_name) do |t, sharded_table_name| 91 | t.primary_key :id 92 | t.varchar :uuid, size: 36 93 | t.varchar :column_name, null: false 94 | t.integer :ref_key, null: false 95 | t.mediumblob :body 96 | 97 | created_at_type = @store.configuration.legacy_created_at_is_bigint ? :bigint : :datetime 98 | t.column :created_at, created_at_type, null: false 99 | 100 | t.index %i[uuid column_name ref_key], name: "#{sharded_table_name}_model", unique: true 101 | end 102 | 103 | @indices.each(&:create_tables!) 104 | end 105 | 106 | def where(query, &block) 107 | primary_index.where(query, &block) 108 | end 109 | 110 | def reject_index_values(values) 111 | values.reject {|k, _| @indices.any? {|i| i.column?(k) } } 112 | end 113 | 114 | def prevent_readonly_attribute_mutation!(key) 115 | @indices.each {|i| i.prevent_readonly_attribute_mutation!(key) } 116 | end 117 | 118 | private 119 | 120 | module InstanceMethods 121 | attr_reader :uuid 122 | 123 | def initialize(uuid, base_body = nil) 124 | @uuid = uuid 125 | @cells = self.class.initialize_cells(self, base_body) 126 | end 127 | 128 | def [](field) 129 | base[field] 130 | end 131 | 132 | def []=(field, value) 133 | base[field] = value 134 | end 135 | 136 | def update(values) 137 | base.update(values) 138 | end 139 | 140 | def save 141 | base.save 142 | end 143 | 144 | def ref_key 145 | base.ref_key 146 | end 147 | 148 | def created_at 149 | base.created_at 150 | end 151 | 152 | def previous 153 | base.previous 154 | end 155 | 156 | def fetch(key, default) 157 | base.fetch(key, default) 158 | end 159 | 160 | def present? 161 | base.present? 162 | end 163 | 164 | def as_json(*) 165 | base.as_json 166 | end 167 | 168 | def reload 169 | cells.each(&:reload) 170 | end 171 | 172 | def cells 173 | @cells.values 174 | end 175 | 176 | def put_cell(cell_values) 177 | self.class.put_cell(shardable_value, cell_values) 178 | end 179 | 180 | def fetch_cell(cell_name, ref_key = nil) 181 | self.class.fetch_cell(shardable_value, uuid, cell_name, ref_key) 182 | end 183 | 184 | def prevent_readonly_attribute_mutation!(key) 185 | self.class.prevent_readonly_attribute_mutation!(key) 186 | end 187 | 188 | private 189 | 190 | def shardable_value 191 | uuid[0, 4].to_i(16) 192 | end 193 | end 194 | end 195 | end 196 | -------------------------------------------------------------------------------- /spec/shameless/store_spec.rb: -------------------------------------------------------------------------------- 1 | describe Shameless::Store do 2 | it 'works' do 3 | _, model = build_store 4 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 5 | 6 | expect(instance.uuid).not_to be_nil 7 | expect(model.where(hotel_id: 1).first.uuid).to eq(instance.uuid) 8 | end 9 | 10 | it 'allows access to base fields' do 11 | _, model = build_store 12 | model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 13 | fetched = model.where(hotel_id: 1).first 14 | 15 | expect(fetched[:hotel_id]).to eq(1) 16 | expect(fetched[:room_type]).to eq('roh') 17 | expect(fetched[:check_in_date]).to eq(Date.today.to_s) 18 | end 19 | 20 | it 'stores non-index fields on the body' do 21 | _, model = build_store 22 | model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 23 | fetched = model.where(hotel_id: 1).first 24 | 25 | expect(fetched[:net_rate]).to eq(90) 26 | end 27 | 28 | it 'properly loads base values when using where' do 29 | _, model = build_store 30 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 31 | 32 | fetched = model.where(hotel_id: 1).first 33 | expect(fetched.uuid).to eq(instance.uuid) 34 | expect(fetched.ref_key).to eq(0) 35 | end 36 | 37 | it 'names the model by downcasing the class name' do 38 | store, _ = build_store do |s| 39 | Store = s 40 | 41 | class MyModel 42 | Store.attach(self) 43 | 44 | index do 45 | integer :my_id 46 | shard_on :my_id 47 | end 48 | end 49 | end 50 | 51 | partition = nil 52 | store.each_partition {|p| partition ||= p } 53 | expect(partition.from('store_mymodel_000001').count).to eq(0) 54 | 55 | Object.send(:remove_const, :MyModel) 56 | Object.send(:remove_const, :Store) 57 | end 58 | 59 | it 'names the default index "primary"' do 60 | store, _ = build_store do |s| 61 | Store = s 62 | 63 | class MyModel 64 | Store.attach(self) 65 | 66 | index do 67 | integer :my_id 68 | shard_on :my_id 69 | end 70 | end 71 | end 72 | 73 | partition = nil 74 | store.each_partition {|p| partition ||= p } 75 | expect(partition.from('store_mymodel_primary_index_000001').count).to eq(0) 76 | 77 | Object.send(:remove_const, :MyModel) 78 | Object.send(:remove_const, :Store) 79 | end 80 | 81 | it 'allows naming the index' do 82 | store, _ = build_store do |s| 83 | Store = s 84 | 85 | class MyModel 86 | Store.attach(self) 87 | 88 | index :foo do 89 | integer :my_id 90 | shard_on :my_id 91 | end 92 | end 93 | end 94 | 95 | partition = nil 96 | store.each_partition {|p| partition ||= p } 97 | expect(partition.from('store_mymodel_foo_index_000001').count).to eq(0) 98 | 99 | Object.send(:remove_const, :MyModel) 100 | Object.send(:remove_const, :Store) 101 | end 102 | 103 | it 'passes connection options to Sequel' do 104 | expect(Sequel).to receive(:connect).with(anything, max_connections: 7).and_call_original 105 | build_store(connection_options: {max_connections: 7}) 106 | end 107 | 108 | it 'passes database extensions to Sequel' do 109 | db = double(Sequel::SQLite::Database).as_null_object 110 | allow(Sequel).to receive(:connect).and_return(db) 111 | build_store(database_extensions: [:foo]) 112 | 113 | expect(db).to have_received(:extension).with(:foo) 114 | end 115 | 116 | it 'passes create table options to create_table' do 117 | db = double(Sequel::SQLite::Database).as_null_object 118 | allow(Sequel).to receive(:connect).and_return(db) 119 | build_store(create_table_options: {temp: true}) 120 | 121 | expect(db).to have_received(:create_table).with("store_rates_000000", temp: true) 122 | end 123 | 124 | describe '#padded_shard' do 125 | it 'returns a 6-digit shard number' do 126 | store, _ = build_store 127 | 128 | expect(store.padded_shard(1)).to eq('000001') 129 | expect(store.padded_shard(35)).to eq('000003') 130 | end 131 | end 132 | 133 | describe '#disconnect' do 134 | it 'disconnects from partitions but keeps instances' do 135 | store, _ = build_store 136 | 137 | partition = nil 138 | store.each_partition {|p| partition = p } 139 | 140 | store.disconnect 141 | 142 | store.each_partition {|p| expect(p).to eq(partition) } 143 | end 144 | end 145 | 146 | describe '#each_partition' do 147 | it 'yields all partitions and their table names' do 148 | store, _ = build_store(partitions_count: 2) 149 | table_names_by_partition = {} 150 | 151 | store.each_partition do |partition, table_names| 152 | table_names_by_partition[partition] = table_names 153 | end 154 | 155 | expect(table_names_by_partition.count).to eq(2) 156 | table_names_by_partition.keys.each do |partition| 157 | expect(partition).to be_an_instance_of(Sequel::SQLite::Database) 158 | end 159 | expect(table_names_by_partition.values.first).to eq(%w[store_rates_000000 store_rates_000001 160 | store_rates_primary_index_000000 store_rates_primary_index_000001]) 161 | expect(table_names_by_partition.values.last).to eq(%w[store_rates_000002 store_rates_000003 162 | store_rates_primary_index_000002 store_rates_primary_index_000003]) 163 | end 164 | end 165 | end 166 | -------------------------------------------------------------------------------- /spec/shameless/cell_spec.rb: -------------------------------------------------------------------------------- 1 | describe Shameless::Cell do 2 | it 'allows storing arbitrary content in cells' do 3 | model = build_model_with_cell 4 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 5 | 6 | instance.meta[:foo] = 'bar' 7 | instance.meta.save 8 | 9 | expect(instance.meta[:foo]).to eq('bar') 10 | 11 | fetched = model.where(hotel_id: 1).first 12 | 13 | expect(fetched.meta[:foo]).to eq('bar') 14 | end 15 | 16 | it 'increments ref_key on save' do 17 | model = build_model_with_cell 18 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 19 | 20 | expect(instance.meta.ref_key).to eq(nil) 21 | 22 | instance.meta[:foo] = 'bar' 23 | instance.meta.save 24 | 25 | expect(instance.meta.ref_key).to eq(0) 26 | 27 | instance.meta.save 28 | 29 | expect(instance.meta.ref_key).to eq(1) 30 | 31 | fetched = model.where(hotel_id: 1).first 32 | 33 | expect(fetched.meta.ref_key).to eq(1) 34 | end 35 | 36 | it 'touches created_at on save' do 37 | model = build_model_with_cell 38 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 39 | 40 | expect(instance.meta.created_at).to eq(nil) 41 | 42 | instance.meta[:foo] = 'bar' 43 | instance.meta.save 44 | 45 | initial_created_at = instance.meta.created_at 46 | expect(instance.meta.created_at).not_to be_nil 47 | 48 | sleep(0.1) 49 | 50 | instance.meta.save 51 | 52 | last_created_at = instance.meta.created_at 53 | expect(last_created_at).to be > initial_created_at 54 | 55 | fetched = model.where(hotel_id: 1).first 56 | 57 | expect(fetched.meta.created_at).to be_within(0.001).of(last_created_at) 58 | end 59 | 60 | it 'remembers id on save' do 61 | model = build_model_with_cell 62 | 63 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 64 | expect(instance.base.id).to eq(1) 65 | 66 | instance.meta[:foo] = 'bar' 67 | instance.meta.save 68 | expect(instance.meta.id).to eq(2) 69 | 70 | second_instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 71 | expect(second_instance.base.id).to eq(3) 72 | 73 | second_instance.meta.save 74 | expect(second_instance.meta.id).to eq(4) 75 | end 76 | 77 | it 'allows to call save without changing anything' do 78 | model = build_model_with_cell 79 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 80 | 81 | instance.meta.save 82 | expect(instance.meta.ref_key).to eq(0) 83 | 84 | instance = model.where(hotel_id: 1).first 85 | 86 | instance.meta.save 87 | expect(instance.meta.ref_key).to eq(1) 88 | end 89 | 90 | describe '#update' do 91 | it 'assigns all values from argument' do 92 | model = build_model_with_cell 93 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 94 | 95 | instance.meta.update(net_rate: 100) 96 | expect(instance.meta[:net_rate]).to eq(100) 97 | expect(instance.meta.ref_key).to eq(0) 98 | 99 | instance.meta.update(gross_rate: 160) 100 | expect(instance.meta[:net_rate]).to eq(100) 101 | expect(instance.meta[:gross_rate]).to eq(160) 102 | expect(instance.meta.ref_key).to eq(1) 103 | end 104 | end 105 | 106 | describe '#previous' do 107 | it 'returns nil for initial version' do 108 | model = build_model_with_cell 109 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 110 | 111 | expect(instance.meta.previous).to be_nil 112 | end 113 | 114 | it 'returns nil for initial version after save' do 115 | model = build_model_with_cell 116 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 117 | instance.meta[:net_rate] = 90 118 | instance.meta.save 119 | 120 | expect(instance.meta.previous).to be_nil 121 | end 122 | 123 | it 'returns the previous version of the cell' do 124 | model = build_model_with_cell 125 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 126 | instance.meta[:net_rate] = 90 127 | instance.meta.save 128 | 129 | instance.meta[:net_rate] = 100 130 | instance.meta.save 131 | 132 | previous = instance.meta.previous 133 | expect(instance.meta.ref_key).to eq(1) 134 | expect(previous.ref_key).to eq(0) 135 | expect(previous[:net_rate]).to eq(90) 136 | end 137 | end 138 | 139 | describe '#reload' do 140 | it 'lazily reloads cell state' do 141 | model = build_model_with_cell 142 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 143 | instance.meta.update(net_rate: 90) 144 | 145 | second_instance = model.where(hotel_id: 1).first 146 | second_instance.meta.update(net_rate: 100) 147 | 148 | instance.meta.reload 149 | expect(instance.meta.ref_key).to eq(1) 150 | expect(instance.meta[:net_rate]).to eq(100) 151 | end 152 | end 153 | 154 | describe '#fetch' do 155 | it 'returns default when value is missing' do 156 | model = build_model_with_cell 157 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 158 | 159 | expect(instance.meta.fetch(:foo, 'bar')).to eq('bar') 160 | end 161 | 162 | it 'returns value when present' do 163 | model = build_model_with_cell 164 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 165 | instance.meta[:foo] = 'bar' 166 | 167 | expect(instance.meta.fetch(:foo, 'baz')).to eq('bar') 168 | end 169 | 170 | it 'returns false when value of false is present' do 171 | model = build_model_with_cell 172 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 173 | instance.meta[:foo] = false 174 | 175 | expect(instance.meta.fetch(:foo, 'bar')).to eq(false) 176 | end 177 | end 178 | 179 | describe '#present?' do 180 | it 'returns false if cell has not been saved yet' do 181 | model = build_model_with_cell 182 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 183 | 184 | expect(instance.meta).not_to be_present 185 | end 186 | 187 | it 'returns false if cell has been modified but not saved' do 188 | model = build_model_with_cell 189 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 190 | 191 | instance.meta[:net_rate] = 90 192 | 193 | expect(instance.meta).not_to be_present 194 | end 195 | 196 | it 'returns true if cell has been saved' do 197 | model = build_model_with_cell 198 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 199 | 200 | instance.meta.update(net_rate: 90) 201 | 202 | expect(instance.meta).to be_present 203 | end 204 | 205 | it 'returns true if cell has been saved and reloaded' do 206 | model = build_model_with_cell 207 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 208 | 209 | instance.meta.update(net_rate: 90) 210 | 211 | instance = model.where(hotel_id: 1).first 212 | 213 | expect(instance.meta).to be_present 214 | end 215 | 216 | describe '#as_json' do 217 | it 'returns the base cell JSON representation' do 218 | model = build_model_with_cell 219 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 220 | 221 | instance.meta.update(net_rate: 90) 222 | 223 | expect(instance.meta.as_json).to eq( 224 | id: 2, 225 | uuid: instance.uuid, 226 | column_name: 'meta', 227 | ref_key: 0, 228 | body: { 229 | 'net_rate' => 90 230 | }, 231 | created_at: instance.meta.created_at 232 | ) 233 | end 234 | end 235 | end 236 | end 237 | -------------------------------------------------------------------------------- /spec/shameless/model_spec.rb: -------------------------------------------------------------------------------- 1 | describe Shameless::Model do 2 | it 'initializes created_at' do 3 | _, model = build_store 4 | now = Time.now 5 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 6 | 7 | expect(instance.created_at).to be >= now 8 | end 9 | 10 | describe '#legacy_created_at_is_bigint' do 11 | it 'uses bigint for created_at' do 12 | _, model = build_store(legacy_created_at_is_bigint: true) 13 | before = Time.now 14 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 15 | after = Time.now 16 | 17 | expect(instance.created_at).to be >= (before.to_f * 1000).to_i 18 | expect(instance.created_at).to be <= (after.to_f * 1000).to_i 19 | end 20 | end 21 | 22 | it 'unifies symbol and string keys' do 23 | _, model = build_store 24 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 25 | 26 | expect(instance["net_rate"]).to eq(90) 27 | 28 | instance[:net_rate] = 100 29 | expect(instance["net_rate"]).to eq(100) 30 | 31 | instance["net_rate"] = 110 32 | expect(instance[:net_rate]).to eq(110) 33 | 34 | expect(instance.base.body.keys.count).to eq(4) 35 | 36 | fetched = model.where(hotel_id: 1).first 37 | 38 | expect(fetched.base.body.keys.count).to eq(4) 39 | end 40 | 41 | it 'allows updates via the instance' do 42 | _, model = build_store 43 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 44 | 45 | instance[:net_rate] = 100 46 | instance.save 47 | 48 | fetched = model.where(hotel_id: 1).first 49 | expect(fetched[:net_rate]).to eq(100) 50 | end 51 | 52 | it 'prevents updates to index fields' do 53 | _, model = build_store 54 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 55 | 56 | message = "The attribute hotel_id cannot be modified because it's part of the primary index" 57 | expect { instance[:hotel_id] = 2 }.to raise_error(Shameless::ReadonlyAttributeMutation, message) 58 | end 59 | 60 | it 'prevents updates to index fields even when accessed as strings' do 61 | _, model = build_store 62 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 63 | 64 | message = "The attribute hotel_id cannot be modified because it's part of the primary index" 65 | expect { instance['hotel_id'] = 2 }.to raise_error(Shameless::ReadonlyAttributeMutation, message) 66 | end 67 | 68 | it 'puts a new revision for a second put on the same index values' do 69 | _, model = build_store 70 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 71 | second_instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 100) 72 | 73 | expect(second_instance.uuid).to eq(instance.uuid) 74 | expect(second_instance.ref_key).to eq(1) 75 | end 76 | 77 | it 'converts shardable value to integer but stores it verbatim' do 78 | _, model = build_store 79 | instance = model.put(hotel_id: '1', room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 80 | 81 | expect(instance[:hotel_id]).to eq('1') 82 | end 83 | 84 | it 'increments ref_key on update' do 85 | _, model = build_store 86 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 87 | 88 | expect(instance.ref_key).to eq(0) 89 | 90 | instance[:net_rate] = 100 91 | instance.save 92 | 93 | expect(instance.ref_key).to eq(1) 94 | fetched = model.where(hotel_id: 1).first 95 | expect(fetched.ref_key).to eq(1) 96 | end 97 | 98 | it 'eagerly initializes all cells' do 99 | model = build_model_with_cell 100 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 101 | 102 | expect(instance.cells.count).to eq(3) 103 | end 104 | 105 | describe '#table_name' do 106 | it 'concatenates store name and model name' do 107 | _, model = build_store 108 | 109 | expect(model.table_name).to eq("store_rates") 110 | end 111 | 112 | it 'does not prefix table names if store name is nil' do 113 | _, model = build_store(name: nil) 114 | 115 | expect(model.table_name).to eq("rates") 116 | end 117 | end 118 | 119 | describe '#update' do 120 | it 'assigns all values from argument' do 121 | _, model = build_store 122 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 123 | 124 | instance.update(net_rate: 100) 125 | 126 | expect(instance.ref_key).to eq(1) 127 | expect(instance[:net_rate]).to eq(100) 128 | expect(instance[:hotel_id]).to eq(1) 129 | end 130 | end 131 | 132 | describe '#previous' do 133 | it 'returns the previous version of the base cell' do 134 | _, model = build_store 135 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 136 | 137 | instance.update(net_rate: 100) 138 | 139 | previous = instance.previous 140 | expect(instance.ref_key).to eq(1) 141 | expect(previous.ref_key).to eq(0) 142 | expect(previous[:net_rate]).to eq(90) 143 | end 144 | end 145 | 146 | describe '#reload' do 147 | it 'lazily reloads all cells' do 148 | model = build_model_with_cell 149 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 150 | 151 | instance.meta.update(net_rate: 95) 152 | 153 | second_instance = model.where(hotel_id: 1).first 154 | second_instance.update(net_rate: 100) 155 | second_instance.meta.update(net_rate: 110) 156 | 157 | instance.reload 158 | expect(instance.ref_key).to eq(1) 159 | expect(instance[:net_rate]).to eq(100) 160 | expect(instance.meta[:net_rate]).to eq(110) 161 | end 162 | end 163 | 164 | describe '#fetch' do 165 | it 'returns value from base cell' do 166 | _, model = build_store 167 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 168 | 169 | expect(instance.fetch(:net_rate, 100)).to eq(90) 170 | expect(instance.fetch(:foo, 'bar')).to eq('bar') 171 | end 172 | end 173 | 174 | describe '#where' do 175 | it 'queries with hash based filter only' do 176 | _, model = build_store 177 | same_day_rate = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 178 | model.put(hotel_id: 1, room_type: 'roh', check_in_date: (Date.today + 1).to_s, net_rate: 90) 179 | 180 | result_ids = model.where(hotel_id: 1, check_in_date: Date.today.to_s).map(&:uuid) 181 | 182 | expect(result_ids).to eq([same_day_rate.uuid]) 183 | end 184 | 185 | it 'queries with block filters' do 186 | _, model = build_store 187 | model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 90) 188 | advance_rate = model.put(hotel_id: 1, room_type: 'roh', check_in_date: (Date.today + 1).to_s, net_rate: 90) 189 | 190 | result_ids = model.where(hotel_id: 1) { check_in_date > Date.today.to_s }.map(&:uuid) 191 | 192 | expect(result_ids).to eq([advance_rate.uuid]) 193 | end 194 | end 195 | 196 | describe '#present?' do 197 | it 'returns false if base cell does not exist' do 198 | _, model = build_store 199 | uuid = SecureRandom.uuid 200 | instance = model.new(uuid) 201 | 202 | expect(instance).not_to be_present 203 | end 204 | 205 | it 'returns true if base cell exists' do 206 | _, model = build_store 207 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 208 | 209 | expect(instance).to be_present 210 | end 211 | end 212 | 213 | describe '.fetch_latest_cells' do 214 | it 'returns cells higher than given ID on a given shard' do 215 | model = build_model_with_cell 216 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 217 | instance.meta.update(net_rate: 90) 218 | 219 | second_instance = model.put(hotel_id: 2, room_type: 'roh', check_in_date: Date.today.to_s, net_rate: 100) 220 | second_instance.ota_rate.update(net_rate: 89) 221 | 222 | third_instance = model.put(hotel_id: 3, room_type: 'roh', check_in_date: Date.today.to_s) 223 | third_instance.update(net_rate: 70) 224 | 225 | expected_counts_per_shard = [0, 0, 0, 0] 226 | expected_counts_per_shard[find_shard(model, instance)] += 2 227 | expected_counts_per_shard[find_shard(model, second_instance)] += 2 228 | expected_counts_per_shard[find_shard(model, third_instance)] += 2 229 | 230 | expected_counts_per_shard.each_with_index do |count, i| 231 | next if i.zero? 232 | 233 | cells = model.fetch_latest_cells(shard: i, cursor: 0, limit: 50) 234 | expect(cells.count).to eq(count) 235 | end 236 | 237 | another_instance = model.put(hotel_id: 4, room_type: 'roh', check_in_date: Date.today.to_s) 238 | another_instance.meta.update(net_rate: 90) 239 | another_instance.ota_rate.update(net_rate: 89) 240 | another_instance.update(net_rate: 91) 241 | another_instance.meta.update(net_rate: 92) 242 | 243 | shard = find_shard(model, another_instance) 244 | cursor = expected_counts_per_shard[shard] 245 | cells = model.fetch_latest_cells(shard: shard, cursor: cursor, limit: 50) 246 | 247 | expect(cells.count).to eq(5) 248 | expect(cells).to be_an_instance_of(Array) 249 | expect(cells.map(&:name)).to eq(%i[base meta ota_rate base meta]) 250 | expect(cells.map(&:uuid)).to eq(Array.new(5) { another_instance.uuid }) 251 | expect(cells.map(&:id)).to eq(Array.new(5) {|i| i + cursor + 1 }) 252 | expect(cells.map(&:ref_key)).to eq([0, 0, 0, 1, 1]) 253 | expect(cells.map {|c| c[:net_rate] }).to eq([nil, 90, 89, 91, 92]) 254 | 255 | cells = model.fetch_latest_cells(shard: shard, cursor: expected_counts_per_shard[shard], limit: 4) 256 | 257 | expect(cells.count).to eq(4) 258 | end 259 | end 260 | 261 | describe '#as_json' do 262 | it 'returns the base cell JSON representation' do 263 | _, model = build_store 264 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s, foo: 'bar') 265 | 266 | expect(instance.as_json).to eq( 267 | id: 1, 268 | uuid: instance.uuid, 269 | column_name: 'base', 270 | ref_key: 0, 271 | body: { 272 | 'hotel_id' => 1, 273 | 'room_type' => 'roh', 274 | 'check_in_date' => Date.today.to_s, 275 | 'foo' => 'bar' 276 | }, 277 | created_at: instance.created_at 278 | ) 279 | end 280 | end 281 | 282 | describe '#max_id' do 283 | it 'returns the maximum id' do 284 | model = build_model_with_cell 285 | 286 | expected_max_id_per_shard = [nil, nil, nil, nil] 287 | 288 | expected_max_id_per_shard.each_with_index do |max_id, shard_id| 289 | expect(model.max_id_on_shard(shard_id)).to eq expected_max_id_per_shard[shard_id] 290 | end 291 | 292 | instance = model.put(hotel_id: 1, room_type: 'roh', check_in_date: Date.today.to_s) 293 | 294 | expected_max_id_per_shard[find_shard(model, instance)] = 1 295 | 296 | expected_max_id_per_shard.each_with_index do |max_id, shard_id| 297 | expect(model.max_id_on_shard(shard_id)).to eq expected_max_id_per_shard[shard_id] 298 | end 299 | end 300 | end 301 | end 302 | 303 | def find_shard(model, instance) 304 | model.store.find_shard(instance.send(:shardable_value)) 305 | end 306 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Shameless 2 | 3 | [![Build Status](https://travis-ci.org/hoteltonight/shameless.svg?branch=master)](https://travis-ci.org/hoteltonight/shameless) 4 | 5 | Shameless is an implementation of a schemaless, distributed, append-only store built on top of MySQL and the Sequel gem. It was extracted from a battle-tested codebase of our main application at HotelTonight. Since it's using Sequel for database access, it could work on any database, e.g. postgres, although we've only used it with MySQL. 6 | 7 | ## Background 8 | 9 | Shameless was born out of our need to have highly scalable, distributed storage for hotel rates. Rates are a way hotels package their rooms, they typically include check-in and check-out date, room type, rate plan, net price, discount, extra services, etc. Our original solution of storing rates in a typical relational SQL table was reaching its limits due to write congestion, migration anxiety, and high maintenance. 10 | 11 | Hotel rates change very frequently, so our solution needed to have consistent write latency. There are also multiple agents mutating various aspects of those rates, so we wanted something that would enable versioning. We also wanted to avoid having to create migrations whenever we were adding more data to rates. 12 | 13 | ## Concept 14 | 15 | The whole idea of Shameless is to split a regular SQL table into index tables and content tables. Index tables map the fields you want to query by to UUIDs, content tables map UUIDs to model contents (bodies). In addition, both index and content tables are sharded. 16 | 17 | The body of the model is schema-less, you can store arbitrary data structures in it. Under the hood, the body is serialized using MessagePack and stored as a blob in a single database column (hence the need for index tables). 18 | 19 | The process of querying for records can be described as: 20 | 21 | 1. Query the index tables by index fields (e.g. hotel ID, check-in date, and length of stay), sharded by hotel ID, getting get back a list of UUIDs 22 | 1. Query the content tables, sharded by UUID, for most recent version of model 23 | 24 | Inserting a record is similar: 25 | 26 | 1. Generate a UUID 27 | 1. Serialize and write model content into appropriate shard of the content tables 28 | 1. Insert a row (index fields + model UUID) to the appropriate shard of the index table 29 | 30 | Inserting a new version of an existing record is even simpler: 31 | 32 | 1. Increment version 33 | 1. Serialize and write model content into appropriate shard of the content tables 34 | 35 | Naturally, shameless hides all that complexity behind a straight-forward API. 36 | 37 | ## Usage 38 | 39 | ### Creating a store 40 | 41 | The core object of shameless is a `Store`. Here's how you can set one up: 42 | 43 | ```ruby 44 | # config/initializers/rate_store.rb 45 | 46 | RateStore = Shameless::Store.new(:rate_store) do |c| 47 | c.partition_urls = [ENV['RATE_STORE_DATABASE_URL_0'], ENV['RATE_STORE_DATABASE_URL_1'] 48 | c.shards_count = 512 # total number of shards across all partitions 49 | c.connection_options = {max_connections: 10} # connection options passed to `Sequel.connect` 50 | c.database_extensions = [:newrelic_instrumentation] 51 | c.create_table_options = {engine: "InnoDB"} # passed to Sequel's `create_table` 52 | end 53 | ``` 54 | 55 | The initializer argument (`:rate_store`) defines the namespace by which all tables will be prefixed, in this case `rate_store_`. If you pass `nil`, there will be no prefix. 56 | 57 | Once you've got the Store configured, you can declare models. 58 | 59 | ### Declaring models 60 | 61 | Models specify the kinds of entities you want to persist in your store. Models are simple Ruby classes (even anonymous) that you attach to a `Store` using `Store#attach(model)`, e.g.: 62 | 63 | ```ruby 64 | # app/models/rate.rb 65 | 66 | class Rate 67 | RateStore.attach(self) 68 | 69 | # ... 70 | end 71 | ``` 72 | 73 | By default, this will map to tables called `rate_store_rate_[000000-000511]` by lowercasing the class name. You can also provide the table namespace using a second argument, e.g.: 74 | 75 | ```ruby 76 | my_model = Class.new do 77 | RateStore.attach(self, :rates) 78 | end 79 | ``` 80 | 81 | A model is useless without indices. Let's see how to define them. 82 | 83 | ### Defining indices 84 | 85 | Indices are a crucial component of shameless. They allow us to perform fast lookups for model UUIDs. Here's how you define an index: 86 | 87 | ```ruby 88 | class Rate 89 | RateStore.attach(self) 90 | 91 | index do 92 | integer :hotel_id 93 | string :room_type 94 | string :check_in_date # at the moment, only integer and string types are supported 95 | 96 | shard_on :hotel_id # required, values need to be numeric 97 | end 98 | end 99 | ``` 100 | 101 | The default index is called a primary index, the corresponding tables would be called `rate_store_rate_primary_index_[000000-000511]`. You can add additional indices you'd like to query by: 102 | 103 | ```ruby 104 | class Rate 105 | RateStore.attach(self) 106 | 107 | index do 108 | # .. 109 | end 110 | 111 | index :secondary do 112 | integer :hotel_id 113 | string :gateway 114 | string :discount_type 115 | 116 | shard_on :hotel_id 117 | end 118 | end 119 | ``` 120 | 121 | ### Defining cells 122 | 123 | Model content is stored in blobs called "cells". You can think of cells as separate model columns that can store rich data structures and can change independently over time. The default cell is called "base" (that's what all model-level accessors delegate to), but you can declare additional cells using `Model.cell`: 124 | 125 | ```ruby 126 | class Rate 127 | RateStore.attach(self) 128 | 129 | index do 130 | # .. 131 | end 132 | 133 | cell :meta 134 | end 135 | ``` 136 | 137 | ### Reading/writing 138 | 139 | To write data to the model, use `Model.put` `Model#save`, `Cell#save`, `Model#update`, or `Cell#update`. `Model.put` will perform an "upsert", i.e. it will try to find an existing record with the given index fields, and insert a new version for that record's base cell if it finds one, or pick a new UUID, write the first version of the base cell, and write to all indices otherwise. 140 | 141 | Here are some examples of how you can read and write data from/to a shameless store: 142 | 143 | ```ruby 144 | # Writing - all index fields are required, the rest is the schemaless content 145 | rate = Rate.put(hotel_id: 1, room_type: '1 bed', check_in_date: Date.today, gateway: 'pegasus', discount_type: 'geo', net_price: 120.0) 146 | rate[:net_price] # => 120.0 # access in the "base" cell 147 | 148 | # Create a new version of the "base" cell 149 | rate[:net_price] = 130.0 150 | rate.save 151 | 152 | # You can also access the "base" cell explicitly 153 | rate.base[:net_price] = 140.0 154 | rate.base.save 155 | 156 | # Reading from/writing to a different cell is simple, too: 157 | rate.meta[:hotel_enabled] = true 158 | rate.meta.save 159 | 160 | # You can also do that in one go using `Model#update` or `Cell#update`. This writes a new 161 | # version of the cell, merging the hash passed in as parameter with existing values. 162 | rate.update(tax_rate: 11.0, gateway: 'pegasus') 163 | rate.body # => {net_price: 140.0, tax_rate: 11.0, gateway: 'pegasus'} 164 | 165 | rate.meta.update(hotel_enabled: false) 166 | ``` 167 | 168 | To query, use `Model.where` (also using Sequel's [virtual row blocks](http://sequel.jeremyevans.net/rdoc/files/doc/virtual_rows_rdoc.html)): 169 | 170 | ```ruby 171 | # Querying by primary index 172 | rates = Rate.where(hotel_id: 1, room_type: '1 bed', check_in_date: Date.today) 173 | 174 | # Querying by a named index 175 | rates = Rate.secondary_index.where(hotel_id: 1, gateway: 'pegasus', discount_type: 'geo') 176 | rates.first[:net_price] # => 130.0 177 | 178 | # Query using Sequel's virtual row block (handy for inequality operators) 179 | rates = Rate.where(hotel_id: 1, room_type: '1 bed') { check_in_date > Date.today } 180 | ``` 181 | 182 | To access a cell field that you're not sure has a value, you can use and `Cell#fetch` (`Model#fetch` delegates to the base cell) to get a value from a cell, or a default, e.g.: 183 | 184 | ```ruby 185 | rate[:net_price] = 130.0 186 | rate.fetch(:net_price, 100) # => 130.0 187 | rate.meta.fetch(:enabled, true) # => true 188 | ``` 189 | 190 | Cells are versioned, the current version is stored in a column called `ref_key`. The first version of a cell has a `ref_key` of zero. To access a previous version of a cell, use `Cell#previous` (`Model#previous` delegates to the base cell). Example: 191 | 192 | ```ruby 193 | # ... 194 | rate[:net_price] # => 120.0 195 | rate.ref_key # => 1 196 | rate.update(net_price: 130.0) 197 | rate.ref_key # => 2 198 | rate.previous[:net_price] # => 120.0 199 | rate.previous.ref_key # => 1 200 | rate.previous.previous.ref_key # => 0 201 | rate.previous.previous.previous # => nil 202 | ``` 203 | 204 | It could happen that another process may have updated a model/cell. To fetch the latest state, use `Cell#reload` (`Model#reload` reloads *all* cells), e.g.: 205 | 206 | ```ruby 207 | rate[:net_price] # => 120.0 208 | 209 | # Another process updates the cell 210 | Rate.where(hotel_id: rate[:hotel_id]).first.update(net_price: 130.0) 211 | 212 | rate[:net_price] # => 120.0 213 | rate.reload 214 | rate[:net_price] # => 130.0 215 | ``` 216 | 217 | To check if a given cell exists, use `Cell#present?` (as you can suspect, `Model#present?` delegates to the base cell). You can also use `Model#cells` to iterate over all cells, e.g.: 218 | 219 | ```ruby 220 | rate.present? # => true 221 | rate.meta.present? # => false 222 | 223 | rate.cells.any?(&:present?) # => true 224 | ``` 225 | 226 | To see the cell's full state (body + metadata), use `Cell#as_json` (`Model#as_json` delegates to the base cell), e.g.: 227 | 228 | ```ruby 229 | rate.as_json # => {id: 123, uuid: "...", created_at: "...", column_name: "base", ref_key: 3, 230 | # body: {hotel_id: 1, check_in_date: "2017-01-03", room_type: "ROH", net_price: 130.0}} 231 | ``` 232 | 233 | ### Creating tables 234 | 235 | To create all shards for all tables, across all partitions, run: 236 | 237 | ```ruby 238 | RateStore.create_tables! 239 | ``` 240 | 241 | This will create the underlying index tables, content tables, together with database indices for fast access. 242 | 243 | ### Concurrent writes 244 | 245 | Since writes to shameless aren't atomic, concurrency control needs to be moved to application code. We're using a setup where almost all our writes go through queues, one queue per shard. We're using `Store#each_shard` to match queue names to shards and to aggregate queue stats across all shards. 246 | 247 | ### Using shameless as a data stream store (similar to Kafka) 248 | 249 | Thanks to storing each write to shameless as a new record in the underlying database table, we're able to use our shameless store as a log for stream processing. For each shard, we have a worker that goes through all new entries in that shard and triggers various event processors to handle all kinds of asynchronous work. For that purpose, we're using `Model.fetch_latest_cells(shard:, cursor:, limit:)`, and incrementing a cursor (stored in Redis) after each record has been processed successfully. We're using the cells' IDs as cursors, using `Cell#id`, which returns the underlying table's primary key value. 250 | 251 | ### Utilities 252 | 253 | Sometimes it may be useful to know where a given model will end up, based on its shardable value. For this, you can use `Store#find_shard(shardable_value)`, e.g.: `RateStore.find_shard(hotel.id) # => 196`. 254 | 255 | ## Installation 256 | 257 | Add this line to your application's Gemfile: 258 | 259 | ```ruby 260 | gem 'shameless' 261 | ``` 262 | 263 | And then execute: 264 | 265 | $ bundle 266 | 267 | Or install it yourself as: 268 | 269 | $ gem install shameless 270 | 271 | ## Development 272 | 273 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 274 | 275 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 276 | 277 | ## Contributing 278 | 279 | Bug reports and pull requests are welcome on GitHub at https://github.com/hoteltonight/shameless. 280 | 281 | ## License 282 | 283 | The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). 284 | 285 | ## Credits 286 | 287 | Shameless was inspired by the following resources: 288 | 289 | - Uber: https://eng.uber.com/schemaless-part-one 290 | - FriendFeed: https://backchannel.org/blog/friendfeed-schemaless-mysql 291 | - Pinterest: https://engineering.pinterest.com/blog/sharding-pinterest-how-we-scaled-our-mysql-fleet 292 | --------------------------------------------------------------------------------