├── .env ├── .gitignore ├── .travis.yml ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── Procfile ├── README.md ├── Rakefile ├── api.rb ├── config.rb ├── observer.rb ├── schema.sql ├── scripts └── create_cluster ├── simulator.rb └── spec ├── api_spec.rb ├── observer_spec.rb ├── simulator_spec.rb └── spec_helper.rb /.env: -------------------------------------------------------------------------------- 1 | API_PORT=5005 2 | NUM_REPLICAS=5 3 | POSTGRES_PORT=5433 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | data 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | addons: 2 | postgresql: "9.6" 3 | before_script: 4 | - DATA_DIR=./data scripts/create_cluster & 5 | - sleep 10 # wait a while -- create_cluster takes a few seconds to run 6 | - createdb -p 5433 rocket-rides-reads-test 7 | - psql -p 5433 rocket-rides-reads-test < schema.sql 8 | env: 9 | global: 10 | - NUM_REPLICAS=5 11 | - POSTGRES_PORT=5433 12 | language: ruby 13 | notifications: 14 | email: 15 | on_success: never 16 | rvm: 17 | - 2.4.0 18 | sudo: false 19 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "excon" 4 | gem "pg" 5 | gem "puma" 6 | gem "sequel" 7 | gem "sinatra" 8 | 9 | group :development do 10 | gem "pry" 11 | gem "pry-byebug" 12 | gem "rack-test" 13 | gem "rake" 14 | gem "rspec" 15 | gem "webmock" 16 | end 17 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | addressable (2.5.2) 5 | public_suffix (>= 2.0.2, < 4.0) 6 | byebug (9.1.0) 7 | coderay (1.1.2) 8 | crack (0.4.3) 9 | safe_yaml (~> 1.0.0) 10 | diff-lcs (1.3) 11 | excon (0.59.0) 12 | hashdiff (0.3.7) 13 | method_source (0.9.0) 14 | mustermann (1.0.1) 15 | pg (0.21.0) 16 | pry (0.11.2) 17 | coderay (~> 1.1.0) 18 | method_source (~> 0.9.0) 19 | pry-byebug (3.5.0) 20 | byebug (~> 9.1) 21 | pry (~> 0.10) 22 | public_suffix (3.0.0) 23 | puma (3.10.0) 24 | rack (2.0.3) 25 | rack-protection (2.0.0) 26 | rack 27 | rack-test (0.7.0) 28 | rack (>= 1.0, < 3) 29 | rake (12.2.1) 30 | rspec (3.7.0) 31 | rspec-core (~> 3.7.0) 32 | rspec-expectations (~> 3.7.0) 33 | rspec-mocks (~> 3.7.0) 34 | rspec-core (3.7.0) 35 | rspec-support (~> 3.7.0) 36 | rspec-expectations (3.7.0) 37 | diff-lcs (>= 1.2.0, < 2.0) 38 | rspec-support (~> 3.7.0) 39 | rspec-mocks (3.7.0) 40 | diff-lcs (>= 1.2.0, < 2.0) 41 | rspec-support (~> 3.7.0) 42 | rspec-support (3.7.0) 43 | safe_yaml (1.0.4) 44 | sequel (5.2.0) 45 | sinatra (2.0.0) 46 | mustermann (~> 1.0) 47 | rack (~> 2.0) 48 | rack-protection (= 2.0.0) 49 | tilt (~> 2.0) 50 | tilt (2.0.8) 51 | webmock (3.1.0) 52 | addressable (>= 2.3.6) 53 | crack (>= 0.3.2) 54 | hashdiff 55 | 56 | PLATFORMS 57 | ruby 58 | 59 | DEPENDENCIES 60 | excon 61 | pg 62 | pry 63 | pry-byebug 64 | puma 65 | rack-test 66 | rake 67 | rspec 68 | sequel 69 | sinatra 70 | webmock 71 | 72 | BUNDLED WITH 73 | 1.12.5 74 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018- Brandur (https://brandur.org) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 9 | of the Software, and to permit persons to whom the Software is furnished to do 10 | so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | api: bundle exec ruby api.rb 2 | observer: bundle exec ruby observer.rb 3 | simulator: bundle exec ruby simulator.rb 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # rocket-rides-scalable [![Build Status](https://travis-ci.org/brandur/rocket-rides-scalable.svg?branch=master)](https://travis-ci.org/brandur/rocket-rides-scalable) 2 | 3 | This is a project based on the original [Rocket Rides][rides] repository to 4 | demonstrate what it might look like to implement reads from a replica that are 5 | guaranteed to never be stale. See [the associated article][reads] for full 6 | details. 7 | 8 | ## Architecture 9 | 10 | If you look in `Procfile`, you'll see these processes: 11 | 12 | * `api`: The main Rocket Rides API. It responds to requests and writes data to 13 | Postgres. On read requests, it may delegate the read to a replica instead of 14 | the primary. 15 | * `observer`: A background process that periodically (by default, every 0.5 16 | seconds) connects to each replica, checks the last WAL LSN (log sequence 17 | number) that it applies, then persists the data set to Postgres. 18 | * `simulator`: Random issues requests to the `api`. First it creates a new 19 | ride, then it consumes the ride by retrieving the same ID that it just 20 | created. Depending on replication status, this retrieval may go to a replica 21 | or to the primary. 22 | 23 | After you run `forego start` you should see the `simulator` issuing jobs 24 | against `api` right away. The `api` will log whether each retrieval went to the 25 | primary or a replica. Here's a sample trace (and note that in Sequel lingo, 26 | `default` is equivalent to "primary"): 27 | 28 | ``` 29 | $ forego start | grep 'Reading ride' 30 | api.1 | Reading ride 96 from server 'replica0' 31 | api.1 | Reading ride 97 from server 'replica0' 32 | api.1 | Reading ride 98 from server 'replica0' 33 | api.1 | Reading ride 99 from server 'replica1' 34 | api.1 | Reading ride 100 from server 'replica4' 35 | api.1 | Reading ride 101 from server 'replica2' 36 | api.1 | Reading ride 102 from server 'replica0' 37 | api.1 | Reading ride 103 from server 'default' 38 | api.1 | Reading ride 104 from server 'default' 39 | api.1 | Reading ride 105 from server 'replica2' 40 | ``` 41 | 42 | `api` won't read from the primary unless it has no choice, but even so, you'll 43 | see that occasionally the replicas fall far enough behind that there are no 44 | appropriate candidates (or more likely, the observer hasn't run recently 45 | enough) and a query will be routed to the primary (`default`). 46 | 47 | ## Setup 48 | 49 | Requirements: 50 | 51 | 1. Postgres (`brew install postgres`) -- **Note:** the names of some 52 | WAL-related functions changed in Postgres 10, and therefore this program 53 | requires Postgres 10 or above to run. 54 | 2. Ruby (`brew install ruby`) 55 | 3. forego (`brew install forego`) 56 | 57 | So we can see some real replication in action, the program should be run 58 | against a small cluster of a primary and some number of read replicas. This 59 | script will use an existing Postgres installation to bring up a new primary on 60 | a non-standard port (`5433` by default) and a set of replicas (5 by default) 61 | that stream from it: 62 | 63 | ``` 64 | forego run scripts/create_cluster 65 | ``` 66 | 67 | Issuing `Ctrl+C` will stop the cluster. The script's contents are pretty easy 68 | to read, so you should take a look to make sure it's not doing anything 69 | untowards. Default configuration for `NUM_REPLICAS` and `POSTGRES_PORT` can be 70 | found in `.env`. 71 | 72 | Install dependencies, create a database and schema, and start running the 73 | processes: 74 | 75 | ``` 76 | bundle install 77 | createdb -p 5433 rocket-rides-scalable 78 | psql -p 5433 rocket-rides-scalable < schema.sql 79 | forego start 80 | ``` 81 | 82 | ## Development & testing 83 | 84 | Install dependencies, create a test database and schema, and then run the test 85 | suite: 86 | 87 | ``` 88 | createdb -p 5433 rocket-rides-scalable-test 89 | psql -p 5433 rocket-rides-scalable-test < schema.sql 90 | forego run bundle exec rspec spec/ 91 | ``` 92 | 93 | [reads]: https://brandur.org/postgres-reads 94 | [rides]: https://github.com/stripe/stripe-connect-rocketrides 95 | 96 | 99 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | begin 2 | require 'rspec/core/rake_task' 3 | RSpec::Core::RakeTask.new(:spec) 4 | task :default => :spec 5 | rescue LoadError 6 | abort("No RSpec. Consider using `bundle exec`.") 7 | end 8 | -------------------------------------------------------------------------------- /api.rb: -------------------------------------------------------------------------------- 1 | require "json" 2 | require "sinatra" 3 | 4 | require_relative "./config" 5 | 6 | class API < Sinatra::Base 7 | set :server, %w[puma] 8 | set :show_exceptions, false 9 | 10 | post "/rides" do 11 | user = authenticate_user(request) 12 | params = validate_params(request) 13 | 14 | DB.transaction(isolation: :serializable) do 15 | ride = Ride.create( 16 | distance: params["distance"], 17 | user_id: user.id, 18 | ) 19 | update_user_min_lsn(user) 20 | 21 | [201, JSON.generate(serialize_ride(ride))] 22 | end 23 | end 24 | 25 | get "/rides/:id" do |id| 26 | user = authenticate_user(request) 27 | 28 | name = select_replica(user) 29 | $stdout.puts "Reading ride #{id} from server '#{name}'" 30 | 31 | ride = Ride.server(name).first(id: id) 32 | if ride.nil? 33 | halt 404, JSON.generate(wrap_error( 34 | Messages.error_not_found(object: "ride", id: id) 35 | )) 36 | end 37 | 38 | [200, JSON.generate(serialize_ride(ride))] 39 | end 40 | end 41 | 42 | # 43 | # models 44 | # 45 | 46 | class Ride < Sequel::Model 47 | end 48 | 49 | class User < Sequel::Model 50 | end 51 | 52 | # 53 | # other modules/classes 54 | # 55 | 56 | module Messages 57 | def self.ok 58 | "Payment accepted. Your pilot is on their way!" 59 | end 60 | 61 | def self.error_auth_invalid 62 | "Credentials in Authorization were invalid." 63 | end 64 | 65 | def self.error_auth_required 66 | "Please specify credentials in the Authorization header." 67 | end 68 | 69 | def self.error_not_found(object:, id:) 70 | "Object of type '#{object}' with ID '#{id}' was not found." 71 | end 72 | 73 | def self.error_require_float(key:) 74 | "Parameter '#{key}' must be a floating-point number." 75 | end 76 | 77 | def self.error_require_param(key:) 78 | "Please specify parameter '#{key}'." 79 | end 80 | end 81 | 82 | # 83 | # helpers 84 | # 85 | 86 | def authenticate_user(request) 87 | auth = request.env["HTTP_AUTHORIZATION"] 88 | if auth.nil? || auth.empty? 89 | halt 401, JSON.generate(wrap_error(Messages.error_auth_required)) 90 | end 91 | 92 | # This is obviously something you shouldn't do in a real application, but for 93 | # now we're just going to trust that the user is whoever they said they were 94 | # from an email in the `Authorization` header. 95 | user = User.first(email: auth) 96 | if user.nil? 97 | halt 401, JSON.generate(wrap_error(Messages.error_auth_invalid)) 98 | end 99 | 100 | user 101 | end 102 | 103 | # Selects the name of a random replica that can be passed to a Sequel `#server` 104 | # invocation to run a read a read query against a replica instead of the 105 | # primary. 106 | # 107 | # Uses the given user's `min_lsn` to determine which replicas are caught up 108 | # enough to be usable such that we will never have a stale read. 109 | # 110 | # In cases where no appropriate replica candidates exist, then we return 111 | # `:default` which maps to the primary. 112 | def select_replica(user) 113 | # If the user's `min_lsn` is `NULL` then they haven't performed an operation 114 | # yet, and we don't yet know if we can use a replica yet. Default to the 115 | # primary. 116 | return :default if user.min_lsn.nil? 117 | 118 | # exclude :default at the zero index 119 | replica_names = DB.servers[1..-1].map { |name| name.to_s } 120 | 121 | res = DB[Sequel.lit(<<~eos), replica_names, user.min_lsn] 122 | SELECT name 123 | FROM replica_statuses 124 | WHERE name IN ? 125 | AND pg_wal_lsn_diff(last_lsn, ?) >= 0; 126 | eos 127 | 128 | # If no candidates are caught up enough, then go to the primary. 129 | return :default if res.nil? || res.empty? 130 | 131 | # Return a random replica name from amongst the candidates. 132 | candidate_names = res.map { |res| res[:name].to_sym } 133 | candidate_names.sample 134 | end 135 | 136 | def serialize_ride(ride) 137 | { 138 | "distance": ride.distance.round(1), 139 | "id": ride.id, 140 | "user_id": ride.user_id, 141 | } 142 | end 143 | 144 | # Updates a user's `min_lsn` (log sequence number) so that we can start making 145 | # determinations as to whether it's safe for them to read from replicas. Note 146 | # that this is an update operation and always executes against the primary. 147 | def update_user_min_lsn(user) 148 | User. 149 | where(id: user.id). 150 | update(Sequel.lit("min_lsn = pg_current_wal_lsn()")) 151 | end 152 | 153 | def validate_params(request) 154 | { 155 | "distance" => validate_params_float(request, "distance"), 156 | } 157 | end 158 | 159 | def validate_params_float(request, key) 160 | val = validate_params_present(request, key) 161 | 162 | # Float as opposed to to_f because it's more strict about what it'll take. 163 | begin 164 | Float(val) 165 | rescue ArgumentError 166 | halt 422, JSON.generate(wrap_error(Messages.error_require_float(key: key))) 167 | end 168 | end 169 | 170 | def validate_params_present(request, key) 171 | val = request.POST[key] 172 | return val if !val.nil? && !val.empty? 173 | halt 422, JSON.generate(wrap_error(Messages.error_require_param(key: key))) 174 | end 175 | 176 | # Wraps a message in the standard structure that we send back for error 177 | # responses from the API. Still needs to be JSON-encoded before transmission. 178 | def wrap_error(message) 179 | { error: message } 180 | end 181 | 182 | # 183 | # run 184 | # 185 | 186 | if __FILE__ == $0 187 | port = ENV["API_PORT"] || abort("need API_PORT") 188 | API.run!(port: port) 189 | end 190 | -------------------------------------------------------------------------------- /config.rb: -------------------------------------------------------------------------------- 1 | require "logger" 2 | require "pg" 3 | require "sequel" 4 | 5 | NUM_REPLICAS = Integer(ENV["NUM_REPLICAS"] || abort("need NUM_REPLICAS")) 6 | POSTGRES_PORT = Integer(ENV["POSTGRES_PORT"] || abort("need POSTGRES_PORT")) 7 | 8 | DB = Sequel.connect("postgres://localhost:#{POSTGRES_PORT}/rocket-rides-scalable", 9 | servers: Hash[NUM_REPLICAS.times.map { |i| 10 | [:"replica#{i}", { port: POSTGRES_PORT + 1 + i }] 11 | }] 12 | ) 13 | 14 | # Currently only required for getting replica last LSNs, and there's probably a 15 | # better way to do that. 16 | DB.extension :server_block 17 | 18 | # a verbose mode to help with debugging 19 | if ENV["VERBOSE"] == "true" 20 | DB.loggers << Logger.new($stdout) 21 | 22 | # Emits the name of the server that a query was sent to in logs. Useful for 23 | # verifying that queries that we think are going to replicas are actually 24 | # going to replicas. 25 | DB.extension :server_logging 26 | end 27 | -------------------------------------------------------------------------------- /observer.rb: -------------------------------------------------------------------------------- 1 | require_relative "./api" 2 | 3 | class Observer 4 | def run 5 | loop do 6 | run_once 7 | sleep(SLEEP_DURATION) 8 | end 9 | end 10 | 11 | def run_once 12 | # exclude :default at the zero index 13 | replica_names = DB.servers[1..-1] 14 | 15 | last_lsns = replica_names.map do |name| 16 | DB.with_server(name) do 17 | DB[Sequel.lit(<<~eos)].first[:lsn] 18 | SELECT pg_last_wal_replay_lsn() AS lsn; 19 | eos 20 | end 21 | end 22 | 23 | insert_tuples = [] 24 | replica_names.each_with_index do |name, i| 25 | insert_tuples << { name: name.to_s, last_lsn: last_lsns[i] } 26 | end 27 | 28 | # update all replica statuses at once with upsert 29 | DB[:replica_statuses]. 30 | insert_conflict(target: :name, update: { last_lsn: Sequel[:excluded][:last_lsn] }). 31 | multi_insert(insert_tuples) 32 | 33 | $stdout.puts "Updated replica LSNs: results=#{insert_tuples}" 34 | 35 | insert_tuples 36 | end 37 | 38 | # 39 | # private 40 | # 41 | 42 | # Sleep duration in seconds to sleep between runs so that we're not just 43 | # constantly churning against all our databases. 44 | SLEEP_DURATION = 0.5 45 | private_constant :SLEEP_DURATION 46 | end 47 | 48 | # 49 | # run 50 | # 51 | 52 | if __FILE__ == $0 53 | # so output appears in Forego 54 | $stderr.sync = true 55 | $stdout.sync = true 56 | 57 | Observer.new.run 58 | end 59 | -------------------------------------------------------------------------------- /schema.sql: -------------------------------------------------------------------------------- 1 | BEGIN; 2 | 3 | -- 4 | -- A relation that contains the last observed lsn (log sequence number) for 5 | -- every known replica. 6 | -- 7 | CREATE TABLE replica_statuses ( 8 | id BIGSERIAL PRIMARY KEY, 9 | last_lsn PG_LSN NOT NULL, 10 | name VARCHAR(100) NOT NULL UNIQUE 11 | ); 12 | 13 | -- 14 | -- A relation to hold records for every user of our app. 15 | -- 16 | CREATE TABLE users ( 17 | id BIGSERIAL PRIMARY KEY, 18 | email VARCHAR(255) NOT NULL UNIQUE, 19 | 20 | -- stores the minimum lsn (log sequence number) required to have replicated 21 | -- to a replica before read requests for the user can be fulfilled on it 22 | min_lsn PG_LSN 23 | ); 24 | 25 | CREATE INDEX users_email 26 | ON users (email); 27 | 28 | -- 29 | -- A relation representing a single ride by a user. 30 | -- 31 | CREATE TABLE rides ( 32 | id BIGSERIAL PRIMARY KEY, 33 | created_at TIMESTAMPTZ NOT NULL DEFAULT now(), 34 | distance DOUBLE PRECISION NOT NULL, 35 | 36 | user_id BIGINT NOT NULL 37 | REFERENCES users ON DELETE RESTRICT 38 | ); 39 | 40 | COMMIT; 41 | -------------------------------------------------------------------------------- /scripts/create_cluster: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # 4 | # Script that initializes and runs a Postgres primary database with a number of 5 | # replica clusters that stream from it. 6 | # 7 | # It creates DATA_DIR directory in the current working directory, and it's 8 | # re-created at the beginning of every run (it is IMPORTANT to note that all 9 | # data is lost from this directory between runs so this script should only be 10 | # used for purely ephemeral purposes like testing). 11 | # 12 | # POSTGRES_PORT specifies the port for the started primary to run on. Replicas 13 | # will get port numbers assigned based off this port and their replica number. 14 | # So for example, if POSTGRES_PORT is 5433, the primary will get 5433, the 15 | # first replica will get 5434, the second 5435, the third 5436, etc. 16 | # 17 | 18 | DATA_DIR = ENV["DATA_DIR"] || abort("need DATA_DIR") 19 | NUM_REPLICAS = Integer(ENV["NUM_REPLICAS"] || abort("need NUM_REPLICAS")) 20 | POSTGRES_PORT = Integer(ENV["POSTGRES_PORT"] || abort("need POSTGRES_PORT")) 21 | 22 | # used to find a default Postgrer user 23 | USER = ENV["USER"] || abort("need USER") 24 | 25 | def print_status(message) 26 | puts "=== #{message}" 27 | end 28 | 29 | # 30 | # clean up 31 | # 32 | 33 | print_status("Removing #{DATA_DIR}") 34 | `rm -rf #{DATA_DIR}` 35 | 36 | # 37 | # initialize data directories 38 | # 39 | 40 | print_status("Initializing data directory for primary") 41 | `initdb -D #{DATA_DIR}/primary/` 42 | 43 | # 44 | # configure primary 45 | # 46 | 47 | print_status("Configuring primary") 48 | 49 | # Configures authentication around replication. We allow permission for our 50 | # user to access replication for both `local` (for use with `pg_basebackup`) 51 | # and then also on IPv4/IPv6 so that we can later connect to the database over 52 | # its other network interfaces. 53 | File.open("#{DATA_DIR}/primary/pg_hba.conf", mode="a") do |f| 54 | f << <<~eos 55 | local replication #{USER} trust 56 | host replication #{USER} 127.0.0.1/32 trust 57 | host replication #{USER} ::1/128 trust 58 | eos 59 | end 60 | 61 | File.open("#{DATA_DIR}/primary/postgresql.conf", mode="a") do |f| 62 | f << <<~eos 63 | max_connections=100 64 | max_wal_senders=99 # must be less than max_connections 65 | port=#{POSTGRES_PORT} 66 | wal_level=hot_standby 67 | 68 | # The primary could remove WAL between a base backup and when a replica 69 | # comes online so that the replica is stuck never able to catch up. This 70 | # removes any possibility of raciness by keeping a couple 16 MB segments 71 | # around. There's so little activity on this toy database that in practice 72 | # this will be more than enough. 73 | wal_keep_segments=2 74 | eos 75 | end 76 | 77 | # 78 | # start primary 79 | # 80 | 81 | print_status("Starting primary") 82 | primary_pid = Process.spawn("postgres -D #{DATA_DIR}/primary/") 83 | 84 | print_status("Waiting a short moment for it to come up") 85 | sleep(1) 86 | 87 | # 88 | # bring up replicas 89 | # 90 | 91 | replica_pids = [] 92 | 93 | NUM_REPLICAS.times do |i| 94 | 95 | # 96 | # create base backup for replica 97 | # 98 | 99 | print_status("Initializing data directory for replica #{i}") 100 | `pg_basebackup -p #{POSTGRES_PORT} --wal-method=stream -D #{DATA_DIR}/replica#{i}/` 101 | 102 | # 103 | # configure replica 104 | # 105 | 106 | print_status("Configuring replica #{i}") 107 | 108 | File.open("#{DATA_DIR}/replica#{i}/postgresql.conf", mode="a") do |f| 109 | f << <<~eos 110 | port=#{POSTGRES_PORT + 1 + i} 111 | shared_buffers=500MB 112 | hot_standby=on 113 | hot_standby_feedback=on 114 | eos 115 | end 116 | 117 | File.open("#{DATA_DIR}/replica#{i}/recovery.conf", mode="a") do |f| 118 | f << <<~eos 119 | standby_mode=on 120 | primary_conninfo='host=127.0.0.1 port=#{POSTGRES_PORT} user=#{USER}' 121 | eos 122 | end 123 | 124 | # 125 | # start replica 126 | # 127 | 128 | print_status("Starting replica #{i}") 129 | replica_pids << Process.spawn("postgres -D #{DATA_DIR}/replica#{i}/") 130 | end 131 | 132 | Process.wait primary_pid 133 | -------------------------------------------------------------------------------- /simulator.rb: -------------------------------------------------------------------------------- 1 | require "net/http" 2 | require "securerandom" 3 | 4 | require_relative "./api" 5 | 6 | class Simulator 7 | def initialize(port:) 8 | self.port = port 9 | end 10 | 11 | def run 12 | loop do 13 | run_once(sleep_before_get: rand * 2) 14 | duration = rand * 2 15 | $stdout.puts "Sleeping for #{duration}" 16 | sleep(duration) 17 | end 18 | end 19 | 20 | def run_once(sleep_before_get: 0.0) 21 | user = User.find_or_create(email: "user@example.com") 22 | 23 | http = Net::HTTP.new("localhost", port) 24 | request = Net::HTTP::Post.new("/rides") 25 | request["Authorization"] = user.email 26 | request.set_form_data({ 27 | "distance" => rand * (MAX_DISTANCE - MIN_DISTANCE) + MIN_DISTANCE 28 | }) 29 | response = http.request(request) 30 | $stdout.puts "Response: status=#{response.code} body=#{response.body}" 31 | 32 | data = JSON.parse(response.body, symbolize_names: true) 33 | 34 | sleep(sleep_before_get) 35 | 36 | request = Net::HTTP::Get.new("/rides/#{data[:id]}") 37 | request["Authorization"] = user.email 38 | response = http.request(request) 39 | $stdout.puts "Response: status=#{response.code} body=#{response.body}" 40 | end 41 | 42 | # 43 | # private 44 | # 45 | 46 | MAX_DISTANCE = 1000.0 47 | private_constant :MAX_DISTANCE 48 | MIN_DISTANCE = 5.0 49 | private_constant :MIN_DISTANCE 50 | 51 | attr_accessor :port 52 | end 53 | 54 | # 55 | # run 56 | # 57 | 58 | if __FILE__ == $0 59 | # so output appears in Forego 60 | $stderr.sync = true 61 | $stdout.sync = true 62 | 63 | port = ENV["API_PORT"] || abort("need API_PORT") 64 | 65 | # wait a moment for the API to come up 66 | sleep(3) 67 | 68 | Simulator.new(port: port).run 69 | end 70 | -------------------------------------------------------------------------------- /spec/api_spec.rb: -------------------------------------------------------------------------------- 1 | require "rack/test" 2 | require "securerandom" 3 | 4 | require_relative "./spec_helper" 5 | 6 | RSpec.describe API do 7 | include Rack::Test::Methods 8 | 9 | USER_EMAIL = "user@example.com" 10 | 11 | VALID_PARAMS = { 12 | "distance" => 123.0, 13 | }.freeze 14 | 15 | def app 16 | API 17 | end 18 | 19 | let(:user) do 20 | User.create( 21 | email: USER_EMAIL, 22 | ) 23 | end 24 | 25 | before do 26 | clear_database 27 | suppress_stdout 28 | user # these accessors are lazy, so ensure user exists 29 | end 30 | 31 | describe "POST /rides" do 32 | it "succeeds and creates a ride" do 33 | post "/rides", VALID_PARAMS, headers 34 | expect(last_response.status).to eq(201) 35 | expect(unwrap_field(last_response.body, :distance)).to eq( 36 | VALID_PARAMS["distance"].round(1) 37 | ) 38 | 39 | expect(Ride.count).to eq(1) 40 | 41 | # A `min_lsn` should have been set on the user after the ride was 42 | # created. 43 | user.reload 44 | expect(user.min_lsn).not_to be_nil 45 | end 46 | 47 | describe "failure" do 48 | it "denies requests without authorization" do 49 | post "/rides", VALID_PARAMS, {} 50 | expect(last_response.status).to eq(401) 51 | expect(unwrap_error(last_response.body)).to \ 52 | eq(Messages.error_auth_required) 53 | end 54 | 55 | it "denies requests with invalid authorization" do 56 | post "/rides", VALID_PARAMS, { 57 | "HTTP_AUTHORIZATION" => "user-does-not-exist@example.com" 58 | } 59 | expect(last_response.status).to eq(401) 60 | expect(unwrap_error(last_response.body)).to \ 61 | eq(Messages.error_auth_invalid) 62 | end 63 | 64 | it "denies requests that are missing parameters" do 65 | post "/rides", {}, headers 66 | expect(last_response.status).to eq(422) 67 | expect(unwrap_error(last_response.body)).to \ 68 | eq(Messages.error_require_param(key: "distance")) 69 | end 70 | 71 | it "denies requests that are the wrong type" do 72 | post "/rides", { "distance" => "foo" }, headers 73 | expect(last_response.status).to eq(422) 74 | expect(unwrap_error(last_response.body)).to \ 75 | eq(Messages.error_require_float(key: "distance")) 76 | end 77 | end 78 | end 79 | 80 | describe "GET /rides/:id" do 81 | let(:ride) do 82 | Ride.create( 83 | distance: 123.0, 84 | user_id: user.id, 85 | ) 86 | end 87 | 88 | it "succeeds and retrieves a ride" do 89 | get "/rides/#{ride.id}", {}, headers 90 | expect(last_response.status).to eq(200) 91 | expect(unwrap_field(last_response.body, :distance)).to eq( 92 | ride.distance.round(1), 93 | ) 94 | end 95 | 96 | it "reads from a replica given a user min_sln" do 97 | # Note that while we do set a `min_sln` for the user, we're not actually 98 | # confirming that the API is reading off the replica in this test (it 99 | # probably is, but it's not actually checked). We should try to do a 100 | # little better. 101 | update_user_min_lsn(user) 102 | 103 | get "/rides/#{ride.id}", {}, headers 104 | expect(last_response.status).to eq(200) 105 | expect(unwrap_field(last_response.body, :distance)).to eq( 106 | ride.distance.round(1), 107 | ) 108 | end 109 | 110 | describe "failure" do 111 | it "denies requests without authorization" do 112 | get "/rides/#{ride.id}", {}, {} 113 | expect(last_response.status).to eq(401) 114 | expect(unwrap_error(last_response.body)).to \ 115 | eq(Messages.error_auth_required) 116 | end 117 | 118 | it "denies requests with invalid authorization" do 119 | get "/rides/#{ride.id}", {}, { 120 | "HTTP_AUTHORIZATION" => "user-does-not-exist@example.com" 121 | } 122 | expect(last_response.status).to eq(401) 123 | expect(unwrap_error(last_response.body)).to \ 124 | eq(Messages.error_auth_invalid) 125 | end 126 | 127 | it "404s requests that ask for IDs that don't exist" do 128 | get "/rides/0", {}, headers 129 | expect(last_response.status).to eq(404) 130 | expect(unwrap_error(last_response.body)).to \ 131 | eq(Messages.error_not_found(object: "ride", id: "0")) 132 | end 133 | end 134 | end 135 | 136 | describe "#select_replica" do 137 | it "returns primary if a user's min_lsn is nil" do 138 | DB[:replica_statuses].insert(name: "replica1", last_lsn: FAR_FUTURE_LSN) 139 | 140 | expect(select_replica(user)).to eq(:default) 141 | end 142 | 143 | it "returns primary if the replica_statuses has no entries" do 144 | update_user_min_lsn(user) 145 | user.reload 146 | 147 | expect(select_replica(user)).to eq(:default) 148 | end 149 | 150 | it "returns a candidate otherwise" do 151 | DB[:replica_statuses].insert(name: "replica1", last_lsn: FAR_FUTURE_LSN) 152 | update_user_min_lsn(user) 153 | user.reload 154 | 155 | expect(select_replica(user)).not_to eq(:default) 156 | end 157 | end 158 | 159 | describe "#update_user_min_lsn" do 160 | it "updates a user's minimum lsn (log sequence number)" do 161 | update_user_min_lsn(user) 162 | user.reload 163 | expect(user.min_lsn).not_to be_nil 164 | end 165 | end 166 | 167 | # 168 | # helpers 169 | # 170 | 171 | FAR_FUTURE_LSN = "99999999/74145E8" 172 | 173 | private def headers 174 | # The demo API trusts that we are who we say we are. A record for this user 175 | # is created in the `before` block. 176 | { "HTTP_AUTHORIZATION" => USER_EMAIL } 177 | end 178 | 179 | private def unwrap_error(body) 180 | unwrap_field(body, :error) 181 | end 182 | 183 | private def unwrap_field(body, field) 184 | data = JSON.parse(body, symbolize_names: true) 185 | expect(data).to have_key(field) 186 | data[field] 187 | end 188 | end 189 | -------------------------------------------------------------------------------- /spec/observer_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative "./spec_helper" 2 | require_relative "../observer" 3 | 4 | RSpec.describe Observer do 5 | before do 6 | clear_database 7 | suppress_stdout 8 | end 9 | 10 | it "runs and updates replica lsns" do 11 | insert_tuples = Observer.new.run_once 12 | expect(insert_tuples).not_to be_empty 13 | expect(insert_tuples.map { |t| t[:name] }).not_to include(:default) 14 | expect(insert_tuples.map { |t| t[:name] }).to \ 15 | eq(DB.servers[1..-1].map { |name| name.to_s }) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/simulator_spec.rb: -------------------------------------------------------------------------------- 1 | require_relative "./spec_helper" 2 | require_relative "../simulator" 3 | 4 | WebMock.disable_net_connect! 5 | 6 | RSpec.describe Simulator do 7 | before do 8 | suppress_stdout 9 | WebMock.enable! 10 | end 11 | 12 | it "initiates a request" do 13 | stub_request(:post, "http://localhost:5000/rides").to_return( 14 | body: JSON.generate({ id: "123" }) 15 | ) 16 | stub_request(:get, "http://localhost:5000/rides/123") 17 | 18 | Simulator.new(port: "5000").run_once 19 | 20 | assert_requested :post, "http://localhost:5000/rides" 21 | assert_requested :get, "http://localhost:5000/rides/123" 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require "rspec" 2 | require 'webmock/rspec' 3 | 4 | ENV["RACK_ENV"] = "test" 5 | 6 | require_relative "../api" 7 | 8 | def clear_database 9 | DB.transaction do 10 | DB.run("TRUNCATE replica_statuses CASCADE") 11 | DB.run("TRUNCATE rides CASCADE") 12 | DB.run("TRUNCATE users CASCADE") 13 | end 14 | end 15 | 16 | def suppress_stdout 17 | $stdout = StringIO.new unless verbose? 18 | end 19 | 20 | def verbose? 21 | ENV["VERBOSE"] == "true" 22 | end 23 | --------------------------------------------------------------------------------