├── Gemfile ├── lib ├── split │ ├── version.rb │ ├── dashboard │ │ ├── public │ │ │ ├── dashboard.js │ │ │ ├── reset.css │ │ │ └── style.css │ │ ├── views │ │ │ ├── index.erb │ │ │ ├── layout.erb │ │ │ └── _experiment.erb │ │ └── helpers.rb │ ├── configuration.rb │ ├── dashboard.rb │ ├── alternative.rb │ ├── experiment.rb │ └── helper.rb └── split.rb ├── .gitignore ├── Rakefile ├── .travis.yml ├── Guardfile ├── spec ├── spec_helper.rb ├── configuration_spec.rb ├── dashboard_helpers_spec.rb ├── dashboard_spec.rb ├── alternative_spec.rb ├── experiment_spec.rb └── helper_spec.rb ├── LICENSE ├── split.gemspec ├── CHANGELOG.mdown └── README.mdown /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gemspec 4 | -------------------------------------------------------------------------------- /lib/split/version.rb: -------------------------------------------------------------------------------- 1 | module Split 2 | VERSION = "0.4.2" 3 | end 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | Gemfile.lock 4 | pkg/* 5 | *.rbc 6 | .idea 7 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env rake 2 | require 'bundler/gem_tasks' 3 | require 'rspec/core/rake_task' 4 | 5 | RSpec::Core::RakeTask.new('spec') 6 | 7 | task :default => :spec -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 1.8.7 4 | - 1.9.2 5 | - 1.9.3 6 | - jruby-18mode # JRuby in 1.8 mode 7 | - jruby-19mode # JRuby in 1.9 mode 8 | - rbx-18mode 9 | - rbx-19mode -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | guard 'rspec', :version => 2 do 2 | watch(%r{^spec/.+_spec\.rb$}) 3 | watch(%r{^lib/(.+)\.rb$}) { |m| "spec/lib/#{m[1]}_spec.rb" } 4 | watch('spec/spec_helper.rb') { "spec" } 5 | end -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RACK_ENV'] = "test" 2 | 3 | require 'rubygems' 4 | require 'bundler/setup' 5 | require 'split' 6 | require 'ostruct' 7 | require 'complex' if RUBY_VERSION.match(/1\.8/) 8 | 9 | def session 10 | @session ||= {} 11 | end 12 | 13 | def params 14 | @params ||= {} 15 | end 16 | 17 | def request(ua = 'Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_6_6; de-de) AppleWebKit/533.20.25 (KHTML, like Gecko) Version/5.0.4 Safari/533.20.27') 18 | r = OpenStruct.new 19 | r.user_agent = ua 20 | r.ip = '192.168.1.1' 21 | @request ||= r 22 | end -------------------------------------------------------------------------------- /spec/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Split::Configuration do 4 | it "should provide default values" do 5 | config = Split::Configuration.new 6 | 7 | config.ignore_ip_addresses.should eql([]) 8 | config.robot_regex.should eql(/\b(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg)\b/i) 9 | config.db_failover.should be_false 10 | config.db_failover_on_db_error.should be_a Proc 11 | config.allow_multiple_experiments.should be_false 12 | config.enabled.should be_true 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/split/dashboard/public/dashboard.js: -------------------------------------------------------------------------------- 1 | function confirmReset() { 2 | var agree=confirm("This will delete all data for this experiment?"); 3 | if (agree) 4 | return true; 5 | else 6 | return false; 7 | } 8 | 9 | function confirmDelete() { 10 | var agree=confirm("Are you sure you want to delete this experiment and all its data?"); 11 | if (agree) 12 | return true; 13 | else 14 | return false; 15 | } 16 | 17 | function confirmWinner() { 18 | var agree=confirm("This will now be returned for all users. Are you sure?"); 19 | if (agree) 20 | return true; 21 | else 22 | return false; 23 | } -------------------------------------------------------------------------------- /lib/split/dashboard/views/index.erb: -------------------------------------------------------------------------------- 1 | <% if @experiments.any? %> 2 |

The list below contains all the registered experiments along with the number of test participants, completed and conversion rate currently in the system.

3 | 4 | <% @experiments.each do |experiment| %> 5 | <%= erb :_experiment, :locals => {:experiment => experiment} %> 6 | <% end %> 7 | <% else %> 8 |

No experiments have started yet, you need to define them in your code and introduce them to your users.

9 |

Check out the Readme for more help getting started.

10 | <% end %> -------------------------------------------------------------------------------- /spec/dashboard_helpers_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'split/dashboard/helpers' 3 | 4 | include Split::DashboardHelpers 5 | 6 | describe Split::DashboardHelpers do 7 | describe 'confidence_level' do 8 | it 'should handle very small numbers' do 9 | confidence_level(Complex(2e-18, -0.03)).should eql('No Change') 10 | end 11 | 12 | it "should consider a z-score of 1.96 < z < 2.57 as 95% confident" do 13 | confidence_level(2.12).should eql('95% confidence') 14 | end 15 | 16 | it "should consider a z-score of -1.96 > z > -2.57 as 95% confident" do 17 | confidence_level(-2.12).should eql('95% confidence') 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/split/dashboard/views/layout.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Split 9 | 10 | 11 | 12 |
13 |

Split Dashboard

14 |
15 | 16 |
17 | <%= yield %> 18 |
19 | 20 | 23 | 24 | -------------------------------------------------------------------------------- /lib/split/configuration.rb: -------------------------------------------------------------------------------- 1 | module Split 2 | class Configuration 3 | attr_accessor :robot_regex 4 | attr_accessor :ignore_ip_addresses 5 | attr_accessor :db_failover 6 | attr_accessor :db_failover_on_db_error 7 | attr_accessor :allow_multiple_experiments 8 | attr_accessor :enabled 9 | attr_accessor :cookie_expires 10 | attr_accessor :cookie_domain 11 | 12 | def initialize 13 | @robot_regex = /\b(Baidu|Gigabot|Googlebot|libwww-perl|lwp-trivial|msnbot|SiteUptime|Slurp|WordPress|ZIBB|ZyBorg)\b/i 14 | @ignore_ip_addresses = [] 15 | @db_failover = false 16 | @db_failover_on_db_error = proc{|error|} # e.g. use Rails logger here 17 | @allow_multiple_experiments = false 18 | @enabled = true 19 | @cookie_expires = nil 20 | @cookie_domain = nil 21 | end 22 | end 23 | end -------------------------------------------------------------------------------- /lib/split/dashboard/helpers.rb: -------------------------------------------------------------------------------- 1 | module Split 2 | module DashboardHelpers 3 | def url(*path_parts) 4 | [ path_prefix, path_parts ].join("/").squeeze('/') 5 | end 6 | 7 | def path_prefix 8 | request.env['SCRIPT_NAME'] 9 | end 10 | 11 | def number_to_percentage(number, precision = 2) 12 | round(number * 100) 13 | end 14 | 15 | def round(number, precision = 2) 16 | BigDecimal.new(number.to_s).round(precision).to_f 17 | end 18 | 19 | def confidence_level(z_score) 20 | return z_score if z_score.is_a? String 21 | 22 | z = round(z_score.to_s.to_f, 3).abs 23 | 24 | if z == 0.0 25 | 'No Change' 26 | elsif z < 1.96 27 | 'no confidence' 28 | elsif z < 2.57 29 | '95% confidence' 30 | elsif z < 3.29 31 | '99% confidence' 32 | else 33 | '99.9% confidence' 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/split/dashboard/public/reset.css: -------------------------------------------------------------------------------- 1 | html, body, div, span, applet, object, iframe, 2 | h1, h2, h3, h4, h5, h6, p, blockquote, pre, 3 | a, abbr, acronym, address, big, cite, code, 4 | del, dfn, em, font, img, ins, kbd, q, s, samp, 5 | small, strike, strong, sub, sup, tt, var, 6 | dl, dt, dd, ul, li, 7 | form, label, legend, 8 | table, caption, tbody, tfoot, thead, tr, th, td { 9 | margin: 0; 10 | padding: 0; 11 | border: 0; 12 | outline: 0; 13 | font-weight: inherit; 14 | font-style: normal; 15 | font-size: 100%; 16 | font-family: inherit; 17 | } 18 | 19 | :focus { 20 | outline: 0; 21 | } 22 | 23 | body { 24 | line-height: 1; 25 | } 26 | 27 | ul { 28 | list-style: none; 29 | } 30 | 31 | table { 32 | border-collapse: collapse; 33 | border-spacing: 0; 34 | } 35 | 36 | caption, th, td { 37 | text-align: left; 38 | font-weight: normal; 39 | } 40 | 41 | blockquote:before, blockquote:after, 42 | q:before, q:after { 43 | content: ""; 44 | } 45 | 46 | blockquote, q { 47 | quotes: "" ""; 48 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Andrew Nesbitt 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /lib/split/dashboard.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra/base' 2 | require 'split' 3 | require 'bigdecimal' 4 | require 'split/dashboard/helpers' 5 | 6 | module Split 7 | class Dashboard < Sinatra::Base 8 | dir = File.dirname(File.expand_path(__FILE__)) 9 | 10 | set :views, "#{dir}/dashboard/views" 11 | set :public_folder, "#{dir}/dashboard/public" 12 | set :static, true 13 | set :method_override, true 14 | 15 | helpers Split::DashboardHelpers 16 | 17 | get '/' do 18 | @experiments = Split::Experiment.all 19 | erb :index 20 | end 21 | 22 | post '/:experiment' do 23 | @experiment = Split::Experiment.find(params[:experiment]) 24 | @alternative = Split::Alternative.new(params[:alternative], params[:experiment]) 25 | @experiment.winner = @alternative.name 26 | redirect url('/') 27 | end 28 | 29 | post '/reset/:experiment' do 30 | @experiment = Split::Experiment.find(params[:experiment]) 31 | @experiment.reset 32 | redirect url('/') 33 | end 34 | 35 | delete '/:experiment' do 36 | @experiment = Split::Experiment.find(params[:experiment]) 37 | @experiment.delete 38 | redirect url('/') 39 | end 40 | end 41 | end -------------------------------------------------------------------------------- /split.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | $:.push File.expand_path("../lib", __FILE__) 3 | require "split/version" 4 | 5 | Gem::Specification.new do |s| 6 | s.name = "split" 7 | s.version = Split::VERSION 8 | s.platform = Gem::Platform::RUBY 9 | s.authors = ["Andrew Nesbitt"] 10 | s.email = ["andrewnez@gmail.com"] 11 | s.homepage = "https://github.com/andrew/split" 12 | s.summary = %q{Rack based split testing framework} 13 | 14 | s.rubyforge_project = "split" 15 | 16 | s.files = `git ls-files`.split("\n") 17 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 18 | s.executables = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) } 19 | s.require_paths = ["lib"] 20 | 21 | s.add_dependency 'redis', '>= 2.1' 22 | s.add_dependency 'redis-namespace', '>= 1.1.0' 23 | s.add_dependency 'sinatra', '>= 1.2.6' 24 | 25 | s.add_development_dependency 'rake' 26 | s.add_development_dependency 'bundler', '~> 1.0' 27 | s.add_development_dependency 'rspec', '~> 2.6' 28 | s.add_development_dependency 'rack-test', '~> 0.6' 29 | s.add_development_dependency 'guard-rspec', '~> 0.4' 30 | end 31 | -------------------------------------------------------------------------------- /lib/split.rb: -------------------------------------------------------------------------------- 1 | require 'split/experiment' 2 | require 'split/alternative' 3 | require 'split/helper' 4 | require 'split/version' 5 | require 'split/configuration' 6 | require 'redis/namespace' 7 | 8 | module Split 9 | extend self 10 | attr_accessor :configuration 11 | 12 | # Accepts: 13 | # 1. A 'hostname:port' string 14 | # 2. A 'hostname:port:db' string (to select the Redis db) 15 | # 3. A 'hostname:port/namespace' string (to set the Redis namespace) 16 | # 4. A redis URL string 'redis://host:port' 17 | # 5. An instance of `Redis`, `Redis::Client`, `Redis::DistRedis`, 18 | # or `Redis::Namespace`. 19 | def redis=(server) 20 | if server.respond_to? :split 21 | if server =~ /redis\:\/\// 22 | redis = Redis.connect(:url => server, :thread_safe => true) 23 | else 24 | server, namespace = server.split('/', 2) 25 | host, port, db = server.split(':') 26 | redis = Redis.new(:host => host, :port => port, 27 | :thread_safe => true, :db => db) 28 | end 29 | namespace ||= :split 30 | 31 | @redis = Redis::Namespace.new(namespace, :redis => redis) 32 | elsif server.respond_to? :namespace= 33 | @redis = server 34 | else 35 | @redis = Redis::Namespace.new(:split, :redis => server) 36 | end 37 | end 38 | 39 | # Returns the current Redis connection. If none has been created, will 40 | # create a new one. 41 | def redis 42 | return @redis if @redis 43 | self.redis = 'localhost:6379' 44 | self.redis 45 | end 46 | 47 | # Call this method to modify defaults in your initializers. 48 | # 49 | # @example 50 | # Split.configure do |config| 51 | # config.ignore_ips = '192.168.2.1' 52 | # end 53 | def configure 54 | self.configuration ||= Configuration.new 55 | yield(configuration) 56 | end 57 | end 58 | 59 | Split.configure {} 60 | 61 | if defined?(Rails) 62 | class ActionController::Base 63 | ActionController::Base.send :include, Split::Helper 64 | ActionController::Base.helper Split::Helper 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/dashboard_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'rack/test' 3 | require 'split/dashboard' 4 | 5 | describe Split::Dashboard do 6 | include Rack::Test::Methods 7 | 8 | def app 9 | @app ||= Split::Dashboard 10 | end 11 | 12 | before(:each) { Split.redis.flushall } 13 | 14 | it "should respond to /" do 15 | get '/' 16 | last_response.should be_ok 17 | end 18 | 19 | it "should reset an experiment" do 20 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 21 | 22 | red = Split::Alternative.new('red', 'link_color') 23 | blue = Split::Alternative.new('blue', 'link_color') 24 | red.participant_count = 5 25 | blue.participant_count = 6 26 | 27 | post '/reset/link_color' 28 | 29 | last_response.should be_redirect 30 | 31 | new_red_count = Split::Alternative.new('red', 'link_color').participant_count 32 | new_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 33 | 34 | new_blue_count.should eql(0) 35 | new_red_count.should eql(0) 36 | end 37 | 38 | it "should delete an experiment" do 39 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 40 | delete '/link_color' 41 | last_response.should be_redirect 42 | Split::Experiment.find('link_color').should be_nil 43 | end 44 | 45 | it "should mark an alternative as the winner" do 46 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 47 | experiment.winner.should be_nil 48 | 49 | post '/link_color', :alternative => 'red' 50 | 51 | last_response.should be_redirect 52 | experiment.winner.name.should eql('red') 53 | end 54 | 55 | it "should display the start date" do 56 | experiment_start_time = Time.parse('2011-07-07') 57 | Time.stub(:now => experiment_start_time) 58 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 59 | 60 | get '/' 61 | 62 | last_response.body.should include('2011-07-07') 63 | end 64 | 65 | it "should handle experiments without a start date" do 66 | experiment_start_time = Time.parse('2011-07-07') 67 | Time.stub(:now => experiment_start_time) 68 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 69 | 70 | Split.redis.hdel(:experiment_start_times, experiment.name) 71 | 72 | get '/' 73 | 74 | last_response.body.should include('Unknown') 75 | end 76 | end -------------------------------------------------------------------------------- /lib/split/alternative.rb: -------------------------------------------------------------------------------- 1 | module Split 2 | class Alternative 3 | attr_accessor :name 4 | attr_accessor :experiment_name 5 | attr_accessor :weight 6 | 7 | def initialize(name, experiment_name) 8 | @experiment_name = experiment_name 9 | if Hash === name 10 | @name = name.keys.first 11 | @weight = name.values.first 12 | else 13 | @name = name 14 | @weight = 1 15 | end 16 | end 17 | 18 | def to_s 19 | name 20 | end 21 | 22 | def participant_count 23 | Split.redis.hget(key, 'participant_count').to_i 24 | end 25 | 26 | def participant_count=(count) 27 | Split.redis.hset(key, 'participant_count', count.to_i) 28 | end 29 | 30 | def completed_count 31 | Split.redis.hget(key, 'completed_count').to_i 32 | end 33 | 34 | def completed_count=(count) 35 | Split.redis.hset(key, 'completed_count', count.to_i) 36 | end 37 | 38 | def increment_participation 39 | Split.redis.hincrby key, 'participant_count', 1 40 | end 41 | 42 | def increment_completion 43 | Split.redis.hincrby key, 'completed_count', 1 44 | end 45 | 46 | def control? 47 | experiment.control.name == self.name 48 | end 49 | 50 | def conversion_rate 51 | return 0 if participant_count.zero? 52 | (completed_count.to_f/participant_count.to_f) 53 | end 54 | 55 | def experiment 56 | Split::Experiment.find(experiment_name) 57 | end 58 | 59 | def z_score 60 | # CTR_E = the CTR within the experiment split 61 | # CTR_C = the CTR within the control split 62 | # E = the number of impressions within the experiment split 63 | # C = the number of impressions within the control split 64 | 65 | control = experiment.control 66 | 67 | alternative = self 68 | 69 | return 'N/A' if control.name == alternative.name 70 | 71 | ctr_e = alternative.conversion_rate 72 | ctr_c = control.conversion_rate 73 | 74 | e = alternative.participant_count 75 | c = control.participant_count 76 | 77 | return 0 if ctr_c.zero? 78 | 79 | standard_deviation = ((ctr_e / ctr_c**3) * ((e*ctr_e)+(c*ctr_c)-(ctr_c*ctr_e)*(c+e))/(c*e)) ** 0.5 80 | 81 | z_score = ((ctr_e / ctr_c) - 1) / standard_deviation 82 | end 83 | 84 | def save 85 | Split.redis.hsetnx key, 'participant_count', 0 86 | Split.redis.hsetnx key, 'completed_count', 0 87 | end 88 | 89 | def reset 90 | Split.redis.hmset key, 'participant_count', 0, 'completed_count', 0 91 | end 92 | 93 | def delete 94 | Split.redis.del(key) 95 | end 96 | 97 | def self.valid?(name) 98 | String === name || hash_with_correct_values?(name) 99 | end 100 | 101 | def self.hash_with_correct_values?(name) 102 | Hash === name && String === name.keys.first && Float(name.values.first) rescue false 103 | end 104 | 105 | private 106 | 107 | def key 108 | "#{experiment_name}:#{name}" 109 | end 110 | end 111 | end -------------------------------------------------------------------------------- /lib/split/dashboard/views/_experiment.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 |

4 | Experiment: <%= experiment.name %> 5 | <% if experiment.version > 1 %>v<%= experiment.version %><% end %> 6 |

7 | 8 |
9 | <%= experiment.start_time ? experiment.start_time.strftime('%Y-%m-%d') : 'Unknown' %> 10 |
" method='post' onclick="return confirmReset()"> 11 | 12 |
13 |
" method='post' onclick="return confirmDelete()"> 14 | 15 | 16 |
17 |
18 |
19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | <% total_participants = total_completed = 0 %> 31 | <% experiment.alternatives.each do |alternative| %> 32 | 33 | 39 | 40 | 41 | 42 | 56 | 59 | 73 | 74 | 75 | <% total_participants += alternative.participant_count %> 76 | <% total_completed += alternative.completed_count %> 77 | <% end %> 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 |
Alternative NameParticipantsNon-finishedCompletedConversion RateConfidenceFinish
34 | <%= alternative.name %> 35 | <% if alternative.control? %> 36 | control 37 | <% end %> 38 | <%= alternative.participant_count %><%= alternative.participant_count - alternative.completed_count %><%= alternative.completed_count %> 43 | <%= number_to_percentage(alternative.conversion_rate) %>% 44 | <% if experiment.control.conversion_rate > 0 && !alternative.control? %> 45 | <% if alternative.conversion_rate > experiment.control.conversion_rate %> 46 | 47 | +<%= number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1) %>% 48 | 49 | <% elsif alternative.conversion_rate < experiment.control.conversion_rate %> 50 | 51 | <%= number_to_percentage((alternative.conversion_rate/experiment.control.conversion_rate)-1) %>% 52 | 53 | <% end %> 54 | <% end %> 55 | 57 | <%= confidence_level(alternative.z_score) %> 58 | 60 | <% if experiment.winner %> 61 | <% if experiment.winner.name == alternative.name %> 62 | Winner 63 | <% else %> 64 | Loser 65 | <% end %> 66 | <% else %> 67 |
68 | 69 | 70 |
71 | <% end %> 72 |
Totals<%= total_participants %><%= total_participants - total_completed %><%= total_completed %>N/AN/AN/A
89 |
-------------------------------------------------------------------------------- /CHANGELOG.mdown: -------------------------------------------------------------------------------- 1 | ## 0.4.2 (June 1, 2012) 2 | 3 | Features: 4 | 5 | - Now works with v3.0 of redis gem 6 | 7 | Bugfixes: 8 | 9 | - Fixed redis failover on Rubinius 10 | 11 | ## 0.4.1 (April 6, 2012) 12 | 13 | Features: 14 | 15 | - Added configuration option to disable Split testing (@ilyakatz, #45) 16 | 17 | Bugfixes: 18 | 19 | - Fix weights for existing experiments (@andreas, #40) 20 | - Fixed dashboard range error (@andrew, #42) 21 | 22 | ## 0.4.0 (March 7, 2012) 23 | 24 | **IMPORTANT** 25 | 26 | If using ruby 1.8.x and weighted alternatives you should always pass the control alternative through as the second argument with any other alternatives as a third argument because the order of the hash is not preserved in ruby 1.8, ruby 1.9 users are not affected by this bug. 27 | 28 | Features: 29 | 30 | - Experiments now record when they were started (@vrish88, #35) 31 | - Old versions of experiments in sessions are now cleaned up 32 | - Avoid users participating in multiple experiments at once (#21) 33 | 34 | Bugfixes: 35 | 36 | - Overriding alternatives doesn't work for weighted alternatives (@layflags, #34) 37 | - confidence_level helper should handle tiny z-scores (#23) 38 | 39 | ## 0.3.3 (February 16, 2012) 40 | 41 | Bugfixes: 42 | 43 | - Fixed redis failover when a block was passed to ab_test (@layflags, #33) 44 | 45 | ## 0.3.2 (February 12, 2012) 46 | 47 | Features: 48 | 49 | - Handle redis errors gracefully (@layflags, #32) 50 | 51 | ## 0.3.1 (November 19, 2011) 52 | 53 | Features: 54 | 55 | - General code tidy up (@ryanlecompte, #22, @mocoso, #28) 56 | - Lazy loading data from Redis (@lautis, #25) 57 | 58 | Bugfixes: 59 | 60 | - Handle unstarted experiments (@mocoso, #27) 61 | - Relaxed Sinatra version requirement (@martinclu, #24) 62 | 63 | 64 | ## 0.3.0 (October 9, 2011) 65 | 66 | Features: 67 | 68 | - Redesigned dashboard (@mrappleton, #17) 69 | - Use atomic increments in redis for better concurrency (@lautis, #18) 70 | - Weighted alternatives 71 | 72 | Bugfixes: 73 | 74 | - Fix to allow overriding of experiments that aren't on version 1 75 | 76 | 77 | ## 0.2.4 (July 18, 2011) 78 | 79 | Features: 80 | 81 | - Added option to finished to not reset the users session 82 | 83 | Bugfixes: 84 | 85 | - Only allow strings as alternatives, fixes strange errors when passing true/false or symbols 86 | 87 | ## 0.2.3 (June 26, 2011) 88 | 89 | Features: 90 | 91 | - Experiments can now be deleted from the dashboard 92 | - ab_test helper now accepts a block 93 | - Improved dashboard 94 | 95 | Bugfixes: 96 | 97 | - After resetting an experiment, existing users of that experiment will also be reset 98 | 99 | ## 0.2.2 (June 11, 2011) 100 | 101 | Features: 102 | 103 | - Updated redis-namespace requirement to 1.0.3 104 | - Added a configuration object for changing options 105 | - Robot regex can now be changed via a configuration options 106 | - Added ability to ignore visits from specified IP addresses 107 | - Dashboard now shows percentage improvement of alternatives compared to the control 108 | - If the alternatives of an experiment are changed it resets the experiment and uses the new alternatives 109 | 110 | Bugfixes: 111 | 112 | - Saving an experiment multiple times no longer creates duplicate alternatives 113 | 114 | ## 0.2.1 (May 29, 2011) 115 | 116 | Bugfixes: 117 | 118 | - Convert legacy sets to lists to avoid exceptions during upgrades from 0.1.x 119 | 120 | ## 0.2.0 (May 29, 2011) 121 | 122 | Features: 123 | 124 | - Override an alternative via a url parameter 125 | - Experiments can now be reset from the dashboard 126 | - The first alternative is now considered the control 127 | - General dashboard usability improvements 128 | - Robots are ignored and given the control alternative 129 | 130 | Bugfixes: 131 | 132 | - Alternatives are now store in a list rather than a set to ensure consistent ordering 133 | - Fixed diving by zero errors 134 | 135 | ## 0.1.1 (May 18, 2011) 136 | 137 | Bugfixes: 138 | 139 | - More Robust conversion rate display on dashboard 140 | - Ensure `Split::Version` is available everywhere, fixed dashboard 141 | 142 | ## 0.1.0 (May 17, 2011) 143 | 144 | Initial Release -------------------------------------------------------------------------------- /spec/alternative_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'split/alternative' 3 | 4 | describe Split::Alternative do 5 | before(:each) { Split.redis.flushall } 6 | 7 | it "should have a name" do 8 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 9 | alternative = Split::Alternative.new('Basket', 'basket_text') 10 | alternative.name.should eql('Basket') 11 | end 12 | 13 | it "return only the name" do 14 | experiment = Split::Experiment.new('basket_text', {'Basket' => 0.6}, {"Cart" => 0.4}) 15 | alternative = Split::Alternative.new('Basket', 'basket_text') 16 | alternative.name.should eql('Basket') 17 | end 18 | 19 | it "should have a default participation count of 0" do 20 | alternative = Split::Alternative.new('Basket', 'basket_text') 21 | alternative.participant_count.should eql(0) 22 | end 23 | 24 | it "should have a default completed count of 0" do 25 | alternative = Split::Alternative.new('Basket', 'basket_text') 26 | alternative.completed_count.should eql(0) 27 | end 28 | 29 | it "should belong to an experiment" do 30 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 31 | experiment.save 32 | alternative = Split::Alternative.new('Basket', 'basket_text') 33 | alternative.experiment.name.should eql(experiment.name) 34 | end 35 | 36 | it "should save to redis" do 37 | alternative = Split::Alternative.new('Basket', 'basket_text') 38 | alternative.save 39 | Split.redis.exists('basket_text:Basket').should be true 40 | end 41 | 42 | it "should increment participation count" do 43 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 44 | experiment.save 45 | alternative = Split::Alternative.new('Basket', 'basket_text') 46 | old_participant_count = alternative.participant_count 47 | alternative.increment_participation 48 | alternative.participant_count.should eql(old_participant_count+1) 49 | 50 | Split::Alternative.new('Basket', 'basket_text').participant_count.should eql(old_participant_count+1) 51 | end 52 | 53 | it "should increment completed count" do 54 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 55 | experiment.save 56 | alternative = Split::Alternative.new('Basket', 'basket_text') 57 | old_completed_count = alternative.participant_count 58 | alternative.increment_completion 59 | alternative.completed_count.should eql(old_completed_count+1) 60 | 61 | Split::Alternative.new('Basket', 'basket_text').completed_count.should eql(old_completed_count+1) 62 | end 63 | 64 | it "can be reset" do 65 | alternative = Split::Alternative.new('Basket', 'basket_text') 66 | alternative.participant_count = 10 67 | alternative.completed_count = 4 68 | alternative.reset 69 | alternative.participant_count.should eql(0) 70 | alternative.completed_count.should eql(0) 71 | end 72 | 73 | it "should know if it is the control of an experiment" do 74 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 75 | experiment.save 76 | alternative = Split::Alternative.new('Basket', 'basket_text') 77 | alternative.control?.should be_true 78 | alternative = Split::Alternative.new('Cart', 'basket_text') 79 | alternative.control?.should be_false 80 | end 81 | 82 | describe 'conversion rate' do 83 | it "should be 0 if there are no conversions" do 84 | alternative = Split::Alternative.new('Basket', 'basket_text') 85 | alternative.completed_count.should eql(0) 86 | alternative.conversion_rate.should eql(0) 87 | end 88 | 89 | it "does something" do 90 | alternative = Split::Alternative.new('Basket', 'basket_text') 91 | alternative.stub(:participant_count).and_return(10) 92 | alternative.stub(:completed_count).and_return(4) 93 | alternative.conversion_rate.should eql(0.4) 94 | end 95 | end 96 | 97 | describe 'z score' do 98 | it 'should be zero when the control has no conversions' do 99 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 100 | 101 | alternative = Split::Alternative.new('red', 'link_color') 102 | alternative.z_score.should eql(0) 103 | end 104 | 105 | it "should be N/A for the control" do 106 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 107 | 108 | control = experiment.control 109 | control.z_score.should eql('N/A') 110 | end 111 | end 112 | end -------------------------------------------------------------------------------- /lib/split/experiment.rb: -------------------------------------------------------------------------------- 1 | module Split 2 | class Experiment 3 | attr_accessor :name 4 | attr_accessor :winner 5 | 6 | def initialize(name, *alternative_names) 7 | @name = name.to_s 8 | @alternatives = alternative_names.map do |alternative| 9 | Split::Alternative.new(alternative, name) 10 | end 11 | end 12 | 13 | def winner 14 | if w = Split.redis.hget(:experiment_winner, name) 15 | Split::Alternative.new(w, name) 16 | else 17 | nil 18 | end 19 | end 20 | 21 | def control 22 | alternatives.first 23 | end 24 | 25 | def reset_winner 26 | Split.redis.hdel(:experiment_winner, name) 27 | end 28 | 29 | def winner=(winner_name) 30 | Split.redis.hset(:experiment_winner, name, winner_name.to_s) 31 | end 32 | 33 | def start_time 34 | t = Split.redis.hget(:experiment_start_times, @name) 35 | Time.parse(t) if t 36 | end 37 | 38 | def alternatives 39 | @alternatives.dup 40 | end 41 | 42 | def alternative_names 43 | @alternatives.map(&:name) 44 | end 45 | 46 | def next_alternative 47 | winner || random_alternative 48 | end 49 | 50 | def random_alternative 51 | weights = alternatives.map(&:weight) 52 | 53 | total = weights.inject(:+) 54 | point = rand * total 55 | 56 | alternatives.zip(weights).each do |n,w| 57 | return n if w >= point 58 | point -= w 59 | end 60 | end 61 | 62 | def version 63 | @version ||= (Split.redis.get("#{name.to_s}:version").to_i || 0) 64 | end 65 | 66 | def increment_version 67 | @version = Split.redis.incr("#{name}:version") 68 | end 69 | 70 | def key 71 | if version.to_i > 0 72 | "#{name}:#{version}" 73 | else 74 | name 75 | end 76 | end 77 | 78 | def reset 79 | alternatives.each(&:reset) 80 | reset_winner 81 | increment_version 82 | end 83 | 84 | def delete 85 | alternatives.each(&:delete) 86 | reset_winner 87 | Split.redis.srem(:experiments, name) 88 | Split.redis.del(name) 89 | increment_version 90 | end 91 | 92 | def new_record? 93 | !Split.redis.exists(name) 94 | end 95 | 96 | def save 97 | if new_record? 98 | Split.redis.sadd(:experiments, name) 99 | Split.redis.hset(:experiment_start_times, @name, Time.now) 100 | @alternatives.reverse.each {|a| Split.redis.lpush(name, a.name) } 101 | end 102 | end 103 | 104 | def self.load_alternatives_for(name) 105 | case Split.redis.type(name) 106 | when 'set' # convert legacy sets to lists 107 | alts = Split.redis.smembers(name) 108 | Split.redis.del(name) 109 | alts.reverse.each {|a| Split.redis.lpush(name, a) } 110 | Split.redis.lrange(name, 0, -1) 111 | else 112 | Split.redis.lrange(name, 0, -1) 113 | end 114 | end 115 | 116 | def self.all 117 | Array(Split.redis.smembers(:experiments)).map {|e| find(e)} 118 | end 119 | 120 | def self.find(name) 121 | if Split.redis.exists(name) 122 | self.new(name, *load_alternatives_for(name)) 123 | end 124 | end 125 | 126 | def self.find_or_create(key, *alternatives) 127 | name = key.to_s.split(':')[0] 128 | 129 | if alternatives.length == 1 130 | if alternatives[0].is_a? Hash 131 | alternatives = alternatives[0].map{|k,v| {k => v} } 132 | else 133 | raise InvalidArgument, 'You must declare at least 2 alternatives' 134 | end 135 | end 136 | 137 | alts = initialize_alternatives(alternatives, name) 138 | 139 | if Split.redis.exists(name) 140 | if load_alternatives_for(name) == alts.map(&:name) 141 | experiment = self.new(name, *alternatives) 142 | else 143 | exp = self.new(name, *load_alternatives_for(name)) 144 | exp.reset 145 | exp.alternatives.each(&:delete) 146 | experiment = self.new(name, *alternatives) 147 | experiment.save 148 | end 149 | else 150 | experiment = self.new(name, *alternatives) 151 | experiment.save 152 | end 153 | return experiment 154 | 155 | end 156 | 157 | def self.initialize_alternatives(alternatives, name) 158 | 159 | unless alternatives.all? { |a| Split::Alternative.valid?(a) } 160 | raise InvalidArgument, 'Alternatives must be strings' 161 | end 162 | 163 | alternatives.map do |alternative| 164 | Split::Alternative.new(alternative, name) 165 | end 166 | end 167 | end 168 | end -------------------------------------------------------------------------------- /lib/split/helper.rb: -------------------------------------------------------------------------------- 1 | module Split 2 | module Helper 3 | def ab_test(experiment_name, control, *alternatives) 4 | 5 | puts 'WARNING: You should always pass the control alternative through as the second argument with any other alternatives as the third because the order of the hash is not preserved in ruby 1.8' if RUBY_VERSION.match(/1\.8/) && alternatives.length.zero? 6 | ret = if Split.configuration.enabled 7 | experiment_variable(alternatives, control, experiment_name) 8 | else 9 | control_variable(control) 10 | end 11 | 12 | if block_given? 13 | if defined?(capture) # a block in a rails view 14 | block = Proc.new { yield(ret) } 15 | concat(capture(ret, &block)) 16 | false 17 | else 18 | yield(ret) 19 | end 20 | else 21 | ret 22 | end 23 | end 24 | 25 | def finished(experiment_name, options = {:reset => true}) 26 | return if exclude_visitor? or !Split.configuration.enabled 27 | return unless (experiment = Split::Experiment.find(experiment_name)) 28 | 29 | change_session do 30 | if alternative_name = @session[experiment.key] 31 | alternative = Split::Alternative.new(alternative_name, experiment_name) 32 | alternative.increment_completion 33 | @session.delete(experiment_name) if options[:reset] 34 | end 35 | end 36 | rescue => e 37 | raise unless Split.configuration.db_failover 38 | Split.configuration.db_failover_on_db_error.call(e) 39 | end 40 | 41 | def override(experiment_name, alternatives) 42 | params[experiment_name] if defined?(params) && alternatives.include?(params[experiment_name]) 43 | end 44 | 45 | def begin_experiment(experiment, alternative_name = nil) 46 | alternative_name ||= experiment.control.name 47 | change_session { @session[experiment.key] = alternative_name } 48 | end 49 | 50 | def exclude_visitor? 51 | is_robot? or is_ignored_ip_address? 52 | end 53 | 54 | def not_allowed_to_test?(experiment_key) 55 | !Split.configuration.allow_multiple_experiments && doing_other_tests?(experiment_key) 56 | end 57 | 58 | def doing_other_tests?(experiment_key) 59 | get_session.keys.reject { |k| k == experiment_key }.length > 0 60 | end 61 | 62 | def clean_old_versions(experiment) 63 | change_session do 64 | old_versions(experiment).each do |old_key| 65 | @session.delete old_key 66 | end 67 | end 68 | end 69 | 70 | def old_versions(experiment) 71 | if experiment.version > 0 72 | get_session.keys.select { |k| k.match(Regexp.new(experiment.name)) }.reject { |k| k == experiment.key } 73 | else 74 | [] 75 | end 76 | end 77 | 78 | def is_robot? 79 | request.user_agent =~ Split.configuration.robot_regex 80 | end 81 | 82 | def is_ignored_ip_address? 83 | if Split.configuration.ignore_ip_addresses.any? 84 | Split.configuration.ignore_ip_addresses.include?(request.ip) 85 | else 86 | false 87 | end 88 | end 89 | 90 | def get_session 91 | if cookies.signed[:split] 92 | Marshal.load(cookies.signed[:split]) 93 | else 94 | {} 95 | end 96 | end 97 | 98 | def set_session(value) 99 | cookie = {value: Marshal.dump(value)} 100 | cookie[:expires] = Split.configuration.cookie_expires if Split.configuration.cookie_expires 101 | cookie[:domain] = Split.configuration.cookie_domain if Split.configuration.cookie_domain 102 | 103 | cookies.signed[:split] = cookie 104 | end 105 | 106 | def change_session 107 | @session = get_session 108 | 109 | yield @session if block_given? 110 | 111 | set_session(@session) 112 | end 113 | 114 | protected 115 | 116 | def control_variable(control) 117 | Hash === control ? control.keys.first : control 118 | end 119 | 120 | def experiment_variable(alternatives, control, experiment_name) 121 | begin 122 | experiment = Split::Experiment.find_or_create(experiment_name, *([control] + alternatives)) 123 | if experiment.winner 124 | ret = experiment.winner.name 125 | else 126 | if forced_alternative = override(experiment.name, experiment.alternative_names) 127 | ret = forced_alternative 128 | else 129 | clean_old_versions(experiment) 130 | begin_experiment(experiment) if exclude_visitor? or not_allowed_to_test?(experiment.key) 131 | 132 | if get_session[experiment.key] 133 | ret = get_session[experiment.key] 134 | else 135 | alternative = experiment.next_alternative 136 | alternative.increment_participation 137 | begin_experiment(experiment, alternative.name) 138 | ret = alternative.name 139 | end 140 | end 141 | end 142 | rescue => e 143 | raise unless Split.configuration.db_failover 144 | Split.configuration.db_failover_on_db_error.call(e) 145 | ret = control_variable(control) 146 | end 147 | ret 148 | end 149 | 150 | end 151 | 152 | end 153 | -------------------------------------------------------------------------------- /lib/split/dashboard/public/style.css: -------------------------------------------------------------------------------- 1 | html { 2 | background: #efefef; 3 | font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; 4 | font-size: 13px; 5 | } 6 | 7 | body { 8 | padding: 0 10px; 9 | margin: 10px auto 0; 10 | max-width:800px; 11 | } 12 | 13 | .header { 14 | background: #ededed; 15 | background: -webkit-gradient(linear, left top, left bottom, 16 | color-stop(0%,#576a76), 17 | color-stop(100%,#4d5256)); 18 | background: -moz-linear-gradient (top, #576a76 0%, #414e58 100%); 19 | background: -webkit-linear-gradient(top, #576a76 0%, #414e58 100%); 20 | background: -o-linear-gradient (top, #576a76 0%, #414e58 100%); 21 | background: -ms-linear-gradient (top, #576a76 0%, #414e58 100%); 22 | background: linear-gradient (top, #576a76 0%, #414e58 100%); 23 | border-bottom: 1px solid #fff; 24 | -moz-border-radius-topleft: 5px; 25 | -webkit-border-top-left-radius: 5px; 26 | border-top-left-radius: 5px; 27 | -moz-border-radius-topright: 5px; 28 | -webkit-border-top-right-radius:5px; 29 | border-top-right-radius: 5px; 30 | 31 | overflow:hidden; 32 | padding: 10px 5%; 33 | text-shadow:0 1px 0 #000; 34 | } 35 | 36 | .header h1 { 37 | color: #eee; 38 | float:left; 39 | font-size:1.2em; 40 | font-weight:normal; 41 | margin:2px 30px 0 0; 42 | } 43 | 44 | .header ul li { 45 | display: inline; 46 | } 47 | 48 | .header ul li a { 49 | color: #eee; 50 | text-decoration: none; 51 | margin-right: 10px; 52 | display: inline-block; 53 | padding: 4px 8px; 54 | -moz-border-radius: 10px; 55 | -webkit-border-radius:10px; 56 | border-radius: 10px; 57 | 58 | } 59 | 60 | .header ul li a:hover { 61 | background: rgba(255,255,255,0.1); 62 | } 63 | 64 | .header ul li a:active { 65 | -moz-box-shadow: inset 0 1px 0 rgba(0,0,0,0.2); 66 | -webkit-box-shadow:inset 0 1px 0 rgba(0,0,0,0.2); 67 | box-shadow: inset 0 1px 0 rgba(0,0,0,0.2); 68 | } 69 | 70 | .header ul li.current a { 71 | background: rgba(255,255,255,0.1); 72 | -moz-box-shadow: inset 0 1px 0 rgba(0,0,0,0.2); 73 | -webkit-box-shadow:inset 0 1px 0 rgba(0,0,0,0.2); 74 | box-shadow: inset 0 1px 0 rgba(0,0,0,0.2); 75 | color: #fff; 76 | } 77 | 78 | #main { 79 | padding: 10px 5%; 80 | background: #f9f9f9; 81 | border:1px solid #ccc; 82 | border-top:none; 83 | -moz-box-shadow: 0 3px 10px rgba(0,0,0,0.2); 84 | -webkit-box-shadow:0 3px 10px rgba(0,0,0,0.2); 85 | box-shadow: 0 3px 10px rgba(0,0,0,0.2); 86 | overflow: hidden; 87 | } 88 | 89 | #main .logo { 90 | float: right; 91 | margin: 10px; 92 | } 93 | 94 | #main span.hl { 95 | background: #efefef; 96 | padding: 2px; 97 | } 98 | 99 | #main h1 { 100 | margin: 10px 0; 101 | font-size: 190%; 102 | font-weight: bold; 103 | color: #0080FF; 104 | } 105 | 106 | #main table { 107 | width: 100%; 108 | margin:0 0 10px; 109 | } 110 | 111 | #main table tr td, #main table tr th { 112 | border-bottom: 1px solid #ccc; 113 | padding: 6px; 114 | } 115 | 116 | #main table tr th { 117 | background: #efefef; 118 | color: #888; 119 | font-size: 80%; 120 | text-transform:uppercase; 121 | } 122 | 123 | #main table tr td.no-data { 124 | text-align: center; 125 | padding: 40px 0; 126 | color: #999; 127 | font-style: italic; 128 | font-size: 130%; 129 | } 130 | 131 | #main a { 132 | color: #111; 133 | } 134 | 135 | #main p { 136 | margin: 5px 0; 137 | } 138 | 139 | #main p.intro { 140 | margin-bottom: 15px; 141 | font-size: 85%; 142 | color: #999; 143 | margin-top: 0; 144 | line-height: 1.3; 145 | } 146 | 147 | #main h1.wi { 148 | margin-bottom: 5px; 149 | } 150 | 151 | #main p.sub { 152 | font-size: 95%; 153 | color: #999; 154 | } 155 | 156 | .experiment { 157 | background:#fff; 158 | border: 1px solid #eee; 159 | border-bottom:none; 160 | margin:30px 0; 161 | } 162 | 163 | .experiment .experiment-header { 164 | background: #f4f4f4; 165 | background: -webkit-gradient(linear, left top, left bottom, 166 | color-stop(0%,#f4f4f4), 167 | color-stop(100%,#e0e0e0)); 168 | background: -moz-linear-gradient (top, #f4f4f4 0%, #e0e0e0 100%); 169 | background: -webkit-linear-gradient(top, #f4f4f4 0%, #e0e0e0 100%); 170 | background: -o-linear-gradient (top, #f4f4f4 0%, #e0e0e0 100%); 171 | background: -ms-linear-gradient (top, #f4f4f4 0%, #e0e0e0 100%); 172 | background: linear-gradient (top, #f4f4f4 0%, #e0e0e0 100%); 173 | border-top:1px solid #fff; 174 | overflow:hidden; 175 | padding:0 10px; 176 | } 177 | 178 | .experiment h2 { 179 | color:#888; 180 | margin: 12px 0 0; 181 | font-size: 1em; 182 | font-weight:bold; 183 | float:left; 184 | text-shadow:0 1px 0 rgba(255,255,255,0.8); 185 | } 186 | 187 | .experiment h2 .version{ 188 | font-style:italic; 189 | font-size:0.8em; 190 | color:#bbb; 191 | font-weight:normal; 192 | } 193 | 194 | .experiment table em{ 195 | font-style:italic; 196 | font-size:0.9em; 197 | color:#bbb; 198 | } 199 | 200 | .experiment table .totals td { 201 | background: #eee; 202 | font-weight: bold; 203 | } 204 | 205 | #footer { 206 | padding: 10px 5%; 207 | color: #999; 208 | font-size: 85%; 209 | line-height: 1.5; 210 | padding-top: 10px; 211 | } 212 | 213 | #footer p a { 214 | color: #999; 215 | } 216 | 217 | .inline-controls { 218 | float:right; 219 | } 220 | 221 | .inline-controls small { 222 | color: #888; 223 | font-size: 11px; 224 | } 225 | 226 | .inline-controls form { 227 | display: inline-block; 228 | font-size: 10px; 229 | line-height: 38px; 230 | } 231 | 232 | .inline-controls input { 233 | margin-left: 10px; 234 | } 235 | 236 | .worse, .better { 237 | color: #773F3F; 238 | font-size: 10px; 239 | font-weight:bold; 240 | } 241 | 242 | .better { 243 | color: #408C48; 244 | } 245 | 246 | a.button, button, input[type="submit"] { 247 | padding: 4px 10px; 248 | overflow: hidden; 249 | background: #d8dae0; 250 | -moz-box-shadow: 0 1px 0 rgba(0,0,0,0.5); 251 | -webkit-box-shadow:0 1px 0 rgba(0,0,0,0.5); 252 | box-shadow: 0 1px 0 rgba(0,0,0,0.5); 253 | border:none; 254 | -moz-border-radius: 30px; 255 | -webkit-border-radius:30px; 256 | border-radius: 30px; 257 | color:#2e3035; 258 | cursor: pointer; 259 | font-family: 'Helvetica Neue', Helvetica, Arial, sans-serif; 260 | text-decoration: none; 261 | text-shadow:0 1px 0 rgba(255,255,255,0.8); 262 | -moz-user-select: none; 263 | -webkit-user-select:none; 264 | user-select: none; 265 | white-space: nowrap; 266 | } 267 | a.button:hover, button:hover, input[type="submit"]:hover, 268 | a.button:focus, button:focus, input[type="submit"]:focus{ 269 | background:#bbbfc7; 270 | } 271 | a.button:active, button:active, input[type="submit"]:active{ 272 | -moz-box-shadow: inset 0 0 4px #484d57; 273 | -webkit-box-shadow:inset 0 0 4px #484d57; 274 | box-shadow: inset 0 0 4px #484d57; 275 | position:relative; 276 | top:1px; 277 | } 278 | 279 | a.button.red, button.red, input[type="submit"].red, 280 | a.button.green, button.green, input[type="submit"].green { 281 | color:#fff; 282 | text-shadow:0 1px 0 rgba(0,0,0,0.4); 283 | } 284 | 285 | a.button.red, button.red, input[type="submit"].red { 286 | background:#a56d6d; 287 | } 288 | a.button.red:hover, button.red:hover, input[type="submit"].red:hover, 289 | a.button.red:focus, button.red:focus, input[type="submit"].red:focus { 290 | background:#895C5C; 291 | } 292 | a.button.green, button.green, input[type="submit"].green { 293 | background:#8daa92; 294 | } 295 | a.button.green:hover, button.green:hover, input[type="submit"].green:hover, 296 | a.button.green:focus, button.green:focus, input[type="submit"].green:focus { 297 | background:#768E7A; 298 | } 299 | 300 | 301 | -------------------------------------------------------------------------------- /spec/experiment_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'split/experiment' 3 | 4 | describe Split::Experiment do 5 | before(:each) { Split.redis.flushall } 6 | 7 | it "should have a name" do 8 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 9 | experiment.name.should eql('basket_text') 10 | end 11 | 12 | it "should have alternatives" do 13 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 14 | experiment.alternatives.length.should be 2 15 | end 16 | 17 | it "should save to redis" do 18 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 19 | experiment.save 20 | Split.redis.exists('basket_text').should be true 21 | end 22 | 23 | it "should save the start time to redis" do 24 | experiment_start_time = Time.parse("Sat Mar 03 14:01:03") 25 | Time.stub(:now => experiment_start_time) 26 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 27 | experiment.save 28 | 29 | Split::Experiment.find('basket_text').start_time.should == experiment_start_time 30 | end 31 | 32 | it "should handle not having a start time" do 33 | experiment_start_time = Time.parse("Sat Mar 03 14:01:03") 34 | Time.stub(:now => experiment_start_time) 35 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 36 | experiment.save 37 | 38 | Split.redis.hdel(:experiment_start_times, experiment.name) 39 | 40 | Split::Experiment.find('basket_text').start_time.should == nil 41 | end 42 | 43 | it "should not create duplicates when saving multiple times" do 44 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 45 | experiment.save 46 | experiment.save 47 | Split.redis.exists('basket_text').should be true 48 | Split.redis.lrange('basket_text', 0, -1).should eql(['Basket', "Cart"]) 49 | end 50 | 51 | describe 'deleting' do 52 | it 'should delete itself' do 53 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 54 | experiment.save 55 | 56 | experiment.delete 57 | Split.redis.exists('basket_text').should be false 58 | Split::Experiment.find('link_color').should be_nil 59 | end 60 | 61 | it "should increment the version" do 62 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 63 | experiment.version.should eql(0) 64 | experiment.delete 65 | experiment.version.should eql(1) 66 | end 67 | end 68 | 69 | describe 'new record?' do 70 | it "should know if it hasn't been saved yet" do 71 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 72 | experiment.new_record?.should be_true 73 | end 74 | 75 | it "should know if it has been saved yet" do 76 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 77 | experiment.save 78 | experiment.new_record?.should be_false 79 | end 80 | end 81 | 82 | describe 'find' do 83 | it "should return an existing experiment" do 84 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 85 | experiment.save 86 | Split::Experiment.find('basket_text').name.should eql('basket_text') 87 | end 88 | 89 | it "should return an existing experiment" do 90 | Split::Experiment.find('non_existent_experiment').should be_nil 91 | end 92 | end 93 | 94 | describe 'control' do 95 | it 'should be the first alternative' do 96 | experiment = Split::Experiment.new('basket_text', 'Basket', "Cart") 97 | experiment.save 98 | experiment.control.name.should eql('Basket') 99 | end 100 | end 101 | 102 | describe 'winner' do 103 | it "should have no winner initially" do 104 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 105 | experiment.winner.should be_nil 106 | end 107 | 108 | it "should allow you to specify a winner" do 109 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 110 | experiment.winner = 'red' 111 | 112 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 113 | experiment.winner.name.should == 'red' 114 | end 115 | end 116 | 117 | describe 'reset' do 118 | it 'should reset all alternatives' do 119 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 120 | green = Split::Alternative.new('green', 'link_color') 121 | experiment.winner = 'green' 122 | 123 | experiment.next_alternative.name.should eql('green') 124 | green.increment_participation 125 | 126 | experiment.reset 127 | 128 | reset_green = Split::Alternative.new('green', 'link_color') 129 | reset_green.participant_count.should eql(0) 130 | reset_green.completed_count.should eql(0) 131 | end 132 | 133 | it 'should reset the winner' do 134 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 135 | green = Split::Alternative.new('green', 'link_color') 136 | experiment.winner = 'green' 137 | 138 | experiment.next_alternative.name.should eql('green') 139 | green.increment_participation 140 | 141 | experiment.reset 142 | 143 | experiment.winner.should be_nil 144 | end 145 | 146 | it "should increment the version" do 147 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 148 | experiment.version.should eql(0) 149 | experiment.reset 150 | experiment.version.should eql(1) 151 | end 152 | end 153 | 154 | describe 'next_alternative' do 155 | it "should always return the winner if one exists" do 156 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 157 | green = Split::Alternative.new('green', 'link_color') 158 | experiment.winner = 'green' 159 | 160 | experiment.next_alternative.name.should eql('green') 161 | green.increment_participation 162 | 163 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 164 | experiment.next_alternative.name.should eql('green') 165 | end 166 | end 167 | 168 | describe 'changing an existing experiment' do 169 | it "should reset an experiment if it is loaded with different alternatives" do 170 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red', 'green') 171 | blue = Split::Alternative.new('blue', 'link_color') 172 | blue.participant_count = 5 173 | blue.save 174 | same_experiment = Split::Experiment.find_or_create('link_color', 'blue', 'yellow', 'orange') 175 | same_experiment.alternatives.map(&:name).should eql(['blue', 'yellow', 'orange']) 176 | new_blue = Split::Alternative.new('blue', 'link_color') 177 | new_blue.participant_count.should eql(0) 178 | end 179 | end 180 | 181 | describe 'alternatives passed as non-strings' do 182 | it "should throw an exception if an alternative is passed that is not a string" do 183 | lambda { Split::Experiment.find_or_create('link_color', :blue, :red) }.should raise_error 184 | lambda { Split::Experiment.find_or_create('link_enabled', true, false) }.should raise_error 185 | end 186 | end 187 | 188 | describe 'specifying weights' do 189 | it "should work for a new experiment" do 190 | experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 2 }) 191 | 192 | experiment.alternatives.map(&:weight).should == [1, 2] 193 | end 194 | 195 | it "should work for an existing experiment" do 196 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 197 | experiment.save 198 | 199 | same_experiment = Split::Experiment.find_or_create('link_color', {'blue' => 1}, {'red' => 2 }) 200 | same_experiment.alternatives.map(&:weight).should == [1, 2] 201 | end 202 | end 203 | 204 | 205 | 206 | end 207 | -------------------------------------------------------------------------------- /README.mdown: -------------------------------------------------------------------------------- 1 | # Split 2 | 3 | Split is a rack based ab testing framework designed to work with Rails, Sinatra or any other rack based app. 4 | 5 | Split is heavily inspired by the Abingo and Vanity rails ab testing plugins and Resque in its use of Redis. 6 | 7 | Split is designed to be hacker friendly, allowing for maximum customisation and extensibility. 8 | 9 | [![Build Status](https://secure.travis-ci.org/andrew/split.png?branch=master)](http://travis-ci.org/andrew/split) [![Dependency Status](https://gemnasium.com/andrew/split.png)](https://gemnasium.com/andrew/split) 10 | 11 | ## Requirements 12 | 13 | Split uses redis as a datastore. 14 | 15 | Split only supports redis 2.0 or greater. 16 | 17 | If you're on OS X, Homebrew is the simplest way to install Redis: 18 | 19 | $ brew install redis 20 | $ redis-server /usr/local/etc/redis.conf 21 | 22 | You now have a Redis daemon running on 6379. 23 | 24 | ## Setup 25 | 26 | If you are using bundler add split to your Gemfile: 27 | 28 | gem 'split' 29 | 30 | Then run: 31 | 32 | bundle install 33 | 34 | Otherwise install the gem: 35 | 36 | gem install split 37 | 38 | and require it in your project: 39 | 40 | require 'split' 41 | 42 | ### SystemTimer 43 | 44 | If you are using Redis on Ruby 1.8.x then you will likely want to also use the SystemTimer gem if you want to make sure the Redis client will not hang. 45 | 46 | Put the following in your gemfile as well: 47 | 48 | gem 'SystemTimer' 49 | 50 | ### Rails 51 | 52 | Split is autoloaded when rails starts up, as long as you've configured redis it will 'just work'. 53 | 54 | ### Sinatra 55 | 56 | To configure sinatra with Split you need to enable sessions and mix in the helper methods. Add the following lines at the top of your sinatra app: 57 | 58 | class MySinatraApp < Sinatra::Base 59 | enable :sessions 60 | helpers Split::Helper 61 | 62 | get '/' do 63 | ... 64 | end 65 | 66 | ## Usage 67 | 68 | To begin your ab test use the `ab_test` method, naming your experiment with the first argument and then the different variants which you wish to test on as the other arguments. 69 | 70 | `ab_test` returns one of the alternatives, if a user has already seen that test they will get the same alternative as before, which you can use to split your code on. 71 | 72 | It can be used to render different templates, show different text or any other case based logic. 73 | 74 | `finished` is used to make a completion of an experiment, or conversion. 75 | 76 | Example: View 77 | 78 | <% ab_test("login_button", "/images/button1.jpg", "/images/button2.jpg") do |button_file| %> 79 | <%= img_tag(button_file, :alt => "Login!") %> 80 | <% end %> 81 | 82 | Example: Controller 83 | 84 | def register_new_user 85 | # See what level of free points maximizes users' decision to buy replacement points. 86 | @starter_points = ab_test("new_user_free_points", '100', '200', '300') 87 | end 88 | 89 | Example: Conversion tracking (in a controller!) 90 | 91 | def buy_new_points 92 | # some business logic 93 | finished("new_user_free_points") 94 | end 95 | 96 | Example: Conversion tracking (in a view) 97 | 98 | Thanks for signing up, dude! <% finished("signup_page_redesign") > 99 | 100 | You can find more examples, tutorials and guides on the [wiki](https://github.com/andrew/split/wiki). 101 | 102 | ## Extras 103 | 104 | ### Weighted alternatives 105 | 106 | Perhaps you only want to show an alternative to 10% of your visitors because it is very experimental or not yet fully load tested. 107 | 108 | To do this you can pass a weight with each alternative in the following ways: 109 | 110 | ab_test('homepage design', {'Old' => 20}, {'New' => 2}) 111 | 112 | ab_test('homepage design', 'Old', {'New' => 0.1}) 113 | 114 | ab_test('homepage design', {'Old' => 10}, 'New') 115 | 116 | Note: If using ruby 1.8.x and weighted alternatives you should always pass the control alternative through as the second argument with any other alternatives as a third argument because the order of the hash is not preserved in ruby 1.8, ruby 1.9.1+ users are not affected by this bug. 117 | 118 | This will only show the new alternative to visitors 1 in 10 times, the default weight for an alternative is 1. 119 | 120 | ### Overriding alternatives 121 | 122 | For development and testing, you may wish to force your app to always return an alternative. 123 | You can do this by passing it as a parameter in the url. 124 | 125 | If you have an experiment called `button_color` with alternatives called `red` and `blue` used on your homepage, a url such as: 126 | 127 | http://myawesomesite.com?button_color=red 128 | 129 | will always have red buttons. This won't be stored in your session or count towards to results. 130 | 131 | ### Reset after completion 132 | 133 | When a user completes a test their session is reset so that they may start the test again in the future. 134 | 135 | To stop this behaviour you can pass the following option to the `finished` method: 136 | 137 | finished('experiment_name', :reset => false) 138 | 139 | The user will then always see the alternative they started with. 140 | 141 | ### Multiple experiments at once 142 | 143 | By default Split will avoid users participating in multiple experiments at once. This means you are less likely to skew results by adding in more variation to your tests. 144 | 145 | To stop this behaviour and allow users to participate in multiple experiments at once enable the `allow_multiple_experiments` config option like so: 146 | 147 | Split.configure do |config| 148 | config.allow_multiple_experiments = true 149 | end 150 | 151 | ## Web Interface 152 | 153 | Split comes with a Sinatra-based front end to get an overview of how your experiments are doing. 154 | 155 | If you are running Rails 2: You can mount this inside your app using Rack::URLMap in your `config.ru` 156 | 157 | require 'split/dashboard' 158 | 159 | run Rack::URLMap.new \ 160 | "/" => Your::App.new, 161 | "/split" => Split::Dashboard.new 162 | 163 | However, if you are using Rails 3: You can mount this inside your app routes by first adding this to the Gemfile: 164 | 165 | gem 'split', :require => 'split/dashboard' 166 | 167 | Then adding this to config/routes.rb 168 | 169 | mount Split::Dashboard, :at => 'split' 170 | 171 | You may want to password protect that page, you can do so with `Rack::Auth::Basic` 172 | 173 | Split::Dashboard.use Rack::Auth::Basic do |username, password| 174 | username == 'admin' && password == 'p4s5w0rd' 175 | end 176 | 177 | ## Configuration 178 | 179 | You can override the default configuration options of Split like so: 180 | 181 | Split.configure do |config| 182 | config.robot_regex = /my_custom_robot_regex/ 183 | config.ignore_ip_addresses << '81.19.48.130' 184 | config.db_failover = true # handle redis errors gracefully 185 | config.db_failover_on_db_error = proc{|error| Rails.logger.error(error.message) } 186 | config.allow_multiple_experiments = true 187 | config.enabled = true 188 | end 189 | 190 | ### DB failover solution 191 | 192 | Due to the fact that Redis has no autom. failover mechanism, it's 193 | possible to switch on the `db_failover` config option, so that `ab_test` 194 | and `finished` will not crash in case of a db failure. `ab_test` always 195 | delivers alternative A (the first one) in that case. 196 | 197 | It's also possible to set a `db_failover_on_db_error` callback (proc) 198 | for example to log these errors via Rails.logger. 199 | 200 | ### Redis 201 | 202 | You may want to change the Redis host and port Split connects to, or 203 | set various other options at startup. 204 | 205 | Split has a `redis` setter which can be given a string or a Redis 206 | object. This means if you're already using Redis in your app, Split 207 | can re-use the existing connection. 208 | 209 | String: `Split.redis = 'localhost:6379'` 210 | 211 | Redis: `Split.redis = $redis` 212 | 213 | For our rails app we have a `config/initializers/split.rb` file where 214 | we load `config/split.yml` by hand and set the Redis information 215 | appropriately. 216 | 217 | Here's our `config/split.yml`: 218 | 219 | development: localhost:6379 220 | test: localhost:6379 221 | staging: redis1.example.com:6379 222 | fi: localhost:6379 223 | production: redis1.example.com:6379 224 | 225 | And our initializer: 226 | 227 | rails_root = ENV['RAILS_ROOT'] || File.dirname(__FILE__) + '/../..' 228 | rails_env = ENV['RAILS_ENV'] || 'development' 229 | 230 | split_config = YAML.load_file(rails_root + '/config/split.yml') 231 | Split.redis = split_config[rails_env] 232 | 233 | ## Namespaces 234 | 235 | If you're running multiple, separate instances of Split you may want 236 | to namespace the keyspaces so they do not overlap. This is not unlike 237 | the approach taken by many memcached clients. 238 | 239 | This feature is provided by the [redis-namespace][rs] library, which 240 | Split uses by default to separate the keys it manages from other keys 241 | in your Redis server. 242 | 243 | Simply use the `Split.redis.namespace` accessor: 244 | 245 | Split.redis.namespace = "split:blog" 246 | 247 | We recommend sticking this in your initializer somewhere after Redis 248 | is configured. 249 | 250 | ## Extensions 251 | 252 | - [Split::Export](http://github.com/andrew/split-export) - easily export ab test data out of Split 253 | - [Split::Analytics](http://github.com/andrew/split-analytics) - push test data to google analytics 254 | 255 | ## Screencast 256 | 257 | Ryan bates has produced an excellent 10 minute screencast about split on the Railscasts site: [A/B Testing with Split](http://railscasts.com/episodes/331-a-b-testing-with-split) 258 | 259 | ## Contributors 260 | 261 | Special thanks to the following people for submitting patches: 262 | 263 | * Lloyd Pick 264 | * Jeffery Chupp 265 | * Andrew Appleton 266 | 267 | ## Development 268 | 269 | Source hosted at [GitHub](http://github.com/andrew/split). 270 | Report Issues/Feature requests on [GitHub Issues](http://github.com/andrew/split/issues). 271 | 272 | Tests can be ran with `rake spec` 273 | 274 | ### Note on Patches/Pull Requests 275 | 276 | * Fork the project. 277 | * Make your feature addition or bug fix. 278 | * Add tests for it. This is important so I don't break it in a 279 | future version unintentionally. 280 | * Commit, do not mess with rakefile, version, or history. 281 | (if you want to have your own version, that is fine but bump version in a commit by itself I can ignore when I pull) 282 | * Send me a pull request. Bonus points for topic branches. 283 | 284 | ## Copyright 285 | 286 | Copyright (c) 2012 Andrew Nesbitt. See [LICENSE](https://github.com/andrew/split/blob/master/LICENSE) for details. 287 | -------------------------------------------------------------------------------- /spec/helper_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | # TODO change some of these tests to use Rack::Test 4 | 5 | describe Split::Helper do 6 | include Split::Helper 7 | 8 | before(:each) do 9 | Split.redis.flushall 10 | @session = {} 11 | params = nil 12 | end 13 | 14 | describe "ab_test" do 15 | it "should assign a random alternative to a new user when there are an equal number of alternatives assigned" do 16 | ab_test('link_color', 'blue', 'red') 17 | ['red', 'blue'].should include(ab_user['link_color']) 18 | end 19 | 20 | it "should increment the participation counter after assignment to a new user" do 21 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 22 | 23 | previous_red_count = Split::Alternative.new('red', 'link_color').participant_count 24 | previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 25 | 26 | ab_test('link_color', 'blue', 'red') 27 | 28 | new_red_count = Split::Alternative.new('red', 'link_color').participant_count 29 | new_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 30 | 31 | (new_red_count + new_blue_count).should eql(previous_red_count + previous_blue_count + 1) 32 | end 33 | 34 | it "should return the given alternative for an existing user" do 35 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 36 | alternative = ab_test('link_color', 'blue', 'red') 37 | repeat_alternative = ab_test('link_color', 'blue', 'red') 38 | alternative.should eql repeat_alternative 39 | end 40 | 41 | it 'should always return the winner if one is present' do 42 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 43 | experiment.winner = "orange" 44 | 45 | ab_test('link_color', 'blue', 'red').should == 'orange' 46 | end 47 | 48 | it "should allow the alternative to be force by passing it in the params" do 49 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 50 | @params = {'link_color' => 'blue'} 51 | alternative = ab_test('link_color', 'blue', 'red') 52 | alternative.should eql('blue') 53 | alternative = ab_test('link_color', {'blue' => 1}, 'red' => 5) 54 | alternative.should eql('blue') 55 | @params = {'link_color' => 'red'} 56 | alternative = ab_test('link_color', 'blue', 'red') 57 | alternative.should eql('red') 58 | alternative = ab_test('link_color', {'blue' => 5}, 'red' => 1) 59 | alternative.should eql('red') 60 | end 61 | 62 | it "should allow passing a block" do 63 | alt = ab_test('link_color', 'blue', 'red') 64 | ret = ab_test('link_color', 'blue', 'red') { |alternative| "shared/#{alternative}" } 65 | ret.should eql("shared/#{alt}") 66 | end 67 | 68 | it "should allow the share of visitors see an alternative to be specificed" do 69 | ab_test('link_color', {'blue' => 0.8}, {'red' => 20}) 70 | ['red', 'blue'].should include(ab_user['link_color']) 71 | end 72 | 73 | it "should allow alternative weighting interface as a single hash" do 74 | ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2) 75 | experiment = Split::Experiment.find('link_color') 76 | experiment.alternative_names.should eql(['blue', 'red']) 77 | end 78 | 79 | it "should only let a user participate in one experiment at a time" do 80 | ab_test('link_color', 'blue', 'red') 81 | ab_test('button_size', 'small', 'big') 82 | ab_user['button_size'].should eql('small') 83 | big = Split::Alternative.new('big', 'button_size') 84 | big.participant_count.should eql(0) 85 | small = Split::Alternative.new('small', 'button_size') 86 | small.participant_count.should eql(0) 87 | end 88 | 89 | it "should let a user participate in many experiment with allow_multiple_experiments option" do 90 | Split.configure do |config| 91 | config.allow_multiple_experiments = true 92 | end 93 | link_color = ab_test('link_color', 'blue', 'red') 94 | button_size = ab_test('button_size', 'small', 'big') 95 | ab_user['button_size'].should eql(button_size) 96 | button_size_alt = Split::Alternative.new(button_size, 'button_size') 97 | button_size_alt.participant_count.should eql(1) 98 | end 99 | end 100 | 101 | describe 'finished' do 102 | it 'should increment the counter for the completed alternative' do 103 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 104 | alternative_name = ab_test('link_color', 'blue', 'red') 105 | 106 | previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 107 | 108 | finished('link_color') 109 | 110 | new_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 111 | 112 | new_completion_count.should eql(previous_completion_count + 1) 113 | end 114 | 115 | it "should clear out the user's participation from their session" do 116 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 117 | alternative_name = ab_test('link_color', 'blue', 'red') 118 | 119 | previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 120 | 121 | session[:split].should eql("link_color" => alternative_name) 122 | finished('link_color') 123 | session[:split].should == {} 124 | end 125 | 126 | it "should not clear out the users session if reset is false" do 127 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 128 | alternative_name = ab_test('link_color', 'blue', 'red') 129 | 130 | previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 131 | 132 | session[:split].should eql("link_color" => alternative_name) 133 | finished('link_color', :reset => false) 134 | session[:split].should eql("link_color" => alternative_name) 135 | end 136 | 137 | it "should do nothing where the experiment was not started by this user" do 138 | session[:split] = nil 139 | lambda { finished('some_experiment_not_started_by_the_user') }.should_not raise_exception 140 | end 141 | end 142 | 143 | describe 'conversions' do 144 | it 'should return a conversion rate for an alternative' do 145 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 146 | alternative_name = ab_test('link_color', 'blue', 'red') 147 | 148 | previous_convertion_rate = Split::Alternative.new(alternative_name, 'link_color').conversion_rate 149 | previous_convertion_rate.should eql(0.0) 150 | 151 | finished('link_color') 152 | 153 | new_convertion_rate = Split::Alternative.new(alternative_name, 'link_color').conversion_rate 154 | new_convertion_rate.should eql(1.0) 155 | end 156 | end 157 | 158 | describe 'when user is a robot' do 159 | before(:each) do 160 | @request = OpenStruct.new(:user_agent => 'Googlebot/2.1 (+http://www.google.com/bot.html)') 161 | end 162 | 163 | describe 'ab_test' do 164 | it 'should return the control' do 165 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 166 | alternative = ab_test('link_color', 'blue', 'red') 167 | alternative.should eql experiment.control.name 168 | end 169 | 170 | it "should not increment the participation count" do 171 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 172 | 173 | previous_red_count = Split::Alternative.new('red', 'link_color').participant_count 174 | previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 175 | 176 | ab_test('link_color', 'blue', 'red') 177 | 178 | new_red_count = Split::Alternative.new('red', 'link_color').participant_count 179 | new_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 180 | 181 | (new_red_count + new_blue_count).should eql(previous_red_count + previous_blue_count) 182 | end 183 | end 184 | describe 'finished' do 185 | it "should not increment the completed count" do 186 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 187 | alternative_name = ab_test('link_color', 'blue', 'red') 188 | 189 | previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 190 | 191 | finished('link_color') 192 | 193 | new_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 194 | 195 | new_completion_count.should eql(previous_completion_count) 196 | end 197 | end 198 | end 199 | describe 'when ip address is ignored' do 200 | before(:each) do 201 | @request = OpenStruct.new(:ip => '81.19.48.130') 202 | Split.configure do |c| 203 | c.ignore_ip_addresses << '81.19.48.130' 204 | end 205 | end 206 | 207 | describe 'ab_test' do 208 | it 'should return the control' do 209 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 210 | alternative = ab_test('link_color', 'blue', 'red') 211 | alternative.should eql experiment.control.name 212 | end 213 | 214 | it "should not increment the participation count" do 215 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 216 | 217 | previous_red_count = Split::Alternative.new('red', 'link_color').participant_count 218 | previous_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 219 | 220 | ab_test('link_color', 'blue', 'red') 221 | 222 | new_red_count = Split::Alternative.new('red', 'link_color').participant_count 223 | new_blue_count = Split::Alternative.new('blue', 'link_color').participant_count 224 | 225 | (new_red_count + new_blue_count).should eql(previous_red_count + previous_blue_count) 226 | end 227 | end 228 | describe 'finished' do 229 | it "should not increment the completed count" do 230 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 231 | alternative_name = ab_test('link_color', 'blue', 'red') 232 | 233 | previous_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 234 | 235 | finished('link_color') 236 | 237 | new_completion_count = Split::Alternative.new(alternative_name, 'link_color').completed_count 238 | 239 | new_completion_count.should eql(previous_completion_count) 240 | end 241 | end 242 | end 243 | 244 | describe 'versioned experiments' do 245 | it "should use version zero if no version is present" do 246 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 247 | alternative_name = ab_test('link_color', 'blue', 'red') 248 | experiment.version.should eql(0) 249 | session[:split].should eql({'link_color' => alternative_name}) 250 | end 251 | 252 | it "should save the version of the experiment to the session" do 253 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 254 | experiment.reset 255 | experiment.version.should eql(1) 256 | alternative_name = ab_test('link_color', 'blue', 'red') 257 | session[:split].should eql({'link_color:1' => alternative_name}) 258 | end 259 | 260 | it "should load the experiment even if the version is not 0" do 261 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 262 | experiment.reset 263 | experiment.version.should eql(1) 264 | alternative_name = ab_test('link_color', 'blue', 'red') 265 | session[:split].should eql({'link_color:1' => alternative_name}) 266 | return_alternative_name = ab_test('link_color', 'blue', 'red') 267 | return_alternative_name.should eql(alternative_name) 268 | end 269 | 270 | it "should reset the session of a user on an older version of the experiment" do 271 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 272 | alternative_name = ab_test('link_color', 'blue', 'red') 273 | session[:split].should eql({'link_color' => alternative_name}) 274 | alternative = Split::Alternative.new(alternative_name, 'link_color') 275 | alternative.participant_count.should eql(1) 276 | 277 | experiment.reset 278 | experiment.version.should eql(1) 279 | alternative = Split::Alternative.new(alternative_name, 'link_color') 280 | alternative.participant_count.should eql(0) 281 | 282 | new_alternative_name = ab_test('link_color', 'blue', 'red') 283 | session[:split]['link_color:1'].should eql(new_alternative_name) 284 | new_alternative = Split::Alternative.new(new_alternative_name, 'link_color') 285 | new_alternative.participant_count.should eql(1) 286 | end 287 | 288 | it "should cleanup old versions of experiments from the session" do 289 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 290 | alternative_name = ab_test('link_color', 'blue', 'red') 291 | session[:split].should eql({'link_color' => alternative_name}) 292 | alternative = Split::Alternative.new(alternative_name, 'link_color') 293 | alternative.participant_count.should eql(1) 294 | 295 | experiment.reset 296 | experiment.version.should eql(1) 297 | alternative = Split::Alternative.new(alternative_name, 'link_color') 298 | alternative.participant_count.should eql(0) 299 | 300 | new_alternative_name = ab_test('link_color', 'blue', 'red') 301 | session[:split].should eql({'link_color:1' => new_alternative_name}) 302 | end 303 | 304 | it "should only count completion of users on the current version" do 305 | experiment = Split::Experiment.find_or_create('link_color', 'blue', 'red') 306 | alternative_name = ab_test('link_color', 'blue', 'red') 307 | session[:split].should eql({'link_color' => alternative_name}) 308 | alternative = Split::Alternative.new(alternative_name, 'link_color') 309 | 310 | experiment.reset 311 | experiment.version.should eql(1) 312 | 313 | finished('link_color') 314 | alternative = Split::Alternative.new(alternative_name, 'link_color') 315 | alternative.completed_count.should eql(0) 316 | end 317 | end 318 | 319 | context 'when redis is not available' do 320 | 321 | before(:each) do 322 | Split.stub(:redis).and_raise(Errno::ECONNREFUSED.new) 323 | end 324 | 325 | context 'and db_failover config option is turned off' do 326 | 327 | before(:each) do 328 | Split.configure do |config| 329 | config.db_failover = false 330 | end 331 | end 332 | 333 | describe 'ab_test' do 334 | it 'should raise an exception' do 335 | lambda { 336 | ab_test('link_color', 'blue', 'red') 337 | }.should raise_error(Errno::ECONNREFUSED) 338 | end 339 | end 340 | 341 | describe 'finished' do 342 | it 'should raise an exception' do 343 | lambda { 344 | finished('link_color') 345 | }.should raise_error(Errno::ECONNREFUSED) 346 | end 347 | end 348 | 349 | describe "disable split testing" do 350 | 351 | before(:each) do 352 | Split.configure do |config| 353 | config.enabled = false 354 | end 355 | end 356 | 357 | after(:each) do 358 | Split.configure do |config| 359 | config.enabled = true 360 | end 361 | end 362 | 363 | it "should not attempt to connect to redis" do 364 | 365 | lambda { 366 | ab_test('link_color', 'blue', 'red') 367 | }.should_not raise_error(Errno::ECONNREFUSED) 368 | end 369 | 370 | it "should return control variable" do 371 | ab_test('link_color', 'blue', 'red').should eq('blue') 372 | lambda { 373 | finished('link_color') 374 | }.should_not raise_error(Errno::ECONNREFUSED) 375 | end 376 | 377 | end 378 | 379 | 380 | end 381 | 382 | context 'and db_failover config option is turned on' do 383 | 384 | before(:each) do 385 | Split.configure do |config| 386 | config.db_failover = true 387 | end 388 | end 389 | 390 | describe 'ab_test' do 391 | it 'should not raise an exception' do 392 | lambda { 393 | ab_test('link_color', 'blue', 'red') 394 | }.should_not raise_error(Errno::ECONNREFUSED) 395 | end 396 | it 'should call db_failover_on_db_error proc with error as parameter' do 397 | Split.configure do |config| 398 | config.db_failover_on_db_error = proc do |error| 399 | error.should be_a(Errno::ECONNREFUSED) 400 | end 401 | end 402 | Split.configuration.db_failover_on_db_error.should_receive(:call) 403 | ab_test('link_color', 'blue', 'red') 404 | end 405 | it 'should always use first alternative' do 406 | ab_test('link_color', 'blue', 'red').should eq('blue') 407 | ab_test('link_color', {'blue' => 0.01}, 'red' => 0.2).should eq('blue') 408 | ab_test('link_color', {'blue' => 0.8}, {'red' => 20}).should eq('blue') 409 | ab_test('link_color', 'blue', 'red') do |alternative| 410 | "shared/#{alternative}" 411 | end.should eq('shared/blue') 412 | end 413 | end 414 | 415 | describe 'finished' do 416 | it 'should not raise an exception' do 417 | lambda { 418 | finished('link_color') 419 | }.should_not raise_error(Errno::ECONNREFUSED) 420 | end 421 | it 'should call db_failover_on_db_error proc with error as parameter' do 422 | Split.configure do |config| 423 | config.db_failover_on_db_error = proc do |error| 424 | error.should be_a(Errno::ECONNREFUSED) 425 | end 426 | end 427 | Split.configuration.db_failover_on_db_error.should_receive(:call) 428 | finished('link_color') 429 | end 430 | end 431 | 432 | 433 | end 434 | 435 | end 436 | 437 | end 438 | --------------------------------------------------------------------------------