├── .gitignore ├── .limited_red ├── .simplecov ├── .travis.yml ├── Gemfile ├── Gemfile.lock ├── LICENSE ├── README.md ├── Rakefile ├── bin └── pairwise ├── features ├── bugs │ ├── bad_arguments.feature │ └── bad_yml.feature ├── customizing_pairwise_output_format.feature ├── generating_pairwise_data.feature ├── step_definitions │ └── pairwise_data_steps.rb └── support │ ├── env.rb │ └── hooks.rb ├── lib ├── pairwise.rb └── pairwise │ ├── cli.rb │ ├── formatter.rb │ ├── formatter │ ├── csv.rb │ └── cucumber.rb │ ├── input_data.rb │ ├── input_file.rb │ ├── ipo.rb │ ├── ipo │ ├── horizontal.rb │ └── vertical.rb │ ├── pair_collection.rb │ └── test_pair.rb ├── pairwise.gemspec └── spec ├── pairwise ├── cli_spec.rb ├── ipo │ └── horizontal_spec.rb ├── ipo_spec.rb └── pair_collection_spec.rb ├── pairwise_spec.rb └── spec_helper.rb /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | tmp/ 3 | pkg 4 | rdoc 5 | .#* 6 | coverage/ 7 | *.swp 8 | -------------------------------------------------------------------------------- /.limited_red: -------------------------------------------------------------------------------- 1 | project name: pairwise 2 | -------------------------------------------------------------------------------- /.simplecov: -------------------------------------------------------------------------------- 1 | SimpleCov.configure do 2 | add_filter 'spec' 3 | add_filter 'features' 4 | end 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | rvm: 2 | - 1.8.7 3 | - 1.9.2 4 | - 1.9.3 5 | # whitelist 6 | branches: 7 | only: 8 | - master 9 | 10 | notifications: 11 | email: 12 | - joe@josephwilk.net 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | group :development,:test do 4 | gem 'rspec' 5 | gem 'cucumber' 6 | gem 'limited_red' 7 | gem 'rake' 8 | gem 'simplecov' 9 | gem 'guard' 10 | gem 'guard-rspec' 11 | end 12 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: http://rubygems.org/ 3 | specs: 4 | builder (3.0.0) 5 | cucumber (1.2.1) 6 | builder (>= 2.1.2) 7 | diff-lcs (>= 1.1.3) 8 | gherkin (~> 2.11.0) 9 | json (>= 1.4.6) 10 | diff-lcs (1.1.3) 11 | ffi (1.1.5) 12 | gherkin (2.11.1) 13 | json (>= 1.4.6) 14 | guard (1.3.2) 15 | listen (>= 0.4.2) 16 | thor (>= 0.14.6) 17 | guard-rspec (1.2.1) 18 | guard (>= 1.1) 19 | httparty (0.8.1) 20 | multi_json 21 | multi_xml 22 | json (1.7.4) 23 | limited_red (0.4.2) 24 | cucumber (>= 1.1.4) 25 | httparty (= 0.8.1) 26 | listen (0.4.7) 27 | rb-fchange (~> 0.0.5) 28 | rb-fsevent (~> 0.9.1) 29 | rb-inotify (~> 0.8.8) 30 | multi_json (1.3.6) 31 | multi_xml (0.5.1) 32 | rake (0.9.2.2) 33 | rb-fchange (0.0.5) 34 | ffi 35 | rb-fsevent (0.9.1) 36 | rb-inotify (0.8.8) 37 | ffi (>= 0.5.0) 38 | rspec (2.11.0) 39 | rspec-core (~> 2.11.0) 40 | rspec-expectations (~> 2.11.0) 41 | rspec-mocks (~> 2.11.0) 42 | rspec-core (2.11.1) 43 | rspec-expectations (2.11.2) 44 | diff-lcs (~> 1.1.3) 45 | rspec-mocks (2.11.2) 46 | simplecov (0.6.4) 47 | multi_json (~> 1.0) 48 | simplecov-html (~> 0.5.3) 49 | simplecov-html (0.5.3) 50 | thor (0.16.0) 51 | 52 | PLATFORMS 53 | ruby 54 | 55 | DEPENDENCIES 56 | cucumber 57 | guard 58 | guard-rspec 59 | limited_red 60 | rake 61 | rspec 62 | simplecov 63 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) 2009 Joseph Wilk 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Pairwise 2 | ------- 3 | 4 | [![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/josephwilk/pairwise) 5 | [![Build Status](https://secure.travis-ci.org/josephwilk/pairwise.png)](http://travis-ci.org/josephwilk/pairwise) 6 | 7 | How to use Pairwise: http://josephwilk.github.com/pairwise/ 8 | 9 | Running tests 10 | ------------ 11 | 
rake
12 | 13 | 14 | Copyright 15 | -------- 16 | 17 | Copyright (c) 2009,2010,2011,2012 Joseph Wilk. See LICENSE for details. 18 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | 3 | require 'cucumber/rake/task' 4 | require 'rspec/core/rake_task' 5 | 6 | RSpec::Core::RakeTask.new(:spec) 7 | Cucumber::Rake::Task.new(:features) 8 | 9 | task :default => ["spec", "features"] -------------------------------------------------------------------------------- /bin/pairwise: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | $:.unshift(File.dirname(__FILE__) + '/../lib') unless $:.include?(File.dirname(__FILE__) + '/../lib') 3 | 4 | require 'pairwise' 5 | require 'pairwise/cli' 6 | 7 | Pairwise::Cli.execute(ARGV) 8 | -------------------------------------------------------------------------------- /features/bugs/bad_arguments.feature: -------------------------------------------------------------------------------- 1 | Feature: Bad arguments 2 | 3 | Scenario: Non existent file 4 | When I run pairwise file_with_does_not_exist 5 | And I should see in the errors 6 | """ 7 | No such file or directory 8 | """ 9 | 10 | Scenario: Existing folder 11 | Given I have a folder "empty" 12 | When I run pairwise empty/ 13 | Then I should see in the output 14 | """ 15 | Usage: pairwise 16 | """ 17 | 18 | Scenario: Unsupported input file type 19 | Given I have the file "example.rar" 20 | When I run pairwise example.rar 21 | Then I should see in the errors 22 | """ 23 | Unsupported file type: rar 24 | """ 25 | 26 | Scenario: Unsupported input file without extension 27 | Given I have the file "example" 28 | When I run pairwise example 29 | Then I should see in the errors 30 | """ 31 | Cannot determine file type for: example 32 | """ -------------------------------------------------------------------------------- /features/bugs/bad_yml.feature: -------------------------------------------------------------------------------- 1 | Feature: Bad yml 2 | 3 | Scenario: Empty yml 4 | Given I have the yaml file "inputs.yml" containing: 5 | """ 6 | """ 7 | When I run pairwise inputs.yml 8 | Then it should not show any errors 9 | And I should see in the output 10 | """ 11 | Error: 'inputs.yml' does not contain the right structure for me to generate the pairwise set! 12 | """ 13 | 14 | Scenario: yml with no lists 15 | Given I have the yaml file "listey_inputs.yml" containing: 16 | """ 17 | mookey 18 | """ 19 | When I run pairwise listey_inputs.yml 20 | Then it should not show any errors 21 | And I should see in the output 22 | """ 23 | Error: 'listey_inputs.yml' does not contain the right structure for me to generate the pairwise set! 24 | """ -------------------------------------------------------------------------------- /features/customizing_pairwise_output_format.feature: -------------------------------------------------------------------------------- 1 | Feature: Customizing pairwise output format 2 | In order to save time importing pairwise sets 3 | As a tester 4 | I want the output pairwise data to be in the most convenient format for me 5 | 6 | Scenario: formatting output as csv 7 | Given I have the yaml file "inputs.yml" containing: 8 | """ 9 | - event with image: [Football, Basketball, Soccer] 10 | - event without image: [Football, Basketball, Soccer] 11 | - media: [Image, Video, Music] 12 | """ 13 | When I run pairwise inputs.yml --format csv 14 | Then I should see the output 15 | """ 16 | event with image,event without image,media 17 | Football,Football,Image 18 | Football,Basketball,Music 19 | Football,Soccer,Video 20 | Basketball,Football,Music 21 | Basketball,Basketball,Video 22 | Basketball,Soccer,Image 23 | Soccer,Football,Video 24 | Soccer,Basketball,Image 25 | Soccer,Soccer,Music 26 | 27 | """ -------------------------------------------------------------------------------- /features/generating_pairwise_data.feature: -------------------------------------------------------------------------------- 1 | Feature: Generating pairwise data 2 | In order to test small, managable datasets while having confidence in test coverage 3 | As a tester 4 | I want a set of tests which is smaller than all the possible combinations of my specified inputs 5 | 6 | Scenario: No input file specified 7 | When I run pairwise 8 | Then I should see in the output 9 | """ 10 | Usage: pairwise [options] FILE.[yml|csv] 11 | """ 12 | 13 | Scenario: Ordered yaml inputs 14 | Given I have the yaml file "inputs.yml" containing: 15 | """ 16 | - event with image: [Football, Basketball, Soccer] 17 | - event without image: [Football, Basketball, Soccer] 18 | - media: [Image, Video, Music] 19 | """ 20 | When I run pairwise inputs.yml 21 | Then I should see the output 22 | """ 23 | | event with image | event without image | media | 24 | | Football | Football | Image | 25 | | Football | Basketball | Music | 26 | | Football | Soccer | Video | 27 | | Basketball | Football | Music | 28 | | Basketball | Basketball | Video | 29 | | Basketball | Soccer | Image | 30 | | Soccer | Football | Video | 31 | | Soccer | Basketball | Image | 32 | | Soccer | Soccer | Music | 33 | 34 | """ 35 | 36 | Scenario: Unorderd yaml inputs 37 | Given I have the yaml file "inputs.yml" containing: 38 | """ 39 | event with image: [Football, Basketball, Soccer] 40 | event without image: [Football, Basketball, Soccer] 41 | media: [Image, Video, Music] 42 | """ 43 | When I run pairwise inputs.yml 44 | Then I should see the output 45 | """ 46 | | event with image | event without image | media | 47 | | Football | Football | Image | 48 | | Football | Basketball | Music | 49 | | Football | Soccer | Video | 50 | | Basketball | Football | Music | 51 | | Basketball | Basketball | Video | 52 | | Basketball | Soccer | Image | 53 | | Soccer | Football | Video | 54 | | Soccer | Basketball | Image | 55 | | Soccer | Soccer | Music | 56 | 57 | """ 58 | 59 | Scenario: Single value yaml inputs 60 | Given I have the yaml file "inputs.yml" containing: 61 | """ 62 | 1: 1 63 | 2: 2 64 | """ 65 | When I run pairwise inputs.yml 66 | Then I should see the output 67 | """ 68 | | 1 | 2 | 69 | | 1 | 2 | 70 | 71 | """ 72 | 73 | Scenario: inputing as csv 74 | Given I have the csv file "inputs.csv" containing: 75 | """ 76 | media, event with image, event without image 77 | Image, Football, Football 78 | Video, Basketball, Basketball 79 | Music, Soccer, Soccer 80 | """ 81 | When I run pairwise inputs.csv 82 | Then I should see the output 83 | """ 84 | | event with image | event without image | media | 85 | | Football | Football | Image | 86 | | Football | Basketball | Music | 87 | | Football | Soccer | Video | 88 | | Basketball | Football | Music | 89 | | Basketball | Basketball | Video | 90 | | Basketball | Soccer | Image | 91 | | Soccer | Football | Video | 92 | | Soccer | Basketball | Image | 93 | | Soccer | Soccer | Music | 94 | 95 | """ 96 | 97 | Scenario: Not replacing wild cards 98 | Given I have the yaml file "inputs.yml" containing: 99 | """ 100 | - A: [A1, A2, A3] 101 | - B: [B1, B2] 102 | - C: [C1, C2, C3] 103 | """ 104 | When I run pairwise inputs.yml --keep-wild-cards 105 | Then I should see the output 106 | """ 107 | | A | B | C | 108 | | A1 | B1 | C1 | 109 | | A1 | B2 | C3 | 110 | | A2 | B1 | C3 | 111 | | A2 | B2 | C2 | 112 | | A3 | B1 | C2 | 113 | | A3 | B2 | C1 | 114 | | A2 | any_value_of_B | C1 | 115 | | A1 | any_value_of_B | C2 | 116 | | A3 | any_value_of_B | C3 | 117 | 118 | """ -------------------------------------------------------------------------------- /features/step_definitions/pairwise_data_steps.rb: -------------------------------------------------------------------------------- 1 | Given /^I have the (?:yaml |csv )?file "([^\"]*)" containing:$/ do |file_name, file_contents| 2 | create_file(file_name, file_contents) 3 | end 4 | 5 | Given /^I have the file "([^\"]*)"$/ do |filename| 6 | step %Q{I have the file "#{filename}" containing:}, "" 7 | end 8 | 9 | Given /^I have a folder "([^\"]*)"$/ do |folder_name| 10 | create_folder(folder_name) 11 | end 12 | 13 | When /^I run (.+)$/ do |command| 14 | run(command) 15 | end 16 | 17 | Then /^I should see the output$/ do |text| 18 | last_stdout.should == text 19 | end 20 | 21 | Then /^I should see in the output$/ do |string| 22 | step "it should not show any errors" 23 | last_stdout.should include(string) 24 | end 25 | 26 | Then /^I should see in the errors$/ do |string| 27 | last_stderr.should include(string) 28 | end 29 | 30 | Then /^it should not show any errors$/ do 31 | last_stderr.should == "" 32 | end 33 | -------------------------------------------------------------------------------- /features/support/env.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'simplecov' 3 | SimpleCov.start 4 | SimpleCov.command_name 'integration_tests' 5 | 6 | require 'tempfile' 7 | require 'rspec' 8 | require "rspec/mocks/standalone" 9 | 10 | require 'fileutils' 11 | require 'forwardable' 12 | 13 | SCRATCH_SPACE = 'tmp' 14 | 15 | require 'limited_red/plugins/cucumber' 16 | 17 | class PairwiseWorld 18 | extend Forwardable 19 | def_delegators PairwiseWorld, :self_test_dir 20 | 21 | def self.examples_dir(subdir=nil) 22 | @examples_dir ||= File.expand_path(File.join(File.dirname(__FILE__), "../../#{SCRATCH_SPACE}")) 23 | subdir ? File.join(@examples_dir, subdir) : @examples_dir 24 | end 25 | 26 | def self.self_test_dir 27 | @self_test_dir ||= examples_dir 28 | end 29 | 30 | def pairwise_lib_dir 31 | @pairwise_lib_dir ||= File.expand_path(File.join(File.dirname(__FILE__), '../../lib')) 32 | end 33 | 34 | def initialize 35 | @current_dir = self_test_dir 36 | end 37 | 38 | private 39 | attr_reader :last_exit_status, :last_stderr 40 | 41 | def last_stdout 42 | @last_stdout 43 | end 44 | 45 | def create_file(file_name, file_content) 46 | in_current_dir do 47 | FileUtils.mkdir_p(File.dirname(file_name)) unless File.directory?(File.dirname(file_name)) 48 | File.open(file_name, 'w') { |f| f << file_content } 49 | end 50 | end 51 | 52 | def create_folder(folder_name) 53 | in_current_dir do 54 | FileUtils.mkdir_p(folder_name) unless File.directory?(folder_name) 55 | end 56 | end 57 | 58 | def set_env_var(variable, value) 59 | @original_env_vars ||= {} 60 | @original_env_vars[variable] = ENV[variable] 61 | ENV[variable] = value 62 | end 63 | 64 | def in_current_dir(&block) 65 | Dir.chdir(@current_dir, &block) 66 | end 67 | 68 | def run(command) 69 | stderr_file = Tempfile.new('pairwise') 70 | stderr_file.close 71 | in_current_dir do 72 | IO.popen("../bin/#{command} 2> #{stderr_file.path}", 'r') do |io| 73 | @last_stdout = io.read 74 | end 75 | 76 | @last_exit_status = $?.exitstatus 77 | end 78 | @last_stderr = IO.read(stderr_file.path) 79 | end 80 | end 81 | 82 | World do 83 | PairwiseWorld.new 84 | end 85 | 86 | Before do 87 | FileUtils.rm_rf SCRATCH_SPACE 88 | FileUtils.mkdir SCRATCH_SPACE 89 | end 90 | -------------------------------------------------------------------------------- /features/support/hooks.rb: -------------------------------------------------------------------------------- 1 | Before do 2 | Kernel.stub!(:rand).and_return(0) 3 | end 4 | 5 | -------------------------------------------------------------------------------- /lib/pairwise.rb: -------------------------------------------------------------------------------- 1 | $:.unshift(File.dirname(__FILE__)) unless 2 | $:.include?(File.dirname(__FILE__)) || $:.include?(File.expand_path(File.dirname(__FILE__))) 3 | 4 | require 'pairwise/test_pair' 5 | require 'pairwise/pair_collection' 6 | require 'pairwise/ipo' 7 | require 'pairwise/ipo/horizontal' 8 | require 'pairwise/ipo/vertical' 9 | require 'pairwise/formatter' 10 | require 'pairwise/input_data' 11 | require 'pairwise/input_file' 12 | require 'pairwise/cli' 13 | 14 | require 'yaml' 15 | if RUBY_VERSION != '1.8.7' && RUBY_VERSION < '2.0.0' 16 | YAML::ENGINE.yamler = 'syck' 17 | end 18 | 19 | module Pairwise 20 | class InvalidInputData < StandardError; end 21 | 22 | VERSION = '0.2.3' 23 | 24 | class << self 25 | def combinations(*inputs) 26 | raise InvalidInputData, "Minimum of 2 inputs are required to generate pairwise test set" unless valid?(inputs) 27 | Pairwise::IPO.new(inputs).build 28 | end 29 | 30 | private 31 | def valid?(inputs) 32 | array_of_arrays?(inputs) && 33 | inputs.length >= 2 && 34 | !inputs[0].empty? && !inputs[1].empty? 35 | end 36 | 37 | def array_of_arrays?(data) 38 | data.reject{|datum| datum.kind_of?(Array)}.empty? 39 | end 40 | 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/pairwise/cli.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | 3 | module Pairwise 4 | class Cli 5 | BUILTIN_FORMATS = { 6 | 'cucumber' => [Pairwise::Formatter::Cucumber, 7 | 'Tables for Cucumber'], 8 | 'csv' => [Pairwise::Formatter::Csv, 9 | 'Comma seperated values']} 10 | 11 | max = BUILTIN_FORMATS.keys.map{|s| s.length}.max 12 | FORMAT_HELP = (BUILTIN_FORMATS.keys.sort.map do |key| 13 | " #{key}#{' ' * (max - key.length)} : #{BUILTIN_FORMATS[key][1]}" 14 | end) 15 | 16 | class << self 17 | def execute(args) 18 | new(args).execute! 19 | end 20 | end 21 | 22 | def initialize(args, out = STDOUT) 23 | @args, @out = args, out 24 | @options = defaults 25 | end 26 | 27 | def parse! 28 | @args.extend(::OptionParser::Arguable) 29 | @args.options do |opts| 30 | opts.banner = ["Usage: pairwise [options] FILE.[yml|csv]", "", 31 | "Example:", 32 | "pairwise data/inputs.yml", "", "", 33 | ].join("\n") 34 | opts.on("-k", "--keep-wild-cards", 35 | "Don't automatically replace any wild-cards which appear", 36 | "in the pairwise data") do 37 | @options[:keep_wild_cards] = true 38 | end 39 | opts.on('-f FORMAT', '--format FORMAT', 40 | "How to format pairwise data (Default: cucumber). Available formats:", 41 | *FORMAT_HELP) do |format| 42 | @options[:format] = format 43 | end 44 | opts.on_tail("--version", "Show version.") do 45 | @out.puts Pairwise::VERSION 46 | Kernel.exit(0) 47 | end 48 | opts.on_tail("-h", "--help", "You're looking at it.") do 49 | exit_with_help 50 | end 51 | end.parse! 52 | 53 | @filename_with_path = @args[0] unless @args.empty? 54 | end 55 | 56 | def execute! 57 | parse! 58 | validate_options! 59 | 60 | if inputs = InputFile.load(@filename_with_path) 61 | builder = Pairwise::IPO.new(inputs.data, @options) 62 | 63 | formatter.display(builder.build, inputs.labels) 64 | else 65 | puts "Error: '#{@filename_with_path}' does not contain the right structure for me to generate the pairwise set!" 66 | end 67 | end 68 | 69 | private 70 | def defaults 71 | { :keep_wild_cards => false, 72 | :format => 'cucumber' } 73 | end 74 | 75 | def validate_options! 76 | exit_with_help if @filename_with_path.nil? || @filename_with_path.empty? 77 | raise Errno::ENOENT, @filename_with_path unless File.exist?(@filename_with_path) 78 | exit_with_help unless File.file?(@filename_with_path) 79 | end 80 | 81 | def exit_with_help 82 | @out.puts @args.options.help 83 | Kernel.exit(0) 84 | end 85 | 86 | def formatter 87 | format = BUILTIN_FORMATS[@options[:format]][0] 88 | format.new(@out) 89 | end 90 | 91 | end 92 | end 93 | -------------------------------------------------------------------------------- /lib/pairwise/formatter.rb: -------------------------------------------------------------------------------- 1 | %w[cucumber csv].each {|file| require "pairwise/formatter/#{file}"} 2 | -------------------------------------------------------------------------------- /lib/pairwise/formatter/csv.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | module Formatter 3 | class Csv 4 | 5 | def initialize(out) 6 | @out = out 7 | end 8 | 9 | def display(test_data, input_labels) 10 | @out.puts input_labels.join(',') 11 | test_data.each do |data| 12 | @out.puts data.join(',') 13 | end 14 | end 15 | end 16 | end 17 | end -------------------------------------------------------------------------------- /lib/pairwise/formatter/cucumber.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | module Formatter 3 | class Cucumber 4 | 5 | def initialize(out) 6 | @out = out 7 | @max = {} 8 | end 9 | 10 | def display(test_data, input_labels) 11 | @test_data = label_wild_cards(test_data, input_labels) 12 | @input_labels = input_labels 13 | 14 | @out.print "|" 15 | @input_labels.each_with_index do |label, column| 16 | @out.print padded_string(label, column) + "|" 17 | end 18 | @out.puts 19 | 20 | @test_data.each do |data| 21 | @out.print "|" 22 | data.each_with_index do |datum, column| 23 | @out.print padded_string(datum, column) + "|" 24 | end 25 | @out.puts 26 | end 27 | end 28 | 29 | private 30 | def label_wild_cards(test_data, labels) 31 | test_data.map do |data| 32 | data.enum_for(:each_with_index).map do |datum, column| 33 | datum == IPO::WILD_CARD ? "any_value_of_#{labels[column]}" : datum 34 | end 35 | end 36 | end 37 | 38 | def padded_string(string, column) 39 | padding_length = max_line_length(column) - string.to_s.length 40 | " #{string} " + (" " * padding_length) 41 | end 42 | 43 | def max_line_length(column) 44 | @max[column] ||= ([@input_labels[column].to_s.length] + @test_data.map{|data| data[column].to_s.length}).max 45 | end 46 | 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/pairwise/input_data.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | class InputData 3 | 4 | def initialize(inputs) 5 | @inputs = inputs.is_a?(Hash) ? hash_inputs_to_list(inputs) : inputs 6 | end 7 | 8 | def data 9 | @data ||= @inputs.map {|input| input.values[0]} 10 | end 11 | 12 | def labels 13 | @labels ||= @inputs.map{|input| input.keys}.flatten 14 | end 15 | 16 | private 17 | 18 | def hash_inputs_to_list(inputs_hash) 19 | inputs_hash.sort.map do |key, value| 20 | value = [value] unless value.is_a?(Array) 21 | {key => value} 22 | end 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/pairwise/input_file.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | class InputFile 3 | 4 | class << self 5 | def load(filename) 6 | inputs = self.new(filename).load_and_parse 7 | InputData.new(inputs) if valid?(inputs) 8 | end 9 | 10 | def valid?(inputs) 11 | inputs && (inputs.is_a?(Array) || inputs.is_a?(Hash)) 12 | end 13 | end 14 | 15 | def initialize(filename) 16 | @filename = filename 17 | self.extend(input_file_module) 18 | end 19 | 20 | private 21 | 22 | def input_file_module 23 | type = @filename[/\.(.+)$/, 1] 24 | raise "Cannot determine file type for: #{@filename}" unless type 25 | case type.downcase 26 | when 'yaml', 'yml' then Yaml 27 | else 28 | Pairwise.const_get(type.capitalize) rescue raise "Unsupported file type: #{type}" 29 | end 30 | end 31 | 32 | end 33 | 34 | module Yaml 35 | def load_and_parse 36 | require 'yaml' 37 | begin 38 | inputs = YAML.load_file(@filename) 39 | rescue 40 | nil 41 | end 42 | end 43 | end 44 | 45 | module Csv 46 | def load_and_parse 47 | require 'csv' 48 | 49 | csv_data = CSV.read @filename 50 | headers = csv_data.shift.map {|head| head.to_s.strip } 51 | string_data = csv_data.map {|row| row.map {|cell| cell.to_s.strip } } 52 | 53 | inputs = Hash.new {|h,k| h[k] = []} 54 | 55 | string_data.each do |row| 56 | row.each_with_index { |value, index| inputs[headers[index]] << value } 57 | end 58 | 59 | inputs 60 | end 61 | end 62 | end -------------------------------------------------------------------------------- /lib/pairwise/ipo.rb: -------------------------------------------------------------------------------- 1 | # A pairwise implementation using the in-parameter-order (IPO) strategy. 2 | # Based on: http://ranger.uta.edu/~ylei/paper/ipo-tse.pdf 3 | module Pairwise 4 | class IPO 5 | 6 | WILD_CARD = 'wild_card' 7 | 8 | def initialize(inputs, options = {}) 9 | @list_of_input_values = inputs 10 | @options = options 11 | end 12 | 13 | def build 14 | input_combinations = PairCollection.new(@list_of_input_values[0], [@list_of_input_values[1]], 0) 15 | @list_of_input_values.size > 2 ? in_parameter_order_generation(input_combinations) : input_combinations.to_a 16 | end 17 | 18 | private 19 | 20 | def in_parameter_order_generation(input_combinations) 21 | @list_of_input_values[2..-1].each_with_index do |input_values, index| 22 | index += 2 23 | previously_grown_input_values = @list_of_input_values[0..(index-1)] 24 | 25 | input_combinations, uncovered_pairs = IPO::Horizontal.growth(input_combinations, input_values, previously_grown_input_values) 26 | input_combinations = IPO::Vertical.growth(input_combinations, uncovered_pairs) 27 | end 28 | input_combinations = replace_wild_cards(input_combinations) unless @options[:keep_wild_cards] 29 | input_combinations 30 | end 31 | 32 | def replace_wild_cards(input_combinations) 33 | map_wild_cards(input_combinations) do |_, index| 34 | if @list_of_input_values[index].length == 1 35 | @list_of_input_values[index][0] 36 | else 37 | pick_random_value(@list_of_input_values[index]) 38 | end 39 | end 40 | end 41 | 42 | def map_wild_cards(input_combinations) 43 | input_combinations.map do |input_combination| 44 | input_combination.enum_for(:each_with_index).map do |input_value, index| 45 | input_value == WILD_CARD ? yield(index, index) : input_value 46 | end 47 | end 48 | end 49 | 50 | def map_each_input_value(input_combinations, &block) 51 | input_combinations.map do |input_combination| 52 | input_combination.enum_for(:each_with_index).map do |input_value, index| 53 | yield input_value, index 54 | end 55 | end 56 | end 57 | 58 | def pick_random_value(input_combination) 59 | input_combination[Kernel.rand(input_combination.size)] 60 | end 61 | 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/pairwise/ipo/horizontal.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | class IPO 3 | 4 | class Horizontal 5 | class << self 6 | 7 | def growth(input_combinations, input_values_for_growth, previously_grown_input_values) 8 | uncovered_pairs = PairCollection.new(input_values_for_growth, previously_grown_input_values, previously_grown_input_values.size) 9 | input_combinations, uncovered_pairs = grow_input_combinations_and_remove_covered_pairs(input_combinations, input_values_for_growth, uncovered_pairs) 10 | [input_combinations, uncovered_pairs] 11 | end 12 | 13 | private 14 | 15 | def grow_input_combinations_and_remove_covered_pairs(input_combinations, input_values_for_growth, uncovered_pairs) 16 | input_combinations = input_combinations.enum_for(:each_with_index).map do |input_combination, input_index| 17 | extended_input_combination = uncovered_pairs.input_combination_that_covers_most_pairs(input_combination, input_values_for_growth) 18 | uncovered_pairs.remove_pairs_covered_by!(extended_input_combination) 19 | extended_input_combination 20 | end 21 | [input_combinations, uncovered_pairs] 22 | end 23 | end 24 | 25 | end 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/pairwise/ipo/vertical.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | class IPO 3 | class Vertical 4 | 5 | def self.growth(input_combinations, uncovered_pairs) 6 | new_input_combinations = uncovered_pairs.reduce([]) do |new_input_combinations, uncovered_pair| 7 | new_input_combinations = uncovered_pair.replace_wild_card(new_input_combinations) 8 | end 9 | 10 | input_combinations + new_input_combinations 11 | end 12 | 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/pairwise/pair_collection.rb: -------------------------------------------------------------------------------- 1 | module Pairwise 2 | class PairCollection < Array 3 | 4 | def initialize(input_parameter_values, input_value_lists, input_parameter_index) 5 | pairs = generate_pairs_between(input_parameter_values, input_value_lists, input_parameter_index) 6 | @combination_history = [] 7 | super(pairs) 8 | end 9 | 10 | def remove_pairs_covered_by!(extended_input_list) 11 | self.reject!{|pair| pair.covered_by?(extended_input_list)} 12 | end 13 | 14 | def input_combination_that_covers_most_pairs(input_combination, input_values_for_growth) 15 | candidates = input_values_for_growth.map{|value| input_combination + [value]} 16 | max_combination = candidates.max {|combination_1, combination_2| pairs_covered_count(combination_1) <=> pairs_covered_count(combination_2)} 17 | possible_max_combinations = candidates.select{|combination| pairs_covered_count(max_combination) == pairs_covered_count(combination)} 18 | 19 | winner = if possible_max_combinations.size > 1 && !@combination_history.empty? 20 | find_most_different_combination(possible_max_combinations) 21 | else 22 | possible_max_combinations[0] 23 | end 24 | @combination_history << winner 25 | winner 26 | end 27 | 28 | def to_a 29 | self.map{|list| list.to_a} 30 | end 31 | 32 | private 33 | 34 | def generate_pairs_between(input_parameter_values, input_value_lists, input_parameter_index) 35 | pairs = [] 36 | input_parameter_values.each do |input_value_a| 37 | input_value_lists.each_with_index do |input_list, input_value_b_index| 38 | input_list.each do |input_value_b| 39 | pairs << TestPair.new(input_parameter_index, input_value_b_index, input_value_a, input_value_b) 40 | end 41 | end 42 | end 43 | pairs 44 | end 45 | 46 | def pairs_covered_count(input_combination) 47 | self.reduce(0) do |covered_count, pair| 48 | covered_count += 1 if pair.covered_by?(input_combination) 49 | covered_count 50 | end 51 | end 52 | 53 | def find_most_different_combination(options) 54 | scores = options.map do |option| 55 | score(option) 56 | end.flatten 57 | 58 | _, winner_index = *scores.each_with_index.max 59 | options[winner_index] 60 | end 61 | 62 | def score(option) 63 | #O(n^2) 64 | @combination_history.map do |previous_combination| 65 | option.each_with_index.inject(0) do |difference_score, (value, index)| 66 | value != previous_combination[index] ? difference_score : difference_score += 1 67 | end 68 | end.max 69 | end 70 | 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /lib/pairwise/test_pair.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | 3 | module Pairwise 4 | class TestPair 5 | extend Forwardable 6 | 7 | def_delegators :@pair, :+, :inspect, :to_a, :== 8 | 9 | def initialize(p1_position, p2_position, p1, p2) 10 | @p1, @p2 = p1, p2 11 | @p1_position, @p2_position = p1_position, p2_position 12 | @pair = [@p1, @p2] 13 | end 14 | 15 | def covered_by?(test_pair) 16 | debugger unless test_pair 17 | test_pair[@p1_position] == @p1 && 18 | test_pair[@p2_position] == @p2 19 | end 20 | 21 | def replace_wild_card(new_input_combinations) 22 | #TODO: Decided if we should replace all matches or single matches? 23 | if wild_card_index = find_wild_card_index(new_input_combinations) 24 | new_input_combinations[wild_card_index][@p2_position] = @p2 25 | else 26 | new_input_combinations << create_input_list 27 | end 28 | new_input_combinations 29 | end 30 | 31 | private 32 | def create_input_list 33 | new_input_list = Array.new(@p1_position){IPO::WILD_CARD} 34 | 35 | new_input_list[@p1_position] = @p1 36 | new_input_list[@p2_position] = @p2 37 | new_input_list 38 | end 39 | 40 | def find_wild_card_index(input_combinations) 41 | wild_card_list = input_combinations.map do |input_combination| 42 | input_combination[@p2_position] == IPO::WILD_CARD && input_combination[@p1_position] == @p1 43 | end 44 | wild_card_list.rindex(true) 45 | end 46 | 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /pairwise.gemspec: -------------------------------------------------------------------------------- 1 | require File.dirname(__FILE__) + '/lib/pairwise' 2 | 3 | Gem::Specification.new do |s| 4 | s.name = "pairwise" 5 | s.summary = %Q{Turn large test data combinations into smaller ones} 6 | s.description = %Q{A tool for selecting a smaller number of test combinations (using pairwise generation) rather than exhaustively testing all possible permutations.} 7 | s.email = "joe@josephwilk.net" 8 | s.homepage = "http://wiki.github.com/josephwilk/pairwise" 9 | s.authors = ["Joseph Wilk"] 10 | s.require_paths = %w[lib] 11 | s.files = %w[README.md] + Dir.glob("{examples,lib,spec}/**/*.rb") 12 | s.version = Pairwise::VERSION 13 | s.executables = ['pairwise'] 14 | 15 | s.add_development_dependency 'rspec' 16 | s.add_development_dependency 'cucumber' 17 | end 18 | 19 | -------------------------------------------------------------------------------- /spec/pairwise/cli_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | module Pairwise 4 | describe Cli do 5 | 6 | before(:each) do 7 | Kernel.stub!(:exit).and_return(nil) 8 | end 9 | 10 | def output_stream 11 | @output_stream ||= StringIO.new 12 | end 13 | 14 | def options 15 | #TODO: push options out to an object and avoid hacking at private instance vars 16 | @cli.instance_variable_get("@options") 17 | end 18 | 19 | def prepare_args(args) 20 | args.is_a?(Array) ? args : args.split(' ') 21 | end 22 | 23 | def after_parsing(args) 24 | @cli = Pairwise::Cli.new(prepare_args(args), output_stream) 25 | @cli.parse! 26 | yield 27 | end 28 | 29 | context '--keep-wild-cards' do 30 | it "displays wild cards in output result" do 31 | after_parsing('--keep-wild-cards') do 32 | options[:keep_wild_cards].should == true 33 | end 34 | end 35 | end 36 | 37 | context '-f FORMAT or --format FORMAT' do 38 | it "defaults to the cucumber format for output" do 39 | after_parsing('') do 40 | options[:format].should == 'cucumber' 41 | end 42 | end 43 | 44 | it "overides the cucumber format when passed a specific format" do 45 | after_parsing('--format csv') do 46 | options[:format].should == 'csv' 47 | end 48 | end 49 | end 50 | 51 | context "--help" do 52 | it "displays usage" do 53 | after_parsing('--help') do 54 | output_stream.string.should =~ /Usage: pairwise \[options\] FILE\.\[yml|csv\]/ 55 | end 56 | end 57 | end 58 | 59 | context '--version' do 60 | it "displays Pairwise's version" do 61 | after_parsing('--version') do 62 | output_stream.string.should =~ /#{Pairwise::VERSION}/ 63 | end 64 | end 65 | end 66 | 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/pairwise/ipo/horizontal_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | module Pairwise 4 | class IPO 5 | 6 | describe Horizontal do 7 | describe ".growth" do 8 | context "when the input_combinations size is smaller than the input values for growth size" do 9 | it "should extenhd with C's inputs" do 10 | input_combinations = [[:A1, :B1], [:A1, :B2]] 11 | data = [[:A1, :A2], [:C1, :C2, :C3 ]] 12 | 13 | test_set, _ = Horizontal.growth(input_combinations, data[1], data[0..1]) 14 | 15 | test_set.should == [[:A1, :B1, :C1], [:A1, :B2, :C3]] 16 | end 17 | end 18 | 19 | context "when the input_combinations size is larger than the input values for growth size" do 20 | before(:each) do 21 | @test_pairs = [[:A1, :B1], [:A1, :B2], [:A2, :B1], [:A2, :B2]] 22 | @data = [[:A1, :A2], [:B1, :B2], [:C1, :C2, :C3 ]] 23 | end 24 | 25 | it "should return pairs extended with C's inputs" do 26 | test_set, _ = Horizontal.growth(@test_pairs, @data[2], @data[0..1]) 27 | 28 | test_set.should == [[:A1, :B1, :C1], [:A1, :B2, :C3], [:A2, :B1, :C3], [:A2, :B2, :C2]] 29 | end 30 | 31 | it "should return all the uncovered pairs" do 32 | _, pi = Horizontal.growth(@test_pairs, @data[2], @data[0..1]) 33 | 34 | pi.to_a.should == [[:C1, :A2], [:C1, :B2], [:C2, :A1], [:C2, :B1]] 35 | end 36 | 37 | end 38 | end 39 | end 40 | 41 | end 42 | end -------------------------------------------------------------------------------- /spec/pairwise/ipo_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | module Pairwise 4 | describe IPO do 5 | 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /spec/pairwise/pair_collection_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | module Pairwise 4 | describe PairCollection do 5 | describe '#input_combination_that_covers_most_pairs' do 6 | it "should do find the combination that covers most pairs" do 7 | pair_collection = PairCollection.new([:A2, :B2], [[:A2, :B2], [:B2, :A1]], 1) 8 | 9 | combination = pair_collection.input_combination_that_covers_most_pairs([:A2, :B2], [:C2, :A1]) 10 | 11 | combination.should == [:A2, :B2, :C2] 12 | end 13 | end 14 | end 15 | end -------------------------------------------------------------------------------- /spec/pairwise_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Pairwise do 4 | before(:each) do 5 | Kernel.stub!(:rand).and_return(0) 6 | end 7 | 8 | context "with invalid inputs" do 9 | it "should be invalid when running with no input" do 10 | lambda{ Pairwise.combinations([]) }.should raise_error(Pairwise::InvalidInputData) 11 | end 12 | 13 | it "should be invalid when running with only 1 input" do 14 | lambda{ Pairwise.combinations([:A1, :A2])}.should raise_error(Pairwise::InvalidInputData) 15 | end 16 | 17 | it "should be invalid when running with a single list input" do 18 | lambda{ Pairwise.combinations([:A1, :A2])}.should raise_error(Pairwise::InvalidInputData) 19 | end 20 | end 21 | 22 | context "with valid inputs" do 23 | context "which are equal lengths" do 24 | it "should generate pairs for 2 parameters of 1 value" do 25 | data = [[:A1], [:B1]] 26 | 27 | Pairwise.combinations(*data).should == [[:A1, :B1]] 28 | end 29 | 30 | it "should generate all pairs for 2 parameters of 2 values" do 31 | data = [[:A1, :A2], [:B1, :B2]] 32 | 33 | Pairwise.combinations(*data).should == [[:A1, :B1], [:A1, :B2], [:A2, :B1], [:A2, :B2]] 34 | end 35 | 36 | it "should generate all pairs for 3 parameters of 1 value" do 37 | data = [[:A1], [:B1], [:C1]] 38 | 39 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1]] 40 | end 41 | 42 | it "should generate pairs for three paramters" do 43 | data = [[:A1, :A2], 44 | [:B1, :B2], 45 | [:C1 , :C2]] 46 | 47 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1], 48 | [:A1, :B2, :C2], 49 | [:A2, :B1, :C2], 50 | [:A2, :B2, :C1]] 51 | end 52 | end 53 | 54 | context "which are unequal lengths" do 55 | it "should generate all pairs for 3 parameters of 1,1,2 values" do 56 | data = [[:A1], [:B1], [:C1, :C2]] 57 | 58 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1], 59 | [:A1, :B1, :C2]] 60 | end 61 | 62 | it "should generate all pairs for 3 parameters of 1,1,3 values" do 63 | data = [[:A1], [:B1], [:C1, :C2, :C3]] 64 | 65 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1], 66 | [:A1, :B1, :C2], 67 | [:A1, :B1, :C3]] 68 | end 69 | 70 | it "should generate all pairs for 3 parameters of 1,2,3 values" do 71 | data = [[:A1], [:B1, :B2], [:C1, :C2, :C3]] 72 | 73 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1], [:A1, :B2, :C3], [:A1, :B2, :C1], [:A1, :B1, :C2], [:A1, :B2, :C2], [:A1, :B1, :C3]] 74 | end 75 | 76 | it "should generate all pairs for 3 parameters of 2,1,2 values" do 77 | data = [[:A1, :A2], [:B1], [:C1, :C2]] 78 | 79 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1], 80 | [:A2, :B1, :C2], 81 | [:A2, :B1, :C1], 82 | [:A1, :B1, :C2]] 83 | 84 | 85 | #:A1, :B1, :C1 86 | #:A1, :B1, :C2 87 | #:A2, :B1, :C1 88 | #:A2,any_value_of_B, :C2 89 | end 90 | 91 | it "should generate all pairs for 4 parameters of 2,1,2,2 values" do 92 | data = [[:A1, :A2], [:B1], [:C1, :C2], [:D1, :D2]] 93 | 94 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1, :D1], [:A2, :B1, :C2, :D2], [:A2, :B1, :C1, :D2], [:A1, :B1, :C2, :D2], [:A2, :B1, :C2, :D1]] 95 | end 96 | 97 | it "should generate pairs for three parameters" do 98 | data = [[:A1, :A2], 99 | [:B1, :B2], 100 | [:C1 , :C2 , :C3 ]] 101 | 102 | Pairwise.combinations(*data).should == [[:A1, :B1, :C1], [:A1, :B2, :C3], [:A2, :B1, :C3], [:A2, :B2, :C2], [:A2, :B2, :C1], [:A1, :B1, :C2]] 103 | end 104 | 105 | describe "replacing wildcards which could have more than one option" do 106 | it "should generate pairs for 2 parameters of 3,2,3 values" do 107 | Pairwise.combinations([:A1, :A2, :A3], 108 | [:B1, :B2], 109 | [:C1, :C2, :C3]).should == [[:A1, :B1, :C1], 110 | [:A1, :B2, :C3], 111 | [:A2, :B1, :C3], 112 | [:A2, :B2, :C2], 113 | [:A3, :B1, :C2], 114 | [:A3, :B2, :C1], 115 | [:A2, :B1, :C1], 116 | [:A1, :B1, :C2], 117 | [:A3, :B1, :C3]] 118 | end 119 | end 120 | end 121 | 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler' 3 | Bundler.require(:test) 4 | 5 | SimpleCov.start 6 | SimpleCov.command_name 'unit_tests' 7 | 8 | require File.dirname(__FILE__) + '/../lib/pairwise' --------------------------------------------------------------------------------