├── lib ├── fuzzbert │ ├── version.rb │ ├── generation.rb │ ├── test.rb │ ├── container.rb │ ├── generator.rb │ ├── dsl.rb │ ├── mutator.rb │ ├── test_suite.rb │ ├── generators.rb │ ├── error_handler.rb │ ├── rake_task.rb │ ├── template.rb │ ├── autorun.rb │ └── executor.rb └── fuzzbert.rb ├── spec ├── fuzz │ ├── fuzz_nothing.rb │ └── fuzz_custom_handler.rb ├── test_spec.rb ├── mutator_spec.rb ├── generator_spec.rb ├── template_spec.rb ├── autorun_spec.rb ├── dsl_spec.rb └── executor_spec.rb ├── Gemfile ├── bin └── fuzzbert ├── .gitignore ├── .travis.yml ├── examples ├── mutator.rb ├── binary.rb ├── template.rb ├── cmd_line.rb └── custom_handler.rb ├── Rakefile ├── fuzzbert.gemspec ├── LICENSE ├── ci └── travis.rb └── README.md /lib/fuzzbert/version.rb: -------------------------------------------------------------------------------- 1 | module FuzzBert 2 | VERSION = "1.0.4" 3 | end 4 | 5 | -------------------------------------------------------------------------------- /lib/fuzzbert/generation.rb: -------------------------------------------------------------------------------- 1 | 2 | module FuzzBert::Generation 3 | 4 | def generator 5 | -> { to_data } 6 | end 7 | 8 | end 9 | -------------------------------------------------------------------------------- /spec/fuzz/fuzz_nothing.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | fuzz "nothing" do 4 | deploy { |_| } 5 | data("some") { -> {"a"} } 6 | end 7 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | group :development do 4 | gem 'rake' 5 | end 6 | 7 | group :test do 8 | gem 'rspec', '~> 2.99.0' 9 | end 10 | 11 | gemspec 12 | -------------------------------------------------------------------------------- /bin/fuzzbert: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | begin 4 | require 'fuzzbert' 5 | FuzzBert::AutoRun.autorun 6 | rescue LoadError 7 | $stderr.puts "Could not find fuzzbert." 8 | exit(1) 9 | end 10 | -------------------------------------------------------------------------------- /lib/fuzzbert/test.rb: -------------------------------------------------------------------------------- 1 | 2 | class FuzzBert::Test 3 | 4 | def initialize(runner) 5 | @runner = runner 6 | end 7 | 8 | def run(data) 9 | @runner.call data 10 | end 11 | 12 | end 13 | 14 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | tmp 2 | *.swp 3 | *.swo 4 | *~ 5 | .rbx/ 6 | .idea/ 7 | *.gem 8 | *.rbc 9 | .bundle 10 | .config 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | tmp 16 | 17 | # YARD artifacts 18 | .yardoc 19 | _yardoc 20 | doc/ 21 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | script: 'ci/travis.rb' 2 | rvm: 3 | - ruby-head 4 | - 1.9.3 5 | - rbx-2 6 | - 2.0.0 7 | - 2.1.0 8 | gemfile: 9 | - Gemfile 10 | notifications: 11 | email: 12 | recipients: 13 | - Martin.Bosslet@googlemail.com 14 | on_success: always 15 | on_failure: always 16 | branches: 17 | only: 18 | - master 19 | -------------------------------------------------------------------------------- /examples/mutator.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | fuzz "Web App" do 4 | deploy do |data| 5 | #send JSON data via HTTP 6 | end 7 | 8 | data "mutated data" do 9 | #in practice, choose a moderately sized value that is generally accepted by the target 10 | m = FuzzBert::Mutator.new '{ user: { id: 42, name: "FuzzBert" }' 11 | m.generator 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/fuzzbert/container.rb: -------------------------------------------------------------------------------- 1 | 2 | class FuzzBert::Container 3 | include FuzzBert::Generation 4 | 5 | def initialize(generators=[]) 6 | @generators = generators 7 | end 8 | 9 | def <<(generator) 10 | @generators << generator 11 | end 12 | 13 | def to_data 14 | "".tap do |buf| 15 | @generators.each { |gen| buf << gen.call } 16 | end 17 | end 18 | 19 | end 20 | 21 | 22 | -------------------------------------------------------------------------------- /spec/fuzz/fuzz_custom_handler.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | module FuzzBert::Spec 4 | class CustomHandler 5 | @@called = false 6 | 7 | def self.called 8 | @@called 9 | end 10 | 11 | def handle(error_data) 12 | @@called = true 13 | end 14 | end 15 | end 16 | 17 | fuzz "failing" do 18 | deploy { |_| raise "boo!" } 19 | data("some") { -> {"a"} } 20 | end 21 | -------------------------------------------------------------------------------- /lib/fuzzbert/generator.rb: -------------------------------------------------------------------------------- 1 | 2 | class FuzzBert::Generator 3 | include FuzzBert::Generation 4 | 5 | attr_reader :description 6 | 7 | def initialize(desc, generator=nil, &blk) 8 | @description = desc 9 | @generator = generator || blk 10 | raise RuntimeError.new("No generator given") unless @generator 11 | end 12 | 13 | def to_data 14 | @generator.call 15 | end 16 | 17 | end 18 | 19 | -------------------------------------------------------------------------------- /lib/fuzzbert/dsl.rb: -------------------------------------------------------------------------------- 1 | module FuzzBert::DSL 2 | def fuzz(*args, &blk) 3 | suite = FuzzBert::TestSuite.create(*args, &blk) 4 | raise RuntimeError.new "No 'deploy' block was given" unless suite.test 5 | raise RuntimeError.new "No 'data' blocks were given" unless suite.generators 6 | FuzzBert::AutoRun.register(suite) 7 | end 8 | end 9 | 10 | extend FuzzBert::DSL 11 | Module.send(:include, FuzzBert::DSL) 12 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | require 'rspec/core/rake_task' 3 | 4 | def java? 5 | !! (RUBY_PLATFORM =~ /java/) 6 | end 7 | 8 | def rubinius? 9 | !! (RUBY_ENGINE =~ /rbx/) 10 | end 11 | 12 | task :default => :spec 13 | 14 | RSpec::Core::RakeTask.new(:spec) do |spec| 15 | spec.ruby_opts = ['--1.9'] if java? 16 | spec.ruby_opts = ['-X19'] if rubinius? 17 | spec.rspec_opts = ['-c', '--format d'] 18 | spec.verbose = true 19 | spec.fail_on_error = true 20 | end 21 | 22 | -------------------------------------------------------------------------------- /lib/fuzzbert/mutator.rb: -------------------------------------------------------------------------------- 1 | 2 | class FuzzBert::Mutator < FuzzBert::Generator 3 | 4 | def initialize(value) 5 | orig = value.dup 6 | orig.force_encoding(Encoding::BINARY) 7 | super("Mutator") do 8 | #select a byte 9 | i = FuzzBert::PRNG.rand(value.size) 10 | old = orig[i].ord 11 | #map a random value from 0..254 to 0..255 excluding the current value 12 | b = FuzzBert::PRNG.rand(255) 13 | b = b < old ? b : b + 1 14 | orig.dup.tap { |s| s.setbyte(i, b) } 15 | end 16 | end 17 | 18 | end 19 | 20 | -------------------------------------------------------------------------------- /examples/binary.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | fuzz "String.to_i" do 4 | deploy do |data| 5 | begin 6 | String.to_i(data) 7 | rescue StandardError 8 | end 9 | end 10 | 11 | data("completely random") { FuzzBert::Generators.random } 12 | 13 | data("1..1000") { FuzzBert::Generators.cycle(1..1000) } 14 | 15 | data "leading zero, fixed length of 100 digits" do 16 | c = FuzzBert::Container.new 17 | c << FuzzBert::Generators.fixed("0") 18 | c << FuzzBert::Generators.random_fixlen(99) 19 | c.generator 20 | end 21 | 22 | end 23 | 24 | -------------------------------------------------------------------------------- /spec/test_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | describe FuzzBert::Test do 5 | 6 | describe "new" do 7 | it "takes a mandatory proc argument" do 8 | -> { FuzzBert::Test.new }.should raise_error 9 | FuzzBert::Test.new( lambda { |data| data }).should be_an_instance_of(FuzzBert::Test) 10 | end 11 | end 12 | 13 | describe "#run" do 14 | it "executes the block passed on creation with the data passed to it" do 15 | value = "test" 16 | t = FuzzBert::Test.new( lambda { |data| data }) 17 | t.run(value).should == value 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /examples/template.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | fuzz "Web App" do 4 | deploy do |data| 5 | #send JSON data via HTTP here instead 6 | p data 7 | end 8 | 9 | data "template" do 10 | t = FuzzBert::Template.new <<-EOS 11 | { user: { id: ${id}, name: "${name}", text: "${text}" } } 12 | EOS 13 | t.set(:id, FuzzBert::Generators.cycle(1..10000)) 14 | name = FuzzBert::Container.new 15 | name << FuzzBert::Generators.fixed("fixed") 16 | name << FuzzBert::Generators.random_fixlen(2) 17 | t.set(:name, name.generator) 18 | t.set(:text) { "Fixed text plus two random bytes: #{FuzzBert::Generators.random_fixlen(2).call}" } 19 | t.generator 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /examples/cmd_line.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | fuzz "OpenSSL command line (asn1parse)" do 4 | 5 | deploy do |data| 6 | IO.popen("openssl asn1parse -inform DER -noout", "w") do |io| 7 | io.write(data) 8 | end 9 | status = $? 10 | unless status.exited? && status.success? 11 | raise RuntimeError.new("bug!") 12 | end 13 | end 14 | 15 | data("completely random") { FuzzBert::Generators.random } 16 | 17 | data "Indefinite length sequence" do 18 | c = FuzzBert::Container.new 19 | c << FuzzBert::Generators.fixed("\x31\x80") 20 | c << FuzzBert::Generators.random 21 | c << FuzzBert::Generators.fixed("\x00\x00") 22 | c.generator 23 | end 24 | 25 | end 26 | 27 | -------------------------------------------------------------------------------- /lib/fuzzbert/test_suite.rb: -------------------------------------------------------------------------------- 1 | 2 | class FuzzBert::TestSuite 3 | 4 | attr_reader :description, :test, :generators 5 | 6 | def initialize(desc) 7 | @description = desc 8 | @generators = [] 9 | end 10 | 11 | def deploy(&blk) 12 | raise RuntimeError.new "No block was given" unless blk 13 | @test = FuzzBert::Test.new(blk) 14 | end 15 | 16 | def data(desc, &blk) 17 | raise RuntimeError.new "No block was given" unless blk 18 | @generators << FuzzBert::Generator.new(desc, blk.call) 19 | end 20 | 21 | def self.create(desc, &blk) 22 | raise RuntimeError.new "No block was given" unless blk 23 | obj = self.new(desc) 24 | obj.instance_eval(&blk) 25 | obj 26 | end 27 | 28 | end 29 | -------------------------------------------------------------------------------- /fuzzbert.gemspec: -------------------------------------------------------------------------------- 1 | $:.unshift File.expand_path('../lib', __FILE__) 2 | 3 | require 'fileutils' 4 | require 'fuzzbert/version' 5 | 6 | Gem::Specification.new do |s| 7 | s.name = 'fuzzbert' 8 | s.version = FuzzBert::VERSION 9 | s.author = 'Martin Bosslet' 10 | s.email = 'Martin.Bosslet@gmail.com' 11 | s.homepage = 'https://github.com/krypt/FuzzBert' 12 | s.description = 'A random testing / fuzzer framework for Ruby.' 13 | s.summary = 'Fuzz your applications and libraries with minimal effort.' 14 | s.files = Dir.glob('{lib}/**/*') 15 | s.files += ['LICENSE'] 16 | s.test_files = Dir.glob('spec/**/*.rb') 17 | s.extra_rdoc_files = [ "README.md" ] 18 | s.bindir = 'bin' 19 | s.executables = ['fuzzbert'] 20 | s.require_path = 'lib' 21 | s.license = 'MIT' 22 | end 23 | -------------------------------------------------------------------------------- /spec/mutator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | describe FuzzBert::Mutator do 5 | 6 | describe "::new" do 7 | it "takes a (valid) base value" do 8 | value = "test" 9 | -> {FuzzBert::Mutator.new(value)}.should_not raise_error 10 | end 11 | end 12 | 13 | describe "#generator" do 14 | it "implements Generation" do 15 | mut = FuzzBert::Mutator.new("value") 16 | mut.generator.should_not be_nil 17 | end 18 | end 19 | 20 | describe "#to_data" do 21 | it "mutates the base value in exactly one position" do 22 | value = "FuzzBert" 23 | mut = FuzzBert::Mutator.new(value) 24 | mutated = mut.to_data 25 | diff = 0 26 | value.each_byte.each_with_index do |b, i| 27 | diff += 1 unless b == mutated[i].ord 28 | end 29 | diff.should == 1 30 | end 31 | end 32 | 33 | end 34 | 35 | -------------------------------------------------------------------------------- /examples/custom_handler.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | 3 | # To use this Handler, you must pass it as an argument 4 | # to the 'fuzzbert' executable, something like 5 | # 6 | # fuzzbert --handler MyHandler "FILE_PATTERN" 7 | # 8 | class MyHandler 9 | def handle(error_data) 10 | #create an issue in the bug tracker 11 | puts error_data[:id] 12 | p error_data[:data] 13 | puts error_data[:pid] 14 | puts error_data[:status] 15 | end 16 | end 17 | 18 | fuzz "Some application" do 19 | 20 | deploy do |data| 21 | #send the generated data to your application here instead 22 | p data 23 | end 24 | 25 | data("completely random") { FuzzBert::Generators.random } 26 | 27 | data "Payload" do 28 | c = FuzzBert::Container.new 29 | c << FuzzBert::Generators.fixed("\x30\x80") 30 | c << FuzzBert::Generators.random 31 | c.generator 32 | end 33 | 34 | end 35 | 36 | -------------------------------------------------------------------------------- /spec/generator_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | describe FuzzBert::Generator do 5 | 6 | describe "::new" do 7 | it "takes a description and a generator" do 8 | desc = "desc" 9 | value = "test" 10 | gen = FuzzBert::Generator.new(desc, FuzzBert::Generators.fixed(value)) 11 | gen.description.should == desc 12 | gen.to_data.should == value 13 | end 14 | end 15 | 16 | describe "#generator" do 17 | it "implements Generation" do 18 | gen = FuzzBert::Generator.new("test") { "test" } 19 | gen.generator.should_not be_nil 20 | end 21 | end 22 | 23 | describe "#to_data" do 24 | it "returns the value returned by its generator" do 25 | value = "test" 26 | desc = "desc" 27 | gen = FuzzBert::Generator.new(desc) { value } 28 | gen.description.should == desc 29 | gen.to_data.should == value 30 | end 31 | end 32 | end 33 | 34 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2013 Martin Boßlet 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /ci/travis.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | class Runner 4 | def initialize(tasks) 5 | @tasks = tasks 6 | @results = {} 7 | end 8 | 9 | def run! 10 | header "FuzzBert CI started." 11 | puts "Ruby version:" 12 | system "ruby -v" 13 | @tasks.each do |t| 14 | cmd = "bundle exec rake #{t.to_s} --trace" 15 | @results[t] = system(cmd) 16 | end 17 | end 18 | 19 | def evaluate 20 | failed = @results.select { |k,v| !v } 21 | puts 22 | if failed.empty? 23 | echo_success "The build was successful." 24 | echo_success "All tasks have completed successfully." 25 | exit(true) 26 | else 27 | echo_failure "The build has failed." 28 | echo_failure "Failed tasks: #{failed.keys.join(', ')}" 29 | exit(false) 30 | end 31 | end 32 | 33 | private 34 | 35 | def header(msg) 36 | puts "\n\e[1;34m#{msg}\e[m\n" 37 | end 38 | 39 | def echo_failure(msg) 40 | puts "\n\e[1;31m#{msg}\e[m\n" 41 | end 42 | 43 | def echo_success(msg) 44 | puts "\n\e[1;32m#{msg}\e[m\n" 45 | end 46 | end 47 | 48 | r = Runner.new [:spec] 49 | r.run! 50 | r.evaluate 51 | -------------------------------------------------------------------------------- /lib/fuzzbert/generators.rb: -------------------------------------------------------------------------------- 1 | require 'base64' 2 | 3 | module FuzzBert::Generators 4 | 5 | module_function 6 | 7 | def random(limit=1024) 8 | -> { random_bytes(limit) { |data| data } } 9 | end 10 | 11 | def random_b64(limit=1024) 12 | -> { random_bytes(b64_len(limit)) { |data| Base64.encode64(data) } } 13 | end 14 | 15 | def random_hex(limit=1024) 16 | -> { random_bytes(hex_len(limit)) { |data| data.unpack("H*")[0] } } 17 | end 18 | 19 | def random_fixlen(len) 20 | -> { random_bytes_fixlen(len) { |data| data } } 21 | end 22 | 23 | def random_b64_fixlen(len) 24 | -> { random_bytes_fixlen(b64_len(len)) { |data| Base64.encode64(data) } } 25 | end 26 | 27 | def random_hex_fixlen(len) 28 | -> { random_bytes_fixlen(hex_len(len)) { |data| data.unpack("H*")[0] } } 29 | end 30 | 31 | def cycle(range) 32 | ary = range.to_a 33 | i = 0 34 | lambda do 35 | ret = ary[i] 36 | i = (i + 1) % ary.size 37 | ret 38 | end 39 | end 40 | 41 | def fixed(data) 42 | -> { data } 43 | end 44 | 45 | def sample(ary) 46 | -> { ary.sample } 47 | end 48 | 49 | private; module_function 50 | 51 | def hex_len(len) 52 | len / 2 53 | end 54 | 55 | def b64_len(len) 56 | len * 3 / 4 57 | end 58 | 59 | def random_bytes(limit) 60 | len = FuzzBert::PRNG.rand(1..limit) 61 | yield FuzzBert::PRNG.bytes(len) 62 | end 63 | 64 | def random_bytes_fixlen(len) 65 | yield FuzzBert::PRNG.bytes(len) 66 | end 67 | 68 | end 69 | -------------------------------------------------------------------------------- /lib/fuzzbert/error_handler.rb: -------------------------------------------------------------------------------- 1 | 2 | module FuzzBert::Handler 3 | 4 | module ConsoleHelper 5 | def info(error_data) 6 | status = error_data[:status] 7 | 8 | crashed = status.termsig 9 | 10 | if crashed 11 | puts "The data caused a hard crash." 12 | else 13 | puts "The data caused an uncaught error." 14 | end 15 | end 16 | end 17 | 18 | class FileOutput 19 | include FuzzBert::Handler::ConsoleHelper 20 | 21 | def initialize(dir=nil) 22 | @dir = dir 23 | if @dir && !@dir.end_with?("/") 24 | @dir << "/" 25 | end 26 | end 27 | 28 | def handle(error_data) 29 | id = error_data[:id] 30 | data = error_data[:data] 31 | status = error_data[:status] 32 | pid = error_data[:pid] 33 | 34 | crashed = status.termsig 35 | prefix = crashed ? "crash" : "bug" 36 | 37 | filename = "#{dir_prefix}#{prefix}#{pid}" 38 | while File.exists?(filename) 39 | filename << ('a'..'z').to_a.sample 40 | end 41 | File.open(filename, "wb") { |f| f.print(data) } 42 | 43 | puts "#{id} failed. Data was saved as #{filename}." 44 | info(error_data) 45 | end 46 | 47 | private 48 | 49 | def dir_prefix 50 | return "./" unless @dir 51 | @dir 52 | end 53 | end 54 | 55 | class Console 56 | include FuzzBert::Handler::ConsoleHelper 57 | 58 | def initialize 59 | $stdout.sync = true 60 | end 61 | 62 | def handle(error_data) 63 | puts "#{error_data[:id]} failed. Data: #{error_data[:data].inspect}" 64 | info(error_data) 65 | end 66 | end 67 | 68 | end 69 | 70 | 71 | -------------------------------------------------------------------------------- /lib/fuzzbert/rake_task.rb: -------------------------------------------------------------------------------- 1 | require 'fuzzbert' 2 | require 'rake' 3 | require 'rake/tasklib' 4 | 5 | class FuzzBert::RakeTask < ::Rake::TaskLib 6 | include ::Rake::DSL if defined?(::Rake::DSL) 7 | 8 | # Name of task. 9 | # 10 | # default: 11 | # :fuzz 12 | attr_accessor :name 13 | 14 | # Glob pattern to match files. 15 | # 16 | # default: 17 | # 'fuzz/**/fuzz_*.rb' 18 | attr_accessor :pattern 19 | 20 | # Command line options to pass to ruby. 21 | # 22 | # default: 23 | # nil 24 | attr_accessor :ruby_opts 25 | 26 | # Path to FuzzBert 27 | # 28 | # default: 29 | # 'fuzzbert' 30 | attr_accessor :fuzzbert_path 31 | 32 | # Command line options to pass to fuzzbert. 33 | # 34 | # default: 35 | # nil 36 | attr_accessor :fuzzbert_opts 37 | 38 | def initialize(*args) 39 | #configure the rake task 40 | setup_ivars(args) 41 | yield self if block_given? 42 | 43 | desc "Run FuzzBert random test suite" unless ::Rake.application.last_comment 44 | 45 | task name do 46 | run_task 47 | end 48 | end 49 | 50 | def setup_ivars(*args) 51 | @name = args.shift || :fuzz 52 | @ruby_opts, @fuzzbert_opts = nil, nil 53 | @fuzzbert_path = 'fuzzbert' 54 | @pattern = 'fuzz/**/fuzz_*.rb' 55 | end 56 | 57 | def run_task 58 | begin 59 | system(command) 60 | rescue 61 | #silent, user could have interrupted a permanent session 62 | end 63 | end 64 | 65 | private 66 | 67 | def command 68 | cmd = [RUBY, ruby_opts, '-S', fuzzbert_path, fuzzbert_opts, pattern] 69 | cmd.flatten.reject(&blank).join(" ") 70 | end 71 | 72 | def blank 73 | lambda {|s| s.nil? || s == ""} 74 | end 75 | 76 | end 77 | 78 | -------------------------------------------------------------------------------- /lib/fuzzbert.rb: -------------------------------------------------------------------------------- 1 | =begin 2 | 3 | = Info 4 | 5 | FuzzBert - Random Testing / Fuzzing in Ruby 6 | 7 | Copyright (C) 2012-2013 8 | Martin Bosslet 9 | All rights reserved. 10 | 11 | Permission is hereby granted, free of charge, to any person obtaining 12 | a copy of this software and associated documentation files (the 13 | "Software"), to deal in the Software without restriction, including 14 | without limitation the rights to use, copy, modify, merge, publish, 15 | distribute, sublicense, and/or sell copies of the Software, and to 16 | permit persons to whom the Software is furnished to do so, subject to 17 | the following conditions: 18 | 19 | The above copyright notice and this permission notice shall be 20 | included in all copies or substantial portions of the Software. 21 | 22 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 23 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 24 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 25 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 26 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 27 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 28 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 29 | 30 | =end 31 | 32 | module FuzzBert 33 | 34 | PRNG = Random.new 35 | 36 | end 37 | 38 | require_relative 'fuzzbert/version' 39 | require_relative 'fuzzbert/generation' 40 | require_relative 'fuzzbert/generators' 41 | require_relative 'fuzzbert/generator' 42 | require_relative 'fuzzbert/mutator' 43 | require_relative 'fuzzbert/template' 44 | require_relative 'fuzzbert/container' 45 | require_relative 'fuzzbert/test' 46 | require_relative 'fuzzbert/error_handler' 47 | require_relative 'fuzzbert/test_suite' 48 | require_relative 'fuzzbert/executor' 49 | require_relative 'fuzzbert/autorun' 50 | require_relative 'fuzzbert/dsl' 51 | 52 | -------------------------------------------------------------------------------- /spec/template_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | describe FuzzBert::Template do 5 | 6 | describe "::new" do 7 | it "takes a String parameter" do 8 | FuzzBert::Template.new("test").should be_an_instance_of(FuzzBert::Template) 9 | end 10 | end 11 | 12 | describe "#set" do 13 | it "allows to define callbacks for template variables" do 14 | t = FuzzBert::Template.new "a${var}c" 15 | t.set(:var) { "b" } 16 | t.to_data.should == "abc" 17 | end 18 | 19 | it "takes an optional proc to define the callback" do 20 | t = FuzzBert::Template.new "a${var}c" 21 | cb = -> { "b" } 22 | t.set(:var, cb) 23 | t.to_data.should == "abc" 24 | end 25 | 26 | it "takes only Symbols to reference the template variables" do 27 | t = FuzzBert::Template.new "a${var}c" 28 | t.set("var") { "b" } 29 | -> { t.to_data }.should raise_error 30 | end 31 | end 32 | 33 | describe "#generator" do 34 | it "implements Generation" do 35 | t = FuzzBert::Template.new "test" 36 | t.generator.should_not be_nil 37 | end 38 | end 39 | 40 | describe "#to_data" do 41 | it "can replace multiple template variables that possess a callback defined by set" do 42 | t = FuzzBert::Template.new "a${var1}c${var2}" 43 | t.set(:var1) { "b" } 44 | t.set(:var2) { "d" } 45 | t.to_data.should == "abcd" 46 | end 47 | 48 | specify "the dollar sign can be escaped with a backslash" do 49 | t = FuzzBert::Template.new "a\\${var}c" 50 | t.to_data.should == "a${var}c" 51 | end 52 | 53 | specify "a backslash can be escaped with another backslash" do 54 | t = FuzzBert::Template.new "a\\\\c" 55 | t.to_data.should == "a\\c" 56 | end 57 | 58 | it "raises an error if no closing brace is found for an open one" do 59 | -> { FuzzBert::Template.new "a${bc" }.should raise_error 60 | end 61 | 62 | it "does allow curly braces within a template variable identifier" do 63 | t = FuzzBert::Template.new "a${v{ar}c" 64 | t.set("v{ar".to_sym) { "b" } 65 | t.to_data.should == "abc" 66 | end 67 | 68 | it "does allow backslashes within a template variable identifier" do 69 | t = FuzzBert::Template.new "a${v\\ar}c" 70 | t.set("v\\ar".to_sym) { "b" } 71 | t.to_data.should == "abc" 72 | end 73 | 74 | it "allows text only" do 75 | t = FuzzBert::Template.new "abc" 76 | t.to_data.should == "abc" 77 | end 78 | 79 | it "allows variables only" do 80 | t = FuzzBert::Template.new "${a}${b}${c}" 81 | t.set(:a) { "a" } 82 | t.set(:b) { "b" } 83 | t.set(:c) { "c" } 84 | t.to_data.should == "abc" 85 | end 86 | 87 | it "allows heredoc strings" do 88 | t = FuzzBert::Template.new <<-EOS 89 | { user: { id: ${id}, name: "${name}" } } 90 | EOS 91 | 92 | t.set(:id) { "5" } 93 | t.set(:name) { "FuzzBert" } 94 | t.to_data.should == "{ user: { id: 5, name: \"FuzzBert\" } }\n" 95 | end 96 | 97 | it "raises an error if a variable callback hasn't been set and indicates which one" do 98 | t = FuzzBert::Template.new "a${var}c" 99 | begin 100 | t.to_data 101 | fail 102 | rescue StandardError => e 103 | e.message.should include("var") 104 | end 105 | end 106 | end 107 | 108 | end 109 | -------------------------------------------------------------------------------- /spec/autorun_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | 5 | describe FuzzBert::AutoRun do 6 | 7 | after(:each) do 8 | #clear any TestSuites that were created 9 | FuzzBert::AutoRun::TEST_CASES.clear 10 | end 11 | 12 | let(:handler) do 13 | c = Class.new 14 | def c.handle(error_data) 15 | raise RuntimeError.new 16 | end 17 | c 18 | end 19 | 20 | 21 | context "with a valid test" do 22 | value = "test" 23 | called = false 24 | 25 | fuzz "autorun" do 26 | 27 | deploy do |data| 28 | data.should == value 29 | end 30 | 31 | data "1" do 32 | called = true 33 | -> { value } 34 | end 35 | 36 | end 37 | 38 | it "has added a TestSuite to AutoRun" do 39 | FuzzBert::AutoRun::TEST_CASES.size.should == 1 40 | FuzzBert::AutoRun.run(pool_size: 1, limit: 1, handler: handler, sleep_delay: 0.05) 41 | called.should == true 42 | end 43 | end 44 | 45 | context "with no test" do 46 | it "raises an error when executed" do 47 | FuzzBert::AutoRun::TEST_CASES.should be_empty 48 | -> { FuzzBert::AutoRun.run }.should raise_error 49 | end 50 | end 51 | 52 | describe "command line" do 53 | 54 | def relative_path(filename) 55 | File.expand_path(filename, File.dirname(File.expand_path(__FILE__))) 56 | end 57 | 58 | let (:fuzz_nothing) { relative_path('fuzz/fuzz_nothing.rb') } 59 | let (:single_test_args) { ['--pool-size', '1', '--limit', '1', '--sleep-delay', '0.05'] } 60 | 61 | it "accepts pool-size, limit and sleep-delay" do 62 | args = single_test_args + [fuzz_nothing] 63 | -> { FuzzBert::AutoRun.autorun_with_args(args) }.should_not raise_error 64 | end 65 | 66 | it "accepts console" do 67 | args = single_test_args + ['--console', fuzz_nothing] 68 | -> { FuzzBert::AutoRun.autorun_with_args(args) }.should_not raise_error 69 | end 70 | 71 | it "accepts bug-dir" do 72 | args = single_test_args + ['--bug-dir', '.', fuzz_nothing] 73 | -> { FuzzBert::AutoRun.autorun_with_args(args) }.should_not raise_error 74 | end 75 | 76 | it "accepts a custom handler" do 77 | args = single_test_args + ['--handler', 'FuzzBert::Spec::CustomHandler', relative_path('fuzz/fuzz_custom_handler.rb')] 78 | -> { FuzzBert::AutoRun.autorun_with_args(args) }.should_not raise_error 79 | FuzzBert::Spec::CustomHandler.called.should be_true 80 | end 81 | 82 | it "accepts multiple single files" do 83 | args = [ 84 | '--pool-size', '1', 85 | '--limit', '2', 86 | '--sleep-delay', '0.05', 87 | '--handler', 'FuzzBert::Spec::CustomHandler', 88 | fuzz_nothing, relative_path('fuzz/fuzz_custom_handler.rb') 89 | ] 90 | -> { FuzzBert::AutoRun.autorun_with_args(args) }.should_not raise_error 91 | FuzzBert::Spec::CustomHandler.called.should be_true 92 | end 93 | 94 | it "accepts a pattern" do 95 | args = [ 96 | '--pool-size', '1', 97 | '--limit', '2', 98 | '--sleep-delay', '0.05', 99 | '--handler', 'FuzzBert::Spec::CustomHandler', 100 | relative_path('fuzz/**/fuzz_*.rb') 101 | ] 102 | -> { FuzzBert::AutoRun.autorun_with_args(args) }.should_not raise_error 103 | FuzzBert::Spec::CustomHandler.called.should be_true 104 | end 105 | end 106 | 107 | end 108 | 109 | -------------------------------------------------------------------------------- /lib/fuzzbert/template.rb: -------------------------------------------------------------------------------- 1 | require 'stringio' 2 | 3 | class FuzzBert::Template 4 | include FuzzBert::Generation 5 | 6 | def initialize(template) 7 | @template = Parser.new(template).parse 8 | @callbacks = {} 9 | end 10 | 11 | def set(name, cb=nil, &blk) 12 | @callbacks[name] = cb || blk 13 | end 14 | 15 | def to_data 16 | "".tap do |buf| 17 | @template.each { |t| buf << t.to_data(@callbacks).to_s } 18 | end 19 | end 20 | 21 | private 22 | 23 | class Parser 24 | 25 | def initialize(template) 26 | @io = StringIO.new(template) 27 | @template = [] 28 | end 29 | 30 | def parse 31 | @state = determine_state 32 | while token = parse_token 33 | @template << token 34 | end 35 | @template 36 | end 37 | 38 | def parse_token 39 | case @state 40 | when :TEXT 41 | parse_text 42 | when :IDENTIFIER 43 | parse_identifier 44 | else 45 | nil 46 | end 47 | end 48 | 49 | def determine_state 50 | begin 51 | @buf = @io.readchar 52 | 53 | case @buf 54 | when '$' 55 | c = @io.readchar 56 | if c == "{" 57 | @buf = "" 58 | :IDENTIFIER 59 | else 60 | @buf << c 61 | :TEXT 62 | end 63 | when '\\' 64 | @buf = "" 65 | :TEXT 66 | else 67 | :TEXT 68 | end 69 | rescue EOFError 70 | :EOF 71 | end 72 | end 73 | 74 | def parse_identifier 75 | name = "" 76 | begin 77 | until (c = @io.readchar) == '}' 78 | name << c 79 | end 80 | 81 | if name.empty? 82 | raise RuntimeError.new("No identifier name given") 83 | end 84 | 85 | @state = determine_state 86 | Identifier.new(name) 87 | rescue EOFError 88 | raise RuntimeError.new("Unclosed identifier") 89 | end 90 | end 91 | 92 | def parse_text 93 | text = @buf 94 | begin 95 | loop do 96 | until (c = @io.readchar) == '$' 97 | if c == '\\' 98 | text << parse_escape 99 | else 100 | text << c 101 | end 102 | end 103 | 104 | d = @io.readchar 105 | if d == "{" 106 | @state = :IDENTIFIER 107 | return Text.new(text) 108 | else 109 | text << c 110 | text << d 111 | end 112 | end 113 | rescue EOFError 114 | @state = :EOF 115 | Text.new(text) 116 | end 117 | end 118 | 119 | def parse_escape 120 | begin 121 | @io.readchar 122 | rescue EOFError 123 | '\\' 124 | end 125 | end 126 | 127 | end 128 | 129 | class Text 130 | 131 | def initialize(text) 132 | @text = text 133 | end 134 | 135 | def to_data(callbacks) 136 | @text 137 | end 138 | 139 | end 140 | 141 | class Identifier 142 | 143 | def initialize(name) 144 | @name = name.to_sym 145 | end 146 | 147 | def to_data(callbacks) 148 | cb = callbacks[@name] 149 | raise RuntimeError.new "No callback set for :#{@name}" unless cb 150 | cb.call 151 | end 152 | 153 | end 154 | 155 | end 156 | 157 | -------------------------------------------------------------------------------- /spec/dsl_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | describe FuzzBert::TestSuite do 5 | describe "fuzz" do 6 | context "with one generator" do 7 | let(:suite) do 8 | FuzzBert::TestSuite.create "test" do 9 | deploy { |data| data } 10 | 11 | data("1") { FuzzBert::Generators.random } 12 | end 13 | end 14 | 15 | it "returns an instance of TestSuite" do 16 | suite.should be_an_instance_of(FuzzBert::TestSuite) 17 | suite.description.should == "test" 18 | end 19 | 20 | it "defines a test that executes the block in deploy" do 21 | suite.test.should_not be_nil 22 | suite.test.should be_an_instance_of(FuzzBert::Test) 23 | ["a", 10, true, Object.new].each { |o| suite.test.run(o).should == o } 24 | end 25 | 26 | it "defines one generator" do 27 | suite.generators.size.should == 1 28 | gen = suite.generators.first 29 | gen.should be_an_instance_of(FuzzBert::Generator) 30 | gen.description.should == "1" 31 | gen.to_data.should be_an_instance_of(String) 32 | end 33 | 34 | end 35 | 36 | context "with two generators" do 37 | let(:suite) do 38 | FuzzBert::TestSuite.create "test" do 39 | deploy { |data| data } 40 | 41 | data("1") { FuzzBert::Generators.fixed(1) } 42 | data("2") { FuzzBert::Generators.fixed(2) } 43 | end 44 | end 45 | 46 | it "defines two generators" do 47 | suite.generators.size.should == 2 48 | gen = suite.generators.first 49 | gen.should be_an_instance_of(FuzzBert::Generator) 50 | gen.description.should == "1" 51 | gen.to_data.should == 1 52 | 53 | gen = suite.generators[1] 54 | gen.should be_an_instance_of(FuzzBert::Generator) 55 | gen.description.should == "2" 56 | gen.to_data.should == 2 57 | end 58 | 59 | end 60 | 61 | context "with complex data" do 62 | let(:suite) do 63 | FuzzBert::TestSuite.create "test" do 64 | deploy { |data| data } 65 | 66 | data "1" do 67 | c = FuzzBert::Container.new 68 | c << FuzzBert::Generators.fixed("1") 69 | c << FuzzBert::Generators.fixed("2") 70 | c << FuzzBert::Generators.fixed("3") 71 | c.generator 72 | end 73 | 74 | end 75 | end 76 | 77 | it "applies the container generators in sequence" do 78 | suite.generators.size.should == 1 79 | gen = suite.generators.first 80 | gen.should be_an_instance_of(FuzzBert::Generator) 81 | gen.description.should == "1" 82 | gen.to_data.should == "123" 83 | end 84 | end 85 | end 86 | 87 | it "raises an error when no block is given" do 88 | -> { FuzzBert::TestSuite.create "test" }.should raise_error 89 | end 90 | 91 | it "raises an error when the deploy block is missing" do 92 | lambda do 93 | FuzzBert::TestSuite.create "test" do 94 | data("1") { FuzzBert::Generators.random } 95 | end.should raise_error 96 | end 97 | end 98 | 99 | it "raises an error when the data blocks are missing" do 100 | lambda do 101 | FuzzBert::TestSuite.create "test" do 102 | deploy { |data| data } 103 | end.should raise_error 104 | end 105 | end 106 | 107 | describe "deploy" do 108 | it "raises an error when no block is given" do 109 | lambda do 110 | FuzzBert::TestSuite.create "test" do 111 | deploy 112 | 113 | data("1") { FuzzBert::Generators.random } 114 | end 115 | end.should raise_error 116 | end 117 | end 118 | 119 | describe "data" do 120 | it "raises an error when no block is given" do 121 | lambda do 122 | FuzzBert::TestSuite.create "test" do 123 | deploy { |data| data } 124 | data("1") 125 | end 126 | end.should raise_error 127 | end 128 | end 129 | 130 | end 131 | 132 | -------------------------------------------------------------------------------- /lib/fuzzbert/autorun.rb: -------------------------------------------------------------------------------- 1 | require 'optparse' 2 | 3 | module FuzzBert::AutoRun 4 | 5 | TEST_CASES = [] 6 | 7 | module_function 8 | 9 | def register(suite) 10 | TEST_CASES << suite 11 | end 12 | 13 | def autorun 14 | autorun_with_args(ARGV) 15 | end 16 | 17 | def autorun_with_args(argv) 18 | options, files = process_args(argv) 19 | load_files(files) 20 | run(options) 21 | end 22 | 23 | def run(options=nil) 24 | raise RuntimeError.new "No test cases were found" if TEST_CASES.empty? 25 | FuzzBert::Executor.new(TEST_CASES, options).run 26 | end 27 | 28 | private; module_function 29 | 30 | def load_files(files) 31 | files.each do |pattern| 32 | Dir.glob(pattern).each { |f| load File.expand_path(f) } 33 | end 34 | end 35 | 36 | def process_args(args = []) 37 | options = {} 38 | 39 | OptionParser.new do |opts| 40 | add_banner(opts) 41 | 42 | add_help(opts) 43 | add_pool_size(opts, options) 44 | add_limit(opts, options) 45 | add_console(opts, options) 46 | add_sleep_delay(opts, options) 47 | add_handler(opts, options) 48 | add_bug_dir(opts, options) 49 | 50 | opts.parse! args 51 | end 52 | 53 | raise ArgumentError.new("No file pattern was given") if args.empty? 54 | [options, args] 55 | end 56 | 57 | def add_banner(opts) 58 | opts.banner = 'Usage: fuzzbert [OPTIONS] PATTERN [PATTERNS]' 59 | opts.separator <<-EOS 60 | 61 | Run your random tests by pointing fuzzbert to a single or many explicit files 62 | or by providing a pattern. The default pattern is 'fuzz/**/fuzz_*.rb, assuming 63 | that your FuzzBert files (files beginning with 'fuzz_') live in a directory 64 | named fuzz located under the current directory that you are in. 65 | 66 | By default, fuzzbert will run the tests you specify forever, be sure to hit 67 | CTRL+C when you are done or specify a limit with '--limit'. 68 | 69 | EOS 70 | 71 | opts.version = FuzzBert::VERSION 72 | end 73 | 74 | def add_help(opts) 75 | opts.on '-h', '--help', 'Run ' do 76 | puts opts 77 | exit 78 | end 79 | end 80 | 81 | def add_pool_size(opts, options) 82 | opts.on '--pool-size SIZE', Integer, "Sets the number of concurrently running processes to SIZE. Default is 4." do |n| 83 | options[:pool_size] = n.to_i 84 | end 85 | end 86 | 87 | def add_limit(opts, options) 88 | opts.on '--limit LIMIT', Integer, "Instead of running permanently, fuzzing will be stopped after running LIMIT of instances." do |n| 89 | options[:limit] = n.to_i 90 | end 91 | end 92 | 93 | def add_console(opts, options) 94 | opts.on '--console', "Output the failing cases including data on the console instead of saving them in a file." do 95 | options[:handler] = FuzzBert::Handler::Console.new 96 | end 97 | end 98 | 99 | def add_sleep_delay(opts, options) 100 | opts.on '--sleep-delay SECONDS', Float, "Specify the number of SECONDS that the main process sleeps before checking that the limit has been reached. Default is 1." do |f| 101 | options[:sleep_delay] = f.to_f 102 | end 103 | end 104 | 105 | def add_handler(opts, options) 106 | opts.on '--handler CLASS', String, "Specify the full path to a CLASS that will serve as your Handler." do |path| 107 | #lazy initialization: the Handler must be defined in one of the fuzzer files 108 | options[:handler] = Class.new do 109 | @@path = path 110 | 111 | def handle(error_data) 112 | @inner ||= class_for_name(@@path).new 113 | @inner.handle(error_data) 114 | end 115 | 116 | def class_for_name(path) 117 | path.split('::').inject(Object) { |mod, class_name| mod.const_get(class_name) } 118 | end 119 | end.new 120 | end 121 | end 122 | 123 | def add_bug_dir(opts, options) 124 | opts.on '--bug-dir DIRECTORY', String, "The DIRECTORY where the resulting bug files will be stored. Default is the current directory." do |dir| 125 | raise ArgumentError.new "#{dir} is not a directory" unless Dir.exists?(dir) 126 | options[:handler] = FuzzBert::Handler::FileOutput.new(dir) 127 | end 128 | end 129 | 130 | end 131 | 132 | -------------------------------------------------------------------------------- /spec/executor_spec.rb: -------------------------------------------------------------------------------- 1 | require 'rspec' 2 | require 'fuzzbert' 3 | 4 | describe FuzzBert::Executor do 5 | 6 | describe "::new" do 7 | let(:test) do 8 | #NOTE: Do we need the test variable here? 9 | test = FuzzBert::Test.new(lambda { |data| data }) 10 | FuzzBert::TestSuite.create("suite") do 11 | deploy { |data| data } 12 | data("1") { FuzzBert::Generators.random } 13 | end 14 | end 15 | 16 | it "takes a mandatory (array of) TestSuite as first argument" do 17 | -> { FuzzBert::Executor.new }.should raise_error ArgumentError 18 | FuzzBert::Executor.new(test).should be_an_instance_of(FuzzBert::Executor) 19 | FuzzBert::Executor.new([test]).should be_an_instance_of(FuzzBert::Executor) 20 | end 21 | 22 | it "raises an ArgumentError if the TestSuite argument is nil" do 23 | -> { FuzzBert::Executor.new(nil) }.should raise_error ArgumentError 24 | end 25 | 26 | it "raises an ArgumentError if the TestSuite argument is empty" do 27 | -> { FuzzBert::Executor.new([]) }.should raise_error ArgumentError 28 | end 29 | 30 | it "allows a pool_size argument" do 31 | size = 1 32 | executor = FuzzBert::Executor.new(test, pool_size: size) 33 | executor.pool_size.should == size 34 | end 35 | 36 | it "allows a limit argument" do 37 | limit = 42 38 | executor = FuzzBert::Executor.new(test, limit: limit) 39 | executor.limit.should == limit 40 | end 41 | 42 | it "allows a handler argument" do 43 | handler = FuzzBert::Handler::Console.new 44 | executor = FuzzBert::Executor.new(test, handler: handler) 45 | executor.handler.should == handler 46 | end 47 | 48 | it "allows a sleep_delay argument" do 49 | delay = 0.1 50 | executor = FuzzBert::Executor.new(test, sleep_delay: delay) 51 | executor.sleep_delay.should == delay 52 | end 53 | 54 | it "defaults pool_size to 4" do 55 | FuzzBert::Executor.new(test).pool_size.should == 4 56 | end 57 | 58 | it "defaults limit to -1" do 59 | FuzzBert::Executor.new(test).limit.should == -1 60 | end 61 | 62 | it "defaults handler to a FileOutputHandler" do 63 | FuzzBert::Executor.new(test).handler.should be_an_instance_of(FuzzBert::Handler::FileOutput) 64 | end 65 | 66 | it "defaults sleep_delay to 1" do 67 | FuzzBert::Executor.new(test).sleep_delay.should == 1 68 | end 69 | end 70 | 71 | describe "#run" do 72 | subject { FuzzBert::Executor.new(suite, pool_size: 1, limit: 1, handler: handler, sleep_delay: 0.05).run } 73 | 74 | class TestHandler 75 | def initialize(&blk) 76 | @handler = blk 77 | end 78 | 79 | def handle(error_data) 80 | @handler.call(error_data) 81 | end 82 | end 83 | 84 | context "doesn't complain when test succeeds" do 85 | let (:suite) do 86 | FuzzBert::TestSuite.create("suite") do 87 | deploy { |data| data } 88 | data("1") { -> { "a" } } 89 | end 90 | end 91 | let (:handler) { TestHandler.new { |_| raise RuntimeError.new } } 92 | it { -> { subject }.should_not raise_error } 93 | end 94 | 95 | context "reports an unrescued exception" do 96 | called = false 97 | let (:suite) do 98 | FuzzBert::TestSuite.create("suite") do 99 | deploy { |_| raise "boo!" } 100 | data("1") { -> { "a" } } 101 | end 102 | end 103 | let (:handler) { TestHandler.new { |_| called = true } } 104 | it { -> { subject }.should_not raise_error; called.should be_true } 105 | end 106 | 107 | context "allows rescued exceptions" do 108 | let (:suite) do 109 | FuzzBert::TestSuite.create("suite") do 110 | deploy { |_| begin; raise "boo!"; rescue RuntimeError; end } 111 | data("1") { -> { "a" } } 112 | end 113 | end 114 | let (:handler) { TestHandler.new { |_| raise RuntimeError.new } } 115 | it { -> { subject }.should_not raise_error } 116 | end 117 | 118 | context "can handle SEGV" do 119 | called = false 120 | let (:suite) do 121 | FuzzBert::TestSuite.create("suite") do 122 | deploy { |_| Process.kill(:SEGV, Process.pid) } 123 | data("1") { -> { "a" } } 124 | end 125 | end 126 | let (:handler) { TestHandler.new { |_| called = true } } 127 | let (:generator) { FuzzBert::Generator.new("test") { "a" } } 128 | it { -> { subject }.should_not raise_error; called.should be_true } 129 | end if false #don't want to SEGV every time 130 | end 131 | 132 | end 133 | -------------------------------------------------------------------------------- /lib/fuzzbert/executor.rb: -------------------------------------------------------------------------------- 1 | class FuzzBert::Executor 2 | 3 | attr_reader :pool_size, :limit, :handler, :sleep_delay 4 | 5 | DEFAULT_POOL_SIZE = 4 6 | DEFAULT_LIMIT = -1 7 | DEFAULT_HANDLER = FuzzBert::Handler::FileOutput 8 | DEFAULT_SLEEP_DELAY = 1 9 | 10 | DEFAULT_ARGS = { 11 | pool_size: DEFAULT_POOL_SIZE, 12 | limit: DEFAULT_LIMIT, 13 | handler: DEFAULT_HANDLER.new, 14 | sleep_delay: DEFAULT_SLEEP_DELAY 15 | } 16 | 17 | def initialize(suites, args = DEFAULT_ARGS) 18 | raise ArgumentError.new("No test cases were passed") unless suites 19 | 20 | args ||= DEFAULT_ARGS 21 | @pool_size = args[:pool_size] || DEFAULT_POOL_SIZE 22 | @limit = args[:limit] || DEFAULT_LIMIT 23 | @handler = args[:handler] || DEFAULT_HANDLER.new 24 | @sleep_delay = args[:sleep_delay] || DEFAULT_SLEEP_DELAY 25 | @data_cache = {} 26 | @n = 0 27 | @exiting = false 28 | @producer = DataProducer.new(suites) 29 | end 30 | 31 | def run 32 | trap_child_exit 33 | trap_interrupt 34 | 35 | @pool_size.times { run_instance(*@producer.next) } 36 | @running = true 37 | @limit == -1 ? sleep : conditional_sleep 38 | end 39 | 40 | private 41 | 42 | def run_instance(description, test, generator) 43 | data = generator.to_data 44 | pid = fork do 45 | begin 46 | test.run(data) 47 | rescue StandardError 48 | abort 49 | end 50 | end 51 | id = "#{description}/#{generator.description}" 52 | @data_cache[pid] = [id, data] 53 | end 54 | 55 | def trap_child_exit 56 | trap(:CHLD) do 57 | while_child_exits do |exitval| 58 | pid = exitval[0] 59 | status = exitval[1] 60 | data_ary = @data_cache.delete(pid) 61 | 62 | handle({ 63 | id: data_ary[0], 64 | data: data_ary[1], 65 | pid: pid, 66 | status: status 67 | }) if status_failed?(status) 68 | 69 | start_new_child 70 | end 71 | end 72 | end 73 | 74 | def while_child_exits 75 | while exitval = Process.wait2(-1, Process::WNOHANG) 76 | yield exitval 77 | end 78 | rescue Errno::ECHILD 79 | # fine 80 | end 81 | 82 | def status_failed?(status) 83 | !status.success? && !interrupted(status) 84 | end 85 | 86 | def start_new_child 87 | @n += 1 88 | if limit_reached? 89 | run_instance(*@producer.next) 90 | else 91 | @running = false 92 | end 93 | end 94 | 95 | def limit_reached? 96 | @limit == -1 || @n < @limit 97 | end 98 | 99 | def trap_interrupt 100 | trap(:INT) do 101 | exit! (1) if @exiting 102 | @exiting = true 103 | graceful_exit 104 | end 105 | end 106 | 107 | def graceful_exit 108 | puts "\nExiting...Interrupt again to exit immediately" 109 | begin 110 | while Process.wait; end 111 | rescue Errno::ECHILD 112 | end 113 | exit 0 114 | end 115 | 116 | def handle(error_data) 117 | @handler.handle(error_data) 118 | end 119 | 120 | def interrupted(status) 121 | return false if status.exited? 122 | status.termsig == nil || status.termsig == 2 123 | end 124 | 125 | def conditional_sleep 126 | sleep @sleep_delay until !@running 127 | end 128 | 129 | class DataProducer 130 | def initialize(suites) 131 | @ring = Ring.new(suites) 132 | update 133 | end 134 | 135 | def update 136 | @suite = @ring.next 137 | @gen_iter = ProcessSafeEnumerator.new(@suite.generators) 138 | end 139 | 140 | def next 141 | gen = nil 142 | until gen 143 | begin 144 | gen = @gen_iter.next 145 | rescue StopIteration 146 | update 147 | end 148 | end 149 | [@suite.description, @suite.test, gen] 150 | end 151 | 152 | class Ring 153 | def initialize(objs) 154 | @i = 0 155 | objs = [objs] unless objs.respond_to?(:each) 156 | @objs = objs.to_a 157 | raise ArgumentError.new("No test cases found") if @objs.empty? 158 | end 159 | 160 | def next 161 | obj = @objs[@i] 162 | @i = (@i + 1) % @objs.size 163 | obj 164 | end 165 | end 166 | 167 | #needed because the Fiber used for normal Enumerators has race conditions 168 | class ProcessSafeEnumerator 169 | def initialize(ary) 170 | @i = 0 171 | @ary = ary.to_a 172 | end 173 | 174 | def next 175 | obj = @ary[@i] 176 | raise StopIteration unless obj 177 | @i += 1 178 | obj 179 | end 180 | end 181 | end 182 | 183 | end 184 | 185 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # FuzzBert [![Build Status](https://secure.travis-ci.org/krypt/FuzzBert.png?branch=master)](http://travis-ci.org/krypt/FuzzBert) [![Code Climate](https://codeclimate.com/badge.png)](https://codeclimate.com/github/krypt/FuzzBert) 2 | 3 | A random testing/fuzzer framework for Ruby. 4 | 5 | Random Testing (or "fuzzing") is not really new, it has been around for quite 6 | some time. Yet it still hasn't found widespread adoption in everyday coding 7 | practices, much too often it is only used for the purpose of finding exploits 8 | for existing applications or libraries. FuzzBert wants to improve this situation. 9 | It's a simple fuzzing framework with an RSpec-like DSL that will allow you to 10 | integrate random tests in your project with minimal effort. 11 | 12 | For further information on random testing, here are two excellent starting points: 13 | 14 | * [Udacity CS258](https://www.udacity.com/course/software-testing--cs258) 15 | * [Babysitting an Army of Monkeys](http://fuzzinginfo.files.wordpress.com/2012/05/cmiller-csw-2010.pdf) 16 | 17 | ## Installation 18 | 19 | gem install fuzzbert 20 | fuzzbert --help 21 | 22 | ## Defining a random test 23 | 24 | FuzzBert defines an RSpec-like DSL that can be used to define different fuzzing 25 | scenarios. The DSL uses three words: `fuzz`, `deploy` and `data`. 26 | 27 | Here is a quick example that fuzzes `JSON.parse`: 28 | 29 | ```ruby 30 | require 'json' 31 | require 'fuzzbert' 32 | 33 | fuzz "JSON.parse" do 34 | 35 | deploy do |data| 36 | begin 37 | JSON.parse data 38 | rescue StandardError 39 | #fine, we just want to capture crashes 40 | end 41 | end 42 | 43 | data "completely random" do 44 | FuzzBert::Generators.random 45 | end 46 | 47 | data "enclosing curly braces" do 48 | c = FuzzBert::Container.new 49 | c << FuzzBert::Generators.fixed("{") 50 | c << FuzzBert::Generators.random 51 | c << FuzzBert::Generators.fixed("}") 52 | c.generator 53 | end 54 | 55 | data "my custom generator" do 56 | prng = Random.new 57 | lambda do 58 | buf = '{ user: { ' 59 | buf << prng.bytes(100) 60 | buf << ' } }' 61 | end 62 | end 63 | 64 | end 65 | ``` 66 | 67 | `fuzz` can be thought of as defining a new scenario, such as "fuzz this command 68 | line tool", "fuzz this particular URL of my web app" or "fuzz this library method 69 | taking external input". 70 | 71 | Within a `fuzz` block, there must be one occurrences of `deploy` and one or several 72 | occurrences of `data`. The `deploy` block is the spot where we deliver the random 73 | payload that has been generated. It is agnostic about the actual target in order to 74 | leave you free to fuzz whatever you require in your particular case. The `data` 75 | blocks define the shape of the random data being generated. There can be more than 76 | one such block because it is often beneficial to not only shoot completely random 77 | data at the target - you often want to deliver more structured data as well, trying 78 | to find the edge cases deeper within your code. Good random test suites make use 79 | of both - totally random data as well as structured data - in order to cover as 80 | much "code surface" as possible. 81 | 82 | The `deploy` block takes the generated data as a parameter. The block itself is 83 | responsible of deploying the payload. An execution is considered successful if 84 | the `deploy` block passes with no uncaught error being raised. If an error slips 85 | through or if the Ruby process crashes altogether, the execution is of course 86 | considered as a failure. 87 | 88 | `data` blocks must return a lambda or proc that takes no argument. You can either 89 | choose completely custom lambdas of your own or use those predefined for you in 90 | `FuzzBert::Generators`. 91 | 92 | ## Running a random test 93 | 94 | Once the FuzzBert files are set up, you may run your tests similar to how you 95 | would run unit tests: 96 | 97 | fuzzbert "fuzz/**/fuzz_*.rb" 98 | 99 | If your FuzzBert files are already in a directory named 'fuzz' and each of them 100 | begins with 'fuzz_', you may omit the pattern altogether. 101 | 102 | Each `fuzz` block defines a `TestSuite`. These are executed in a round-robin manner. 103 | Each individual `TestSuite` will then apply the `deploy` block with a sample of 104 | data generated successively by each one of the `data` blocks. Once all `data` blocks 105 | are used up, the next `TestSuite` will be executed etc. By default, a FuzzBert 106 | fuzzing session runs forever, until the process is either killed or by manually hitting 107 | `CTRL+C` for example. This was a deliberate design choice since random testing suites 108 | need to be run for quite some time to be effective. It's something you want to run over 109 | the weekend rather than for a couple of minutes. Still, it can make sense to explicitly 110 | limit the number of runs, for example when integrating FuzzBert with a CI server or 111 | with Travis. You can do so by passing the `--limit` parameter to the `fuzzbert` 112 | executable: 113 | 114 | fuzzbert --limit 1000000 "fuzz/**/fuzz_*.rb" 115 | 116 | Every single execution of `deploy` is run in a separate process. The main reason for 117 | this is that we typically want to detect hard crashes when a C extension or even Ruby 118 | itself encounters an input it can't handle. Besides being able to cope with these cases, 119 | running in separate processes proves beneficial otherwise as well: by default, FuzzBert 120 | runs the tests in four separate processes at once, therefore utilizing your CPU's cores 121 | effectively. You can tweak that setting with `--pool-size` to set this number to 1 122 | (for completely sequential runs) or to the exact number of cores your CPU offers. 123 | 124 | fuzzbert --pool-size 1 my/fuzzbert/file 125 | 126 | ## What happens if we encounter a bug? 127 | 128 | If a test should end up failing (either the process crashed completely or caused an 129 | uncaught error), FuzzBert will output the failing test on your terminal and tell you 130 | where it stored the data that caused this. This conveniently allows you to run FuzzBert 131 | over the weekend and when you return on Monday, the troubleshooters will sit there all 132 | lined up for you to go through and filter. By using the `--console` command line switch 133 | you can tell FuzzBert to not explicitly store the data, but echoing the data that 134 | caused the crash to the terminal instead. 135 | 136 | fuzzbert --console "fuzz/**/fuzz_*.rb" 137 | 138 | If you don't want to litter your current working directory with the files generated 139 | by FuzzBert, you can also specify a specific path to where they should be saved 140 | instead: 141 | 142 | fuzzbert --bug-dir bugs "fuzz/**/fuzz_*.rb" 143 | 144 | This is still not quite what you want to happen in case a test crashes? There's 145 | also the possibility to define a handler of your own: 146 | 147 | ```ruby 148 | require 'fuzzbert' 149 | 150 | class MyHandler 151 | def handle(error_data) 152 | #create an issue in the bug tracker 153 | puts error_data[:id] 154 | p error_data[:data] 155 | puts error_data[:pid] 156 | puts error_data[:status] 157 | end 158 | end 159 | 160 | fuzz "Define here as usual" do 161 | ... 162 | end 163 | ``` 164 | 165 | Now you just need to tell FuzzBert to use your custom handler: 166 | 167 | fuzzbert --handler MyHandler my/fuzzbert/file 168 | 169 | ## Templates 170 | 171 | Using the approach described so far is most useful for binary protocols, but as 172 | soon as you work with mainly String-based data this can quickly become a chore. 173 | What you actually want in these situations is some sort of template mechanism 174 | that comes with mostly fixed data and only replaces a few selected parts with 175 | randomly generated data. This, too, is possible with FuzzBert, it comes with a 176 | minimal templating language: 177 | 178 | ```ruby 179 | require 'fuzzbert' 180 | 181 | fuzz "My Web App" do 182 | 183 | deploy do |data| 184 | # Send the data to your web app with httpclient or similar. 185 | # You define the "error conditions": if a response to some 186 | # data is not as expected, you could simply raise an error 187 | # here. 188 | end 189 | 190 | data "JSON generated from a template" do 191 | t = FuzzBert::Template.new '{ user: { id: ${id}, name: "${name}" } }' 192 | t.set(:id, FuzzBert::Generators.cycle(1..10000)) 193 | t.set(:name) { "Fixed text plus two random bytes: #{FuzzBert::Generators.random_fixlen(2).call}" } 194 | t.generator 195 | end 196 | 197 | end 198 | ``` 199 | 200 | Simply specify your template variables using `${..}` and assign a callback for 201 | them via `set`. Of course you may escape the dollar sign with a backslash as 202 | usual. The Template#set method takes as its first argument the symbol 203 | representing the template variable. To specify the data to be generated, it 204 | either takes a second argument in form of a proc or lambda (to use the 205 | built-in generators for example, as in `:id` in the example) or it takes a 206 | block that allows to define the data ad-hoc (as in `:name`). 207 | 208 | ## Mutators 209 | 210 | Mutation is the principle used in "Babysitting an Army of Monkeys". The basis for 211 | the mutation tests is a valid sample of input that is then modified in exactly one 212 | position in each test instance. You can apply this principle as follows: 213 | 214 | ```ruby 215 | require 'fuzzbert' 216 | 217 | fuzz "Web App" do 218 | deploy do |data| 219 | #send JSON data via HTTP 220 | end 221 | 222 | data "mutated data" do 223 | m = FuzzBert::Mutator.new '{ user: { id: 42, name: "FuzzBert" }' 224 | m.generator 225 | end 226 | 227 | end 228 | ``` 229 | 230 | This will take the original JSON data and modify one byte each time data is being 231 | generated. 232 | 233 | ## Rake integration 234 | 235 | You may integrate Rake tasks for FuzzBert similar to how you would include a task for 236 | Rspec: 237 | 238 | ```ruby 239 | require 'rake' 240 | require 'fuzzbert/rake_task' 241 | 242 | FuzzBert::RakeTask.new(:fuzz) do |spec| 243 | spec.fuzzbert_opts = ['--limit 10000000', '--console'] 244 | spec.pattern = 'fuzz/**/fuzz_*.rb' 245 | end 246 | ``` 247 | 248 | ## Supported versions 249 | 250 | FuzzBert has been confirmed to run on CRuby 1.9.3 and Rubinius 2.0.0dev. Since 251 | it heavily relies on forking, it does not run on JRuby so far, but support is planned and 252 | on its way. 253 | 254 | You may also use FuzzBert for fuzzing arbitrary applications or libraries that aren't 255 | connected to Ruby at all - have a look in the examples that ship with FuzzBert. 256 | 257 | ## License 258 | 259 | Copyright (c) 2012-2013 Martin Boßlet. Distributed under the MIT License. See LICENSE for 260 | further details. 261 | 262 | --------------------------------------------------------------------------------