├── Gemfile ├── hipchat.yml.sample ├── features ├── support │ └── base.rb ├── step_definitions │ ├── cucumber_steps.rb │ └── bilgerat_steps.rb ├── error_handling.feature └── outlines │ └── main.feature ├── bilgerat.gemspec ├── LICENSE ├── Gemfile.lock ├── README.md └── lib └── bilgerat.rb /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | gemspec 3 | -------------------------------------------------------------------------------- /hipchat.yml.sample: -------------------------------------------------------------------------------- 1 | default: 2 | user: 'Bilge Rat #{TEST_ENV_NUMBER}' 3 | auth_token: 'goes here' 4 | room: 'test room' 5 | error_color: 'red' 6 | -------------------------------------------------------------------------------- /features/support/base.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'aruba/cucumber' 4 | 5 | After do 6 | FileUtils.rm_rf @current_dir if @current_dir && File.directory?(@current_dir) 7 | end 8 | -------------------------------------------------------------------------------- /features/step_definitions/cucumber_steps.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | # Copied from the cucumber project 4 | 5 | Given /^I am in (.*)$/ do |example_dir_relative_path| 6 | @current_dir = fixtures_dir(example_dir_relative_path) 7 | end 8 | 9 | Given /^a standard Cucumber project directory structure$/ do 10 | @current_dir = `mktemp -d cuc.XXXXXX`.strip 11 | #puts "created cuc dir #{@current_dir}" 12 | in_current_dir do 13 | FileUtils.rm_rf 'features' if File.directory?('features') 14 | FileUtils.mkdir_p 'features/support' 15 | FileUtils.mkdir 'features/step_definitions' 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /features/error_handling.feature: -------------------------------------------------------------------------------- 1 | Feature: Error handling 2 | 3 | @announce-stderr 4 | Scenario: Undefined steps work 5 | And a file named "features/scenario_with_unmatched_step_def.feature" with: 6 | """ 7 | Feature: Outline 8 | 9 | @tagtag 10 | Scenario: blah blah 11 | Given this passes 12 | """ 13 | And a file named "features/step_definitions/steps.rb" with: 14 | """ 15 | Given /^this passes$/ do 16 | end 17 | Given /^this passes$/ do 18 | end 19 | """ 20 | When I run bilgerat with: `cucumber --format Bilgerat --out na --format pretty` 21 | Then there should be a hipchat post matching /.*Ambiguous match of "this passes".*/ -------------------------------------------------------------------------------- /bilgerat.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |s| 2 | s.name = "bilgerat" 3 | s.version = '0.2.1' 4 | s.platform = Gem::Platform::RUBY 5 | s.required_ruby_version = '>= 1.9.3' 6 | s.authors = ["Joseph Shraibman"] 7 | s.email = ["jshraibman@mdsol.com"] 8 | s.homepage = "https://github.com/mdsol/bilgerat" 9 | s.summary = "Cucumber output formatter that sends failure messages to Hipchat" 10 | 11 | s.add_dependency "cucumber", ">= 1.0.0" 12 | s.add_dependency 'hipchat', '~> 0.7.0' 13 | 14 | s.add_development_dependency 'aruba', '~> 0.5' 15 | s.add_development_dependency 'debugger' 16 | 17 | s.files = `git ls-files`.split("\n") 18 | s.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 19 | s.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } 20 | end 21 | -------------------------------------------------------------------------------- /features/step_definitions/bilgerat_steps.rb: -------------------------------------------------------------------------------- 1 | def debug_file_name 2 | '/tmp/tempfile.xml' 3 | end 4 | 5 | When /^I run bilgerat with: `(.*)`$/ do |cmd| 6 | step %Q{I run `env DEBUG_BILGERAT=#{debug_file_name} #{cmd}`} 7 | end 8 | 9 | When /^I clear hipchat posts$/ do 10 | File.delete(debug_file_name) if File.exists?(debug_file_name) 11 | end 12 | 13 | Before do 14 | step 'I clear hipchat posts' 15 | end 16 | 17 | # For debugging 18 | Then /^I print the hipchat posts$/ do 19 | puts File.read(debug_file_name) 20 | end 21 | 22 | Then /^there should (not )?be a hipchat post matching \/(.*)\/$/ do |should_not, pattern| 23 | file_text = nil 24 | File.open(debug_file_name, 'r') do |file| 25 | file_text = file.read 26 | end 27 | re = Regexp.new("#{pattern}", Regexp::MULTILINE) 28 | 29 | if should_not 30 | file_text.should_not match(re) 31 | else 32 | file_text.should match(re) 33 | end 34 | end -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Medidata Solutions Worldwide 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /features/outlines/main.feature: -------------------------------------------------------------------------------- 1 | Feature: Scenario outlines 2 | 3 | Background: 4 | Given a standard Cucumber project directory structure 5 | 6 | Scenario: Full information is only printed for the first example 7 | And a file named "features/scenario_with_failing_examples.feature" with: 8 | """ 9 | Feature: Outline 10 | 11 | @tagtag 12 | Scenario Outline: blah blah 13 | Given this 14 | Examples: 15 | | fails or passes | 16 | | passes | 17 | | fails | 18 | | fails | 19 | """ 20 | And a file named "features/step_definitions/steps.rb" with: 21 | """ 22 | Given /^this (fails|passes)$/ do |str| 23 | str.should == 'passes' 24 | end 25 | """ 26 | When I run bilgerat with: `cucumber --format Bilgerat --out na --format pretty` 27 | Then there should be a hipchat post matching /.*@tagtag.*Example #2 failed.*/ 28 | And there should not be a hipchat post matching /@tagtag.*Example #3 failed.*/ 29 | And there should be a hipchat post matching /.*Example #3 failed.*/ 30 | 31 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | bilgerat (0.2.1) 5 | cucumber (>= 1.0.0) 6 | hipchat (~> 0.7.0) 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | aruba (0.5.3) 12 | childprocess (>= 0.3.6) 13 | cucumber (>= 1.1.1) 14 | rspec-expectations (>= 2.7.0) 15 | builder (3.2.2) 16 | childprocess (0.3.9) 17 | ffi (~> 1.0, >= 1.0.11) 18 | columnize (0.3.6) 19 | cucumber (1.3.2) 20 | builder (>= 2.1.2) 21 | diff-lcs (>= 1.1.3) 22 | gherkin (~> 2.12.0) 23 | multi_json (~> 1.3) 24 | debugger (1.2.2) 25 | columnize (>= 0.3.1) 26 | debugger-linecache (~> 1.1.1) 27 | debugger-ruby_core_source (~> 1.1.5) 28 | debugger-linecache (1.1.2) 29 | debugger-ruby_core_source (>= 1.1.1) 30 | debugger-ruby_core_source (1.1.7) 31 | diff-lcs (1.2.4) 32 | ffi (1.9.0) 33 | gherkin (2.12.0) 34 | multi_json (~> 1.3) 35 | hipchat (0.7.0) 36 | httparty 37 | httparty 38 | httparty (0.11.0) 39 | multi_json (~> 1.0) 40 | multi_xml (>= 0.5.2) 41 | multi_json (1.7.7) 42 | multi_xml (0.5.4) 43 | rspec-expectations (2.13.0) 44 | diff-lcs (>= 1.1.3, < 2.0) 45 | 46 | PLATFORMS 47 | ruby 48 | 49 | DEPENDENCIES 50 | aruba (~> 0.5) 51 | bilgerat! 52 | debugger 53 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | bilgerat 2 | ======== 3 | 4 | Bilgerat is a [cucumber](http://cukes.info/) output formatter that sends messages about failing scenarios to [HipChat](https://www.hipchat.com/) rooms. 5 | 6 | 7 | usage 8 | ----- 9 | 10 | In your Gemfile: 11 | 12 | ```ruby 13 | gem 'bilgerat', git: 'git@github.com:mdsol/bilgerat.git' 14 | ``` 15 | 16 | On the command line: 17 | 18 | ``` 19 | cucumber --format Bilgerat --out na --format pretty 20 | ``` 21 | 22 | 23 | configuration 24 | ----- 25 | You must supply a configuration file that contains credentials to use the HipChat API. By default Bilgerat looks for this file is config/hipchat.yml. You can override this location by setting the HIPCHAT_CONFIG_PATH environment variable. 26 | 27 | The configuration file contains settings per context. You should set all configuration items in the default context. You can override these settings for other contexts. Use the BILGERAT_CONTEXT environment variable to choose the context. 28 | 29 | For example our CI server runs cucumber scenarios in parallel for the first round, then reruns failing scenarios in the final round. Our config file looks like this: 30 | 31 | ``` 32 | default: 33 | user: 'Bilge Rat #{TEST_ENV_NUMBER}' 34 | auth_token: 'goes here' 35 | room: 'test room' 36 | error_color: 'red' 37 | first_round: 38 | error_color: 'purple' 39 | final_round: 40 | user: 'Final Bilge Rat' 41 | ``` 42 | 43 | -------------------------------------------------------------------------------- /lib/bilgerat.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | # Based on https://github.com/cucumber/cucumber/blob/master/lib/cucumber/formatter/pretty.rb and the other cucumber 4 | # built in formatters. 5 | # There was no handy api so I had to reverse engineer 6 | 7 | class Bilgerat 8 | 9 | def initialize(step_mother, path_or_io, options) 10 | end 11 | 12 | 13 | def before_background(background) 14 | @background_failed = nil 15 | reset_scenario_info 16 | @in_background = background 17 | end 18 | 19 | def after_background(background) 20 | @in_background = nil 21 | @background_tags = @tags 22 | @tags = nil 23 | end 24 | 25 | def tag_name(tag) 26 | (@tags ||= []) << tag 27 | end 28 | 29 | def scenario_name(keyword, name, file_colon_line, source_indent) 30 | reset_scenario_info 31 | @current_scenario_info = {keyword: keyword, name: name, file_colon_line: file_colon_line, tags: @tags} 32 | @tags = nil 33 | end 34 | 35 | def after_table_row(table_row) 36 | return unless @in_examples and Cucumber::Ast::OutlineTable::ExampleRow === table_row 37 | @example_num += 1 if !@header_row 38 | if table_row.exception 39 | hipchat_exception(table_row.exception) 40 | elsif !@header_row && table_row.failed? 41 | hipchat_exception('') 42 | end 43 | @header_row = false 44 | end 45 | 46 | def before_examples(*args) 47 | @in_examples = true 48 | @header_row = true 49 | end 50 | 51 | def after_examples(*args) 52 | @in_examples = false 53 | end 54 | 55 | def exception(exception, status) 56 | hipchat_exception(exception) 57 | end 58 | 59 | # file_colon_line is new in cucumber 1.2.0. Give it default of nil to be reverse compatible 60 | def step_name(keyword, step_match, status, source_indent, background, file_colon_line=nil) 61 | @current_failed_step_info = nil 62 | #TODO: detect if we are running in strict mode somehow, and if so also send a message when status == :pending 63 | if status == :failed 64 | @current_failed_step_info = {step_match: step_match, file_colon_line: file_colon_line, status: status} 65 | end 66 | end 67 | 68 | private 69 | 70 | # send failure report to hipchat 71 | def hipchat_exception(exception) 72 | # If the background fails only send one message the first time 73 | return if @background_failed 74 | 75 | # If this is a failing scenario output includes: 76 | # 1a) file & line number for scenario 77 | # 1b) all tags, including those declared on the feature 78 | # 1c) scenario name 79 | # 2) failing step 80 | # 3) The exception 81 | 82 | # If this is a failing example, then output includes: 83 | # 1) For the first failing example in an outline the same as above, omitted for the subsequent examples 84 | # 2) "Example #X failed:" 85 | # 3) the exception 86 | 87 | # If this was a failing background step: 88 | # 1a) file & line number for background 89 | # 1b) "Background step failed:" 90 | # 2) failing step 91 | # 3) the exception 92 | 93 | sb = '' 94 | 95 | # part 1 96 | unless @had_failing_example 97 | if @current_scenario_info 98 | sb << "# #{ @current_scenario_info[:file_colon_line] }\n" 99 | all_tags = (@background_tags || []) + (@current_scenario_info[:tags] || []) 100 | sb << all_tags.join(' ') + "\n" if all_tags.size > 0 101 | sb << "#{ @current_scenario_info[:keyword]}: #{ @current_scenario_info[:name]}\n" 102 | elsif @in_background 103 | sb << "# #{ @in_background.file_colon_line }\nBackground step failed:\n" 104 | @background_failed = true 105 | else 106 | sb = 'error: no scenario info' 107 | end 108 | end 109 | 110 | # part2 111 | if @current_failed_step_info # Failing scenario or background, not example 112 | sb << "#{ current_step_match_to_str } # " 113 | fcl = @current_failed_step_info[:file_colon_line] # line in the feature file, may be nil 114 | sb << fcl << ' → ' if fcl 115 | sb << @current_failed_step_info[:step_match].file_colon_line << "\n" 116 | elsif @example_num 117 | @had_failing_example = true 118 | sb << "Example ##{@example_num} failed:\n" 119 | end 120 | 121 | adapter.hip_post( "#{ sb }#{ build_exception_detail(exception) }", color: :error ) 122 | end 123 | 124 | # Convert the step match (saved from step_name(), above into a string for outputting. 125 | def current_step_match_to_str 126 | current_step_match = @current_failed_step_info[:step_match] 127 | # current_step_match might be a StepMatch or a NoStepMatch. If a NoStepMatch we must pass in dummy argument to format_args 128 | args = current_step_match.is_a?(Cucumber::NoStepMatch)? [nil] : [] 129 | current_step_match.format_args(*args) 130 | end 131 | 132 | def adapter 133 | HipchatAdapter 134 | end 135 | 136 | # Based on cucumber code 137 | def build_exception_detail(exception) 138 | return exception if exception.is_a? String 139 | backtrace = Array.new 140 | 141 | message = exception.message 142 | if defined?(RAILS_ROOT) && message.include?('Exception caught') 143 | matches = message.match(/Showing (.+)<\/i>(?:.+) #(\d+)/) 144 | backtrace += ["#{RAILS_ROOT}/#{matches[1]}:#{matches[2]}"] if matches 145 | matches = message.match(/([^(\/)]+)<\//m) 146 | message = matches ? matches[1] : "" 147 | end 148 | 149 | unless exception.instance_of?(RuntimeError) 150 | message = "#{message} (#{exception.class})" 151 | end 152 | 153 | message << "\n" << backtrace.join("\n") 154 | end 155 | 156 | def reset_scenario_info 157 | @current_failed_step_info = @current_scenario_info = @example_num = @had_failing_example = nil 158 | @example_num = 0 159 | end 160 | 161 | end 162 | 163 | # In theory in the future there might be different adapters that can plug in to the output formatter, but for now 164 | # there is just this one. 165 | class HipchatAdapter 166 | 167 | class << self 168 | DEFAULTS = { 169 | :message_format => 'text', 170 | :notify => '1' 171 | }.freeze 172 | 173 | 174 | # Send a message to a HipChat room 175 | # TODO: fork a thread so we don't block tests while we wait for the network. Also on the puts calls because on at 176 | # least one occasion a call blocked and locked up cucumber. 177 | def hip_post(message, options = {}) 178 | if ENV['DEBUG_BILGERAT'] 179 | unless @debug_file 180 | @debug_file = File.open(ENV['DEBUG_BILGERAT'], 'w') 181 | @debug_file.puts "" 182 | at_exit { @debug_file.puts "" } 183 | end 184 | @debug_file.puts "#{ message }" 185 | end 186 | 187 | return unless configured? 188 | 189 | def option(sym) 190 | return options[sym] if options.keys.include?(sym) 191 | DEFAULTS[sym] 192 | end 193 | 194 | # Replace the 'error' color with a real color 195 | options[:color] = error_color if options[:color] == :error 196 | 197 | begin 198 | client[config['room']].send(username, message, DEFAULTS.merge(options)) 199 | #puts "sent msg to hipchat" 200 | rescue => ex 201 | STDERR.puts "Caught #{ex.class}; disabling hipchat notification" 202 | @configured = false 203 | end 204 | end 205 | 206 | # Config hash, from yml file 207 | 208 | private 209 | 210 | def error_color 211 | @error_color ||= config['error_color'] || 'red' 212 | end 213 | 214 | # Returns something that looks like a hash. It returns values from the raw bash by first looking under the 215 | # current context, then under 'default' 216 | def config 217 | config_file = ENV['HIPCHAT_CONFIG_PATH'] || 'config/hipchat.yml' 218 | return nil unless File.exists?(config_file) 219 | 220 | 221 | @config ||= Class.new do 222 | @raw_config_yaml = YAML.load_file(config_file) 223 | 224 | @context = ENV['BILGERAT_CONTEXT'] if ENV['BILGERAT_CONTEXT'] && ENV['BILGERAT_CONTEXT'].length > 0 225 | 226 | def self.[](sym) 227 | sym = sym.to_s 228 | if @context 229 | hash = @raw_config_yaml[@context] 230 | return hash[sym] if hash && hash.keys.include?(sym) 231 | end 232 | @raw_config_yaml['default'][sym] 233 | end 234 | end 235 | end 236 | 237 | # Are we configured to send messages to HipChat? If not just drop messages. 238 | def configured? 239 | if @configured.nil? 240 | @configured = !!(config && config['room'] && config['auth_token']) 241 | else 242 | @configured 243 | end 244 | end 245 | 246 | def client 247 | require 'hipchat' 248 | @client ||= HipChat::Client.new(config['auth_token']) 249 | end 250 | 251 | # The username as we want it to appear in the HipChat room. 252 | def username 253 | @username ||= begin 254 | env_var = config['user'] 255 | case env_var 256 | when nil then 257 | 'Bilge Rat' 258 | when Regexp.compile('#{TEST_ENV_NUMBER}') then 259 | test_env_number = ENV['TEST_ENV_NUMBER'] 260 | test_env_number = '1' if test_env_number == '' 261 | env_var.gsub('#{TEST_ENV_NUMBER}', test_env_number || '') 262 | else 263 | env_var 264 | end 265 | end 266 | end 267 | 268 | end 269 | end 270 | --------------------------------------------------------------------------------