├── .rspec ├── features ├── step_definitions │ ├── unused_steps.rb │ ├── fail_steps.rb │ ├── wait_steps.rb │ └── ambiguous_steps.rb ├── support │ └── env.rb ├── ambiguous.feature ├── scenario.feature ├── failure.feature ├── pending.feature ├── scenario_with_background.feature ├── scenario_outline.feature ├── scenario_outline_with_background.feature └── characteristics │ ├── cucumber_step_characteristics.json │ └── cucumber_step_characteristics.html ├── cucumber_version ├── 1.3.5 │ ├── output_path.rb │ └── Gemfile ├── 2.0.2 │ ├── output_path.rb │ └── Gemfile ├── 2.1.0 │ ├── output_path.rb │ └── Gemfile ├── 2.2.0 │ ├── output_path.rb │ └── Gemfile ├── 2.3.3 │ ├── output_path.rb │ └── Gemfile └── 2.4.0 │ ├── output_path.rb │ └── Gemfile ├── lib ├── cucumber_characteristics │ ├── version.rb │ ├── autoload.rb │ ├── formatter.rb │ ├── cucumber_common_step_patch.rb │ ├── configuration.rb │ ├── exporter.rb │ ├── cucumber_1x_step_patch.rb │ ├── profile_data.rb │ ├── cucumber_2x_step_patch.rb │ └── view │ │ └── step_report.html.haml └── cucumber_characteristics.rb ├── Gemfile ├── .rubocop.yml ├── .gitignore ├── LICENSE.txt ├── cucumber_characteristics.gemspec ├── Rakefile ├── spec ├── spec_helper.rb └── html_output_spec.rb └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format progress 3 | -------------------------------------------------------------------------------- /features/step_definitions/unused_steps.rb: -------------------------------------------------------------------------------- 1 | Given(/^I am unused/) do 2 | end 3 | -------------------------------------------------------------------------------- /features/support/env.rb: -------------------------------------------------------------------------------- 1 | require 'cucumber' 2 | require 'cucumber_characteristics/autoload' 3 | -------------------------------------------------------------------------------- /cucumber_version/1.3.5/output_path.rb: -------------------------------------------------------------------------------- 1 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = File.dirname(__FILE__) 2 | -------------------------------------------------------------------------------- /cucumber_version/2.0.2/output_path.rb: -------------------------------------------------------------------------------- 1 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = File.dirname(__FILE__) 2 | -------------------------------------------------------------------------------- /cucumber_version/2.1.0/output_path.rb: -------------------------------------------------------------------------------- 1 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = File.dirname(__FILE__) 2 | -------------------------------------------------------------------------------- /cucumber_version/2.2.0/output_path.rb: -------------------------------------------------------------------------------- 1 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = File.dirname(__FILE__) 2 | -------------------------------------------------------------------------------- /cucumber_version/2.3.3/output_path.rb: -------------------------------------------------------------------------------- 1 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = File.dirname(__FILE__) 2 | -------------------------------------------------------------------------------- /cucumber_version/2.4.0/output_path.rb: -------------------------------------------------------------------------------- 1 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = File.dirname(__FILE__) 2 | -------------------------------------------------------------------------------- /features/step_definitions/fail_steps.rb: -------------------------------------------------------------------------------- 1 | Given(/^I fail$/) do 2 | raise 'Expected step failure' 3 | end 4 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/version.rb: -------------------------------------------------------------------------------- 1 | module CucumberCharacteristics 2 | VERSION = '0.0.6'.freeze 3 | end 4 | -------------------------------------------------------------------------------- /features/step_definitions/wait_steps.rb: -------------------------------------------------------------------------------- 1 | Given(/^I wait ([\d\.]+) seconds$/) do |s| 2 | sleep(s.to_f) 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in cucumber_characteristics.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /features/step_definitions/ambiguous_steps.rb: -------------------------------------------------------------------------------- 1 | Given(/^I wait some/) do 2 | end 3 | 4 | Given(/^I wait some seconds$/) do 5 | end 6 | -------------------------------------------------------------------------------- /cucumber_version/2.0.2/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'cucumber', '2.0.2' 4 | gem 'cucumber_characteristics', path: '../..' 5 | -------------------------------------------------------------------------------- /cucumber_version/2.1.0/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'cucumber', '2.1.0' 4 | gem 'cucumber_characteristics', path: '../..' 5 | -------------------------------------------------------------------------------- /cucumber_version/2.2.0/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'cucumber', '2.2.0' 4 | gem 'cucumber_characteristics', path: '../..' 5 | -------------------------------------------------------------------------------- /cucumber_version/2.3.3/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'cucumber', '2.3.3' 4 | gem 'cucumber_characteristics', path: '../..' 5 | -------------------------------------------------------------------------------- /cucumber_version/2.4.0/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | gem 'cucumber', '2.4.0' 4 | gem 'cucumber_characteristics', path: '../..' 5 | -------------------------------------------------------------------------------- /cucumber_version/1.3.5/Gemfile: -------------------------------------------------------------------------------- 1 | source 'http://rubygems.org' 2 | 3 | ruby '< 2.3.0' 4 | 5 | gem 'cucumber', '1.3.5' 6 | gem 'cucumber_characteristics', path: '../..' 7 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/autoload.rb: -------------------------------------------------------------------------------- 1 | require 'cucumber_characteristics' 2 | 3 | AfterConfiguration do |configuration| 4 | configuration.formats << ['CucumberCharacteristics::Formatter', nil] 5 | end 6 | -------------------------------------------------------------------------------- /features/ambiguous.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenaio with an ambiguous step 2 | 3 | Scenario: Timings for normal scenario 4 | Given I wait some seconds 5 | -------------------------------------------------------------------------------- /features/scenario.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenario 2 | 3 | Scenario: Timings for normal scenario 4 | Given I wait 0.5 seconds 5 | When I wait 0.5 seconds 6 | Then I wait 0.5 seconds 7 | -------------------------------------------------------------------------------- /features/failure.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenaio with a pending step 2 | 3 | Scenario: Timings for normal scenario 4 | Given I wait 1 seconds 5 | When I fail 6 | Then I wait 1 seconds 7 | -------------------------------------------------------------------------------- /features/pending.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenaio with a pending step 2 | 3 | Scenario: Timings for normal scenario 4 | Given I wait 1 seconds 5 | When I call a pending step 6 | Then I wait 1 seconds 7 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/formatter.rb: -------------------------------------------------------------------------------- 1 | module CucumberCharacteristics 2 | class Formatter 3 | def initialize(runtime, io, options) 4 | @runtime = runtime 5 | @io = io 6 | @options = options 7 | @features = {} 8 | end 9 | 10 | def after_features(features) 11 | profile = ProfileData.new(@runtime, features) 12 | Exporter.new(profile).export 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | Metrics/LineLength: 2 | Max: 207 3 | 4 | # Offense count: 9 5 | Style/Documentation: 6 | Exclude: 7 | - 'spec/**/*' 8 | - 'test/**/*' 9 | - 'lib/cucumber_characteristics.rb' 10 | - 'lib/cucumber_characteristics/configuration.rb' 11 | - 'lib/cucumber_characteristics/cucumber_step_patch.rb' 12 | - 'lib/cucumber_characteristics/exporter.rb' 13 | - 'lib/cucumber_characteristics/formatter.rb' 14 | - 'lib/cucumber_characteristics/profile_data.rb' 15 | -------------------------------------------------------------------------------- /features/scenario_with_background.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenaio with a background 2 | 3 | Background: 4 | Given I wait 0.4 seconds 5 | 6 | Scenario: Timings for normal scenario 7 | Given I wait 0.1 seconds 8 | When I wait 0.2 seconds 9 | Then I wait 0.3 seconds 10 | 11 | Scenario: Timings for another normal scenario 12 | Given I wait 0.5 seconds 13 | When I wait 0.6 seconds 14 | Then I wait 0.7 seconds 15 | -------------------------------------------------------------------------------- /features/scenario_outline.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenario outline 2 | 3 | Scenario Outline: Timings for scenario outline 4 | Given I wait seconds 5 | When I wait seconds 6 | Then I wait seconds 7 | And I wait 0.2 seconds 8 | Examples: 9 | | given_wait | when_wait | then_wait | 10 | | 0.1 | 0.2 | 0.3 | 11 | | 0.3 | 0.2 | 0.1 | 12 | -------------------------------------------------------------------------------- /features/scenario_outline_with_background.feature: -------------------------------------------------------------------------------- 1 | Feature: As a user I want to understand where my tests are spending their time in a scenario outline with a background 2 | 3 | Background: 4 | Given I wait 0.4 seconds 5 | 6 | Scenario Outline: Timings for scenario outline 7 | Given I wait seconds 8 | When I wait seconds 9 | Then I wait seconds 10 | And I wait 0.2 seconds 11 | Examples: 12 | | given_wait | when_wait | then_wait | 13 | | 0.1 | 0.2 | 0.3 | 14 | | 0.5 | 0.6 | 0.7 | 15 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/cucumber_common_step_patch.rb: -------------------------------------------------------------------------------- 1 | # http://stackoverflow.com/questions/4470108/when-monkey-patching-a-method-can-you-call-the-overridden-method-from-the-new-i 2 | 3 | module Cucumber 4 | class StepMatch 5 | if method_defined?(:invoke) 6 | old_invoke = instance_method(:invoke) 7 | attr_reader :duration 8 | 9 | define_method(:invoke) do |multiline_arg| 10 | start_time = Time.now 11 | ret = old_invoke.bind(self).call(multiline_arg) 12 | @duration = Time.now - start_time 13 | ret 14 | end 15 | end 16 | end 17 | end 18 | 19 | module Cucumber 20 | module Ast 21 | class StepInvocation 22 | attr_reader :step_match 23 | end 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics.rb: -------------------------------------------------------------------------------- 1 | require 'cucumber_characteristics/configuration' 2 | require 'cucumber_characteristics/cucumber_common_step_patch' 3 | if Cucumber::VERSION < '2.0.0' 4 | require 'cucumber_characteristics/cucumber_1x_step_patch' 5 | else 6 | require 'cucumber_characteristics/cucumber_2x_step_patch' 7 | end 8 | require 'cucumber_characteristics/exporter' 9 | require 'cucumber_characteristics/formatter' 10 | require 'cucumber_characteristics/profile_data' 11 | 12 | module CucumberCharacteristics 13 | class << self 14 | attr_writer :configuration 15 | end 16 | 17 | def self.configuration 18 | @configuration ||= Configuration.new 19 | end 20 | 21 | def self.reset 22 | @configuration = Configuration.new 23 | end 24 | 25 | def self.configure 26 | yield(configuration) 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/configuration.rb: -------------------------------------------------------------------------------- 1 | module CucumberCharacteristics 2 | class Configuration 3 | attr_accessor :export_json, :export_html, :target_filename, :relative_path, :precision 4 | 5 | def initialize 6 | @export_json = true 7 | @export_html = true 8 | @precision = 4 9 | @target_filename = 'cucumber_step_characteristics' 10 | @relative_path = 'features/characteristics' 11 | end 12 | 13 | def full_target_filename 14 | "#{full_dir}/#{@target_filename}" 15 | end 16 | 17 | def full_dir 18 | dir = resolve_path_from_root @relative_path 19 | FileUtils.mkdir_p dir unless File.exist? dir 20 | dir 21 | end 22 | 23 | def resolve_path_from_root(rel_path) 24 | if defined?(Rails) 25 | Rails.root.join(rel_path) 26 | elsif defined?(Rake.original_dir) 27 | File.expand_path(rel_path, Rake.original_dir) 28 | else 29 | File.expand_path(rel_path, Dir.pwd) 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | /cucumber_version/1.3.5/features/characteristics/cucumber_step_characteristics.html 19 | /cucumber_version/1.3.5/features/characteristics/cucumber_step_characteristics.json 20 | /cucumber_version/2.3.3/features/characteristics/cucumber_step_characteristics.html 21 | /cucumber_version/2.3.3/features/characteristics/cucumber_step_characteristics.json 22 | /cucumber_version/2.1.0/features/characteristics/cucumber_step_characteristics.html 23 | /cucumber_version/2.1.0/features/characteristics/cucumber_step_characteristics.json 24 | /cucumber_version/2.2.0/features/characteristics/cucumber_step_characteristics.html 25 | /cucumber_version/2.2.0/features/characteristics/cucumber_step_characteristics.json 26 | /cucumber_version/2.4.0/features/characteristics/cucumber_step_characteristics.html 27 | /cucumber_version/2.4.0/features/characteristics/cucumber_step_characteristics.json 28 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Stuart Ingram 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /cucumber_characteristics.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'cucumber_characteristics/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = 'cucumber_characteristics' 8 | spec.version = CucumberCharacteristics::VERSION 9 | spec.authors = ['Stuart Ingram'] 10 | spec.email = ['stuart.ingram@gmail.com'] 11 | spec.description = 'Gem to profile cucumber steps and features' 12 | spec.summary = 'Gem to profile cucumber steps and features' 13 | spec.homepage = 'https://github.com/singram/cucumber_characteristics' 14 | spec.license = 'MIT' 15 | 16 | spec.files = `git ls-files`.split($INPUT_RECORD_SEPARATOR) 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ['lib'] 20 | spec.required_ruby_version = '~> 2.0' 21 | 22 | spec.add_dependency 'cucumber', '>=1.3.5' 23 | spec.add_dependency 'haml' 24 | 25 | spec.add_development_dependency 'bundler' 26 | spec.add_development_dependency 'nokogiri' 27 | spec.add_development_dependency 'pry' 28 | spec.add_development_dependency 'rake' 29 | spec.add_development_dependency 'rspec' 30 | spec.add_development_dependency 'rubocop' 31 | end 32 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake' 2 | require 'bundler/gem_tasks' 3 | 4 | # Approach to version testing credited to 5 | # https://github.com/makandra/cucumber_factory 6 | 7 | namespace :versions do 8 | namespace :bundle do 9 | desc 'Bundle all spec apps' 10 | task :install do 11 | for_each_directory_of('./cucumber_version/**/Gemfile') do |directory| 12 | Bundler.with_clean_env do 13 | system("cd #{directory} && bundle install") 14 | end 15 | end 16 | end 17 | end 18 | 19 | desc 'Test all supported cucumber versions' 20 | task :test do 21 | for_each_directory_of('./cucumber_version/**/Gemfile') do |directory| 22 | Bundler.with_clean_env do 23 | clean_outputs(directory) 24 | system("cd #{directory} && bundle exec cucumber --color --format progress -g -r ../../features/ ../../features/") 25 | system("bundle exec rspec -r #{directory}/output_path.rb ") 26 | end 27 | end 28 | end 29 | end 30 | 31 | def clean_outputs(dir) 32 | ["#{dir}/features/characteristics/cucumber_step_characteristics.json", 33 | "#{dir}/features/characteristics/cucumber_step_characteristics.html" 34 | ].each do |file| 35 | File.delete(file) if File.exist?(file) 36 | end 37 | end 38 | 39 | def for_each_directory_of(path) 40 | Dir[path].sort.each do |rakefile| 41 | directory = File.dirname(rakefile) 42 | puts '', "\033[44m#{directory}\033[0m", '' 43 | yield(directory) 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # This file was generated by the `rspec --init` command. Conventionally, all 2 | # specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. 3 | # Require this file using `require "spec_helper"` to ensure that it is only 4 | # loaded once. 5 | # 6 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 7 | RSpec.configure do |config| 8 | config.treat_symbols_as_metadata_keys_with_true_values = true 9 | config.run_all_when_everything_filtered = true 10 | config.filter_run :focus 11 | 12 | # Run specs in random order to surface order dependencies. If you find an 13 | # order dependency and want to debug it, you can fix the order by providing 14 | # the seed, which is printed after each run. 15 | # --seed 1234 16 | config.order = 'random' 17 | end 18 | 19 | unless defined?(CHARACTERISTICS_OUTPUT_PATH_PREFIX) 20 | CHARACTERISTICS_OUTPUT_PATH_PREFIX = '.'.freeze 21 | end 22 | 23 | CUCUMBER_VERSION = if defined?(Cucumber) 24 | Cucumber::Version.to_s 25 | else 26 | CHARACTERISTICS_OUTPUT_PATH_PREFIX.split('/').last 27 | end 28 | 29 | TIMING_TOLERANCE = 0.05 30 | 31 | require 'pp' 32 | 33 | def read_html_output 34 | output_file = "#{CHARACTERISTICS_OUTPUT_PATH_PREFIX}/features/characteristics/cucumber_step_characteristics.html" 35 | puts "Testing - #{output_file}" 36 | File.open(output_file) { |f| Nokogiri::HTML(f) } 37 | rescue Errno::ENOENT 38 | puts "Could not find file '#{output_file}'" 39 | exit 40 | end 41 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/exporter.rb: -------------------------------------------------------------------------------- 1 | require 'haml' 2 | require 'digest/md5' 3 | 4 | module CucumberCharacteristics 5 | class Exporter 6 | attr_reader :profile 7 | def initialize(profile) 8 | @profile = profile 9 | @config = CucumberCharacteristics.configuration 10 | end 11 | 12 | def export 13 | filename = @config.full_target_filename 14 | if @config.export_html 15 | File.open(filename + '.html', 'w') { |file| file.write(to_html) } 16 | puts "Step characteristics report written to #{filename}.html" 17 | end 18 | if @config.export_json 19 | File.open(filename + '.json', 'w') { |file| file.write(to_json) } 20 | puts "Step characteristics report written to #{filename}.json" 21 | end 22 | end 23 | 24 | def to_html 25 | template = File.read(File.expand_path('../view/step_report.html.haml', __FILE__)) 26 | haml_engine = Haml::Engine.new(template) 27 | haml_engine.render(self) 28 | end 29 | 30 | def to_json 31 | @profile.step_profiles.to_json 32 | end 33 | 34 | # HELPERS 35 | 36 | def format_ts(t) 37 | t ? format("%0.#{@config.precision}f", t) : '-' 38 | end 39 | 40 | def format_step_usage(step_feature_data) 41 | step_feature_data[:feature_location].map do |location, timings| 42 | location.to_s + (timings.count > 1 ? " (x#{timings.count})" : '') 43 | end.join("\n") 44 | end 45 | 46 | def step_status_summary(profile) 47 | status = profile.step_count_by_status 48 | status.keys.sort.map { |s| status[s] > 0 ? "#{s.capitalize}: #{status[s]}" : nil }.compact.join(', ') 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/cucumber_1x_step_patch.rb: -------------------------------------------------------------------------------- 1 | module Cucumber 2 | class Runtime 3 | def scenario_profiles 4 | return @scenario_profiles if @scenario_profiles 5 | feature_profiles = {} 6 | scenarios.each do |f| 7 | if f.is_a?(Cucumber::Ast::OutlineTable::ExampleRow) 8 | feature_id = f.scenario_outline.file_colon_line 9 | feature_profiles[feature_id] ||= { name: f.scenario_outline.name, total_duration: 0, step_count: 0, example_count: 0, examples: {} } 10 | example_id = f.name 11 | feature_profiles[feature_id][:examples][example_id] = scenario_outline_example_profile(f) 12 | else 13 | feature_id = f.file_colon_line 14 | feature_profiles[feature_id] = scenario_profile(f) 15 | end 16 | end 17 | @scenario_profiles = feature_profiles 18 | end 19 | 20 | private 21 | 22 | def scenario_profile(scenario) 23 | scenario_profile = { name: scenario.name, total_duration: 0, step_count: 0 } 24 | scenario_profile[:total_duration] = scenario.steps.select { |s| s.status == :passed }.map { |s| s.step_match.duration }.inject(&:+) 25 | scenario_profile[:step_count] = scenario.steps.count 26 | scenario_profile[:status] = scenario.status 27 | scenario_profile 28 | end 29 | 30 | def scenario_outline_example_profile(scenario) 31 | example_profile = { total_duration: 0, step_count: 0 } 32 | example_profile[:total_duration] = scenario.instance_variable_get(:@step_invocations).select { |s| s.status == :passed }.map { |s| s.step_match.duration }.inject(&:+) 33 | example_profile[:step_count] = scenario.instance_variable_get(:@step_invocations).count 34 | example_profile[:status] = scenario.status 35 | example_profile 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /features/characteristics/cucumber_step_characteristics.json: -------------------------------------------------------------------------------- 1 | [["features/step_definitions/wait_steps.rb:1",{"total_count":33,"passed":{"count":31,"feature_location":{"features/failure.feature:4":[1.002],"features/pending.feature:4":[1.002],"features/scenario.feature:4":[0.501],"features/scenario.feature:5":[0.501],"features/scenario.feature:6":[0.501],"features/scenario_outline.feature:4":[0.101,0.305],"features/scenario_outline.feature:5":[0.2,0.2],"features/scenario_outline.feature:6":[0.305,0.101],"features/scenario_outline.feature:7":[0.2,0.2],"features/scenario_outline_with_background.feature:4":[0.402,0.402],"features/scenario_outline_with_background.feature:7":[0.101,0.501],"features/scenario_outline_with_background.feature:8":[0.2,0.601],"features/scenario_outline_with_background.feature:9":[0.305,0.701],"features/scenario_outline_with_background.feature:10":[0.2,0.2],"features/scenario_with_background.feature:4":[0.402,0.402],"features/scenario_with_background.feature:7":[0.101],"features/scenario_with_background.feature:8":[0.2],"features/scenario_with_background.feature:9":[0.305],"features/scenario_with_background.feature:12":[0.501],"features/scenario_with_background.feature:13":[0.601],"features/scenario_with_background.feature:14":[0.701]}},"failed":{"count":0,"feature_location":{}},"skipped":{"count":2,"feature_location":{"features/failure.feature:6":[],"features/pending.feature:6":[]}},"undefined":{"count":0,"feature_location":{}},"regexp":"/^I wait ([\\d\\.]+) seconds$/","fastest":0.101,"slowest":1.002,"average":0.3853225806451612,"total_duration":11.944999999999997,"standard_deviation":0.2372462157958885,"variation":0.901,"variance":0.0562857669094693}],["I call a pending step",{"total_count":1,"passed":{"count":0,"feature_location":{}},"failed":{"count":0,"feature_location":{}},"skipped":{"count":0,"feature_location":{}},"undefined":{"count":1,"feature_location":{"features/pending.feature:5":[]}},"fastest":null,"slowest":null,"average":null,"total_duration":null,"standard_deviation":null,"variation":null}],["features/step_definitions/fail_steps.rb:1",{"total_count":1,"passed":{"count":0,"feature_location":{}},"failed":{"count":1,"feature_location":{"features/failure.feature:5":[]}},"skipped":{"count":0,"feature_location":{}},"undefined":{"count":0,"feature_location":{}},"regexp":"/^I fail$/","fastest":null,"slowest":null,"average":null,"total_duration":null,"standard_deviation":null,"variation":null}]] -------------------------------------------------------------------------------- /lib/cucumber_characteristics/profile_data.rb: -------------------------------------------------------------------------------- 1 | require 'pp' 2 | 3 | module CucumberCharacteristics 4 | class ProfileData 5 | CUCUMBER_VERSION = Gem::Version.new(Cucumber::VERSION) 6 | 7 | extend Forwardable 8 | 9 | def_delegators :@runtime, :scenarios, :steps 10 | attr_reader :duration 11 | 12 | STATUS_ORDER = { passed: 0, failed: 2000, skipped: 1000, undefined: 500 }.freeze 13 | 14 | STATUS = STATUS_ORDER.keys 15 | 16 | def initialize(runtime, features) 17 | @runtime = runtime 18 | @duration = features.duration 19 | @features = features 20 | end 21 | 22 | def ambiguous_count 23 | @runtime.steps.count { |s| ambiguous?(s) } 24 | end 25 | 26 | def unmatched_steps 27 | unmatched = {} 28 | @runtime.unmatched_step_definitions.each do |u| 29 | location = u.file_colon_line 30 | unmatched[location] = u.regexp_source 31 | end 32 | unmatched.sort 33 | end 34 | 35 | def unmatched_steps? 36 | unmatched_steps.count > 0 37 | end 38 | 39 | def feature_profiles 40 | return @feature_profiles if @feature_profiles 41 | feature_profiles = @runtime.scenario_profiles 42 | @feature_profiles = with_feature_calculations(feature_profiles) 43 | end 44 | 45 | def with_feature_calculations(feature_profiles) 46 | feature_profiles.each do |feature, meta| 47 | next unless meta[:examples] 48 | feature_profiles[feature][:example_count] = meta[:examples].keys.count 49 | feature_profiles[feature][:total_duration] = meta[:examples].map { |_e, m| m[:total_duration] || 0 }.inject(&:+) 50 | feature_profiles[feature][:step_count] = meta[:examples].map { |_e, m| m[:step_count] }.inject(&:+) 51 | feature_profiles[feature][:examples] = feature_profiles[feature][:examples].sort_by { |_k, v| v[:total_duration] }.reverse 52 | feature_profiles[feature][:status] = if meta[:examples].all? { |_e, m| m[:status] == :passed } 53 | :passed 54 | elsif meta[:examples].any? { |_e, m| m[:status] == :failed } 55 | :failed 56 | elsif meta[:examples].any? { |_e, m| m[:status] == :skipped } 57 | :skipped 58 | else 59 | :unknown 60 | end 61 | end 62 | feature_profiles.sort_by { |_k, v| (STATUS_ORDER[v[:status]] || 0) + (v[:total_duration] || 0) }.reverse 63 | end 64 | 65 | def step_profiles 66 | step_profiles = {} 67 | @runtime.steps.each do |s| 68 | next if ambiguous?(s) 69 | step_name = s.status == :undefined ? s.name : s.step_match.step_definition.file_colon_line 70 | # Initialize data structure 71 | step_profiles[step_name] ||= { total_count: 0 } 72 | STATUS.each { |status| step_profiles[step_name][status] ||= { count: 0, feature_location: {} } } 73 | feature_location = s.file_colon_line 74 | step_profiles[step_name][s.status][:count] += 1 75 | step_profiles[step_name][:total_count] += 1 76 | step_profiles[step_name][s.status][:feature_location][feature_location] ||= [] 77 | next unless s.status != :undefined 78 | step_profiles[step_name][:regexp] = s.step_match.step_definition.regexp_source 79 | if s.status == :passed 80 | step_profiles[step_name][s.status][:feature_location][feature_location] << s.step_match.duration 81 | end 82 | end 83 | with_step_calculations(step_profiles) 84 | end 85 | 86 | def ambiguous?(step) 87 | step.status == :failed && step.step_match.step_definition.nil? 88 | end 89 | 90 | def with_step_calculations(step_profiles) 91 | step_profiles.each do |step, meta| 92 | meta.merge!(fastest: nil, slowest: nil, average: nil, total_duration: nil, standard_deviation: nil, variation: nil) 93 | next unless meta[:passed][:count] > 0 94 | timings = [] 95 | STATUS.each do |status| 96 | timings << meta[status][:feature_location].values 97 | end 98 | timings = timings.flatten.compact 99 | 100 | step_profiles[step][:fastest] = timings.min 101 | step_profiles[step][:slowest] = timings.max 102 | step_profiles[step][:variation] = step_profiles[step][:slowest] - step_profiles[step][:fastest] 103 | step_profiles[step][:total_duration] = timings.inject(:+) 104 | step_profiles[step][:average] = step_profiles[step][:total_duration] / meta[:passed][:count] 105 | sum = timings.inject(0) { |accum, i| accum + (i - step_profiles[step][:average])**2 } 106 | step_profiles[step][:variance] = sum / timings.length.to_f 107 | step_profiles[step][:standard_deviation] = Math.sqrt(step_profiles[step][:variance]) 108 | end 109 | step_profiles.sort_by { |_k, v| v[:total_duration] || 0 }.reverse 110 | end 111 | 112 | def step_duration 113 | step_duration = [] 114 | step_profiles.each do |_step, meta| 115 | STATUS.each do |status| 116 | meta[status][:feature_location].each do |_location, timings| 117 | step_duration << timings 118 | end 119 | end 120 | end 121 | step_duration.flatten.compact.inject(:+) || 0 122 | end 123 | 124 | def nonstep_duration 125 | duration - step_duration 126 | end 127 | 128 | def step_count_by_status 129 | status = {} 130 | @runtime.steps.each do |s| 131 | status[s.status] ||= 0 132 | status[s.status] += 1 133 | end 134 | status 135 | end 136 | 137 | def step_count(status) 138 | step_count_by_status[status] 139 | end 140 | 141 | def scenario_count_by_status 142 | status = {} 143 | @runtime.scenarios.each do |s| 144 | status[s.status] ||= 0 145 | status[s.status] += 1 146 | end 147 | status 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CucumberCharacteristics 2 | 3 | Gem to profile cucumber steps and features. 4 | 5 | ## Compatibility 6 | 7 | + (J)Ruby - 1.9.3 -> 2.3.0 8 | + Cucumber - 1.3.5, 2.4.x+ 9 | 10 | ## High level features 11 | 12 | Step analysis including 13 | + Location of step in steps file & regex 14 | + Step usage location and number of times executed (background/outline, etc.) 15 | + Counts for success/failure/pending/etc. 16 | + Total time taken in test run 17 | + Average, fastest, slowest times per step 18 | + Variation, variance & standard deviation calculations 19 | 20 | Feature analysis including 21 | + Feature location 22 | + Time taken to run feature 23 | + Result of feature test (pass, fail, etc.) 24 | + Number of steps run 25 | + Breakdown of feature by individual example run if a scenario outline 26 | 27 | Other features. 28 | + Reporting of ambiguous step calls 29 | + Reporting of unused step definitions 30 | 31 | ## Installation 32 | 33 | ### Step 1 34 | 35 | Add this line to your application's Gemfile: 36 | 37 | gem 'cucumber_characteristics' 38 | 39 | And then execute: 40 | 41 | $ bundle install 42 | 43 | Or install it yourself as: 44 | 45 | $ gem install cucumber_characteristics 46 | 47 | 48 | ### Step 2 49 | 50 | Add the following line to your cucumber environment file typically found at `features\support\env.rb` 51 | 52 | require 'cucumber_characteristics/autoload' 53 | 54 | ## Usage 55 | 56 | 1. For always-on automatic loading (recommended), add `require 'cucumber_characteristics/autoload'` to `features/support/yourapp_env.rb`. It is recommended by cucumber that you do not enhance features/support/env.rb so that upgrades are painless (relatively) 57 | 58 | 2. Add it to your `cucumber.yml` by adding `--format CucumberCharacteristics::Formatter` i.e. 59 | 60 | `std_opts = "-r features/. -r --quiet --format CucumberCharacteristics::Formatter --format progress"` 61 | 62 | 3. Use it via command line with `--format CucumberCharacteristics::Formatter`. 63 | 64 | ## Configuration 65 | 66 | You can configure the export of step characteristics via the following (defaults are same as example) 67 | 68 | CucumberCharacteristics.configure do |config| 69 | config.export_json = true 70 | config.export_html = true 71 | config.precision = 4 72 | config.target_filename = 'cucumber_step_characteristics' 73 | config.relative_path = 'features/characteristics' 74 | end 75 | 76 | This again can be added to your cucumber environment file typically found at `features\support\env.rb` 77 | 78 | ## Results 79 | 80 | Exported characteristic information is listed out at the end of the cucumber run in a message similar to 81 | 82 | Step characteristic report written to /home/singram/projects/gems/cucumber_characteristics/features/characteristics/cucumber_step_characteristics.html 83 | Step characteristic report written to /home/singram/projects/gems/cucumber_characteristics/features/characteristics/cucumber_step_characteristics.json 84 | 85 | depending on the options specified. 86 | 87 | The JSON option is provided for convenience in case there is a further use case/analysis required that is not provided by the gem. 88 | 89 | An example can be found [here](features/characteristics/cucumber_step_characteristics.json) 90 | 91 | ## Problem 92 | 93 | The formatting hooks on the face of it provide the necessary event points to profile any given feature file. 94 | This is true for a Scenario, but consider the following ScenaioOutline 95 | 96 | Feature: As a user I want to understand where my tests are spending their time 97 | 98 | Scenario Outline: Timings for scenario outline 99 | Given I wait seconds 100 | When I wait seconds 101 | Then I wait seconds 102 | And I wait 0.2 seconds 103 | Examples: 104 | | given_wait | when_wait | then_wait | 105 | | 1 | 2 | 3 | 106 | | 5 | 6 | 7 | 107 | 108 | Running 109 | 110 | cucumber --format debug features/outline.feature 111 | 112 | A couple of problems become evident 113 | 114 | 1. There are step definitions walked prior to the examples_array. These steps are not actually invoked rendering these hooks points misleading for profiling purposes 115 | 2. There are only 3 table_cell element blocks. These can be profiled, but what about the last step that does not have an input from the examples? There are no hook points to profile this step. 116 | 117 | This is why when you use the 'progress' formatter you would get 4 'skipped' for the initial step hooks triggered and then only 6 green dots representing steps when there should be 8 as it key's off table cells not steps. 118 | 119 | Possible solutions 120 | 121 | 1. Introduce new hook point for all true step invocations regardless of context. 122 | 2. Adjust table_cell hooks to include 'null' cells when considering steps without definitions. 123 | 3. Include profile information in runtime master object to parse out at end. 124 | 125 | As it turns out it was pretty simple to enhance the runtime object to reliably return profile information. 126 | 127 | ## Development 128 | 129 | 1. Install development environment 130 | 131 | `bundle install` 132 | 133 | 2. Run formatter over default cucumber version 134 | 135 | `bundle exec cucumber` 136 | 137 | 3. Run tests across all supported cucumber versions 138 | 139 | `bundle exec rake versions:bundle:install` 140 | 141 | `bundle exec rake versions:test` 142 | 143 | * NOTE. When running the cucumber tests failures, pending etc are expected. All specs should pass * 144 | 145 | 146 | ## Contributing 147 | 148 | 1. Fork it 149 | 2. Create your feature branch (`git checkout -b my-new-feature`) 150 | 3. Commit your changes (`git commit -am 'Add some feature'`) 151 | 4. Push to the branch (`git push origin my-new-feature`) 152 | 5. Create new Pull Request (after running tests!) 153 | 154 | ## Credits 155 | 1. Ryan Boucher [cucumber_timing_presenter](https://github.com/distributedlife/cucumber_timing_presenter) for inspiration. 156 | 2. AlienFast [cucumber_statistics](https://github.com/alienfast/cucumber_statistics) for inspiration. 157 | 3. [Brandon Hilker](http://brandonhilkert.com/blog/ruby-gem-configuration-patterns/) for gem building tutorials 158 | 4. Nathan Menge for helping QA the ruby 2.3.0 / cucumber 2.3.0 updates 159 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/cucumber_2x_step_patch.rb: -------------------------------------------------------------------------------- 1 | module Cucumber 2 | class StepDefinitionLight 3 | unless method_defined?(:file_colon_line) 4 | def file_colon_line 5 | location.file_colon_line 6 | end 7 | end 8 | end 9 | 10 | module Core 11 | module Ast 12 | module Location 13 | # Cucumber::Core::Ast::Location::Precise 14 | class Precise 15 | unless method_defined?(:file_colon_line) 16 | def file_colon_line 17 | to_s 18 | end 19 | end 20 | end 21 | end 22 | end 23 | end 24 | 25 | module Formatter 26 | module LegacyApi 27 | module Ast 28 | class Scenario 29 | attr_accessor :steps, :background_steps 30 | end 31 | end 32 | 33 | class RuntimeFacade 34 | def scenario_profiles 35 | return @scenario_profiles if @scenario_profiles 36 | feature_profiles = {} 37 | 38 | assign_steps_to_scenarios! 39 | results.scenarios.each do |scenario| 40 | # Feature id outline is the top level feature defintion 41 | # aggregating up multiple examples 42 | feature_id = feature_id(scenario) 43 | if outline_feature?(scenario) 44 | feature_profiles[feature_id] ||= { name: scenario.name, total_duration: 0, step_count: 0, examples: {} } 45 | agg_steps = aggregate_steps(scenario.steps) 46 | feature_profiles[feature_id][:total_duration] += agg_steps[:total_duration] 47 | feature_profiles[feature_id][:step_count] += agg_steps[:step_count] 48 | 49 | # First order step associations to scenario examples 50 | example_id = scenario_from(scenario.steps.first).name.match(/(Examples.*\))/).captures.first 51 | feature_profiles[feature_id][:examples][example_id] = { total_duration: 0, step_count: 0 } 52 | feature_profiles[feature_id][:examples][example_id] = aggregate_steps(scenario.steps) 53 | feature_profiles[feature_id][:examples][example_id][:status] = scenario.status.to_sym 54 | else 55 | feature_profiles[feature_id] = { name: scenario.name, total_duration: 0, step_count: 0 } 56 | feature_profiles[feature_id][:status] = scenario.status.to_sym 57 | feature_profiles[feature_id].merge!(aggregate_steps(scenario.steps)) 58 | end 59 | end 60 | # Collect up background tasks not directly attributable to 61 | # specific scenarios 62 | feature_files.each do |file| 63 | steps = background_steps_for(file) 64 | next if steps.empty? 65 | feature_id = "#{file}:0 (Background)" 66 | feature_profiles[feature_id] = { name: 'Background', total_duration: 0, step_count: 0 } 67 | feature_profiles[feature_id].merge!(aggregate_steps(steps)) 68 | feature_profiles[feature_id][:status] = 69 | steps.map(&:status).uniq.join(',') 70 | end 71 | 72 | @scenario_profiles = feature_profiles 73 | end 74 | 75 | private 76 | 77 | def feature_id(scenario) 78 | if outline_feature?(scenario) 79 | scenario_outline_to_feature_id(scenario) 80 | else 81 | scenario.location.to_s 82 | end 83 | end 84 | 85 | def outline_feature?(scenario) 86 | scenario.name =~ /Examples \(#\d+\)$/ 87 | end 88 | 89 | def outline_step?(step) 90 | step.step.class == Cucumber::Core::Ast::ExpandedOutlineStep 91 | end 92 | 93 | def background_step?(step) 94 | !step.background.nil? 95 | end 96 | 97 | def scenario_from(step) 98 | if outline_step?(step) 99 | # Match directly to scenario by line number 100 | scenarios.select do |s| 101 | s.location.file == step.location.file && s.location.line == step.location.line 102 | end.first 103 | else 104 | # Match indirectly to preceeding scenario by line number 105 | # (explicit sort needed for ruby 2.x) 106 | scenarios.select { |s| s.location.file == step.location.file && s.location.line < step.location.line }.sort { |a, b| a.location.line <=> b.location.line }.last 107 | end 108 | end 109 | 110 | def background_steps_for(scenario_file) 111 | steps.select do |s| 112 | s.location.file == scenario_file && 113 | background_step?(s) 114 | end 115 | end 116 | 117 | def assign_steps_to_scenarios! 118 | steps.each do |step| 119 | scenario = scenario_from(step) 120 | if scenario 121 | scenario.steps ||= [] 122 | scenario.steps << step 123 | end 124 | end 125 | end 126 | 127 | Feature = Struct.new(:name, :line) #=> Customer 128 | def scenario_outlines_from_file(file) 129 | count = 1 130 | results = [] 131 | File.open(file).each_line do |li| 132 | results << Feature.new(li, count) if li[/^\s*Scenario Outline:/] 133 | count += 1 134 | end 135 | results 136 | end 137 | 138 | # List of scenario name and lines from file 139 | def scenario_outlines(file) 140 | @feature_outlines ||= {} 141 | return @feature_outlines[file] if @feature_outlines[file] 142 | @feature_outlines[file] = scenario_outlines_from_file(file) 143 | end 144 | 145 | def scenario_outline_to_feature_id(scenario) 146 | scenarios = scenario_outlines(scenario.location.file) 147 | scenario_outline = scenarios.select { |s| s.line < scenario.location.line } 148 | scenario_outline = scenario_outline.sort { |a, b| a.line <=> b.line }.last 149 | "#{scenario.location.file}:#{scenario_outline.line}" 150 | end 151 | 152 | def feature_files 153 | scenarios.map { |s| s.location.file }.uniq 154 | end 155 | 156 | def aggregate_steps(steps) 157 | { total_duration: steps.reject { |s| [:skipped, :undefined].include?(s.status) }.map { |s| s.duration.nanoseconds.to_f / 1_000_000_000 }.inject(&:+), 158 | step_count: steps.count } 159 | end 160 | end 161 | end 162 | end 163 | end 164 | -------------------------------------------------------------------------------- /lib/cucumber_characteristics/view/step_report.html.haml: -------------------------------------------------------------------------------- 1 | !!! 2 | %html{:lang => "en"} 3 | %head 4 | %meta{:charset => "utf-8"}/ 5 | %meta{:content => "IE=edge", "http-equiv" => "X-UA-Compatible"}/ 6 | %meta{:content => "width=device-width, initial-scale=1", :name => "viewport"}/ 7 | %meta{:content => "Cucumber Step Characteristics", :name => "description"}/ 8 | %title Cucumber Step Characteristics 9 | %link{:href => "http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap.min.css", :rel => "stylesheet"}/ 10 | %link{:href => "http://netdna.bootstrapcdn.com/bootstrap/3.1.1/css/bootstrap-theme.min.css", :rel => "stylesheet"}/ 11 | / Custom styles for this layout 12 | :css 13 | body { 14 | min-height: 2000px; 15 | padding-top: 70px; 16 | } 17 | 18 | td { 19 | white-space: nowrap; 20 | } 21 | 22 | table.tablesorter thead tr .headerSortUp { 23 | background-color: #8dbdd8; 24 | } 25 | table.tablesorter thead tr .headerSortDown { 26 | background-color: #8dbdd8; 27 | } 28 | / HTML5 shim and Respond.js IE8 support of HTML5 elements and media queries 29 | /[if lt IE 9] 30 | 31 | 32 | %body 33 | .container 34 | .page-header 35 | %h1 Cucumber step characteristics 36 | .alert.alert-info 37 | %span 38 | #{profile.scenarios.count} Scenarios, 39 | #{profile.steps.count} Steps completed 40 | %span.text-muted 41 | (#{step_status_summary(profile)})
42 | - if profile.ambiguous_count > 0 43 | #{profile.ambiguous_count}/#{profile.step_count(:failed)} failures due to ambiguous matches
44 | Test duration #{format_ts(profile.duration)}s. 45 | %span.text-muted 46 | (#{format_ts(profile.step_duration)}s steps, #{format_ts(profile.nonstep_duration)}s non-steps) 47 | %span.text-muted.pull-right.small 48 | Finished on #{Time.now} 49 | 50 | %ul.nav.nav-tabs 51 | %li.active 52 | %a{:href => '#steps', :'data-toggle' => 'tab'} Steps 53 | %li 54 | %a{:href => '#features', :'data-toggle' => 'tab'} Features 55 | - if profile.unmatched_steps? 56 | %li 57 | %a{:href => '#unused_steps', :'data-toggle' => 'tab'} Unused Steps (#{profile.unmatched_steps.count}) 58 | %div.tab-content 59 | %div.tab-pane.active#steps 60 | %table.table.table-striped.table-bordered.table-condensed.tablesorter#profile_table 61 | %thead 62 | %tr 63 | %th Step 64 | %th Total time 65 | %th Passed 66 | %th Average 67 | %th Fastest 68 | %th Slowest 69 | %th Variation 70 | %th Variance 71 | %th Std Deviation 72 | %th Skipped 73 | %th Error 74 | %th Undef 75 | %th Total count 76 | %tbody 77 | - profile.step_profiles.each do |step, meta| 78 | %tr.step_result 79 | %td.step 80 | %abbr{:title => "#{meta[:regexp]}"} #{step} 81 | %td.total_time #{format_ts(meta[:total_duration])} 82 | %td.passed_count 83 | - if meta[:passed][:count] > 0 84 | %abbr{:title => "#{ format_step_usage(meta[:passed]) }" } #{meta[:passed][:count]} 85 | - else 86 | 0 87 | %td.average_time #{format_ts(meta[:average])} 88 | %td.fastest_time #{format_ts(meta[:fastest])} 89 | %td.slowest_time #{format_ts(meta[:slowest])} 90 | %td.variation_time #{format_ts(meta[:variation])} 91 | %td.variance_time #{format_ts(meta[:variance])} 92 | %td.std_dev_time #{format_ts(meta[:standard_deviation])} 93 | %td.skipped_count 94 | - if meta[:skipped][:count] > 0 95 | %abbr{:title => "#{ format_step_usage(meta[:skipped]) }" } #{meta[:skipped][:count]} 96 | - else 97 | 0 98 | %td.failed_count 99 | - if meta[:failed][:count] > 0 100 | %abbr{:title => "#{ format_step_usage(meta[:failed]) }" } #{meta[:failed][:count]} 101 | - else 102 | 0 103 | %td.undef_count 104 | - if meta[:undefined][:count] > 0 105 | %abbr{:title => "#{ format_step_usage(meta[:undefined]) }" } #{meta[:undefined][:count]} 106 | - else 107 | 0 108 | %td.total_count #{meta[:total_count]} 109 | 110 | %div.tab-pane#features 111 | %table.table.table-striped.table-bordered.table-condensed.tablesorter#feature_table 112 | %thead 113 | %tr 114 | %th Feature 115 | %th Total time 116 | %th Step count 117 | %th Status 118 | %th 119 | %tbody 120 | - profile.feature_profiles.each do |feature, meta| 121 | %tr.feature_result{:class => "#{meta[:status] == :failed ? :danger : meta[:status] == :undefined ? :warning : nil }"} 122 | %td.feature #{feature} 123 | %td.total_time #{format_ts(meta[:total_duration])} 124 | %td.step_count #{meta[:step_count]} 125 | %td.status #{meta[:status]} 126 | %td.examples 127 | - if meta[:examples] 128 | %button.btn.btn-primary{:'data-toggle'=>'modal', :'data-target'=>"##{Digest::MD5.hexdigest(feature)}"} 129 | #{meta[:example_count]} Examples 130 | 131 | - profile.feature_profiles.select{|f,m| m.key?(:examples) }.each do |feature, meta| 132 | %div{ :class=> 'modal fade bs-example-modal-lg', :tabindex=>"-1", :role=>"dialog", :'aria-labelledby'=>"myLargeModalLabel", :'aria-hidden'=>"true", :id => "#{Digest::MD5.hexdigest(feature)}"} 133 | %div.modal-dialog.modal-lg 134 | %div.modal-content 135 | %div.modal-header 136 | %h4 #{feature} 137 | %div.modal-body 138 | %table.table.table-striped.table-bordered.table-condensed{:id => feature.split('/').last.gsub(/[\.:]/, '_') } 139 | %thead 140 | %tr 141 | %th Example 142 | %th Total time 143 | %th Step count 144 | %th Status 145 | %tbody 146 | - meta[:examples].each do |example, meta| 147 | %tr.example_result{:class => "#{meta[:status] == :failed ? :danger : meta[:status] == :undefined ? :warning : nil }"} 148 | %td.example #{example} 149 | %td.total_time #{format_ts(meta[:total_duration])} 150 | %td.step_count #{meta[:step_count]} 151 | %td.status #{meta[:status]} 152 | %tfooter 153 | %tr.example_totals 154 | %th 155 | %th.total_time #{format_ts(meta[:total_duration])} 156 | %th.step_count #{meta[:step_count]} 157 | %th.status #{meta[:status]} 158 | %div.modal-footer 159 | %button.btn.btn-default{:type=>"button", :'data-dismiss'=>"modal"} 160 | Close 161 | 162 | - if profile.unmatched_steps? 163 | %div.tab-pane#unused_steps 164 | %table.table.table-striped.table-bordered.table-condensed.tablesorter#unmatched_steps_table 165 | %thead 166 | %tr 167 | %th Step Location 168 | %th Step 169 | %tbody 170 | - profile.unmatched_steps.each do |step_location, step| 171 | %tr 172 | %td #{step_location} 173 | %td #{step} 174 | 175 | %script{:src => "https://ajax.googleapis.com/ajax/libs/jquery/1.11.0/jquery.min.js"} 176 | %script{:src => "http://netdna.bootstrapcdn.com/bootstrap/3.1.1/js/bootstrap.min.js"} 177 | %script{:src => "https://raw.githubusercontent.com/christianbach/tablesorter/master/jquery.tablesorter.min.js"} 178 | :javascript 179 | $(document).ready(function() 180 | { 181 | $("#profile_table").tablesorter( {sortList: [[1,1],[3,1]], widgets: ['zebra']} ); 182 | $("#feature_table").tablesorter( {sortList: [[3,0],[1,1]], widgets: ['zebra']} ); 183 | $("#unmatched_steps_table").tablesorter( {sortList: [[1,0]], widgets: ['zebra']} ); 184 | } 185 | ); 186 | -------------------------------------------------------------------------------- /spec/html_output_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'nokogiri' 3 | 4 | describe 'HTML Output' do 5 | step_expectations = { 'pending step' => { total_count: 1, undef_count: 1 }, 6 | 'wait_steps' => { total_count: 33, passed_count: 31, 7 | skipped_count: 2, 8 | timings: { fastest_time: 0.1, 9 | slowest_time: 1.0, total_time: 11.95 } }, 10 | 'fail_steps' => { total_count: 1, failed_count: 1 }, 11 | 'ambiguous_steps' => { total_count: 1, passed_count: 1 } } 12 | 13 | if CUCUMBER_VERSION < '2.0.0' 14 | feature_expectations = { 15 | 'failure.feature' => { step_count: 3, total_time: 1.0, status: 'failed' }, 16 | 'pending.feature' => { step_count: 3, total_time: 1.0, 17 | status: 'undefined' }, 18 | 'scenario_outline_with_background.feature:6' => { step_count: 10, total_time: 3.6, 19 | status: 'passed', 20 | examples: { '| 0.5 | 0.6 | 0.7 |' => { total_time: 2.4, step_count: 5, 21 | status: 'passed' }, 22 | '| 0.1 | 0.2 | 0.3 |' => { total_time: 1.2, step_count: 5, 23 | status: 'passed' } } }, 24 | 'scenario_with_background.feature:11' => { step_count: 4, total_time: 2.2, 25 | status: 'passed' }, 26 | 'scenario_with_background.feature:6' => { step_count: 4, total_time: 1.0, 27 | status: 'passed' }, 28 | 'scenario_outline.feature:3' => { step_count: 8, total_time: 1.6, 29 | status: 'passed', 30 | examples: { '| 0.1 | 0.2 | 0.3 |' => { total_time: 0.8, step_count: 4, 31 | status: 'passed' }, 32 | '| 0.3 | 0.2 | 0.1 |' => { total_time: 0.8, step_count: 4, 33 | status: 'passed' } } }, 34 | 'scenario.feature' => { step_count: 3, total_time: 1.5, 35 | status: 'passed' }, 36 | 'ambiguous.feature' => { step_count: 1, total_time: 0.0, 37 | status: 'passed' } } 38 | else 39 | feature_expectations = { 40 | 'failure.feature' => { step_count: 3, total_time: 1.0, status: 'failed' }, 41 | 'pending.feature' => { step_count: 3, total_time: 1.0, 42 | status: 'undefined' }, 43 | 'scenario_outline_with_background.feature:6' => { step_count: 8, total_time: 2.8, 44 | status: 'passed', 45 | examples: { 'Examples (#2)' => { total_time: 2.0, step_count: 4, 46 | status: 'passed' }, 47 | 'Examples (#1)' => { total_time: 0.8, step_count: 4, 48 | status: 'passed' } } }, 49 | 'scenario_outline_with_background.feature:0' => { step_count: 2, total_time: 0.8, 50 | status: 'passed' }, 51 | 'scenario_with_background.feature:11' => { step_count: 3, total_time: 1.8, 52 | status: 'passed' }, 53 | 'scenario_with_background.feature:6' => { step_count: 3, total_time: 0.6, 54 | status: 'passed' }, 55 | 'scenario_with_background.feature:0' => { step_count: 2, total_time: 0.8, 56 | status: 'passed' }, 57 | 'scenario_outline.feature:3' => { step_count: 8, total_time: 1.6, 58 | status: 'passed', 59 | examples: { 'Examples (#1)' => { total_time: 0.8, step_count: 4, 60 | status: 'passed' }, 61 | 'Examples (#2)' => { total_time: 0.8, step_count: 4, 62 | status: 'passed' } } }, 63 | 'scenario.feature' => { step_count: 3, total_time: 1.5, 64 | status: 'passed' }, 65 | 'ambiguous.feature' => { step_count: 1, total_time: 0.0, 66 | status: 'passed' } } 67 | end 68 | 69 | before :all do 70 | @html_output = read_html_output 71 | end 72 | 73 | it 'should have a title' do 74 | expect(@html_output.css('title').text).to eq 'Cucumber Step Characteristics' 75 | end 76 | 77 | context 'step output' do 78 | let(:steps) { @html_output.css('table#profile_table tr.step_result') } 79 | 80 | it "has #{step_expectations.size} defined" do 81 | expect(steps.size).to eq step_expectations.size 82 | end 83 | 84 | step_expectations.each do |step_match, expectations| 85 | context step_match.to_s do 86 | let(:step) { steps.select { |tr| tr.css('.step')[0].text =~ /#{step_match}/ }.first } 87 | 88 | timings = expectations.delete(:timings) 89 | expectations.each do |type, expectation| 90 | it "#{type} of #{expectation}" do 91 | expect(step.css(".#{type}").text.strip).to eq expectation.to_s 92 | end 93 | end 94 | 95 | if timings 96 | 97 | context 'timing' do 98 | timings.each do |type, expectation| 99 | it "#{type} ~#{expectation}s" do 100 | expect(step.css(".#{type}").text.strip.to_f).to be_within(TIMING_TOLERANCE).of(expectation) 101 | end 102 | end 103 | end 104 | 105 | end 106 | end 107 | end 108 | end 109 | 110 | context 'feature output' do 111 | let(:features) { @html_output.css('table#feature_table tr.feature_result') } 112 | 113 | it "has #{feature_expectations.size} defined" do 114 | expect(features.size).to eq feature_expectations.size 115 | end 116 | 117 | feature_expectations.each do |feature_name, expectations| 118 | context feature_name do 119 | let(:feature) { features.select { |tr| tr.css('.feature')[0].text =~ /#{feature_name}/ }.first } 120 | 121 | it "has #{expectations[:step_count]} steps" do 122 | expect(feature.css('.step_count').text).to eq expectations[:step_count].to_s 123 | end 124 | 125 | it "has #{expectations[:status]} result" do 126 | expect(feature.css('.status').text).to eq expectations[:status].to_s 127 | end 128 | 129 | it "takes ~#{expectations[:total_time]}s" do 130 | expect(feature.css('.total_time').text.strip.to_f).to be_within(TIMING_TOLERANCE).of(expectations[:total_time]) 131 | end 132 | 133 | if expectations[:examples] 134 | 135 | context 'examples' do 136 | let(:examples) { @html_output.css("table##{feature_name.gsub(/[\.:]/, '_')}") } 137 | 138 | expectations[:examples].each do |id, details| 139 | context "example #{id}" do 140 | let(:example) { examples.css('.example_result').select { |tr| tr.css('.example')[0].text == id }.first } 141 | 142 | it "has #{details[:step_count]} steps" do 143 | expect(example.css('.step_count').text).to eq details[:step_count].to_s 144 | end 145 | 146 | it "takes #{details[:total_time]}s" do 147 | expect(example.css('.total_time').text.strip.to_f).to be_within(TIMING_TOLERANCE).of(details[:total_time]) 148 | end 149 | 150 | it "has #{details[:status]} result" do 151 | expect(example.css('.status').text).to eq details[:status].to_s 152 | end 153 | end 154 | end 155 | 156 | context 'totals' do 157 | let(:feature_totals) do 158 | examples.css('.example_totals') 159 | .first 160 | end 161 | 162 | total_steps = expectations[:examples].map { |_k, v| v[:step_count] }.inject(&:+) 163 | total_time = expectations[:examples].map { |_k, v| v[:total_time] }.inject(&:+) 164 | 165 | it "has #{total_steps} steps" do 166 | expect(feature_totals.css('.step_count').text).to eq total_steps.to_s 167 | end 168 | 169 | it "takes ~#{total_time}s" do 170 | expect(feature_totals.css('.total_time').text.strip.to_f).to be_within(TIMING_TOLERANCE).of(total_time) 171 | end 172 | end 173 | end 174 | 175 | end 176 | end 177 | end 178 | end 179 | end 180 | -------------------------------------------------------------------------------- /features/characteristics/cucumber_step_characteristics.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | Cucumber Step Characteristics 9 | 10 | 11 | 12 | 29 | 30 | 34 | 35 | 36 |
37 |
38 |

Cucumber step characteristics

39 |
40 |
41 | 42 | 10 Scenarios, 43 | 36 Steps completed 44 | 45 | (Failed: 2, Passed: 31, Skipped: 2, Undefined: 1)
46 | 47 | 1/2 failures due to ambiguous matches
48 | Test duration 12.5340s. 49 | 50 | (11.9450s steps, 0.5890s non-steps) 51 | 52 | 53 | 54 | Finished on 2016-04-10 22:28:08 -0400 55 | 56 |
57 | 68 |
69 |
70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 93 | 94 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 106 | 109 | 112 | 113 | 114 | 115 | 118 | 119 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 131 | 134 | 137 | 138 | 139 | 140 | 143 | 144 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 156 | 159 | 162 | 163 | 164 | 165 |
StepTotal timePassedAverageFastestSlowestVariationVarianceStd DeviationSkippedErrorUndefTotal count
91 | features/step_definitions/wait_steps.rb:1 92 | 11.9450 95 | 31 96 | 0.38530.10101.00200.90100.05630.2372 104 | 2 105 | 107 | 0 108 | 110 | 0 111 | 33
116 | I call a pending step 117 | - 120 | 0 121 | ------ 129 | 0 130 | 132 | 0 133 | 135 | 1 136 | 1
141 | features/step_definitions/fail_steps.rb:1 142 | - 145 | 0 146 | ------ 154 | 0 155 | 157 | 1 158 | 160 | 0 161 | 1
166 |
167 |
168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 221 | 222 | 223 | 224 | 225 | 226 | 227 | 232 | 233 | 234 | 235 | 236 | 237 | 238 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 248 | 249 | 250 |
FeatureTotal timeStep countStatus
features/failure.feature:31.00203failed 185 |
features/ambiguous.feature:3-1failed 193 |
features/pending.feature:31.00203undefined 201 |
features/scenario_outline_with_background.feature:63.613010passed 209 | 212 |
features/scenario_with_background.feature:112.20504passed 220 |
features/scenario_outline.feature:31.61208passed 228 | 231 |
features/scenario.feature:31.50303passed 239 |
features/scenario_with_background.feature:61.00804passed 247 |
251 | 299 | 347 |
348 |
349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | 357 | 358 | 359 | 360 | 361 | 362 |
Step LocationStep
features/step_definitions/unused_steps.rb:1/^I am unused/
363 |
364 |
365 |
366 | 367 | 368 | 369 | 376 | 377 | 378 | --------------------------------------------------------------------------------