├── Gemfile
├── lib
├── resque-job-stats.rb
├── resque
│ └── plugins
│ │ ├── job_stats
│ │ ├── version.rb
│ │ ├── enqueued.rb
│ │ ├── performed.rb
│ │ ├── failed.rb
│ │ ├── statistic.rb
│ │ ├── duration.rb
│ │ ├── history.rb
│ │ └── timeseries.rb
│ │ └── job_stats.rb
└── resque-job-stats
│ ├── server
│ └── views
│ │ ├── job_stats.erb
│ │ └── job_histories.erb
│ └── server.rb
├── examples
└── sinatra
│ ├── config.ru
│ ├── Gemfile
│ ├── app.rb
│ ├── README.md
│ ├── job.rb
│ └── Rakefile
├── .document
├── docs
└── images
│ ├── stats-individual.png
│ └── stats-overview.png
├── .travis.yml
├── test
├── helper.rb
├── test_job_stats.rb
└── test_server.rb
├── CHANGELOG.md
├── Rakefile
├── .gitignore
├── LICENSE.txt
├── resque-job-stats.gemspec
└── README.md
/Gemfile:
--------------------------------------------------------------------------------
1 | source 'https://rubygems.org'
2 |
3 | gemspec
4 |
--------------------------------------------------------------------------------
/lib/resque-job-stats.rb:
--------------------------------------------------------------------------------
1 | require 'resque/plugins/job_stats'
2 |
--------------------------------------------------------------------------------
/examples/sinatra/config.ru:
--------------------------------------------------------------------------------
1 | require './app'
2 | run ExampleApp
3 |
--------------------------------------------------------------------------------
/.document:
--------------------------------------------------------------------------------
1 | lib/**/*.rb
2 | bin/*
3 | -
4 | features/**/*.feature
5 | LICENSE.txt
6 |
--------------------------------------------------------------------------------
/docs/images/stats-individual.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alanpeabody/resque-job-stats/HEAD/docs/images/stats-individual.png
--------------------------------------------------------------------------------
/docs/images/stats-overview.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/alanpeabody/resque-job-stats/HEAD/docs/images/stats-overview.png
--------------------------------------------------------------------------------
/examples/sinatra/Gemfile:
--------------------------------------------------------------------------------
1 | source "https://rubygems.org"
2 |
3 | gem "rake"
4 | gem "sinatra"
5 | gem "resque"
6 | gem "resque-job-stats", path: "../../"
7 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/version.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 | VERSION = '0.5.0'
5 | end
6 | end
7 | end
8 |
--------------------------------------------------------------------------------
/.travis.yml:
--------------------------------------------------------------------------------
1 | rvm:
2 | - 2.2.10
3 | - 2.3.8
4 | - 2.4.6
5 | - 2.5.5
6 | - 2.6.3
7 | services:
8 | - redis-server
9 | before_install:
10 | - gem update bundler
11 |
--------------------------------------------------------------------------------
/examples/sinatra/app.rb:
--------------------------------------------------------------------------------
1 | require './job'
2 | require 'resque/server'
3 | require 'resque-job-stats'
4 | require 'resque-job-stats/server'
5 |
6 | Resque.redis = Redis.new
7 |
8 | class ExampleApp < Resque::Server
9 | end
10 |
--------------------------------------------------------------------------------
/test/helper.rb:
--------------------------------------------------------------------------------
1 | require 'minitest/autorun'
2 | require 'minitest/unit'
3 | require 'minitest/mock'
4 | require 'rack/test'
5 | require 'resque'
6 | require 'timecop'
7 |
8 | Resque.redis = 'localhost:6379'
9 | Resque.redis.namespace = 'resque:job_stats'
10 |
11 | $LOAD_PATH.unshift(File.dirname(__FILE__))
12 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
13 | require 'resque-job-stats'
14 |
15 |
--------------------------------------------------------------------------------
/examples/sinatra/README.md:
--------------------------------------------------------------------------------
1 | # Resque-Job-Stats demo
2 |
3 | Basic rack app showing resque-job-stats UI.
4 |
5 | ```ruby
6 | cd examples/sinatra
7 | bundle install
8 | rackup # start resque web server
9 |
10 | # in another terminal window
11 | QUEUE=* rake resque:work
12 |
13 | # in another terminal window
14 | rake enqueue_success
15 | rake enqueue_failure
16 | # can also specify counts
17 | SUCCESS_JOBS=10 rake enqueue_success
18 | ```
19 |
--------------------------------------------------------------------------------
/examples/sinatra/job.rb:
--------------------------------------------------------------------------------
1 | require 'resque'
2 | require 'resque-job-stats'
3 |
4 | module Job
5 | @queue = :default
6 | extend Resque::Plugins::JobStats
7 |
8 | def self.perform(params)
9 | sleep 1
10 | puts "Processed a job!"
11 | end
12 | end
13 |
14 | module FailingJob
15 | @queue = :failing
16 | extend Resque::Plugins::JobStats
17 |
18 | def self.perform(params)
19 | sleep 1
20 | raise 'not processable!'
21 | end
22 | end
23 |
--------------------------------------------------------------------------------
/examples/sinatra/Rakefile:
--------------------------------------------------------------------------------
1 | require './job'
2 | require 'resque'
3 | require 'redis'
4 | require "resque/tasks"
5 |
6 | Resque.redis = Redis.new
7 |
8 | desc "Enqueue successful job"
9 | task :enqueue_success do
10 | count = ENV['SUCCESS_JOBS'] || 1
11 | count.to_i.times do
12 | Resque.enqueue(Job, {})
13 | end
14 | end
15 |
16 | desc "Enqueue failing job"
17 | task :enqueue_failure do
18 | count = ENV['FAILURE_JOBS'] || 1
19 | count.to_i.times do
20 | Resque.enqueue(FailingJob, {})
21 | end
22 | end
23 |
--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
1 | # Changelog
2 |
3 | ## Master
4 |
5 | ## 0.5.0
6 |
7 | * Add example app (#27, 158f466)
8 | * Remove support for ruby 1.9 and 2.1 (#36, 3cf0aab)
9 | * Support Resque 2 (#35, 3e30e9c)
10 |
11 | ## 0.4.2
12 |
13 | * Fix missing ERB files in gemspec (e30e599)
14 |
15 | ## 0.4.1
16 |
17 | * Remove `date` from gemspec (31e054b)
18 |
19 | ## 0.4.0
20 |
21 | * Add stats and UI for individual job histories (#24, f10d9b6) -- @emilong
22 | * Modernize the gem, and add Travis CI testing for newer rubies (#23, cbac34d)
23 |
24 | Changelog started on Oct 20, 2016.
25 |
--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
1 | # encoding: utf-8
2 |
3 | require "bundler/gem_tasks"
4 |
5 | require 'rake'
6 | require 'rake/testtask'
7 |
8 | require File.dirname(__FILE__) + '/lib/resque-job-stats'
9 |
10 | Rake::TestTask.new(:test) do |test|
11 | test.libs << 'lib' << 'test'
12 | test.pattern = 'test/**/test_*.rb'
13 | test.verbose = true
14 | end
15 |
16 | task :default => :test
17 |
18 | require 'rdoc/task'
19 | Rake::RDocTask.new do |rdoc|
20 | version = Resque::Plugins::JobStats::VERSION
21 |
22 | rdoc.rdoc_dir = 'rdoc'
23 | rdoc.title = "resque-job-stats #{version}"
24 | rdoc.rdoc_files.include('README*')
25 | rdoc.rdoc_files.include('lib/**/*.rb')
26 | end
27 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/enqueued.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 |
5 | # Extend your job with this module to track how many
6 | # jobs are queued successfully
7 | module Enqueued
8 |
9 | # Sets the number of jobs queued
10 | def jobs_enqueued=(int)
11 | Resque.redis.set(jobs_enqueued_key,int)
12 | end
13 |
14 | # Returns the number of jobs enqueued
15 | def jobs_enqueued
16 | Resque.redis.get(jobs_enqueued_key).to_i
17 | end
18 |
19 | # Returns the key used for tracking jobs enqueued
20 | def jobs_enqueued_key
21 | "stats:jobs:#{self.name}:enqueued"
22 | end
23 |
24 | # Increments the enqueued count when job is queued
25 | def after_enqueue_job_stats_enqueued(*args)
26 | Resque.redis.incr(jobs_enqueued_key)
27 | end
28 |
29 | end
30 | end
31 | end
32 | end
33 |
34 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/performed.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 |
5 | # Extend your job with this module to track how many
6 | # jobs are performed successfully
7 | module Performed
8 |
9 | # Sets the number of jobs performed
10 | def jobs_performed=(int)
11 | Resque.redis.set(jobs_performed_key,int)
12 | end
13 |
14 | # Returns the number of jobs performed
15 | def jobs_performed
16 | Resque.redis.get(jobs_performed_key).to_i
17 | end
18 |
19 | # Returns the key used for tracking jobs performed
20 | def jobs_performed_key
21 | "stats:jobs:#{self.name}:performed"
22 | end
23 |
24 | # Increments the performed count when job is complete
25 | def after_perform_job_stats_performed(*args)
26 | Resque.redis.incr(jobs_performed_key)
27 | end
28 |
29 | end
30 | end
31 | end
32 | end
33 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/failed.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 |
5 | # Extend your job with this module to track how many
6 | # jobs fail
7 | module Failed
8 |
9 | # Sets the number of jobs failed
10 | def jobs_failed=(int)
11 | Resque.redis.set(jobs_failed_key,int)
12 | end
13 |
14 | # Returns the number of jobs failed
15 | def jobs_failed
16 | jobs_failed = Resque.redis.get(jobs_failed_key).to_i
17 | return jobs_failed / 2 if Resque::VERSION == '1.20.0'
18 | jobs_failed
19 | end
20 |
21 | # Returns the key used for tracking jobs failed
22 | def jobs_failed_key
23 | "stats:jobs:#{self.name}:failed"
24 | end
25 |
26 | # Increments the failed count when job is complete
27 | def on_failure_job_stats_failed(e,*args)
28 | Resque.redis.incr(jobs_failed_key)
29 | end
30 |
31 | end
32 | end
33 | end
34 | end
35 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats.rb:
--------------------------------------------------------------------------------
1 | require 'resque'
2 | require 'resque/plugins/job_stats/performed'
3 | require 'resque/plugins/job_stats/enqueued'
4 | require 'resque/plugins/job_stats/failed'
5 | require 'resque/plugins/job_stats/duration'
6 | require 'resque/plugins/job_stats/timeseries'
7 | require 'resque/plugins/job_stats/statistic'
8 | require 'resque/plugins/job_stats/history'
9 |
10 | module Resque
11 | module Plugins
12 | module JobStats
13 | include Resque::Plugins::JobStats::Performed
14 | include Resque::Plugins::JobStats::Enqueued
15 | include Resque::Plugins::JobStats::Failed
16 | include Resque::Plugins::JobStats::Duration
17 | include Resque::Plugins::JobStats::Timeseries::Enqueued
18 | include Resque::Plugins::JobStats::Timeseries::Performed
19 | include Resque::Plugins::JobStats::History
20 |
21 | def self.extended(base)
22 | self.measured_jobs << base
23 | end
24 |
25 | def self.measured_jobs
26 | @measured_jobs ||= []
27 | end
28 | end
29 | end
30 | end
31 |
--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | # rcov generated
2 | coverage
3 |
4 | # rdoc generated
5 | rdoc
6 |
7 | # yard generated
8 | doc
9 | .yardoc
10 |
11 | # bundler
12 | .bundle
13 |
14 | # jeweler generated
15 | pkg
16 |
17 | Gemfile.lock
18 |
19 | # Have editor/IDE/OS specific files you need to ignore? Consider using a global gitignore:
20 | #
21 | # * Create a file at ~/.gitignore
22 | # * Include files you want ignored
23 | # * Run: git config --global core.excludesfile ~/.gitignore
24 | #
25 | # After doing this, these files will be ignored in all your git projects,
26 | # saving you from having to 'pollute' every project you touch with them
27 | #
28 | # Not sure what to needs to be ignored for particular editors/OSes? Here's some ideas to get you started. (Remember, remove the leading # of the line)
29 | #
30 | # For MacOS:
31 | #
32 | #.DS_Store
33 |
34 | # For TextMate
35 | #*.tmproj
36 | #tmtags
37 |
38 | # For emacs:
39 | #*~
40 | #\#*
41 | #.\#*
42 |
43 | # For vim:
44 | #*.swp
45 |
46 | # For redcar:
47 | #.redcar
48 |
49 | # For rubinius:
50 | #*.rbc
51 |
52 | dump.rdb
53 |
--------------------------------------------------------------------------------
/LICENSE.txt:
--------------------------------------------------------------------------------
1 | Copyright (c) 2011 Alan Peabody
2 |
3 | Permission is hereby granted, free of charge, to any person obtaining
4 | a copy of this software and associated documentation files (the
5 | "Software"), to deal in the Software without restriction, including
6 | without limitation the rights to use, copy, modify, merge, publish,
7 | distribute, sublicense, and/or sell copies of the Software, and to
8 | permit persons to whom the Software is furnished to do so, subject to
9 | the following conditions:
10 |
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 |
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 |
--------------------------------------------------------------------------------
/lib/resque-job-stats/server/views/job_stats.erb:
--------------------------------------------------------------------------------
1 |
Resque Job Stats
2 |
3 |
4 | This page displays statistics about jobs that have been executed.
5 |
6 |
7 |
8 |
9 | | Name |
10 | <%= stat_header(:jobs_enqueued) %>
11 | <%= stat_header(:jobs_performed) %>
12 | <%= stat_header(:jobs_failed) %>
13 | <%= stat_header(:job_rolling_avg) %>
14 | <%= stat_header(:longest_job) %>
15 |
16 | <% @jobs.each do |job| %>
17 |
18 | |
19 | <%= job.name %>
20 | <% if job.job_class <= Resque::Plugins::JobStats::History ||
21 | job.job_class.is_a?(Resque::Plugins::JobStats::History) %>
22 | [history]
23 | <% end %>
24 | |
25 | <%= display_stat(job, :jobs_enqueued, :number_display) %>
26 | <%= display_stat(job, :jobs_performed, :number_display) %>
27 | <%= display_stat(job, :jobs_failed, :number_display) %>
28 | <%= display_stat(job, :job_rolling_avg, :time_display) %>
29 | <%= display_stat(job, :longest_job, :time_display) %>
30 |
31 | <% end %>
32 |
33 |
--------------------------------------------------------------------------------
/resque-job-stats.gemspec:
--------------------------------------------------------------------------------
1 | lib = File.expand_path('../lib', __FILE__)
2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
3 | require 'resque/plugins/job_stats/version'
4 |
5 | Gem::Specification.new do |s|
6 | s.name = "resque-job-stats"
7 | s.version = "#{Resque::Plugins::JobStats::VERSION}"
8 |
9 | s.authors = ["alanpeabody"]
10 | s.description = "Tracks jobs performed, failed, and the duration of the last 100 jobs for each job type."
11 | s.email = "gapeabody@gmail.com"
12 | s.extra_rdoc_files = [
13 | "LICENSE.txt",
14 | "README.md"
15 | ]
16 |
17 | files = `git ls-files`.split("\n") rescue []
18 | files &= (
19 | Dir['lib/**/*.{rb,erb}'] +
20 | Dir['*.md'])
21 |
22 | s.files = files
23 | s.require_paths = ["lib"]
24 |
25 | s.homepage = "http://github.com/alanpeabody/resque-job-stats"
26 | s.licenses = ["MIT"]
27 | s.required_ruby_version = ">= 2.2"
28 |
29 | s.summary = "Job-centric stats for Resque"
30 |
31 | s.add_dependency('resque', '>= 1.17', '< 3')
32 |
33 | s.add_development_dependency "rake"
34 | s.add_development_dependency "minitest", '~> 5.0'
35 | s.add_development_dependency "timecop", '~> 0.6'
36 | s.add_development_dependency 'rack-test', '>= 0'
37 | end
38 |
--------------------------------------------------------------------------------
/lib/resque-job-stats/server/views/job_histories.erb:
--------------------------------------------------------------------------------
1 | Resque Job Histories
2 |
3 |
4 | This page displays histories of jobs that have been executed.
5 |
6 |
7 | <%= @job_class %>
8 |
9 |
10 |
11 | | Start time |
12 | Duration |
13 | Arguments |
14 | Success |
15 | Exception |
16 |
17 | <% @histories.each do |history| %>
18 |
19 | | <%= history["run_at"] %> |
20 | <%= history["duration"] %> |
21 | <%= history["args"].inspect %> |
22 | <%= check_or_cross_stat(history["success"]) %> |
23 | <%= history["exception"]["name"] if history["exception"] %> |
24 |
25 | <% end %>
26 |
27 |
28 | <%if @start > 0 || @start + @limit <= @size %>
29 |
37 | <%end%>
38 |
39 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/statistic.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 | # A class composed of a job class and the various job statistics
5 | # collected for the given job.
6 | class Statistic
7 | include Comparable
8 |
9 | # An array of the default statistics that will be displayed in the web tab
10 | DEFAULT_STATS = [:jobs_enqueued, :jobs_performed, :jobs_failed, :job_rolling_avg, :longest_job]
11 |
12 | attr_accessor *[:job_class].concat(DEFAULT_STATS)
13 |
14 | class << self
15 | # Find and load a Statistic for all resque jobs that are in the Resque::Plugins::JobStats.measured_jobs collection
16 | def find_all(metrics)
17 | Resque::Plugins::JobStats.measured_jobs.map{|j| new(j, metrics)}
18 | end
19 | end
20 |
21 | # A series of metrics describing one job class.
22 | def initialize(job_class, metrics)
23 | self.job_class = job_class
24 | self.load(metrics)
25 | end
26 |
27 | def load(metrics)
28 | metrics.each do |metric|
29 | self.send("#{metric}=", job_class.send(metric))
30 | end
31 | end
32 |
33 | def name
34 | self.job_class.name
35 | end
36 |
37 | def <=>(other)
38 | self.name <=> other.name
39 | end
40 | end
41 | end
42 | end
43 | end
44 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/duration.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 | module Duration
5 |
6 | # Resets all job durations
7 | def reset_job_durations
8 | Resque.redis.del(jobs_duration_key)
9 | end
10 |
11 | # Returns the number of jobs failed
12 | def job_durations
13 | Resque.redis.lrange(jobs_duration_key,0,durations_recorded - 1).map(&:to_f)
14 | end
15 |
16 | # Returns the key used for tracking job durations
17 | def jobs_duration_key
18 | "stats:jobs:#{self.name}:duration"
19 | end
20 |
21 | # Increments the failed count when job is complete
22 | def around_perform_job_stats_duration(*args)
23 | start = Time.now
24 | yield
25 | duration = Time.now - start
26 |
27 | Resque.redis.lpush(jobs_duration_key, duration)
28 | Resque.redis.ltrim(jobs_duration_key, 0, durations_recorded)
29 | end
30 |
31 | def durations_recorded
32 | @durations_recorded || 100
33 | end
34 |
35 | def job_rolling_avg
36 | job_times = job_durations
37 | return 0.0 if job_times.size == 0.0
38 | job_times.inject(0.0) {|s,j| s + j} / job_times.size
39 | end
40 |
41 | def longest_job
42 | job_durations.max.to_f
43 | end
44 |
45 | end
46 | end
47 | end
48 | end
49 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/history.rb:
--------------------------------------------------------------------------------
1 | module Resque
2 | module Plugins
3 | module JobStats
4 | module History
5 | include Resque::Helpers
6 |
7 | def job_histories(start=0, limit=histories_recordable)
8 | Resque.redis.lrange(jobs_history_key, start, start + limit - 1).map { |h| decode(h) }
9 | end
10 |
11 | # Returns the key used for tracking job histories
12 | def jobs_history_key
13 | "stats:jobs:#{self.name}:history"
14 | end
15 |
16 | def around_perform_job_stats_history(*args)
17 | # we collect our own duration and start time rather
18 | # than correlate with the duration stat to make sure
19 | # we're associating them with the right job arguments
20 | start = Time.now
21 | begin
22 | yield
23 | duration = Time.now - start
24 | push_history "success" => true, "args" => args, "run_at" => start, "duration" => duration
25 | rescue Exception => e
26 | duration = Time.now - start
27 | exception = { "name" => e.to_s, "backtrace" => e.backtrace }
28 | push_history "success" => false, "exception" => exception, "args" => args, "run_at" => start, "duration" => duration
29 | raise e
30 | end
31 | end
32 |
33 | def histories_recordable
34 | @histories_recordable || 100
35 | end
36 |
37 | def histories_recorded
38 | Resque.redis.llen(jobs_history_key)
39 | end
40 |
41 | def reset_job_histories
42 | Resque.redis.del(jobs_history_key)
43 | end
44 |
45 | private
46 |
47 | def push_history(history)
48 | Resque.redis.lpush(jobs_history_key, encode(history))
49 | Resque.redis.ltrim(jobs_history_key, 0, histories_recordable)
50 | end
51 | end
52 | end
53 | end
54 | end
55 |
56 |
--------------------------------------------------------------------------------
/lib/resque-job-stats/server.rb:
--------------------------------------------------------------------------------
1 | require 'resque/server'
2 |
3 | module Resque
4 | module Plugins
5 | module JobStats
6 | module Server
7 | VIEW_PATH = File.join(File.dirname(__FILE__), 'server', 'views')
8 |
9 | def job_stats_to_display
10 | @job_stats_to_display ||= Resque::Plugins::JobStats::Statistic::DEFAULT_STATS
11 | end
12 |
13 | # Set this to an array of the public accessor names in Resque::Plugins::JobStats::Statistic
14 | # that you wish to display. The default is [:jobs_enqueued, :jobs_performed, :jobs_failed, :job_rolling_avg, :longest_job]
15 | # Examples:
16 | # Resque::Server.job_stats_to_display = [:jobs_performed, :job_rolling_avg]
17 | def job_stats_to_display=(stats)
18 | @job_stats_to_display = stats
19 | end
20 |
21 | module Helpers
22 | def display_stat?(stat_name)
23 | self.class.job_stats_to_display == :all ||
24 | [self.class.job_stats_to_display].flatten.map(&:to_sym).include?(stat_name.to_sym)
25 | end
26 |
27 | def time_display(float)
28 | float.zero? ? "" : ("%.2f" % float.to_s) + "s"
29 | end
30 |
31 | def number_display(num)
32 | num.zero? ? "" : num
33 | end
34 |
35 | def stat_header(stat_name)
36 | if(display_stat?(stat_name))
37 | "" + stat_name.to_s.gsub(/_/,' ').capitalize + " | "
38 | end
39 | end
40 |
41 | def display_stat(stat, stat_name, format)
42 | if(display_stat?(stat_name))
43 | formatted_stat = self.send(format, stat.send(stat_name))
44 | "#{formatted_stat} | "
45 | end
46 | end
47 |
48 | def check_or_cross_stat(value)
49 | value ? "✓" : "✗"
50 | end
51 | end
52 |
53 | class << self
54 | def registered(app)
55 | app.get '/job_stats' do
56 | @jobs = Resque::Plugins::JobStats::Statistic.find_all(self.class.job_stats_to_display).sort
57 | erb(File.read(File.join(VIEW_PATH, 'job_stats.erb')))
58 | end
59 | # We have little choice in using this funky name - Resque
60 | # already has a "Stats" tab, and it doesn't like
61 | # tab names with spaces in it (it translates the url as job%20stats)
62 | app.tabs << "Job_Stats"
63 |
64 | app.get '/job_history/:job_class' do
65 | @job_class = Resque::Plugins::JobStats.measured_jobs.find { |j| j.to_s == params[:job_class] }
66 | pass unless @job_class
67 |
68 | @start = 0
69 | @start = params[:start].to_i if params[:start]
70 | @limit = 100
71 | @limit = params[:limit].to_i if params[:limit]
72 |
73 | @histories = @job_class.job_histories(@start,@limit)
74 | @size = @job_class.histories_recorded
75 |
76 | erb(File.read(File.join(VIEW_PATH, 'job_histories.erb')))
77 | end
78 |
79 | app.helpers(Helpers)
80 | end
81 | end
82 | end
83 | end
84 | end
85 | end
86 |
87 | Resque::Server.register Resque::Plugins::JobStats::Server
88 |
--------------------------------------------------------------------------------
/lib/resque/plugins/job_stats/timeseries.rb:
--------------------------------------------------------------------------------
1 | require 'time'
2 |
3 | module Resque
4 | module Plugins
5 | module JobStats
6 |
7 | # Extend your job with this module to track how many
8 | # jobs are performed over a period of time
9 | module Timeseries
10 |
11 | module Common
12 | # A timestamp rounded to the lowest minute
13 | def timestamp
14 | time = Time.now.utc
15 | Time.at(time.to_i - time.sec).utc # to_i removes usecs
16 | end
17 |
18 | private
19 |
20 | TIME_FORMAT = {:minutes => "%d:%H:%M", :hours => "%d:%H"}
21 | FACTOR = {:minutes => 1, :hours => 60}
22 |
23 | def range(sample_size, time_unit, end_time) # :nodoc:
24 | (0..sample_size).map { |n| end_time - (n * 60 * FACTOR[time_unit])}
25 | end
26 |
27 | def timeseries_data(type, sample_size, time_unit) # :nodoc:
28 | timeseries_range = range(sample_size, time_unit, timestamp)
29 | timeseries_keys = timeseries_range.map { |time| jobs_timeseries_key(type, time, time_unit)}
30 | timeseries_data = Resque.redis.mget(*(timeseries_keys))
31 |
32 | return Hash[(0..sample_size).map { |i| [timeseries_range[i], timeseries_data[i].to_i]}]
33 | end
34 |
35 | def jobs_timeseries_key(type, key_time, time_unit) # :nodoc:
36 | "#{prefix}:#{type}:#{key_time.strftime(TIME_FORMAT[time_unit])}"
37 | end
38 |
39 | def prefix # :nodoc:
40 | "stats:jobs:#{self.name}:timeseries"
41 | end
42 |
43 | def incr_timeseries(type) # :nodoc:
44 | increx(jobs_timeseries_key(type, timestamp, :minutes), (60 * 61)) # 1h + 1m for some buffer
45 | increx(jobs_timeseries_key(type, timestamp, :hours), (60 * 60 * 25)) # 24h + 60m for some buffer
46 | end
47 |
48 | # Increments a key and sets its expiry time
49 | def increx(key, ttl)
50 | Resque.redis.incr(key)
51 | Resque.redis.expire(key, ttl)
52 | end
53 | end
54 | end
55 | end
56 | end
57 | end
58 |
59 | module Resque::Plugins::JobStats::Timeseries::Enqueued
60 | include Resque::Plugins::JobStats::Timeseries::Common
61 |
62 | # Increments the enqueued count for the timestamp when job is queued
63 | def after_enqueue_job_stats_timeseries(*args)
64 | incr_timeseries(:enqueued)
65 | end
66 |
67 | # Hash of timeseries data over the last 60 minutes for queued jobs
68 | def queued_per_minute
69 | timeseries_data(:enqueued, 60, :minutes)
70 | end
71 |
72 | # Hash of timeseries data over the last 24 hours for queued jobs
73 | def queued_per_hour
74 | timeseries_data(:enqueued, 24, :hours)
75 | end
76 | end
77 |
78 | module Resque::Plugins::JobStats::Timeseries::Performed
79 | include Resque::Plugins::JobStats::Timeseries::Common
80 |
81 | # Increments the performed count for the timestamp when job is complete
82 | def after_perform_job_stats_timeseries(*args)
83 | incr_timeseries(:performed)
84 | end
85 |
86 | # Hash of timeseries data over the last 60 minutes for completed jobs
87 | def performed_per_minute
88 | timeseries_data(:performed, 60, :minutes)
89 | end
90 |
91 | # Hash of timeseries data over the last 24 hours for completed jobs
92 | def performed_per_hour
93 | timeseries_data(:performed, 24, :hours)
94 | end
95 | end
96 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # resque-job-stats
2 |
3 | [](http://travis-ci.org/alanpeabody/resque-job-stats)
4 |
5 | Job centric stats for Resque.
6 |
7 | Stats are tracked per Job type (class/module) in addition to the worker based stats Resque provides.
8 |
9 | Stats tracked are:
10 |
11 | * Jobs performed
12 | * Jobs enqueued
13 | * Jobs failed
14 | * Duration of last x jobs completed
15 | * Average job duration over last 100 jobs completed
16 | * Longest job duration over last 100 jobs completed
17 | * Jobs enqueued as timeseries data (minute, hourly)
18 | * Jobs performed as timeseries data (minute, hourly)
19 |
20 | This information can be used to help track performance and diagnose specific bottlenecks.
21 |
22 | We are sending this information to Nagios for graphing and alerts (via a custom rake task).
23 |
24 | ## Installation
25 |
26 | Requires resque '>= 1.17', '< 3'
27 |
28 | In your Gemfile add:
29 |
30 | ```ruby
31 | gem 'resque-job-stats'
32 | ```
33 |
34 | ## Usage
35 |
36 | Simply extend your class
37 |
38 | ```ruby
39 | class MyJob
40 | extend Resque::Plugins::JobStats
41 |
42 | @queue = :my_job
43 | def self.perform(*args)
44 | # ..
45 | end
46 | end
47 | ```
48 |
49 | And you will have a set of keys starting with `'stats:jobs:my_job'` inside your Resque redis namespace.
50 |
51 | Alternatively you can include just the metric you wish to record.
52 |
53 | ```ruby
54 | class MyVerboseJob
55 | extend Resque::Plugins::JobStats::Performed
56 | extend Resque::Plugins::JobStats::Enqueued
57 | extend Resque::Plugins::JobStats::Failed
58 | extend Resque::Plugins::JobStats::Duration
59 | extend Resque::Plugins::JobStats::Timeseries::Enqueued
60 | extend Resque::Plugins::JobStats::Timeseries::Performed
61 |
62 | @queue = :my_job
63 | def self.perform(*args)
64 | # ...
65 | end
66 | end
67 | ```
68 |
69 | ### Duration module
70 |
71 | The duration module provides two metrics, the longest job and the job rolling avg.
72 |
73 | These are accessible via two singleton methods, `MyJob.job_rolling_avg` and `MyJob.longest_job`.
74 |
75 | By default the last 100 jobs durations are stored and used to provide the above metrics.
76 |
77 | You may set the number of jobs to include by setting the `@durations_recorded` variable.
78 |
79 |
80 | ```ruby
81 | class MyJob
82 | extend Resque::Plugins::JobStats::Duration
83 |
84 | @queue = :my_job
85 | @durations_recorded = 1000
86 |
87 | def self.perform(*payload)
88 | # ...
89 | end
90 | end
91 | ```
92 |
93 | ### Timeseries module
94 |
95 | The timeseries module provides timeseries counts of jobs performed. The metrics are rolling and kept for a period of time before being expired.
96 | The timestamp used for the timeseries data is UTC.
97 |
98 | ## Resque Web Tab
99 |
100 | The Resque web page for showing the statistics will only display jobs that extend Resque::Plugins::JobStats (in other words, just
101 | the jobs that include all of the metrics):
102 |
103 | ```ruby
104 | class MyJob
105 | extend Resque::Plugins::JobStats
106 | ...
107 | end
108 | ```
109 |
110 | The interface can be included in your app like this:
111 |
112 | ```ruby
113 | require 'resque-job-stats/server'
114 | ```
115 |
116 | If you wish to display only certain metrics, you can filter the metrics accordingly. The default metrics can be found in Resque::Plugins::JobStats::Statistic.
117 |
118 | ```ruby
119 | Resque::Server.job_stats_to_display = [:jobs_enqueued, :job_rolling_avg]
120 | ```
121 |
122 | ## Screenshots
123 |
124 | ### Overview
125 |
126 | 
127 |
128 | ### Individual Job Histories
129 |
130 | 
131 |
132 | ## TODO
133 |
134 | * Find clean way to add queue wait time stats.
135 |
136 | ## Contributing to resque-job-stats
137 |
138 | * Check out the latest master to make sure the feature hasn't been implemented or the bug hasn't been fixed yet
139 | * Check out the issue tracker to make sure someone already hasn't requested it and/or contributed it
140 | * Fork the project
141 | * Start a feature/bugfix branch
142 | * Commit and push until you are happy with your contribution
143 | * Make sure to add tests for it. This is important so I don't break it in a future version unintentionally.
144 | * Please try not to mess with the Rakefile, version, or history. If you want to have your own version, or is otherwise necessary, that is fine, but please isolate to its own commit so I can cherry-pick around it.
145 |
146 | ## Contributers
147 |
148 | * [damonmorgan](https://github.com/damonmorgan)
149 | * [unclebilly](https://github.com/unclebilly)
150 | * [jesperronn](https://github.com/jesperronn)
151 |
152 | ## Copyright
153 |
154 | Copyright (c) 2011-2012 Alan Peabody. See LICENSE.txt for further details.
155 |
156 |
--------------------------------------------------------------------------------
/test/test_job_stats.rb:
--------------------------------------------------------------------------------
1 | require 'helper'
2 |
3 | class BaseJob
4 | @queue = :test
5 |
6 | def self.perform(sleep_time=0.01)
7 | sleep sleep_time
8 | end
9 | end
10 |
11 | class SimpleJob < BaseJob
12 | extend Resque::Plugins::JobStats
13 | @queue = :test
14 | end
15 |
16 | class FailJob < BaseJob
17 | extend Resque::Plugins::JobStats::Failed
18 | extend Resque::Plugins::JobStats::History
19 | @queue = :test
20 |
21 | def self.perform(*args)
22 | raise 'fail'
23 | end
24 | end
25 |
26 | class CustomDurJob < BaseJob
27 | extend Resque::Plugins::JobStats::Duration
28 | @queue = :test
29 | @durations_recorded = 5
30 | end
31 |
32 | class CustomHistJob < BaseJob
33 | extend Resque::Plugins::JobStats::History
34 | @queue = :test
35 | @histories_recordable = 5
36 | end
37 |
38 | class TestResqueJobStats < MiniTest::Unit::TestCase
39 |
40 | def setup
41 | # Ensure empty redis for each test
42 | Resque.redis.flushdb
43 | @worker = Resque::Worker.new(:test)
44 | end
45 |
46 | def test_lint
47 | Resque::Plugin.lint(Resque::Plugins::JobStats)
48 | assert_equal true, true
49 | rescue => e
50 | assert_equal false, e
51 | end
52 |
53 | def test_jobs_performed
54 | assert_equal 'stats:jobs:SimpleJob:performed', SimpleJob.jobs_performed_key
55 | SimpleJob.jobs_performed = 0
56 | 3.times do
57 | Resque.enqueue(SimpleJob)
58 | @worker.work(0)
59 | end
60 | assert_equal 3, SimpleJob.jobs_performed
61 | end
62 |
63 | def test_jobs_enqueued
64 | assert_equal 'stats:jobs:SimpleJob:enqueued', SimpleJob.jobs_enqueued_key
65 | SimpleJob.jobs_enqueued = 0
66 | 3.times do
67 | Resque.enqueue(SimpleJob)
68 | @worker.work(0)
69 | end
70 | assert_equal 3, SimpleJob.jobs_enqueued
71 | end
72 |
73 | def test_jobs_failed
74 | assert_equal 'stats:jobs:FailJob:failed', FailJob.jobs_failed_key
75 | FailJob.jobs_failed = 0
76 | 3.times do
77 | Resque.enqueue(FailJob)
78 | @worker.work(0)
79 | end
80 | assert_equal 3, FailJob.jobs_failed
81 | end
82 |
83 | def test_duration
84 | assert_equal 'stats:jobs:SimpleJob:duration', SimpleJob.jobs_duration_key
85 | SimpleJob.reset_job_durations
86 | assert_equal 0.0, SimpleJob.job_rolling_avg
87 | assert_equal 0.0, SimpleJob.longest_job
88 |
89 | 3.times do |i|
90 | d = (i + 1)/10.0
91 | Resque.enqueue(SimpleJob,d)
92 | @worker.work(0)
93 | end
94 |
95 | assert_in_delta 0.3, SimpleJob.job_durations[0], 0.01
96 | assert_in_delta 0.2, SimpleJob.job_durations[1], 0.01
97 | assert_in_delta 0.1, SimpleJob.job_durations[2], 0.01
98 | assert_in_delta 0.3, SimpleJob.longest_job, 0.01
99 | assert_in_delta 0.2, SimpleJob.job_rolling_avg, 0.01
100 | end
101 |
102 | def test_custom_duration
103 | CustomDurJob.reset_job_durations
104 |
105 | 2.times do
106 | Resque.enqueue(CustomDurJob,1.0)
107 | @worker.work(0)
108 | end
109 |
110 | 5.times do
111 | Resque.enqueue(CustomDurJob,0.1)
112 | @worker.work(0)
113 | end
114 |
115 | assert_in_delta 0.1, CustomDurJob.longest_job, 0.01
116 | assert_in_delta 0.1, CustomDurJob.job_rolling_avg, 0.01
117 | end
118 |
119 | def test_perform_timeseries
120 | time = SimpleJob.timestamp
121 | 3.times do
122 | Resque.enqueue(SimpleJob)
123 | @worker.work(0)
124 | end
125 | assert_equal 3, SimpleJob.performed_per_minute[time]
126 | assert_equal 0, SimpleJob.performed_per_minute[(time - 60)]
127 |
128 | assert_equal 3, SimpleJob.performed_per_hour[time]
129 | assert_equal 0, SimpleJob.performed_per_hour[(time - 3600)]
130 | end
131 |
132 | def test_enqueue_timeseries
133 | time = SimpleJob.timestamp
134 | Timecop.freeze(time)
135 | Resque.enqueue(SimpleJob,0)
136 | Timecop.freeze(time + 60)
137 | @worker.work(0)
138 | assert_equal 1, SimpleJob.queued_per_minute[time]
139 | assert_equal 0, SimpleJob.queued_per_minute[(time + 60)]
140 | assert_equal 1, SimpleJob.performed_per_minute[(time + 60)]
141 | Timecop.return
142 | end
143 |
144 | def test_measured_jobs
145 | assert_equal [SimpleJob], Resque::Plugins::JobStats.measured_jobs
146 | end
147 |
148 | def test_history
149 | assert_equal 'stats:jobs:SimpleJob:history', SimpleJob.jobs_history_key
150 | SimpleJob.reset_job_histories
151 | assert_equal 0, SimpleJob.job_histories.count
152 | assert_equal 0, SimpleJob.histories_recorded
153 |
154 | 3.times do |i|
155 | d = (i + 1)/10.0
156 | Resque.enqueue(SimpleJob,d)
157 | @worker.work(0)
158 | end
159 |
160 | assert_equal 3, SimpleJob.job_histories.count
161 | assert_equal 3, SimpleJob.histories_recorded
162 | assert_equal 1, SimpleJob.job_histories(1,1).count
163 |
164 | assert SimpleJob.job_histories[0]["success"]
165 | assert_in_delta 0.3, SimpleJob.job_histories[0]["args"][0], 0.01
166 | assert_in_delta 0.3, SimpleJob.job_histories[0]["duration"], 0.01
167 |
168 | assert SimpleJob.job_histories[1]["success"]
169 | assert_in_delta 0.2, SimpleJob.job_histories[1]["args"][0], 0.01
170 | assert_in_delta 0.2, SimpleJob.job_histories[1]["duration"], 0.01
171 |
172 | assert SimpleJob.job_histories[2]["success"]
173 | assert_in_delta 0.1, SimpleJob.job_histories[2]["args"][0], 0.01
174 | assert_in_delta 0.1, SimpleJob.job_histories[2]["duration"], 0.01
175 | end
176 |
177 | def test_custom_history
178 | CustomHistJob.reset_job_histories
179 |
180 | 2.times do
181 | Resque.enqueue(CustomHistJob,1.0)
182 | @worker.work(0)
183 | end
184 |
185 | 5.times do
186 | Resque.enqueue(CustomHistJob,0.1)
187 | @worker.work(0)
188 | end
189 |
190 | assert_equal 5, CustomHistJob.job_histories.count
191 | assert_in_delta 0.1, CustomHistJob.job_histories.first["args"][0], 0.01
192 | assert_in_delta 0.1, CustomHistJob.job_histories.last["args"][0], 0.01
193 | end
194 |
195 | def test_failure_history
196 | FailJob.reset_job_histories
197 |
198 | 2.times do
199 | Resque.enqueue(FailJob)
200 | @worker.work(0)
201 | end
202 |
203 | assert_equal 2, FailJob.job_histories.count
204 | assert_equal 0, FailJob.job_histories.first["args"].count
205 | assert ! FailJob.job_histories.first["success"]
206 | assert_equal "fail", FailJob.job_histories.first["exception"]["name"]
207 | end
208 | end
209 |
--------------------------------------------------------------------------------
/test/test_server.rb:
--------------------------------------------------------------------------------
1 | require File.expand_path(File.join(File.dirname(__FILE__), 'helper'))
2 | require 'resque-job-stats/server'
3 |
4 | # A pretend job that has all of the statistics we want to display (i.e. extends
5 | # Resque::Plugins::JobStats)
6 | class AnyJobber
7 | class << self
8 | def jobs_enqueued
9 | 111
10 | end
11 |
12 | def jobs_performed
13 | 12345
14 | end
15 |
16 | def jobs_failed
17 | 0
18 | end
19 |
20 | def job_rolling_avg
21 | 0.3333232
22 | end
23 |
24 | def longest_job
25 | 0.455555
26 | end
27 | end
28 | end
29 |
30 | class YetAnotherJobber < AnyJobber
31 | end
32 |
33 | # mocked job with history
34 | class HistoricJob
35 | include Resque::Plugins::JobStats::History
36 |
37 | def self.job_histories(start=0,limit=2)
38 | [{"run_at" => "Thu Jan 02 03:45:01 -0700 2012",
39 | "duration" => 1.2345,
40 | "args" => [1, 2, "3"],
41 | "success" => true},
42 | {"run_at" => "Thu Jan 02 03:45:01 -0700 2011",
43 | "duration" => 6.7890,
44 | "args" => ["a", "b"],
45 | "success" => false,
46 | "exception" => {"name" => "bad stuff", "backtrace" => ["a", "b"]}}][start,limit]
47 | end
48 |
49 | def self.histories_recorded
50 | 2
51 | end
52 |
53 | def self.jobs_performed
54 | 1
55 | end
56 | end
57 |
58 | class MyServer
59 | def self.job_stats_to_display
60 | [:jobs_enqueued]
61 | end
62 |
63 | include Resque::Plugins::JobStats::Server::Helpers
64 | end
65 |
66 | ENV['RACK_ENV'] = 'test'
67 | class TestServer < MiniTest::Unit::TestCase
68 | include Rack::Test::Methods
69 |
70 | def setup
71 | Resque::Server.job_stats_to_display = Resque::Plugins::JobStats::Statistic::DEFAULT_STATS
72 | @server = MyServer.new
73 | end
74 |
75 | def app
76 | Resque::Server
77 | end
78 |
79 | def test_job_stats
80 | Resque::Plugins::JobStats.stub :measured_jobs, [AnyJobber] do
81 | get '/job_stats'
82 | assert_equal 200, last_response.status, last_response.body
83 | assert last_response.body =~ /\s*AnyJobber\s*<\/td>/, "job name was not found"
84 | assert last_response.body.include?(" | 111 | "), "jobs_enqueued was not found"
85 | assert last_response.body.include?("12345 | "), "jobs_performed was not found"
86 | assert last_response.body.include?(" | "), "jobs_failed was not found"
87 | assert last_response.body.include?("0.33s | "), "job_rolling_avg was not found"
88 | assert last_response.body.include?("0.46s | "), "longest_job was not found"
89 | end
90 | end
91 |
92 | def test_job_stats_filtered
93 | Resque::Server.job_stats_to_display = [:longest_job]
94 | Resque::Plugins::JobStats.stub :measured_jobs, [AnyJobber] do
95 | get '/job_stats'
96 | assert_equal 200, last_response.status, last_response.body
97 | assert last_response.body =~ /\s*AnyJobber\s*<\/td>/, "job name was not found"
98 | assert !last_response.body.include?(" | 111 | "), "jobs_enqueued was not found"
99 | assert !last_response.body.include?("12345 | "), "jobs_performed was not found"
100 | assert !last_response.body.include?(" | "), "jobs_failed was not found"
101 | assert !last_response.body.include?("0.33s | "), "job_rolling_avg was not found"
102 | assert last_response.body.include?("0.46s | "), "longest_job was not found"
103 | end
104 | end
105 |
106 | def test_stat_header
107 | assert_equal "Jobs enqueued | ", @server.stat_header(:jobs_enqueued)
108 | assert_equal nil, @server.stat_header(:FOOOOOOO)
109 | end
110 |
111 | def test_display_stat?
112 | assert !@server.display_stat?(:jobs_barfing)
113 | assert @server.display_stat?(:jobs_enqueued)
114 | end
115 |
116 | def test_job_sorting
117 | Resque::Plugins::JobStats.stub :measured_jobs, [YetAnotherJobber, AnyJobber] do
118 | get '/job_stats'
119 | assert_equal 200, last_response.status, last_response.body
120 | assert(last_response.body =~ /AnyJobber(.|\n)+YetAnotherJobber/, "AnyJobber should be found before YetAnotherJobber")
121 | end
122 | end
123 |
124 | def test_tabs
125 | assert app.tabs.include?("Job_Stats"), "The tab should be in resque's server"
126 | end
127 |
128 | def test_job_stats_history_link
129 | Resque::Server.job_stats_to_display = [:jobs_performed]
130 | Resque::Plugins::JobStats.stub :measured_jobs, [HistoricJob,AnyJobber] do
131 | get "/job_stats"
132 | assert_equal 200, last_response.status, last_response.body
133 | assert last_response.body.include?("[history]"), "history link not found"
134 | assert !last_response.body.include?("[history]"), "unexpected history link"
135 | end
136 | end
137 |
138 | def test_history
139 | Resque::Plugins::JobStats.stub :measured_jobs, [HistoricJob] do
140 | get "/job_history/HistoricJob"
141 | assert_equal 200, last_response.status, last_response.body
142 | assert last_response.body.include?("HistoricJob
"), "job name was not found"
143 | assert last_response.body.include?("Thu Jan 02 03:45:01 -0700 2012 | "), "start time not found"
144 | assert last_response.body.include?("[1, 2, \"3\"] | "), "args not found"
145 | assert last_response.body.include?("✓ | "), "success marker not found"
146 | end
147 | end
148 |
149 | def test_history_pagination
150 | Resque::Plugins::JobStats.stub :measured_jobs, [HistoricJob] do
151 | get "/job_history/HistoricJob?start=1"
152 | assert_equal 200, last_response.status, last_response.body
153 | assert !last_response.body.include?("✓ | "), "success marker unexpected"
154 | assert last_response.body.include?("✗ | "), "failure marker not found"
155 | assert last_response.body.include?("✓ | "), "success marker not found"
160 | assert !last_response.body.include?("✗ | "), "failure marker unexpected"
161 | assert last_response.body.include?("