├── .travis.yml
├── lib
    ├── resque-cleaner.rb
    ├── resque_cleaner
    │   ├── server
    │   │   └── views
    │   │   │   ├── cleaner_exec.erb
    │   │   │   ├── _limiter.erb
    │   │   │   ├── cleaner.erb
    │   │   │   ├── _stats.erb
    │   │   │   ├── _cleaner_styles.erb
    │   │   │   ├── _paginate.erb
    │   │   │   └── cleaner_list.erb
    │   └── server.rb
    └── resque_cleaner.rb
├── .gitignore
├── misc
    ├── resque-cleaner-list.png
    ├── resque-cleaner-main.png
    └── resque-cleaner-list-with-regex.png
├── Gemfile
├── Rakefile
├── LICENSE
├── resque-cleaner.gemspec
├── test
    ├── resque_web_test.rb
    ├── test_helper.rb
    ├── redis-test.conf
    └── resque_cleaner_test.rb
├── CHANGELOG.md
├── examples
    └── failure_job.json
└── README.md


/.travis.yml:
--------------------------------------------------------------------------------
1 | services:
2 |   - redis-server
3 | 


--------------------------------------------------------------------------------
/lib/resque-cleaner.rb:
--------------------------------------------------------------------------------
1 | require 'resque_cleaner'
2 | 


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
1 | .DS_Store
2 | local_note.md
3 | Gemfile.lock
4 | 


--------------------------------------------------------------------------------
/misc/resque-cleaner-list.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ono/resque-cleaner/HEAD/misc/resque-cleaner-list.png


--------------------------------------------------------------------------------
/misc/resque-cleaner-main.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ono/resque-cleaner/HEAD/misc/resque-cleaner-main.png


--------------------------------------------------------------------------------
/misc/resque-cleaner-list-with-regex.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/ono/resque-cleaner/HEAD/misc/resque-cleaner-list-with-regex.png


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
 1 | source 'https://rubygems.org'
 2 | 
 3 | gemspec
 4 | 
 5 | group :test do
 6 |   gem 'rake'
 7 |   gem 'json'
 8 |   gem 'timecop'
 9 |   gem 'rack-test'
10 | end
11 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/cleaner_exec.erb:
--------------------------------------------------------------------------------
1 | <%= erb File.read(ResqueCleaner::Server.erb_path("_cleaner_styles.erb")) %>
2 | 
3 | <div class="cleaner">
4 |   <p class="message">Processed <%= @count %> jobs.</p>
5 |   <p class="back_to_list">
6 |   <a href="<%=@list_url%>">Back to List</a>
7 |   </p>
8 | </div>
9 | 


--------------------------------------------------------------------------------
/Rakefile:
--------------------------------------------------------------------------------
 1 | #
 2 | # Setup
 3 | #
 4 | 
 5 | # load 'tasks/redis.rake'
 6 | #require 'rake/testtask'
 7 | 
 8 | $LOAD_PATH.unshift 'lib'
 9 | #require 'resque/tasks'
10 | 
11 | def command?(command)
12 |   system("type #{command} > /dev/null 2>&1")
13 | end
14 | 
15 | 
16 | #
17 | # Tests
18 | #
19 | 
20 | task :default => :test
21 | 
22 | desc "Run the test suite"
23 | task :test do
24 |   rg = command?(:rg)
25 |   Dir['test/**/*_test.rb'].each do |f|
26 |     rg ? sh("rg #{f}") : ruby(f)
27 |   end
28 | end
29 | 
30 | if command? :kicker
31 |   desc "Launch Kicker (like autotest)"
32 |   task :kicker do
33 |     puts "Kicking... (ctrl+c to cancel)"
34 |     exec "kicker -e rake test lib examples"
35 |   end
36 | end
37 | 
38 | 
39 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/_limiter.erb:
--------------------------------------------------------------------------------
 1 |   <div class="warning">
 2 |     <p>
 3 |     There are more than <%= @cleaner.limiter.maximum%> jobs. ResqueCleaner handles only recent <%= @cleaner.limiter.maximum %> jobs. See the Limiter section on <a href="https://github.com/ono/resque-cleaner" target="_blank">README</a> for more detail.
 4 |     </p>
 5 |     <p>
 6 |       <form method="post" action="cleaner_stale">
 7 |         <input type="hidden" name="action" value="clear_stale" />
 8 |         <input type="submit" onclick="return confirm('Are you sure?')" value="Clear all jobs older than <%= @cleaner.limiter.count %>th job."/>
 9 |         (<%= @cleaner.failure.count - @cleaner.limiter.maximum %> jobs will be cleared)
10 |       </form>
11 |     </p>
12 |   </div>
13 | 
14 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/cleaner.erb:
--------------------------------------------------------------------------------
 1 | <%= erb File.read(ResqueCleaner::Server.erb_path("_cleaner_styles.erb")) %>
 2 | 
 3 | <div class="cleaner">
 4 |   <div class="title clearfix">
 5 |     <h1>Group by class</h1>
 6 |   </div>
 7 | 
 8 |   <%= erb(
 9 |         File.read(ResqueCleaner::Server.erb_path("_stats.erb")),
10 |         locals: {q: 'c', type: 'klass'}
11 |       ) %>
12 | 
13 |   <div class="title clearfix">
14 |     <h1>Group by exception</h1>
15 |   </div>
16 | 
17 |   <%= erb(
18 |         File.read(ResqueCleaner::Server.erb_path("_stats.erb")),
19 |         locals: {q: 'ex', type: 'exception'}
20 |       ) %>
21 | 
22 |   <% if @cleaner.limiter.on? %>
23 |     <%= erb File.read(ResqueCleaner::Server.erb_path("_limiter.erb")) %>
24 |   <% end %>
25 | </div>
26 | 


--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2010 Tatsuya Ono
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining
 4 | a copy of this software and associated documentation files (the
 5 | "Software"), to deal in the Software without restriction, including
 6 | without limitation the rights to use, copy, modify, merge, publish,
 7 | distribute, sublicense, and/or sell copies of the Software, and to
 8 | permit persons to whom the Software is furnished to do so, subject to
 9 | the following conditions:
10 | 
11 | The above copyright notice and this permission notice shall be
12 | included in all copies or substantial portions of the Software.
13 | 
14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
21 | 


--------------------------------------------------------------------------------
/resque-cleaner.gemspec:
--------------------------------------------------------------------------------
 1 | Gem::Specification.new do |s|
 2 |   s.name              = "resque-cleaner"
 3 |   s.version           = "0.4.1"
 4 |   s.date              = Time.now.strftime('%Y-%m-%d')
 5 |   s.summary           = "Resque plugin cleaning up failed jobs."
 6 |   s.homepage          = "https://github.com/ono/resque-cleaner"
 7 |   s.email             = "ononoma@gmail.com"
 8 |   s.authors           = [ "Tatsuya Ono" ]
 9 |   s.license           = "MIT"
10 |   s.required_ruby_version = '>= 1.9.3'
11 | 
12 |   s.files             = %w( README.md CHANGELOG.md Rakefile LICENSE )
13 |   s.files            += Dir.glob("lib/**/*")
14 |   s.files            += Dir.glob("test/**/*")
15 | 
16 |   s.require_paths     = ["lib"]
17 | 
18 |   s.extra_rdoc_files  = [ "LICENSE", "README.md", "CHANGELOG.md" ]
19 |   s.rdoc_options      = ["--charset=UTF-8"]
20 | 
21 |   s.add_dependency "resque"
22 | 
23 |   s.add_development_dependency "minitest", "~> 5.0"
24 |   s.add_development_dependency "rack-test", "~> 0.6.0"
25 | 
26 |   s.description = <<DESCRIPTION
27 |     resque-cleaner maintains the cleanliness of failed jobs on Resque.
28 | DESCRIPTION
29 | end
30 | 
31 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/_stats.erb:
--------------------------------------------------------------------------------
 1 | <table class="class_list">
 2 |   <tr>
 3 |     <th><%= type.gsub('klass','class').capitalize %></th>
 4 |     <th>Failed</th>
 5 |     <th>In last 1 hour</th>
 6 |     <th>In last 3 hours</th>
 7 |     <th>In last 24 hours</th>
 8 |     <th>In last 3 days</th>
 9 |     <th>In last 7 days</th>
10 |   </tr>
11 |   <% @stats[type.to_sym].each do |field,count| %>
12 |     <tr>
13 |       <% filter = "#{q}=#{URI.encode(field)}" %>
14 |       <td><%= field %></td>
15 |       <td class="number">
16 |         <a href="cleaner_list?<%=filter%>"><%= count[:total] %></a>
17 |       </td>
18 |       <td class="number">
19 |         <a href="cleaner_list?<%=filter%>&f=1"><%= count[:h1] %></a>
20 |       </td>
21 |       <td class="number">
22 |         <a href="cleaner_list?<%=filter%>&f=3"><%= count[:h3] %></a>
23 |       </td>
24 |       <td class="number">
25 |         <a href="cleaner_list?<%=filter%>&f=24"><%= count[:d1] %></a>
26 |       </td>
27 |       <td class="number">
28 |         <a href="cleaner_list?<%=filter%>&f=72"><%= count[:d3] %></a>
29 |       </td>
30 |       <td class="number">
31 |         <a href="cleaner_list?<%=filter%>&f=168"><%= count[:d7] %></a>
32 |       </td>
33 |     </tr>
34 |   <% end %>
35 |   <tr class="total">
36 |     <td>Total</td>
37 |     <td class="number"><a href="cleaner_list"><%= @total[:total] %></a></td>
38 |     <td class="number"><a href="cleaner_list?f=1"><%= @total[:h1] %></a></td>
39 |     <td class="number"><a href="cleaner_list?f=3"><%= @total[:h3] %></a></td>
40 |     <td class="number"><a href="cleaner_list?f=24"><%= @total[:d1] %></a></td>
41 |     <td class="number"><a href="cleaner_list?f=72"><%= @total[:d3] %></a></td>
42 |     <td class="number"><a href="cleaner_list?f=168"><%= @total[:d7] %></a></td>
43 |   </tr>
44 | </table>
45 | 


--------------------------------------------------------------------------------
/test/resque_web_test.rb:
--------------------------------------------------------------------------------
 1 | require File.expand_path(File.dirname(__FILE__) + '/test_helper')
 2 | 
 3 | require 'digest/sha1'
 4 | require 'json'
 5 | require 'rack/test'
 6 | 
 7 | class Minitest::Spec
 8 |   include Rack::Test::Methods
 9 |   def app
10 |     Resque::Server.new
11 |   end
12 | end
13 | 
14 | def setup_some_failed_jobs
15 |   Resque.redis.flushall
16 | 
17 |   @worker = Resque::Worker.new(:jobs,:jobs2)
18 | 
19 |   create_and_process_jobs :jobs, @worker, 1, Time.now, BadJobWithSyntaxError, "great_args"
20 | 
21 |   10.times {|i|
22 |     create_and_process_jobs :jobs, @worker, 1, Time.now, BadJob, "test_#{i}"
23 |   }
24 | 
25 |   @cleaner = Resque::Plugins::ResqueCleaner.new
26 |   @cleaner.print_message = false
27 | end
28 | 
29 | describe "resque-web" do
30 |   before do
31 |     setup_some_failed_jobs
32 |   end
33 | 
34 |   it "#cleaner should respond with success" do
35 |     get "/cleaner"
36 |     assert last_response.body.include?('BadJob')
37 |     assert last_response.body =~ /\bException\b/
38 |   end
39 | 
40 |   it "#cleaner_list should respond with success" do
41 |     get "/cleaner_list"
42 |     assert last_response.ok?, last_response.errors
43 |   end
44 | 
45 |   it '#cleaner_list shows the failed jobs' do
46 |     get "/cleaner_list"
47 |     assert last_response.body.include?('BadJob')
48 |   end
49 | 
50 |   it "#cleaner_list shows the failed jobs when we use a select_by_regex" do
51 |     get "/cleaner_list", :regex => "BadJob*"
52 |     assert last_response.body.include?('"BadJobWithSyntaxError"')
53 |     assert last_response.body.include?('"BadJob"')
54 |   end
55 | 
56 | 
57 |   it '#cleaner_exec clears job' do
58 |     post "/cleaner_exec", :action => "clear", :sha1 => Digest::SHA1.hexdigest(@cleaner.select[0].to_json)
59 |     assert_equal 10, @cleaner.select.size
60 |   end
61 |   it "#cleaner_dump should respond with success" do
62 |     get "/cleaner_dump"
63 |     assert last_response.ok?, last_response.errors
64 |   end
65 | end
66 | 
67 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/_cleaner_styles.erb:
--------------------------------------------------------------------------------
 1 | <style type="text/css" media="screen">
 2 |   #main .cleaner .title h1 { float: left }
 3 | 
 4 |   #main .cleaner .sub_header {color: #888; font-size: 80%; font-weight: bold}
 5 | 
 6 |   #main .cleaner .control_panel {
 7 |     margin: 5px 0 10px; padding: 5px; background: #eee; display: inline-block;
 8 |     -webkit-border-radius:5px; border:1px solid #ccc;
 9 |   }
10 | 
11 |   #main .cleaner #exec { color: black; }
12 |   #main .cleaner #exec select { margin-right: 20px }
13 |   #main .cleaner #exec a { margin-right: 5px}
14 |   #main .cleaner #exec a.disabled { color: #aaa}
15 | 
16 |   #main .cleaner table.class_list { width: auto }
17 |   #main .cleaner td.number { text-align: right }
18 |   #main .cleaner tr.total { border-top: 3px double #ccc}
19 | 
20 |   #main .cleaner form { float: none; margin: 0}
21 | 
22 |   #main .cleaner .warning { margin: 30px 0; padding: 10px; background: #fdd; display: inline-block; }
23 | 
24 |   #main .clearfix:after {
25 |     content: ".";
26 |     display: block;
27 |     height: 0;
28 |     clear: both;
29 |     visibility: hidden;
30 |   }
31 |   #main .cleaner ul.failed {
32 |     margin-bottom: 10px
33 |   }
34 | 
35 |   #main .cleaner .list_info{
36 |     color: #999; text-align: center;
37 |   }
38 |   #main .cleaner .list_info a{
39 |     color: #666
40 |   }
41 |   #main .cleaner .list_info .list_summary{
42 |     vertical-align: baseline;
43 |     display: inline-block;
44 |     margin: 0; padding:0;
45 |   }
46 |   #main .cleaner ul.pagination {
47 |     display: inline-block;
48 |     vertical-align: baseline;
49 |     margin:0; padding:0;
50 |   }
51 |   #main .cleaner ul.pagination li {
52 |     list-style:none; float: left; padding: 0 3px;
53 |   }
54 |   #main .cleaner ul.pagination li.current {
55 |     color: #333; font-weight: bold;
56 |   }
57 |   #main .cleaner ul.pagination li.summary{
58 |     padding-right: 10px
59 |   }
60 |   #main .cleaner div.dump{
61 |     float: right
62 |   }
63 | </style>
64 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/_paginate.erb:
--------------------------------------------------------------------------------
 1 | <div class="list_info clearfix">
 2 | 
 3 |   <ul class="pagination">
 4 |     <li class="summary">
 5 |       <%= @paginate.first_index%> - <%= @paginate.last_index%> / <%= @paginate.total_size%>
 6 |     </li>
 7 |     <% if @paginate.max_page>1 %>
 8 |       <% if @paginate.first_page? %>
 9 |         <li class="prev off">&lt;&lt;newer</li>
10 |       <% else %>
11 |         <li class="prev"><a href="<%= @paginate.page_url(:prev)%>">&lt;&lt;newer</a></li>
12 |       <% end %>
13 | 
14 |       <% 1.upto(6<@paginate.max_page ? 6 : @paginate.max_page) do |pg| %>
15 |         <% if @paginate.page==pg %>
16 |           <li class="current"><%=pg%></li>
17 |         <% else %>
18 |           <li><a href="<%= @paginate.page_url(pg)%>"><%=pg%></a></li>
19 |         <% end %>
20 |       <% end %>
21 | 
22 |       <% if @paginate.max_page > 6 %>
23 |         <% if @paginate.max_page!=@paginate.page %>
24 |           <% if @paginate.page>7 %>
25 |             <li class="off">...</li>
26 |           <% end %>
27 |           <% if @paginate.page>6 %>
28 |             <li class="current"><%=@paginate.page%></li>
29 |           <% end %>
30 |         <% end %>
31 |         <% if @paginate.max_page>7 && @paginate.page+1!=@paginate.max_page %>
32 |           <li class="off">...</li>
33 |         <% end %>
34 | 
35 |         <% if @paginate.max_page==@paginate.page %>
36 |           <li class="current"><%=@paginate.max_page%></li>
37 |         <% else %>
38 |           <li><a href="<%= @paginate.page_url(@paginate.max_page)%>"><%=@paginate.max_page%></a></li>
39 |         <% end %>
40 |       <% end %>
41 | 
42 |       <% if @paginate.last_page? %>
43 |         <li class="next off">older&gt;&gt;</li>
44 |       <% else %>
45 |         <li class="next"><a href="<%= @paginate.page_url(:next)%>">older&gt;&gt;</a></li>
46 |       <% end %>
47 |     <% end %>
48 | 
49 |   </ul>
50 |     <div class="dump">
51 |         <a href="<%= @dump_url%>" target="_blank">Dump</a>
52 |     </div>
53 | </div>
54 | 
55 | 


--------------------------------------------------------------------------------
/CHANGELOG.md:
--------------------------------------------------------------------------------
 1 | ## 0.4.1 (2018-01-25)
 2 | 
 3 | * Remove Requeue version lock (#46)
 4 | 
 5 | ## 0.4.0 (2017-11-20)
 6 | 
 7 | * Render CSS as ERB partial to make Rails 5 compatible (#45)
 8 | 
 9 | ## 0.3.2 (2016-03-09)
10 | 
11 | * Handle failure JSON with nil on payload safely (#40)
12 | 
13 | ## 0.3.1 (2016-01-28)
14 | 
15 | * Bug fix: fix the issue regex is not applied on bulk clearance (#36)
16 | 
17 | ## 0.3.0 (2014-05-27)
18 | 
19 | * Search by regex (#27)
20 | * Show stats by exception (#29)
21 | * Stop supporting ruby 1.8.x and 1.9.2
22 | * Escape query parameters (#30)
23 | 
24 | ## 0.2.12 (2013-12-03)
25 | 
26 | * Remove Resque::Helpers include (#23)
27 | * Tweak Gemspec (#24)
28 | * Don't use yaml format to show arguments
29 | * Tweak README
30 | 
31 | ## 0.2.11 (2013-07-19)
32 | 
33 | * Use transaction for retry-and-clear (#22).
34 | * Fix for CI
35 | 
36 | ## 0.2.10 (2012-10-15)
37 | 
38 | * Bug fix: use URL helper to support resque-web which is not hosted under '/'.
39 | 
40 | ## 0.2.9 (2012-05-10)
41 | 
42 | * Make limiter configurable on resque-web.
43 | 
44 | ## 0.2.8 (2012-03-19)
45 | 
46 | * UI tweak (#15)
47 | 
48 | ## 0.2.7 (2012-01-17)
49 | 
50 | * Support Travis CI
51 | * Support failure jobs without payload information (#11).
52 | 
53 | ## 0.2.6 (2011-08-23)
54 | 
55 | * Follow the latest resque for date format.
56 | 
57 | ## 0.2.5 (2011-05-05)
58 | 
59 | * BUGFIX: Pagination has been broken.
60 | * Dump a list as JSON format.
61 | 
62 | ## 0.2.4 (2011-04-19)
63 | 
64 | * BUGFIX: "Select All" ignores exception filter.
65 | 
66 | ## 0.2.3 (2011-04-11)
67 | 
68 | * Exception filter.
69 | 
70 | ## 0.2.2 (2011-04-07)
71 | 
72 | * Changed a way to load yajl/json\_gem.
73 | 
74 | ## 0.2.1 (2011-04-06)
75 | 
76 | * BUGFIX: Bulk action didn't work properly when you select all jobs.
77 | * Removed absolute paths from html.
78 | 
79 | ## 0.2.0 (2011-04-06)
80 | 
81 | * Extended with resque-web
82 | 
83 | ## 0.1.1 (2010-12-30)
84 | 
85 | * Fixed for ruby 1.9.2
86 | * Fixed a bug on #retried? method
87 | 
88 | ## 0.1.0 (2010-11-24)
89 | 
90 | * First official release
91 | 
92 | 
93 | 


--------------------------------------------------------------------------------
/examples/failure_job.json:
--------------------------------------------------------------------------------
 1 | {"failed_at": "2009/03/13 00:00:00",
 2 |  "payload": {"args": ["Johnson"], "class": "BadJob"},
 3 |  "queue": "jobs",
 4 |  "worker": "localhost:7327:jobs,jobs2",
 5 |  "exception": "RuntimeError",
 6 |  "error": "Bad job, Johnson",
 7 |  "backtrace": 
 8 |   ["./test/test_helper.rb:108:in `perform'",
 9 |    "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/job.rb:133:in `perform'",
10 |    "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/worker.rb:157:in `perform'",
11 |    "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/worker.rb:124:in `work'",
12 |    "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/worker.rb:110:in `loop'",
13 |    "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/worker.rb:110:in `work'",
14 |    "test/resque_cleaner_test.rb:9:in `create_and_process_jobs'",
15 |    "/opt/local/lib/ruby/gems/1.8/gems/timecop-0.3.5/lib/timecop/timecop.rb:91:in `travel'",
16 |    "/opt/local/lib/ruby/gems/1.8/gems/timecop-0.3.5/lib/timecop/timecop.rb:49:in `send'",
17 |    "/opt/local/lib/ruby/gems/1.8/gems/timecop-0.3.5/lib/timecop/timecop.rb:49:in `freeze'",
18 |    "test/resque_cleaner_test.rb:5:in `create_and_process_jobs'",
19 |    "test/resque_cleaner_test.rb:28:in `setup'",
20 |    "/opt/local/lib/ruby/1.8/test/unit/testcase.rb:77:in `run'",
21 |    "/opt/local/lib/ruby/1.8/test/unit/testsuite.rb:34:in `run'",
22 |    "/opt/local/lib/ruby/1.8/test/unit/testsuite.rb:33:in `each'",
23 |    "/opt/local/lib/ruby/1.8/test/unit/testsuite.rb:33:in `run'",
24 |    "/opt/local/lib/ruby/1.8/test/unit/testsuite.rb:34:in `run'",
25 |    "/opt/local/lib/ruby/1.8/test/unit/testsuite.rb:33:in `each'",
26 |    "/opt/local/lib/ruby/1.8/test/unit/testsuite.rb:33:in `run'",
27 |    "/opt/local/lib/ruby/1.8/test/unit/ui/testrunnermediator.rb:46:in `run_suite'",
28 |    "/opt/local/lib/ruby/1.8/test/unit/ui/console/testrunner.rb:67:in `start_mediator'",
29 |    "/opt/local/lib/ruby/1.8/test/unit/ui/console/testrunner.rb:41:in `start'",
30 |    "/opt/local/lib/ruby/1.8/test/unit/ui/testrunnerutilities.rb:29:in `run'",
31 |    "/opt/local/lib/ruby/1.8/test/unit/autorunner.rb:216:in `run'",
32 |    "/opt/local/lib/ruby/1.8/test/unit/autorunner.rb:12:in `run'",
33 |    "./test/test_helper.rb:41",
34 |    "test/resque_cleaner_test.rb:3"]
35 | }
36 | 


--------------------------------------------------------------------------------
/test/test_helper.rb:
--------------------------------------------------------------------------------
  1 | # Mostly copied from Resque in order to have similar test environment.
  2 | # https://github.com/defunkt/resque/blob/master/test/test_helper.rb
  3 | 
  4 | dir = File.dirname(File.expand_path(__FILE__))
  5 | $LOAD_PATH.unshift dir + '/../lib'
  6 | $TESTING = true
  7 | require 'rubygems'
  8 | require 'minitest'
  9 | require 'minitest/spec'
 10 | require 'minitest/autorun'
 11 | require 'resque'
 12 | require 'timecop'
 13 | 
 14 | begin
 15 |   require 'leftright'
 16 | rescue LoadError
 17 | end
 18 | require 'resque'
 19 | require 'resque_cleaner'
 20 | 
 21 | $TEST_PID = Process.pid
 22 | 
 23 | #
 24 | # make sure we can run redis
 25 | #
 26 | 
 27 | if !system("which redis-server")
 28 |   puts '', "** can't find `redis-server` in your path"
 29 |   puts "** try running `sudo rake install`"
 30 |   abort ''
 31 | end
 32 | 
 33 | 
 34 | #
 35 | # start our own redis when the tests start,
 36 | # kill it when they end
 37 | #
 38 | MiniTest.after_run {
 39 |   if Process.pid == $TEST_PID
 40 |     processes = `ps -A -o pid,command | grep [r]edis-test`.split($/)
 41 |     pids = processes.map { |process| process.split(" ")[0] }
 42 |     puts "Killing test redis server..."
 43 |     pids.each { |pid| Process.kill("TERM", pid.to_i) }
 44 |     dump = "test/dump.rdb"
 45 |     File.delete(dump) if File.exist?(dump)
 46 |   end
 47 | }
 48 | 
 49 | puts "Starting redis for testing at localhost:9736..."
 50 | `redis-server #{dir}/redis-test.conf`
 51 | Resque.redis = 'localhost:9736'
 52 | 
 53 | 
 54 | ##
 55 | # Helper to perform job classes
 56 | #
 57 | module PerformJob
 58 |   def perform_job(klass, *args)
 59 |     resque_job = Resque::Job.new(:testqueue, 'class' => klass, 'args' => args)
 60 |     resque_job.perform
 61 |   end
 62 | end
 63 | 
 64 | #
 65 | # fixture classes
 66 | #
 67 | 
 68 | class SomeJob
 69 |   def self.perform(repo_id, path)
 70 |   end
 71 | end
 72 | 
 73 | class SomeIvarJob < SomeJob
 74 |   @queue = :ivar
 75 | end
 76 | 
 77 | class SomeMethodJob < SomeJob
 78 |   def self.queue
 79 |     :method
 80 |   end
 81 | end
 82 | 
 83 | class BadJob
 84 |   def self.perform(name=nil)
 85 |     msg = name ? "Bad job, #{name}" : "Bad job!"
 86 |     raise msg
 87 |   end
 88 | end
 89 | 
 90 | class GoodJob
 91 |   def self.perform(name)
 92 |     "Good job, #{name}"
 93 |   end
 94 | end
 95 | 
 96 | class BadJobWithSyntaxError
 97 |   def self.perform
 98 |     raise SyntaxError, "Extra Bad job!"
 99 |   end
100 | end
101 | 
102 | #
103 | # helper methods
104 | #
105 | 
106 | def create_and_process_jobs(queue,worker,num,date,job,*args)
107 |   Timecop.freeze(date) do
108 |     num.times do
109 |       Resque::Job.create(queue, job, *args)
110 |     end
111 |     worker.work(0)
112 |   end
113 | end
114 | 
115 | def queue_size(*queues)
116 |   queues.inject(0){|sum,queue| sum + Resque.size(queue).to_i}
117 | end
118 | 
119 | def add_empty_payload_failure
120 |   data = {
121 |     :failed_at => Time.now.strftime("%Y/%m/%d %H:%M:%S %Z"),
122 |     :payload   => nil,
123 |     :exception => "Resque::DirtyExit",
124 |     :error     => "Resque::DirtyExit",
125 |     :backtrace => [],
126 |     :worker    => "worker",
127 |     :queue     => "queue"
128 |   }
129 |   data = Resque.encode(data)
130 |   Resque.redis.rpush(:failed, data)
131 | end
132 | 


--------------------------------------------------------------------------------
/test/redis-test.conf:
--------------------------------------------------------------------------------
  1 | # Redis configuration file example
  2 | 
  3 | # By default Redis does not run as a daemon. Use 'yes' if you need it.
  4 | # Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
  5 | daemonize yes
  6 | 
  7 | # When run as a daemon, Redis write a pid file in /var/run/redis.pid by default.
  8 | # You can specify a custom pid file location here.
  9 | pidfile ./test/redis-test.pid
 10 | 
 11 | # Accept connections on the specified port, default is 6379
 12 | port 9736
 13 | 
 14 | # If you want you can bind a single interface, if the bind option is not
 15 | # specified all the interfaces will listen for connections.
 16 | #
 17 | # bind 127.0.0.1
 18 | 
 19 | # Close the connection after a client is idle for N seconds (0 to disable)
 20 | timeout 300
 21 | 
 22 | # Save the DB on disk:
 23 | #
 24 | #   save <seconds> <changes>
 25 | #
 26 | #   Will save the DB if both the given number of seconds and the given
 27 | #   number of write operations against the DB occurred.
 28 | #
 29 | #   In the example below the behaviour will be to save:
 30 | #   after 900 sec (15 min) if at least 1 key changed
 31 | #   after 300 sec (5 min) if at least 10 keys changed
 32 | #   after 60 sec if at least 10000 keys changed
 33 | save 900 1
 34 | save 300 10
 35 | save 60 10000
 36 | 
 37 | # The filename where to dump the DB
 38 | dbfilename dump.rdb
 39 | 
 40 | # For default save/load DB in/from the working directory
 41 | # Note that you must specify a directory not a file name.
 42 | dir ./test/
 43 | 
 44 | # Set server verbosity to 'debug'
 45 | # it can be one of:
 46 | # debug (a lot of information, useful for development/testing)
 47 | # notice (moderately verbose, what you want in production probably)
 48 | # warning (only very important / critical messages are logged)
 49 | loglevel debug
 50 | 
 51 | # Specify the log file name. Also 'stdout' can be used to force
 52 | # the demon to log on the standard output. Note that if you use standard
 53 | # output for logging but daemonize, logs will be sent to /dev/null
 54 | logfile stdout
 55 | 
 56 | # Set the number of databases. The default database is DB 0, you can select
 57 | # a different one on a per-connection basis using SELECT <dbid> where
 58 | # dbid is a number between 0 and 'databases'-1
 59 | databases 16
 60 | 
 61 | ################################# REPLICATION #################################
 62 | 
 63 | # Master-Slave replication. Use slaveof to make a Redis instance a copy of
 64 | # another Redis server. Note that the configuration is local to the slave
 65 | # so for example it is possible to configure the slave to save the DB with a
 66 | # different interval, or to listen to another port, and so on.
 67 | 
 68 | # slaveof <masterip> <masterport>
 69 | 
 70 | ################################## SECURITY ###################################
 71 | 
 72 | # Require clients to issue AUTH <PASSWORD> before processing any other
 73 | # commands.  This might be useful in environments in which you do not trust
 74 | # others with access to the host running redis-server.
 75 | #
 76 | # This should stay commented out for backward compatibility and because most
 77 | # people do not need auth (e.g. they run their own servers).
 78 | 
 79 | # requirepass foobared
 80 | 
 81 | ################################### LIMITS ####################################
 82 | 
 83 | # Set the max number of connected clients at the same time. By default there
 84 | # is no limit, and it's up to the number of file descriptors the Redis process
 85 | # is able to open. The special value '0' means no limts.
 86 | # Once the limit is reached Redis will close all the new connections sending
 87 | # an error 'max number of clients reached'.
 88 | 
 89 | # maxclients 128
 90 | 
 91 | # Don't use more memory than the specified amount of bytes.
 92 | # When the memory limit is reached Redis will try to remove keys with an
 93 | # EXPIRE set. It will try to start freeing keys that are going to expire
 94 | # in little time and preserve keys with a longer time to live.
 95 | # Redis will also try to remove objects from free lists if possible.
 96 | #
 97 | # If all this fails, Redis will start to reply with errors to commands
 98 | # that will use more memory, like SET, LPUSH, and so on, and will continue
 99 | # to reply to most read-only commands like GET.
100 | #
101 | # WARNING: maxmemory can be a good idea mainly if you want to use Redis as a
102 | # 'state' server or cache, not as a real DB. When Redis is used as a real
103 | # database the memory usage will grow over the weeks, it will be obvious if
104 | # it is going to use too much memory in the long run, and you'll have the time
105 | # to upgrade. With maxmemory after the limit is reached you'll start to get
106 | # errors for write operations, and this may even lead to DB inconsistency.
107 | 
108 | # maxmemory <bytes>
109 | 
110 | ############################### ADVANCED CONFIG ###############################
111 | 
112 | # Glue small output buffers together in order to send small replies in a
113 | # single TCP packet. Uses a bit more CPU but most of the times it is a win
114 | # in terms of number of queries per second. Use 'yes' if unsure.
115 | #glueoutputbuf yes
116 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server/views/cleaner_list.erb:
--------------------------------------------------------------------------------
  1 | <%= erb File.read(ResqueCleaner::Server.erb_path("_cleaner_styles.erb")) %>
  2 | 
  3 | <!-- Many code was copied from failed.erb of the original resque -->
  4 | <div class="cleaner">
  5 |   <div class="title clearfix">
  6 |     <h1>Failed Jobs</h1>
  7 |   </div>
  8 | 
  9 |   <div class="clearfix">
 10 |     <div class="control_panel sub_header">
 11 |       <form method="get">
 12 |         <span class="class_filter">
 13 |           Class: <%= class_filter("filter_class","c",@klasses,@klass)%>
 14 |         </span>
 15 |         <span class="exception_filter">
 16 |           Exception: <%= exception_filter("filter_class","ex",@exceptions,@exception)%>
 17 |         </span>
 18 |         <span class="time_filter">
 19 |           From: <%= time_filter("filter_from","f",@from)%>
 20 |         </span>
 21 |         <span class="time_filter">
 22 |           To: <%= time_filter("filter_to","t",@to)%>
 23 |         </span>
 24 |         <span class="regex_filter">
 25 |           Regex: <%= text_filter("filter_regex", "regex", Rack::Utils.escape_html(@regex))%>
 26 |         </span>
 27 |         <input type="submit" value="Filter" />
 28 |       </form>
 29 |     </div>
 30 |   </div>
 31 | 
 32 |   <% if @count > 0 %>
 33 |     <div class="clearfix">
 34 |       <div class="control_panel sub_header">
 35 |         <form method="post" id="exec" action="cleaner_exec">
 36 |           <input type="hidden" name="c" value="<%=@klass%>" />
 37 |           <input type="hidden" name="f" value="<%=@from%>" />
 38 |           <input type="hidden" name="t" value="<%=@to%>" />
 39 |           <input type="hidden" name="p" value="<%=@paginate.page%>" />
 40 |           <input type="hidden" name="ex" value="<%=@exception%>" />
 41 |           <input type="hidden" name="regex" value="<%=Rack::Utils.escape_html(@regex)%>" />
 42 |           <select id="form_action" name="action">
 43 |             <option id="default_option" value="" selected="selected">-- Select Action --</option>
 44 |             <option value="clear">Clear</option>
 45 |             <option value="retry_and_clear">Retry and Clear</option>
 46 |             <option value="retry">Retry</option>
 47 |           </select>
 48 |           <a href="#" id="select_all">select all</a>
 49 |           <a href="#" id="reset_all">reset</a>
 50 | 
 51 |           <% if @paginate.max_page > 1 %>
 52 |             <input type="checkbox" name="select_all_pages" value="1" id="select_all_pages" />
 53 |             <label for="select_all_pages">Select all <%=@count%> jobs</label>
 54 |           <% end %>
 55 |           <input type="hidden" name="sha1" id="sha1_list" />
 56 | 
 57 |         </form>
 58 |       </div>
 59 |     </div>
 60 |   <% end %>
 61 | 
 62 |   <% start = 0 %>
 63 |   <% failed = @paginate.paginated_jobs%>
 64 |   <% index = 0 %>
 65 |   <% date_format = "%Y/%m/%d %T %z" %>
 66 | 
 67 |   <% if @paginate.max_page > 0 %>
 68 |     <%= erb File.read(ResqueCleaner::Server.erb_path("_paginate.erb")) %>
 69 |     <ul class='failed'>
 70 |       <%for job in failed%>
 71 |         <% index += 1 %>
 72 |         <li>
 73 |           <dl>
 74 |             <% if job.nil? %>
 75 |             <dt>Error</dt>
 76 |             <dd>Job <%= index%> could not be parsed; perhaps it contains invalid JSON?</dd>
 77 |             <% else %>
 78 |             <dt>
 79 |               <input type="checkbox" id="<%=Digest::SHA1.hexdigest job.to_json %>" />
 80 |             </dt>
 81 |             <dd>&nbsp;</dd>
 82 |             <dt>Worker</dt>
 83 |             <dd>
 84 |               <a href="<%= u(:workers, job['worker']) %>"><%= job['worker'].split(':')[0...2].join(':') %></a> on <b class='queue-tag'><%= job['queue'] %></b > at <b><span class="time"><%= Time.parse(job['failed_at']).strftime(date_format) %></span></b>
 85 |               <% if job['retried_at'] %>
 86 |                 <div class='retried'>
 87 |                   Retried <b><span class="time"><%= job['retried_at'] %></span></b>
 88 |                 </div>
 89 |               <% end %>
 90 |             </dd>
 91 |             <dt>Class</dt>
 92 |             <dd><code><%= job['payload'] ? job['payload']['class'] : 'nil' %></code></dd>
 93 |             <dt>Arguments</dt>
 94 |             <dd><pre><%=h job['payload'] ? show_job_args(job['payload']['args']) : 'nil' %></pre></dd>
 95 |             <dt>Exception</dt>
 96 |             <dd><code><%= job['exception'] %></code></dd>
 97 |             <dt>Error</dt>
 98 |             <dd class='error'>
 99 |               <% if job['backtrace'] %>
100 |                 <a href="#" class="backtrace"><%= h(job['error']) %></a>
101 |                 <pre style='display:none'><%=h job['backtrace'].join("\n") %></pre>
102 |               <% else %>
103 |                 <%=h job['error'] %>
104 |               <% end %>
105 |             </dd>
106 |             <% end %>
107 |           </dl>
108 |           <div class='r'>
109 |           </div>
110 |         </li>
111 |       <%end%>
112 |     </ul>
113 |     <%= erb File.read(ResqueCleaner::Server.erb_path("_paginate.erb")) %>
114 |   <% else %>
115 |     Clean!
116 |   <% end %>
117 | </div>
118 | 
119 | <script>
120 |   $(document).ready(function(){
121 |     $('#select_all_pages').click(function() {
122 |       updateCheckboxStaus();
123 |     });
124 | 
125 |     $('#select_all').click(function() {
126 |       if (!$(this).hasClass('disabled')) {
127 |         $('.failed input').attr('checked','checked');
128 |       }
129 |       return false;
130 |     });
131 |     $('#reset_all').click(function() {
132 |       if (!$(this).hasClass('disabled')) {
133 |         $('.failed input').removeAttr('checked');
134 |       }
135 |       return false;
136 |     });
137 | 
138 |     $('#form_action').change( function() {
139 |       if ($('#form_action option:selected').val()=='') return;
140 | 
141 |       if ($('#select_all_pages:checked, .failed input:checked').length==0) {
142 |         alert('Please select jobs.');
143 |         $('#default_option').attr('selected','selected');
144 |         return false;
145 |       }
146 | 
147 |       if (!confirm('Do you really want to proceed?')) {
148 |         $('#default_option').attr('selected','selected');
149 |         return false;
150 |       }
151 | 
152 |       if ($('#select_all_pages:checked').length==0) {
153 |         setSha1();
154 |       }
155 |       $('#exec').submit();
156 |     });
157 |   });
158 | 
159 |   function setSha1() {
160 |     var sha1 = "";
161 |     $('.failed input:checked').each( function() {
162 |       if (sha1.length>0) sha1 += ",";
163 |       sha1 += $(this).attr("id");
164 |     });
165 | 
166 |     $('#sha1_list').val(sha1);
167 |   }
168 | 
169 |   function updateCheckboxStaus() {
170 |     if ($('#select_all_pages:checked').length==1) {
171 |       $('#exec a').addClass('disabled');
172 |       $('.failed input').attr('disabled','disabled');
173 |     } else {
174 |       $('#exec a').removeClass('disabled');
175 |       $('.failed input').removeAttr('disabled');
176 |     }
177 |   };
178 | 
179 | </script>
180 | 


--------------------------------------------------------------------------------
/test/resque_cleaner_test.rb:
--------------------------------------------------------------------------------
  1 | require File.expand_path(File.dirname(__FILE__) + '/test_helper')
  2 | require 'time'
  3 | describe "ResqueCleaner" do
  4 |   before do
  5 |     Resque.redis.flushall
  6 | 
  7 |     @worker = Resque::Worker.new(:jobs,:jobs2)
  8 | 
  9 |     # 3 BadJob at 2009-03-13
 10 |     create_and_process_jobs :jobs, @worker, 3, Time.parse('2009-03-13'), BadJob
 11 |     # 2 BadJob by Jason at 2009-03-13
 12 |     create_and_process_jobs :jobs2, @worker, 2, Time.parse('2009-03-13'), BadJob, "Jason"
 13 | 
 14 |     # 1 BadJob by Johnson at 2009-03-13
 15 |     create_and_process_jobs :jobs, @worker, 1, Time.parse('2009-03-13'), BadJob, "Johnson"
 16 | 
 17 |     # 7 BadJob at 2009-11-13
 18 |     create_and_process_jobs :jobs, @worker, 7, Time.parse('2009-11-13'), BadJobWithSyntaxError
 19 |     # 7 BadJob by Freddy at 2009-11-13
 20 |     create_and_process_jobs :jobs2, @worker, 7, Time.parse('2009-11-13'), BadJob, "Freddy"
 21 | 
 22 |     # 11 BadJob at 2010-08-13
 23 |     create_and_process_jobs :jobs, @worker, 11, Time.parse('2010-08-13'), BadJob
 24 |     # 11 BadJob by Jason at 2010-08-13
 25 |     create_and_process_jobs :jobs2, @worker, 11, Time.parse('2010-08-13'), BadJob, "Jason"
 26 | 
 27 |     @cleaner = Resque::Plugins::ResqueCleaner.new
 28 |     @cleaner.print_message = false
 29 |   end
 30 | 
 31 |   it "#select_by_regex returns only Jason jobs" do
 32 |     ret = @cleaner.select_by_regex(/Jason/)
 33 |     assert_equal 13, ret.size
 34 |   end
 35 | 
 36 |   it "#select_by_regex returns an empty array if passed a non-regex" do
 37 |     ['string', nil, 13, Class.new].each do |non_regex|
 38 |       ret = @cleaner.select_by_regex(nil)
 39 |       assert_equal 0, ret.size
 40 |     end
 41 |   end
 42 | 
 43 |   it "#select returns failure jobs" do
 44 |     ret = @cleaner.select
 45 |     assert_equal 42, ret.size
 46 |   end
 47 | 
 48 |   it "#select works with a limit" do
 49 |     @cleaner.limiter.maximum = 10
 50 |     ret = @cleaner.select
 51 | 
 52 |     # only maximum number
 53 |     assert_equal 10, ret.size
 54 | 
 55 |     # lait one
 56 |     assert_equal Time.parse(ret[0]['failed_at']), Time.parse('2010-08-13')
 57 |   end
 58 | 
 59 |   it "#select with a block returns failure jobs which the block evaluates true" do
 60 |     ret = @cleaner.select {|job| job["payload"]["args"][0]=="Jason"}
 61 |     assert_equal 13, ret.size
 62 |   end
 63 | 
 64 |   it "#clear deletes failure jobs" do
 65 |     cleared = @cleaner.clear
 66 |     assert_equal 42, cleared
 67 |     assert_equal 0, @cleaner.select.size
 68 |   end
 69 | 
 70 |   it "#clear with a block deletes failure jobs which the block evaluates true" do
 71 |     cleared = @cleaner.clear{|job| job["payload"]["args"][0]=="Jason"}
 72 |     assert_equal 13, cleared
 73 |     assert_equal 42-13, @cleaner.select.size
 74 |     assert_equal 0, @cleaner.select{|job| job["payload"]["args"][0]=="Jason"}.size
 75 |   end
 76 | 
 77 |   it "#requeue retries failure jobs" do
 78 |     assert_equal 0, queue_size(:jobs,:jobs2)
 79 |     requeued = @cleaner.requeue
 80 |     assert_equal 42, requeued
 81 |     assert_equal 42, @cleaner.select.size # it doesn't clear jobs
 82 |     assert_equal 42, queue_size(:jobs,:jobs2)
 83 |   end
 84 | 
 85 |   it "#requeue with a block retries failure jobs which the block evaluates true" do
 86 |     requeued = @cleaner.requeue{|job| job["payload"]["args"][0]=="Jason"}
 87 |     assert_equal 13, requeued
 88 |     assert_equal 13, queue_size(:jobs,:jobs2)
 89 |   end
 90 | 
 91 |   it "#requeue with clear option requeues and deletes failure jobs" do
 92 |     assert_equal 0, queue_size(:jobs,:jobs2)
 93 |     requeued = @cleaner.requeue(true)
 94 |     assert_equal 42, requeued
 95 |     assert_equal 42, queue_size(:jobs,:jobs2)
 96 |     assert_equal 0, @cleaner.select.size
 97 |   end
 98 | 
 99 |   it "#requeue with :queue option requeues the jobs to the queue" do
100 |     assert_equal 0, queue_size(:jobs,:jobs2,:retry)
101 |     requeued = @cleaner.requeue false, :queue => :retry
102 |     assert_equal 42, requeued
103 |     assert_equal 42, @cleaner.select.size # it doesn't clear jobs
104 |     assert_equal 0, queue_size(:jobs,:jobs2)
105 |     assert_equal 42, queue_size(:retry)
106 |   end
107 | 
108 |   it "#clear_stale deletes failure jobs which is queued before the last x enqueued" do
109 |     @cleaner.limiter.maximum = 10
110 |     @cleaner.clear_stale
111 |     assert_equal 10, @cleaner.failure.count
112 |     assert_equal Time.parse(@cleaner.failure_jobs[0]['failed_at']), Time.parse('2010-08-13')
113 |   end
114 | 
115 |   it "FailedJobEx module extends job and provides some useful methods" do
116 |     # before 2009-04-01
117 |     ret = @cleaner.select {|j| j.before?('2009-04-01')}
118 |     assert_equal 6, ret.size
119 | 
120 |     # after 2010-01-01
121 |     ret = @cleaner.select {|j| j.after?('2010-01-01')}
122 |     assert_equal 22, ret.size
123 | 
124 |     # filter by class
125 |     ret = @cleaner.select {|j| j.klass?(BadJobWithSyntaxError)}
126 |     assert_equal 7, ret.size
127 | 
128 |     # filter by exception
129 |     ret = @cleaner.select {|j| j.exception?(SyntaxError)}
130 |     assert_equal 7, ret.size
131 | 
132 |     # filter by queue
133 |     ret = @cleaner.select {|j| j.queue?(:jobs2)}
134 |     assert_equal 20, ret.size
135 | 
136 |     # combination
137 |     ret = @cleaner.select {|j| j.queue?(:jobs2) && j.before?('2009-12-01')}
138 |     assert_equal 9, ret.size
139 | 
140 |     # combination 2
141 |     ret = @cleaner.select {|j| j['payload']['args']==['Jason'] && j.queue?(:jobs2)}
142 |     assert_equal 13, ret.size
143 | 
144 |     # retried?
145 |     requeued = @cleaner.requeue{|j| j["payload"]["args"][0]=="Johnson"}
146 |     ret = @cleaner.select {|j| j.retried?}
147 |     assert_equal 1, ret.size
148 |   end
149 | 
150 |   it "#stats_by_date returns stats grouped by date" do
151 |     ret = @cleaner.stats_by_date
152 |     assert_equal 6, ret['2009/03/13']
153 |     assert_equal 14, ret['2009/11/13']
154 | 
155 |     # with block
156 |     ret = @cleaner.stats_by_date{|j| j['payload']['args']==['Jason']}
157 |     assert_equal 2, ret['2009/03/13']
158 |     assert_equal nil, ret['2009/11/13']
159 |     assert_equal 11, ret['2010/08/13']
160 |   end
161 | 
162 |   it "#stats_by_class returns stats grouped by class" do
163 |     ret = @cleaner.stats_by_class
164 |     assert_equal 35, ret['BadJob']
165 |     assert_equal 7, ret['BadJobWithSyntaxError']
166 |   end
167 | 
168 |   it "#stats_by_class works with broken log" do
169 |     add_empty_payload_failure
170 |     ret = @cleaner.stats_by_class
171 |     assert_equal 1, ret['UNKNOWN']
172 |   end
173 | 
174 |   it "#stats_by_exception returns stats grouped by exception" do
175 |     ret = @cleaner.stats_by_exception
176 |     assert_equal 35, ret['RuntimeError']
177 |     assert_equal 7, ret['SyntaxError']
178 |   end
179 | 
180 |   it "#lock ensures that a new failure job doesn't affect in a limit mode" do
181 |     @cleaner.limiter.maximum = 23
182 |     @cleaner.limiter.lock do
183 |       first = @cleaner.select[0]
184 |       assert_equal "Freddy", first["payload"]["args"][0]
185 | 
186 |       create_and_process_jobs :jobs, @worker, 30, Time.parse('2010-10-10'), BadJob, "Jack"
187 | 
188 |       first = @cleaner.select[0]
189 |       assert_equal "Freddy", first["payload"]["args"][0]
190 |     end
191 |     first = @cleaner.select[0]
192 |     assert_equal "Jack", first["payload"]["args"][0]
193 |   end
194 | 
195 |   it "allows you to configure limiter" do
196 |     c = Resque::Plugins::ResqueCleaner.new
197 |     refute_equal c.limiter.maximum, 10_000
198 | 
199 |     module Resque::Plugins
200 |       ResqueCleaner::Limiter.default_maximum = 10_000
201 |     end
202 | 
203 |     c = Resque::Plugins::ResqueCleaner.new
204 |     assert_equal c.limiter.maximum, 10_000
205 |   end
206 | end
207 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner/server.rb:
--------------------------------------------------------------------------------
  1 | require 'yaml'
  2 | 
  3 | # Extends Resque Web Based UI.
  4 | # Structure has been borrowed from ResqueScheduler.
  5 | module ResqueCleaner
  6 |   module Server
  7 | 
  8 |     def self.erb_path(filename)
  9 |       File.join(File.dirname(__FILE__), 'server', 'views', filename)
 10 |     end
 11 | 
 12 |     # Pagination helper for list page.
 13 |     class Paginate
 14 |       attr_accessor :page_size, :page, :jobs, :url
 15 |       def initialize(jobs, url, page=1, page_size=20)
 16 |         @jobs = jobs
 17 |         @url = url
 18 |         @page = (!page || page < 1) ? 1 : page
 19 |         @page_size = 20
 20 |       end
 21 | 
 22 |       def first_index
 23 |         @page_size * (@page-1)
 24 |       end
 25 | 
 26 |       def last_index
 27 |         last = first_index + @page_size - 1
 28 |         last > @jobs.size-1 ? @jobs.size-1 : last
 29 |       end
 30 | 
 31 |       def paginated_jobs
 32 |         @jobs[first_index,@page_size]
 33 |       end
 34 | 
 35 |       def first_page?
 36 |         @page <= 1
 37 |       end
 38 | 
 39 |       def last_page?
 40 |         @page >= max_page
 41 |       end
 42 | 
 43 |       def page_url(page)
 44 |         u = @url
 45 |         u += @url.include?("?") ? "&" : "?"
 46 |         if page.is_a?(Symbol)
 47 |           page = @page - 1 if page==:prev
 48 |           page = @page + 1 if page==:next
 49 |         end
 50 |         u += "p=#{page}"
 51 |       end
 52 | 
 53 |       def total_size
 54 |         @jobs.size
 55 |       end
 56 | 
 57 |       def max_page
 58 |         ((total_size-1) / @page_size) + 1
 59 |       end
 60 |     end
 61 | 
 62 |     def self.included(base)
 63 | 
 64 |       base.class_eval do
 65 |         helpers do
 66 |           def time_filter(id, name, value)
 67 |             html = "<select id=\"#{id}\" name=\"#{name}\">"
 68 |             html += "<option value=\"\">-</option>"
 69 |             [1, 3, 6, 12, 24].each do |h|
 70 |               selected = h.to_s == value ? 'selected="selected"' : ''
 71 |               html += "<option #{selected} value=\"#{h}\">#{h} #{h==1 ? "hour" : "hours"} ago</option>"
 72 |             end
 73 |             [3, 7, 14, 28].each do |d|
 74 |               selected = (d*24).to_s == value ? 'selected="selected"' : ''
 75 |               html += "<option #{selected} value=\"#{d*24}\">#{d} days ago</option>"
 76 |             end
 77 |             html += "</select>"
 78 |           end
 79 | 
 80 |           def class_filter(id, name, klasses, value)
 81 |             html = "<select id=\"#{id}\" name=\"#{name}\">"
 82 |             html += "<option value=\"\">-</option>"
 83 |             klasses.each do |k|
 84 |               selected = k == value ? 'selected="selected"' : ''
 85 |               html += "<option #{selected} value=\"#{k}\">#{k}</option>"
 86 |             end
 87 |             html += "</select>"
 88 |           end
 89 | 
 90 |           def exception_filter(id, name, exceptions, value)
 91 |             html = "<select id=\"#{id}\" name=\"#{name}\">"
 92 |             html += "<option value=\"\">-</option>"
 93 |             exceptions.each do |ex|
 94 |               selected = ex == value ? 'selected="selected"' : ''
 95 |               html += "<option #{selected} value=\"#{ex}\">#{ex}</option>"
 96 |             end
 97 |             html += "</select>"
 98 |           end
 99 | 
100 |           def show_job_args(args)
101 |             Array(args).map { |a| a.inspect }.join("\n")
102 |           end
103 | 
104 |           def text_filter(id, name, value)
105 |             html = "<input id=\"#{id}\"  type=\"text\" name=\"#{name}\" value=\"#{value}\">"
106 |             html += "</input>"
107 |           end
108 |         end
109 | 
110 |         mime_type :json, 'application/json'
111 | 
112 |         get "/cleaner" do
113 |           load_library
114 |           load_cleaner_filter
115 | 
116 |           @jobs = cleaner.select
117 |           @stats = { :klass => {}, :exception => {} }
118 |           @total = Hash.new(0)
119 |           @jobs.each do |job|
120 |             payload = job["payload"] || {}
121 |             klass = payload["class"] || 'UNKNOWN'
122 |             exception = job["exception"] || 'UNKNOWN'
123 |             failed_at = Time.parse job["failed_at"]
124 |             @stats[:klass][klass] ||= Hash.new(0)
125 |             @stats[:exception][exception] ||= Hash.new(0)
126 | 
127 |             [
128 |               @stats[:klass][klass],
129 |               @stats[:exception][exception],
130 |               @total
131 |             ].each do |stat|
132 |               stat[:total] += 1
133 |               stat[:h1] += 1 if failed_at >= hours_ago(1)
134 |               stat[:h3] += 1 if failed_at >= hours_ago(3)
135 |               stat[:d1] += 1 if failed_at >= hours_ago(24)
136 |               stat[:d3] += 1 if failed_at >= hours_ago(24*3)
137 |               stat[:d7] += 1 if failed_at >= hours_ago(24*7)
138 |             end
139 |           end
140 | 
141 |           erb File.read(ResqueCleaner::Server.erb_path('cleaner.erb'))
142 |         end
143 | 
144 |         get "/cleaner_list" do
145 |           load_library
146 |           load_cleaner_filter
147 |           build_urls
148 | 
149 |           block = filter_block
150 | 
151 |           @failed = cleaner.select(&block).reverse
152 | 
153 |           @paginate = Paginate.new(@failed, @list_url, params[:p].to_i)
154 | 
155 |           @klasses = cleaner.stats_by_class.keys
156 |           @exceptions = cleaner.stats_by_exception.keys
157 |           @count = cleaner.select(&block).size
158 | 
159 |           erb File.read(ResqueCleaner::Server.erb_path('cleaner_list.erb'))
160 |         end
161 | 
162 |         post "/cleaner_exec" do
163 |           load_library
164 |           load_cleaner_filter
165 |           build_urls
166 | 
167 |           if params[:select_all_pages]!="1"
168 |             @sha1 = {}
169 |             params[:sha1].split(",").each {|s| @sha1[s] = true }
170 |           end
171 | 
172 |           block = filter_block
173 | 
174 |           @count =
175 |             case params[:action]
176 |             when "clear" then cleaner.clear(&block)
177 |             when "retry_and_clear" then cleaner.requeue(true,&block)
178 |             when "retry" then cleaner.requeue(false,{},&block)
179 |             end
180 | 
181 |           erb File.read(ResqueCleaner::Server.erb_path('cleaner_exec.erb'))
182 |         end
183 | 
184 |         get "/cleaner_dump" do
185 |           load_library
186 |           load_cleaner_filter
187 | 
188 |           block = filter_block
189 | 
190 |           content_type :json
191 |           JSON.pretty_generate(cleaner.select(&block))
192 |         end
193 | 
194 |         post "/cleaner_stale" do
195 |           load_library
196 |           cleaner.clear_stale
197 |           redirect url_path(:cleaner)
198 |         end
199 |       end
200 | 
201 |     end
202 | 
203 |     def cleaner
204 |       @cleaner ||= Resque::Plugins::ResqueCleaner.new
205 |       @cleaner.print_message = false
206 |       @cleaner
207 |     end
208 | 
209 |     def load_library
210 |       require 'digest/sha1'
211 |       begin
212 |         require 'yajl/json_gem' unless [].respond_to?(:to_json)
213 |       rescue Exception
214 |         require 'json'
215 |       end
216 |     end
217 | 
218 |     def load_cleaner_filter
219 |       @from = params[:f]=="" ? nil : params[:f]
220 |       @to = params[:t]=="" ? nil : params[:t]
221 |       @klass = params[:c]=="" ? nil : params[:c]
222 |       @exception = params[:ex]=="" ? nil : params[:ex]
223 |       @regex = params[:regex]=="" ? nil : params[:regex]
224 |     end
225 | 
226 |     def build_urls
227 |       params = {
228 |         c: @klass,
229 |         ex: @exception,
230 |         f: @from,
231 |         t: @to,
232 |         regex: @regex
233 |       }.map {|key,value| "#{key}=#{URI.encode(value.to_s)}"}.join("&")
234 | 
235 |       @list_url = "cleaner_list?#{params}"
236 |       @dump_url = "cleaner_dump?#{params}"
237 |     end
238 | 
239 |     def filter_block
240 |       block = lambda{|j|
241 |         (!@from || j.after?(hours_ago(@from))) &&
242 |         (!@to || j.before?(hours_ago(@to))) &&
243 |         (!@klass || j.klass?(@klass)) &&
244 |         (!@exception || j.exception?(@exception)) &&
245 |         (!@sha1 || @sha1[Digest::SHA1.hexdigest(j.to_json)]) &&
246 |         (!@regex || j.to_s =~ /#{@regex}/)
247 |       }
248 |     end
249 | 
250 |     def hours_ago(h)
251 |       Time.now - h.to_i*60*60
252 |     end
253 |     Resque::Server.tabs << 'Cleaner'
254 |   end
255 | end
256 | 
257 | Resque::Server.class_eval do
258 |   include ResqueCleaner::Server
259 | end
260 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ResqueCleaner
  2 | =============
  3 | 
  4 | [github.com/ono/resque-cleaner](https://github.com/ono/resque-cleaner)
  5 | 
  6 | This project is not active.
  7 | Please read [this](https://github.com/ono/resque-cleaner/issues/47) before you use the library.
  8 | 
  9 | Description
 10 | -----------
 11 | 
 12 | ResqueCleaner is a [Resque](https://github.com/defunkt/resque) plugin which
 13 | aims to help you to clean up failed jobs on Resque by:
 14 | 
 15 | * Showing stats of failed jobs
 16 | * Retrying failed jobs
 17 | * Removing failed jobs
 18 | * Filtering failed jobs
 19 | 
 20 | 
 21 | Installation
 22 | ------------
 23 | 
 24 | Install as a gem:
 25 | 
 26 |     $ gem install resque-cleaner
 27 | 
 28 | 
 29 | Resque-Web integration
 30 | ----------------------
 31 | 
 32 | ![Screen 1](https://github.com/ono/resque-cleaner/raw/master/misc/resque-cleaner-main.png)
 33 | ![Screen 2](misc/resque-cleaner-list-with-regex.png)
 34 | 
 35 | 
 36 | Configuration
 37 | -------------
 38 | 
 39 | At first, you have to load ResqueCleaner to enable the Cleaner tab. Here is
 40 | an example step.
 41 | 
 42 | 1. Create a configuration file for resque-web
 43 | <br/>```touch [app_dir]/config/resque-web.rb```
 44 | 
 45 | 2. Add the following line into the file
 46 | <br/>```require 'resque-cleaner'```
 47 | 
 48 | 3. Then pass the file when you start resque-web
 49 | <br/>```% resque-web [app_dir]/config/resque-web.rb```
 50 | 
 51 | You can also configure [limiter](https://github.com/ono/resque-cleaner#limiter)
 52 | in the file.
 53 | 
 54 | e.g.
 55 | 
 56 | ```ruby
 57 | require 'resque-cleaner'
 58 | module Resque::Plugins
 59 |   ResqueCleaner::Limiter.default_maximum = 10_000
 60 | end
 61 | ```
 62 | 
 63 | Console
 64 | -------
 65 | 
 66 | Hopefully a situation of your failed jobs is simple enough to get figured out through
 67 | the web interface. But, if not, a powerful filtering feature of ResqueCleaner may help
 68 | you to understand what is going on with your console(irb).
 69 | 
 70 | **Create Instance**
 71 | 
 72 | ```ruby
 73 |     > cleaner = Resque::Plugins::ResqueCleaner.new
 74 | ```
 75 | 
 76 | **Show Stats**
 77 | 
 78 | Shows stats of failed jobs grouped by date.
 79 | 
 80 | ```ruby
 81 |     > cleaner.stats_by_date
 82 |     2009/03/13:    6
 83 |     2009/11/13:   14
 84 |     2010/08/13:   22
 85 |          total:   42
 86 |     => {'2009/03/10' => 6, ...}
 87 | ```
 88 | 
 89 | You could also group them by class.
 90 | 
 91 | ```ruby
 92 |     > cleaner.stats_by_class
 93 |          BadJob:    3
 94 |     HorribleJob:    7
 95 |           total:   10
 96 |     => {'BadJob' => 3, ...}
 97 | ```
 98 | 
 99 | Or you could also group them by exception.
100 | 
101 | ```ruby
102 |     > cleaner.stats_by_exception
103 | 	 RuntimeError:   35
104 |     SyntaxError:    7
105 |           total:   42
106 |     => {'RuntimeError' => 35, ...}
107 | ```
108 | 
109 | You can get the ones filtered with a block: it targets only jobs which the block
110 | evaluates true.
111 | 
112 | e.g. Show stats only of jobs entered with some arguments:
113 | 
114 | ```ruby
115 |     > cleaner.stats_by_date {|j| j["payload"]["args"].size > 0}
116 |     2009/03/13:    3
117 |     2009/11/13:    7
118 |     2010/08/13:   11
119 |          total:   22
120 |     => {'2009/03/10' => 3, ...}
121 | ```
122 | 
123 | A convenience helper for searching for failed jobs which satisfy a
124 | regular expression:
125 | 
126 | ```ruby
127 | cleaner.select_by_regex(/Jason/) # => all failed jobs that have "Jason" in them
128 | ```
129 | 
130 | **Retry(Requeue) Jobs**
131 | 
132 | You can retry all failed jobs with this method.
133 | 
134 | ```ruby
135 |     > cleaner.requeue
136 | ```
137 | 
138 | Of course, you can filter jobs with a block; it requeues only jobs which the
139 | block evaluates true. 
140 | 
141 | e.g. Retry only jobs with some arguments:
142 | 
143 | ```ruby
144 |     > cleaner.requeue {|j| j["payload"]["args"].size > 0}
145 | ```
146 | 
147 | The job hash is extended with a module which defines some useful methods. You
148 | can use it in the block.
149 | 
150 | e.g. Retry only jobs entered within a day:
151 | 
152 | ```ruby
153 |     > cleaner.requeue {|j| j.after?(1.day.ago)}
154 | ```
155 | 
156 | e.g. Retry EmailJob entered with arguments within 3 days:
157 | 
158 | ```ruby
159 |     > cleaner.requeue {|j| j.after?(3.days.ago) && j.klass?(EmailJob) && j["payload"]["args"].size>0}
160 | ```
161 | 
162 | See Helper Methods section bellow for more information.
163 | 
164 | NOTE:
165 | [1.day.ago](https://github.com/rails/rails/blob/master/activesupport/lib/active_support/core_ext/numeric/time.rb)
166 | is not in standard library. Using it for making explanation more understandable. It is equivalent to `Time.now - 60*60*24*3`.
167 | 
168 | **Clear Jobs**
169 | 
170 | You can clear all failed jobs with this method:
171 | 
172 | ```ruby
173 |     > cleaner.clear
174 | ```
175 | 
176 | Like you can do with the retry method, the clear method takes a block. Here are
177 | some examples:
178 | 
179 | ```ruby
180 |     > cleaner.clear {|j| j.retried?}
181 |     => clears all jobs already retried and returns number of the jobs.
182 | 
183 |     > cleaner.clear {|j| j.queue?(:low) && j.before?('2010-10-10')}
184 |     => clears all jobs entried in :low queue before 10th October, 2010.
185 | 
186 |     > cleaner.clear {|j| j.exception?("RuntimeError") && j.queue?(:low)}
187 |     => clears all jobs raised RuntimeError and queued :low queue
188 | ```
189 | 
190 | **Retry and Clear Jobs**
191 | 
192 | You can retry(requeue) and clear failed jobs at the same time; just pass true
193 | as an argument. 
194 | 
195 | e.g. Retry EmailJob and remove from failed jobs:
196 | 
197 | ```ruby
198 |     > cleaner.requeue(true) {|j| j.klass?(EmailJob)}
199 | ```
200 | 
201 | **Retry with other queue**
202 | 
203 | You can requeue failed jobs into other queue. In this way, you can retry failed
204 | jobs without blocking jobs being entered by your service running in the live.
205 | 
206 | e.g. Retry failed jobs on :retry queue
207 | 
208 | ```ruby
209 |     > cleaner.requeue(false, :queue => :retry)
210 | ```
211 | 
212 | Don't forget to launch resque worker for the queue.
213 | 
214 |     % QUEUE=retry rake resque:work
215 | 
216 | **Select Jobs**
217 | 
218 | You can just select the jobs of course. Here are some examples:
219 | 
220 | ```ruby
221 |     > cleaner.select {|j| j["payload"]["args"][0]=="Johonson"}
222 |     > cleaner.select {|j| j.after?(2.days.ago)}
223 |     > cleaner.select #=> returns all jobs
224 | ```
225 | 
226 | **Helper Methods**
227 | 
228 | Here is a list of methods a failed job retained through ResqueCleaner has:
229 | 
230 |     retried?: returns true if the job has already been retried.
231 |     requeued?: alias of retried?.
232 |     before?(time): returns true if the job failed before the time.
233 |     after?(time): returns true if the job failed after the time.
234 |     klass?(klass_or_name): returns true if the class of job matches.
235 |     queue?(queue_name): returns true if the queue of job matches.
236 |     exception?(exception_name): returns true if the exception matches.
237 | 
238 | 
239 | Failed Job
240 | -----------
241 | 
242 | Here is a sample of failed jobs:
243 | 
244 |     {"failed_at": "2009/03/13 00:00:00",
245 |      "payload": {"args": ["Johnson"], "class": "BadJob"},
246 |      "queue": "jobs",
247 |      "worker": "localhost:7327:jobs,jobs2",
248 |      "exception": "RuntimeError",
249 |      "error": "Bad job, Johnson",
250 |      "backtrace": 
251 |       ["./test/test_helper.rb:108:in `perform'",
252 |        "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/job.rb:133:in `perform'",
253 |        "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/worker.rb:157:in `perform'",
254 |        "/opt/local/lib/ruby/gems/1.8/gems/resque-1.10.0/lib/resque/worker.rb:124:in `work'",
255 |        "....(omitted)....",
256 |        "./test/test_helper.rb:41",
257 |        "test/resque_cleaner_test.rb:3"]
258 |     }
259 | 
260 | 
261 | Limiter
262 | -------
263 | 
264 | ResqueCleaner expects a disaster situation like a huge number of failed jobs are
265 | out there. Since ResqueCleaner's filter function is running on your application
266 | process but on your Redis, it would not respond ages if you try to deal with all
267 | of those jobs.
268 | 
269 | ResqueCleaner supposes recent jobs are more important than old jobs. Therefore
270 | ResqueCleaner deals with **ONLY LAST X(default=1000) JOBS**. In this way, you
271 | could avoid slow responses. You can change the number through `limiter` attribute.
272 | 
273 | Let's see how it works with an following example.
274 | 
275 | **Sample Situation**
276 | 
277 | * Number of failed jobs: 100,000
278 | 
279 | Default limiter is 1000 so that the limiter returns 1000 as a count.
280 | 
281 | ```ruby
282 |     > cleaner.limiter.count
283 |     => 1,000
284 |     > cleaner.failure.count
285 |     => 100,000
286 | ```
287 | 
288 | You could know if the limiter is on with on? method.
289 | 
290 | ```ruby
291 |     > cleaner.limiter.on?
292 |     => true
293 | ```
294 | 
295 | You can change the maximum number of the limiter with maximum attribute.
296 | 
297 | ```ruby
298 |     > cleaner.limiter.maximum = 3000
299 |     => 3,000
300 |     > cleaner.limiter.count
301 |     => 3,000
302 |     > cleaner.limiter.on?
303 |     => true
304 | ```
305 | 
306 | With limiter, ResqueCleaner's filtering targets only the last X(3000 in this
307 | sample) failed jobs.
308 | 
309 | ```ruby
310 |     > cleaner.select.size
311 |     => 3,000
312 | ```
313 | 
314 | The clear\_stale method deletes all jobs entered prior to the last X(3000 in
315 | this sample) failed jobs. This calls Redis API and no iteration occurs on Ruby
316 | application; it should be quick even if there are huge number of failed jobs.
317 | 
318 | ```ruby
319 |     > cleaner.clear_stale
320 |     > cleaner.failure.count
321 |     => 3,000
322 |     > cleaner.limiter.count
323 |     => 3,000
324 |     > cleaner.limiter.on?
325 |     => false
326 | ```
327 | 
328 | Many Thanks!
329 | ------------
330 | 
331 | To our [Contributors](https://github.com/ono/resque-cleaner/contributors)
332 | 
333 | 
334 | 


--------------------------------------------------------------------------------
/lib/resque_cleaner.rb:
--------------------------------------------------------------------------------
  1 | require 'time'
  2 | require 'resque'
  3 | require 'resque/server'
  4 | 
  5 | module Resque
  6 |   module Plugins
  7 |     # ResqueCleaner class provides useful functionalities to retry or clean
  8 |     # failed jobs. Let's clean up your failed list!
  9 |     class ResqueCleaner
 10 |       # ResqueCleaner fetches all elements from Redis and checks them
 11 |       # by linear when filtering them. Since there is a performance concern,
 12 |       # ResqueCleaner handles only the latest x(default 1000) jobs.
 13 |       #
 14 |       # You can change the value through limiter attribute.
 15 |       # e.g. cleaner.limiter.maximum = 5000
 16 |       attr_reader :limiter
 17 | 
 18 |       # Set false if you don't show any message.
 19 |       attr_accessor :print_message
 20 | 
 21 |       # Initializes instance
 22 |       def initialize
 23 |         @failure = Resque::Failure.backend
 24 |         @print_message = true
 25 |         @limiter = Limiter.new self
 26 |       end
 27 | 
 28 |       # Returns redis instance.
 29 |       def redis
 30 |         Resque.redis
 31 |       end
 32 | 
 33 |       # Returns failure backend. Only supports redis backend.
 34 |       def failure
 35 |         @failure
 36 |       end
 37 | 
 38 |       # Stats by date.
 39 |       def stats_by_date(&block)
 40 |         jobs, stats = select(&block), {}
 41 |         jobs.each do |job|
 42 |           date = job["failed_at"][0,10]
 43 |           stats[date] ||= 0
 44 |           stats[date] += 1
 45 |         end
 46 | 
 47 |         print_stats(stats) if print?
 48 |         stats
 49 |       end
 50 | 
 51 |       # Stats by class.
 52 |       def stats_by_class(&block)
 53 |         jobs, stats = select(&block), {}
 54 |         jobs.each do |job|
 55 |           klass = job["payload"] && job["payload"]["class"] ? job["payload"]["class"] : "UNKNOWN"
 56 |           stats[klass] ||= 0
 57 |           stats[klass] += 1
 58 |         end
 59 | 
 60 |         print_stats(stats) if print?
 61 |         stats
 62 |       end
 63 | 
 64 |       # Stats by exception.
 65 |       def stats_by_exception(&block)
 66 |         jobs, stats = select(&block), {}
 67 |         jobs.each do |job|
 68 |           exception = job["exception"]
 69 |           stats[exception] ||= 0
 70 |           stats[exception] += 1
 71 |         end
 72 | 
 73 |         print_stats(stats) if print?
 74 |         stats
 75 |       end
 76 | 
 77 |       # Print stats
 78 |       def print_stats(stats)
 79 |         log too_many_message if @limiter.on?
 80 |         stats.keys.sort.each do |k|
 81 |           log "%15s: %4d" % [k,stats[k]]
 82 |         end
 83 |         log "%15s: %4d" % ["total", @limiter.count]
 84 |       end
 85 | 
 86 |       # Returns every jobs for which block evaluates to true.
 87 |       def select(&block)
 88 |         jobs = @limiter.jobs
 89 |         block_given? ? @limiter.jobs.select(&block) : jobs
 90 |       end
 91 |       alias :failure_jobs :select
 92 | 
 93 |       def select_by_regex(regex)
 94 |         select do |job|
 95 |           job.to_s =~ regex
 96 |         end
 97 |       end
 98 | 
 99 |       # Clears every jobs for which block evaluates to true.
100 |       def clear(&block)
101 |         cleared = 0
102 |         @limiter.lock do
103 |           @limiter.jobs.each_with_index do |job,i|
104 |             if !block_given? || block.call(job)
105 |               index = @limiter.start_index + i - cleared
106 |               # fetches again since you can't ensure that it is always true:
107 |               # a == endode(decode(a))
108 |               value = redis.lindex(:failed, index)
109 |               redis.lrem(:failed, 1, value)
110 |               cleared += 1
111 |             end
112 |           end
113 |         end
114 |         cleared
115 |       end
116 | 
117 |       # Retries every jobs for which block evaluates to true.
118 |       def requeue(clear_after_requeue=false, options={}, &block)
119 |         requeued = 0
120 |         queue = options["queue"] || options[:queue]
121 |         @limiter.lock do
122 |           @limiter.jobs.each_with_index do |job,i|
123 |             if !block_given? || block.call(job)
124 |               index = @limiter.start_index + i - requeued
125 | 
126 |               value = redis.lindex(:failed, index)
127 |               redis.multi do
128 |                 Job.create(queue||job['queue'], job['payload']['class'], *job['payload']['args'])
129 | 
130 |                 if clear_after_requeue
131 |                   # remove job
132 |                   # TODO: should use ltrim. not sure why i used lrem here...
133 |                   redis.lrem(:failed, 1, value)
134 |                 else
135 |                   # mark retried
136 |                   job['retried_at'] = Time.now.strftime("%Y/%m/%d %H:%M:%S")
137 |                   redis.lset(:failed, @limiter.start_index+i, Resque.encode(job))
138 |                 end
139 |               end
140 | 
141 |               requeued += 1
142 |             end
143 |           end
144 |         end
145 |         requeued
146 |       end
147 | 
148 |       # Clears all jobs except the last X jobs
149 |       def clear_stale
150 |         return 0 unless @limiter.on?
151 |         c = @limiter.maximum
152 |         redis.ltrim(:failed, -c, -1)
153 |         c
154 |       end
155 | 
156 |       # Exntends job(Hash instance) with some helper methods.
157 |       module FailedJobEx
158 |         # Returns true if the job has been already retried. Otherwise returns
159 |         # false.
160 |         def retried?
161 |           !self['retried_at'].nil?
162 |         end
163 |         alias :requeued? :retried?
164 | 
165 |         # Returns true if the job processed(failed) before the given time.
166 |         # Otherwise returns false.
167 |         # You can pass Time object or String.
168 |         def before?(time)
169 |           time = Time.parse(time) if time.is_a?(String)
170 |           Time.parse(self['failed_at']) < time
171 |         end
172 | 
173 |         # Returns true if the job processed(failed) after the given time.
174 |         # Otherwise returns false.
175 |         # You can pass Time object or String.
176 |         def after?(time)
177 |           time = Time.parse(time) if time.is_a?(String)
178 |           Time.parse(self['failed_at']) >= time
179 |         end
180 | 
181 |         # Returns true if the class of the job matches. Otherwise returns false.
182 |         def klass?(klass_or_name)
183 |           if self["payload"] && self["payload"]["class"]
184 |             self["payload"]["class"] == klass_or_name.to_s
185 |           else
186 |             klass_or_name=="UNKNOWN"
187 |           end
188 |         end
189 | 
190 |         # Returns true if the exception raised by the failed job matches. Otherwise returns false.
191 |         def exception?(exception)
192 |           self["exception"] == exception.to_s
193 |         end
194 | 
195 |         # Returns true if the queue of the job matches. Otherwise returns false.
196 |         def queue?(queue)
197 |           self["queue"] == queue.to_s
198 |         end
199 |       end
200 | 
201 |       # Through the Limiter class, you accesses only the last x(default 1000)
202 |       # jobs.
203 |       class Limiter
204 |         @@default_maximum ||= 1000
205 | 
206 |         class << self
207 |           def default_maximum
208 |             @@default_maximum
209 |           end
210 | 
211 |           def default_maximum=(v)
212 |             @@default_maximum = v
213 |           end
214 |         end
215 | 
216 |         attr_accessor :maximum
217 |         def initialize(cleaner)
218 |           @cleaner = cleaner
219 |           @maximum = @@default_maximum
220 |           @locked = false
221 |         end
222 | 
223 |         # Returns true if limiter is ON: number of failed jobs is more than
224 |         # maximum value.
225 |         def on?
226 |           @cleaner.failure.count > @maximum
227 |         end
228 | 
229 |         # Returns limited count.
230 |         def count
231 |           if @locked
232 |             @jobs.size
233 |           else
234 |             on? ? @maximum : @cleaner.failure.count
235 |           end
236 |         end
237 | 
238 |         # Returns jobs. If numbers of jobs is more than maximum, it returns only
239 |         # the maximum.
240 |         def jobs
241 |           if @locked
242 |             @jobs
243 |           else
244 |             all( - count, count)
245 |           end
246 |         end
247 | 
248 |         # Wraps Resque's all and returns always array.
249 |         def all(index=0,count=1)
250 |           jobs = @cleaner.failure.all( index, count)
251 |           jobs = [] unless jobs
252 |           jobs = [jobs] unless jobs.is_a?(Array)
253 |           jobs.each{|j| j.extend FailedJobEx}
254 |           jobs
255 |         end
256 | 
257 |         # Returns a start index of jobs in :failed list.
258 |         def start_index
259 |           if @locked
260 |             @start_index
261 |           else
262 |             on? ? @cleaner.failure.count-@maximum : 0
263 |           end
264 |         end
265 | 
266 |         # Assuming new failures pushed while cleaner is dealing with failures,
267 |         # you need to lock the range.
268 |         def lock
269 |           old = @locked
270 | 
271 |           unless @locked
272 |             total_count = @cleaner.failure.count
273 |             if total_count>@maximum
274 |               @start_index = total_count-@maximum
275 |               @jobs = all( @start_index, @maximum)
276 |             else
277 |               @start_index = 0
278 |               @jobs = all( 0, total_count)
279 |             end
280 |           end
281 | 
282 |           @locked = true
283 |           yield
284 |         ensure
285 |           @locked = old
286 |         end
287 |       end
288 | 
289 |       # Outputs message. Overrides this method when you want to change a output
290 |       # stream.
291 |       def log(msg)
292 |         puts msg if print?
293 |       end
294 | 
295 |       def print?
296 |         @print_message
297 |       end
298 | 
299 |       def too_many_message
300 |         "There are too many failed jobs(count=#{@failure.count}). This only looks at last #{@limiter.maximum} jobs."
301 |       end
302 |     end
303 |   end
304 | end
305 | 
306 | require 'resque_cleaner/server'
307 | 
308 | 


--------------------------------------------------------------------------------