├── Gemfile ├── .gitignore ├── lib ├── attentive_sidekiq │ ├── version.rb │ ├── middleware │ │ ├── server │ │ │ └── attentionist.rb │ │ └── client │ │ │ └── attentionist.rb │ ├── updater_observer.rb │ ├── web │ │ ├── locales │ │ │ ├── ru.yml │ │ │ ├── en.yml │ │ │ └── cs.yml │ │ └── views │ │ │ └── disappeared-list.erb │ ├── web.rb │ ├── manager.rb │ └── api.rb └── attentive_sidekiq.rb ├── Rakefile ├── circle.yml ├── test ├── test_helper.rb ├── manager_test.rb ├── web_test.rb ├── api_test.rb └── server_middleware_test.rb ├── LICENSE.txt ├── attentive_sidekiq.gemspec └── README.md /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .bundle 3 | .config 4 | Gemfile.lock 5 | rdoc 6 | tmp 7 | *.swp 8 | *.rdb 9 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/version.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | VERSION = '0.3.3'.freeze 3 | end 4 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake/testtask' 2 | 3 | Rake::TestTask.new do |t| 4 | t.libs << 'test' 5 | t.pattern = "test/*_test.rb" 6 | end 7 | 8 | desc "Run tests" 9 | task :default => :test 10 | -------------------------------------------------------------------------------- /circle.yml: -------------------------------------------------------------------------------- 1 | machine: 2 | timezone: 3 | Europe/Moscow 4 | ruby: 5 | version: 2.2.2 6 | environment: 7 | REDIS_URL: redis://localhost:6379 8 | services: 9 | - redis 10 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/middleware/server/attentionist.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | module Middleware 3 | module Server 4 | class Attentionist 5 | def call(worker_instance, item, queue) 6 | Suspicious.add(item) 7 | yield 8 | ensure 9 | Suspicious.remove(item['jid']) 10 | end 11 | end 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/middleware/client/attentionist.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | module Middleware 3 | module Client 4 | class Attentionist 5 | def call(worker_class, item, queue, redis_pool = nil) 6 | # TODO: we could backup job info here aswell 7 | # this would lead us to the need of more complex records filtering 8 | yield 9 | end 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/updater_observer.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | class UpdaterObserver 3 | def update time, result, ex 4 | if result 5 | AttentiveSidekiq.logger.info("#{time} [AttentiveSidekiq] Finished updating with result #{result}") 6 | elsif ex.is_a?(Concurrent::TimeoutError) 7 | AttentiveSidekiq.logger.error("#{time} [AttentiveSidekiq] Execution timed out") 8 | else 9 | AttentiveSidekiq.logger.error("#{time } [AttentiveSidekiq] Execution failed with error #{ex}\n") 10 | end 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/web/locales/ru.yml: -------------------------------------------------------------------------------- 1 | ru: 2 | jid: JID 3 | queue: Очередь 4 | class: Задача 5 | arguments: Аргументы 6 | disappeared_jobs: Пропавшие задачи 7 | created_at: Создана 8 | noticed_at: Пропажа замечена 9 | time_unknown: время неизвестно 10 | actions: Действия 11 | delete: Удалить 12 | delete_confirmation: Вы уверены, что хотите удалить эту запись? 13 | status: Статус 14 | detected: Обнаружена 15 | requeued: Поставлена в очередь 16 | requeue: Перезапустить 17 | requeue_confirmation: Вы уверены, что хотите перезапустить задачу? 18 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/web/locales/en.yml: -------------------------------------------------------------------------------- 1 | en: 2 | jid: JID 3 | queue: Queue name 4 | class: Class name 5 | arguments: Arguments 6 | disappeared_jobs: Disappeared jobs 7 | created_at: Created at 8 | noticed_at: Noticed disappeared at 9 | time_unknown: Time unknown 10 | actions: Actions 11 | delete: Delete 12 | delete_confirmation: Are you sure you want to delete this record? 13 | status: Status 14 | detected: Detected 15 | requeued: Requeued 16 | requeue: Requeue 17 | requeue_confirmation: Are you sure you want to requeue this job? 18 | delete_all: Delete all 19 | requeue_all: Requeue all 20 | delete_all_confirmation: Are you sure you want to delete all disappeared jobs? 21 | requeue_all_confirmation: Are you sure you want to requeue all disappeared jobs? 22 | no_disappeared_jobs: No dissapeared jobs. 23 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | ENV['RACK_ENV'] = ENV['RAILS_ENV'] = 'test' 2 | 3 | require 'pry' 4 | require 'sidekiq' 5 | require 'sidekiq/api' 6 | require 'sidekiq/cli' 7 | require 'sidekiq/processor' 8 | require 'sidekiq/manager' 9 | require 'sidekiq/util' 10 | require 'sidekiq/redis_connection' 11 | require 'redis-namespace' 12 | require 'attentive_sidekiq' 13 | require 'minitest/autorun' 14 | require 'minitest/pride' 15 | require 'minitest/stub_any_instance' 16 | 17 | REDIS_URL = ENV["REDIS_URL"] || "redis://localhost:6379/15" 18 | REDIS_NAMESPACE = ENV["REDIS_NAMESPACE"] || 'testy' 19 | REDIS = Sidekiq::RedisConnection.create(:url => REDIS_URL, :namespace => REDIS_NAMESPACE) 20 | 21 | Sidekiq.configure_server do |config| 22 | config.server_middleware do |chain| 23 | chain.add AttentiveSidekiq::Middleware::Server::Attentionist 24 | end 25 | end 26 | 27 | Sidekiq.logger = nil 28 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/web/locales/cs.yml: -------------------------------------------------------------------------------- 1 | cs: 2 | jid: JID 3 | queue: Název fronty 4 | class: Název třídy 5 | arguments: Argumenty 6 | disappeared_jobs: Ztracené úkoly 7 | created_at: Vytvořeno 8 | noticed_at: Noticed disappeared at 9 | time_unknown: neznámý čas 10 | actions: Akce 11 | delete: Smazat 12 | delete_confirmation: Jste si jisti, že chcete smazat tento záznam? 13 | status: Stav 14 | detected: Detekováno 15 | requeued: Znovu zařazeno 16 | requeue: Zařadit znovu 17 | requeue_confirmation: Jste si jisti, že chcete tento úkol znovu zařadit? 18 | delete_all: Smazat vše 19 | requeue_all: Zařadit vše znovu 20 | delete_all_confirmation: Jste si jisti, že chcete smazat všechny ztracené úkoly? 21 | requeue_all_confirmation: Jste si jisti, že chcete znovu zařadit všechny ztracené úkoly? 22 | no_disappeared_jobs: Nebyly nalezeny žádné ztracené úkoly. 23 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 twonegatives 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /attentive_sidekiq.gemspec: -------------------------------------------------------------------------------- 1 | lib = File.expand_path("../lib", __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | require 'attentive_sidekiq/version' 4 | 5 | Gem::Specification.new do |s| 6 | s.name = 'attentive_sidekiq' 7 | s.version = AttentiveSidekiq::VERSION 8 | s.summary = "Make your sidekiq to be attentive to lost jobs" 9 | s.description = "This gem allows you to watch the jobs which suddenly dissappeared from redis without being completed by redis worker" 10 | s.authors = ["twonegatives"] 11 | s.email = 'whitewhiteheaven@gmail.com' 12 | s.files = Dir['**/*'].keep_if{ |file| File.file?(file) } 13 | s.test_files = s.files.grep(%r{^(test|spec|features)/}) 14 | s.homepage = 15 | 'http://rubygems.org/gems/attentive_sidekiq' 16 | s.license = 'MIT' 17 | 18 | s.add_development_dependency 'sidekiq', '~> 4.2' 19 | s.add_development_dependency 'rake', '~> 11.3' 20 | s.add_development_dependency 'minitest', '~> 5.0' 21 | s.add_development_dependency 'minitest-stub_any_instance' 22 | s.add_development_dependency 'redis-namespace', '~> 1.5' 23 | s.add_development_dependency "rack-test", '~> 0.6' 24 | s.add_development_dependency 'pry', '~> 0.10' 25 | s.add_development_dependency "pry-byebug" 26 | 27 | s.add_dependency "activesupport" 28 | s.add_dependency 'concurrent-ruby', '~> 1.0' 29 | end 30 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq.rb: -------------------------------------------------------------------------------- 1 | require 'set' 2 | require 'active_support/inflector' 3 | require 'active_support/core_ext/hash' 4 | require 'concurrent' 5 | require 'attentive_sidekiq/api' 6 | require 'attentive_sidekiq/middleware/server/attentionist' 7 | require 'attentive_sidekiq/middleware/client/attentionist' 8 | require 'attentive_sidekiq/updater_observer' 9 | require 'attentive_sidekiq/manager' 10 | require 'sidekiq/web' unless defined?(Sidekiq::Web) 11 | require 'attentive_sidekiq/web' 12 | 13 | module AttentiveSidekiq 14 | DEFAULTS = { 15 | timeout_interval: 60, 16 | execution_interval: 600 17 | } 18 | 19 | REDIS_SUSPICIOUS_KEY = "attentive_observed_hash" 20 | REDIS_DISAPPEARED_KEY = "attentive_disappeared_hash" 21 | 22 | class << self 23 | attr_writer :timeout_interval, :execution_interval, :logger 24 | 25 | def timeout_interval 26 | return @timeout_interval if @timeout_interval 27 | @timeout_interval = options[:timeout_interval] || DEFAULTS[:timeout_interval] 28 | end 29 | 30 | def execution_interval 31 | return @execution_interval if @execution_interval 32 | @execution_interval = options[:execution_interval] || DEFAULTS[:execution_interval] 33 | end 34 | 35 | def logger 36 | @logger ||= Sidekiq.logger 37 | end 38 | 39 | def options 40 | Sidekiq.options[:attentive] || Sidekiq.options['attentive'] || {} 41 | end 42 | end 43 | end 44 | 45 | AttentiveSidekiq::Manager.instance.start! if Sidekiq.server? 46 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/web.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | module Web 3 | VIEW_PATH = File.expand_path("../web/views", __FILE__) 4 | 5 | def self.registered(app) 6 | app.get("/disappeared-jobs") do 7 | @disappeared_jobs = AttentiveSidekiq::Disappeared.jobs 8 | erb File.read(File.join(VIEW_PATH, 'disappeared-list.erb')) 9 | end 10 | 11 | app.post("/disappeared-jobs/requeue-all") do 12 | AttentiveSidekiq::Disappeared.jobs.each do |job| 13 | if job['status'] == 'detected' 14 | AttentiveSidekiq::Disappeared.requeue(job['jid']) 15 | end 16 | end 17 | redirect "#{root_path}disappeared-jobs" 18 | end 19 | 20 | app.post("/disappeared-jobs/delete-all") do 21 | AttentiveSidekiq::Disappeared.jobs.each do |job| 22 | AttentiveSidekiq::Disappeared.remove(job['jid']) 23 | end 24 | redirect "#{root_path}disappeared-jobs" 25 | end 26 | 27 | app.post("/disappeared-jobs/:jid/delete") do 28 | AttentiveSidekiq::Disappeared.remove(params['jid']) 29 | redirect "#{root_path}disappeared-jobs" 30 | end 31 | 32 | app.post("/disappeared-jobs/:jid/requeue") do 33 | AttentiveSidekiq::Disappeared.requeue(params['jid']) 34 | redirect "#{root_path}disappeared-jobs" 35 | end 36 | end 37 | end 38 | end 39 | 40 | Sidekiq::Web.register AttentiveSidekiq::Web 41 | Sidekiq::Web.locales << File.expand_path(File.dirname(__FILE__) + "/web/locales") 42 | Sidekiq::Web.tabs['disappeared_jobs'] = 'disappeared-jobs' 43 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/manager.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | class Manager 3 | @@instance = AttentiveSidekiq::Manager.new 4 | 5 | def self.instance 6 | @@instance 7 | end 8 | 9 | def start! 10 | task = Concurrent::TimerTask.new(options) do 11 | AttentiveSidekiq::Manager.instance.update_disappeared_jobs 12 | end 13 | task.add_observer(AttentiveSidekiq::UpdaterObserver.new) 14 | task.execute 15 | end 16 | 17 | def update_disappeared_jobs 18 | suspicious = AttentiveSidekiq::Suspicious.jobs 19 | active_ids = AttentiveSidekiq::Active.job_ids 20 | those_lost = suspicious.delete_if{|i| active_ids.include?(i["jid"])} 21 | 22 | # Sidekiq might have been too fast finishing up a job that appeared in the suspicious list 23 | # but didn't make it to the active list, so that's a false-positive. 24 | # We need to get the new suspicious list again, and remove any lost jobs that are no longer there. 25 | # Those jobs that appeared in the first suspicious list, but not the second one were simply finished 26 | # quickly by Sidekiq before showing up as active by a worker. 27 | suspicious = AttentiveSidekiq::Suspicious.jobs 28 | those_lost.delete_if{|i| !suspicious.any?{|j| i['jid'] == j['jid']} } 29 | 30 | those_lost.each do |job| 31 | Disappeared.add(job) 32 | Suspicious.remove(job['jid']) 33 | end 34 | end 35 | 36 | private_class_method :new 37 | 38 | private 39 | 40 | def options 41 | { 42 | execution_interval: AttentiveSidekiq.execution_interval, 43 | timeout_interval: AttentiveSidekiq.timeout_interval 44 | } 45 | end 46 | 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /test/manager_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | 3 | class ManagerTest < Minitest::Test 4 | describe "with real redis" do 5 | before do 6 | Sidekiq.redis = REDIS 7 | Sidekiq.redis{ |c| c.flushdb } 8 | 9 | common_hash = {'class' => 'UnexistantWorker', 'args' => [], 'created_at' => Time.now.to_i} 10 | @item_in_progress = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(common_hash) 11 | @item_disappeared = {'jid' => "YE11OW5-247", 'queue' => 'yellow_queue'}.merge!(common_hash) 12 | 13 | AttentiveSidekiq::Suspicious.add @item_in_progress 14 | AttentiveSidekiq::Suspicious.add @item_disappeared 15 | 16 | @active_job_ids = [@item_in_progress['jid']] 17 | end 18 | 19 | it "removes lone job from suspicious and adds to disappeared" do 20 | AttentiveSidekiq::Active.stub(:job_ids, @active_job_ids) do 21 | AttentiveSidekiq::Manager.instance.update_disappeared_jobs 22 | 23 | assert_includes disappeared_now, @item_disappeared 24 | refute_includes suspicious_now, @item_disappeared 25 | end 26 | end 27 | 28 | it "leaves jobs which are being currently processed in suspicious" do 29 | AttentiveSidekiq::Active.stub(:job_ids, @active_job_ids) do 30 | AttentiveSidekiq::Manager.instance.update_disappeared_jobs 31 | 32 | assert_includes suspicious_now, @item_in_progress 33 | refute_includes disappeared_now, @item_in_progress 34 | end 35 | end 36 | 37 | def suspicious_now 38 | Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.map{|i| JSON.parse(i)} 39 | end 40 | 41 | def disappeared_now 42 | from_redis = Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_DISAPPEARED_KEY)} 43 | from_redis.map{|i| JSON.parse(i).except('noticed_at', 'status')} 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/api.rb: -------------------------------------------------------------------------------- 1 | module AttentiveSidekiq 2 | class RedisBasedHash 3 | class << self 4 | def jobs 5 | Sidekiq.redis{|conn| conn.hvals(hash_name)}.map{|i| JSON.parse(i)} 6 | end 7 | 8 | def job_ids 9 | jobs.map{|i| i["jid"]} 10 | end 11 | 12 | def get_job jid 13 | JSON.parse(Sidekiq.redis{|conn| conn.hget(hash_name, jid)}) 14 | end 15 | 16 | def add item 17 | Sidekiq.redis{ |conn| conn.hset(hash_name, item['jid'], item.to_json) } 18 | end 19 | 20 | def remove jid 21 | Sidekiq.redis{|conn| conn.hdel(hash_name, jid)} 22 | end 23 | end 24 | end 25 | 26 | class Disappeared < RedisBasedHash 27 | STATUS_DETECTED = 'detected' 28 | STATUS_REQUEUED = 'requeued' 29 | SIDEKIQ_PUSH_OPTIONS = %w[queue class args retry backtrace].freeze 30 | 31 | class << self 32 | alias_method :base_add, :add 33 | 34 | def add item 35 | extended_item = {'noticed_at' => Time.now.to_i, 'status' => STATUS_DETECTED}.merge(item) 36 | super extended_item 37 | end 38 | 39 | def requeue jid 40 | record = get_job(jid) 41 | Sidekiq::Client.push(create_options(record)) 42 | 43 | base_add(record.merge('status' => STATUS_REQUEUED)) 44 | end 45 | 46 | def hash_name 47 | AttentiveSidekiq::REDIS_DISAPPEARED_KEY 48 | end 49 | 50 | private 51 | 52 | def create_options(item) 53 | SIDEKIQ_PUSH_OPTIONS.each_with_object({}) do |option, mem| 54 | mem[option] = item[option] if item.include?(option) 55 | end 56 | end 57 | end 58 | end 59 | 60 | class Suspicious < RedisBasedHash 61 | class << self 62 | def hash_name 63 | AttentiveSidekiq::REDIS_SUSPICIOUS_KEY 64 | end 65 | end 66 | end 67 | 68 | class Active 69 | class << self 70 | def jobs 71 | Sidekiq::Workers.new.to_a.map{|i| i[2]["payload"]} 72 | end 73 | 74 | def job_ids 75 | Set.new(jobs.map{|i| i["jid"]}) 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /test/web_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | require "rack/test" 3 | 4 | class WebTest < Minitest::Test 5 | include Rack::Test::Methods 6 | 7 | def setup 8 | Sidekiq.redis = REDIS 9 | Sidekiq.redis{ |c| c.flushdb } 10 | 11 | common_hash = {'class' => 'UnexistantWorker', 'args' => [], 'created_at' => Time.now.to_i} 12 | @item_disappeared = {'jid' => "YE11OW5-247", 'queue' => 'yellow_queue'}.merge!(common_hash) 13 | @item_in_progress = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(common_hash) 14 | 15 | AttentiveSidekiq::Suspicious.add(@item_in_progress) 16 | AttentiveSidekiq::Disappeared.add(@item_disappeared) 17 | end 18 | 19 | def test_displays_jobs_in_disappeared_hash 20 | get '/disappeared-jobs' 21 | assert_equal 200, last_response.status 22 | assert_match @item_disappeared['jid'], last_response.body 23 | end 24 | 25 | def test_does_not_display_jobs_not_in_disappeared_hash 26 | get '/disappeared-jobs' 27 | assert_equal 200, last_response.status 28 | refute_match @item_in_progress['jid'], last_response.body 29 | end 30 | 31 | def test_delete_all_route_functions_fine 32 | AttentiveSidekiq::Disappeared.stub(:remove, nil) do 33 | post "/disappeared-jobs/delete-all" 34 | follow_redirect! 35 | assert_equal 200, last_response.status 36 | end 37 | end 38 | 39 | def test_requeue_all_route_functions_fine 40 | AttentiveSidekiq::Disappeared.stub(:requeue, nil) do 41 | post "/disappeared-jobs/requeue-all" 42 | follow_redirect! 43 | assert_equal 200, last_response.status 44 | end 45 | end 46 | 47 | def test_delete_route_functions_fine 48 | AttentiveSidekiq::Disappeared.stub(:remove, nil) do 49 | post "/disappeared-jobs/#{@item_disappeared['jid']}/delete" 50 | follow_redirect! 51 | assert_equal 200, last_response.status 52 | end 53 | end 54 | 55 | def test_requeue_route_functions_fine 56 | AttentiveSidekiq::Disappeared.stub(:requeue, nil) do 57 | post "/disappeared-jobs/#{@item_disappeared['jid']}/requeue" 58 | follow_redirect! 59 | assert_equal 200, last_response.status 60 | end 61 | end 62 | 63 | private 64 | 65 | def app 66 | Sidekiq::Web 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /lib/attentive_sidekiq/web/views/disappeared-list.erb: -------------------------------------------------------------------------------- 1 |

<%= t('disappeared_jobs') %>

2 | 3 | <% if @disappeared_jobs.count > 0 %> 4 |
5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | <% @disappeared_jobs.each do |job| %> 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 41 | 42 | <% end %> 43 | 44 |
<%= t('jid') %><%= t('queue') %><%= t('class') %><%= t('arguments') %><%= t('created_at') %><%= t('noticed_at') %><%= t('status') %><%= t('actions') %>
<%= job['jid'] %><%= job['queue'] %><%= job['class'] %><%= job['args'] %><%= job['created_at'] ? Time.at(job['created_at']).strftime("%H:%M:%S %d.%m.%Y %z") : t("time_unknown") %><%= job['noticed_at'] ? Time.at(job['noticed_at']).strftime("%H:%M:%S %d.%m.%Y %z") : t("time_unknown") %><%= job['status'] ? t(job['status']) : t('detected') %> 30 |
" method="post"> 31 | <%= csrf_tag %> 32 | 33 |
34 | <% if job['status'] != AttentiveSidekiq::Disappeared::STATUS_REQUEUED %> 35 |
" method="post"> 36 | <%= csrf_tag %> 37 | 38 |
39 | <% end %> 40 |
45 |
46 | 47 |
" method="post"> 48 | <%= csrf_tag %> 49 | 50 |
51 | 52 |
" method="post"> 53 | <%= csrf_tag %> 54 | 55 |
56 | <% else %> 57 |
<%= t('no_disappeared_jobs') %>
58 | <% end %> 59 | -------------------------------------------------------------------------------- /test/api_test.rb: -------------------------------------------------------------------------------- 1 | require_relative "test_helper" 2 | 3 | class ApiTest < Minitest::Test 4 | describe "with real redis" do 5 | before do 6 | Sidekiq.redis = REDIS 7 | Sidekiq.redis { |c| c.flushdb } 8 | 9 | simple_hash = {'class' => 'ApiTest::SimpleWorker', 'args' => [], 'created_at' => Time.now.to_i} 10 | args_hash = {'args' => [1, "boo", "woo"], 'class' => 'ApiTest::WorkerWithArgs', 'created_at' => Time.now.to_i} 11 | optional_options = { 'retry' => true, 'backtrace' => false } 12 | 13 | @item_in_progress = {'jid' => "REDA-257513", 'queue' => 'red_queue'}.merge!(simple_hash) 14 | @disappeared_wo_args = {'jid' => "YE11OW5-247", 'queue' => 'yellow_queue'}.merge!(simple_hash) 15 | @disappeared_with_args = {'jid' => "B1UE-441", 'queue' => 'blue_queue'}.merge!(args_hash) 16 | @disappeared_with_all_args_possible = { 'jid' => 'B1UE-441-ALL', 'queue' => 'blue_queue' }.merge(args_hash.merge(optional_options)) 17 | 18 | AttentiveSidekiq::Suspicious.add(@item_in_progress) 19 | 20 | AttentiveSidekiq::Disappeared.add(@disappeared_wo_args) 21 | AttentiveSidekiq::Disappeared.add(@disappeared_with_args) 22 | AttentiveSidekiq::Disappeared.add(@disappeared_with_all_args_possible) 23 | end 24 | 25 | class BaseWorker 26 | include Sidekiq::Worker 27 | end 28 | 29 | class SimpleWorker < BaseWorker 30 | def perform 31 | end 32 | end 33 | 34 | class WorkerWithArgs < BaseWorker 35 | def perform a, b, c 36 | end 37 | end 38 | 39 | describe 'requeue' do 40 | it "adds jobs without args to sidekiq queue" do 41 | item = @disappeared_wo_args 42 | assert_equal 0, returned_queue(item).size 43 | 44 | AttentiveSidekiq::Disappeared.requeue(item['jid']) 45 | 46 | assert_equal 1, returned_queue(item).size 47 | element = item_in_returned_queue(item) 48 | assert_equal item['class'], element['class'] 49 | assert_equal item['args'], element['args'] 50 | end 51 | 52 | it 'adds 2 jobs with args to sidekiq queue' do 53 | item = @disappeared_with_args 54 | item_all_args = @disappeared_with_all_args_possible 55 | 56 | AttentiveSidekiq::Disappeared.requeue(item['jid']) 57 | AttentiveSidekiq::Disappeared.requeue(item_all_args['jid']) 58 | 59 | element = item_in_returned_queue(item) 60 | element_with_all_args = item_in_returned_queue(item_all_args, 1) 61 | 62 | assert_equal(item['class'], element['class']) 63 | assert_equal(item['args'], element['args']) 64 | 65 | (AttentiveSidekiq::Disappeared::SIDEKIQ_PUSH_OPTIONS - %w[backtrace]).each do |key| 66 | assert_equal(item_all_args[key], element_with_all_args[key]) 67 | end 68 | end 69 | 70 | it "marks jobs as requeued" do 71 | item = @disappeared_wo_args 72 | assert_equal job_status(item), AttentiveSidekiq::Disappeared::STATUS_DETECTED 73 | 74 | AttentiveSidekiq::Disappeared.requeue(item['jid']) 75 | assert_equal job_status(item), AttentiveSidekiq::Disappeared::STATUS_REQUEUED 76 | end 77 | 78 | def returned_queue(item) 79 | Sidekiq::Queue.new(item['queue']) 80 | end 81 | 82 | def item_in_returned_queue(item, position = 0) 83 | returned_queue(item).to_a[position] 84 | end 85 | 86 | def job_status(item) 87 | AttentiveSidekiq::Disappeared.get_job(item['jid'])['status'] 88 | end 89 | end 90 | end 91 | end 92 | -------------------------------------------------------------------------------- /test/server_middleware_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | 3 | class ServerMiddlewareTest < Minitest::Test 4 | describe "with real redis" do 5 | before do 6 | Sidekiq.redis = REDIS 7 | Sidekiq.redis{ |c| c.flushdb } 8 | 9 | @mutex = Mutex.new 10 | @stopper = ConditionVariable.new 11 | end 12 | 13 | class HardWorker 14 | include Sidekiq::Worker 15 | 16 | def perform(seed, work_amount = 10) 17 | raise "wrong amount of work" if work_amount <= 0 18 | 1.upto(work_amount) do |i| 19 | 1.upto(work_amount) do |j| 20 | 1.upto(work_amount) do |k| 21 | i*j*k 22 | end 23 | end 24 | end 25 | end 26 | end 27 | 28 | class SidekiqEmulator 29 | @@instance = SidekiqEmulator.new 30 | 31 | def self.instance 32 | @@instance 33 | end 34 | 35 | def process_jobs 36 | processor.send(:process, work_unit) 37 | end 38 | 39 | private_class_method :new 40 | 41 | private 42 | 43 | def processor 44 | ::Sidekiq::Processor.new(manager) 45 | end 46 | 47 | def manager 48 | options = { :concurrency => 1, :queues => ['default'] } 49 | Sidekiq::Manager.new(options) 50 | end 51 | 52 | def work_unit 53 | fetch = Sidekiq::BasicFetch.new(:queues => ['default']) 54 | fetch.retrieve_work 55 | end 56 | 57 | end 58 | 59 | class DefaultQueue 60 | @@instance = DefaultQueue.new 61 | 62 | def self.instance 63 | @@instance 64 | end 65 | 66 | def size 67 | queue.size rescue 0 68 | end 69 | 70 | def queue 71 | ::Sidekiq::Queue.new 72 | end 73 | 74 | private_class_method :new 75 | end 76 | 77 | it "does not mark job as suspicious while its queued" do 78 | assert_equal 0, DefaultQueue.instance.size 79 | HardWorker.perform_async(1) 80 | assert_equal 1, DefaultQueue.instance.size 81 | assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 82 | end 83 | 84 | it "marks job as suspicious as soon as it is started" do 85 | assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 86 | HardWorker.perform_async(2, 100_000) 87 | Thread.new{ 88 | SidekiqEmulator.instance.process_jobs 89 | } 90 | sleep(1) # TODO: refactor this somehow 91 | assert_equal 1, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 92 | end 93 | 94 | it "removes suspicious mark as soon as job is finished succesfully" do 95 | assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 96 | HardWorker.perform_async(2, 1) 97 | Thread.new{ 98 | @mutex.synchronize{ 99 | SidekiqEmulator.instance.process_jobs 100 | @stopper.signal 101 | } 102 | } 103 | @mutex.synchronize{ @stopper.wait(@mutex) } 104 | assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 105 | end 106 | 107 | it "removes suspicious mark as soon as job failed" do 108 | assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 109 | HardWorker.perform_async(2, -1) 110 | Thread.new{ 111 | @mutex.synchronize{ 112 | SidekiqEmulator.instance.process_jobs rescue nil 113 | @stopper.signal 114 | } 115 | } 116 | @mutex.synchronize{ @stopper.wait(@mutex) } 117 | assert_equal 0, Sidekiq.redis{|conn| conn.hvals(AttentiveSidekiq::REDIS_SUSPICIOUS_KEY)}.size 118 | end 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Attentive Sidekiq 2 | =========================== 3 | 4 | [![Gem Version](https://badge.fury.io/rb/attentive_sidekiq.svg)](https://badge.fury.io/rb/attentive_sidekiq) 5 | [![Code Climate](https://codeclimate.com/github/twonegatives/attentive_sidekiq/badges/gpa.svg)](https://codeclimate.com/github/twonegatives/attentive_sidekiq) 6 | [![CircleCI](https://circleci.com/gh/twonegatives/attentive_sidekiq.svg?style=shield)](https://circleci.com/gh/twonegatives/attentive_sidekiq) 7 | 8 | Be aware of suddenly disappeared sidekiq jobs! 9 | 10 | ![shortsummary](https://cloud.githubusercontent.com/assets/1937799/20489411/fe726e82-b023-11e6-9528-7df519fec7dd.gif) 11 | 12 | ### About 13 | The case of disappearing sidekiq jobs was proved by [github issues](https://github.com/mperham/sidekiq/issues/1831), [stackoverflow questions](http://stackoverflow.com/questions/35555000/current-sidekiq-job-lost-when-deploying-to-heroku) and (sadly) personal experience. Attentive Sidekiq was made to protect your jobs and critical data from silent disappearing. In case there appears a job being started but not finished and not being processing at the moment, you will know this. 14 | 15 | ### Usage 16 | Attentive Sidekiq provides you with a couple of useful API methods. 17 | 18 | To get a hash containing all information about jobs marked as disappeared: 19 | ```ruby 20 | AttentiveSidekiq::Disappeared.jobs 21 | ``` 22 | 23 | To get only JIDs of lost jobs: 24 | ```ruby 25 | AttentiveSidekiq::Disappeared.job_ids 26 | ``` 27 | 28 | To place a disappeared job back into queue: 29 | ```ruby 30 | AttentiveSidekiq::Disappeared.requeue(jid) 31 | ``` 32 | 33 | To remove a job from disappeared hash (e.g. after manual requeue): 34 | ```ruby 35 | AttentiveSidekiq::Disappeared.remove(jid) 36 | ``` 37 | 38 | ### Sidekiq Web integration 39 | You may also watch info about disappeared jobs in a web UI. 40 | Simply make sure you have Sidekiq UI enabled, then head right to the Disappeared Jobs tab in the navbar. 41 | The Web UI uses the API exclusively: anything you can do in the UI can be scripted with the API. 42 | 43 | ![webui](https://cloud.githubusercontent.com/assets/1937799/20490807/a01216d0-b028-11e6-96b7-c23fd67bdf89.png) 44 | 45 | ### Pre-requirements and notes 46 | 47 | - Attentive Sidekiq assumes you've got Sidekiq installed already. 48 | - You should make sure sidekiq process is started in order for disappeared jobs updater to work properly. 49 | - It was tested with Sidekiq version 4. Seamless functionality with lower sidekiq versions is not guaranteed. 50 | 51 | ### Installation 52 | 53 | Add this line to your application's Gemfile: 54 | 55 | gem 'attentive_sidekiq' 56 | 57 | And then execute: 58 | 59 | $ bundle 60 | 61 | Configure your middleware chains, lookup [Middleware usage](https://github.com/mperham/sidekiq/wiki/Middleware) on Sidekiq wiki for more info. 62 | 63 | ```ruby 64 | Sidekiq.configure_server do |config| 65 | config.server_middleware do |chain| 66 | chain.add AttentiveSidekiq::Middleware::Server::Attentionist 67 | end 68 | end 69 | ``` 70 | 71 | After that you can use your jobs as usual. 72 | 73 | ### Configuration 74 | 75 | Specify desired configuration inside of `sidekiq.yml` file: 76 | 77 | ```YML 78 | attentive: 79 | # Time in seconds between checks for disappeared jobs 80 | :execution_interval: 300 # default: 600 81 | # Time limit in seconds to perform disappeared jobs check 82 | :timeout_interval: 25 # default: 60 83 | ``` 84 | 85 | By default, Attentive Sidekiq uses `Sidekiq.logger` to log its work. You may change it in your initializer: 86 | 87 | ```ruby 88 | AttentiveSidekiq.logger = Logger.new("log/attentive_sidekiq.log") 89 | ``` 90 | 91 | ### Suggestions? Bugs? 92 | 93 | If you've got a question, feature suggestion or found a bug please add an [issue on GitHub](https://github.com/twonegatives/attentive_sidekiq/issues) or fork the project and send a pull request. 94 | --------------------------------------------------------------------------------