├── .rspec ├── .gitignore ├── lib ├── sidekiq-limit_fetch.rb └── sidekiq │ ├── limit_fetch │ ├── instances.rb │ ├── unit_of_work.rb │ ├── global │ │ ├── monitor.rb │ │ ├── selector.rb │ │ └── semaphore.rb │ └── queues.rb │ ├── extensions │ ├── manager.rb │ └── queue.rb │ └── limit_fetch.rb ├── Gemfile ├── demo ├── config │ ├── environment.rb │ ├── boot.rb │ ├── application.rb │ └── environments │ │ └── development.rb ├── app │ └── workers │ │ ├── a_worker.rb │ │ ├── b_worker.rb │ │ ├── fast_worker.rb │ │ ├── slow_worker.rb │ │ └── c_worker.rb ├── Gemfile ├── README.md └── Rakefile ├── .travis.yml ├── Rakefile ├── spec ├── spec_helper.rb └── sidekiq │ ├── limit_fetch │ ├── global │ │ └── monitor_spec.rb │ ├── semaphore_spec.rb │ └── queues_spec.rb │ ├── limit_fetch_spec.rb │ └── extensions │ └── queue_spec.rb ├── sidekiq-limit_fetch.gemspec ├── LICENSE.txt ├── bench └── compare.rb └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper --color 2 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | pkg/ 3 | .bundle/ 4 | -------------------------------------------------------------------------------- /lib/sidekiq-limit_fetch.rb: -------------------------------------------------------------------------------- 1 | require_relative 'sidekiq/limit_fetch' 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | gem 'sidekiq' 5 | -------------------------------------------------------------------------------- /demo/config/environment.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../application', __FILE__) 2 | Demo::Application.initialize! 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.3.1 4 | - 2.2.5 5 | - jruby-9.1.5.0 6 | services: 7 | - redis-server 8 | -------------------------------------------------------------------------------- /demo/config/boot.rb: -------------------------------------------------------------------------------- 1 | ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../../Gemfile', __FILE__) 2 | require 'bundler/setup' if File.exists?(ENV['BUNDLE_GEMFILE']) 3 | -------------------------------------------------------------------------------- /demo/app/workers/a_worker.rb: -------------------------------------------------------------------------------- 1 | class AWorker 2 | include Sidekiq::Worker 3 | sidekiq_options queue: :a 4 | 5 | def perform 6 | sleep 10 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /demo/app/workers/b_worker.rb: -------------------------------------------------------------------------------- 1 | class BWorker 2 | include Sidekiq::Worker 3 | sidekiq_options queue: :b 4 | 5 | def perform 6 | sleep 10 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /demo/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'rails' 4 | gem 'launchy' 5 | gem 'sidekiq', github: 'mperham/sidekiq' 6 | gem 'sidekiq-limit_fetch', path: '..' 7 | 8 | -------------------------------------------------------------------------------- /demo/app/workers/fast_worker.rb: -------------------------------------------------------------------------------- 1 | class FastWorker 2 | include Sidekiq::Worker 3 | sidekiq_options queue: :fast 4 | 5 | def perform 6 | sleep 0.2 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /demo/app/workers/slow_worker.rb: -------------------------------------------------------------------------------- 1 | class SlowWorker 2 | include Sidekiq::Worker 3 | sidekiq_options queue: :slow 4 | 5 | def perform 6 | sleep 1 7 | end 8 | end 9 | -------------------------------------------------------------------------------- /demo/app/workers/c_worker.rb: -------------------------------------------------------------------------------- 1 | class CWorker 2 | include Sidekiq::Worker 3 | sidekiq_options queue: :c 4 | 5 | def perform 6 | sleep 10 7 | end 8 | end 9 | 10 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler/gem_tasks' 2 | require 'rspec/core/rake_task' 3 | 4 | RSpec::Core::RakeTask.new 5 | 6 | task :default do 7 | rspec = Rake::Task[:spec] 8 | rspec.invoke 9 | ENV['namespace'] = 'namespace' 10 | rspec.reenable 11 | rspec.invoke 12 | end 13 | -------------------------------------------------------------------------------- /demo/config/application.rb: -------------------------------------------------------------------------------- 1 | require File.expand_path('../boot', __FILE__) 2 | 3 | require 'action_controller/railtie' 4 | require 'action_mailer/railtie' 5 | require 'sprockets/railtie' 6 | 7 | Bundler.require(:default, Rails.env) 8 | 9 | module Demo 10 | Application = Class.new Rails::Application 11 | end 12 | -------------------------------------------------------------------------------- /demo/config/environments/development.rb: -------------------------------------------------------------------------------- 1 | Demo::Application.configure do 2 | config.cache_classes = false 3 | config.eager_load = false 4 | config.consider_all_requests_local = true 5 | config.action_controller.perform_caching = false 6 | config.action_mailer.raise_delivery_errors = false 7 | config.active_support.deprecation = :log 8 | config.assets.debug = true 9 | end 10 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch/instances.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq::LimitFetch::Instances 2 | def self.extended(klass) 3 | klass.instance_variable_set :@instances, {} 4 | end 5 | 6 | def new(*args) 7 | @instances[args] ||= super 8 | end 9 | 10 | alias [] new 11 | 12 | def instances 13 | @instances.values 14 | end 15 | 16 | def reset_instances! 17 | @instances = {} 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/sidekiq/extensions/manager.rb: -------------------------------------------------------------------------------- 1 | class Sidekiq::Manager 2 | module InitLimitFetch 3 | def initialize(options={}) 4 | options[:fetch] = Sidekiq::LimitFetch 5 | super 6 | end 7 | 8 | def start 9 | Sidekiq::LimitFetch::Queues.start options 10 | Sidekiq::LimitFetch::Global::Monitor.start! 11 | super 12 | end 13 | end 14 | 15 | prepend InitLimitFetch 16 | end 17 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch/unit_of_work.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq 2 | class LimitFetch::UnitOfWork < BasicFetch::UnitOfWork 3 | def initialize(queue, job) 4 | super 5 | redis_retryable { Queue[queue_name].increase_busy } 6 | end 7 | 8 | def acknowledge 9 | redis_retryable { Queue[queue_name].decrease_busy } 10 | redis_retryable { Queue[queue_name].release } 11 | end 12 | 13 | def requeue 14 | super 15 | acknowledge 16 | end 17 | 18 | private 19 | 20 | def redis_retryable(&block) 21 | Sidekiq::LimitFetch.redis_retryable(&block) 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/sidekiq/extensions/queue.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq 2 | class Queue 3 | extend LimitFetch::Instances, Forwardable 4 | attr_reader :rname 5 | 6 | def_delegators :lock, 7 | :limit, :limit=, :limit_changed?, 8 | :process_limit, :process_limit=, 9 | :acquire, :release, 10 | :pause, :pause_for_ms, :unpause, 11 | :block, :unblock, 12 | :paused?, :blocking?, 13 | :unblocked, :block_except, 14 | :probed, :busy, 15 | :increase_busy, :decrease_busy, 16 | :local_busy?, :explain, 17 | :remove_locks_except! 18 | 19 | def lock 20 | @lock ||= LimitFetch::Global::Semaphore.new name 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'sidekiq/limit_fetch' 2 | 3 | Sidekiq.logger = nil 4 | Sidekiq.redis = { namespace: ENV['namespace'] } 5 | 6 | RSpec.configure do |config| 7 | config.order = :random 8 | config.disable_monkey_patching! 9 | config.raise_errors_for_deprecations! 10 | config.before do 11 | Sidekiq::Queue.reset_instances! 12 | Sidekiq.redis do |it| 13 | clean_redis = ->(queue) do 14 | it.pipelined do 15 | it.del "limit_fetch:limit:#{queue}" 16 | it.del "limit_fetch:process_limit:#{queue}" 17 | it.del "limit_fetch:busy:#{queue}" 18 | it.del "limit_fetch:probed:#{queue}" 19 | it.del "limit_fetch:pause:#{queue}" 20 | it.del "limit_fetch:block:#{queue}" 21 | end 22 | end 23 | 24 | clean_redis.call(name) if defined?(name) 25 | queues.each(&clean_redis) if defined?(queues) and queues.is_a? Array 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /sidekiq-limit_fetch.gemspec: -------------------------------------------------------------------------------- 1 | Gem::Specification.new do |gem| 2 | gem.name = 'sidekiq-limit_fetch' 3 | gem.version = '3.4.0' 4 | gem.license = 'MIT' 5 | gem.authors = 'brainopia' 6 | gem.email = 'brainopia@evilmartians.com' 7 | gem.summary = 'Sidekiq strategy to support queue limits' 8 | gem.homepage = 'https://github.com/brainopia/sidekiq-limit_fetch' 9 | gem.description = <<-DESCRIPTION 10 | Sidekiq strategy to restrict number of workers 11 | which are able to run specified queues simultaneously. 12 | DESCRIPTION 13 | 14 | gem.files = `git ls-files`.split($/) 15 | gem.test_files = gem.files.grep %r{^spec/} 16 | gem.require_paths = %w(lib) 17 | 18 | gem.add_dependency 'sidekiq', '>= 4' 19 | gem.add_development_dependency 'redis-namespace', '~> 1.5', '>= 1.5.2' 20 | gem.add_development_dependency 'rspec' 21 | gem.add_development_dependency 'rake' 22 | end 23 | -------------------------------------------------------------------------------- /spec/sidekiq/limit_fetch/global/monitor_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Sidekiq::LimitFetch::Global::Monitor do 2 | let(:monitor) { described_class.start! ttl, timeout } 3 | let(:ttl) { 1 } 4 | let(:queue) { Sidekiq::Queue[name] } 5 | let(:name) { 'default' } 6 | 7 | before { monitor } 8 | after { monitor.kill } 9 | 10 | context 'old locks' do 11 | let(:timeout) { 0.5 } 12 | 13 | it 'should remove invalidated old locks' do 14 | 2.times { queue.acquire } 15 | sleep 2*ttl 16 | expect(queue.probed).to eq 2 17 | 18 | allow(described_class).to receive(:update_heartbeat) 19 | sleep 2*ttl 20 | expect(queue.probed).to eq 0 21 | end 22 | 23 | it 'should remove invalid locks' do 24 | 2.times { queue.acquire } 25 | allow(described_class).to receive(:update_heartbeat) 26 | Sidekiq.redis do |it| 27 | it.del Sidekiq::LimitFetch::Global::Monitor::PROCESS_SET 28 | end 29 | sleep 2*ttl 30 | expect(queue.probed).to eq 0 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /demo/README.md: -------------------------------------------------------------------------------- 1 | This is a demo rails app with a configured sidekiq-limit_fetch. 2 | 3 | Its purpose is to check whether plugin works in certain situations. 4 | 5 | Application is preconfigured with two workers: 6 | - `app/workers/fast_worker.rb` which does `sleep 0.2` 7 | - `app/workers/slow_worker.rb` which does `sleep 1` 8 | 9 | There is also a rake task which can be invoked as `bundle exec rake demo:limit`: 10 | 11 | - it prefills sidekiq tasks 12 | 13 | ```ruby 14 | 100.times do 15 | SlowWorker.perform_async 16 | FastWorker.perform_async 17 | end 18 | ``` 19 | - sets sidekiq config 20 | 21 | ```yaml 22 | :verbose: false 23 | :concurrency: 4 24 | :queues: 25 | - slow 26 | - fast 27 | :limits: 28 | slow: 1 29 | ``` 30 | 31 | - and launches a sidekiq admin page with overview of queues in browser. 32 | The page is set to live-poll so effects of limits can be seen directly. 33 | 34 | 35 | To change simulation modify `Rakefile` or workers. 36 | 37 | Any bugs related to the plugin should be demonstrated with a reproduction from this base app. 38 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 brainopia 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /bench/compare.rb: -------------------------------------------------------------------------------- 1 | require 'benchmark' 2 | require 'sidekiq/cli' 3 | require 'sidekiq/api' 4 | 5 | total = (ARGV.shift || 50).to_i 6 | concurrency = ARGV.shift || 1 7 | limit = ARGV.shift 8 | 9 | if limit 10 | limit = nil if limit == 'nil' 11 | 12 | $:.unshift File.expand_path '../lib' 13 | require 'sidekiq-limit_fetch' 14 | Sidekiq::Queue['inline'].limit = limit 15 | Sidekiq.redis {|it| it.del 'limit_fetch:probed:inline' } 16 | Sidekiq::LimitFetch::Queues.send(:define_method, :set) {|*| } 17 | end 18 | 19 | Sidekiq::Queue.new('inline').clear 20 | 21 | class FastJob 22 | include Sidekiq::Worker 23 | sidekiq_options queue: :inline 24 | 25 | def perform(i) 26 | puts "job N#{i} is finished" 27 | end 28 | end 29 | 30 | class FinishJob 31 | include Sidekiq::Worker 32 | sidekiq_options queue: :inline 33 | 34 | def perform 35 | Process.kill 'INT', 0 36 | end 37 | end 38 | 39 | total.times {|i| FastJob.perform_async i+1 } 40 | FinishJob.perform_async 41 | 42 | Sidekiq::CLI.instance.tap do |cli| 43 | %w(validate! boot_system).each {|stub| cli.define_singleton_method(stub) {}} 44 | cli.parse ['-q inline', '-q other', "-c #{concurrency}"] 45 | 46 | puts Benchmark.measure { 47 | begin 48 | cli.run 49 | rescue Exception 50 | end 51 | } 52 | end 53 | -------------------------------------------------------------------------------- /spec/sidekiq/limit_fetch_spec.rb: -------------------------------------------------------------------------------- 1 | Thread.abort_on_exception = true 2 | 3 | RSpec.describe Sidekiq::LimitFetch do 4 | let(:options) {{ queues: queues, limits: limits }} 5 | let(:queues) { %w(queue1 queue1 queue2 queue2) } 6 | let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }} 7 | 8 | before do 9 | subject::Queues.start options 10 | 11 | Sidekiq.redis do |it| 12 | it.del 'queue:queue1' 13 | it.lpush 'queue:queue1', 'task1' 14 | it.lpush 'queue:queue1', 'task2' 15 | it.expire 'queue:queue1', 30 16 | end 17 | end 18 | 19 | it 'should acquire lock on queue for execution' do 20 | work = subject.retrieve_work 21 | expect(work.queue_name).to eq 'queue1' 22 | expect(work.job).to eq 'task1' 23 | 24 | expect(Sidekiq::Queue['queue1'].busy).to eq 1 25 | expect(Sidekiq::Queue['queue2'].busy).to eq 0 26 | 27 | expect(subject.retrieve_work).not_to be 28 | work.requeue 29 | 30 | expect(Sidekiq::Queue['queue1'].busy).to eq 0 31 | expect(Sidekiq::Queue['queue2'].busy).to eq 0 32 | 33 | work = subject.retrieve_work 34 | expect(work.job).to eq 'task1' 35 | 36 | expect(Sidekiq::Queue['queue1'].busy).to eq 1 37 | expect(Sidekiq::Queue['queue2'].busy).to eq 0 38 | 39 | expect(subject.retrieve_work).not_to be 40 | work.acknowledge 41 | 42 | expect(Sidekiq::Queue['queue1'].busy).to eq 0 43 | expect(Sidekiq::Queue['queue2'].busy).to eq 0 44 | 45 | work = subject.retrieve_work 46 | expect(work.job).to eq 'task2' 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch.rb: -------------------------------------------------------------------------------- 1 | require 'forwardable' 2 | require 'sidekiq' 3 | require 'sidekiq/manager' 4 | require 'sidekiq/api' 5 | 6 | module Sidekiq::LimitFetch 7 | autoload :UnitOfWork, 'sidekiq/limit_fetch/unit_of_work' 8 | 9 | require_relative 'limit_fetch/instances' 10 | require_relative 'limit_fetch/queues' 11 | require_relative 'limit_fetch/global/semaphore' 12 | require_relative 'limit_fetch/global/selector' 13 | require_relative 'limit_fetch/global/monitor' 14 | require_relative 'extensions/queue' 15 | require_relative 'extensions/manager' 16 | 17 | extend self 18 | 19 | def new(_) 20 | self 21 | end 22 | 23 | def retrieve_work 24 | queue, job = redis_brpop(Queues.acquire) 25 | Queues.release_except(queue) 26 | UnitOfWork.new(queue, job) if job 27 | end 28 | 29 | # Backwards compatibility for sidekiq v6.1.0 30 | # @see https://github.com/mperham/sidekiq/pull/4602 31 | def bulk_requeue(*args) 32 | if Sidekiq::BasicFetch.respond_to?(:bulk_requeue) # < 6.1.0 33 | Sidekiq::BasicFetch.bulk_requeue(*args) 34 | else # 6.1.0+ 35 | Sidekiq::BasicFetch.new(Sidekiq.options).bulk_requeue(*args) 36 | end 37 | end 38 | 39 | def redis_retryable 40 | yield 41 | rescue Redis::BaseConnectionError 42 | sleep 1 43 | retry 44 | end 45 | 46 | private 47 | 48 | TIMEOUT = Sidekiq::BasicFetch::TIMEOUT 49 | 50 | def redis_brpop(queues) 51 | if queues.empty? 52 | sleep TIMEOUT # there are no queues to handle, so lets sleep 53 | [] # and return nothing 54 | else 55 | redis_retryable { Sidekiq.redis { |it| it.brpop *queues, TIMEOUT } } 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/sidekiq/limit_fetch/semaphore_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe 'semaphore' do 2 | let(:name) { 'default' } 3 | subject { Sidekiq::LimitFetch::Global::Semaphore.new name } 4 | 5 | it 'should have no limit by default' do 6 | expect(subject.limit).not_to be 7 | end 8 | 9 | it 'should set limit' do 10 | subject.limit = 4 11 | expect(subject.limit).to eq 4 12 | end 13 | 14 | it 'should acquire and count active tasks' do 15 | 3.times { subject.acquire } 16 | expect(subject.probed).to eq 3 17 | end 18 | 19 | it 'should acquire tasks with regard to limit' do 20 | subject.limit = 4 21 | 6.times { subject.acquire } 22 | expect(subject.probed).to eq 4 23 | end 24 | 25 | it 'should acquire tasks with regard to process limit' do 26 | subject.process_limit = 4 27 | 6.times { subject.acquire } 28 | expect(subject.probed).to eq 4 29 | end 30 | 31 | it 'should release active tasks' do 32 | 6.times { subject.acquire } 33 | 3.times { subject.release } 34 | expect(subject.probed).to eq 3 35 | end 36 | 37 | it 'should pause tasks' do 38 | 3.times { subject.acquire } 39 | subject.pause 40 | 2.times { subject.acquire } 41 | expect(subject.probed).to eq 3 42 | 2.times { subject.release } 43 | expect(subject.probed).to eq 1 44 | end 45 | 46 | it 'should unpause tasks' do 47 | subject.pause 48 | 3.times { subject.acquire } 49 | subject.unpause 50 | 2.times { subject.acquire } 51 | expect(subject.probed).to eq 2 52 | end 53 | 54 | it 'should pause tasks for a limited time' do 55 | 3.times { subject.acquire } 56 | subject.pause_for_ms 50 57 | 2.times { subject.acquire } 58 | expect(subject.probed).to eq 3 59 | sleep(100.0 / 1000) 60 | 2.times { subject.acquire } 61 | expect(subject.probed).to eq 5 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch/global/monitor.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq::LimitFetch::Global 2 | module Monitor 3 | extend self 4 | 5 | HEARTBEAT_PREFIX = 'limit:heartbeat:' 6 | PROCESS_SET = 'limit:processes' 7 | HEARTBEAT_TTL = 20 8 | REFRESH_TIMEOUT = 5 9 | 10 | def start!(ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT) 11 | Thread.new do 12 | loop do 13 | Sidekiq::LimitFetch.redis_retryable do 14 | add_dynamic_queues 15 | update_heartbeat ttl 16 | invalidate_old_processes 17 | end 18 | 19 | sleep timeout 20 | end 21 | end 22 | end 23 | 24 | def all_processes 25 | Sidekiq.redis {|it| it.smembers PROCESS_SET } 26 | end 27 | 28 | def old_processes 29 | all_processes.reject do |process| 30 | Sidekiq.redis {|it| it.get heartbeat_key process } 31 | end 32 | end 33 | 34 | def remove_old_processes! 35 | Sidekiq.redis do |it| 36 | old_processes.each {|process| it.srem PROCESS_SET, process } 37 | end 38 | end 39 | 40 | def add_dynamic_queues 41 | queues = Sidekiq::LimitFetch::Queues 42 | queues.add Sidekiq::Queue.all.map(&:name) if queues.dynamic? 43 | end 44 | 45 | private 46 | 47 | def update_heartbeat(ttl) 48 | Sidekiq.redis do |it| 49 | it.multi do 50 | it.set heartbeat_key, true 51 | it.sadd PROCESS_SET, Selector.uuid 52 | it.expire heartbeat_key, ttl 53 | end 54 | end 55 | end 56 | 57 | def invalidate_old_processes 58 | Sidekiq.redis do |it| 59 | remove_old_processes! 60 | processes = all_processes 61 | 62 | Sidekiq::Queue.instances.each do |queue| 63 | queue.remove_locks_except! processes 64 | end 65 | end 66 | end 67 | 68 | def heartbeat_key(process=Selector.uuid) 69 | HEARTBEAT_PREFIX + process 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /demo/Rakefile: -------------------------------------------------------------------------------- 1 | require File.expand_path('../config/application', __FILE__) 2 | Demo::Application.load_tasks 3 | 4 | namespace :demo do 5 | task limit: :environment do 6 | puts '=> Creating sidekiq tasks' 7 | 8 | 100.times do 9 | SlowWorker.perform_async 10 | FastWorker.perform_async 11 | end 12 | 13 | run_sidekiq_monitoring 14 | run_sidekiq_workers config: <<-YAML 15 | :verbose: false 16 | :concurrency: 4 17 | :queues: 18 | - slow 19 | - fast 20 | :limits: 21 | slow: 1 22 | YAML 23 | end 24 | 25 | task blocking: :environment do 26 | puts '=> Creating sidekiq tasks' 27 | 28 | AWorker.perform_async 29 | BWorker.perform_async 30 | CWorker.perform_async 31 | 32 | run_sidekiq_monitoring 33 | run_sidekiq_workers config: <<-YAML 34 | :verbose: false 35 | :concurrency: 4 36 | :queues: 37 | - a 38 | - b 39 | - c 40 | :blocking: 41 | - a 42 | YAML 43 | end 44 | 45 | task advanced_blocking: :environment do 46 | puts '=> Creating sidekiq tasks' 47 | 48 | AWorker.perform_async 49 | BWorker.perform_async 50 | CWorker.perform_async 51 | 52 | run_sidekiq_monitoring 53 | run_sidekiq_workers config: <<-YAML 54 | :verbose: false 55 | :concurrency: 4 56 | :queues: 57 | - a 58 | - b 59 | - c 60 | :blocking: 61 | - [a, b] 62 | YAML 63 | end 64 | def with_sidekiq_config(config) 65 | whitespace_offset = config[/\A */].size 66 | config.gsub! /^ {#{whitespace_offset}}/, '' 67 | 68 | puts "=> Use sidekiq config:\n#{config}" 69 | File.write 'config/sidekiq.yml', config 70 | yield 71 | ensure 72 | FileUtils.rm 'config/sidekiq.yml' 73 | end 74 | 75 | def run_sidekiq_monitoring 76 | require 'sidekiq/web' 77 | Thread.new do 78 | Rack::Server.start app: Sidekiq::Web, Port: 3000 79 | end 80 | sleep 1 81 | Launchy.open 'http://127.0.0.1:3000/busy?poll=true' 82 | end 83 | 84 | def run_sidekiq_workers(options) 85 | require 'sidekiq/cli' 86 | cli = Sidekiq::CLI.instance 87 | 88 | %w(validate! boot_system).each do |stub| 89 | cli.define_singleton_method(stub) {} 90 | end 91 | 92 | with_sidekiq_config options[:config] do 93 | cli.send :setup_options, [] 94 | end 95 | 96 | cli.run 97 | end 98 | end 99 | 100 | -------------------------------------------------------------------------------- /spec/sidekiq/extensions/queue_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Sidekiq::Queue do 2 | context 'singleton' do 3 | shared_examples :constructor do 4 | it 'with default name' do 5 | new_object = -> { described_class.send constructor } 6 | expect(new_object.call).to eq new_object.call 7 | end 8 | 9 | it 'with given name' do 10 | new_object = ->(name) { described_class.send constructor, name } 11 | expect(new_object.call('name')).to eq new_object.call('name') 12 | end 13 | end 14 | 15 | context '.new' do 16 | let(:constructor) { :new } 17 | it_behaves_like :constructor 18 | end 19 | 20 | context '.[]' do 21 | let(:constructor) { :[] } 22 | it_behaves_like :constructor 23 | end 24 | 25 | context '#lock' do 26 | let(:name) { 'example' } 27 | let(:queue) { Sidekiq::Queue[name] } 28 | 29 | it 'should be available' do 30 | expect(queue.acquire).to be 31 | end 32 | 33 | it 'should be pausable' do 34 | queue.pause 35 | expect(queue.acquire).not_to be 36 | end 37 | 38 | it 'should be continuable' do 39 | queue.pause 40 | queue.unpause 41 | expect(queue.acquire).to be 42 | end 43 | 44 | it 'should be limitable' do 45 | queue.limit = 1 46 | expect(queue.acquire).to be 47 | expect(queue.acquire).not_to be 48 | end 49 | 50 | it 'should be resizable' do 51 | queue.limit = 0 52 | expect(queue.acquire).not_to be 53 | queue.limit = nil 54 | expect(queue.acquire).to be 55 | end 56 | 57 | it 'should be countable' do 58 | queue.limit = 3 59 | 5.times { queue.acquire } 60 | expect(queue.probed).to eq 3 61 | end 62 | 63 | it 'should be releasable' do 64 | queue.acquire 65 | expect(queue.probed).to eq 1 66 | queue.release 67 | expect(queue.probed).to eq 0 68 | end 69 | 70 | it 'should tell if paused' do 71 | expect(queue).not_to be_paused 72 | queue.pause 73 | expect(queue).to be_paused 74 | queue.unpause 75 | expect(queue).not_to be_paused 76 | end 77 | 78 | it 'should tell if blocking' do 79 | expect(queue).not_to be_blocking 80 | queue.block 81 | expect(queue).to be_blocking 82 | queue.unblock 83 | expect(queue).not_to be_blocking 84 | end 85 | 86 | it 'should be marked as changed' do 87 | queue = Sidekiq::Queue["uniq_#{name}"] 88 | expect(queue).not_to be_limit_changed 89 | queue.limit = 3 90 | expect(queue).to be_limit_changed 91 | end 92 | end 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch/queues.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq::LimitFetch::Queues 2 | extend self 3 | 4 | THREAD_KEY = :acquired_queues 5 | 6 | def start(options) 7 | @queues = options[:queues] 8 | @dynamic = options[:dynamic] 9 | 10 | @limits = options[:limits] || {} 11 | @process_limits = options[:process_limits] || {} 12 | @blocks = options[:blocking] || [] 13 | 14 | options[:strict] ? strict_order! : weighted_order! 15 | 16 | apply_process_limit_to_queues 17 | apply_limit_to_queues 18 | apply_blocks_to_queues 19 | end 20 | 21 | def acquire 22 | selector.acquire(ordered_queues, namespace) 23 | .tap {|it| save it } 24 | .map {|it| "queue:#{it}" } 25 | end 26 | 27 | def release_except(full_name) 28 | queues = restore 29 | queues.delete full_name[/queue:(.*)/, 1] if full_name 30 | 31 | Sidekiq::LimitFetch.redis_retryable do 32 | selector.release queues, namespace 33 | end 34 | end 35 | 36 | def dynamic? 37 | @dynamic 38 | end 39 | 40 | def add(queues) 41 | queues.each do |queue| 42 | unless @queues.include? queue 43 | apply_process_limit_to_queue(queue) 44 | apply_limit_to_queue(queue) 45 | 46 | @queues.push queue 47 | end 48 | end 49 | end 50 | 51 | def strict_order! 52 | @queues.uniq! 53 | def ordered_queues; @queues end 54 | end 55 | 56 | def weighted_order! 57 | def ordered_queues; @queues.shuffle.uniq end 58 | end 59 | 60 | def namespace 61 | @namespace ||= Sidekiq.redis do |it| 62 | if it.respond_to?(:namespace) and it.namespace 63 | "#{it.namespace}:" 64 | else 65 | '' 66 | end 67 | end 68 | end 69 | 70 | private 71 | 72 | def apply_process_limit_to_queues 73 | @queues.uniq.each do |queue_name| 74 | apply_process_limit_to_queue(queue_name) 75 | end 76 | end 77 | 78 | def apply_process_limit_to_queue(queue_name) 79 | queue = Sidekiq::Queue[queue_name] 80 | queue.process_limit = @process_limits[queue_name.to_s] || @process_limits[queue_name.to_sym] 81 | end 82 | 83 | def apply_limit_to_queues 84 | @queues.uniq.each do |queue_name| 85 | apply_limit_to_queue(queue_name) 86 | end 87 | end 88 | 89 | def apply_limit_to_queue(queue_name) 90 | queue = Sidekiq::Queue[queue_name] 91 | 92 | unless queue.limit_changed? 93 | queue.limit = @limits[queue_name.to_s] || @limits[queue_name.to_sym] 94 | end 95 | end 96 | 97 | def apply_blocks_to_queues 98 | @queues.uniq.each do |queue_name| 99 | Sidekiq::Queue[queue_name].unblock 100 | end 101 | 102 | @blocks.to_a.each do |it| 103 | if it.is_a? Array 104 | it.each {|name| Sidekiq::Queue[name].block_except it } 105 | else 106 | Sidekiq::Queue[it].block 107 | end 108 | end 109 | end 110 | 111 | def selector 112 | Sidekiq::LimitFetch::Global::Selector 113 | end 114 | 115 | def save(queues) 116 | Thread.current[THREAD_KEY] = queues 117 | end 118 | 119 | def restore 120 | Thread.current[THREAD_KEY] || [] 121 | ensure 122 | Thread.current[THREAD_KEY] = nil 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /spec/sidekiq/limit_fetch/queues_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Sidekiq::LimitFetch::Queues do 2 | let(:queues) { %w[queue1 queue2] } 3 | let(:limits) {{ 'queue1' => 3 }} 4 | let(:strict) { true } 5 | let(:blocking) {} 6 | let(:process_limits) {{ 'queue2' => 3 }} 7 | 8 | let(:options) do 9 | { queues: queues, 10 | limits: limits, 11 | strict: strict, 12 | blocking: blocking, 13 | process_limits: process_limits } 14 | end 15 | 16 | before { subject.start options } 17 | 18 | it 'should acquire queues' do 19 | subject.acquire 20 | expect(Sidekiq::Queue['queue1'].probed).to eq 1 21 | expect(Sidekiq::Queue['queue2'].probed).to eq 1 22 | end 23 | 24 | it 'should acquire dynamically blocking queues' do 25 | subject.acquire 26 | expect(Sidekiq::Queue['queue1'].probed).to eq 1 27 | expect(Sidekiq::Queue['queue2'].probed).to eq 1 28 | 29 | Sidekiq::Queue['queue1'].block 30 | 31 | subject.acquire 32 | expect(Sidekiq::Queue['queue1'].probed).to eq 2 33 | expect(Sidekiq::Queue['queue2'].probed).to eq 1 34 | end 35 | 36 | it 'should block except given queues' do 37 | Sidekiq::Queue['queue1'].block_except 'queue2' 38 | subject.acquire 39 | expect(Sidekiq::Queue['queue1'].probed).to eq 1 40 | expect(Sidekiq::Queue['queue2'].probed).to eq 1 41 | 42 | Sidekiq::Queue['queue1'].block_except 'queue404' 43 | subject.acquire 44 | expect(Sidekiq::Queue['queue1'].probed).to eq 2 45 | expect(Sidekiq::Queue['queue2'].probed).to eq 1 46 | end 47 | 48 | it 'should release queues' do 49 | subject.acquire 50 | subject.release_except nil 51 | expect(Sidekiq::Queue['queue1'].probed).to eq 0 52 | expect(Sidekiq::Queue['queue2'].probed).to eq 0 53 | end 54 | 55 | it 'should release queues except selected' do 56 | subject.acquire 57 | subject.release_except 'queue:queue1' 58 | expect(Sidekiq::Queue['queue1'].probed).to eq 1 59 | expect(Sidekiq::Queue['queue2'].probed).to eq 0 60 | end 61 | 62 | it 'should release when no queues was acquired' do 63 | queues.each {|name| Sidekiq::Queue[name].pause } 64 | subject.acquire 65 | expect { subject.release_except nil }.not_to raise_exception 66 | end 67 | 68 | context 'blocking' do 69 | let(:blocking) { %w(queue1) } 70 | 71 | it 'should acquire blocking queues' do 72 | 3.times { subject.acquire } 73 | expect(Sidekiq::Queue['queue1'].probed).to eq 3 74 | expect(Sidekiq::Queue['queue2'].probed).to eq 1 75 | end 76 | end 77 | 78 | it 'should set limits' do 79 | subject 80 | expect(Sidekiq::Queue['queue1'].limit).to eq 3 81 | expect(Sidekiq::Queue['queue2'].limit).not_to be 82 | end 83 | 84 | it 'should set process_limits' do 85 | subject 86 | expect(Sidekiq::Queue['queue2'].process_limit).to eq 3 87 | end 88 | 89 | context 'without strict flag' do 90 | let(:strict) { false } 91 | 92 | it 'should retrieve weighted queues' do 93 | expect(subject.ordered_queues).to match_array(%w(queue1 queue2)) 94 | end 95 | end 96 | 97 | it 'with strict flag should retrieve strictly ordered queues' do 98 | expect(subject.ordered_queues).to eq %w(queue1 queue2) 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch/global/selector.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq::LimitFetch::Global 2 | module Selector 3 | extend self 4 | 5 | MUTEX_FOR_UUID = Mutex.new 6 | 7 | def acquire(queues, namespace) 8 | redis_eval :acquire, [namespace, uuid, queues] 9 | end 10 | 11 | def release(queues, namespace) 12 | redis_eval :release, [namespace, uuid, queues] 13 | end 14 | 15 | def uuid 16 | # - if we'll remove "@uuid ||=" from inside of mutex 17 | # then @uuid can be overwritten 18 | # - if we'll remove "@uuid ||=" from outside of mutex 19 | # then each read will lead to mutex 20 | @uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid } 21 | end 22 | 23 | private 24 | 25 | def redis_eval(script_name, args) 26 | Sidekiq.redis do |it| 27 | begin 28 | it.evalsha send("redis_#{script_name}_sha"), argv: args 29 | rescue Redis::CommandError => error 30 | raise unless error.message.include? 'NOSCRIPT' 31 | it.eval send("redis_#{script_name}_script"), argv: args 32 | end 33 | end 34 | end 35 | 36 | def redis_acquire_sha 37 | @acquire_sha ||= Digest::SHA1.hexdigest redis_acquire_script 38 | end 39 | 40 | def redis_release_sha 41 | @release_sha ||= Digest::SHA1.hexdigest redis_release_script 42 | end 43 | 44 | def redis_acquire_script 45 | <<-LUA 46 | local namespace = table.remove(ARGV, 1)..'limit_fetch:' 47 | local worker_name = table.remove(ARGV, 1) 48 | local queues = ARGV 49 | local available = {} 50 | local unblocked = {} 51 | local locks 52 | local process_locks 53 | local blocking_mode 54 | 55 | for _, queue in ipairs(queues) do 56 | if not blocking_mode or unblocked[queue] then 57 | local probed_key = namespace..'probed:'..queue 58 | local pause_key = namespace..'pause:'..queue 59 | local limit_key = namespace..'limit:'..queue 60 | local process_limit_key = namespace..'process_limit:'..queue 61 | local block_key = namespace..'block:'..queue 62 | 63 | local paused, limit, process_limit, can_block = 64 | unpack(redis.call('mget', 65 | pause_key, 66 | limit_key, 67 | process_limit_key, 68 | block_key 69 | )) 70 | 71 | if not paused then 72 | limit = tonumber(limit) 73 | process_limit = tonumber(process_limit) 74 | 75 | if can_block or limit then 76 | locks = redis.call('llen', probed_key) 77 | end 78 | 79 | if process_limit then 80 | local all_locks = redis.call('lrange', probed_key, 0, -1) 81 | process_locks = 0 82 | for _, process in ipairs(all_locks) do 83 | if process == worker_name then 84 | process_locks = process_locks + 1 85 | end 86 | end 87 | end 88 | 89 | if not blocking_mode then 90 | blocking_mode = can_block and locks > 0 91 | end 92 | 93 | if blocking_mode and can_block ~= 'true' then 94 | for unblocked_queue in string.gmatch(can_block, "[^,]+") do 95 | unblocked[unblocked_queue] = true 96 | end 97 | end 98 | 99 | if (not limit or limit > locks) and 100 | (not process_limit or process_limit > process_locks) then 101 | redis.call('rpush', probed_key, worker_name) 102 | table.insert(available, queue) 103 | end 104 | end 105 | end 106 | end 107 | 108 | return available 109 | LUA 110 | end 111 | 112 | def redis_release_script 113 | <<-LUA 114 | local namespace = table.remove(ARGV, 1)..'limit_fetch:' 115 | local worker_name = table.remove(ARGV, 1) 116 | local queues = ARGV 117 | 118 | for _, queue in ipairs(queues) do 119 | local probed_key = namespace..'probed:'..queue 120 | redis.call('lrem', probed_key, 1, worker_name) 121 | end 122 | LUA 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/sidekiq/limit_fetch/global/semaphore.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq::LimitFetch::Global 2 | class Semaphore 3 | PREFIX = 'limit_fetch' 4 | 5 | attr_reader :local_busy 6 | 7 | def initialize(name) 8 | @name = name 9 | @lock = Mutex.new 10 | @local_busy = 0 11 | end 12 | 13 | def limit 14 | value = redis {|it| it.get "#{PREFIX}:limit:#@name" } 15 | value.to_i if value 16 | end 17 | 18 | def limit=(value) 19 | @limit_changed = true 20 | 21 | if value 22 | redis {|it| it.set "#{PREFIX}:limit:#@name", value } 23 | else 24 | redis {|it| it.del "#{PREFIX}:limit:#@name" } 25 | end 26 | end 27 | 28 | def limit_changed? 29 | @limit_changed 30 | end 31 | 32 | def process_limit 33 | value = redis {|it| it.get "#{PREFIX}:process_limit:#@name" } 34 | value.to_i if value 35 | end 36 | 37 | def process_limit=(value) 38 | if value 39 | redis {|it| it.set "#{PREFIX}:process_limit:#@name", value } 40 | else 41 | redis {|it| it.del "#{PREFIX}:process_limit:#@name" } 42 | end 43 | end 44 | 45 | def acquire 46 | Selector.acquire([@name], namespace).size > 0 47 | end 48 | 49 | def release 50 | redis {|it| it.lrem "#{PREFIX}:probed:#@name", 1, Selector.uuid } 51 | end 52 | 53 | def busy 54 | redis {|it| it.llen "#{PREFIX}:busy:#@name" } 55 | end 56 | 57 | def busy_processes 58 | redis {|it| it.lrange "#{PREFIX}:busy:#@name", 0, -1 } 59 | end 60 | 61 | def increase_busy 62 | increase_local_busy 63 | redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid } 64 | end 65 | 66 | def decrease_busy 67 | decrease_local_busy 68 | redis {|it| it.lrem "#{PREFIX}:busy:#@name", 1, Selector.uuid } 69 | end 70 | 71 | def probed 72 | redis {|it| it.llen "#{PREFIX}:probed:#@name" } 73 | end 74 | 75 | def probed_processes 76 | redis {|it| it.lrange "#{PREFIX}:probed:#@name", 0, -1 } 77 | end 78 | 79 | def pause 80 | redis {|it| it.set "#{PREFIX}:pause:#@name", true } 81 | end 82 | 83 | def pause_for_ms ms 84 | redis {|it| it.psetex "#{PREFIX}:pause:#@name", ms, true } 85 | end 86 | 87 | def unpause 88 | redis {|it| it.del "#{PREFIX}:pause:#@name" } 89 | end 90 | 91 | def paused? 92 | redis {|it| it.get "#{PREFIX}:pause:#@name" } 93 | end 94 | 95 | def block 96 | redis {|it| it.set "#{PREFIX}:block:#@name", true } 97 | end 98 | 99 | def block_except(*queues) 100 | raise ArgumentError if queues.empty? 101 | redis {|it| it.set "#{PREFIX}:block:#@name", queues.join(',') } 102 | end 103 | 104 | def unblock 105 | redis {|it| it.del "#{PREFIX}:block:#@name" } 106 | end 107 | 108 | def blocking? 109 | redis {|it| it.get "#{PREFIX}:block:#@name" } 110 | end 111 | 112 | def increase_local_busy 113 | @lock.synchronize { @local_busy += 1 } 114 | end 115 | 116 | def decrease_local_busy 117 | @lock.synchronize { @local_busy -= 1 } 118 | end 119 | 120 | def local_busy? 121 | @local_busy > 0 122 | end 123 | 124 | def explain 125 | <<-END.gsub(/^ {8}/, '') 126 | Current sidekiq process: #{Selector.uuid} 127 | 128 | All processes: 129 | #{Monitor.all_processes.join "\n"} 130 | 131 | Stale processes: 132 | #{Monitor.old_processes.join "\n"} 133 | 134 | Locked queue processes: 135 | #{probed_processes.sort.join "\n"} 136 | 137 | Busy queue processes: 138 | #{busy_processes.sort.join "\n"} 139 | 140 | Limit: 141 | #{limit.inspect} 142 | 143 | Process limit: 144 | #{process_limit.inspect} 145 | 146 | Blocking: 147 | #{blocking?} 148 | END 149 | end 150 | 151 | def remove_locks_except!(processes) 152 | locked_processes = probed_processes.uniq 153 | (locked_processes - processes).each do |dead_process| 154 | remove_lock! dead_process 155 | end 156 | end 157 | 158 | def remove_lock!(process) 159 | redis do |it| 160 | it.lrem "#{PREFIX}:probed:#@name", 0, process 161 | it.lrem "#{PREFIX}:busy:#@name", 0, process 162 | end 163 | end 164 | 165 | private 166 | 167 | def redis(&block) 168 | Sidekiq.redis(&block) 169 | end 170 | 171 | def namespace 172 | Sidekiq::LimitFetch::Queues.namespace 173 | end 174 | end 175 | end 176 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | Sidekiq strategy to support a granular queue control – 4 | limiting, pausing, blocking, querying. 5 | 6 | [![Build Status](https://secure.travis-ci.org/brainopia/sidekiq-limit_fetch.svg)](http://travis-ci.org/brainopia/sidekiq-limit_fetch) 7 | [![Gem Version](https://badge.fury.io/rb/sidekiq-limit_fetch.svg)](http://badge.fury.io/rb/sidekiq-limit_fetch) 8 | [![Dependency Status](https://gemnasium.com/brainopia/sidekiq-limit_fetch.svg)](https://gemnasium.com/brainopia/sidekiq-limit_fetch) 9 | [![Code Climate](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch.svg)](https://codeclimate.com/github/brainopia/sidekiq-limit_fetch) 10 | 11 | ## Installation 12 | 13 | Add this line to your application's Gemfile: 14 | 15 | gem 'sidekiq-limit_fetch' 16 | 17 | ### Requirements 18 | 19 | **Important note:** At this moment, `sidekiq-limit_fetch` is incompatible with 20 | - sidekiq pro's `reliable_fetch` 21 | - `sidekiq-rate-limiter` 22 | - any other plugin that rewrites fetch strategy of sidekiq. 23 | 24 | ## Usage 25 | 26 | ### Require 27 | You must `require 'sidekiq-limit_fetch'` if it isn't already. It will not work until then. 28 | 29 | ### Limits 30 | 31 | Specify limits which you want to place on queues inside sidekiq.yml: 32 | 33 | ```yaml 34 | :limits: 35 | queue_name1: 5 36 | queue_name2: 10 37 | ``` 38 | 39 | Or set it dynamically in your code: 40 | ```ruby 41 | Sidekiq::Queue['queue_name1'].limit = 5 42 | Sidekiq::Queue['queue_name2'].limit = 10 43 | ``` 44 | 45 | In these examples, tasks for the ```queue_name1``` will be run by at most 5 46 | workers at the same time and the ```queue_name2``` will have no more than 10 47 | workers simultaneously. 48 | 49 | Ability to set limits dynamically allows you to resize worker 50 | distribution among queues any time you want. 51 | 52 | ### Limits per process 53 | 54 | If you use multiple sidekiq processes then you can specify limits per process: 55 | 56 | ```yaml 57 | :process_limits: 58 | queue_name: 2 59 | ``` 60 | 61 | Or set it in your code: 62 | 63 | ```ruby 64 | Sidekiq::Queue['queue_name'].process_limit = 2 65 | ``` 66 | 67 | ### Busy workers by queue 68 | 69 | You can see how many workers currently handling a queue: 70 | 71 | ```ruby 72 | Sidekiq::Queue['name'].busy # number of busy workers 73 | ``` 74 | 75 | ### Pauses 76 | 77 | You can also pause your queues temporarily. Upon continuing their limits 78 | will be preserved. 79 | 80 | ```ruby 81 | Sidekiq::Queue['name'].pause # prevents workers from running tasks from this queue 82 | Sidekiq::Queue['name'].paused? # => true 83 | Sidekiq::Queue['name'].unpause # allows workers to use the queue 84 | Sidekiq::Queue['name'].pause_for_ms(1000) # will pause for a second 85 | ``` 86 | 87 | ### Blocking queue mode 88 | 89 | If you use strict queue ordering (it will be used if you don't specify queue weights) 90 | then you can set blocking status for queues. It means if a blocking 91 | queue task is executing then no new task from lesser priority queues will 92 | be ran. Eg, 93 | 94 | ```yaml 95 | :queues: 96 | - a 97 | - b 98 | - c 99 | :blocking: 100 | - b 101 | ``` 102 | 103 | In this case when a task for `b` queue is ran no new task from `c` queue 104 | will be started. 105 | 106 | You can also enable and disable blocking mode for queues on the fly: 107 | 108 | ```ruby 109 | Sidekiq::Queue['name'].block 110 | Sidekiq::Queue['name'].blocking? # => true 111 | Sidekiq::Queue['name'].unblock 112 | ``` 113 | 114 | ### Advanced blocking queues 115 | 116 | You can also block on array of queues. It means when any of them is 117 | running only queues higher and queues from their blocking group can 118 | run. It will be easier to understand with an example: 119 | 120 | ```yaml 121 | :queues: 122 | - a 123 | - b 124 | - c 125 | - d 126 | :blocking: 127 | - [b, c] 128 | ``` 129 | 130 | In this case tasks from `d` will be blocked when a task from queue `b` or `c` is executed. 131 | 132 | You can dynamically set exceptions for queue blocking: 133 | 134 | ```ruby 135 | Sidekiq::Queue['queue1'].block_except 'queue2' 136 | ``` 137 | 138 | ### Dynamic queues 139 | 140 | You can support dynamic queues (that are not listed in sidekiq.yml but 141 | that have tasks pushed to them (usually with `Sidekiq::Client.push`)). 142 | 143 | To use this mode you need to specify a following line in sidekiq.yml: 144 | 145 | ```yaml 146 | :dynamic: true 147 | ``` 148 | 149 | Dynamic queues will be ran at the lowest priority. 150 | 151 | ### Maintenance 152 | 153 | If you use ```flushdb```, restart the sidekiq process to re-populate the dynamic configuration. 154 | 155 | ### Thanks 156 | 157 | 158 | Sponsored by Evil Martians 159 | 160 | --------------------------------------------------------------------------------