├── .github └── workflows │ ├── lint.yml │ ├── release.yml │ └── test.yml ├── .gitignore ├── .rspec ├── .rubocop.yml ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin ├── console └── setup ├── lib └── sidekiq │ ├── fair_tenant.rb │ └── fair_tenant │ ├── client_middleware.rb │ ├── config.rb │ └── version.rb ├── sidekiq-fair_tenant.gemspec ├── spec ├── sidekiq │ └── fair_tenant_spec.rb ├── spec_helper.rb └── support │ └── jobs.rb └── tea.yaml /.github/workflows/lint.yml: -------------------------------------------------------------------------------- 1 | name: Lint 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - '**' 8 | tags-ignore: 9 | - 'v*' 10 | 11 | jobs: 12 | rubocop: 13 | # Skip running tests for local pull requests (use push event instead), run only for foreign ones 14 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login 15 | name: RuboCop 16 | runs-on: ubuntu-latest 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: ruby/setup-ruby@v1 20 | with: 21 | ruby-version: "3.3" 22 | bundler-cache: true 23 | - name: Lint Ruby code with RuboCop 24 | run: | 25 | bundle exec rubocop 26 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: Release gem 2 | 3 | on: 4 | push: 5 | tags: 6 | - v* 7 | 8 | jobs: 9 | release: 10 | runs-on: ubuntu-latest 11 | permissions: 12 | contents: write 13 | id-token: write 14 | packages: write 15 | steps: 16 | - uses: actions/checkout@v4 17 | with: 18 | fetch-depth: 0 # Fetch current tag as annotated. See https://github.com/actions/checkout/issues/290 19 | - uses: ruby/setup-ruby@v1 20 | with: 21 | ruby-version: "3.3" 22 | - name: "Extract data from tag: version, message, body" 23 | id: tag 24 | run: | 25 | git fetch --tags --force # Really fetch annotated tag. See https://github.com/actions/checkout/issues/290#issuecomment-680260080 26 | echo ::set-output name=version::${GITHUB_REF#refs/tags/v} 27 | echo ::set-output name=subject::$(git for-each-ref $GITHUB_REF --format='%(contents:subject)') 28 | BODY="$(git for-each-ref $GITHUB_REF --format='%(contents:body)')" 29 | # Extract changelog entries between this and previous version headers 30 | escaped_version=$(echo ${GITHUB_REF#refs/tags/v} | sed -e 's/[]\/$*.^[]/\\&/g') 31 | changelog=$(awk "BEGIN{inrelease=0} /## \[${escaped_version}\]/{inrelease=1;next} /## \[[0-9]+\.[0-9]+\.[0-9]+.*?\]/{inrelease=0;exit} {if (inrelease) print}" CHANGELOG.md) 32 | # Multiline body for release. See https://github.community/t/set-output-truncates-multiline-strings/16852/5 33 | BODY="${BODY}"$'\n'"${changelog}" 34 | BODY="${BODY//'%'/'%25'}" 35 | BODY="${BODY//$'\n'/'%0A'}" 36 | BODY="${BODY//$'\r'/'%0D'}" 37 | echo "::set-output name=body::$BODY" 38 | # Add pre-release option if tag name has any suffix after vMAJOR.MINOR.PATCH 39 | if [[ ${GITHUB_REF#refs/tags/} =~ ^v[0-9]+\.[0-9]+\.[0-9]+.+ ]]; then 40 | echo ::set-output name=prerelease::true 41 | fi 42 | - name: Build gem 43 | run: gem build 44 | - name: Calculate checksums 45 | run: sha256sum sidekiq-fair_tenant-${{ steps.tag.outputs.version }}.gem > SHA256SUM 46 | - name: Check version 47 | run: ls -l sidekiq-fair_tenant-${{ steps.tag.outputs.version }}.gem 48 | - name: Create Release 49 | id: create_release 50 | uses: actions/create-release@v1 51 | env: 52 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 53 | with: 54 | tag_name: ${{ github.ref }} 55 | release_name: ${{ steps.tag.outputs.subject }} 56 | body: ${{ steps.tag.outputs.body }} 57 | draft: false 58 | prerelease: ${{ steps.tag.outputs.prerelease }} 59 | - name: Upload built gem as release asset 60 | uses: actions/upload-release-asset@v1 61 | env: 62 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 63 | with: 64 | upload_url: ${{ steps.create_release.outputs.upload_url }} 65 | asset_path: sidekiq-fair_tenant-${{ steps.tag.outputs.version }}.gem 66 | asset_name: sidekiq-fair_tenant-${{ steps.tag.outputs.version }}.gem 67 | asset_content_type: application/x-tar 68 | - name: Upload checksums as release asset 69 | uses: actions/upload-release-asset@v1 70 | env: 71 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 72 | with: 73 | upload_url: ${{ steps.create_release.outputs.upload_url }} 74 | asset_path: SHA256SUM 75 | asset_name: SHA256SUM 76 | asset_content_type: text/plain 77 | - name: Publish to GitHub packages 78 | env: 79 | GEM_HOST_API_KEY: Bearer ${{ secrets.GITHUB_TOKEN }} 80 | run: | 81 | gem push sidekiq-fair_tenant-${{ steps.tag.outputs.version }}.gem --host https://rubygems.pkg.github.com/${{ github.repository_owner }} 82 | - name: Configure RubyGems Credentials 83 | uses: rubygems/configure-rubygems-credentials@main 84 | - name: Publish to RubyGems 85 | run: | 86 | gem push sidekiq-fair_tenant-${{ steps.tag.outputs.version }}.gem 87 | -------------------------------------------------------------------------------- /.github/workflows/test.yml: -------------------------------------------------------------------------------- 1 | name: Test 2 | 3 | on: 4 | pull_request: 5 | push: 6 | branches: 7 | - '**' 8 | tags-ignore: 9 | - 'v*' 10 | 11 | jobs: 12 | test: 13 | # Skip running tests for local pull requests (use push event instead), run only for foreign ones 14 | if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.owner.login != github.event.pull_request.base.repo.owner.login 15 | name: "Ruby ${{ matrix.ruby }} × Sidekiq v${{ matrix.sidekiq }} × ActiveJob v${{ matrix.activejob }} × Redis v${{ matrix.redis }}" 16 | runs-on: ubuntu-latest 17 | strategy: 18 | fail-fast: false 19 | matrix: 20 | include: 21 | - ruby: "3.3" 22 | sidekiq: '7' 23 | activejob: '7.1' 24 | redis: '7.2' 25 | - ruby: "3.2" 26 | sidekiq: '7' 27 | activejob: '7.0' 28 | redis: '7.2' 29 | - ruby: "3.1" 30 | sidekiq: '6' 31 | activejob: '6.1' 32 | redis: '7.0' 33 | - ruby: "3.0" 34 | sidekiq: '5' 35 | activejob: '6.0' 36 | redis: '6.2' 37 | services: 38 | redis: 39 | image: redis:${{ matrix.redis }} 40 | options: >- 41 | --health-cmd "redis-cli ping" 42 | --health-interval 10s 43 | --health-timeout 5s 44 | --health-retries 5 45 | ports: 46 | - 6379:6379 47 | env: 48 | REDIS_URL: redis://localhost:6379 49 | SIDEKIQ_VERSION: '${{ matrix.sidekiq }}' 50 | ACTIVEJOB_VERSION: '${{ matrix.activejob }}' 51 | steps: 52 | - uses: actions/checkout@v4 53 | - uses: ruby/setup-ruby@v1 54 | with: 55 | ruby-version: ${{ matrix.ruby }} 56 | bundler-cache: true 57 | - name: Run RSpec 58 | run: bundle exec rspec 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | # rspec failure tracking 11 | .rspec_status 12 | 13 | Gemfile.lock 14 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | TargetRubyVersion: 2.6 3 | NewCops: enable 4 | 5 | Style/StringLiterals: 6 | Enabled: true 7 | EnforcedStyle: double_quotes 8 | 9 | Style/StringLiteralsInInterpolation: 10 | Enabled: true 11 | EnforcedStyle: double_quotes 12 | 13 | Layout/LineLength: 14 | Max: 120 15 | 16 | Metrics/AbcSize: 17 | Max: 20 18 | 19 | Metrics/BlockLength: 20 | Exclude: 21 | - 'spec/**/*' 22 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ## [0.1.0] - 2024-01-12 11 | 12 | - Initial release: multiple rules, configurable throttling window. 13 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | 5 | gemspec 6 | 7 | # rubocop:disable Bundler/DuplicatedGem 8 | sidekiq_version = ENV.fetch("SIDEKIQ_VERSION", "~> 7.2") 9 | case sidekiq_version 10 | when "HEAD" 11 | gem "sidekiq", git: "https://github.com/sidekiq/sidekiq.git" 12 | else 13 | sidekiq_version = "~> #{sidekiq_version}.0" if sidekiq_version.match?(/^\d+(?:\.\d+)?$/) 14 | gem "sidekiq", sidekiq_version 15 | end 16 | 17 | activejob_version = ENV.fetch("ACTIVEJOB_VERSION", "~> 7.1") 18 | case activejob_version 19 | when "HEAD" 20 | git "https://github.com/rails/rails.git" do 21 | gem "activejob" 22 | gem "activesupport" 23 | gem "rails" 24 | end 25 | else 26 | activejob_version = "~> #{activejob_version}.0" if activejob_version.match?(/^\d+\.\d+$/) 27 | gem "activejob", activejob_version 28 | gem "activesupport", activejob_version 29 | end 30 | # rubocop:enable Bundler/DuplicatedGem 31 | 32 | gem "rake", "~> 13.0" 33 | 34 | gem "rspec", "~> 3.0" 35 | 36 | gem "rspec-sidekiq" 37 | 38 | gem "rubocop", "~> 1.21", require: false 39 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2024 Andrey Novikov 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sidekiq::FairTenant 2 | 3 | Throttle “greedy” clients’ jobs to ensure more or less fair distribution of resources between clients. 4 | 5 | This tiny [Sidekiq] middleware will re-route client's jobs after certain threshold to throttled queues (defined by you), where they will be processed with reduced priority. 6 | 7 | “Weighted queues” feature of Sidekiq allows to de-prioritize jobs in throttled queues, so they will not block jobs from other clients, at the same time preserving overall throughput. 8 | 9 | 10 | 11 | 15 | Sponsored by Evil Martians 21 | 22 | 23 | 24 | ## Installation 25 | 26 | 1. Install the gem and add to the application's Gemfile by executing: 27 | 28 | ```sh 29 | bundle add sidekiq-fair_tenant 30 | ``` 31 | 32 | 2. Add `fair_tenant_queues` section to `sidekiq_options` in your job class: 33 | 34 | ```diff 35 | class SomeJob 36 | sidekiq_options \ 37 | queue: 'default', 38 | + fair_tenant_queues: [ 39 | + { queue: 'throttled_2x', threshold: 100, per: 1.hour }, 40 | + { queue: 'throttled_4x', threshold: 10, per: 1.minute }, 41 | + ] 42 | end 43 | ``` 44 | 45 | 3. Add tenant detection login into your job class: 46 | 47 | ```diff 48 | class SomeJob 49 | + def self.fair_tenant(*_perform_arguments) 50 | + # Return any string that will be used as tenant name 51 | + "tenant_1" 52 | + end 53 | end 54 | ``` 55 | 56 | 4. Add throttled queues with reduced weights to your Sidekiq configuration: 57 | 58 | ```diff 59 | # config/sidekiq.yml 60 | :queues: 61 | - [default, 4] 62 | + - [throttled_2x, 2] 63 | + - [throttled_4x, 1] 64 | ``` 65 | 66 | See [Sidekiq Advanced options for Queues](https://github.com/sidekiq/sidekiq/wiki/Advanced-Options#queues) to learn more about queue weights. 67 | 68 | ## Usage 69 | 70 | ### Specifying throttling rules 71 | 72 | In your job class, add `fair_tenant_queues` section to `sidekiq_options` as array of hashes with following keys: 73 | 74 | - `queue` - throttled queue name to re-route jobs into. 75 | - `threshold` - maximum number of jobs allowed to be enqueued within `per` seconds. 76 | - `per` - sliding time window in seconds to count jobs (you can use ActiveSupport Durations in Rails). 77 | 78 | You can specify multiple rules and they all will be checked. _Last_ matching rule will be used, so order rules from least to most restrictive. 79 | 80 | Example: 81 | 82 | ```ruby 83 | sidekiq_options \ 84 | queue: 'default', 85 | fair_tenant_queues: [ 86 | # First rule is less restrictive, reacting to a large number of jobs enqueued in a long time window 87 | { queue: 'throttled_2x', threshold: 1_000, per: 1.day }, 88 | # Next rule is more restrictive, reacting to spikes of jobs in a short time window 89 | { queue: 'throttled_4x', threshold: 10, per: 1.minute }, 90 | ] 91 | ``` 92 | 93 | ### Specifying tenant 94 | 95 | 1. Explicitly during job enqueuing: 96 | 97 | ```ruby 98 | SomeJob.set(fair_tenant: 'tenant_1').perform_async 99 | ``` 100 | 101 | 2. Dynamically using `fair_tenant` class-level method in your job class (receives same arguments as `perform`) 102 | 103 | ```ruby 104 | class SomeJob 105 | def self.fair_tenant(*_perform_arguments) 106 | # Return any string that will be used as tenant name 107 | "tenant_1" 108 | end 109 | end 110 | ``` 111 | 112 | 3. Set `fair_tenant` job option in a custom [middleware](https://github.com/sidekiq/sidekiq/wiki/Middleware) earlier in the stack. 113 | 114 | 4. Or let this gem automatically pick tenant name from [apartment-sidekiq](https://github.com/influitive/apartment-sidekiq) if you're using apartment gem. 115 | 116 | ## Configuration 117 | 118 | Configuration is handled by [anyway_config] gem. With it you can load settings from environment variables (which names are constructed from config key upcased and prefixed with `SIDEKIQ_FAIR_TENANT_`), YAML files, and other sources. See [anyway_config] docs for details. 119 | 120 | | Config key | Type | Default | Description | 121 | |----------------------------|----------|---------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| 122 | | `max_throttling_window` | integer | `86_400` (1 day) | Maximum throttling window in seconds | 123 | | `enqueues_key` | string | `sidekiq-fair_tenant:enqueued:%s:tenant:%s` | Ruby [format string](https://docs.ruby-lang.org/en/3.3/format_specifications_rdoc.html) used as a name for Redis key holding job ids for throttling window. Available placeholders: `queue`, `job_class`, `fair_tenant` | 124 | | `logger` | logger | `Sidekiq.logger` | Logger instance used for warning logging. | 125 | 126 | ## How it works 127 | 128 | If number of jobs enqueued by a single client exceeds some threshold per a sliding time window, their jobs would be re-routed to another queue, with lower priority. 129 | 130 | This gem tracks single client's jobs in a Redis [sorted set](https://redis.io/docs/data-types/sorted-sets/) with job id as a key and enqueuing timestamp as a score. When a new job is enqueued, it is added to the set, and then the set is trimmed to contain only jobs enqueued within the last `max_throttling_window` seconds. 131 | 132 | On every enqueue attempt, the set is checked for number of jobs enqueued within the last `per` seconds of every rule. If the number of jobs in this time window exceeds `threshold`, the job is enqueued to a throttled queue, otherwise it is enqueued to the default queue. If multiple rules match, last one is used. 133 | 134 | You are expected to configure Sidekiq to process throttled queues with lower priority using [queue weights](https://github.com/mperham/sidekiq/wiki/Advanced-Options#queues). 135 | 136 | ### Advantages 137 | - If fast queues are empty then slow queues are processed at full speed (no artificial delays) 138 | - If fast queues are full, slow queues are still processed, but slower (configurable), so application doesn’t “stall” for throttled users 139 | - Minimal changes to the application code are required. 140 | 141 | ### Disadvantages 142 | - As Sidekiq does not support mixing ordered and weighted queue modes (as stated in Sidekiq Wiki on queue configuration), you can’t make the same worker process execute some super important queue always first, ignoring other queues. Run separate worker to solve this. 143 | - You have to keep track of all your queues and their weights. 144 | 145 | ## Development 146 | 147 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 148 | 149 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and the created tag, and push the `.gem` file to [rubygems.org](https://rubygems.org). 150 | 151 | ## Contributing 152 | 153 | Bug reports and pull requests are welcome on GitHub at https://github.com/Envek/sidekiq-fair_tenant. 154 | 155 | ## License 156 | 157 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 158 | 159 | [sidekiq]: https://github.com/sidekiq/sidekiq "Simple, efficient background processing for Ruby" 160 | [anyway_config]: https://github.com/palkan/anyway_config "Configuration library for Ruby gems and applications" 161 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | require "rspec/core/rake_task" 5 | 6 | RSpec::Core::RakeTask.new(:spec) 7 | 8 | require "rubocop/rake_task" 9 | 10 | RuboCop::RakeTask.new 11 | 12 | task default: %i[spec rubocop] 13 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require "bundler/setup" 5 | require "sidekiq/fair_tenant" 6 | 7 | # You can add fixtures and/or initialization code here to make experimenting 8 | # with your gem easier. You can also use a different console, if you like. 9 | 10 | require "irb" 11 | IRB.start(__FILE__) 12 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /lib/sidekiq/fair_tenant.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sidekiq" 4 | 5 | require_relative "fair_tenant/version" 6 | require_relative "fair_tenant/config" 7 | require_relative "fair_tenant/client_middleware" 8 | 9 | module Sidekiq 10 | # Client middleware and job DSL for throttling jobs of overly active tenants 11 | module FairTenant 12 | class Error < ::StandardError; end 13 | 14 | class << self 15 | extend ::Forwardable 16 | 17 | def config 18 | @config ||= Config.new 19 | end 20 | 21 | def_delegators :config, :max_throttling_window, :enqueues_key, :logger 22 | end 23 | end 24 | end 25 | 26 | Sidekiq.configure_client do |config| 27 | config.client_middleware do |chain| 28 | chain.add Sidekiq::FairTenant::ClientMiddleware 29 | end 30 | end 31 | 32 | Sidekiq.configure_server do |config| 33 | config.client_middleware do |chain| 34 | chain.add Sidekiq::FairTenant::ClientMiddleware 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/sidekiq/fair_tenant/client_middleware.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sidekiq 4 | module FairTenant 5 | # Client middleware re-routing jobs of overly active tenants to slower queues based on thresholds 6 | class ClientMiddleware 7 | # Re-routes job to the most appropriate queue, based on tenant's throttling rules 8 | # rubocop:disable Metrics/MethodLength 9 | def call(worker, job, queue, redis_pool) 10 | job_class = job_class(worker, job) 11 | arguments = job["wrapped"] ? job.dig("args", 0, "arguments") : job["args"] 12 | return yield unless enabled?(job_class, job, queue) 13 | 14 | job["fair_tenant"] ||= tenant_id(job_class, job, arguments) 15 | unless job["fair_tenant"] 16 | logger.warn "#{job_class} with args #{arguments.inspect} won't be throttled: missing `fair_tenant` in job" 17 | return yield 18 | end 19 | 20 | redis_pool.then do |redis| 21 | register_job(job_class, job, queue, redis) 22 | job["queue"] = assign_queue(job_class, job, queue, redis) 23 | end 24 | 25 | yield 26 | end 27 | # rubocop:enable Metrics/MethodLength 28 | 29 | private 30 | 31 | def enabled?(job_class, job, queue) 32 | return false if job["fair_tenant_queues"].blank? # Not configured for throttling 33 | 34 | original_queue = original_queue(job_class, job, queue) 35 | return false if original_queue != job["queue"] # Someone already rerouted this job, nothing to do here 36 | 37 | true 38 | end 39 | 40 | # Writes job to sliding window sorted set 41 | def register_job(job_class, job, queue, redis) 42 | enqueues_key = enqueues_key(job_class, job, queue) 43 | max_throttling_window = Sidekiq::FairTenant.max_throttling_window 44 | redis.multi do |tx| 45 | tx.zremrangebyscore(enqueues_key, "-inf", (Time.now - max_throttling_window).to_i) 46 | tx.zadd(enqueues_key, Time.now.to_i, "jid:#{job["jid"]}") 47 | tx.expire(enqueues_key, max_throttling_window) 48 | end 49 | end 50 | 51 | # Chooses the last queue, for the most restrictive (threshold/time) rule that is met. 52 | # Assumes the slowest queue, with most restrictive rule, comes last in the `fair_tenants` array. 53 | def assign_queue(job_class, job, queue, redis) 54 | enqueues_key = enqueues_key(job_class, job, queue) 55 | 56 | matching_rules = 57 | job["fair_tenant_queues"].map(&:symbolize_keys).filter do |config| 58 | threshold = config[:threshold] 59 | window_start = Time.now - (config[:per] || Sidekiq::FairTenant.max_throttling_window) 60 | threshold < redis.zcount(enqueues_key, window_start.to_i, Time.now.to_i) 61 | end 62 | 63 | matching_rules.any? ? matching_rules.last[:queue] : queue 64 | end 65 | 66 | def enqueues_key(job_class, job, queue) 67 | format(Sidekiq::FairTenant.enqueues_key, queue: queue, fair_tenant: job["fair_tenant"], job_class: job_class) 68 | end 69 | 70 | def job_class(worker, job) 71 | job_class = job["wrapped"] || worker 72 | return job_class if job_class.is_a?(Class) 73 | return job_class.constantize if job_class.respond_to?(:constantize) 74 | 75 | Object.const_get(job_class.to_s) 76 | end 77 | 78 | # Calculates tenant identifier (`fair_tenant`) for the job 79 | def tenant_id(job_class, job, arguments) 80 | return job_class.fair_tenant(*arguments) if job_class.respond_to?(:fair_tenant) 81 | 82 | job["apartment"] # for compatibility with sidekiq-apartment 83 | end 84 | 85 | def original_queue(job_class, _job, queue) 86 | if job_class.respond_to?(:queue_name) 87 | job_class.queue_name # ActiveJob 88 | elsif job_class.respond_to?(:queue) 89 | job_class.queue.to_s # Sidekiq 90 | else 91 | queue 92 | end 93 | end 94 | 95 | def logger 96 | Sidekiq::FairTenant.logger 97 | end 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/sidekiq/fair_tenant/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "anyway" 4 | 5 | module Sidekiq 6 | module FairTenant 7 | # Runtime configuration for the sidekiq-fair_tenant gem 8 | class Config < ::Anyway::Config 9 | config_name :sidekiq_fair_tenant 10 | 11 | # Maximum amount of time to store information about tenant enqueues 12 | attr_config max_throttling_window: 86_400 # 1 day 13 | 14 | # Sorted set that contains job ids enqueued by each tenant in last 1 day (max throttling window) 15 | attr_config enqueues_key: "sidekiq-fair_tenant:enqueued:%s:tenant:%s" 16 | 17 | # Logger to use for throttling warnings 18 | attr_config logger: ::Sidekiq.logger 19 | end 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /lib/sidekiq/fair_tenant/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sidekiq 4 | module FairTenant 5 | VERSION = "0.1.0" 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /sidekiq-fair_tenant.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "lib/sidekiq/fair_tenant/version" 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "sidekiq-fair_tenant" 7 | spec.version = Sidekiq::FairTenant::VERSION 8 | spec.authors = ["Andrey Novikov"] 9 | spec.email = ["envek@envek.name"] 10 | 11 | spec.summary = "Throttle Sidekiq jobs of greedy tenants" 12 | spec.description = "Re-route jobs of way too active tenants to slower queues, letting other tenant's jobs to go first" 13 | spec.homepage = "https://github.com/Envek/sidekiq-fair_tenant" 14 | spec.license = "MIT" 15 | spec.required_ruby_version = ">= 2.6.0" 16 | 17 | spec.metadata["homepage_uri"] = spec.homepage 18 | spec.metadata["source_code_uri"] = "https://github.com/Envek/sidekiq-fair_tenant" 19 | spec.metadata["changelog_uri"] = "https://github.com/Envek/sidekiq-fair_tenant/blob/master/CHANGELOG.md" 20 | spec.metadata["rubygems_mfa_required"] = "true" 21 | 22 | # Specify which files should be added to the gem when it is released. 23 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 24 | spec.files = Dir.chdir(__dir__) do 25 | `git ls-files -z`.split("\x0").reject do |f| 26 | (File.expand_path(f) == __FILE__) || 27 | f.start_with?(*%w[bin/ spec/ Gemfile Rakefile]) || 28 | f.match(/^(\.)/) 29 | end 30 | end 31 | spec.bindir = "exe" 32 | spec.executables = spec.files.grep(%r{\Aexe/}) { |f| File.basename(f) } 33 | spec.require_paths = ["lib"] 34 | 35 | spec.add_dependency "anyway_config", ">= 1.0", "< 3" 36 | spec.add_dependency "sidekiq", ">= 5" 37 | end 38 | -------------------------------------------------------------------------------- /spec/sidekiq/fair_tenant_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Sidekiq::FairTenant do 4 | context "when throttling is not set up" do 5 | it "enqueues job to default job" do 6 | expect { SamplePlainJob.perform_async }.to enqueue_sidekiq_job.on("default") 7 | end 8 | end 9 | 10 | context "when throttling is set up" do 11 | subject(:enqueuer) { ->(**args) { SampleThrottledJob.set(**args).perform_async } } 12 | 13 | it "re-routes jobs above threshold for a single tenant" do 14 | # 1st job should be enqueued to the default queue 15 | expect { enqueuer.call(fair_tenant: :foo) }.to enqueue_sidekiq_job.on("whatever") 16 | 17 | # 2nd job should be enqueued to the slow queue as it exceeds threshold of 1 18 | expect { enqueuer.call(fair_tenant: :foo) }.to enqueue_sidekiq_job.on("whatever_supaslow") 19 | 20 | # However, 3rd job should be enqueued to the default queue as it is for a different tenant 21 | expect { enqueuer.call(fair_tenant: :bar) }.to enqueue_sidekiq_job.on("whatever") 22 | 23 | # And 4th job should be enqueued to the less slow queue as it super slow queue window has passed 24 | travel 1.hour + 1.second 25 | expect { enqueuer.call(fair_tenant: :foo) }.to enqueue_sidekiq_job.on("whatever_semislow") 26 | 27 | # After all time windows have passed, 5th job should be enqueued to the default queue 28 | travel 1.day + 1.second 29 | expect { enqueuer.call(fair_tenant: :foo) }.to enqueue_sidekiq_job.on("whatever") 30 | end 31 | end 32 | 33 | context "when another queue has been specified for a job at enqueue time" do 34 | subject(:enqueuer) { -> { SampleThrottledJob.set(queue: :another, fair_tenant: :foo).perform_async } } 35 | 36 | it "doesn't re-routes jobs" do 37 | # 1st job should be enqueued to the default queue 38 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("another") 39 | 40 | # 2nd job should also be enqueued to the slow queue even if it exceeds threshold of 1 41 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("another") 42 | end 43 | end 44 | 45 | context "using ActiveJob" do 46 | subject(:enqueuer) { ->(*args) { SampleActiveJob.perform_later(*args) } } 47 | 48 | it "re-routes jobs above threshold for a single tenant" do 49 | skip("ActiveJob supported only in Sidekiq 6+") if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.0.1") 50 | 51 | # 1st job should be enqueued to the default queue 52 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("whatever") 53 | 54 | # 2nd job should be enqueued to the slow queue as it exceeds threshold of 1 55 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("whatever_supaslow") 56 | 57 | # However, 3rd job should be enqueued to the default queue as it is for a different tenant 58 | expect { enqueuer.call("bar") }.to enqueue_sidekiq_job.on("whatever") 59 | 60 | # And 4th job should be enqueued to the less slow queue as it super slow queue window has passed 61 | travel 1.hour + 1.second 62 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("whatever_semislow") 63 | 64 | # After all time windows have passed, 5th job should be enqueued to the default queue 65 | travel 1.day + 1.second 66 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("whatever") 67 | end 68 | end 69 | 70 | context "when another queue has been specified for an ActiveJob at enqueue time" do 71 | subject(:enqueuer) { -> { SampleActiveJob.set(queue: :another).perform_later } } 72 | 73 | it "doesn't re-routes jobs" do 74 | skip("ActiveJob supported only in Sidekiq 6+") if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.0.1") 75 | 76 | # 1st job should be enqueued to the default queue 77 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("another") 78 | 79 | # 2nd job should also be enqueued to the set up queue even if it exceeds threshold of 1 80 | expect { enqueuer.call }.to enqueue_sidekiq_job.on("another") 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/setup" 4 | require "active_job" 5 | require "active_job/queue_adapters/sidekiq_adapter" 6 | require "active_support/core_ext/numeric/time" 7 | require "active_support/testing/time_helpers" 8 | require "sidekiq/testing" 9 | require "rspec-sidekiq" 10 | 11 | require "sidekiq/fair_tenant" 12 | 13 | require_relative "support/jobs" 14 | 15 | RSpec.configure do |config| 16 | config.include ActiveSupport::Testing::TimeHelpers 17 | 18 | # Enable flags like --only-failures and --next-failure 19 | config.example_status_persistence_file_path = ".rspec_status" 20 | 21 | # Disable RSpec exposing methods globally on `Module` and `main` 22 | config.disable_monkey_patching! 23 | 24 | config.expect_with :rspec do |c| 25 | c.syntax = :expect 26 | end 27 | 28 | config.filter_run focus: true 29 | config.run_all_when_everything_filtered = true 30 | 31 | config.mock_with :rspec 32 | 33 | Kernel.srand config.seed 34 | config.order = :random 35 | 36 | config.after do 37 | travel_back 38 | Sidekiq::Queues.clear_all 39 | 40 | Sidekiq.redis do |redis| 41 | if redis.respond_to?(:scan_each) 42 | redis.scan_each(match: "sidekiq-fair_tenant:*", &redis.method(:del)) 43 | else 44 | redis.scan(match: "sidekiq-fair_tenant:*").each(&redis.method(:del)) 45 | end 46 | end 47 | end 48 | end 49 | 50 | ActiveJob::Base.logger = Logger.new(nil) 51 | -------------------------------------------------------------------------------- /spec/support/jobs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class SamplePlainJob 4 | include Sidekiq::Worker 5 | 6 | def perform(*_args) 7 | "My job is simple" 8 | end 9 | end 10 | 11 | class SampleThrottledJob 12 | include Sidekiq::Worker 13 | 14 | sidekiq_options queue: :whatever, 15 | fair_tenant_queues: [ 16 | { queue: :whatever_semislow, threshold: 1, per: 1.day }, 17 | { queue: :whatever_supaslow, threshold: 1, per: 1.hour } 18 | ] 19 | end 20 | 21 | class SampleActiveJob < ActiveJob::Base 22 | self.queue_adapter = :sidekiq 23 | 24 | queue_as :whatever 25 | 26 | if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("6.0.1") 27 | include Sidekiq::Worker::Options 28 | 29 | sidekiq_options fair_tenant_queues: [ 30 | { queue: :whatever_semislow, threshold: 1, per: 1.day }, 31 | { queue: :whatever_supaslow, threshold: 1, per: 1.hour } 32 | ] 33 | end 34 | 35 | def self.fair_tenant(arg1 = "foo", *) 36 | arg1 37 | end 38 | 39 | def perform(_arg1, *) 40 | "I'm doing my job" 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /tea.yaml: -------------------------------------------------------------------------------- 1 | # https://tea.xyz/what-is-this-file 2 | --- 3 | version: 1.0.0 4 | codeOwners: 5 | - '0x4d9B3A6207B48E31147327f8efaF31D5EFC3784e' 6 | quorum: 1 7 | --------------------------------------------------------------------------------