├── .github ├── dependabot.yml └── workflows │ ├── continuous_integration.yml │ └── regression_test.yml ├── .gitignore ├── .standard.yml ├── Appraisals ├── CHANGELOG.md ├── Gemfile ├── MIT_LICENSE.txt ├── README.md ├── Rakefile ├── VERSION ├── bin └── sidekiq-process-manager ├── gemfiles ├── sidekiq_5.0.gemfile ├── sidekiq_5.x.gemfile ├── sidekiq_6.0.gemfile ├── sidekiq_6.x.gemfile ├── sidekiq_7.0.gemfile └── sidekiq_7.x.gemfile ├── lib ├── sidekiq-process_manager.rb └── sidekiq │ ├── process_manager.rb │ └── process_manager │ ├── manager.rb │ └── version.rb ├── sidekiq-process_manager.gemspec └── spec ├── sidekiq └── process_manager │ └── manager_spec.rb ├── spec_helper.rb └── support ├── mock_application.rb ├── mock_sidekiq_cli.rb ├── mock_sidekiq_process.rb └── mocks.rb /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # Dependabot update strategy 2 | version: 2 3 | updates: 4 | - package-ecosystem: bundler 5 | directory: "/" 6 | schedule: 7 | interval: daily 8 | allow: 9 | # Automatically keep all runtime dependencies updated 10 | - dependency-name: "*" 11 | dependency-type: "production" 12 | versioning-strategy: lockfile-only 13 | -------------------------------------------------------------------------------- /.github/workflows/continuous_integration.yml: -------------------------------------------------------------------------------- 1 | name: Continuous Integration 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - actions-* 8 | tags: 9 | - v* 10 | pull_request: 11 | branches-ignore: 12 | - actions-* 13 | 14 | env: 15 | BUNDLE_CLEAN: "true" 16 | BUNDLE_PATH: vendor/bundle 17 | BUNDLE_JOBS: 3 18 | BUNDLE_RETRY: 3 19 | 20 | jobs: 21 | build: 22 | runs-on: ubuntu-latest 23 | strategy: 24 | fail-fast: false 25 | matrix: 26 | include: 27 | - ruby: "ruby" 28 | standardrb: true 29 | - ruby: "3.2" 30 | appraisal: "sidekiq_7.x" 31 | - ruby: "3.1" 32 | appraisal: "sidekiq_7.0" 33 | - ruby: "3.0" 34 | appraisal: "sidekiq_6.x" 35 | - ruby: "2.7" 36 | appraisal: "sidekiq_6.0" 37 | - ruby: "2.6" 38 | appraisal: "sidekiq_5.x" 39 | - ruby: "2.5" 40 | appraisal: "sidekiq_5.0" 41 | steps: 42 | - uses: actions/checkout@v4 43 | - name: Set up Ruby ${{ matrix.ruby }} 44 | uses: ruby/setup-ruby@v1 45 | with: 46 | ruby-version: "${{ matrix.ruby }}" 47 | - name: Install packages 48 | run: | 49 | sudo apt-get update 50 | sudo apt-get install libsqlite3-dev 51 | - name: Setup bundler 52 | if: matrix.bundler != '' 53 | run: | 54 | gem uninstall bundler --all 55 | gem install bundler --no-document --version ${{ matrix.bundler }} 56 | - name: Set Appraisal bundle 57 | if: matrix.appraisal != '' 58 | run: | 59 | echo "using gemfile gemfiles/${{ matrix.appraisal }}.gemfile" 60 | bundle config set gemfile "gemfiles/${{ matrix.appraisal }}.gemfile" 61 | - name: Install gems 62 | run: | 63 | bundle update 64 | - name: Run Tests 65 | run: bundle exec rake 66 | - name: standardrb 67 | if: matrix.standardrb == true 68 | run: bundle exec rake standard -------------------------------------------------------------------------------- /.github/workflows/regression_test.yml: -------------------------------------------------------------------------------- 1 | name: Regression Test 2 | on: 3 | workflow_dispatch: 4 | schedule: 5 | - cron: "0 15 * * 1" 6 | env: 7 | BUNDLE_CLEAN: "true" 8 | BUNDLE_PATH: vendor/bundle 9 | BUNDLE_JOBS: 3 10 | BUNDLE_RETRY: 3 11 | jobs: 12 | specs: 13 | name: Run specs 14 | runs-on: ubuntu-latest 15 | strategy: 16 | fail-fast: false 17 | steps: 18 | - name: Checkout 19 | uses: actions/checkout@v4 20 | - name: Set up Ruby ${{ matrix.ruby }} 21 | uses: ruby/setup-ruby@v1 22 | with: 23 | ruby-version: ruby 24 | - name: Install bundler 25 | run: | 26 | bundle update 27 | - name: Run specs 28 | run: | 29 | bundle exec rake spec 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .bundle/ 3 | .rspec 4 | .rspec_status 5 | .ruby-version 6 | .yardoc/ 7 | .env 8 | Gemfile.lock 9 | coverage/ 10 | gemfiles/*.gemfile.lock 11 | log/*.log 12 | pkg/ 13 | rdoc/ 14 | doc/ 15 | -------------------------------------------------------------------------------- /.standard.yml: -------------------------------------------------------------------------------- 1 | ruby_version: 2.5 2 | 3 | format: progress 4 | 5 | ignore: 6 | - '**/*': 7 | - Style/RedundantBegin 8 | - 'spec/**/*': 9 | - Lint/UselessAssignment 10 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | appraise "sidekiq-7.x" do 4 | gem "sidekiq", "~> 7.0" 5 | end 6 | 7 | appraise "sidekiq-7.0" do 8 | gem "sidekiq", "~> 7.0.0" 9 | end 10 | 11 | appraise "sidekiq-6.x" do 12 | gem "sidekiq", "~> 6.0" 13 | end 14 | 15 | appraise "sidekiq-6.0" do 16 | gem "sidekiq", "~> 6.0.0" 17 | end 18 | 19 | appraise "sidekiq-5.x" do 20 | gem "sidekiq", "~> 5.0" 21 | end 22 | 23 | appraise "sidekiq-5.0" do 24 | gem "sidekiq", "~> 5.0.0" 25 | end 26 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | All notable changes to this project will be documented in this file. 3 | 4 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), 5 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## 1.1.2 8 | 9 | ### Added 10 | - Added thread to monitor child processes to make sure they exit after a SIGTERM signal has been sent. If a process does not exit after the configured Sidekiq timeout time, then it will be killed with a SIGKILL signal. 11 | 12 | ### Changed 13 | - A SIGINT sent to the manager process will sent SIGTERM to the child processes to give them a chance to shutdown gracefully. 14 | 15 | ## 1.1.1 16 | 17 | ### Added 18 | - Guards to ensure signal processing thread doesn't die. 19 | 20 | ## 1.1.0 21 | 22 | ### Added 23 | - Sidekiq 7 support. 24 | - Max memory setting to automatically restart processes suffering from memory bloat. 25 | - Use a notification pipe to handle signals (@KevinCarterDev) 26 | 27 | ### Removed 28 | - Sidekiq < 5.0 support. 29 | - Ruby < 2.5 support. 30 | 31 | ## 1.0.4 32 | 33 | ### Fixed 34 | - Set $0 to "sidekiq" in preforked process so instrumentation libraries detecting sidekiq server from the command line will work. 35 | 36 | ## 1.0.3 37 | 38 | ### Fixed 39 | - Restore bin dir to gem distribution. 40 | 41 | ## 1.0.2 42 | 43 | ### Added 44 | - Support for sidekiq >= 6.1. 45 | - Set $0 to "sidekiq" so instrumentation libraries detecting sidekiq server from the command line will work. 46 | 47 | ### Changed 48 | - Minimum Ruby version 2.3. 49 | 50 | ## 1.0.1 51 | 52 | ### Changed 53 | - Remove auto require of `sidekiq/cli` so `require: false` does not need to be specified in a Gemfile. 54 | 55 | ## 1.0.0 56 | 57 | ### Added 58 | - Initial release. 59 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | git_source(:github) { |repo_name| "https://github.com/#{repo_name}" } 4 | 5 | gemspec 6 | 7 | gem "rspec", "~> 3.0" 8 | gem "rake" 9 | gem "standard", "~>1.0" 10 | gem "pry-byebug" 11 | gem "yard" 12 | gem "appraisal" 13 | -------------------------------------------------------------------------------- /MIT_LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2019 Brian Durand 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Sidekiq::ProcessManager 2 | 3 | [![Continuous Integration](https://github.com/bdurand/sidekiq-process_manager/actions/workflows/continuous_integration.yml/badge.svg)](https://github.com/bdurand/sidekiq-process_manager/actions/workflows/continuous_integration.yml) 4 | [![Regression Test](https://github.com/bdurand/sidekiq-process_manager/actions/workflows/regression_test.yml/badge.svg)](https://github.com/bdurand/sidekiq-process_manager/actions/workflows/regression_test.yml) 5 | [![Ruby Style Guide](https://img.shields.io/badge/code_style-standard-brightgreen.svg)](https://github.com/testdouble/standard) 6 | [![Gem Version](https://badge.fury.io/rb/sidekiq-process_manager.svg)](https://badge.fury.io/rb/sidekiq-process_manager) 7 | 8 | This gem provides a command line script for managing [sidekiq](https://github.com/mperham/sidekiq) processes. It starts up a process that then forks multiple sidekiq processes and manages their life cycle. This is important for large sidekiq installations, since without it on MRI ruby, sidekiq will only use one CPU core. By starting multiple processes you make all cores available. 9 | 10 | The sidekiq processes can all be managed by sending signals to the manager process. This process simply forwards the signals on to the child processes, allowing you to control the sidekiq processes as you normally would. 11 | 12 | If one of the sidekiq processes exits unexpectedly, the process manager automatically starts a new sidekiq process to replace it. 13 | 14 | ## Pre-Forking 15 | 16 | You can use pre-forking to improve memory utilization on the child sidekiq processes. This is similar to using pre-forking in a web server like puma or unicorn. You application will be pre-loaded by the master process and the child sidekiq processes will utilize the loaded code via copy-on-write memory. The overall effect is that you should be able to run more sidekiq processes in a lower memory footprint. 17 | 18 | One issue with pre-forking is that any file descriptors (including network connections) your application has open when it forks will be shared between all the processes. This can lead to race conditions and other problems. To fix it, you must close and reopen all database connections, etc. after the child sidekiq processes have been forked. 19 | 20 | To do this, your application will need to add an initializer with calls to `Sidekiq::ProcessManager.after_fork` and `Sidekiq::ProcessManager.before_fork`. 21 | 22 | The `before_fork` hook is called on the master process right before it starts forking processes. You can use this to close connections on the master process that you don't need. 23 | 24 | The `after_fork` hook is called after a forked sidekiq process has initialized the application. You can use this to re-establish connections so that each process gets it's own streams. 25 | 26 | At a minimum, you'll probably want the following hooks to close and re-open the ActiveRecord database connection: 27 | 28 | ```ruby 29 | Sidekiq::ProcessManager.before_fork do 30 | ActiveRecord::Base.connection.disconnect! 31 | end 32 | 33 | Sidekiq::ProcessManager.after_fork do 34 | ActiveRecord::Base.connection.reconnect! 35 | end 36 | ``` 37 | 38 | If you're already using a pre-forking web server, you'll need to do most of the same things for sidekiq as well. 39 | 40 | ## Pre-Booting 41 | 42 | If your application can't be pre-forked, you can at least load the gem files and libraries your application depends on instead by specifying a preboot file. This file will be loaded by the master process and any code loaded will be copy-on-write shared with the child processes. 43 | 44 | For a Rails application, you would normally want to preboot the `config/boot.rb` file. 45 | 46 | ## Memory Bloat 47 | 48 | You can also specify a maximum memory footprint that you want to allow for each child process. You can use this feature to automatically guard against poorly designed workers that bloat the Ruby memory heap. Note that you can also use an external process monitor to kill processes with memory bloat; the process manager will restart any process regardless of how it exits. 49 | 50 | ## Usage 51 | 52 | Install the gem in your sidekiq process and run it with `bundle exec sidekiq-process-manager` or, if you use [bundle binstubs](https://bundler.io/man/bundle-binstubs.1.html), `bin/sidekiq-process-manager`. Command line arguments are passed through to `sidekiq`. If you want to supply on of the `sidekiq-process_manager` specific options, those options should come first and the `sidekiq` options should appear after a `--` flag 53 | 54 | ```bash 55 | bundle exec sidekiq-process-manager -C config/sidekiq.yml 56 | ``` 57 | 58 | or 59 | 60 | 61 | ```bash 62 | bundle exec sidekiq-process-manager --no-prefork -- -C config/sidekiq.yml 63 | ``` 64 | 65 | You can specify the number of sidekiq processes to run with the `--processes` argument or with the `SIDEKIQ_PROCESSES` environment variable. The default number of processes is 1. 66 | 67 | ```bash 68 | bundle exec sidekiq-process-manager --processes 4 69 | ``` 70 | 71 | or 72 | 73 | ```bash 74 | SIDEKIQ_PROCESSES=4 bundle exec sidekiq-process-manager 75 | ``` 76 | 77 | You can turn pre-forking on or off with the --prefork or --no-prefork flag. You can also specify to turn on pre-forking with the `SIDEKIQ_PREFORK` environment variable. 78 | 79 | ```bash 80 | bundle exec sidekiq-process-manager --processes 4 --prefork 81 | ``` 82 | 83 | or 84 | 85 | ```bash 86 | SIDEKIQ_PREFORK=1 SIDEKIQ_PROCESSES=4 bundle exec sidekiq-process-manager 87 | ``` 88 | 89 | You can turn pre-booting on with the `--preboot` argument or with the `SIDEKIQ_PREBOOT` environment variable. 90 | 91 | ```bash 92 | bundle exec sidekiq-process-manager --processes 4 --preboot config/boot.rb 93 | ``` 94 | 95 | or 96 | 97 | ```bash 98 | SIDEKIQ_PREBOOT=config/boot.rb SIDEKIQ_PROCESSES=4 bundle exec sidekiq-process-manager 99 | ``` 100 | 101 | You can set the maximum memory allowed per sidekiq process with the `--max-memory` argument or with the `SIDEKIQ_MAX_MEMORY` environment variable. You can suffix the value with "m" to specify megabytes or "g" to specify gigabytes. 102 | 103 | ```bash 104 | bundle exec sidekiq-process-manager --processes 4 --max-memory 2g 105 | ``` 106 | 107 | or 108 | 109 | ```bash 110 | SIDEKIQ_MAX_MEMORY=2000m SIDEKIQ_PROCESSES=4 bundle exec sidekiq-process-manager 111 | ``` 112 | 113 | ## Alternatives 114 | 115 | Any process manager can be an alternative (service, update, systemd, monit, god, etc.). 116 | 117 | The advantages this gem can provide are: 118 | 119 | 1. Pre-forking can be useful on systems with many cores but not enough memory to run enough sidekiq processes to use them all. 120 | 121 | 2. Running in the foreground with output going to standard out instead of as daemon can integrate better into containerized environments. 122 | 123 | 3. Built with sidekiq in mind so signal passing is consistent with signals used when running a simple sidekiq process. 124 | 125 | ## Installation 126 | 127 | Add this line to your application's Gemfile: 128 | 129 | ```ruby 130 | gem "sidekiq-process_manager", require: false 131 | ``` 132 | 133 | Then execute: 134 | ```bash 135 | $ bundle 136 | ``` 137 | 138 | Or install it yourself as: 139 | ```bash 140 | $ gem install sidekiq-process_manager 141 | ``` 142 | 143 | ## Contributing 144 | 145 | Open a pull request on [GitHub](https://github.com/bdurand/sidekiq-process_manager). 146 | 147 | Please use the [standardrb](https://github.com/testdouble/standard) syntax and lint your code with `standardrb --fix` before submitting. 148 | 149 | ## License 150 | 151 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | begin 2 | require "bundler/setup" 3 | rescue LoadError 4 | puts "You must `gem install bundler` and `bundle install` to run rake tasks" 5 | end 6 | 7 | require "yard" 8 | YARD::Rake::YardocTask.new(:yard) 9 | 10 | require "bundler/gem_tasks" 11 | 12 | task :release do 13 | unless `git rev-parse --abbrev-ref HEAD`.chomp == "master" 14 | warn "Gem can only be released from the master branch" 15 | exit 1 16 | end 17 | end 18 | 19 | require "rspec/core/rake_task" 20 | 21 | RSpec::Core::RakeTask.new(:spec) 22 | 23 | task default: :spec 24 | 25 | desc "run the specs using appraisal" 26 | task :appraisals do 27 | exec "bundle exec appraisal rake spec" 28 | end 29 | 30 | namespace :appraisals do 31 | desc "install all the appraisal gemspecs" 32 | task :install do 33 | exec "bundle exec appraisal install" 34 | end 35 | end 36 | 37 | require "standard/rake" 38 | -------------------------------------------------------------------------------- /VERSION: -------------------------------------------------------------------------------- 1 | 1.1.2 2 | -------------------------------------------------------------------------------- /bin/sidekiq-process-manager: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require 'optparse' 5 | require_relative "../lib/sidekiq-process_manager" 6 | 7 | DEFAULT_PROCESS_COUNT = 1 8 | 9 | def parse_max_memory(max_memory) 10 | value = nil 11 | 12 | matched = max_memory.to_s.match(/\A([\d]+(?:\.[\d]+)?)([mg])\z/i) 13 | if matched 14 | value = matched[1].to_f 15 | if matched[2].downcase == 'm' 16 | value *= 1024 * 1024 17 | elsif matched[2].downcase == 'g' 18 | value *= 1024 * 1024 * 1024 19 | end 20 | end 21 | 22 | value 23 | end 24 | 25 | options = { 26 | process_count: Integer(ENV.fetch('SIDEKIQ_PROCESSES', DEFAULT_PROCESS_COUNT)), 27 | prefork: !ENV.fetch("SIDEKIQ_PREFORK", "").empty?, 28 | preboot: ENV["SIDEKIQ_PREBOOT"], 29 | max_memory: parse_max_memory(ENV["SIDEKIQ_MAX_MEMORY"]), 30 | mode: nil, 31 | } 32 | 33 | parser = OptionParser.new do |opts| 34 | opts.banner = "Usage: sidekiq-process-manager [options] [--] [sidekiq options]" 35 | 36 | opts.on('--processes PROCESSES', Integer, "Number of processes to spin up (can also specify with SIDEKIQ_PROCESSES)") do |count| 37 | options[:process_count] = count 38 | end 39 | 40 | opts.on('--[no-]prefork', "Use prefork for spinning up sidekiq processes (can also specify with SIDEKIQ_PREFORK)") do |prefork| 41 | options[:prefork] = prefork 42 | end 43 | 44 | opts.on('--preboot FILE', "Preboot the processes (can also specify with SIDEKIQ_PREBOOT)") do |preboot| 45 | options[:preboot] = preboot 46 | end 47 | 48 | opts.on('--max-memory MEMORY', "Max memory for each process (can also specify with SIDEKIQ_MAX_MEMORY); suffix with m or g to specify megabytes or gigabytes") do |max_memory| 49 | options[:max_memory] = parse_max_memory(max_memory) 50 | end 51 | 52 | opts.on('--testing', "Enable test mode") do |testing| 53 | options[:mode] = :testing if testing 54 | end 55 | 56 | opts.on("--help", "Prints this help") do 57 | puts opts 58 | exit 59 | end 60 | 61 | opts.separator(<<~DESCR) 62 | 63 | After the manager options, pass in any options for the sidekiq processes. 64 | Additionally, passing in the optional `--` will explicitly end the manager options and begin the sidekiq opts. 65 | E.g. 66 | $ sidekiq-process-manager --no-prefork -- -C config/sidekiq.rb 67 | Calls sidekiq with `sidekiq -C config/sidekiq.rb` 68 | DESCR 69 | end 70 | 71 | sidekiq_args = [] 72 | begin 73 | parser.order!(ARGV) { |nonopt| sidekiq_args << nonopt } 74 | rescue OptionParser::InvalidOption => err 75 | # Handle the case where a user doesn't put in the `--` to separate the args 76 | sidekiq_args.concat(err.args) 77 | end 78 | 79 | ARGV[0, 0] = sidekiq_args 80 | 81 | begin 82 | manager = Sidekiq::ProcessManager::Manager.new(**options) 83 | manager.start 84 | rescue => e 85 | STDERR.puts e.message 86 | STDERR.puts e.backtrace.join($/) 87 | exit 1 88 | end 89 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_5.0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec", "~> 3.0" 6 | gem "rake" 7 | gem "standard", "~>1.0" 8 | gem "pry-byebug" 9 | gem "yard" 10 | gem "appraisal" 11 | gem "sidekiq", "~> 5.0.0" 12 | 13 | gemspec path: "../" 14 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_5.x.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec", "~> 3.0" 6 | gem "rake" 7 | gem "standard", "~>1.0" 8 | gem "pry-byebug" 9 | gem "yard" 10 | gem "appraisal" 11 | gem "sidekiq", "~> 5.0" 12 | 13 | gemspec path: "../" 14 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_6.0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec", "~> 3.0" 6 | gem "rake" 7 | gem "standard", "~>1.0" 8 | gem "pry-byebug" 9 | gem "yard" 10 | gem "appraisal" 11 | gem "sidekiq", "~> 6.0.0" 12 | 13 | gemspec path: "../" 14 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_6.x.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec", "~> 3.0" 6 | gem "rake" 7 | gem "standard", "~>1.0" 8 | gem "pry-byebug" 9 | gem "yard" 10 | gem "appraisal" 11 | gem "sidekiq", "~> 6.0" 12 | 13 | gemspec path: "../" 14 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_7.0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec", "~> 3.0" 6 | gem "rake" 7 | gem "standard", "~>1.0" 8 | gem "pry-byebug" 9 | gem "yard" 10 | gem "appraisal" 11 | gem "sidekiq", "~> 7.0.0" 12 | 13 | gemspec path: "../" 14 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_7.x.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "rspec", "~> 3.0" 6 | gem "rake" 7 | gem "standard", "~>1.0" 8 | gem "pry-byebug" 9 | gem "yard" 10 | gem "appraisal" 11 | gem "sidekiq", "~> 7.0" 12 | 13 | gemspec path: "../" 14 | -------------------------------------------------------------------------------- /lib/sidekiq-process_manager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "sidekiq/process_manager" 4 | -------------------------------------------------------------------------------- /lib/sidekiq/process_manager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "process_manager/version" 4 | require_relative "process_manager/manager" 5 | 6 | module Sidekiq 7 | module ProcessManager 8 | class << self 9 | def before_fork(&block) 10 | @before_fork ||= [] 11 | @before_fork << block 12 | end 13 | 14 | def after_fork(&block) 15 | @after_fork ||= [] 16 | @after_fork << block 17 | end 18 | 19 | def run_before_fork_hooks 20 | if defined?(@before_fork) && @before_fork 21 | @before_fork.each do |block| 22 | block.call 23 | end 24 | end 25 | @before_fork = nil 26 | end 27 | 28 | def run_after_fork_hooks 29 | if defined?(@after_fork) && @after_fork 30 | @after_fork.each do |block| 31 | block.call 32 | end 33 | end 34 | @after_fork = nil 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/sidekiq/process_manager/manager.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sidekiq" 4 | require "get_process_mem" 5 | 6 | module Sidekiq 7 | module ProcessManager 8 | # Process manager for sidekiq. This class is responsible for starting and monitoring 9 | # that the specified number of sidekiq processes are running. It will also forward 10 | # signals sent to the main process to the child processes. 11 | class Manager 12 | attr_reader :cli 13 | 14 | # Create a new process manager. 15 | # 16 | # @param process_count [Integer] The number of sidekiq processes to start. 17 | # @param prefork [Boolean] If true, the process manager will load the application before forking. 18 | # @param preboot [String] If set, the process manager will require the specified file before forking the child processes. 19 | # @param mode [Symbol] If set to :testing, the process manager will use a mock CLI. 20 | # @param silent [Boolean] If true, the process manager will not output any messages. 21 | def initialize(process_count: 1, prefork: false, preboot: nil, max_memory: nil, mode: nil, silent: false) 22 | require "sidekiq/cli" 23 | 24 | # Get the number of processes to fork 25 | @process_count = process_count 26 | raise ArgumentError.new("Process count must be greater than 1") if @process_count < 1 27 | 28 | @prefork = (prefork && process_count > 1) 29 | @preboot = preboot if process_count > 1 && !prefork 30 | @max_memory = ((max_memory.to_i > 0) ? max_memory.to_i : nil) 31 | 32 | if mode == :testing 33 | require_relative "../../../spec/support/mocks" 34 | @cli = MockSidekiqCLI.new(silent) 35 | @memory_check_interval = 1 36 | else 37 | @cli = Sidekiq::CLI.instance 38 | @memory_check_interval = 60 39 | end 40 | 41 | @silent = silent 42 | @pids = [] 43 | @terminated_pids = [] 44 | @started = false 45 | @mutex = Mutex.new 46 | end 47 | 48 | # Start the process manager. This method will start the specified number 49 | # of sidekiq processes and monitor them. It will only exit once all child 50 | # processes have exited. If a child process dies unexpectedly, it will be 51 | # restarted. 52 | # 53 | # Child processes are manged by sending the signals you would normally send 54 | # to a sidekiq process to the process manager instead. 55 | # 56 | # @return [void] 57 | def start 58 | raise "Process manager already started" if started? 59 | @started = true 60 | 61 | load_sidekiq 62 | 63 | master_pid = ::Process.pid 64 | 65 | @signal_pipe_read, @signal_pipe_write = IO.pipe 66 | 67 | @signal_thread = Thread.new do 68 | Thread.current.name = "signal-handler" 69 | 70 | while @signal_pipe_read.wait_readable 71 | begin 72 | signal = @signal_pipe_read.gets.strip 73 | send_signal_to_children(signal.to_sym) 74 | rescue => e 75 | log_warning("Error sending signal #{signal} to child processes: #{e.message}") 76 | end 77 | end 78 | end 79 | 80 | # Trap signals that will be forwarded to child processes 81 | [:INT, :TERM, :USR1, :USR2, :TSTP, :TTIN].each do |signal| 82 | ::Signal.trap(signal) do 83 | if ::Process.pid == master_pid 84 | signal = :TERM if signal == :INT 85 | @signal_pipe_write.puts(signal) 86 | end 87 | end 88 | end 89 | 90 | # Ensure that child processes receive the term signal when the master process exits. 91 | at_exit do 92 | if ::Process.pid == master_pid 93 | if @process_count > 0 94 | send_signal_to_children(:TERM) 95 | end 96 | wait_for_children_to_exit 97 | log_info("Process manager exiting") 98 | end 99 | end 100 | 101 | GC.start 102 | GC.compact if GC.respond_to?(:compact) 103 | # I'm not sure why, but running GC operations blocks until we try to write some I/O. 104 | File.write("/dev/null", "0") 105 | 106 | @process_count.times do 107 | start_child_process! 108 | end 109 | 110 | start_memory_monitor if @max_memory 111 | 112 | log_info("Process manager started") 113 | monitor_child_processes 114 | end 115 | 116 | # Helper to wait on the manager to wait on child processes to start up. 117 | # 118 | # @param timeout [Integer] The number of seconds to wait for child processes to start. 119 | # @return [void] 120 | def wait(timeout = 5) 121 | timeout_time = monotonic_time + timeout 122 | while monotonic_time <= timeout_time 123 | return if @pids.size == @process_count 124 | sleep(0.01) 125 | end 126 | 127 | raise Timeout::Error.new("child processes failed to start in #{timeout} seconds") 128 | end 129 | 130 | # Helper to gracefully stop all child processes. 131 | # 132 | # @return [void] 133 | def stop 134 | stop_memory_monitor 135 | @process_count = 0 136 | send_signal_to_children(:TSTP) 137 | send_signal_to_children(:TERM) 138 | end 139 | 140 | # Get all chile process pids. 141 | # 142 | # @return [Array] 143 | def pids 144 | @mutex.synchronize { @pids.dup } 145 | end 146 | 147 | # Return true if the process manager has started. 148 | # 149 | # @return [Boolean] 150 | def started? 151 | @started 152 | end 153 | 154 | private 155 | 156 | def log_info(message) 157 | return if @silent 158 | if $stderr.tty? 159 | $stderr.write("#{message}#{$/}") 160 | $stderr.flush 161 | else 162 | Sidekiq.logger.info(message) 163 | end 164 | end 165 | 166 | def log_warning(message) 167 | return if @silent 168 | if $stderr.tty? 169 | $stderr.write("#{message}#{$/}") 170 | $stderr.flush 171 | else 172 | Sidekiq.logger.warn(message) 173 | end 174 | end 175 | 176 | def monotonic_time 177 | ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) 178 | end 179 | 180 | def sidekiq_options 181 | if Sidekiq.respond_to?(:default_configuration) 182 | Sidekiq.default_configuration 183 | else 184 | Sidekiq.options 185 | end 186 | end 187 | 188 | def load_sidekiq 189 | @cli.parse 190 | 191 | # Disable daemonization and pidfile creation for child processes (sidekiq < 6.0) 192 | if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.0") 193 | sidekiq_options[:daemon] = false 194 | sidekiq_options[:pidfile] = false 195 | end 196 | 197 | if @prefork 198 | log_info("Pre-forking application") 199 | # Set $0 so instrumentation libraries detecting sidekiq from the command run will work properly. 200 | save_command_line = $0 201 | $0 = File.join(File.dirname($0), "sidekiq") 202 | # Prior to sidekiq 6.1 the method to boot the application was boot_system 203 | if @cli.methods.include?(:boot_application) || @cli.private_methods.include?(:boot_application) 204 | @cli.send(:boot_application) 205 | else 206 | @cli.send(:boot_system) 207 | end 208 | $0 = save_command_line 209 | Sidekiq::ProcessManager.run_before_fork_hooks 210 | elsif @preboot && !@preboot.empty? 211 | if ::File.exist?(@preboot) 212 | require ::File.expand_path(@preboot).sub(/\.rb\Z/, "") 213 | else 214 | log_warning("Could not find preboot file #{@preboot}") 215 | end 216 | end 217 | end 218 | 219 | def set_program_name! 220 | $PROGRAM_NAME = "sidekiq process manager #{sidekiq_options[:tag]} [#{@pids.size} processes]" 221 | end 222 | 223 | def start_child_process! 224 | pid = fork do 225 | # Set $0 so instrumentation libraries detecting sidekiq from the command run will work properly. 226 | $0 = File.join(File.dirname($0), "sidekiq") 227 | @process_count = 0 228 | @pids.clear 229 | @signal_thread.kill 230 | @signal_pipe_read.close 231 | @signal_pipe_write.close 232 | Sidekiq::ProcessManager.run_after_fork_hooks 233 | @cli.run 234 | end 235 | @mutex.synchronize { @pids << pid } 236 | log_info("Forked sidekiq process with pid #{pid}") 237 | set_program_name! 238 | end 239 | 240 | def send_signal_to_children(signal) 241 | log_info("Process manager trapped signal #{signal}") 242 | @process_count = 0 if signal == :INT || signal == :TERM 243 | pids.each do |pid| 244 | send_signal_to_pid(signal, pid) 245 | end 246 | end 247 | 248 | def send_signal_to_pid(signal, pid) 249 | signal = signal.to_sym 250 | begin 251 | log_info("Sending signal #{signal} to sidekiq process #{pid}") 252 | ::Process.kill(signal, pid) 253 | if [:TERM, :INT].include?(signal) 254 | Thread.new do 255 | Thread.current.name = "pid-#{pid}-killer" 256 | ensure_pid_dies(pid) 257 | end 258 | end 259 | rescue Errno::ESRCH 260 | # The process is already dead 261 | end 262 | end 263 | 264 | def start_memory_monitor 265 | log_info("Starting memory monitor with max memory #{(@max_memory / (1024**2)).round}mb") 266 | 267 | @memory_monitor = Thread.new do 268 | Thread.current.name = "memory-monitor" 269 | loop do 270 | sleep(@memory_check_interval) 271 | 272 | pids.each do |pid| 273 | begin 274 | memory = GetProcessMem.new(pid) 275 | if memory.bytes > @max_memory 276 | log_warning("Killing bloated sidekiq process #{pid}: #{memory.mb.round}mb used") 277 | send_signal_to_pid(:TERM, pid) 278 | break 279 | end 280 | rescue => e 281 | log_warning("Error monitoring memory for sidekiq process #{pid}: #{e.inspect}") 282 | end 283 | end 284 | end 285 | end 286 | end 287 | 288 | def stop_memory_monitor 289 | if defined?(@memory_monitor) && @memory_monitor 290 | @memory_monitor.kill 291 | end 292 | end 293 | 294 | def wait_for_children_to_exit 295 | pids.each do |pid| 296 | while process_alive?(pid) 297 | sleep(0.01) 298 | end 299 | end 300 | log_info("All sidekiq processes have exited") 301 | end 302 | 303 | def process_alive?(pid) 304 | begin 305 | ::Process.getpgid(pid) 306 | true 307 | rescue Errno::ESRCH 308 | false 309 | end 310 | end 311 | 312 | def ensure_pid_dies(pid) 313 | # Wait for the process to die, or kill it after a timeout. 314 | timeout = (sidekiq_options[:timeout] || 25).to_f 315 | end_time = monotonic_time + timeout 316 | while monotonic_time < end_time && process_alive?(pid) 317 | sleep(0.1) 318 | end 319 | 320 | if process_alive?(pid) 321 | begin 322 | ::Process.kill(:KILL, pid) 323 | log_warning("Sidekiq process #{pid} failed to exit after #{timeout} seconds; killed with SIGKILL") 324 | rescue Errno::ESRCH 325 | # The process is already dead 326 | end 327 | end 328 | end 329 | 330 | # Listen for child processes dying and restart if necessary. 331 | def monitor_child_processes 332 | loop do 333 | pid = ::Process.wait 334 | @mutex.synchronize { @pids.delete(pid) } 335 | 336 | log_info("Sidekiq process #{pid} exited") 337 | 338 | # If there are not enough processes running, start a replacement one. 339 | if @process_count > @pids.size 340 | start_child_process! 341 | end 342 | 343 | set_program_name! 344 | 345 | if @pids.empty? 346 | break 347 | end 348 | end 349 | end 350 | end 351 | end 352 | end 353 | -------------------------------------------------------------------------------- /lib/sidekiq/process_manager/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sidekiq 4 | module ProcessManager 5 | VERSION = File.read(File.expand_path("../../../VERSION", __dir__)).strip 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /sidekiq-process_manager.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "sidekiq-process_manager" 5 | spec.version = File.read(File.expand_path("VERSION", __dir__)).strip 6 | spec.authors = ["Brian Durand"] 7 | spec.email = ["bbdurand@gmail.com"] 8 | 9 | spec.summary = "Process manager for forking and monitoring multiple sidekiq processes." 10 | spec.homepage = "https://github.com/bdurand/sidekiq-process_manager" 11 | spec.license = "MIT" 12 | 13 | # Specify which files should be added to the gem when it is released. 14 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 15 | ignore_files = %w[ 16 | . 17 | Appraisals 18 | Gemfile 19 | Gemfile.lock 20 | Rakefile 21 | gemfiles/ 22 | spec/ 23 | ] 24 | spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do 25 | `git ls-files -z`.split("\x0").reject { |f| ignore_files.any? { |path| f.start_with?(path) } } 26 | end 27 | 28 | spec.bindir = "bin" 29 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 30 | spec.require_paths = ["lib"] 31 | 32 | spec.required_ruby_version = ">= 2.5" 33 | 34 | spec.add_dependency "sidekiq", ">= 5.0" 35 | spec.add_dependency "get_process_mem" 36 | 37 | spec.add_development_dependency "bundler" 38 | end 39 | -------------------------------------------------------------------------------- /spec/sidekiq/process_manager/manager_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | describe Sidekiq::ProcessManager::Manager do 6 | let!(:manager) do 7 | manager = Sidekiq::ProcessManager::Manager.new(process_count: process_count, prefork: prefork, preboot: preboot, max_memory: max_memory, mode: :testing, silent: true) 8 | Thread.new { manager.start } 9 | manager.wait(2) 10 | manager 11 | end 12 | 13 | let(:prefork) { false } 14 | let(:preboot) { nil } 15 | let(:process_count) { 2 } 16 | let(:max_memory) { nil } 17 | 18 | after do 19 | manager.stop 20 | manager.wait(5) 21 | end 22 | 23 | describe "managing processes" do 24 | it "should start a specified number of processes" do 25 | expect(manager.pids.size).to eq 2 26 | end 27 | 28 | it "should forward signals to child processes" do 29 | signals = [:USR1, :USR2, :TSTP, :TTIN] 30 | signals.each do |signal| 31 | ::Process.kill(signal, ::Process.pid) 32 | end 33 | manager.pids.each do |pid| 34 | manager.cli.processes.each do |process| 35 | expect(process.signals).to match_array(signals) 36 | end 37 | end 38 | end 39 | 40 | it "should exit when all child processes have terminated with an INT signal" do 41 | ::Process.kill(:INT, ::Process.pid) 42 | sleep(2) # allow the signal pipe time to process the signal 43 | manager.wait 44 | expect(manager.pids.size).to eq 0 45 | end 46 | 47 | it "should exit when all child processes have terminated with a TERM signal" do 48 | ::Process.kill(:TERM, ::Process.pid) 49 | sleep(2) # allow the signal pipe time to process the signal 50 | manager.wait 51 | expect(manager.pids.size).to eq 0 52 | end 53 | 54 | it "should restart child processes if they unexpectedly die" do 55 | pids = manager.pids 56 | kill_pid = pids.first 57 | keep_pid = pids.last 58 | ::Process.kill(:TERM, kill_pid) 59 | sleep(2) 60 | manager.wait 61 | expect(manager.pids.size).to eq 2 62 | expect(manager.pids).to_not include(kill_pid) 63 | expect(manager.pids).to include(keep_pid) 64 | end 65 | 66 | it "should not be able to start the process manager twice" do 67 | expect { manager.start }.to raise_error("Process manager already started") 68 | end 69 | end 70 | 71 | describe "with max memory set for child processes" do 72 | let(:max_memory) { 1024 * 1024 } 73 | 74 | it "should restart child processes if they use too much memory" do 75 | pids = manager.pids 76 | expect(pids.size).to eq 2 77 | allow(::Process).to receive(:kill).and_call_original 78 | expect(::Process).to receive(:kill).with(:TERM, pids.first).and_call_original 79 | sleep(2) 80 | manager.wait 81 | # This check is flakey with Sidekiq 6.0 and below 82 | if Gem::Version.new(Sidekiq::VERSION) >= Gem::Version.new("6.5") 83 | expect(manager.pids).to_not include(pids.first) 84 | end 85 | end 86 | end 87 | 88 | describe "without pre-forking processes" do 89 | it "should not preload the application" do 90 | expect(manager.cli.application).to eq nil 91 | end 92 | 93 | it "should not call before or after fork hooks on the child processes" do 94 | manager.pids.each do |pid| 95 | expect(manager.cli.output_for(pid)).to eq [] 96 | end 97 | end 98 | end 99 | 100 | describe "with pre-forking processes" do 101 | let(:prefork) { true } 102 | 103 | it "should preload the application" do 104 | expect(manager.cli.application).to_not eq nil 105 | end 106 | 107 | it "should call before fork hooks on the parent process" do 108 | expect(manager.cli.application.hooks).to eq [:before_fork_1, :before_fork_2] 109 | end 110 | 111 | it "should call after fork hooks on the child processes" do 112 | manager.pids.each do |pid| 113 | expect(manager.cli.output_for(pid)).to eq ["after_fork_1", "after_fork_2"] 114 | end 115 | end 116 | end 117 | 118 | describe "with pre-booting code" do 119 | let(:preboot_file) do 120 | file = Tempfile.new(["preboot", ".rb"]) 121 | file.write("$prebooted = true") 122 | file.flush 123 | file 124 | end 125 | let(:preboot) { preboot_file.path } 126 | 127 | after(:each) do 128 | # rubocop:disable Style/GlobalVars 129 | $prebooted = false 130 | # rubocop:enable Style/GlobalVars 131 | preboot_file.close 132 | end 133 | 134 | it "should load the config/boot.rb file" do 135 | manager 136 | # rubocop:disable Style/GlobalVars 137 | expect($prebooted).to eq true 138 | # rubocop:enable Style/GlobalVars 139 | end 140 | 141 | it "should not call before or after fork hooks on the child processes" do 142 | manager.pids.each do |pid| 143 | expect(manager.cli.output_for(pid)).to eq [] 144 | end 145 | end 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../Gemfile", __dir__) 4 | 5 | require "bundler/setup" if File.exist?(ENV["BUNDLE_GEMFILE"]) 6 | 7 | Bundler.require(:default, :test) 8 | 9 | require "tempfile" 10 | 11 | require_relative "../lib/sidekiq-process_manager" 12 | require_relative "support/mocks" 13 | 14 | RSpec.configure do |config| 15 | config.order = :random 16 | end 17 | 18 | Sidekiq.logger.level = :error 19 | sidekiq_options = (Sidekiq.respond_to?(:default_configuration) ? Sidekiq.default_configuration : Sidekiq.options) 20 | sidekiq_options[:timeout] = 0.1 21 | -------------------------------------------------------------------------------- /spec/support/mock_application.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MockApplication 4 | attr_reader :hooks 5 | 6 | def initialize(output: nil, silent: false) 7 | @hooks = [] 8 | @output = output 9 | @silent = silent 10 | end 11 | 12 | def start 13 | Sidekiq::ProcessManager.before_fork do 14 | record_hook(:before_fork_1) 15 | end 16 | 17 | Sidekiq::ProcessManager.before_fork do 18 | record_hook(:before_fork_2) 19 | end 20 | 21 | Sidekiq::ProcessManager.after_fork do 22 | record_hook(:after_fork_1) 23 | end 24 | 25 | Sidekiq::ProcessManager.after_fork do 26 | record_hook(:after_fork_2) 27 | end 28 | end 29 | 30 | private 31 | 32 | def record_hook(message) 33 | @hooks << message 34 | if @output 35 | @output.write("pid(#{::Process.pid}): #{message}\n") 36 | @output.flush 37 | end 38 | if $stdout.tty? && !@silent 39 | $stdout.write("pid(#{::Process.pid}): #{message}\n") 40 | $stdout.flush 41 | end 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/support/mock_sidekiq_cli.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MockSidekiqCLI 4 | attr_reader :application 5 | attr_writer :silent 6 | 7 | def initialize(silent = false) 8 | @processes = {} 9 | @application = nil 10 | @pipe_reader, @pipe_writer = IO.pipe 11 | @process_output = StringIO.new 12 | @silent = silent 13 | end 14 | 15 | def output_for(pid) 16 | read_process_output! 17 | marker = "pid(#{pid}):" 18 | @process_output.string.split("\n").select { |line| line.start_with?("pid(#{pid}):") }.collect { |line| line.sub(marker, "").strip } 19 | end 20 | 21 | def run 22 | unless Sidekiq::CLI.instance.public_methods.include?(:run) 23 | raise "Sidekiq::CLI #{Sidekiq::VERSION} does not defined a run method" 24 | end 25 | $PROGRAM_NAME = "mock sidekiq" 26 | boot_mock_application 27 | process = MockSidekiqProcess.new(@application) 28 | @processes[process.pid] = process 29 | process.run 30 | rescue => e 31 | warn e.inpsect, e.backtrace.join("\n") 32 | end 33 | 34 | def parse 35 | unless Sidekiq::CLI.instance.public_methods.include?(:parse) 36 | raise "Sidekiq::CLI #{Sidekiq::VERSION} does not defined a parse method" 37 | end 38 | end 39 | 40 | def reset! 41 | @processes.clear 42 | @process_output = StringIO.new 43 | end 44 | 45 | def processes 46 | @processes.values 47 | end 48 | 49 | private 50 | 51 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 52 | def #{(Sidekiq::VERSION.to_f < 6.0) ? "boot_system" : "boot_application"} 53 | boot_mock_application 54 | end 55 | RUBY 56 | 57 | def boot_mock_application 58 | return if @application 59 | @application = MockApplication.new(output: @pipe_writer, silent: @silent) 60 | @application.start 61 | rescue => e 62 | warn e.inpsect, e.backtrace.join("\n") 63 | end 64 | 65 | def read_process_output! 66 | begin 67 | loop { @process_output << @pipe_reader.read_nonblock(4096) } 68 | rescue IO::EAGAINWaitReadable 69 | nil 70 | end 71 | end 72 | end 73 | -------------------------------------------------------------------------------- /spec/support/mock_sidekiq_process.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | class MockSidekiqProcess 4 | attr_reader :pid, :signals, :application 5 | 6 | def initialize(application) 7 | @pid = ::Process.pid 8 | @application = application 9 | @signals = [] 10 | @running = false 11 | end 12 | 13 | def run 14 | @running = true 15 | 16 | [:INT, :TERM, :USR1, :USR2, :TSTP, :TTIN].each do |signal| 17 | ::Signal.trap(signal) do 18 | @signals << signal 19 | if signal == :INT || signal == :TERM 20 | @running = false 21 | end 22 | end 23 | end 24 | 25 | timeout_time = monotonic_time + 10 26 | while running? 27 | sleep(0.01) 28 | if monotonic_time > timeout_time 29 | @running = false 30 | end 31 | end 32 | end 33 | 34 | def running? 35 | @running 36 | end 37 | 38 | private 39 | 40 | def monotonic_time 41 | ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/support/mocks.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "mock_application" 4 | require_relative "mock_sidekiq_cli" 5 | require_relative "mock_sidekiq_process" 6 | --------------------------------------------------------------------------------