├── .rspec ├── lib ├── dynomatic │ ├── version.rb │ ├── adapters │ │ └── delayed_job.rb │ ├── configuration.rb │ ├── adapters.rb │ ├── hobby_scaler.rb │ ├── scaler.rb │ └── master.rb └── dynomatic.rb ├── .travis.yml ├── Rakefile ├── bin ├── setup └── console ├── .gitignore ├── Gemfile ├── spec ├── spec_helper.rb └── dynomatic_spec.rb ├── LICENSE.txt ├── dynomatic.gemspec ├── Gemfile.lock └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /lib/dynomatic/version.rb: -------------------------------------------------------------------------------- 1 | module Dynomatic 2 | VERSION = "0.1.0" 3 | end 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | rvm: 4 | - 2.3.1 5 | before_install: gem install bundler -v 1.16.0 6 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | RSpec::Core::RakeTask.new(:spec) 5 | 6 | task :default => :spec 7 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle/ 2 | /.yardoc 3 | /_yardoc/ 4 | /coverage/ 5 | /doc/ 6 | /pkg/ 7 | /spec/reports/ 8 | /tmp/ 9 | 10 | # rspec failure tracking 11 | .rspec_status 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | git_source(:github) {|repo_name| "https://github.com/#{repo_name}" } 4 | 5 | # Specify your gem's dependencies in dynomatic.gemspec 6 | gemspec 7 | -------------------------------------------------------------------------------- /lib/dynomatic/adapters/delayed_job.rb: -------------------------------------------------------------------------------- 1 | module Dynomatic 2 | module Adapters 3 | class DelayedJob 4 | def job_count 5 | Delayed::Job.where("run_at <= NOW()").count 6 | end 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/dynomatic/configuration.rb: -------------------------------------------------------------------------------- 1 | module Dynomatic 2 | class Configuration 3 | attr_accessor :adapter, :rules, :worker_names, :heroku_token, :heroku_app 4 | 5 | def sorted_rules 6 | @rules.sort_by { |rule| rule[:over] } 7 | end 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /lib/dynomatic/adapters.rb: -------------------------------------------------------------------------------- 1 | module Dynomatic 2 | module Adapters 3 | extend ActiveSupport::Autoload 4 | 5 | autoload :DelayedJob 6 | 7 | module_function 8 | 9 | def detect 10 | case 11 | when defined? ::Delayed::Job 12 | Dynomatic::Adapters::DelayedJob.new 13 | else 14 | nil 15 | end 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "dynomatic" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start(__FILE__) 15 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | require "dynomatic" 3 | 4 | RSpec.configure do |config| 5 | # Enable flags like --only-failures and --next-failure 6 | config.example_status_persistence_file_path = ".rspec_status" 7 | 8 | # Disable RSpec exposing methods globally on `Module` and `main` 9 | config.disable_monkey_patching! 10 | 11 | config.expect_with :rspec do |c| 12 | c.syntax = :expect 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/dynomatic.rb: -------------------------------------------------------------------------------- 1 | require "active_job" 2 | require "dynomatic/version" 3 | 4 | module Dynomatic 5 | extend ActiveSupport::Autoload 6 | 7 | autoload :Adapters 8 | autoload :Configuration 9 | autoload :Master 10 | autoload :Scaler 11 | autoload :HobbyScaler 12 | 13 | mattr_accessor(:configuration) { Configuration.new } 14 | 15 | def self.configure 16 | configuration.adapter ||= Dynomatic::Adapters.detect 17 | 18 | yield(configuration) 19 | 20 | Dynomatic::Master.new(configuration).install! 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/dynomatic/hobby_scaler.rb: -------------------------------------------------------------------------------- 1 | module Dynomatic 2 | class HobbyScaler 3 | attr :scaler, :worker_processes 4 | 5 | def initialize(heroku_token, heroku_app, worker_processes) 6 | @scaler = Scaler.new(heroku_token, heroku_app) 7 | @worker_processes = worker_processes 8 | end 9 | 10 | def scale_to(dyno_count) 11 | start_processes = Array(worker_processes.first(dyno_count)) 12 | stop_processes = Array(worker_processes[dyno_count..-1]) 13 | 14 | start_processes.each do |process| 15 | scaler.start!(process) 16 | end 17 | 18 | stop_processes.each do |process| 19 | scaler.stop!(process) 20 | end 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/dynomatic/scaler.rb: -------------------------------------------------------------------------------- 1 | require "platform-api" 2 | 3 | module Dynomatic 4 | class Scaler 5 | attr :heroku_token, :heroku_app 6 | 7 | def initialize(heroku_token, heroku_app) 8 | @heroku_token = heroku_token 9 | @heroku_app = heroku_app 10 | end 11 | 12 | def scale_to(dyno_count, type: "worker") 13 | Rails.logger.info "Scaling #{type} to #{dyno_count}" 14 | 15 | client.formation.update(heroku_app, type, {quantity: dyno_count}) 16 | end 17 | 18 | def start!(type) 19 | scale_to(1, type: type) 20 | end 21 | 22 | def stop!(type) 23 | scale_to(0, type: type) 24 | end 25 | 26 | private 27 | 28 | def client 29 | @client = ::PlatformAPI.connect_oauth(heroku_token) 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 David Verhasselt 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /dynomatic.gemspec: -------------------------------------------------------------------------------- 1 | 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "dynomatic/version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "dynomatic" 8 | spec.version = Dynomatic::VERSION 9 | spec.authors = ["David Verhasselt"] 10 | spec.email = ["david@crowdway.com"] 11 | 12 | spec.summary = %q{Gem to autoscale Heroku worker dynos based on number of pending jobs} 13 | spec.description = %q{Gem to autoscale Heroku worker dynos based on number of pending jobs} 14 | spec.homepage = "https://github.com/dv/dynomatic" 15 | spec.license = "MIT" 16 | 17 | spec.files = `git ls-files -z`.split("\x0").reject do |f| 18 | f.match(%r{^(test|spec|features)/}) 19 | end 20 | spec.bindir = "exe" 21 | spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } 22 | spec.require_paths = ["lib"] 23 | 24 | spec.add_dependency "activejob" 25 | spec.add_dependency "activesupport" 26 | spec.add_dependency "platform-api" 27 | 28 | spec.add_development_dependency "pry" 29 | spec.add_development_dependency "delayed_job" 30 | spec.add_development_dependency "bundler", "~> 1.16" 31 | spec.add_development_dependency "rake", "~> 10.0" 32 | spec.add_development_dependency "rspec", "~> 3.0" 33 | end 34 | -------------------------------------------------------------------------------- /lib/dynomatic/master.rb: -------------------------------------------------------------------------------- 1 | module Dynomatic 2 | class Master 3 | attr_accessor :current_dyno_count, :configuration 4 | 5 | def initialize(configuration) 6 | @configuration = configuration 7 | end 8 | 9 | def install! 10 | this = self 11 | 12 | ActiveJob::Base.before_perform do 13 | this.adjust_dynos! 14 | end 15 | end 16 | 17 | # Main method 18 | def adjust_dynos! 19 | job_count = configuration.adapter.job_count 20 | new_dyno_count = dyno_count_for_job_count(job_count) 21 | 22 | if current_dyno_count != new_dyno_count 23 | set_dyno_count!(new_dyno_count) 24 | end 25 | end 26 | 27 | private 28 | 29 | def dyno_count_for_job_count(job_count) 30 | rule = configuration.sorted_rules.reverse.find { |rule| rule[:at_least] <= job_count } 31 | 32 | rule[:dynos] 33 | end 34 | 35 | def set_dyno_count!(new_dyno_count) 36 | old_dyno_count = current_dyno_count 37 | self.current_dyno_count = new_dyno_count 38 | 39 | scaler.scale_to(new_dyno_count) 40 | rescue => error 41 | # If something went wrong, reset to old value 42 | self.current_dyno_count = old_dyno_count 43 | 44 | Rails.logger.error "Something went wrong while adjusting dyno count: #{error.message}." 45 | end 46 | 47 | def scaler 48 | @scaler ||= 49 | if configuration.worker_names.present? 50 | HobbyScaler.new(configuration.heroku_token, configuration.heroku_app, configuration.worker_names) 51 | else 52 | Scaler.new(configuration.heroku_token, configuration.heroku_app) 53 | end 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | dynomatic (0.1.0) 5 | activejob 6 | activesupport 7 | platform-api 8 | 9 | GEM 10 | remote: https://rubygems.org/ 11 | specs: 12 | activejob (5.1.4) 13 | activesupport (= 5.1.4) 14 | globalid (>= 0.3.6) 15 | activesupport (5.1.4) 16 | concurrent-ruby (~> 1.0, >= 1.0.2) 17 | i18n (~> 0.7) 18 | minitest (~> 5.1) 19 | tzinfo (~> 1.1) 20 | coderay (1.1.2) 21 | concurrent-ruby (1.0.5) 22 | delayed_job (4.1.4) 23 | activesupport (>= 3.0, < 5.2) 24 | diff-lcs (1.3) 25 | erubis (2.7.0) 26 | excon (0.60.0) 27 | globalid (0.4.1) 28 | activesupport (>= 4.2.0) 29 | heroics (0.0.24) 30 | erubis (~> 2.0) 31 | excon 32 | moneta 33 | multi_json (>= 1.9.2) 34 | i18n (0.9.1) 35 | concurrent-ruby (~> 1.0) 36 | method_source (0.9.0) 37 | minitest (5.11.1) 38 | moneta (0.8.1) 39 | multi_json (1.13.1) 40 | platform-api (2.1.0) 41 | heroics (~> 0.0.23) 42 | moneta (~> 0.8.1) 43 | pry (0.11.3) 44 | coderay (~> 1.1.0) 45 | method_source (~> 0.9.0) 46 | rake (10.5.0) 47 | rspec (3.7.0) 48 | rspec-core (~> 3.7.0) 49 | rspec-expectations (~> 3.7.0) 50 | rspec-mocks (~> 3.7.0) 51 | rspec-core (3.7.1) 52 | rspec-support (~> 3.7.0) 53 | rspec-expectations (3.7.0) 54 | diff-lcs (>= 1.2.0, < 2.0) 55 | rspec-support (~> 3.7.0) 56 | rspec-mocks (3.7.0) 57 | diff-lcs (>= 1.2.0, < 2.0) 58 | rspec-support (~> 3.7.0) 59 | rspec-support (3.7.0) 60 | thread_safe (0.3.6) 61 | tzinfo (1.2.4) 62 | thread_safe (~> 0.1) 63 | 64 | PLATFORMS 65 | ruby 66 | 67 | DEPENDENCIES 68 | bundler (~> 1.16) 69 | delayed_job 70 | dynomatic! 71 | pry 72 | rake (~> 10.0) 73 | rspec (~> 3.0) 74 | 75 | BUNDLED WITH 76 | 1.16.0 77 | -------------------------------------------------------------------------------- /spec/dynomatic_spec.rb: -------------------------------------------------------------------------------- 1 | RSpec.describe Dynomatic do 2 | class DummyJob < ActiveJob::Base 3 | def perform 4 | sleep 0.1 5 | end 6 | end 7 | 8 | class TestAdapter 9 | attr_accessor :job_count 10 | end 11 | 12 | let(:scaler_double) { instance_double(Dynomatic::Scaler) } 13 | 14 | before do 15 | # Remove previous installation 16 | ActiveJob::Base.reset_callbacks("perform") 17 | 18 | # Stub out scaler 19 | allow(Dynomatic::Scaler).to receive(:new).and_return(scaler_double) 20 | 21 | Dynomatic.configure do |config| 22 | config.adapter = TestAdapter.new 23 | config.rules = [ 24 | {at_least: 0, dynos: 1}, 25 | {at_least: 10, dynos: 5}, 26 | {at_least: 20, dynos: 10}, 27 | {at_least: 30, dynos: 15}, 28 | ] 29 | end 30 | end 31 | 32 | context "when job count is high" do 33 | before { Dynomatic.configuration.adapter.job_count = 22 } 34 | 35 | it "increases dyno count" do |example| 36 | expect(scaler_double).to receive(:scale_to).with(10) 37 | 38 | DummyJob.perform_now 39 | end 40 | 41 | it "does not increase dyno count twice" do |example| 42 | expect(scaler_double).to receive(:scale_to) 43 | DummyJob.perform_now 44 | 45 | expect(scaler_double).not_to receive(:scale_to) 46 | DummyJob.perform_now 47 | end 48 | 49 | it "lowers dyno count again when jobs decrease" do |example| 50 | expect(scaler_double).to receive(:scale_to).with(10).ordered 51 | DummyJob.perform_now 52 | 53 | Dynomatic.configuration.adapter.job_count = 12 54 | expect(scaler_double).to receive(:scale_to).with(5).ordered 55 | DummyJob.perform_now 56 | 57 | Dynomatic.configuration.adapter.job_count = 0 58 | expect(scaler_double).to receive(:scale_to).with(1).ordered 59 | DummyJob.perform_now 60 | end 61 | end 62 | 63 | it "has a version number" do 64 | expect(Dynomatic::VERSION).not_to be nil 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dynomatic 2 | 3 | Use this gem to automatically scale up/down your Heroku worker dynos based on the number of ActiveeJob jobs currently waiting in the queue. 4 | 5 | Currently only DelayedJob is supported but it's really easy to add support for your preferred background job library (see `lib/dynomatic/adapters/delayed_job`, it's basically one method you need to implement. PRs welcome!). 6 | 7 | ## Installation 8 | 9 | Add this line to your application's Gemfile: 10 | 11 | ```ruby 12 | gem 'dynomatic' 13 | ``` 14 | 15 | And then execute: 16 | 17 | $ bundle 18 | 19 | Or install it yourself as: 20 | 21 | $ gem install dynomatic 22 | 23 | ## Usage 24 | 25 | Configure it as follows: 26 | 27 | ```ruby 28 | # In an initializer: 29 | Dynomatic.configure do |config| 30 | # Optional, since it will try to detect the correct adapter: 31 | # config.adapter = Dynomatic::Adapters::DelayedJob 32 | 33 | # Check your heroku account for the correct token, or run 34 | # 35 | # heroku auth:token 36 | # 37 | config.heroku_token = ENV["HEROKU_TOKEN"] 38 | config.heroku_app = ENV["HEROKU_APP_NAME"] 39 | 40 | # Add your own rules here. Dynomatic will select the lowest dyno count that 41 | # matches the "at_least" number of jobs. 42 | # 43 | # Some examples based on the following rules: 44 | # 25 jobs = 10 dynos, 31 jobs = 15 dynos 45 | config.rules = [ 46 | {at_least: 0, dynos: 1}, 47 | {at_least: 10, dynos: 5}, 48 | {at_least: 20, dynos: 10}, 49 | {at_least: 30, dynos: 15}, 50 | ] 51 | 52 | # Optional (see README), let Dynomatic know which are the worker dynos: 53 | # config.worker_names = %w(worker1 worker2 worker3 worker4) 54 | end 55 | ``` 56 | 57 | Unless you call `Dynomatic.configure`, it won't hook into ActiveJob. It's sufficient to just skip that call in other environments to disable the scaling. 58 | 59 | ## What if I'm using Hobby dynos? 60 | 61 | Heroku only allows one to scale up/down dynos if they're the more expensive "professional" dynos. If you're using Hobby dynos, you can only start and stop them. 62 | 63 | To work around that, we can just create a bunch of different dyno types, and then enable/disable them as a means of scaling up or down. 64 | 65 | First you need to setup the dynos in your `Procfile`. Wheras before it probably looked a bit like this: 66 | 67 | ```Procfile 68 | web: bundle exec puma -C config/puma.rb 69 | worker: bundle exec rails jobs:work 70 | ``` 71 | 72 | Now add a bunch of worker-types with the exact same command: 73 | 74 | ```Procfile 75 | web: bundle exec puma -C config/puma.rb 76 | worker1: bundle exec rails jobs:work 77 | worker2: bundle exec rails jobs:work 78 | worker3: bundle exec rails jobs:work 79 | worker4: bundle exec rails jobs:work 80 | ``` 81 | 82 | You can add however many you want, but we can't scale beyond the amount that you define here. 83 | 84 | Next, in the Dynomatic configuration, add the names of those worker dyno types: 85 | 86 | ```ruby 87 | Dynomatic.configure do |config| 88 | # ... 89 | config.worker_names = %w(worker1 worker2 worker3 worker4) 90 | 91 | # Since we only have 4 worker dyno types, we can only scale up to 4: 92 | config.rules = [ 93 | {at_least: 0, dynos: 1}, 94 | {at_least: 50, dynos: 2}, 95 | {at_least: 100, dynos: 4}, 96 | ] 97 | end 98 | ``` 99 | 100 | Now instead of scaling up/down a singular `worker` dyno depending on the rules, Dynomatic will instead start/stop the next dyno type in the `worker_names` type. 101 | 102 | Considering the above example, if there's 55 jobs in the queue, Dynomatic will start both `worker1` and `worker2` and stop `worker3` and `worker4`, while if there's only 20, it will only start `worker1` and stop the remainder. 103 | 104 | ## FAQ 105 | 106 | _Does this take into account jobs in the future, or failed, retrying jobs?_ 107 | 108 | No, it only counts jobs that need to be run right now. 109 | 110 | _How do you ensure multi-thread/multi-process workers don't all try to scale the workers_ 111 | I don't, each worker thread of each worker process will do their own call to the API. It's no problem if Heroku receives multiple calls to set the dyno-count to e.g. 20, it will ignore the duplicates. Since there's 4500 requests per hour available for the API, this should also never exhaust the API calls. 112 | 113 | ## Development 114 | 115 | After checking out the repo, run `bin/setup` to install dependencies. Then, run `rake spec` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 116 | 117 | To install this gem onto your local machine, run `bundle exec rake install`. To release a new version, update the version number in `version.rb`, and then run `bundle exec rake release`, which will create a git tag for the version, push git commits and tags, and push the `.gem` file to [rubygems.org](https://rubygems.org). 118 | 119 | ## Contributing 120 | 121 | Bug reports and pull requests are welcome on GitHub at https://github.com/dv/dynomatic. 122 | 123 | ## License 124 | 125 | The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). 126 | 127 | 128 | ## TODO 129 | - Hysteresis (will be hard without using a distributed state storage device) 130 | - Other ActiveJob adapters beyond DelayedJob 131 | --------------------------------------------------------------------------------