├── .document ├── .gitignore ├── .rspec ├── .rubocop.yml ├── .travis.yml ├── .yardopts ├── CHANGES.md ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── lib ├── sqeduler.rb └── sqeduler │ ├── config.rb │ ├── lock_maintainer.rb │ ├── lock_value.rb │ ├── middleware │ └── kill_switch.rb │ ├── redis_lock.rb │ ├── redis_scripts.rb │ ├── service.rb │ ├── trigger_lock.rb │ ├── version.rb │ └── worker │ ├── callbacks.rb │ ├── everything.rb │ ├── kill_switch.rb │ └── synchronization.rb ├── spec ├── config_spec.rb ├── fixtures │ ├── empty_schedule.yaml │ ├── env.rb │ ├── fake_worker.rb │ └── schedule.yaml ├── integration_spec.rb ├── lock_maintainer_spec.rb ├── middleware │ └── kill_switch_spec.rb ├── service_spec.rb ├── spec_helper.rb ├── sqeduler_spec.rb ├── trigger_lock_spec.rb ├── worker │ ├── kill_switch_spec.rb │ └── synchronization_spec.rb └── worker_spec.rb └── sqeduler.gemspec /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | README.md 3 | CHANGES.md 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.bundle 2 | /.yardoc/ 3 | /Gemfile.lock 4 | /doc/ 5 | /pkg/ 6 | *.gem 7 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | --format documentation 3 | --warnings 4 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | Gemspec/RequiredRubyVersion: 2 | Enabled: false 3 | 4 | Layout/HashAlignment: 5 | Enabled: false 6 | 7 | Layout/LineLength: 8 | Enabled: false 9 | 10 | Lint/EndAlignment: 11 | EnforcedStyleAlignWith: variable 12 | 13 | # Setting off false-positives :( 14 | Lint/RedundantCopDisableDirective: 15 | Enabled: false 16 | 17 | Metrics/AbcSize: 18 | Enabled: false 19 | 20 | Metrics/ClassLength: 21 | Max: 150 22 | 23 | Metrics/MethodLength: 24 | Max: 25 25 | 26 | Metrics/BlockLength: 27 | Enabled: false 28 | 29 | Naming: 30 | Enabled: false 31 | 32 | Style/GlobalStdStream: 33 | Enabled: false 34 | 35 | Style/HashSyntax: 36 | EnforcedStyle: hash_rockets 37 | 38 | Style/IfUnlessModifier: 39 | Enabled: false 40 | 41 | Style/NumericPredicate: 42 | Enabled: false 43 | 44 | Style/SoleNestedConditional: 45 | Enabled: false 46 | 47 | Style/StringLiterals: 48 | EnforcedStyle: double_quotes 49 | 50 | Style/Documentation: 51 | Exclude: 52 | - 'lib/sqeduler/version.rb' 53 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | rvm: 3 | - 2.6.6 4 | - 2.7.2 5 | services: 6 | - redis-server 7 | before_install: 8 | - gem update --system 9 | - gem --version 10 | - gem update bundler 11 | - bundle --version 12 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --markup markdown --title "sqeduler Documentation" --protected 2 | -------------------------------------------------------------------------------- /CHANGES.md: -------------------------------------------------------------------------------- 1 | ### 0.3.10 / 2023-06-20 2 | * Support sidekiq-scheduler 5 3 | 4 | ### 0.3.9 / 2022-05-19 5 | * Support sidekiq-scheduler 4 6 | 7 | ### 0.3.8 / 2018-01-10 8 | 9 | * "NoMethodError: undefined method `constantize' for ..." error fixed in tests 10 | * Yard gem version updated to "~> 0.9.11" 11 | 12 | ### 0.3.7 / 2016-09-21 13 | 14 | * Fixed a bug introduced by sidekiq-scheduler 2.0.9 that resulted in the schedule being empty 15 | 16 | ### 0.3.6 / 2016-06-16 17 | 18 | * Symbolize keys in redis config hash 19 | * Add method to list disabled workers 20 | 21 | ### 0.3.5 / 2016-05-03 22 | 23 | * Move sidekiq-scheduler from 1.x to 2.x 24 | 25 | ### 0.3.4 / 2016-03-28 26 | 27 | * Add ability to use a client-provided connection pool rather than creating one 28 | 29 | ### 0.3.3 / 2016-03-25 30 | 31 | * Fixed lock refresher not calling `redis_pool` properly so it wouldn't actually run 32 | 33 | ### 0.3.2 / 2016-03-10 34 | 35 | * Fixed lock refresher failing to lock properly for exclusive runs 36 | * Added debug logs for lock refresher 37 | 38 | ### 0.3.1 / 2016-02-17 39 | 40 | * Fixed lock refresh checking timeout rather than expiration for finding eligible jobs 41 | 42 | ### 0.3.0 / 2016-01-25 43 | 44 | * Added lock refresh to maintain exclusive locks until long running jobs finish 45 | 46 | ### 0.2.2 / 2015-11-11 47 | 48 | * Support ERB in job schedules 49 | * Handle exceptions more gracefully in lock acquisition 50 | 51 | ### 0.2.0 / 2015-04-18 52 | 53 | * Add KillSwitch middleware 54 | * Cleanup 55 | 56 | ### 0.1.4 / 2015-03-26 57 | 58 | * Initial release 59 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Contributing 2 | ============ 3 | 4 | If you would like to contribute code to Sqeduler you can do so through GitHub by 5 | forking the repository and sending a pull request. 6 | 7 | When submitting code, please make every effort to follow existing conventions 8 | and style in order to keep the code as readable as possible `bundle exec rubocop`. 9 | 10 | Before your code can be accepted into the project you must also sign the 11 | [Individual Contributor License Agreement (CLA)][1]. 12 | 13 | [1]: https://spreadsheets.google.com/spreadsheet/viewform?formkey=dDViT2xzUHAwRkI3X3k5Z0lQM091OGc6MQ&ndplr=1 14 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://www.rubygems.org" 4 | 5 | gemspec 6 | 7 | gem "bundler" 8 | gem "kramdown" 9 | gem "pry" 10 | gem "rake" 11 | gem "rspec", "~> 3.12" 12 | gem "rubocop", "~> 1.51.0" 13 | gem "rubocop-rspec" 14 | gem "timecop" 15 | gem "yard", "~> 0.9.11" 16 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | 3 | Version 2.0, January 2004 4 | 5 | http://www.apache.org/licenses/ 6 | 7 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 8 | 9 | 1. Definitions. 10 | 11 | "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 16 | 17 | "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. 18 | 19 | "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. 20 | 21 | "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. 22 | 23 | "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). 24 | 25 | "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. 26 | 27 | "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." 28 | 29 | "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 30 | 31 | 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 32 | 33 | 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 34 | 35 | 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: 36 | 37 | You must give any other recipients of the Work or Derivative Works a copy of this License; and 38 | You must cause any modified files to carry prominent notices stating that You changed the files; and 39 | You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and 40 | If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. 41 | 42 | You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 43 | 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 44 | 45 | 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 46 | 47 | 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 48 | 49 | 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 50 | 51 | 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. 52 | 53 | END OF TERMS AND CONDITIONS 54 | 55 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sqeduler 2 | 3 | [![Build Status](https://travis-ci.org/square/sqeduler.svg?branch=master)](https://travis-ci.org/square/sqeduler) 4 | 5 | ## Description 6 | 7 | Provides loosely-coupled helpers for Sidekiq workers. Provides highly available scheduling across multiple hosts. 8 | 9 | ## Features 10 | 11 | * Centralizes configuration for Sidekiq and Sidekiq::Scheduler 12 | * Provides composable modules for Sidekiq jobs. 13 | * Simple callbacks for `before_start`, `on_success`, `on_failure` 14 | * Synchronization across multiple hosts: 15 | * Provides global level scheduler locks through a thread-safe redis lock 16 | * `synchronize_jobs_mode` for if a job should run exclusively. Currently only supports `:one_at_a_time`. 17 | * Callbacks for `on_schedule_collision` and `on_lock_timeout` 18 | * Crosshost worker killswitches. `enabled` and `disable` methods to enable and disable workers. Enabled by default. 19 | 20 | ## Examples 21 | 22 | To install this gem, add it to your Gemfile: 23 | 24 | ```ruby 25 | gem 'sqeduler' 26 | ``` 27 | 28 | ### Scheduling 29 | 30 | To use this gem for initializing `Sidekiq` and `Sidekiq::Scheduler`: 31 | 32 | In an initializer: 33 | 34 | ```ruby 35 | require 'sqeduler' 36 | config = Sqeduler::Config.new( 37 | # configuration for connecting to redis client. Must be a hash, not a `ConnectionPool`. 38 | :redis_hash => SIDEKIQ_REDIS, 39 | :logger => logger, # defaults to Rails.logger if nil 40 | ) 41 | 42 | # OPTIONAL PARAMETERS 43 | # Additional configuration for Sidekiq. 44 | # Optional server config for sidekiq. Allows you to hook into `Sidekiq.configure_server` 45 | config.on_server_start = proc {|config| ... } 46 | # optional client config for sidekiq. Allows you to hook into `Sidekiq.configure_client` 47 | config.on_client_start = proc {|config| ... } 48 | # required if you want to start the Sidekiq::Scheduler 49 | config.schedule_path = Rails.root.join('config').join('sidekiq_schedule.yml') 50 | # optional to maintain locks for exclusive jobs, see "Lock Maintainer" below 51 | config.maintain_locks = true 52 | 53 | Sqeduler::Service.config = config 54 | # Starts Sidekiq and Sidekiq::Scheduler 55 | Sqeduler::Service.start 56 | ``` 57 | 58 | You can also pass in your own `ConnectionPool` instance as `config.redis_pool` rather than providing configuration in `redis_hash`. If you do so, it's recommended to use a `Redis::Namespace` so that the keys sqeduler sets are namespaced uniquely. 59 | 60 | See documentation for [Sidekiq::Scheduler](https://github.com/Moove-it/sidekiq-scheduler#scheduled-jobs-recurring-jobs) 61 | for specifics on how to construct your schedule YAML file. 62 | 63 | ### Lock Maintainer 64 | 65 | Exclusive locks only last for the expiration you set. If your expiration is 30 seconds and the job runs for 60 seconds, you can have multiple jobs running at once. Rather than having to set absurdly high lock expirations, you can enable the `maintain_locks` option which handles this for you. 66 | 67 | Every 30 seconds, Sqeduler will look for any exclusive Sidekiq jobs that have been running for more than 30 seconds, and have a lock expiration of more than 30 seconds and refresh the lock. 68 | 69 | ### Worker Helpers 70 | 71 | To use `Sqeduler::Worker` modules: 72 | * You **DO NOT need** to use this gem for starting Sidekiq or Sidekiq::Scheduler (i.e: `Sqeduler::Service.start`) 73 | * You **DO need** to provide at `config.redis_hash`, and `config.logger` if you don't want to log to `Rails.logger`. 74 | * This gem creates a separate `ConnectionPool` so that it can create locks for synchronization and store state for disabling/enabling workers. 75 | * You **DO need** to `include`/`prepend` these modules in the actual working class 76 | * They will not work if done in a parent class because of the way `prepend` works in conjunction with inheritance. 77 | 78 | The modules: 79 | 80 | * `Sqeduler::Worker::Callbacks`: simple callbacks for `before_start`, `on_success`, `on_failure` 81 | * `Sqeduler::Worker::Synchronization`: synchronize workers across multiple hosts: 82 | * `synchronize_jobs_mode` for if a job should run exclusively. Currently only supports `:one_at_a_time`. 83 | * Callbacks for `on_schedule_collision` and `on_lock_timeout` 84 | * `Sqeduler::Worker::KillSwitch`: cross-host worker disabling/enabling. 85 | * `enabled` and `disable` class methods to enable and disable workers. 86 | * Workers are enabled by default. 87 | 88 | You can either include everything`include Sqeduler::Worker::Everything`) or prepend à la carte, but make sure to 89 | use [prepend](http://ruby-doc.org/core-2.0.0/Module.html#method-i-prepend), not `include`. 90 | 91 | Sample code and callback docs below. 92 | 93 | ```ruby 94 | class MyWorker 95 | include Sidekiq::Worker 96 | 97 | # include everything 98 | include Sqeduler::Worker::Everything 99 | # or cherry pick the modules that you want 100 | 101 | # optionally synchronize jobs across hosts 102 | prepend Sqeduler::Worker::Synchronization 103 | # then define how the job should be synchronized 104 | # :timeout in seconds, how long should we poll for a lock, default is 5 105 | # :expiration in seconds, how long should the lock be held for 106 | synchronize :one_at_a_time, :expiration => 1.hour, :timeout => 1.second 107 | 108 | # cross-host methods for enabling and disabling workers 109 | # MyWorker.disable and MyWorker.enable 110 | prepend Sqeduler::Worker::KillSwitch 111 | 112 | 113 | # Simple callbacks for `before_start`, `on_success`, `on_failure` 114 | # must be the last worker to be prepended 115 | prepend Sqeduler::Worker::Callbacks 116 | 117 | def perform(*args) 118 | # Your typical sidekiq worker code 119 | end 120 | 121 | private 122 | 123 | # callbacks for Sqeduler::Worker::Callbacks 124 | 125 | def before_start 126 | # before perform is called 127 | end 128 | 129 | def on_success(total_time) 130 | # It worked! Save this status or enqueue other jobs. 131 | end 132 | 133 | def on_failure(e) 134 | # Bugsnag can already be notified with config.exception_notifier, 135 | # but maybe you need to log this differently. 136 | end 137 | 138 | # callbacks for Sqeduler::Worker::Synchronization 139 | 140 | # NOTE: Even if `on_schedule_collision` or `on_lock_timeout` occur your job will still 141 | # receive on_success if you prepend Sqeduler::Worker::Callbacks. These events do not 142 | # equate to failures. 143 | 144 | def on_schedule_collision(duration) 145 | # Called when your worker uses synchronization and :expiration is too low, i.e. it took longer 146 | # to carry out `perform` then your lock's expiration period. In this situation, it's possible for 147 | # the job to get scheduled again even though you expected the job to run exclusively. 148 | end 149 | 150 | def on_lock_timeout(key) 151 | # Called when your worker cannot obtain the lock. 152 | end 153 | end 154 | ``` 155 | 156 | ## License 157 | 158 | Copyright 2015 Square Inc. 159 | 160 | Licensed under the Apache License, Version 2.0 (the "License"); 161 | you may not use this file except in compliance with the License. 162 | You may obtain a copy of the License at 163 | 164 | http://www.apache.org/licenses/LICENSE-2.0 165 | 166 | Unless required by applicable law or agreed to in writing, software 167 | distributed under the License is distributed on an "AS IS" BASIS, 168 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 169 | See the License for the specific language governing permissions and 170 | limitations under the License. 171 | 172 | 173 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | 5 | require "rspec/core/rake_task" 6 | RSpec::Core::RakeTask.new 7 | 8 | require "rubocop/rake_task" 9 | RuboCop::RakeTask.new 10 | 11 | require "yard" 12 | task :doc => :yard 13 | 14 | task :default => %i[spec rubocop] 15 | -------------------------------------------------------------------------------- /lib/sqeduler.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "active_support" 4 | require "sqeduler/version" 5 | require "sqeduler/config" 6 | require "sqeduler/redis_scripts" 7 | require "sqeduler/lock_value" 8 | require "sqeduler/redis_lock" 9 | require "sqeduler/trigger_lock" 10 | require "sqeduler/lock_maintainer" 11 | require "sqeduler/middleware/kill_switch" 12 | require "sqeduler/service" 13 | require "sqeduler/worker/callbacks" 14 | require "sqeduler/worker/synchronization" 15 | require "sqeduler/worker/kill_switch" 16 | require "sqeduler/worker/everything" 17 | -------------------------------------------------------------------------------- /lib/sqeduler/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | # Simple config for Sqeduler::Service 5 | class Config 6 | attr_accessor :logger, :redis_hash, :redis_pool, :schedule_path, 7 | :on_server_start, :on_client_start, :maintain_locks 8 | 9 | def initialize(opts = {}) 10 | self.redis_hash = opts[:redis_hash] 11 | self.redis_pool = opts[:redis_pool] 12 | self.schedule_path = opts[:schedule_path] 13 | self.on_server_start = opts[:on_server_start] 14 | self.on_client_start = opts[:on_client_start] 15 | self.logger = opts[:logger] 16 | self.maintain_locks = opts[:maintain_locks] 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/sqeduler/lock_maintainer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | # This is to ensure that if you set your jobs to run one a time and something goes wrong 5 | # causing a job to run for a long time, your lock won't expire. 6 | # This doesn't stop long running jobs, it just ensures you only end up with one long running job 7 | # rather than 20 of them. 8 | class LockMaintainer 9 | RUN_INTERVAL = 30 10 | RUN_JITTER = (1..5).freeze 11 | 12 | def initialize 13 | @class_with_locks = {} 14 | end 15 | 16 | # This is only done when we initialize Sqeduler, don't need to worry about threading 17 | def run 18 | @maintainer_thread ||= Thread.new do 19 | loop do 20 | begin 21 | if redis_lock.lock 22 | begin 23 | synchronize 24 | ensure 25 | redis_lock.unlock 26 | end 27 | end 28 | rescue StandardError => ex 29 | Service.logger.error "[#{self.class}] #{ex.class}, #{ex.message}" 30 | end 31 | 32 | sleep RUN_INTERVAL + rand(RUN_JITTER) 33 | end 34 | end 35 | end 36 | 37 | private 38 | 39 | def synchronize 40 | now = Time.now.to_i 41 | 42 | Service.redis_pool.with do |redis| 43 | redis.pipelined do 44 | workers.each do |_worker, _tid, args| 45 | # No sense in pinging if it's not been running long enough to matter 46 | next if (now - args["run_at"]) < RUN_INTERVAL 47 | 48 | klass = str_to_class(args["payload"]["class"]) 49 | next unless klass 50 | 51 | lock_key = klass.sync_lock_key(*args["payload"]["args"]) 52 | 53 | # This works because EXPIRE does not recreate the key, it only resets the expiration. 54 | # We don't have to worry about atomic operations or anything like that. 55 | # If the job finishes in the interim and deletes the key nothing will happen. 56 | redis.expire(lock_key, klass.synchronize_jobs_expiration) 57 | 58 | Service.logger.debug "[#{self.class}] Refreshing lock on '#{lock_key}" \ 59 | "to #{klass.synchronize_jobs_expiration} seconds" 60 | end 61 | end 62 | end 63 | end 64 | 65 | # Not all classes will use exclusive locks 66 | def str_to_class(class_name) 67 | return @class_with_locks[class_name] unless @class_with_locks[class_name].nil? 68 | 69 | klass = class_name.constantize 70 | if klass.respond_to?(:synchronize_jobs_mode) 71 | # We only care about exclusive jobs that are long running 72 | if klass.synchronize_jobs_mode == :one_at_a_time && klass.synchronize_jobs_expiration >= RUN_INTERVAL 73 | Service.logger.debug "[#{self.class}] Adding #{class_name} to the whitelist of classes that have locks" 74 | return @class_with_locks[class_name] = klass 75 | end 76 | end 77 | 78 | Service.logger.debug "[#{self.class}] Adding #{class_name} to the blacklist of classes that have locks" 79 | @class_with_locks[class_name] = false 80 | end 81 | 82 | def redis_lock 83 | @redis_lock ||= RedisLock.new("sqeduler-lock-maintainer", :expiration => 60, :timeout => 0) 84 | end 85 | 86 | def workers 87 | @workers ||= Sidekiq::Workers.new 88 | end 89 | end 90 | end 91 | -------------------------------------------------------------------------------- /lib/sqeduler/lock_value.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | # A unique lock value for reserving a lock across multiple hosts 5 | class LockValue 6 | def value 7 | @value ||= [hostname, process_id, thread_id].join(":") 8 | end 9 | 10 | private 11 | 12 | def hostname 13 | Socket.gethostname 14 | end 15 | 16 | def process_id 17 | Process.pid 18 | end 19 | 20 | def thread_id 21 | Thread.current.object_id 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/sqeduler/middleware/kill_switch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "active_support/inflector" 4 | 5 | module Sqeduler 6 | module Middleware 7 | # Verifies that a worker class is enabled before pushing the job into Redis. 8 | # Prevents disabled jobs from getting enqueued. To disable a worker, use 9 | # Sqeduler::Worker::KillSwitch. 10 | class KillSwitch 11 | def call(worker, _msg, _queue, _redis_pool) 12 | worker_klass = normalized_worker_klass(worker) 13 | if worker_enabled?(worker_klass) 14 | yield 15 | else 16 | Service.logger.warn "#{worker_klass.name} is currently disabled. Will not be enqueued." 17 | false 18 | end 19 | end 20 | 21 | def normalized_worker_klass(worker) 22 | # worker_class can be String or a Class 23 | # SEE: https://github.com/mperham/sidekiq/wiki/Middleware 24 | if worker.is_a?(String) 25 | worker.constantize 26 | else 27 | worker 28 | end 29 | end 30 | 31 | def worker_enabled?(worker_klass) 32 | !worker_klass.respond_to?(:enabled?) || worker_klass.enabled? 33 | end 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/sqeduler/redis_lock.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | # Uses eval_sha to execute server-side scripts on redis. 5 | # Avoids some of the potentially racey and brittle dependencies on Time-based 6 | # redis locks in other locking libraries. 7 | class RedisLock 8 | include RedisScripts 9 | 10 | class LockTimeoutError < StandardError; end 11 | SLEEP_TIME = 0.1 12 | attr_reader :key, :timeout 13 | 14 | def initialize(key, options = {}) 15 | @key = key 16 | @expiration = options[:expiration] 17 | raise ArgumentError, "Expiration must be provided!" unless @expiration 18 | 19 | @timeout = options[:timeout] || 5 20 | end 21 | 22 | def lock 23 | Service.logger.info( 24 | "Try to acquire lock with #{key}, expiration: #{@expiration} sec, timeout: #{timeout} sec" 25 | ) 26 | return true if locked? 27 | 28 | if poll_for_lock 29 | Service.logger.info "Acquired lock #{key} with value #{lock_value}" 30 | true 31 | else 32 | Service.logger.info "Failed to acquire lock #{key} with value #{lock_value}" 33 | false 34 | end 35 | end 36 | 37 | def unlock 38 | if release_lock 39 | Service.logger.info "Released lock #{key} with value #{lock_value}" 40 | true 41 | else 42 | Service.logger.info "Cannot release lock #{key} with value #{lock_value}" 43 | false 44 | end 45 | end 46 | 47 | def refresh 48 | if refresh_lock 49 | Service.logger.info "Refreshed lock #{key} with value #{lock_value}" 50 | true 51 | else 52 | Service.logger.info "Cannot refresh lock #{key} with value #{lock_value}" 53 | false 54 | end 55 | end 56 | 57 | def locked? 58 | redis_pool.with do |redis| 59 | if redis.get(key) == lock_value 60 | Service.logger.info "Lock #{key} with value #{lock_value} is valid" 61 | true 62 | else 63 | Service.logger.info "Lock #{key} with value #{lock_value} has expired or is not present" 64 | false 65 | end 66 | end 67 | end 68 | 69 | def self.with_lock(key, options) 70 | raise "Block is required" unless block_given? 71 | 72 | mutex = new(key, options) 73 | unless mutex.lock 74 | raise LockTimeoutError, "Timed out trying to get #{key} lock. Exceeded #{mutex.timeout} sec" 75 | end 76 | 77 | begin 78 | yield 79 | ensure 80 | mutex.unlock 81 | end 82 | end 83 | 84 | def expiration_milliseconds 85 | # expiration needs to be an integer 86 | @expiration ? (@expiration * 1000).to_i : 0 87 | end 88 | 89 | private 90 | 91 | def lock_value 92 | @lock_value ||= LockValue.new.value 93 | end 94 | 95 | def poll_for_lock 96 | start = Time.now 97 | ran_at_least_once = false 98 | while Time.now - start < timeout || !ran_at_least_once 99 | locked = take_lock 100 | break if locked 101 | 102 | ran_at_least_once = true 103 | sleep SLEEP_TIME 104 | end 105 | locked 106 | end 107 | 108 | def take_lock 109 | redis_pool.with do |redis| 110 | redis.set(key, lock_value, :nx => true, :px => expiration_milliseconds) 111 | end 112 | end 113 | 114 | def redis_pool 115 | Service.redis_pool 116 | end 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/sqeduler/redis_scripts.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | # Extracts atomic Lua scripts for Redis. 5 | module RedisScripts 6 | def release_lock 7 | sha_and_evaluate(:release, key, lock_value) 8 | end 9 | 10 | def refresh_lock 11 | sha_and_evaluate(:refresh, key, lock_value) 12 | end 13 | 14 | private 15 | 16 | def sha_and_evaluate(script_name, key, value) 17 | redis_pool.with do |redis| 18 | sha = load_sha(redis, script_name) 19 | # all scripts return 0 or 1 20 | redis.evalsha(sha, :keys => [key], :argv => [value]) == 1 21 | end 22 | end 23 | 24 | def load_sha(redis, script_name) 25 | @redis_sha_cache ||= {} 26 | @redis_sha_cache[script_name] ||= begin 27 | script = case script_name 28 | when :refresh 29 | refresh_lock_script 30 | when :release 31 | release_lock_script 32 | else 33 | raise "No script for #{script_name}" 34 | end 35 | # strip leading whitespace of 8 characters 36 | redis.redis.script(:load, script.gsub(/^ {8}/, "")) 37 | end 38 | end 39 | 40 | def refresh_lock_script 41 | <<-EOF 42 | if redis.call('GET', KEYS[1]) == false then 43 | return redis.call('SET', KEYS[1], ARGV[1], 'NX', 'PX', #{expiration_milliseconds}) and 1 or 0 44 | elseif redis.call('GET', KEYS[1]) == ARGV[1] then 45 | redis.call('PEXPIRE', KEYS[1], #{expiration_milliseconds}) 46 | if redis.call('GET', KEYS[1]) == ARGV[1] then 47 | return 1 48 | end 49 | end 50 | return 0 51 | EOF 52 | end 53 | 54 | def release_lock_script 55 | <<-EOF 56 | if redis.call('GET', KEYS[1]) == ARGV[1] then 57 | redis.call('DEL', KEYS[1]) 58 | return 1 59 | else 60 | return 0 61 | end 62 | EOF 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /lib/sqeduler/service.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sidekiq" 4 | require "sidekiq-scheduler" 5 | module Sqeduler 6 | # Singleton class for configuring this Gem. 7 | class Service 8 | SCHEDULER_TIMEOUT = 60 9 | MINIMUM_REDIS_VERSION = "2.6.12" 10 | 11 | class << self 12 | attr_accessor :config 13 | 14 | def start 15 | raise "No config provided" unless config 16 | 17 | config_sidekiq_server 18 | config_sidekiq_client 19 | end 20 | 21 | def verify_redis_pool(redis_pool) 22 | return @verified if defined?(@verified) 23 | 24 | redis_pool.with do |redis| 25 | version = redis.info["redis_version"] 26 | unless Gem::Version.new(version) >= Gem::Version.new(MINIMUM_REDIS_VERSION) 27 | raise "Must be using redis >= #{MINIMUM_REDIS_VERSION}" 28 | end 29 | 30 | @verified = true 31 | end 32 | end 33 | 34 | def config_sidekiq_server 35 | logger.info "Initializing Sidekiq server" 36 | ::Sidekiq.configure_server do |config| 37 | setup_sidekiq_redis(config) 38 | if Service.scheduling? 39 | logger.info "Initializing Sidekiq::Scheduler with schedule #{::Sqeduler::Service.config.schedule_path}" 40 | 41 | config.on(:startup) do 42 | ::Sidekiq::Scheduler.rufus_scheduler_options = { 43 | :trigger_lock => TriggerLock.new 44 | } 45 | ::Sidekiq.schedule = ::Sqeduler::Service.parse_schedule(::Sqeduler::Service.config.schedule_path) 46 | ::Sidekiq::Scheduler.reload_schedule! 47 | end 48 | 49 | config.on(:shutdown) do 50 | # Make sure any scheduling locks are released on shutdown. 51 | ::Sidekiq::Scheduler.rufus_scheduler.stop 52 | end 53 | else 54 | logger.warn "No schedule_path provided. Not starting Sidekiq::Scheduler." 55 | end 56 | 57 | # the server can also enqueue jobs 58 | config.client_middleware do |chain| 59 | chain.add(Sqeduler::Middleware::KillSwitch) 60 | end 61 | 62 | LockMaintainer.new.run if Service.config.maintain_locks 63 | Service.config.on_server_start&.call(config) 64 | end 65 | end 66 | 67 | def config_sidekiq_client 68 | logger.info "Initializing Sidekiq client" 69 | ::Sidekiq.configure_client do |config| 70 | setup_sidekiq_redis(config) 71 | Service.config.on_client_start&.call(config) 72 | 73 | config.client_middleware do |chain| 74 | chain.add(Sqeduler::Middleware::KillSwitch) 75 | end 76 | end 77 | end 78 | 79 | def setup_sidekiq_redis(config) 80 | return if Service.config.redis_hash.nil? || Service.config.redis_hash.empty? 81 | 82 | config.redis = Service.config.redis_hash 83 | end 84 | 85 | def parse_schedule(path) 86 | raise "Schedule file #{path} does not exist!" unless File.exist?(path) 87 | 88 | file_contents = File.read(path) 89 | YAML.safe_load(ERB.new(file_contents).result) 90 | end 91 | 92 | def scheduling? 93 | !config.schedule_path.to_s.empty? 94 | end 95 | 96 | # A singleton redis ConnectionPool for Sidekiq::Scheduler, 97 | # Sqeduler::Worker::Synchronization, Sqeduler::Worker::KillSwitch. Should be 98 | # separate from Sidekiq's so that we don't saturate the client and server connection 99 | # pools. 100 | def redis_pool 101 | @redis_pool ||= config_redis_pool 102 | end 103 | 104 | def config_redis_pool 105 | redis_pool = if config.redis_pool 106 | config.redis_pool 107 | else 108 | # Redis requires config hash to have symbols as keys. 109 | redis = { :namespace => "sqeduler" }.merge(symbolize_keys(config.redis_hash)) 110 | ::Sidekiq::RedisConnection.create(redis) 111 | end 112 | verify_redis_pool(redis_pool) 113 | redis_pool 114 | end 115 | 116 | def logger 117 | return config.logger if config.logger 118 | return Rails.logger if defined?(Rails) 119 | 120 | raise ArgumentError, "No logger provided and Rails.logger cannot be inferred" 121 | end 122 | 123 | private 124 | 125 | def symbolize_keys(hash) 126 | hash.transform_keys(&:to_sym) 127 | end 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /lib/sqeduler/trigger_lock.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | # Super simple facade to match RufusScheduler's expectations of how 5 | # a trigger_lock behaves. 6 | class TriggerLock < RedisLock 7 | SCHEDULER_LOCK_KEY = "sidekiq_scheduler_lock" 8 | 9 | def initialize 10 | super(SCHEDULER_LOCK_KEY, :expiration => 60, :timeout => 0) 11 | end 12 | 13 | def lock 14 | # Locking should: 15 | # - not block 16 | # - return true if already acquired 17 | # - refresh the lock if already acquired 18 | refresh || super 19 | rescue StandardError 20 | false 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/sqeduler/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | VERSION = "0.3.10" 5 | end 6 | -------------------------------------------------------------------------------- /lib/sqeduler/worker/callbacks.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "benchmark" 4 | module Sqeduler 5 | module Worker 6 | # Basic callbacks for worker events. 7 | module Callbacks 8 | def perform(*args) 9 | before_start 10 | duration = Benchmark.realtime { super } 11 | on_success(duration) 12 | rescue StandardError => e 13 | on_failure(e) 14 | raise 15 | end 16 | 17 | private 18 | 19 | # provides an oppurtunity to log when the job has started (maybe create a 20 | # stateful db record for this job run?) 21 | def before_start 22 | Service.logger.info "Starting #{self.class.name} at #{Time.now} in process ID #{Process.pid}" 23 | super if defined?(super) 24 | end 25 | 26 | # callback for successful run of this job 27 | def on_success(total_time) 28 | Service.logger.info "#{self.class.name} completed at #{Time.now}. Total time #{total_time}" 29 | super if defined?(super) 30 | end 31 | 32 | # callback for when failues in this job occur 33 | def on_failure(e) 34 | Service.logger.error "#{self.class.name} failed with exception #{e}" 35 | super if defined?(super) 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/sqeduler/worker/everything.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | module Worker 5 | # convenience module for including everything 6 | module Everything 7 | def self.included(mod) 8 | mod.prepend Sqeduler::Worker::Synchronization 9 | mod.prepend Sqeduler::Worker::KillSwitch 10 | # needs to be the last one 11 | mod.prepend Sqeduler::Worker::Callbacks 12 | end 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/sqeduler/worker/kill_switch.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Sqeduler 4 | module Worker 5 | # Uses Redis hashes to enabled and disable workers across multiple hosts. 6 | module KillSwitch 7 | SIDEKIQ_DISABLED_WORKERS = "sidekiq.disabled-workers" 8 | 9 | def self.prepended(base) 10 | if base.ancestors.include?(Sqeduler::Worker::Callbacks) 11 | raise "Sqeduler::Worker::Callbacks must be the last module that you prepend." 12 | end 13 | 14 | base.extend(ClassMethods) 15 | end 16 | 17 | def self.disabled 18 | Service.redis_pool.with do |redis| 19 | redis.hgetall(SIDEKIQ_DISABLED_WORKERS) 20 | end 21 | end 22 | 23 | # rubocop:disable Style/Documentation 24 | module ClassMethods 25 | def enable 26 | Service.redis_pool.with do |redis| 27 | redis.hdel(SIDEKIQ_DISABLED_WORKERS, name) 28 | Service.logger.warn "#{name} has been enabled" 29 | end 30 | end 31 | 32 | def disable 33 | Service.redis_pool.with do |redis| 34 | redis.hset(SIDEKIQ_DISABLED_WORKERS, name, Time.now.to_s) 35 | Service.logger.warn "#{name} has been disabled" 36 | end 37 | end 38 | 39 | def disabled? 40 | Service.redis_pool.with do |redis| 41 | redis.hexists(SIDEKIQ_DISABLED_WORKERS, name) 42 | end 43 | end 44 | 45 | def enabled? 46 | !disabled? 47 | end 48 | end 49 | # rubocop:enable Style/Documentation 50 | 51 | def perform(*args) 52 | if self.class.disabled? 53 | Service.logger.warn "#{self.class.name} is currently disabled." 54 | else 55 | super 56 | end 57 | end 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /lib/sqeduler/worker/synchronization.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "benchmark" 4 | require "active_support/core_ext/class/attribute" 5 | 6 | module Sqeduler 7 | module Worker 8 | # Module that provides common synchronization infrastructure 9 | # of workers across multiple hosts `Sqeduler::BaseWorker.synchronize_jobs`. 10 | module Synchronization 11 | def self.prepended(base) 12 | if base.ancestors.include?(Sqeduler::Worker::Callbacks) 13 | raise "Sqeduler::Worker::Callbacks must be the last module that you prepend." 14 | end 15 | 16 | base.extend(ClassMethods) 17 | base.class_attribute :synchronize_jobs_mode 18 | base.class_attribute :synchronize_jobs_expiration 19 | base.class_attribute :synchronize_jobs_timeout 20 | end 21 | 22 | # rubocop:disable Style/Documentation 23 | module ClassMethods 24 | def synchronize(mode, opts = {}) 25 | self.synchronize_jobs_mode = mode 26 | self.synchronize_jobs_timeout = opts[:timeout] || 5 27 | self.synchronize_jobs_expiration = opts[:expiration] 28 | return if synchronize_jobs_expiration 29 | 30 | raise ArgumentError, ":expiration is required!" 31 | end 32 | 33 | def sync_lock_key(*args) 34 | if args.empty? 35 | name 36 | else 37 | "#{name}-#{args.join}" 38 | end 39 | end 40 | end 41 | # rubocop:enable Style/Documentation 42 | 43 | def perform(*args) 44 | if self.class.synchronize_jobs_mode == :one_at_a_time 45 | perform_locked(self.class.sync_lock_key(*args)) do 46 | perform_timed do 47 | super 48 | end 49 | end 50 | else 51 | super 52 | end 53 | end 54 | 55 | private 56 | 57 | # callback for when a lock cannot be obtained 58 | def on_lock_timeout(key) 59 | Service.logger.warn( 60 | "#{self.class.name} unable to acquire lock '#{key}'. Aborting." 61 | ) 62 | super if defined?(super) 63 | end 64 | 65 | # callback for when the job expiration is too short, less < time it took 66 | # perform the actual work 67 | SCHEDULE_COLLISION_MARKER = "%s took %s but has an expiration of %p sec. Beware of race conditions!" 68 | def on_schedule_collision(duration) 69 | Service.logger.warn( 70 | format( 71 | SCHEDULE_COLLISION_MARKER, 72 | self.class.name, 73 | time_duration(duration), 74 | self.class.synchronize_jobs_expiration 75 | ) 76 | ) 77 | super if defined?(super) 78 | end 79 | 80 | def perform_timed(&block) 81 | duration = Benchmark.realtime(&block) 82 | on_schedule_collision(duration) if duration > self.class.synchronize_jobs_expiration 83 | end 84 | 85 | def perform_locked(sync_lock_key, &work) 86 | RedisLock.with_lock( 87 | sync_lock_key, 88 | :expiration => self.class.synchronize_jobs_expiration, 89 | :timeout => self.class.synchronize_jobs_timeout, 90 | &work 91 | ) 92 | rescue RedisLock::LockTimeoutError 93 | on_lock_timeout(sync_lock_key) 94 | end 95 | 96 | # rubocop:disable Metrics/AbcSize 97 | def time_duration(timespan) 98 | rest, secs = timespan.divmod(60) # self is the time difference t2 - t1 99 | rest, mins = rest.divmod(60) 100 | days, hours = rest.divmod(24) 101 | 102 | result = [] 103 | result << "#{days} Days" if days > 0 104 | result << "#{hours} Hours" if hours > 0 105 | result << "#{mins} Minutes" if mins > 0 106 | result << "#{secs.round(2)} Seconds" if secs > 0 107 | result.join(" ") 108 | end 109 | # rubocop:enable Metrics/AbcSize 110 | end 111 | end 112 | end 113 | -------------------------------------------------------------------------------- /spec/config_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe Sqeduler::Config do 6 | describe "#initialize" do 7 | subject do 8 | described_class.new(options) 9 | end 10 | 11 | let(:options) do 12 | { 13 | :logger => double, 14 | :schedule_path => "/tmp/schedule.yaml", 15 | :redis_hash => { 16 | :host => "localhost", 17 | :db => 1 18 | } 19 | }.merge(extras) 20 | end 21 | 22 | let(:extras) { {} } 23 | 24 | describe "redis_hash" do 25 | it "should set the redis_hash" do 26 | expect(subject.redis_hash).to eq(options[:redis_hash]) 27 | end 28 | end 29 | 30 | describe "schedule_path" do 31 | it "should set the schedule_path" do 32 | expect(subject.schedule_path).to eq(options[:schedule_path]) 33 | end 34 | end 35 | 36 | describe "logger" do 37 | it "should set the logger" do 38 | expect(subject.logger).to eq(options[:logger]) 39 | end 40 | end 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /spec/fixtures/empty_schedule.yaml: -------------------------------------------------------------------------------- 1 | {} 2 | -------------------------------------------------------------------------------- /spec/fixtures/env.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sqeduler" 4 | require_relative "fake_worker" 5 | 6 | Sidekiq.logger = Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG } 7 | 8 | Sqeduler::Service.config = Sqeduler::Config.new( 9 | :redis_hash => { 10 | :host => "localhost", 11 | :db => 1, 12 | :namespace => "sqeduler-tests" 13 | }, 14 | :logger => Sidekiq.logger, 15 | :schedule_path => "#{__dir__}/schedule.yaml", 16 | :on_server_start => proc do |_config| 17 | Sqeduler::Service.logger.info "Received on_server_start callback" 18 | end, 19 | :on_client_start => proc do |_config| 20 | Sqeduler::Service.logger.info "Received on_client_start callback" 21 | end 22 | ) 23 | Sqeduler::Service.start 24 | -------------------------------------------------------------------------------- /spec/fixtures/fake_worker.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # Sample worker for specs 4 | class FakeWorker 5 | JOB_RUN_PATH = "/tmp/job_run" 6 | JOB_BEFORE_START_PATH = "/tmp/job_before_start" 7 | JOB_SUCCESS_PATH = "/tmp/job_success" 8 | JOB_FAILURE_PATH = "/tmp/job_failure" 9 | JOB_LOCK_FAILURE_PATH = "/tmp/lock_failure" 10 | SCHEDULE_COLLISION_PATH = "/tmp/schedule_collision" 11 | include Sidekiq::Worker 12 | include Sqeduler::Worker::Everything 13 | 14 | def perform(sleep_time = 0.1) 15 | long_process(sleep_time) 16 | end 17 | 18 | private 19 | 20 | def long_process(sleep_time) 21 | sleep sleep_time 22 | log_event(JOB_RUN_PATH) 23 | end 24 | 25 | def log_event(file_path) 26 | File.open(file_path, "a+") { |f| f.write "1" } 27 | end 28 | 29 | def on_success(_total_duration) 30 | log_event(JOB_SUCCESS_PATH) 31 | end 32 | 33 | def on_failure(_e) 34 | log_event(JOB_FAILURE_PATH) 35 | end 36 | 37 | def before_start 38 | log_event(JOB_BEFORE_START_PATH) 39 | end 40 | 41 | def on_lock_timeout(_key) 42 | log_event(JOB_LOCK_FAILURE_PATH) 43 | end 44 | 45 | def on_schedule_collision(_duration) 46 | log_event(SCHEDULE_COLLISION_PATH) 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /spec/fixtures/schedule.yaml: -------------------------------------------------------------------------------- 1 | FakeWorker: 2 | every: <%=5%>s 3 | -------------------------------------------------------------------------------- /spec/integration_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "./spec/fixtures/fake_worker" 5 | 6 | RSpec.describe "Sidekiq integration" do 7 | def maybe_cleanup_file(file_path) 8 | File.delete(file_path) if File.exist?(file_path) 9 | end 10 | 11 | before do 12 | maybe_cleanup_file(FakeWorker::JOB_RUN_PATH) 13 | maybe_cleanup_file(FakeWorker::JOB_SUCCESS_PATH) 14 | maybe_cleanup_file(FakeWorker::JOB_FAILURE_PATH) 15 | maybe_cleanup_file(FakeWorker::JOB_LOCK_FAILURE_PATH) 16 | maybe_cleanup_file(FakeWorker::JOB_BEFORE_START_PATH) 17 | maybe_cleanup_file(FakeWorker::SCHEDULE_COLLISION_PATH) 18 | end 19 | 20 | it "should start sidekiq, schedule FakeWorker, and verify that it ran" do 21 | path = "#{__dir__}/fixtures/env.rb" 22 | pid = Process.spawn "bundle exec sidekiq -r #{path}" 23 | puts "Spawned process #{pid}" 24 | timeout = 30 25 | start = Time.now 26 | while (Time.now - start) < timeout 27 | break if File.exist?(FakeWorker::JOB_RUN_PATH) 28 | 29 | sleep 0.5 30 | end 31 | Process.kill("INT", pid) 32 | Process.wait(pid, 0) 33 | expect(File.exist?(FakeWorker::JOB_RUN_PATH)).to be_truthy 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /spec/lock_maintainer_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe Sqeduler::LockMaintainer do 6 | let(:instance) { described_class.new } 7 | 8 | before do 9 | stub_const( 10 | "SyncExclusiveWorker", 11 | Class.new do 12 | include Sidekiq::Worker 13 | prepend Sqeduler::Worker::Synchronization 14 | synchronize :one_at_a_time, :expiration => 300, :timeout => 5 15 | 16 | def perform(*_args) 17 | yield 18 | end 19 | end 20 | ) 21 | 22 | stub_const( 23 | "SyncShortWorker", 24 | Class.new do 25 | include Sidekiq::Worker 26 | prepend Sqeduler::Worker::Synchronization 27 | synchronize :one_at_a_time, :expiration => 5, :timeout => 5 28 | 29 | def perform 30 | raise "This shouldn't be called" 31 | end 32 | end 33 | ) 34 | 35 | stub_const( 36 | "SyncWhateverWorker", 37 | Class.new do 38 | include Sidekiq::Worker 39 | prepend Sqeduler::Worker::Synchronization 40 | 41 | def perform 42 | raise "This shouldn't be called" 43 | end 44 | end 45 | ) 46 | 47 | Sqeduler::Service.config = Sqeduler::Config.new( 48 | :redis_hash => REDIS_CONFIG, 49 | :logger => Logger.new("/dev/null"), 50 | :schedule_path => Pathname.new("./spec/fixtures/empty_schedule.yaml") 51 | ) 52 | end 53 | 54 | context "#run" do 55 | let(:run) { instance.run } 56 | 57 | it "calls into the synchronizer" do 58 | expect(instance).to receive(:synchronize).at_least(1) 59 | 60 | run.join(1) 61 | run.terminate 62 | 63 | expect(instance.send(:redis_lock).locked?).to eq(false) 64 | end 65 | 66 | it "doesn't die on errors" do 67 | expect(instance).to receive(:synchronize).and_raise(StandardError, "Boom") 68 | 69 | run.join(1) 70 | expect(run.status).to_not be_falsy 71 | run.terminate 72 | end 73 | 74 | it "obeys the exclusive lock" do 75 | lock = Sqeduler::RedisLock.new("sqeduler-lock-maintainer", :expiration => 60, :timeout => 0) 76 | lock.lock 77 | 78 | expect(instance).to_not receive(:synchronize) 79 | 80 | run.join(1) 81 | run.terminate 82 | end 83 | end 84 | 85 | context "#synchronize" do 86 | subject(:sync) { instance.send(:synchronize) } 87 | 88 | let(:run_at) { Time.now.to_i } 89 | let(:job_args) { [1, { "a" => "b" }] } 90 | 91 | let(:workers) do 92 | [ 93 | [ 94 | "process-key", 95 | "worker-tid-1234", 96 | { 97 | "run_at" => run_at, 98 | "payload" => { 99 | "class" => "SyncExclusiveWorker", 100 | "args" => job_args 101 | } 102 | } 103 | ], 104 | [ 105 | "process-key", 106 | "worker-tid-6789", 107 | { 108 | "run_at" => run_at, 109 | "payload" => { 110 | "class" => "SyncShortWorker", 111 | "args" => job_args 112 | } 113 | } 114 | ], 115 | [ 116 | "process-key", 117 | "worker-tid-4321", 118 | { 119 | "run_at" => run_at, 120 | "payload" => { 121 | "class" => "SyncWhateverWorker", 122 | "args" => [] 123 | } 124 | } 125 | ] 126 | ] 127 | end 128 | 129 | before { allow(instance).to receive(:workers).and_return(workers) } 130 | 131 | it "does nothing if the jobs just started" do 132 | expect(instance).to_not receive(:str_to_class) 133 | sync 134 | end 135 | 136 | context "when outside the run threshold" do 137 | let(:run_at) { (Time.now - described_class::RUN_INTERVAL - 5).to_i } 138 | 139 | let(:lock_key) { SyncExclusiveWorker.sync_lock_key(job_args) } 140 | 141 | it "refresh the lock" do 142 | SyncExclusiveWorker.new.perform(job_args) do 143 | Sqeduler::Service.redis_pool.with do |redis| 144 | # Change the lock timing to make sure ours works 145 | redis.expire(lock_key, 10) 146 | expect(redis.ttl(lock_key)).to eq(10) 147 | 148 | # Run the re-locker 149 | sync 150 | 151 | # Confirm it reset 152 | expect(redis.ttl(lock_key)).to eq(300) 153 | end 154 | end 155 | 156 | # Shouldn't be around once the job finished 157 | Sqeduler::Service.redis_pool.with do |redis| 158 | expect(redis.exists(lock_key)).to eq(0) 159 | end 160 | end 161 | end 162 | end 163 | 164 | context "#str_to_class" do 165 | it "only returns exclusive lock classes" do 166 | expect(instance.send(:str_to_class, "SyncExclusiveWorker")).to eq(SyncExclusiveWorker) 167 | expect(instance.send(:str_to_class, "SyncWhateverWorker")).to eq(false) 168 | end 169 | end 170 | end 171 | -------------------------------------------------------------------------------- /spec/middleware/kill_switch_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe Sqeduler::Middleware::KillSwitch do 6 | before do 7 | Sqeduler::Service.config = Sqeduler::Config.new( 8 | :redis_hash => REDIS_CONFIG, 9 | :logger => Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG } 10 | ) 11 | end 12 | 13 | describe "#call" do 14 | shared_examples_for "job is enqueued" do 15 | it "should enqueue the job" do 16 | expect do |b| 17 | described_class.new.call(worker_klass, nil, nil, nil, &b) 18 | end.to yield_control 19 | end 20 | end 21 | 22 | shared_examples_for "job is not enqueued" do 23 | it "should not enqueue the job" do 24 | expect do |b| 25 | described_class.new.call(worker_klass, nil, nil, nil, &b) 26 | end.to_not yield_control 27 | end 28 | end 29 | 30 | let(:worker_klass) { MyWorker } 31 | 32 | context "job does not prepend KillSwitch" do 33 | before do 34 | stub_const( 35 | "MyWorker", 36 | Class.new do 37 | include Sidekiq::Worker 38 | def perform; end 39 | end 40 | ) 41 | end 42 | 43 | it_behaves_like "job is enqueued" 44 | 45 | context "worker_klass is a string" do 46 | let(:worker_klass) { "MyWorker" } 47 | 48 | it_behaves_like "job is enqueued" 49 | end 50 | end 51 | 52 | context "job prepends KillSwitch" do 53 | before do 54 | stub_const( 55 | "MyWorker", 56 | Class.new do 57 | include Sidekiq::Worker 58 | prepend Sqeduler::Worker::KillSwitch 59 | def perform; end 60 | end 61 | ) 62 | end 63 | 64 | context "job is disabled" do 65 | before { MyWorker.disable } 66 | 67 | it_behaves_like "job is not enqueued" 68 | 69 | context "worker_klass is a string" do 70 | let(:worker_klass) { "MyWorker" } 71 | 72 | it_behaves_like "job is not enqueued" 73 | end 74 | end 75 | 76 | context "job is enabled" do 77 | before { MyWorker.enable } 78 | 79 | it_behaves_like "job is enqueued" 80 | 81 | context "worker_klass is a string" do 82 | let(:worker_klass) { "MyWorker" } 83 | 84 | it_behaves_like "job is enqueued" 85 | end 86 | end 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /spec/service_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe Sqeduler::Service do 6 | let(:logger) do 7 | Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG } 8 | end 9 | 10 | before do 11 | described_class.instance_variables.each do |ivar| 12 | described_class.remove_instance_variable(ivar) 13 | end 14 | end 15 | 16 | describe ".start" do 17 | subject { described_class.start } 18 | 19 | context "no config provided" do 20 | it "should raise" do 21 | expect { subject }.to raise_error(RuntimeError, "No config provided") 22 | end 23 | end 24 | 25 | context "config provided" do 26 | let(:schedule_filepath) { Pathname.new("./spec/fixtures/schedule.yaml") } 27 | let(:server_receiver) { double } 28 | let(:client_receiver) { double } 29 | before do 30 | allow(server_receiver).to receive(:call) 31 | allow(client_receiver).to receive(:call) 32 | 33 | described_class.config = Sqeduler::Config.new( 34 | :redis_hash => REDIS_CONFIG, 35 | :logger => logger, 36 | :schedule_path => schedule_filepath, 37 | :on_server_start => proc { |config| server_receiver.call(config) }, 38 | :on_client_start => proc { |config| client_receiver.call(config) } 39 | ) 40 | end 41 | 42 | it "configures the server" do 43 | expect(Sidekiq).to receive(:configure_server) 44 | subject 45 | end 46 | 47 | it "configures the client" do 48 | expect(Sidekiq).to receive(:configure_client) 49 | subject 50 | end 51 | 52 | it "calls the appropriate on_server_start callbacks" do 53 | allow(Sidekiq).to receive(:server?).and_return(true) 54 | expect(server_receiver).to receive(:call) 55 | subject 56 | end 57 | 58 | it "calls the appropriate on_client_start callbacks" do 59 | expect(client_receiver).to receive(:call) 60 | subject 61 | end 62 | end 63 | end 64 | 65 | describe ".redis_pool" do 66 | subject { described_class.redis_pool } 67 | 68 | before do 69 | described_class.config = Sqeduler::Config.new.tap do |config| 70 | config.redis_hash = REDIS_CONFIG 71 | config.logger = logger 72 | end 73 | end 74 | 75 | context "with pool provided in config" do 76 | let(:original_pool) do 77 | ConnectionPool.new(:size => 10, :timeout => 0.1) do 78 | Redis::Namespace.new("sqeduler", :client => Redis.new(REDIS_CONFIG)) 79 | end 80 | end 81 | 82 | before do 83 | described_class.config = Sqeduler::Config.new.tap do |config| 84 | config.redis_pool = original_pool 85 | config.logger = logger 86 | end 87 | end 88 | 89 | it "doesn't create a connection pool" do 90 | expect(subject.object_id).to eq(original_pool.object_id) 91 | end 92 | 93 | it "checks redis version" do 94 | allow_any_instance_of(Redis).to receive(:info).and_return( 95 | "redis_version" => "2.6.11" 96 | ) 97 | expect { subject }.to raise_error(RuntimeError, "Must be using redis >= 2.6.12") 98 | end 99 | end 100 | 101 | it "creates a connection pool" do 102 | expect(subject).to be_kind_of(ConnectionPool) 103 | end 104 | 105 | it "is memoized" do 106 | pool_1 = described_class.redis_pool 107 | pool_2 = described_class.redis_pool 108 | expect(pool_1.object_id).to eq(pool_2.object_id) 109 | end 110 | 111 | it "is not Sidekiq.redis" do 112 | described_class.start 113 | expect(Sidekiq.redis_pool.object_id).to_not eq(subject.object_id) 114 | end 115 | 116 | context "redis version is too low" do 117 | before do 118 | allow_any_instance_of(Redis).to receive(:info).and_return( 119 | "redis_version" => "2.6.11" 120 | ) 121 | if described_class.instance_variable_defined?(:@redis_pool) 122 | described_class.remove_instance_variable(:@redis_pool) 123 | end 124 | 125 | if described_class.instance_variable_defined?(:@verified) 126 | described_class.remove_instance_variable(:@verified) 127 | end 128 | end 129 | 130 | it "should raise" do 131 | expect { described_class.redis_pool }.to raise_error(RuntimeError, "Must be using redis >= 2.6.12") 132 | end 133 | end 134 | 135 | context "when provided redis_hash has strings as keys" do 136 | let(:expected_redis_config) do 137 | REDIS_CONFIG.merge(:namespace => "sqeduler") 138 | end 139 | 140 | before do 141 | described_class.config = Sqeduler::Config.new.tap do |config| 142 | config.redis_hash = REDIS_CONFIG.transform_keys(&:to_s) 143 | config.logger = logger 144 | end 145 | end 146 | 147 | it "converts keys to symbols to create redis" do 148 | expect(::Sidekiq::RedisConnection).to receive(:create).with(expected_redis_config).and_call_original 149 | subject 150 | end 151 | end 152 | 153 | context "with namespace provided in redis_hash" do 154 | let(:redis_hash) { REDIS_CONFIG.merge(:namespace => "foo") } 155 | 156 | before do 157 | described_class.config = Sqeduler::Config.new.tap do |config| 158 | config.redis_hash = redis_hash 159 | config.logger = logger 160 | end 161 | end 162 | 163 | it "uses the provided namespace" do 164 | expect(::Sidekiq::RedisConnection).to receive(:create).with(redis_hash).and_call_original 165 | subject 166 | end 167 | end 168 | end 169 | 170 | describe ".logger" do 171 | subject { described_class.logger } 172 | 173 | before do 174 | described_class.config = Sqeduler::Config.new.tap do |config| 175 | config.logger = logger 176 | end 177 | end 178 | 179 | context "provided in config" do 180 | it "return the config value" do 181 | expect(subject).to eq(logger) 182 | end 183 | end 184 | 185 | context "no config provided" do 186 | let(:logger) { nil } 187 | 188 | it "should raise ArgumentError" do 189 | expect { subject }.to raise_error(ArgumentError, /^No logger provided/) 190 | end 191 | 192 | context "in a Rails app" do 193 | let(:logger) { double } 194 | before do 195 | rails = double 196 | stub_const("Rails", rails) 197 | allow(rails).to receive(:logger).and_return(logger) 198 | end 199 | 200 | it "should use the Rails logger" do 201 | expect(subject).to eq(logger) 202 | end 203 | end 204 | end 205 | end 206 | end 207 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "pry" 4 | require "redis" 5 | require "rspec" 6 | require "sqeduler" 7 | require "timecop" 8 | 9 | TEST_REDIS = Redis.new(:host => "localhost", :db => 1) 10 | 11 | Timecop.safe_mode = true 12 | 13 | RSpec.configure do |config| 14 | config.before(:each) do 15 | TEST_REDIS.flushdb 16 | Sqeduler::Service.config = nil 17 | stub_const("REDIS_CONFIG", :host => "localhost", :db => 1) 18 | end 19 | config.disable_monkey_patching! 20 | end 21 | -------------------------------------------------------------------------------- /spec/sqeduler_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "sqeduler" 5 | 6 | RSpec.describe Sqeduler do 7 | it "should have a VERSION constant" do 8 | expect(subject.const_get("VERSION")).not_to be_empty 9 | end 10 | end 11 | -------------------------------------------------------------------------------- /spec/trigger_lock_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe Sqeduler::TriggerLock do 6 | context "#lock" do 7 | subject { described_class.new.lock } 8 | 9 | before do 10 | Sqeduler::Service.config = Sqeduler::Config.new( 11 | :redis_hash => REDIS_CONFIG, 12 | :logger => Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG } 13 | ) 14 | end 15 | 16 | let(:trigger_lock_1) { described_class.new } 17 | let(:trigger_lock_2) { described_class.new } 18 | 19 | it "should get the lock" do 20 | lock_successes = [trigger_lock_1, trigger_lock_2].map do |trigger_lock| 21 | Thread.new { trigger_lock.lock } 22 | end.map(&:value) 23 | 24 | expect(lock_successes).to match_array([true, false]) 25 | end 26 | 27 | it "should not be the owner if the lock has expired" do 28 | allow(trigger_lock_1).to receive(:expiration_milliseconds).and_return(1000) 29 | expect(trigger_lock_1.lock).to be true 30 | expect(trigger_lock_1.locked?).to be true 31 | sleep 1 32 | expect(trigger_lock_1.locked?).to be false 33 | end 34 | 35 | it "should refresh the lock expiration time when it is the owner" do 36 | allow(trigger_lock_1).to receive(:expiration_milliseconds).and_return(1000) 37 | expect(trigger_lock_1.lock).to be true 38 | sleep 1.1 39 | expect(trigger_lock_1.locked?).to be false 40 | expect(trigger_lock_1.refresh).to be true 41 | end 42 | 43 | it "should not refresh the lock when it is not owner" do 44 | threads = [] 45 | threads << Thread.new do 46 | allow(trigger_lock_1).to receive(:expiration_milliseconds).and_return(1000) 47 | trigger_lock_1.lock 48 | sleep 1 49 | end 50 | threads << Thread.new do 51 | sleep 1.1 52 | trigger_lock_2.lock 53 | end 54 | threads.each(&:join) 55 | expect(trigger_lock_2.locked?).to be(true) 56 | expect(trigger_lock_1.refresh).to be(false) 57 | end 58 | 59 | it "should release the lock when it is the owner" do 60 | expect(trigger_lock_1.lock).to be true 61 | expect(trigger_lock_1.unlock).to be true 62 | expect(trigger_lock_1.locked?).to be false 63 | end 64 | 65 | it "should not release the lock when it is not the owner" do 66 | threads = [] 67 | threads << Thread.new do 68 | allow(trigger_lock_1).to receive(:expiration_milliseconds).and_return(1000) 69 | trigger_lock_1.lock 70 | sleep 1 71 | end 72 | threads << Thread.new do 73 | sleep 1.1 74 | trigger_lock_2.lock 75 | end 76 | threads.each(&:join) 77 | expect(trigger_lock_2.locked?).to be(true) 78 | expect(trigger_lock_1.unlock).to be(false) 79 | end 80 | 81 | it "should not acquire the lock if there is an error" do 82 | allow(trigger_lock_1).to receive(:refresh_lock).and_raise("boom") 83 | expect(trigger_lock_1.lock).to be false 84 | expect(trigger_lock_1.locked?).to be false 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /spec/worker/kill_switch_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "./spec/fixtures/fake_worker" 5 | 6 | RSpec.describe Sqeduler::Worker::KillSwitch do 7 | describe ".disabled" do 8 | before do 9 | Sqeduler::Service.config = Sqeduler::Config.new( 10 | :redis_hash => REDIS_CONFIG, 11 | :logger => Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG } 12 | ) 13 | end 14 | after { FakeWorker.enable } 15 | 16 | it "lists the disabled workers" do 17 | expect(Sqeduler::Worker::KillSwitch.disabled).to eq({}) 18 | time = Time.now 19 | Timecop.freeze(time) { FakeWorker.disable } 20 | expect(Sqeduler::Worker::KillSwitch.disabled).to eq("FakeWorker" => time.to_s) 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/worker/synchronization_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe Sqeduler::Worker::Synchronization do 6 | describe ".synchronize" do 7 | before do 8 | stub_const( 9 | "ParentWorker", 10 | Class.new do 11 | prepend Sqeduler::Worker::Synchronization 12 | synchronize :one_at_a_time, :expiration => 10, :timeout => 1 13 | end 14 | ) 15 | 16 | stub_const("ChildWorker", Class.new(ParentWorker)) 17 | end 18 | 19 | it "should preserve the synchronize attributes" do 20 | expect(ChildWorker.synchronize_jobs_mode).to eq(:one_at_a_time) 21 | expect(ChildWorker.synchronize_jobs_expiration).to eq(10) 22 | expect(ChildWorker.synchronize_jobs_timeout).to eq(1) 23 | end 24 | 25 | it "should allow the child class to update the synchronize attributes" do 26 | ChildWorker.synchronize :one_at_a_time, :expiration => 20, :timeout => 2 27 | expect(ChildWorker.synchronize_jobs_mode).to eq(:one_at_a_time) 28 | expect(ChildWorker.synchronize_jobs_expiration).to eq(20) 29 | expect(ChildWorker.synchronize_jobs_timeout).to eq(2) 30 | expect(ParentWorker.synchronize_jobs_mode).to eq(:one_at_a_time) 31 | expect(ParentWorker.synchronize_jobs_expiration).to eq(10) 32 | expect(ParentWorker.synchronize_jobs_timeout).to eq(1) 33 | end 34 | end 35 | end 36 | -------------------------------------------------------------------------------- /spec/worker_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | require "./spec/fixtures/fake_worker" 5 | 6 | RSpec.describe Sqeduler::Worker do 7 | def verify_callback_occured(file_path, times = 1) 8 | expect(File.exist?(file_path)).to be_truthy 9 | expect(File.read(file_path).length).to eq(times) 10 | end 11 | 12 | def verify_callback_skipped(file_path) 13 | expect(File.exist?(file_path)).to be_falsey 14 | end 15 | 16 | def maybe_cleanup_file(file_path) 17 | File.delete(file_path) if File.exist?(file_path) 18 | end 19 | 20 | before do 21 | Sqeduler::Service.config = Sqeduler::Config.new( 22 | :redis_hash => REDIS_CONFIG, 23 | :logger => Logger.new(STDOUT).tap { |l| l.level = Logger::DEBUG } 24 | ) 25 | end 26 | 27 | after do 28 | maybe_cleanup_file(FakeWorker::JOB_RUN_PATH) 29 | maybe_cleanup_file(FakeWorker::JOB_SUCCESS_PATH) 30 | maybe_cleanup_file(FakeWorker::JOB_FAILURE_PATH) 31 | maybe_cleanup_file(FakeWorker::JOB_LOCK_FAILURE_PATH) 32 | maybe_cleanup_file(FakeWorker::JOB_BEFORE_START_PATH) 33 | maybe_cleanup_file(FakeWorker::SCHEDULE_COLLISION_PATH) 34 | end 35 | 36 | describe "#perform" do 37 | context "synchronized workers" do 38 | before do 39 | FakeWorker.synchronize :one_at_a_time, 40 | :expiration => expiration, 41 | :timeout => timeout 42 | end 43 | 44 | let(:expiration) { work_time * 4 } 45 | let(:work_time) { 0.1 } 46 | 47 | def run_synchronized_workers 48 | worker1 = Thread.new do 49 | FakeWorker.new.perform(work_time) 50 | end 51 | worker2 = Thread.new do 52 | sleep wait_time 53 | FakeWorker.new.perform(work_time) 54 | end 55 | worker1.join && worker2.join 56 | end 57 | 58 | context "overlapping schedule" do 59 | let(:wait_time) { 0 } 60 | 61 | context "timeout is less than work_time (too short)" do 62 | let(:timeout) { work_time / 2 } 63 | 64 | it "one worker should be blocked" do 65 | run_synchronized_workers 66 | verify_callback_occured(FakeWorker::JOB_LOCK_FAILURE_PATH) 67 | end 68 | 69 | it "only one worker should run" do 70 | run_synchronized_workers 71 | verify_callback_occured(FakeWorker::JOB_RUN_PATH) 72 | end 73 | 74 | it "one worker should succeed" do 75 | run_synchronized_workers 76 | verify_callback_occured(FakeWorker::JOB_SUCCESS_PATH, 2) 77 | end 78 | 79 | it "no worker should fail" do 80 | run_synchronized_workers 81 | verify_callback_skipped(FakeWorker::JOB_FAILURE_PATH) 82 | end 83 | 84 | it "all workers should have received before_start" do 85 | run_synchronized_workers 86 | verify_callback_occured(FakeWorker::JOB_BEFORE_START_PATH, 2) 87 | end 88 | 89 | it "a schedule collision should not have occurred" do 90 | run_synchronized_workers 91 | verify_callback_skipped(FakeWorker::SCHEDULE_COLLISION_PATH) 92 | end 93 | end 94 | 95 | context "timeout is greater than work_time" do 96 | let(:timeout) { work_time * 4 } 97 | 98 | it "no worker should be blocked" do 99 | run_synchronized_workers 100 | verify_callback_skipped(FakeWorker::JOB_LOCK_FAILURE_PATH) 101 | end 102 | 103 | it "both workers should succeed" do 104 | run_synchronized_workers 105 | verify_callback_occured(FakeWorker::JOB_SUCCESS_PATH, 2) 106 | end 107 | 108 | it "no worker should fail" do 109 | run_synchronized_workers 110 | verify_callback_skipped(FakeWorker::JOB_FAILURE_PATH) 111 | end 112 | 113 | it "all workers should have received before_start" do 114 | run_synchronized_workers 115 | verify_callback_occured(FakeWorker::JOB_BEFORE_START_PATH, 2) 116 | end 117 | 118 | it "a schedule collision should not have occurred" do 119 | run_synchronized_workers 120 | verify_callback_skipped(FakeWorker::SCHEDULE_COLLISION_PATH) 121 | end 122 | 123 | context "expiration too short" do 124 | let(:expiration) { work_time / 2 } 125 | 126 | it "no worker should be blocked" do 127 | run_synchronized_workers 128 | verify_callback_skipped(FakeWorker::JOB_LOCK_FAILURE_PATH) 129 | end 130 | 131 | it "all workers should run" do 132 | run_synchronized_workers 133 | verify_callback_occured(FakeWorker::JOB_RUN_PATH, 2) 134 | end 135 | 136 | it "all workers should have received before_start" do 137 | run_synchronized_workers 138 | verify_callback_occured(FakeWorker::JOB_BEFORE_START_PATH, 2) 139 | end 140 | 141 | it "no worker should fail" do 142 | run_synchronized_workers 143 | verify_callback_occured(FakeWorker::JOB_SUCCESS_PATH, 2) 144 | verify_callback_skipped(FakeWorker::JOB_FAILURE_PATH) 145 | end 146 | 147 | it "a schedule collision should occur" do 148 | run_synchronized_workers 149 | verify_callback_occured(FakeWorker::SCHEDULE_COLLISION_PATH, 2) 150 | end 151 | end 152 | end 153 | end 154 | 155 | context "non-overlapping schedule" do 156 | let(:wait_time) { work_time * 2 } 157 | 158 | context "timeout is less than work_time (too short)" do 159 | let(:timeout) { work_time } 160 | 161 | it "no workers should be blocked" do 162 | run_synchronized_workers 163 | verify_callback_skipped(FakeWorker::JOB_LOCK_FAILURE_PATH) 164 | end 165 | 166 | it "all workers should run" do 167 | run_synchronized_workers 168 | verify_callback_occured(FakeWorker::JOB_RUN_PATH, 2) 169 | end 170 | 171 | it "no worker should fail" do 172 | run_synchronized_workers 173 | verify_callback_occured(FakeWorker::JOB_SUCCESS_PATH, 2) 174 | verify_callback_skipped(FakeWorker::JOB_FAILURE_PATH) 175 | end 176 | 177 | it "all workers should have received before_start" do 178 | run_synchronized_workers 179 | verify_callback_occured(FakeWorker::JOB_BEFORE_START_PATH, 2) 180 | end 181 | 182 | it "a schedule collision should not have occurred" do 183 | run_synchronized_workers 184 | verify_callback_skipped(FakeWorker::SCHEDULE_COLLISION_PATH) 185 | end 186 | end 187 | 188 | context "timeout is greater than work_time" do 189 | let(:timeout) { work_time * 2 } 190 | 191 | it "no worker should be blocked" do 192 | run_synchronized_workers 193 | verify_callback_skipped(FakeWorker::JOB_LOCK_FAILURE_PATH) 194 | end 195 | 196 | it "both workers should succeed" do 197 | run_synchronized_workers 198 | verify_callback_occured(FakeWorker::JOB_SUCCESS_PATH, 2) 199 | end 200 | 201 | it "no worker should fail" do 202 | run_synchronized_workers 203 | verify_callback_skipped(FakeWorker::JOB_FAILURE_PATH) 204 | end 205 | 206 | it "all workers should have received before_start" do 207 | run_synchronized_workers 208 | verify_callback_occured(FakeWorker::JOB_BEFORE_START_PATH, 2) 209 | end 210 | 211 | context "expiration too short" do 212 | let(:expiration) { work_time / 2 } 213 | 214 | it "no worker should be blocked" do 215 | run_synchronized_workers 216 | verify_callback_skipped(FakeWorker::JOB_LOCK_FAILURE_PATH) 217 | end 218 | 219 | it "all workers should run" do 220 | run_synchronized_workers 221 | verify_callback_occured(FakeWorker::JOB_RUN_PATH, 2) 222 | end 223 | 224 | it "all workers should have received before_start" do 225 | run_synchronized_workers 226 | verify_callback_occured(FakeWorker::JOB_BEFORE_START_PATH, 2) 227 | end 228 | 229 | it "no worker should fail" do 230 | run_synchronized_workers 231 | verify_callback_occured(FakeWorker::JOB_SUCCESS_PATH, 2) 232 | verify_callback_skipped(FakeWorker::JOB_FAILURE_PATH) 233 | end 234 | 235 | it "a schedule collision should occur" do 236 | run_synchronized_workers 237 | verify_callback_occured(FakeWorker::SCHEDULE_COLLISION_PATH, 2) 238 | end 239 | end 240 | end 241 | end 242 | end 243 | end 244 | 245 | describe ".disable" do 246 | before do 247 | FakeWorker.disable 248 | end 249 | 250 | it "should not run" do 251 | FakeWorker.new.perform(0) 252 | verify_callback_skipped(FakeWorker::JOB_RUN_PATH) 253 | end 254 | 255 | it "should be disabled?" do 256 | expect(FakeWorker.disabled?).to be true 257 | expect(FakeWorker.enabled?).to be false 258 | end 259 | end 260 | 261 | describe ".enable" do 262 | before do 263 | FakeWorker.disable 264 | FakeWorker.enable 265 | end 266 | 267 | it "should run" do 268 | FakeWorker.new.perform(0) 269 | verify_callback_occured(FakeWorker::JOB_RUN_PATH) 270 | end 271 | 272 | it "should be enabled?" do 273 | expect(FakeWorker.disabled?).to be false 274 | expect(FakeWorker.enabled?).to be true 275 | end 276 | end 277 | end 278 | -------------------------------------------------------------------------------- /sqeduler.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require File.expand_path("lib/sqeduler/version", __dir__) 4 | 5 | Gem::Specification.new do |gem| 6 | gem.name = "sqeduler" 7 | gem.version = Sqeduler::VERSION 8 | gem.summary = "Common Sidekiq infrastructure for multi-host applications." 9 | gem.description = <<-DESC 10 | Works with Sidekiq scheduler to provides a highly available scheduler that can be run on 11 | multiple hosts. Also provides a convenient abstract class for Sidekiq workers. 12 | DESC 13 | gem.license = "Apache" 14 | gem.authors = ["Jared Jenkins"] 15 | gem.email = "jaredjenkins@squareup.com" 16 | gem.homepage = "https://rubygems.org/gems/sqeduler" 17 | 18 | gem.files = `git ls-files`.split($RS) 19 | gem.executables = gem.files.grep(%r{^bin/}).map { |f| File.basename(f) } 20 | gem.test_files = gem.files.grep(%r{(test|spec|features)/}) 21 | gem.require_paths = ["lib"] 22 | 23 | gem.add_runtime_dependency "activesupport" 24 | gem.add_runtime_dependency "redis-namespace" 25 | gem.add_runtime_dependency "sidekiq", "< 7" 26 | gem.add_runtime_dependency "sidekiq-scheduler", ">= 2.0", "< 6.0" 27 | end 28 | --------------------------------------------------------------------------------