├── .rspec ├── lib ├── simplekiq │ ├── version.rb │ ├── batch_tracker_job.rb │ ├── orchestration.rb │ ├── orchestration_job.rb │ ├── orchestration_executor.rb │ └── batching_job.rb └── simplekiq.rb ├── .gitignore ├── bin ├── setup └── console ├── Rakefile ├── Gemfile ├── tasks └── ci.rake ├── LICENSE.txt ├── spec ├── orchestration_spec.rb ├── simplekiq_spec.rb ├── spec_helper.rb ├── orchestration_executor_spec.rb ├── orchestration_job_spec.rb └── batching_job_spec.rb ├── .circleci └── config.yml ├── CONTRIBUTORS.md ├── simplekiq.gemspec ├── CHANGELOG.md ├── Gemfile.lock ├── CONTRIBUTING.md └── README.md /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | --require spec_helper 4 | -------------------------------------------------------------------------------- /lib/simplekiq/version.rb: -------------------------------------------------------------------------------- 1 | module Simplekiq 2 | VERSION = "1.0.0" 3 | end 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .rspec_status 2 | /.ruby-version 3 | /*.gem 4 | /.bundle/ 5 | /coverage/ 6 | /doc/ 7 | /pkg/ 8 | /spec/reports/ 9 | /tmp/ 10 | /vendor/bundle/ 11 | -------------------------------------------------------------------------------- /bin/setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | IFS=$'\n\t' 4 | set -vx 5 | 6 | bundle install 7 | 8 | # Do any other automated setup that you need to do here 9 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "rspec/core/rake_task" 4 | 5 | FileList["tasks/*.rake"].each { |task| load task } 6 | 7 | RSpec::Core::RakeTask.new(:spec) 8 | 9 | task default: :spec 10 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | source "https://artifacts.dox.support/repository/gems" do 4 | gem "sidekiq-pro", "7.3.6" 5 | end 6 | 7 | gem "sidekiq", "7.3.7" 8 | 9 | # Specify your gem's dependencies in simplekiq.gemspec 10 | gemspec 11 | -------------------------------------------------------------------------------- /tasks/ci.rake: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | namespace :ci do 4 | desc "Run specs" 5 | task :specs do 6 | reports = "tmp/test-results/rspec" 7 | sh "mkdir -p #{reports}" 8 | sh "bundle exec rspec ./spec " \ 9 | "--format progress " \ 10 | "--format RspecJunitFormatter " \ 11 | "-o #{reports}/results.xml" 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | require "bundler/setup" 4 | require "simplekiq" 5 | 6 | # You can add fixtures and/or initialization code here to make experimenting 7 | # with your gem easier. You can also use a different console, if you like. 8 | 9 | # (If you use this, don't forget to add pry to your Gemfile!) 10 | # require "pry" 11 | # Pry.start 12 | 13 | require "irb" 14 | IRB.start(__FILE__) 15 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright 2022 Doximity, Inc. 2 | 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | 7 | http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software 10 | distributed under the License is distributed on an "AS IS" BASIS, 11 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | See the License for the specific language governing permissions and 13 | limitations under the License. 14 | -------------------------------------------------------------------------------- /lib/simplekiq.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sidekiq" 4 | require "sidekiq-pro" 5 | 6 | require "simplekiq/orchestration_executor" 7 | require "simplekiq/orchestration" 8 | require "simplekiq/orchestration_job" 9 | require "simplekiq/batching_job" 10 | require "simplekiq/batch_tracker_job" 11 | 12 | module Simplekiq 13 | class << self 14 | def auto_define_callbacks(batch, args:, job:) 15 | batch.on("death", job.class, "args" => args) if job.respond_to?(:on_death) 16 | batch.on("complete", job.class, "args" => args) if job.respond_to?(:on_complete) 17 | batch.on("success", job.class, "args" => args) if job.respond_to?(:on_success) 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/simplekiq/batch_tracker_job.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This job serves two purposes: 4 | # * TODO: It provides a convenient way to track top-level orchestration batches 5 | # * The top-level orchestration batch would otherwise be empty (aside from 6 | # child-batches) and all sidekiq-pro batches must have at least 1 job 7 | 8 | module Simplekiq 9 | class BatchTrackerJob 10 | include Sidekiq::Job 11 | 12 | def perform(klass_name, bid, args) 13 | # In the future, this will likely surface the toplevel batch to a callback method on the 14 | # orchestration job. We're holding off on this until we have time to design a comprehensive 15 | # plan for providing simplekiq-wide instrumentation, ideally while being backwards compatible 16 | # for in-flight orchestrations. 17 | 18 | # For now, it's just satisfying the all-batches-must-have-jobs limitation in sidekiq-pro 19 | # described at the head of the file. 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/simplekiq/orchestration.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Simplekiq 4 | class Orchestration 5 | attr_accessor :serial_workflow, :parallel_workflow 6 | def initialize 7 | @serial_workflow = [] 8 | end 9 | 10 | def run(*step) 11 | workflow = parallel_workflow || serial_workflow 12 | workflow << step 13 | end 14 | 15 | def in_parallel 16 | @parallel_workflow = [] 17 | yield 18 | serial_workflow << @parallel_workflow if @parallel_workflow.any? 19 | ensure 20 | @parallel_workflow = nil 21 | serial_workflow 22 | end 23 | 24 | def serialized_workflow 25 | @serialized_workflow ||= serial_workflow.map do |step| 26 | case step[0] 27 | when Array 28 | step.map do |(job, *args)| 29 | {"klass" => job.name, "args" => args} 30 | end 31 | when Class 32 | job, *args = step 33 | {"klass" => job.name, "args" => args} 34 | end 35 | end 36 | end 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /spec/orchestration_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Simplekiq::Orchestration do 4 | let(:orchestration) { described_class.new } 5 | 6 | before do 7 | stub_const("OrcTest::JobA", Class.new) 8 | stub_const("OrcTest::JobB", Class.new) 9 | end 10 | 11 | describe "in_parallel" do 12 | subject { orchestration.serialized_workflow } 13 | 14 | context "when in_parallel is called without any run steps" do 15 | before do 16 | orchestration.in_parallel do 17 | # nothing 18 | end 19 | end 20 | 21 | it "does not add a step" do 22 | expect(subject).to be_empty 23 | end 24 | end 25 | 26 | context "when in_parallel is called with run steps" do 27 | before do 28 | orchestration.in_parallel do 29 | orchestration.run OrcTest::JobA, "syn" 30 | orchestration.run OrcTest::JobB, "ack" 31 | end 32 | end 33 | 34 | it "adds a step" do 35 | expect(subject).to eq [ 36 | [ 37 | {"klass" => "OrcTest::JobA", "args" => ["syn"]}, 38 | {"klass" => "OrcTest::JobB", "args" => ["ack"]} 39 | ] 40 | ] 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/simplekiq/orchestration_job.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | module Simplekiq 6 | module OrchestrationJob 7 | include Sidekiq::Job 8 | 9 | extend Forwardable 10 | 11 | def_delegators :orchestration, :run, :in_parallel 12 | 13 | def perform(*args) 14 | perform_orchestration(*args) 15 | 16 | # This makes it so that if there is a parent batch which this orchestration is run under, then the layered batches will be: 17 | # parent_batch( orchestration_batch( batch_of_first_step_of_the_orchestration ) ) 18 | # If there is no parent batch, then it will simply be: 19 | # orchestration_batch( batch_of_first_step_of_the_orchestration ) 20 | conditionally_within_parent_batch do 21 | OrchestrationExecutor.execute(args: args, job: self, workflow: orchestration.serialized_workflow) 22 | end 23 | end 24 | 25 | def workflow_plan(*args) 26 | perform_orchestration(*args) 27 | orchestration.serialized_workflow 28 | end 29 | 30 | private 31 | 32 | def conditionally_within_parent_batch 33 | if batch 34 | batch.jobs do 35 | yield 36 | end 37 | else 38 | yield 39 | end 40 | end 41 | 42 | def orchestration 43 | @orchestration ||= Orchestration.new 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | 3 | executors: 4 | ruby: 5 | resource_class: small 6 | docker: 7 | - image: cimg/ruby:3.3 8 | ruby-with-redis: 9 | resource_class: small 10 | docker: 11 | - image: cimg/ruby:3.3 12 | - image: redis:6.2 13 | command: redis-server 14 | 15 | jobs: 16 | build: 17 | executor: ruby-with-redis 18 | steps: 19 | - checkout 20 | - run: 21 | name: Install Bundler specific version 22 | command: | 23 | gem install bundler --version "${BUNDLE_VERSION}" --force 24 | - run: 25 | name: Install Ruby Dependencies 26 | command: | 27 | bundle config set --local path 'vendor/bundle' 28 | bundle install --jobs=4 --retry=3 29 | - run: 30 | name: Run Tests 31 | command: bundle exec rake ci:specs 32 | - store_test_results: 33 | name: Store test results 34 | path: tmp/test-results 35 | - run: 36 | name: Run StandardRB 37 | command: bundle exec standardrb 38 | - persist_to_workspace: 39 | root: . 40 | paths: 41 | - vendor/bundle 42 | 43 | workflows: 44 | main: 45 | jobs: 46 | - build: 47 | context: 48 | - sidekiq-enterprise 49 | - nexus_readonly 50 | -------------------------------------------------------------------------------- /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | ## List of All Known Code Contributors to Simplekiq 2 | 3 | ### Jack Noble (Doximity) 4 | * Collaborated on initial concept 5 | * Wrote the majority of the code as of initial release 6 | * Helpful contributions to maintenance of specs, README, etc 7 | 8 | ### John Wilkinson (Doximity) 9 | * Collaborated on initial concept 10 | * Conducted the gem extraction and release 11 | 12 | ### Jason Hagglund (Doximity) 13 | * Finagled a way into getting us the ability to specify `sidekiq-pro` as an explicit dependency despite it not being publicly available and without exposing it to the public in the process. 14 | 15 | ### Brian Dillard (Doximity) 16 | * Added additional comment documentation 17 | * Added support for `on_complete` batch callback support in `Simplekiq::BatchingJob` 18 | 19 | ### Austen Madden (Doximity) 20 | * Fixed bug with batch statuses in callbacks for empty batches 21 | 22 | ### Tiffany Troha (Doximity) 23 | * Added support for specifying `sidekiq_options` for the child job in `Simplekiq::BatchingJob` 24 | 25 | ### Lucas Lazzaris (Doximity) 26 | * Added support for sidekiq 7.x 27 | 28 | ### [Daniel Pepper](https://github.com/dpep) 29 | * On request, graciously took down his unused `simplekiq` placeholder from rubygems so we could continue using the name :raised_hands: 30 | 31 | ### [Jeremy Smith](https://github.com/jeremysmithco) 32 | * Helpfully and respectfully nudged us towards loosening our sidekiq version requirement 33 | -------------------------------------------------------------------------------- /spec/simplekiq_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Simplekiq do 4 | before do 5 | stub_const("OrcTest::BasicJob", Class.new) 6 | 7 | stub_const("OrcTest::CallbacksJob", Class.new do 8 | def on_complete(status, options) 9 | end 10 | 11 | def on_success(status, options) 12 | end 13 | 14 | def on_death(status, options) 15 | end 16 | end) 17 | end 18 | 19 | it "has a version number" do 20 | expect(Simplekiq::VERSION).not_to be nil 21 | end 22 | 23 | describe ".auto_define_callbacks" do 24 | let(:batch) { instance_double(Sidekiq::Batch) } 25 | let(:args) { [1, 2, 3] } 26 | let(:job) { OrcTest::CallbacksJob.new } 27 | 28 | def call 29 | Simplekiq.auto_define_callbacks(batch, args: args, job: job) 30 | end 31 | 32 | it "defines callbacks on the batch for every callback the job defines" do 33 | expect(batch).to receive(:on).once.ordered.with("death", OrcTest::CallbacksJob, "args" => [1, 2, 3]) 34 | expect(batch).to receive(:on).once.ordered.with("complete", OrcTest::CallbacksJob, "args" => [1, 2, 3]) 35 | expect(batch).to receive(:on).once.ordered.with("success", OrcTest::CallbacksJob, "args" => [1, 2, 3]) 36 | call 37 | end 38 | 39 | context "when the job does not define the callback methods" do 40 | let(:job) { OrcTest::BasicJob.new } 41 | 42 | it "does not define any callbacks" do 43 | expect(batch).not_to receive(:on) 44 | call 45 | end 46 | end 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /simplekiq.gemspec: -------------------------------------------------------------------------------- 1 | require_relative "lib/simplekiq/version" 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "simplekiq" 5 | spec.version = Simplekiq::VERSION 6 | spec.authors = ["Jack Noble", "John Wilkinson"] 7 | spec.email = ["jcwilkinson@doximity.com"] 8 | spec.summary = "Sidekiq-based workflow orchestration library" 9 | spec.description = "Provides tools for representing long chains of parallel and serial jobs in a flat, simple way." 10 | spec.homepage = "https://github.com/doximity/simplekiq" 11 | spec.license = "APACHE-2.0" 12 | spec.required_ruby_version = Gem::Requirement.new(">= 2.5.0") 13 | 14 | spec.metadata["homepage_uri"] = spec.homepage 15 | spec.metadata["source_code_uri"] = spec.homepage 16 | # TODO: spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here." 17 | 18 | # Specify which files should be added to the gem when it is released. 19 | # The `git ls-files -z` loads the files in the RubyGem that have been added into git. 20 | spec.files = Dir.chdir(File.expand_path("..", __FILE__)) do 21 | `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } 22 | end 23 | spec.executables = [] 24 | spec.require_paths = ["lib"] 25 | 26 | spec.add_development_dependency "rspec", "~> 3.2" 27 | spec.add_development_dependency "rake", "~> 12.0" 28 | spec.add_development_dependency "rspec_junit_formatter" 29 | spec.add_development_dependency "pry" 30 | spec.add_development_dependency "standard" 31 | 32 | spec.add_dependency "sidekiq", ">= 7.1", "< 9.0" 33 | spec.add_dependency "sidekiq-pro", ">= 7.1", "< 9.0" 34 | end 35 | -------------------------------------------------------------------------------- /lib/simplekiq/orchestration_executor.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module Simplekiq 4 | class OrchestrationExecutor 5 | def self.execute(args:, job:, workflow:) 6 | orchestration_batch = Sidekiq::Batch.new 7 | orchestration_batch.description = "#{job.class.name} Simplekiq orchestration" 8 | Simplekiq.auto_define_callbacks(orchestration_batch, args: args, job: job) 9 | 10 | orchestration_batch.jobs do 11 | Simplekiq::BatchTrackerJob.perform_async(job.class.name, orchestration_batch.bid, args) 12 | 13 | new.run_step(workflow, 0) unless workflow.empty? 14 | end 15 | end 16 | 17 | def run_step(workflow, step) 18 | *jobs = workflow.at(step) 19 | # This will never be empty because Orchestration#serialized_workflow skips inserting 20 | # a new step for in_parallel if there were no inner jobs specified. 21 | 22 | next_step = step + 1 23 | step_batch = Sidekiq::Batch.new 24 | step_batch.description = "Simplekiq orchestrated step #{next_step}" 25 | step_batch.on( 26 | "success", 27 | self.class, 28 | {"orchestration_workflow" => workflow, "step" => next_step} 29 | ) 30 | 31 | step_batch.jobs do 32 | jobs.each do |job| 33 | Object.const_get(job["klass"]).perform_async(*job["args"]) 34 | end 35 | end 36 | end 37 | 38 | def on_success(status, options) 39 | return if options["step"] == options["orchestration_workflow"].length 40 | 41 | Sidekiq::Batch.new(status.parent_bid).jobs do 42 | run_step(options["orchestration_workflow"], options["step"]) 43 | end 44 | end 45 | end 46 | end 47 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 6 | 7 | ## [1.0.0] 8 | * Only support Sidekiq 7.1 and Sidekiq 8 (dropped support for older versions) 9 | [#42](https://github.com/doximity/simplekiq/pull/42) 10 | 11 | ## [0.1.0] (pre-release for 1.0) 12 | * Fix typo in CONTRIBUTORS 13 | [#5](https://github.com/doximity/simplekiq/pull/5) 14 | * Fix incorrectly named spec file 15 | [#9](https://github.com/doximity/simplekiq/pull/9) 16 | * README fix 17 | [#10](https://github.com/doximity/simplekiq/pull/10) 18 | * Updating CONTRIBUTING license 19 | [#16](https://github.com/doximity/simplekiq/pull/16) 20 | * Fix CHANGELOG typo 21 | [#18](https://github.com/doximity/simplekiq/pull/18) 22 | * Add sidekiq-pro as an explicit dependency and loosen sidekiq requirements 23 | [#19](https://github.com/doximity/simplekiq/pull/19) 24 | * Add new toplevel batch to encapsulate all batches within an orchestration 25 | [#21](https://github.com/doximity/simplekiq/pull/21) 26 | * Fix bug with toplevel batch and include batch descriptions 27 | [#23](https://github.com/doximity/simplekiq/pull/23) 28 | 29 | ## [0.0.3] 30 | * Misc minimal fixes to get the gem building and releasable according to our open source standards 31 | [#3](https://github.com/doximity/simplekiq/pull/3) 32 | * Copy over library code from prior sources with maintained history 33 | [#2](https://github.com/doximity/simplekiq/pull/2) 34 | 35 | ## [0.0.2] 36 | * Scrubbed version from rubygems, do not use 37 | ## [0.0.1] 38 | * Scrubbed version from rubygems, do not use 39 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require "bundler/setup" 2 | require "simplekiq" 3 | require "pry" 4 | 5 | module SidekiqBatchTestHelpers 6 | # These helper methods only work in the following test mode: 7 | # sidekiq: :fake, stub_batches: false 8 | 9 | class NoBatchesError < StandardError 10 | def message 11 | "No batches queued. Ensure you the test has `stub_batches: false` and that you are actually queueing a batch" 12 | end 13 | end 14 | 15 | # https://github.com/mperham/sidekiq/issues/2700 16 | def stub_batches 17 | @batches = [] 18 | allow_any_instance_of(Sidekiq::Batch).to receive(:jobs) do |batch, &block| 19 | block.call 20 | @batches << batch 21 | end 22 | end 23 | 24 | def fail_one_batch 25 | raise NoBatchesError if @batches.empty? 26 | raise "Tried to fail one batch but there were multiple batches" if @batches.length > 1 27 | 28 | @batches.first.callbacks["death"].each do |callback| 29 | callback.each do |klass, args| 30 | klass.new.send(:on_death, "death", args) 31 | end 32 | end 33 | end 34 | 35 | def succeed_all_batches 36 | current_batches = @batches 37 | @batches = [] 38 | 39 | send_batch_callbacks_for(current_batches, "success") 40 | send_batch_callbacks_for(current_batches, "complete") 41 | end 42 | 43 | def send_batch_callbacks_for(current_batches, status) 44 | callback_symbol = :"on_#{status}" 45 | current_batches.each do |batch| 46 | next unless batch.callbacks[status] 47 | 48 | batch.callbacks[status].each do |callback| 49 | callback.each do |klass, args| # callback is a hash 50 | klass.new.send(callback_symbol, status, args) 51 | end 52 | end 53 | end 54 | end 55 | 56 | def run_all_jobs_and_batches 57 | loops_left = 100 58 | while Sidekiq::Job.jobs.any? || @batches.any? 59 | loops_left -= 1 60 | raise "no more loops!" if loops_left.negative? 61 | 62 | Sidekiq::Job.drain_all # This will raise if any job fails 63 | succeed_all_batches # Because nothing raised, we can assume success for all batches 64 | end 65 | end 66 | end 67 | 68 | RSpec.configure do |config| 69 | # Enable flags like --only-failures and --next-failure 70 | config.example_status_persistence_file_path = ".rspec_status" 71 | 72 | # Disable RSpec exposing methods globally on `Module` and `main` 73 | config.disable_monkey_patching! 74 | 75 | config.expect_with :rspec do |c| 76 | c.syntax = :expect 77 | end 78 | 79 | config.include SidekiqBatchTestHelpers, sidekiq: :fake 80 | end 81 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: . 3 | specs: 4 | simplekiq (0.2.0) 5 | sidekiq (>= 7.1, < 9.0) 6 | sidekiq-pro (>= 7.1, < 9.0) 7 | 8 | GEM 9 | remote: https://artifacts.dox.support/repository/gems/ 10 | specs: 11 | json (2.16.0) 12 | language_server-protocol (3.17.0.5) 13 | lint_roller (1.1.0) 14 | prism (1.6.0) 15 | racc (1.8.1) 16 | sidekiq-pro (7.3.6) 17 | sidekiq (>= 7.3.7, < 8) 18 | unicode-emoji (4.1.0) 19 | 20 | GEM 21 | remote: https://rubygems.org/ 22 | specs: 23 | ast (2.4.3) 24 | coderay (1.1.3) 25 | connection_pool (2.5.4) 26 | diff-lcs (1.5.0) 27 | logger (1.7.0) 28 | method_source (1.0.0) 29 | parallel (1.27.0) 30 | parser (3.3.10.0) 31 | ast (~> 2.4.1) 32 | racc 33 | pry (0.14.1) 34 | coderay (~> 1.1) 35 | method_source (~> 1.0) 36 | rack (3.2.4) 37 | rainbow (3.1.1) 38 | rake (12.3.3) 39 | redis-client (0.26.1) 40 | connection_pool 41 | regexp_parser (2.11.3) 42 | rspec (3.11.0) 43 | rspec-core (~> 3.11.0) 44 | rspec-expectations (~> 3.11.0) 45 | rspec-mocks (~> 3.11.0) 46 | rspec-core (3.11.0) 47 | rspec-support (~> 3.11.0) 48 | rspec-expectations (3.11.0) 49 | diff-lcs (>= 1.2.0, < 2.0) 50 | rspec-support (~> 3.11.0) 51 | rspec-mocks (3.11.1) 52 | diff-lcs (>= 1.2.0, < 2.0) 53 | rspec-support (~> 3.11.0) 54 | rspec-support (3.11.0) 55 | rspec_junit_formatter (0.5.1) 56 | rspec-core (>= 2, < 4, != 2.12.0) 57 | rubocop (1.80.2) 58 | json (~> 2.3) 59 | language_server-protocol (~> 3.17.0.2) 60 | lint_roller (~> 1.1.0) 61 | parallel (~> 1.10) 62 | parser (>= 3.3.0.2) 63 | rainbow (>= 2.2.2, < 4.0) 64 | regexp_parser (>= 2.9.3, < 3.0) 65 | rubocop-ast (>= 1.46.0, < 2.0) 66 | ruby-progressbar (~> 1.7) 67 | unicode-display_width (>= 2.4.0, < 4.0) 68 | rubocop-ast (1.47.1) 69 | parser (>= 3.3.7.2) 70 | prism (~> 1.4) 71 | rubocop-performance (1.25.0) 72 | lint_roller (~> 1.1) 73 | rubocop (>= 1.75.0, < 2.0) 74 | rubocop-ast (>= 1.38.0, < 2.0) 75 | ruby-progressbar (1.13.0) 76 | sidekiq (7.3.7) 77 | connection_pool (>= 2.3.0) 78 | logger 79 | rack (>= 2.2.4) 80 | redis-client (>= 0.22.2) 81 | standard (1.51.1) 82 | language_server-protocol (~> 3.17.0.2) 83 | lint_roller (~> 1.0) 84 | rubocop (~> 1.80.2) 85 | standard-custom (~> 1.0.0) 86 | standard-performance (~> 1.8) 87 | standard-custom (1.0.2) 88 | lint_roller (~> 1.0) 89 | rubocop (~> 1.50) 90 | standard-performance (1.8.0) 91 | lint_roller (~> 1.1) 92 | rubocop-performance (~> 1.25.0) 93 | unicode-display_width (3.2.0) 94 | unicode-emoji (~> 4.1) 95 | 96 | PLATFORMS 97 | ruby 98 | 99 | DEPENDENCIES 100 | pry 101 | rake (~> 12.0) 102 | rspec (~> 3.2) 103 | rspec_junit_formatter 104 | sidekiq (= 7.3.7) 105 | sidekiq-pro (= 7.3.6)! 106 | simplekiq! 107 | standard 108 | 109 | BUNDLED WITH 110 | 2.7.2 111 | -------------------------------------------------------------------------------- /spec/orchestration_executor_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe Simplekiq::OrchestrationExecutor do 4 | let(:workflow) do 5 | [ 6 | {"klass" => "OrcTest::JobA", "args" => [1]} 7 | ] 8 | end 9 | 10 | let!(:job) do 11 | stub_const("FakeOrchestration", Class.new do 12 | def on_success(status, options) 13 | end 14 | end) 15 | 16 | FakeOrchestration.new 17 | end 18 | 19 | before { stub_const("OrcTest::JobA", Class.new) } 20 | 21 | describe ".execute" do 22 | subject { described_class.execute(args: ["some", "args"], job: job, workflow: workflow) } 23 | let(:batch_double) { instance_double(Sidekiq::Batch, bid: 42) } 24 | 25 | before do 26 | allow(Sidekiq::Batch).to receive(:new).and_return(batch_double) 27 | expect(batch_double).to receive(:description=).with("FakeOrchestration Simplekiq orchestration") 28 | expect(batch_double).to receive(:on).with("success", FakeOrchestration, "args" => ["some", "args"]) 29 | end 30 | 31 | it "kicks off the first step with a new batch with the empty tracking batch inside it" do 32 | batch_stack_depth = 0 # to keep track of how deeply nested within batches we are 33 | expect(batch_double).to receive(:jobs) do |&block| 34 | batch_stack_depth += 1 35 | block.call 36 | batch_stack_depth -= 1 37 | end 38 | 39 | expect(Simplekiq::BatchTrackerJob).to receive(:perform_async) do 40 | expect(batch_stack_depth).to eq 1 41 | end 42 | 43 | instance = instance_double(Simplekiq::OrchestrationExecutor) 44 | allow(Simplekiq::OrchestrationExecutor).to receive(:new).and_return(instance) 45 | expect(instance).to receive(:run_step) do |workflow_arg, step| 46 | expect(batch_stack_depth).to eq 1 47 | expect(step).to eq 0 48 | end 49 | 50 | subject 51 | end 52 | 53 | context "when the workflow is empty" do 54 | let(:workflow) { [] } 55 | 56 | it "creates new batches or run any steps" do 57 | expect(batch_double).to receive(:jobs) { |&block| block.call } 58 | expect(described_class).not_to receive(:new) 59 | 60 | subject 61 | end 62 | end 63 | end 64 | 65 | describe "run_step" do 66 | let(:step_batch) { instance_double(Sidekiq::Batch) } 67 | let(:step) { 0 } 68 | let(:instance) { described_class.new } 69 | 70 | it "runs the next job within a new step batch" do 71 | batch_stack_depth = 0 # to keep track of how deeply nested within batches we are 72 | expect(step_batch).to receive(:jobs) do |&block| 73 | batch_stack_depth += 1 74 | block.call 75 | batch_stack_depth -= 1 76 | end 77 | 78 | expect(OrcTest::JobA).to receive(:perform_async) do |arg| 79 | expect(batch_stack_depth).to eq 1 80 | expect(arg).to eq 1 81 | end 82 | 83 | allow(Sidekiq::Batch).to receive(:new).and_return(step_batch) 84 | expect(step_batch).to receive(:on).with("success", described_class, { 85 | "orchestration_workflow" => workflow, 86 | "step" => 1 87 | }) 88 | expect(step_batch).to receive(:description=).with("Simplekiq orchestrated step 1") 89 | 90 | instance.run_step(workflow, 0) 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /spec/orchestration_job_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sidekiq/testing" 4 | 5 | RSpec.describe Simplekiq::OrchestrationJob do 6 | let!(:job) do 7 | stub_const("FakeOrchestration", Class.new do 8 | include Simplekiq::OrchestrationJob 9 | 10 | def perform_orchestration(first, second) 11 | run OrcTest::JobA, first 12 | run OrcTest::JobB, second 13 | end 14 | end) 15 | 16 | FakeOrchestration.new 17 | end 18 | 19 | before do 20 | stub_const("OrcTest::JobA", Class.new) 21 | stub_const("OrcTest::JobB", Class.new) 22 | stub_const("OrcTest::JobC", Class.new) 23 | end 24 | 25 | def perform 26 | job.perform("some", "args") 27 | end 28 | 29 | describe "on_complete" do 30 | let!(:job) do 31 | Class.new do 32 | include Simplekiq::OrchestrationJob 33 | 34 | def self.on_complete_called(args) 35 | end 36 | 37 | def perform_orchestration 38 | end 39 | 40 | def on_complete(_, options) 41 | self.class.on_complete_called(options["args"]) 42 | end 43 | end 44 | end 45 | 46 | before do 47 | Sidekiq::Testing.inline! 48 | stub_batches 49 | end 50 | 51 | it "runs the on_complete callback even if no jobs are run", sidekiq: :fake do 52 | expect(job).to receive(:on_complete_called).with([]) 53 | 54 | job.new.perform 55 | run_all_jobs_and_batches 56 | end 57 | end 58 | 59 | it "adds a new job to the sequence with #run" do 60 | expect(Simplekiq::OrchestrationExecutor).to receive(:execute).with( 61 | args: ["some", "args"], 62 | job: job, 63 | workflow: [ 64 | {"klass" => "OrcTest::JobA", "args" => ["some"]}, 65 | {"klass" => "OrcTest::JobB", "args" => ["args"]} 66 | ] 67 | ) 68 | 69 | perform 70 | end 71 | 72 | it "enables composition of orchestrations by re-opening the parent batch" do 73 | batch_double = instance_double(Sidekiq::Batch) 74 | 75 | batch_stack_depth = 0 # to keep track of how deeply nested within batches we are 76 | allow(batch_double).to receive(:jobs) do |&block| 77 | batch_stack_depth += 1 78 | block.call 79 | batch_stack_depth -= 1 80 | end 81 | 82 | allow(job).to receive(:batch).and_return(batch_double) 83 | 84 | expect(Simplekiq::OrchestrationExecutor).to receive(:execute) do 85 | expect(batch_stack_depth).to eq 1 86 | end 87 | 88 | perform 89 | end 90 | 91 | context "with jobs specified in_parallel" do 92 | let!(:job) do 93 | stub_const("FakeOrchestration", Class.new do 94 | include Simplekiq::OrchestrationJob 95 | 96 | def perform_orchestration(first, second) 97 | run OrcTest::JobA, first 98 | in_parallel do 99 | run OrcTest::JobB, first 100 | run OrcTest::JobC, second 101 | end 102 | end 103 | end) 104 | 105 | FakeOrchestration.new 106 | end 107 | 108 | it "adds a new jobs in parallel form to the workflow" do 109 | expect(Simplekiq::OrchestrationExecutor).to receive(:execute).with( 110 | args: ["some", "args"], 111 | job: job, 112 | workflow: [ 113 | {"klass" => "OrcTest::JobA", "args" => ["some"]}, 114 | [ 115 | {"klass" => "OrcTest::JobB", "args" => ["some"]}, 116 | {"klass" => "OrcTest::JobC", "args" => ["args"]} 117 | ] 118 | ] 119 | ) 120 | 121 | perform 122 | end 123 | end 124 | end 125 | -------------------------------------------------------------------------------- /lib/simplekiq/batching_job.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # This module enables you to break up your work into batches and run those 4 | # batches as background jobs while keeping all your code in the same file. 5 | # Including this module *implements perform* you should not override it. 6 | # It expects you to implement two methods: #perform_batching and 7 | # #perform_batch. 8 | # 9 | # Optionally you may also implement any combination of Sidekiq::Batch 10 | # callbacks. 11 | # - #on_complete 12 | # - #on_success 13 | # - #on_death 14 | # 15 | # #perform_batching should contain your code for breaking up your work into 16 | # smaller jobs. It handles all the Sidekiq::Batch boilerplate for you. Where 17 | # you would normally call ExampleBatchJob.perform_async you should use 18 | # #queue_batch. If you'd like to custommize the sidekiq batch object, you can 19 | # access it in perform_batching through the `sidekiq_batch` method. 20 | # 21 | # #perform_batch should contain the code that would be in your batch job. Under 22 | # the hood, #queue_batch queues a job which will run #perform_batch. 23 | # 24 | # [Sidekiq::Batch documentation](https://github.com/mperham/sidekiq/wiki/Batches) 25 | # explains batches, their lifecycle, callbacks, etc. 26 | # 27 | # class ExampleJob 28 | # include Simplekiq::BatchingJob 29 | # 30 | # def perform_batching(some_id) 31 | # sidekiq_batch.description = "My custom batch description" # optional 32 | # 33 | # Record.find(some_id).other_records.in_batches do |other_records| 34 | # queue_batch(other_records.ids) 35 | # end 36 | # end 37 | # 38 | # def perform_batch(other_record_ids) 39 | # OtherRecord.where(id: other_record_ids).do_work 40 | # end 41 | # 42 | # def on_death(_status, options) 43 | # same_id_as_before = options["args"].first 44 | # Record.find(same_id_as_before).death! 45 | # end 46 | 47 | # def on_complete(_status, options) 48 | # same_id_as_before = options["args"].first 49 | # Record.find(same_id_as_before).complete! 50 | # end 51 | 52 | # def on_success(_status, options) 53 | # same_id_as_before = options["args"].first 54 | # Record.find(same_id_as_before).success! 55 | # end 56 | # end 57 | # 58 | # ExampleJob.perform_async(some_id) 59 | # 60 | # Come home to the impossible flavor of batch creation 61 | 62 | module Simplekiq 63 | module BatchingJob 64 | include Sidekiq::Job 65 | 66 | BATCH_CLASS_NAME = "SimplekiqBatch" 67 | 68 | class << self 69 | def included(klass) 70 | batch_job_class = Class.new(BaseBatch) 71 | klass.const_set(BATCH_CLASS_NAME, batch_job_class) 72 | 73 | klass.extend ClassMethods 74 | end 75 | end 76 | 77 | module ClassMethods 78 | def batch_sidekiq_options(options) 79 | batch_class = const_get(BATCH_CLASS_NAME) 80 | batch_class.instance_eval do 81 | sidekiq_options(options) 82 | end 83 | end 84 | end 85 | 86 | def perform(*args) 87 | self.batches = [] 88 | 89 | perform_batching(*args) 90 | 91 | # If we're part of an existing sidekiq batch make this a child batch 92 | # This is necessary for it work with orchestration; we could add an option 93 | # to toggle the behavior on and off. 94 | if batch 95 | batch.jobs do 96 | flush_batches(args) 97 | end 98 | else 99 | flush_batches(args) 100 | end 101 | end 102 | 103 | protected # TODO: should this be private? 104 | 105 | attr_accessor :batches 106 | 107 | def flush_batches(args) 108 | batch_job_class = self.class.const_get(BATCH_CLASS_NAME) 109 | sidekiq_batch.description ||= "Simplekiq Batch Jobs for #{self.class.name}, args: #{args}" 110 | 111 | Simplekiq.auto_define_callbacks(sidekiq_batch, args: args, job: self) 112 | 113 | sidekiq_batch.jobs do 114 | batches.each do |job_args| 115 | batch_job_class.perform_async(*job_args) 116 | end 117 | end 118 | end 119 | 120 | def queue_batch(*args) 121 | batches << args 122 | end 123 | 124 | def batch_description=(description) 125 | sidekiq_batch.description = description 126 | end 127 | 128 | private 129 | 130 | def sidekiq_batch 131 | @sidekiq_batch ||= Sidekiq::Batch.new 132 | end 133 | end 134 | 135 | class BaseBatch 136 | include Sidekiq::Job 137 | 138 | def perform(*args) 139 | module_parent_of_class.new.perform_batch(*args) 140 | end 141 | 142 | private 143 | 144 | def module_parent_of_class 145 | # Borrowed from https://apidock.com/rails/Module/module_parent_name 146 | parent_name = (self.class.name =~ /::[^:]+\Z/) ? $`.freeze : nil 147 | parent_name ? Object.const_get(parent_name) : Object 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /spec/batching_job_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "sidekiq/testing" 4 | 5 | RSpec.describe Simplekiq::BatchingJob do 6 | before do 7 | Sidekiq::Testing.inline! 8 | end 9 | 10 | describe "batching" do 11 | let(:test_job) do 12 | Class.new do 13 | include Simplekiq::BatchingJob 14 | 15 | def perform_batching(arg) 16 | queue_batch(arg) 17 | end 18 | 19 | def perform_batch(arg) 20 | Output.call(arg) 21 | end 22 | end 23 | end 24 | 25 | it "runs batches" do 26 | stub_const("TestJob", test_job) 27 | stub_const("Output", output = double("Output", call: nil)) 28 | 29 | test_job.new.perform("test") 30 | 31 | expect(output).to have_received(:call).with("test") 32 | end 33 | 34 | it "queues a job with a readable name" do 35 | stub_const("TestJob", test_job) 36 | allow(TestJob::SimplekiqBatch).to receive(:perform_async) 37 | 38 | test_job.new.perform("test") 39 | 40 | expect(TestJob::SimplekiqBatch).to have_received(:perform_async) 41 | end 42 | end 43 | 44 | describe "on_success" do 45 | let(:test_job) do 46 | Class.new do 47 | include Simplekiq::BatchingJob 48 | 49 | def perform_batching(things) 50 | things.each { |t| queue_batch(t) } 51 | end 52 | 53 | def perform_batch(arg) 54 | end 55 | 56 | def on_success(_, options) 57 | Output.call(options["args"].first) 58 | end 59 | end 60 | end 61 | 62 | it "runs the on_success callback even if no batches are run", sidekiq: :fake do 63 | stub_const("TestJob", test_job) 64 | stub_const("Output", output = double("Output", call: nil)) 65 | stub_batches 66 | 67 | test_job.new.perform([]) 68 | run_all_jobs_and_batches 69 | 70 | expect(output).to have_received(:call).with([]) 71 | end 72 | 73 | it "runs the on_success callback when batches complete successfully", sidekiq: :fake do 74 | stub_const("TestJob", test_job) 75 | stub_const("Output", output = double("Output", call: nil)) 76 | stub_batches 77 | 78 | test_job.new.perform(["test"]) 79 | run_all_jobs_and_batches 80 | 81 | expect(output).to have_received(:call).with(["test"]) 82 | end 83 | end 84 | 85 | describe "on_complete" do 86 | let(:test_job) do 87 | Class.new do 88 | include Simplekiq::BatchingJob 89 | 90 | def perform_batching(things) 91 | things.each { |t| queue_batch(t) } 92 | end 93 | 94 | def perform_batch(arg) 95 | end 96 | 97 | def on_complete(_, options) 98 | Output.call(options["args"].first) 99 | end 100 | end 101 | end 102 | 103 | it "runs the on_complete callback even if no batches are run", sidekiq: :fake do 104 | stub_const("TestJob", test_job) 105 | stub_const("Output", output = double("Output", call: nil)) 106 | stub_batches 107 | 108 | test_job.new.perform([]) 109 | run_all_jobs_and_batches 110 | 111 | expect(output).to have_received(:call).with([]) 112 | end 113 | 114 | it "runs the on_complete callback when each job has been run once", sidekiq: :fake do 115 | stub_const("TestJob", test_job) 116 | stub_const("Output", output = double("Output", call: nil)) 117 | stub_batches 118 | 119 | test_job.new.perform(["test"]) 120 | run_all_jobs_and_batches 121 | 122 | expect(output).to have_received(:call).with(["test"]) 123 | end 124 | end 125 | 126 | describe "on_death" do 127 | let(:test_job) do 128 | Class.new do 129 | include Simplekiq::BatchingJob 130 | 131 | def perform_batching(things) 132 | things.each { |t| queue_batch(t) } 133 | end 134 | 135 | def perform_batch(arg) 136 | end 137 | 138 | def on_death(_, options) 139 | Output.call(options["args"].first) 140 | end 141 | end 142 | end 143 | 144 | it "runs the on_death callback when a batch fails", sidekiq: :fake do 145 | stub_const("TestJob", test_job) 146 | stub_const("Output", output = double("Output", call: nil)) 147 | stub_batches 148 | 149 | test_job.new.perform(["test"]) 150 | fail_one_batch 151 | 152 | expect(output).to have_received(:call).with(["test"]) 153 | end 154 | end 155 | 156 | describe "batch_sidekiq_options" do 157 | let(:test_job) do 158 | Class.new do 159 | include Simplekiq::BatchingJob 160 | 161 | batch_sidekiq_options queue: "test_queue" 162 | 163 | def perform_batching(arg) 164 | queue_batch(arg) 165 | end 166 | 167 | def perform_batch(arg) 168 | Output.call(arg) 169 | end 170 | end 171 | end 172 | 173 | it "sets the sidekiq options for the base class" do 174 | stub_const("Test", test_job) 175 | 176 | expect(Test::SimplekiqBatch.sidekiq_options).to eq("queue" => "test_queue", "retry" => true) 177 | end 178 | end 179 | end 180 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing to Doximity 2 | 3 | We welcome contributions to this repository. Feel free to submit issues for bugs you encounter and pull requests for code and documentation contributions. 4 | In order to prevent licensing issues, Doximity Inc. (“Doximity”, “we”, “us”) requires all contributors to agree to an Individual Contributor License Agreement (“CLA”), which is reproduced below. By submitting your contributions to us, you agree that you have read and are bound by the CLA. If you do not agree with the CLA, you may not submit contributions. 5 | 6 | ## Doximity Individual Contributor License Agreement 7 | 8 | This license is for your protection as a Contributor as well as the protection of Doximity; it does not change your rights to use your own Contributions for any other purpose. 9 | 10 | You accept and agree to the following terms and conditions for Your present and future Contributions submitted to Doximity. Except for the license granted herein to Doximity and recipients of software distributed by Doximity, You reserve all right, titles, and interests in and to Your Contributions. 11 | 12 | ### Definitions 13 | 14 | "You" (or "Your" or the “Contributor”) shall mean the copyright owner or legal entity authorized by the copyright owner that is making this Agreement with Doximity. For legal entities, the entity making a Contribution and all other entities that control, are controlled by, or are under common control with that entity are considered to be a single Contributor. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. 15 | 16 | 1. "Contribution" shall mean the code, documentation, or any original work of authorship, including any modifications or additions to an existing work, that is intentionally submitted by You to Doximity for inclusion in, or documentation of, any of the products owned or managed by Doximity (the "Work"). For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to Doximity or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Doximity for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution." 17 | 18 | 2. Grant of Copyright License. Subject to the terms and conditions of this Agreement, You hereby grant to Doximity and to recipients of software distributed by Doximity a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute Your Contributions and such derivative works. 19 | 20 | 3. Grant of Patent License. Subject to the terms and conditions of this Agreement, You hereby grant to Doximity and to recipients of software distributed by Doximity a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by You that are necessarily infringed by Your Contribution(s) alone or by a combination of Your Contribution(s) with the Work to which such Contribution(s) was submitted. If any entity institutes patent litigation against You or any other entity (including a cross-claim or counterclaim in a lawsuit) alleging that your Contribution, or the Work to which you have contributed, constitutes a direct or contributory patent infringement, then any patent licenses granted to that entity under this Agreement for that Contribution or Work shall terminate as of the date such litigation is filed. 21 | 22 | 4. You represent that You are legally entitled to grant the above license. If your employer(s) has rights to intellectual property that you create that includes your Contributions, you represent that you have received permission to make Contributions on behalf of that employer, that your employer has waived such rights for your Contributions to Doximity, or that your employer has executed a separate Corporate CLA with Doximity. 23 | 24 | 5. You represent that each of Your Contributions is Your original creation (see section 7 for submissions on behalf of others). You represent that Your Contribution submissions include complete details of any third-party license or other restriction (including, but not limited to, related patents and trademarks) of which you are personally aware and which are associated with any part of Your Contributions. 25 | 26 | 6. You are not expected to provide support for Your Contributions, except to the extent You desire to provide support. You may provide support for free, for a fee, or not at all. Unless required by applicable law or agreed to in writing, You provide Your Contributions on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. 27 | 28 | 7. Should You wish to submit work that is not Your original creation, You may submit it to Doximity separately from any Contribution, identifying the complete details of its source and of any license or other restriction (including, but not limited to, related patents, trademarks, and license agreements) of which you are personally aware, and conspicuously marking the work as "Submitted on behalf of a third-party: [named here]". 29 | 30 | 8. You agree to notify Doximity of any facts or circumstances of which you become aware that would make these representations inaccurate in any respect. 31 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simplekiq 2 | 3 | Any time that you find yourself needing to string together a long chain of jobs, particularly when there are multiple stages of Sidekiq-pro batches and callbacks involved, come home instead to the simple flavor of orchestrated job flow with Simplekiq. 4 | 5 | ## Installation 6 | 7 | Add this line to your application's Gemfile: 8 | 9 | ```ruby 10 | gem "simplekiq" 11 | ``` 12 | 13 | Note that this gem requires you be a Sidekiq Pro paid subscriber to be able to use it, so after following the installation docs for getting the private gem configured with your system, ensure you have `sidekiq-pro` at version `~> 5.0.0` or higher (need at least version `5.2.1` if you want to capture `on_death` callbacks [percolating up to parent batches](https://github.com/mperham/sidekiq/blob/main/Pro-Changes.md#521) - a supported feature which is not required for typical orchestration behavior) and that it's being required: 14 | 15 | ```ruby 16 | gem "sidekiq-pro", "~> 5.0.0" 17 | ``` 18 | 19 | And then execute: 20 | 21 | $ bundle install 22 | 23 | Or install it yourself as: 24 | 25 | $ gem install simplekiq 26 | 27 | ## Usage 28 | 29 | There are currently two primary components of the system which were designed to work in harmony: 30 | 31 | * [Simplekiq::OrchestrationJob](./lib/simplekiq/orchestration_job.rb) - A mixin for a Sidekiq jobs to be able to orchestrate a flow of jobs in one place. It makes long complicated flows between jobs easier to understand, iterate on, and test. It eliminates the need to hop between dozens of files to determine when, where, and why a particular job gets called. 32 | * [Simplekiq::BatchingJob](./lib/simplekiq/batching_job.rb) - A mixin designed to make breaking a large job into a batched process dead simple and contained within a single class while still being trivially composable in orchestrations. 33 | 34 | ## Tool Drilldown 35 | 36 | ### Simplekiq::OrchestrationJob 37 | 38 | Mixing in the [Simplekiq::Orchestration](./lib/simplekiq/orchestration_job.rb) module lets you define a human-readable workflow of jobs in a single file with almost* no special requirements or restrictions on how the child jobs are designed. In most cases, Sidekiq jobs not designed for use in orchestrations should be compatible for use in orchestrations. A job implementing `OrchestrationJob` might look like: 39 | 40 | ```ruby 41 | class SomeOrchestrationJob 42 | include Simplekiq::OrchestrationJob 43 | 44 | def perform_orchestration(some_id) 45 | @some_model = SomeModel.find(some_id) # 1. 46 | 47 | run SomeInitialSetupJob, some_model.id # 2. 48 | 49 | in_parallel do 50 | some_related_models.each do |related_model| 51 | run SomeParallelizableJob, related_model.id # 3. 52 | end 53 | end 54 | 55 | run SomeFinalizationJob, some_model.id # 4. 56 | end 57 | 58 | def on_death(status, options) # 5. 59 | SomeModel.find(options["args"].first).failure_happened! 60 | end 61 | 62 | def on_complete(status, options) # 6. 63 | failures = Array(status&.failure_info) # sidekiq-pro batch status api 64 | return if failures.empty? 65 | 66 | SomeModel.find(options["args"].first).it_was_these_failures(failures) 67 | end 68 | 69 | private 70 | 71 | attr_reader :some_model 72 | 73 | def some_related_models 74 | @some_related_models ||= some_model.some_relation 75 | end 76 | end 77 | ``` 78 | 79 | Let's use the above example to describe some specifics of how the flow works. 80 | 81 | 1. `SomeOrchestrationJob` pulls up some instance of parent model `SomeModel`. 82 | 2. It does some initial work in `SomeInitialSetupJob`, which blocks the rest of the workflow until it completes successfully. 83 | 3. Then it will run a `SomeParallelizableJob` for each of some number of associated models `some_related_models`. These jobs will all run parallel to each other independently. 84 | 4. Finally, after all of the parallel jobs from #3 complete successfully, `SomeFinalizationJob` will run and then after it finishes the orchestration will be complete. 85 | 5. If it ran into an error at some point, `on_death` will get fired with the first failure. (please use `sidekiq-pro` of at least `5.2.1` for this feature) 86 | 6. It will call `on_complete` at the end of the orchestration no matter what, this is the place to collect all the failures and persist them somewhere. 87 | 88 | **Note** - it's fine to add utility methods and `attr_accessor`s to keep the code tidy and maintainable. 89 | 90 | When `SomeOrchestrationJob` itself gets called though, the first thing it does it turn these directives into a big serialized structure indicating which job will be called under what conditions (eg, serial or in parallel) and with what arguments, and then keeps passing that between the simplekiq-internal jobs that actually conduct the flow. 91 | 92 | This means when you want to deploy a change to this flow all previous in-flight workflows will continue undisturbed because the workflow is frozen in sidekiq job arguments and will remain frozen until the workflow completes. This is generally a boon, but note that if you remove a job from a workflow you'll need to remember to either keep the job itself (eg, the `SomeFinalizationJob` class file from our above example) in the codebase or replace it with a stub so that any in-flight workflows won't crash due to not being able to pull up the prior-specified workflow. 93 | 94 | "almost* no special requirements or restrictions on how the child jobs are designed" - The one thing you'll want to keep in mind when feeding arbitrary jobs into orchestrations is that if the job creates any new sidekiq batches then those new sidekiq batches should be added as child sidekiq batches of the parent sidekiq batch of the job. The parent sidekiq batch of the job is the sidekiq batch that drives the orchestration from step to step, so if you don't do this it will move onto the next step in the orchestration once your job finishes even if the new sidekiq batches it started didn't finish. This sounds more complicated than it is, you can see an example of code that does this in [`BatchingJob#perform`](./lib/simplekiq/batching_job.rb): 95 | 96 | ```ruby 97 | if batch # is there a parent batch? 98 | batch.jobs do # open the parent batch back up 99 | create_a_new_batch_and_add_jobs_to_it_to_run # make our new batch as a child batch of the parent batch 100 | end # close the parent batch again 101 | else # there's no parent batches, this job was run directly outside of an orchestration 102 | create_a_new_batch_and_add_jobs_to_it_to_run # make our new batch without a parent batch 103 | end 104 | ``` 105 | 106 | ### Simplekiq::BatchingJob 107 | 108 | See the [Simplekiq::BatchingJob](./lib/simplekiq/batching_job.rb) module itself for a description and example usage in the header comments. Nutshell is that you should use this if you're planning on making a batched asynchronous process as it shaves off a lot of ceremony and unexpressive structure. eg - Instead of having `BeerBottlerJob` which queues some number of `BeerBottlerBatchJob`s to handle the broken down sub-tasks you can just have `BeerBottlerJob` with a method for batching, executing individual batches, and a callback that gets run after all batches have completed successfully. 109 | 110 | ## History 111 | 112 | Simplekiq was initially released for private use within Doximity applications in Oct 2020 where it continued to be iterated on towards stability and general use until Jan 2022 when it was deemed settled enough for public release. 113 | 114 | The primary driving factor that inspired this work was a series of over a dozen differently defined and structured jobs part of a single workflow of which the logical flow was extraordinarily difficult to cognitively trace. This led to exteme difficulty in debugging and following problematic instances of the workflow in production as well as needlessly high cost to refactoring and iterative adjustments. 115 | 116 | The crux of the problem was that each job was highly coupled to its position in the overall flow as well as the absence of any central mechanism to indicate what the overall flow was. After building Simplekiq and implementing it into the flow, significant changes to the flow became quick adjustments requiring only a couple lines of code to change and folks unfamiliar with the system could quickly get up to speed by reading through the orchestration job. 117 | 118 | ## Versioning 119 | 120 | This project follows semantic versioning. See https://semver.org/ for details. 121 | 122 | ## Development 123 | 124 | After checking out the repo, run `bin/setup` to install dependencies. Note that this depends on `sidekiq-pro` which requires a [commercial license](https://sidekiq.org/products/pro.html) to install and use. 125 | 126 | Then, run `rake ci:specs` to run the tests. You can also run `bin/console` for an interactive prompt that will allow you to experiment. 127 | 128 | To install this gem onto your local machine, run `bundle exec rake install`. 129 | 130 | To get a new release cut, please either open a PR or an issue with your ask with as much context as possible and someone from Doximity will consider your request. If it makes sense for the direction of the project we'll get it done and inform you of when a release has been made available with the changes. 131 | 132 | For internal employees: consult the company wiki on the current standard process for conducting releases for our public gems. 133 | 134 | ## Contributing 135 | 136 | 1. See [CONTRIBUTING.md](./CONTRIBUTING.md) 137 | 2. Fork it 138 | 3. Create your feature branch (`git checkout -b my-new-feature`) 139 | 4. Commit your changes (`git commit -am 'Add some feature'`) 140 | 5. Push to the branch (`git push origin my-new-feature`) 141 | 6. Create a new Pull Request 142 | 143 | ## License 144 | 145 | The gem is licensed under an Apache 2 license. Contributors are required to sign an contributor license agreement. See LICENSE.txt and CONTRIBUTING.md for more information. 146 | --------------------------------------------------------------------------------