├── .codeclimate.yml ├── .dockerignore ├── .gemrelease ├── .gitignore ├── .rspec ├── .rubocop.yml ├── .travis.yml ├── Appraisals ├── Dockerfile ├── Gemfile ├── LICENSE ├── README.md ├── Rakefile ├── app ├── Gemfile ├── Gemfile.lock ├── app.rb ├── config.ru ├── sidekiq.rb ├── views │ └── index.erb └── workers │ ├── some_worker.rb │ └── unique_worker.rb ├── docker-compose-common.yml ├── docker-compose.yml ├── gemfiles ├── sidekiq_4_0.gemfile ├── sidekiq_4_1.gemfile ├── sidekiq_4_2.gemfile ├── sidekiq_5_0.gemfile └── sidekiq_5_1.gemfile ├── lib ├── sidekiq-merger.rb └── sidekiq │ ├── merger.rb │ └── merger │ ├── config.rb │ ├── flusher.rb │ ├── logging_observer.rb │ ├── merge.rb │ ├── middleware.rb │ ├── redis.rb │ ├── version.rb │ ├── views │ └── index.erb │ └── web.rb ├── misc ├── bulk_notification_flow.png ├── cancel_task_flow.png └── web_ui.png ├── sidekiq-merger.gemspec └── spec ├── sidekiq ├── merger │ ├── flusher_spec.rb │ ├── logging_observer_spec.rb │ ├── merge_spec.rb │ ├── middleware_spec.rb │ └── redis_spec.rb └── merger_spec.rb ├── spec_helper.rb └── support ├── matchers.rb └── worker_class.rb /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | brakeman: 3 | enabled: true 4 | duplication: 5 | enabled: true 6 | config: 7 | languages: 8 | - ruby 9 | fixme: 10 | enabled: true 11 | rubocop: 12 | enabled: true 13 | ratings: 14 | paths: 15 | - Gemfile.lock 16 | - "**.erb" 17 | - "**.rb" 18 | exclude_paths: 19 | - "app/" 20 | - "spec/" 21 | - "misc/" 22 | -------------------------------------------------------------------------------- /.dockerignore: -------------------------------------------------------------------------------- 1 | .bundle/ 2 | .yardoc 3 | _yardoc/ 4 | coverage/ 5 | doc/ 6 | pkg/ 7 | spec/reports/ 8 | tmp/ 9 | rdoc/ 10 | 11 | /Gemfile.lock 12 | 13 | *.gem 14 | *.rbc 15 | 16 | .ruby-version 17 | .ruby-gemset 18 | .rvmrc 19 | 20 | .env 21 | 22 | .dockerignore 23 | -------------------------------------------------------------------------------- /.gemrelease: -------------------------------------------------------------------------------- 1 | bump: 2 | tag: true 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .bundle/ 2 | .yardoc 3 | _yardoc/ 4 | coverage/ 5 | doc/ 6 | pkg/ 7 | spec/reports/ 8 | tmp/ 9 | rdoc/ 10 | 11 | /Gemfile.lock 12 | #Appraisal 13 | *.gemfile.lock 14 | 15 | *.gem 16 | *.rbc 17 | 18 | .ruby-version 19 | .ruby-gemset 20 | .rvmrc 21 | 22 | .env 23 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --format documentation 2 | --color 3 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | AllCops: 2 | DisabledByDefault: true 3 | Exclude: 4 | - 'gemfiles/**/*' 5 | 6 | Style/StringLiterals: 7 | Enabled: true 8 | EnforcedStyle: double_quotes 9 | 10 | Style/StringLiteralsInInterpolation: 11 | Enabled: true 12 | EnforcedStyle: double_quotes 13 | 14 | Layout/SpaceBeforeBlockBraces: 15 | Enabled: true 16 | EnforcedStyle: 'space' 17 | 18 | Layout/SpaceInsideArrayLiteralBrackets: 19 | Enabled: true 20 | 21 | Layout/SpaceInsideReferenceBrackets: 22 | Enabled: true 23 | 24 | Layout/SpaceInsideHashLiteralBraces: 25 | Enabled: true 26 | 27 | Layout/SpaceInsideBlockBraces: 28 | Enabled: true 29 | 30 | Layout/SpaceAroundEqualsInParameterDefault: 31 | Enabled: true 32 | 33 | Layout/SpaceBeforeComma: 34 | Enabled: false 35 | 36 | Layout/SpaceAroundOperators: 37 | Enabled: true 38 | 39 | Layout/SpaceAfterComma: 40 | Enabled: true 41 | 42 | Layout/ExtraSpacing: 43 | Enabled: true 44 | AllowForAlignment: true 45 | 46 | Lint/DuplicateMethods: 47 | Enabled: true 48 | 49 | Layout/LineLength: 50 | Enabled: true 51 | Max: 200 52 | AllowHeredoc: true 53 | AllowURI: true 54 | URISchemes: http, https 55 | 56 | Metrics/MethodLength: 57 | Enabled: true 58 | Max: 25 59 | 60 | Metrics/ClassLength: 61 | Enabled: true 62 | Max: 160 63 | 64 | Metrics/ModuleLength: 65 | Enabled: true 66 | Max: 160 67 | 68 | Style/ClassAndModuleChildren: 69 | EnforcedStyle: compact 70 | Exclude: 71 | - lib/sidekiq/merger/version.rb 72 | 73 | Layout/ArrayAlignment: 74 | Enabled: true 75 | 76 | Layout/HashAlignment: 77 | Enabled: true 78 | 79 | Layout/BlockEndNewline: 80 | Enabled: false 81 | 82 | Style/DoubleNegation: 83 | Enabled: false 84 | 85 | Metrics/AbcSize: 86 | Max: 60 87 | 88 | Metrics/CyclomaticComplexity: 89 | Max: 12 90 | 91 | Metrics/PerceivedComplexity: 92 | Max: 12 93 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: ruby 3 | rvm: 4 | - 2.5.9 5 | - 2.6.7 6 | - 2.7.3 7 | gemfile: 8 | - gemfiles/sidekiq_4_0.gemfile 9 | - gemfiles/sidekiq_4_1.gemfile 10 | - gemfiles/sidekiq_4_2.gemfile 11 | - gemfiles/sidekiq_5_0.gemfile 12 | - gemfiles/sidekiq_5_1.gemfile 13 | services: 14 | - redis-server 15 | cache: bundler 16 | script: 17 | - "bundle exec rake spec" 18 | - "bundle exec rubocop -D" 19 | notifications: 20 | email: false 21 | -------------------------------------------------------------------------------- /Appraisals: -------------------------------------------------------------------------------- 1 | appraise "sidekiq-4-0" do 2 | gem "sidekiq", "~> 4.0.0" 3 | end 4 | 5 | appraise "sidekiq-4-1" do 6 | gem "sidekiq", "~> 4.1.0" 7 | end 8 | 9 | appraise "sidekiq-4-2" do 10 | gem "sidekiq", "~> 4.2.0" 11 | end 12 | 13 | appraise "sidekiq-5-0" do 14 | gem "sidekiq", "~> 5.0.0" 15 | end 16 | 17 | appraise "sidekiq-5-1" do 18 | gem "sidekiq", "~> 5.1.0" 19 | end -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.3.3 2 | MAINTAINER dtaniwaki 3 | 4 | ENV PORT 3000 5 | ENV REDIS_HOST 127.0.0.1 6 | ENV REDIS_PORT 6379 7 | 8 | RUN gem install bundler 9 | ADD . /gem 10 | WORKDIR /gem/app 11 | RUN bundle install -j4 12 | 13 | EXPOSE $PORT 14 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gemspec 4 | 5 | gem "gem-release" 6 | gem "pry" 7 | 8 | gem "sidekiq", "~> #{ENV["SIDEKIQ_VERSION"]}" unless ENV["SIDEKIQ_VERSION"].nil? 9 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 dtaniwaki 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sidekiq-merger 2 | 3 | [![Gem Version][gem-image]][gem-link] 4 | [![Dependency Status][deps-image]][deps-link] 5 | [![Build Status][build-image]][build-link] 6 | [![Coverage Status][cov-image]][cov-link] 7 | [![Code Climate][gpa-image]][gpa-link] 8 | 9 | [![Docker][docker-hub-image]][docker-hub-link] 10 | 11 | Merge [sidekiq](http://sidekiq.org/) jobs occurring before the execution times. Inspired by [sidekiq-grouping](https://github.com/gzigzigzeo/sidekiq-grouping). 12 | 13 | [Demo](http://sidekiq-merger.dtaniwaki.com/) 14 | 15 | ## Use Case 16 | 17 | ### Cancel Task 18 | 19 | ![Cancel Task](misc/cancel_task_flow.png) 20 | 21 | ### Bulk Notification 22 | 23 | ![Bulk Notification](misc/bulk_notification_flow.png) 24 | 25 | ## Installation 26 | 27 | Add this line to your application's Gemfile: 28 | 29 | ```ruby 30 | gem 'sidekiq-merger' 31 | ``` 32 | 33 | And then execute: 34 | 35 | $ bundle 36 | 37 | Or install it yourself as: 38 | 39 | $ gem install sidekiq-merger 40 | 41 | ## Usage 42 | 43 | Add merger option into your workers. 44 | 45 | ```ruby 46 | class SomeWorker 47 | include Sidekiq::Worker 48 | 49 | sidekiq_options merger: { key: -> (args) { args[0] } } 50 | 51 | def perform(*merged_args) 52 | merged_args.each do |args| 53 | # Do something 54 | end 55 | end 56 | end 57 | ``` 58 | 59 | Then, enqueue jobs by `perform_in` or `perform_at`. 60 | 61 | ```ruby 62 | SomeWorker.perform_in 100, 4 63 | SomeWorker.perform_in 100, 3 64 | SomeWorker.perform_in 100, 5 65 | # Passed 100 seconds from the first enqueue. 66 | SomeWorker.perform_in 100, 6 67 | SomeWorker.perform_in 100, 1 68 | ``` 69 | 70 | `SomeWorker` will be executed in 100 seconds with args of `[4], [3], [5]`, then with args of `[6], [1]`. 71 | 72 | `perform_async` works without merging args. 73 | 74 | ```ruby 75 | SomeWorker.perform_async 4 76 | SomeWorker.perform_async 3 77 | SomeWorker.perform_async 5 78 | ``` 79 | 80 | In this case, `SomeWorker` will be executed 3 times with args of `[4]`, `[3]` and `[5]`. 81 | 82 | ### Quick Check 83 | 84 | Run docker containers to check the behavior of this gem. 85 | 86 | $ docker-compose up 87 | 88 | Then, open `http://localhost:3000/`. You can push jobs from the UI and see what happens in the sidekiq console. 89 | 90 | ## Options 91 | 92 | ### `key` (optional, default: `nil`) 93 | 94 | Defines merge key so different arguments can be merged. 95 | 96 | Format: `String` or `Proc` 97 | 98 | e.g. `sidekiq_options merger: { key: -> (args) { args[0..1] } }` 99 | 100 | ### `unique` (optional, default: `false`) 101 | 102 | Prevents enqueue of jobs with identical arguments. 103 | 104 | Format: `Boolean` 105 | 106 | e.g. `true` 107 | 108 | ### `batch_size` (optional, default: `nil`) 109 | 110 | Allow to specify how many jobs max to provide as arguments per aggregation 111 | 112 | Format: `Int` 113 | 114 | e.g. `50` 115 | 116 | ## Web UI 117 | 118 | ![Web UI](misc/web_ui.png) 119 | 120 | Add this line to your `config/routes.rb` to activate web UI: 121 | 122 | ```ruby 123 | require "sidekiq/merger/web" 124 | ``` 125 | 126 | ## Test 127 | 128 | $ bundle exec appraisal rspec 129 | 130 | The test coverage is available at `./coverage/index.html`. 131 | 132 | ## Lint 133 | 134 | $ bundle exec appraisal rubocop 135 | 136 | ## Contributing 137 | 138 | 1. Fork it 139 | 2. Create your feature branch (`git checkout -b my-new-feature`) 140 | 3. Commit your changes (`git commit -am 'Add some feature'`) 141 | 4. Push to the branch (`git push origin my-new-feature`) 142 | 5. Create new [Pull Request](../../pull/new/master) 143 | 144 | ## Copyright 145 | 146 | Copyright (c) 2017 dtaniwaki. See [LICENSE](LICENSE) for details. 147 | 148 | [gem-image]: https://badge.fury.io/rb/sidekiq-merger.svg 149 | [gem-link]: http://badge.fury.io/rb/sidekiq-merger 150 | [build-image]: https://secure.travis-ci.org/dtaniwaki/sidekiq-merger.svg 151 | [build-link]: http://travis-ci.org/dtaniwaki/sidekiq-merger 152 | [deps-image]: https://gemnasium.com/dtaniwaki/sidekiq-merger.svg 153 | [deps-link]: https://gemnasium.com/dtaniwaki/sidekiq-merger 154 | [cov-image]: https://coveralls.io/repos/dtaniwaki/sidekiq-merger/badge.png 155 | [cov-link]: https://coveralls.io/r/dtaniwaki/sidekiq-merger 156 | [gpa-image]: https://codeclimate.com/github/dtaniwaki/sidekiq-merger.svg 157 | [gpa-link]: https://codeclimate.com/github/dtaniwaki/sidekiq-merger 158 | [docker-hub-image]: http://dockeri.co/image/dtaniwaki/sidekiq-merger 159 | [docker-hub-link]: https://hub.docker.com/r/dtaniwaki/sidekiq-merger/ 160 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | require "rubocop/rake_task" 4 | 5 | RSpec::Core::RakeTask.new(:spec) 6 | RuboCop::RakeTask.new(:rubocop) 7 | 8 | task default: [:rubocop, :spec] 9 | -------------------------------------------------------------------------------- /app/Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "rack-flash3" 4 | gem "sinatra" 5 | gem "sinatra-contrib" 6 | gem "sidekiq" 7 | gem "sidekiq-status" 8 | 9 | gem "sidekiq-merger", path: "../" 10 | -------------------------------------------------------------------------------- /app/Gemfile.lock: -------------------------------------------------------------------------------- 1 | PATH 2 | remote: ../ 3 | specs: 4 | sidekiq-merger (0.0.10) 5 | activesupport (>= 3.2, < 6) 6 | concurrent-ruby (~> 1.0) 7 | sidekiq (>= 3.4, < 5) 8 | 9 | GEM 10 | remote: https://rubygems.org/ 11 | specs: 12 | activesupport (5.0.1) 13 | concurrent-ruby (~> 1.0, >= 1.0.2) 14 | i18n (~> 0.7) 15 | minitest (~> 5.1) 16 | tzinfo (~> 1.1) 17 | backports (3.6.8) 18 | concurrent-ruby (1.0.4) 19 | connection_pool (2.2.1) 20 | i18n (0.8.0) 21 | minitest (5.10.1) 22 | multi_json (1.12.1) 23 | rack (1.6.5) 24 | rack-flash3 (1.0.5) 25 | rack 26 | rack-protection (1.5.3) 27 | rack 28 | rack-test (0.6.3) 29 | rack (>= 1.0) 30 | redis (3.3.3) 31 | sidekiq (4.2.9) 32 | concurrent-ruby (~> 1.0) 33 | connection_pool (~> 2.2, >= 2.2.0) 34 | rack-protection (>= 1.5.0) 35 | redis (~> 3.2, >= 3.2.1) 36 | sidekiq-status (0.6.0) 37 | sidekiq (>= 2.7) 38 | sinatra (1.4.8) 39 | rack (~> 1.5) 40 | rack-protection (~> 1.4) 41 | tilt (>= 1.3, < 3) 42 | sinatra-contrib (1.4.7) 43 | backports (>= 2.0) 44 | multi_json 45 | rack-protection 46 | rack-test 47 | sinatra (~> 1.4.0) 48 | tilt (>= 1.3, < 3) 49 | thread_safe (0.3.5) 50 | tilt (2.0.6) 51 | tzinfo (1.2.2) 52 | thread_safe (~> 0.1) 53 | 54 | PLATFORMS 55 | ruby 56 | 57 | DEPENDENCIES 58 | rack-flash3 59 | sidekiq 60 | sidekiq-merger! 61 | sidekiq-status 62 | sinatra 63 | sinatra-contrib 64 | 65 | BUNDLED WITH 66 | 1.13.6 67 | -------------------------------------------------------------------------------- /app/app.rb: -------------------------------------------------------------------------------- 1 | require_relative "./sidekiq" 2 | require "sinatra/base" 3 | require "sinatra/cookies" 4 | require "securerandom" 5 | require "rack/flash" 6 | require "sidekiq/web" 7 | require "sidekiq-status/web" 8 | require "sidekiq/merger/web" 9 | 10 | class App < Sinatra::Application 11 | enable :sessions 12 | use Rack::Flash 13 | 14 | before do 15 | @queue = cookies[:queue] ||= SecureRandom.urlsafe_base64(8) 16 | end 17 | 18 | get "/" do 19 | erb :index 20 | end 21 | 22 | post "/some_worker/perform_in" do 23 | n = rand(10) 24 | Sidekiq::Client.push( 25 | "queue" => @queue, 26 | "class" => SomeWorker, 27 | "args" => [n], 28 | "at" => Time.now + (params[:in] || 60) 29 | ) 30 | flash[:notice] = "Added #{n} to SomeWorker to Queue #{@queue}" 31 | redirect "/" 32 | end 33 | 34 | post "/some_worker/perform_async" do 35 | n = rand(10) 36 | Sidekiq::Client.push( 37 | "queue" => @queue, 38 | "class" => SomeWorker, 39 | "args" => [n] 40 | ) 41 | flash[:notice] = "Added #{n} to SomeWorker to Queue #{@queue}" 42 | redirect "/" 43 | end 44 | 45 | post "/unique_worker/perform_in" do 46 | n = rand(10) 47 | Sidekiq::Client.push( 48 | "queue" => @queue, 49 | "class" => UniqueWorker, 50 | "args" => [n], 51 | "at" => Time.now + (params[:in] || 60) 52 | ) 53 | flash[:notice] = "Added #{n} to UniqueWorker to Queue #{@queue}" 54 | redirect "/" 55 | end 56 | 57 | post "/unique_worker/perform_async" do 58 | n = rand(10) 59 | Sidekiq::Client.push( 60 | "queue" => @queue, 61 | "class" => UniqueWorker, 62 | "args" => [n] 63 | ) 64 | flash[:notice] = "Added #{n} to UniqueWorker to Queue #{@queue}" 65 | redirect "/" 66 | end 67 | end 68 | -------------------------------------------------------------------------------- /app/config.ru: -------------------------------------------------------------------------------- 1 | require "./app" 2 | 3 | run Rack::URLMap.new("/" => App, "/sidekiq" => Sidekiq::Web) 4 | -------------------------------------------------------------------------------- /app/sidekiq.rb: -------------------------------------------------------------------------------- 1 | require "active_support/core_ext/numeric/time" 2 | require "sidekiq" 3 | require "sidekiq-status" 4 | require "sidekiq-merger" 5 | require_relative "workers/some_worker" 6 | require_relative "workers/unique_worker" 7 | 8 | expiration = 30.minutes 9 | 10 | Sidekiq.configure_client do |config| 11 | config.redis = { url: "redis://#{ENV["REDIS_HOST"]}:#{ENV["REDIS_PORT"]}" } 12 | config.client_middleware do |chain| 13 | chain.add Sidekiq::Status::ClientMiddleware, expiration: expiration 14 | end 15 | end 16 | 17 | Sidekiq.configure_server do |config| 18 | config.redis = { url: "redis://#{ENV["REDIS_HOST"]}:#{ENV["REDIS_PORT"]}" } 19 | config.server_middleware do |chain| 20 | chain.add Sidekiq::Status::ServerMiddleware, expiration: expiration 21 | end 22 | config.client_middleware do |chain| 23 | chain.add Sidekiq::Status::ClientMiddleware, expiration: expiration 24 | end 25 | end 26 | -------------------------------------------------------------------------------- /app/views/index.erb: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | Sidekiq Merger 7 | 8 | 9 | 10 |
11 |
12 |

Sidekiq Merger

13 | View Source on GitHub → 14 |
15 |
16 |
17 | <% if flash[:notice] %> 18 |
19 | × 20 |

<%= flash[:notice] %>

21 |
22 | <% end %> 23 |

24 | Click the `perform_in` buttons to create or merge tasks until the execution time (in 60s).
25 | Click the `perform_async` buttons to execute a single task.

26 | Open " target="_blank">sidekiq console to check what happens. 27 |

28 |
29 |

SomeWorker

30 |

31 | sidekiq_options merger: { unique: false } 32 |

33 |

34 | Tasks will be merged regardless of uniqueness. 35 |

36 |
37 |
38 | 39 |
40 |
41 | 42 |
43 |
44 |

UniqueWorker

45 |

46 | sidekiq_options merger: { unique: true } 47 |

48 |

49 | Tasks will be merged if they haven't added already. 50 |

51 |
52 |
53 | 54 |
55 |
56 | 57 |
58 |
59 |
60 |
61 | 62 | 63 | 64 | 65 | -------------------------------------------------------------------------------- /app/workers/some_worker.rb: -------------------------------------------------------------------------------- 1 | class SomeWorker 2 | include Sidekiq::Worker 3 | 4 | sidekiq_options merger: { key: "foo" } 5 | 6 | def perform(*ids) 7 | puts "Get IDs: #{ids.inspect}" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /app/workers/unique_worker.rb: -------------------------------------------------------------------------------- 1 | class UniqueWorker 2 | include Sidekiq::Worker 3 | 4 | sidekiq_options merger: { key: "foo", unique: true } 5 | 6 | def perform(*ids) 7 | puts "Get IDs: #{ids.inspect}" 8 | end 9 | end 10 | -------------------------------------------------------------------------------- /docker-compose-common.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | app: 4 | build: . 5 | environment: 6 | - REDIS_HOST=redis 7 | - REDIS_PORT=6379 8 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '2.1' 2 | services: 3 | worker: 4 | extends: 5 | file: 'docker-compose-common.yml' 6 | service: app 7 | command: bundle exec sidekiq -r ./sidekiq.rb 8 | links: 9 | - redis 10 | environment: 11 | - REDIS_HOST=redis 12 | - REDIS_PORT=6379 13 | web: 14 | extends: 15 | file: 'docker-compose-common.yml' 16 | service: app 17 | command: bundle exec rackup -p 3000 --host 0.0.0.0 18 | ports: 19 | - 3000:3000 20 | links: 21 | - redis 22 | environment: 23 | - PORT=3000 24 | - REDIS_HOST=redis 25 | - REDIS_PORT=6379 26 | redis: 27 | image: redis:3.2.7 28 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_4_0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "gem-release" 6 | gem "pry" 7 | gem "sidekiq", "~> 4.0.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_4_1.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "gem-release" 6 | gem "pry" 7 | gem "sidekiq", "~> 4.1.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_4_2.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "gem-release" 6 | gem "pry" 7 | gem "sidekiq", "~> 4.2.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_5_0.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "gem-release" 6 | gem "pry" 7 | gem "sidekiq", "~> 5.0.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /gemfiles/sidekiq_5_1.gemfile: -------------------------------------------------------------------------------- 1 | # This file was generated by Appraisal 2 | 3 | source "https://rubygems.org" 4 | 5 | gem "gem-release" 6 | gem "pry" 7 | gem "sidekiq", "~> 5.1.0" 8 | 9 | gemspec path: "../" 10 | -------------------------------------------------------------------------------- /lib/sidekiq-merger.rb: -------------------------------------------------------------------------------- 1 | require_relative "sidekiq/merger" 2 | 3 | Sidekiq.configure_client do |config| 4 | config.client_middleware do |chain| 5 | chain.add Sidekiq::Merger::Middleware 6 | end 7 | end 8 | 9 | Sidekiq.configure_server do |config| 10 | config.client_middleware do |chain| 11 | chain.add Sidekiq::Merger::Middleware 12 | end 13 | end 14 | 15 | if Sidekiq.server? 16 | task = Sidekiq::Merger.create_task 17 | task.execute 18 | end 19 | -------------------------------------------------------------------------------- /lib/sidekiq/merger.rb: -------------------------------------------------------------------------------- 1 | require "sidekiq" 2 | require "concurrent" 3 | require_relative "merger/version" 4 | require_relative "merger/middleware" 5 | require_relative "merger/config" 6 | require_relative "merger/flusher" 7 | require_relative "merger/logging_observer" 8 | 9 | module Sidekiq::Merger 10 | LOGGER_TAG = self.name.freeze 11 | 12 | class << self 13 | attr_accessor :logger 14 | 15 | def create_task 16 | interval = Sidekiq::Merger::Config.poll_interval 17 | observer = Sidekiq::Merger::LoggingObserver.new(logger) 18 | flusher = Sidekiq::Merger::Flusher.new(logger) 19 | task = Concurrent::TimerTask.new( 20 | execution_interval: interval 21 | ) { flusher.flush } 22 | task.add_observer(observer) 23 | task 24 | end 25 | 26 | def configure(&block) 27 | yield config 28 | end 29 | 30 | def config 31 | @config ||= Config.new 32 | end 33 | end 34 | 35 | self.logger = Sidekiq.logger 36 | end 37 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/config.rb: -------------------------------------------------------------------------------- 1 | require "active_support/configurable" 2 | 3 | class Sidekiq::Merger::Config 4 | include ActiveSupport::Configurable 5 | 6 | def self.options 7 | Sidekiq.options["merger"] || {} 8 | end 9 | 10 | config_accessor :poll_interval do 11 | options[:poll_interval] || 5 12 | end 13 | 14 | config_accessor :lock_ttl do 15 | options[:lock_ttl] || 1 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/flusher.rb: -------------------------------------------------------------------------------- 1 | class Sidekiq::Merger::Flusher 2 | def initialize(logger) 3 | @logger = logger 4 | end 5 | 6 | def flush 7 | merges = Sidekiq::Merger::Merge.all.select(&:can_flush?) 8 | unless merges.empty? 9 | @logger.info( 10 | "[Sidekiq::Merger] Trying to flush merged queues: #{merges.map(&:full_merge_key).join(",")}" 11 | ) 12 | merges.each(&:flush) 13 | end 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/logging_observer.rb: -------------------------------------------------------------------------------- 1 | class Sidekiq::Merger::LoggingObserver 2 | def initialize(logger) 3 | @logger = logger 4 | end 5 | 6 | def update(time, _result, ex) 7 | if ex.is_a?(Concurrent::TimeoutError) 8 | @logger.error( 9 | "[#{Sidekiq::Merger::LOGGER_TAG}] Execution timed out\n" 10 | ) 11 | elsif ex.present? 12 | @logger.error( 13 | "[#{Sidekiq::Merger::LOGGER_TAG}] Execution failed with error #{ex}\n" 14 | ) 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/merge.rb: -------------------------------------------------------------------------------- 1 | require_relative "redis" 2 | require "active_support/core_ext/hash/indifferent_access" 3 | 4 | class Sidekiq::Merger::Merge 5 | class << self 6 | def all 7 | redis = Sidekiq::Merger::Redis.new 8 | 9 | redis.all_merges.map { |full_merge_key| initialize_with_full_merge_key(full_merge_key, redis: redis) } 10 | end 11 | 12 | def initialize_with_full_merge_key(full_merge_key, options = {}) 13 | keys = full_merge_key.split(":") 14 | raise "Invalid merge key" if keys.size < 3 15 | worker_class = keys[0].camelize.constantize 16 | queue = keys[1] 17 | merge_key = keys[2] 18 | new(worker_class, queue, merge_key, options) 19 | end 20 | 21 | def initialize_with_args(worker_class, queue, args, options = {}) 22 | new(worker_class, queue, merge_key(worker_class, args), options) 23 | end 24 | 25 | def merge_key(worker_class, args) 26 | options = get_options(worker_class) 27 | merge_key = options["key"] 28 | if merge_key.respond_to?(:call) 29 | merge_key = merge_key.call(args) 30 | end 31 | merge_key = "" if merge_key.nil? 32 | merge_key = merge_key.to_json unless merge_key.is_a?(String) 33 | merge_key 34 | end 35 | 36 | def get_options(worker_class) 37 | (worker_class.get_sidekiq_options["merger"] || {}).with_indifferent_access 38 | end 39 | end 40 | 41 | attr_reader :worker_class, :queue, :merge_key 42 | 43 | def initialize(worker_class, queue, merge_key, redis: Sidekiq::Merger::Redis.new) 44 | @worker_class = worker_class 45 | @queue = queue 46 | @merge_key = merge_key 47 | @redis = redis 48 | end 49 | 50 | def add(args, execution_time) 51 | if !options[:unique] || !@redis.merge_exists?(full_merge_key, args) 52 | @redis.push_message(full_merge_key, args, execution_time) 53 | end 54 | end 55 | 56 | def delete(args) 57 | @redis.delete_message(full_merge_key, args) 58 | end 59 | 60 | def delete_all 61 | @redis.delete_merge(full_merge_key) 62 | end 63 | 64 | def size 65 | @redis.merge_size(full_merge_key) 66 | end 67 | 68 | def flush 69 | msgs = [] 70 | 71 | if @redis.lock_merge(full_merge_key, Sidekiq::Merger::Config.lock_ttl) 72 | msgs = @redis.pluck_merge(full_merge_key) 73 | end 74 | 75 | unless msgs.empty? 76 | batches = options[:batch_size].nil? ? [msgs] : msgs.each_slice(options[:batch_size].to_i).to_a 77 | batches.each do |batch_msgs| 78 | # preserve FIFO when enqueuing batches 79 | Sidekiq::Client.push( 80 | "class" => worker_class, 81 | "queue" => queue, 82 | "args" => batch_msgs, 83 | "merged" => true 84 | ) 85 | end 86 | end 87 | end 88 | 89 | def can_flush? 90 | !execution_time.nil? && execution_time < Time.now 91 | end 92 | 93 | def full_merge_key 94 | @full_merge_key ||= [worker_class.name.to_s.underscore, queue, merge_key].join(":") 95 | end 96 | 97 | def all_args 98 | @redis.get_merge(full_merge_key) 99 | end 100 | 101 | def execution_time 102 | @execution_time ||= @redis.merge_execution_time(full_merge_key) 103 | end 104 | 105 | def ==(other) 106 | self.worker_class == other.worker_class && 107 | self.queue == other.queue && 108 | self.merge_key == other.merge_key 109 | end 110 | 111 | private 112 | 113 | def options 114 | @options ||= self.class.get_options(worker_class) 115 | rescue NameError 116 | {} 117 | end 118 | end 119 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/middleware.rb: -------------------------------------------------------------------------------- 1 | require_relative "merge" 2 | 3 | class Sidekiq::Merger::Middleware 4 | def call(worker_class, msg, queue, _ = nil) 5 | return yield if defined?(Sidekiq::Testing) && Sidekiq::Testing.inline? 6 | 7 | worker_class = worker_class.camelize.constantize if worker_class.is_a?(String) 8 | options = worker_class.get_sidekiq_options 9 | 10 | merger_enabled = options.key?("merger") 11 | 12 | return yield unless merger_enabled 13 | 14 | if !msg["at"].nil? && msg["at"].to_f > Time.now.to_f 15 | Sidekiq::Merger::Merge 16 | .initialize_with_args(worker_class, queue, msg["args"]) 17 | .add(msg["args"], msg["at"]) 18 | false 19 | else 20 | msg["args"] = [msg["args"].flatten] unless msg.delete("merged") 21 | yield 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/redis.rb: -------------------------------------------------------------------------------- 1 | require "active_support/core_ext/module/delegation" 2 | 3 | class Sidekiq::Merger::Redis 4 | class << self 5 | KEY_PREFIX = "sidekiq-merger".freeze 6 | 7 | def purge! 8 | redis do |conn| 9 | conn.eval 10 | script = <<-SCRIPT 11 | for i=1, #ARGV do 12 | redis.call('del', unpack(redis.call('keys', ARGV[i]))) 13 | end 14 | return true 15 | SCRIPT 16 | conn.eval(script, [], [merges_key, unique_msg_key("*"), msg_key("*"), lock_key("*")]) 17 | end 18 | end 19 | 20 | def merges_key 21 | "#{KEY_PREFIX}:merges" 22 | end 23 | 24 | def unique_msg_key(key) 25 | "#{KEY_PREFIX}:unique_msg:#{key}" 26 | end 27 | 28 | def msg_key(key) 29 | "#{KEY_PREFIX}:msg:#{key}" 30 | end 31 | 32 | def time_key(key) 33 | "#{KEY_PREFIX}:time:#{key}" 34 | end 35 | 36 | def lock_key(key) 37 | "#{KEY_PREFIX}:lock:#{key}" 38 | end 39 | 40 | def redis(&block) 41 | Sidekiq.redis(&block) 42 | end 43 | end 44 | 45 | def push_message(key, msg, execution_time) 46 | msg_json = msg.to_json 47 | redis do |conn| 48 | conn.multi do 49 | conn.sadd(merges_key, key) 50 | conn.setnx(time_key(key), execution_time.to_i) 51 | conn.lpush(msg_key(key), msg_json) 52 | conn.sadd(unique_msg_key(key), msg_json) 53 | end 54 | end 55 | end 56 | 57 | def delete_message(key, msg) 58 | msg_json = msg.to_json 59 | redis do |conn| 60 | conn.multi do 61 | conn.srem(unique_msg_key(key), msg_json) 62 | conn.lrem(msg_key(key), 0, msg_json) 63 | end 64 | end 65 | end 66 | 67 | def merge_execution_time(key) 68 | redis do |conn| 69 | t = conn.get(time_key(key)) 70 | Time.at(t.to_i) unless t.nil? 71 | end 72 | end 73 | 74 | def merge_size(key) 75 | redis { |conn| conn.llen(msg_key(key)) } 76 | end 77 | 78 | def merge_exists?(key, msg) 79 | msg_json = msg.to_json 80 | redis { |conn| conn.sismember(unique_msg_key(key), msg_json) } 81 | end 82 | 83 | def all_merges 84 | redis { |conn| conn.smembers(merges_key) } 85 | end 86 | 87 | def lock_merge(key, ttl) 88 | redis { |conn| conn.set(lock_key(key), true, nx: true, ex: ttl) } 89 | end 90 | 91 | def get_merge(key) 92 | msgs = [] 93 | redis { |conn| msgs = conn.lrange(msg_key(key), 0, -1) } 94 | msgs.map { |msg| JSON.parse(msg) } 95 | end 96 | 97 | def pluck_merge(key) 98 | msgs = [] 99 | redis do |conn| 100 | conn.multi do 101 | msgs = conn.lrange(msg_key(key), 0, -1) 102 | conn.del(unique_msg_key(key)) 103 | conn.del(msg_key(key)) 104 | conn.del(time_key(key)) 105 | conn.srem(merges_key, key) 106 | end 107 | end 108 | extract_future_value(msgs).map { |msg| JSON.parse(msg) } 109 | end 110 | 111 | def delete_merge(key) 112 | redis do |conn| 113 | conn.multi do 114 | conn.del(unique_msg_key(key)) 115 | conn.del(msg_key(key)) 116 | conn.del(time_key(key)) 117 | conn.del(lock_key(key)) 118 | conn.srem(merges_key, key) 119 | end 120 | end 121 | end 122 | 123 | private 124 | 125 | delegate :merges_key, :msg_key, :unique_msg_key, :time_key, :lock_key, :redis, to: "self.class" 126 | 127 | def extract_future_value(future) 128 | while future.value.is_a?(Redis::FutureNotReady) 129 | sleep(0.001) 130 | end 131 | future.value 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/version.rb: -------------------------------------------------------------------------------- 1 | module Sidekiq 2 | module Merger 3 | VERSION = "0.1.0".freeze 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/views/index.erb: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Merged jobs

4 |
5 |
6 | 7 |
8 |
9 |
10 | <% if true %> 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | <% @merges.each do |merge| %> 21 | 22 | 23 | 24 | 25 | 26 | 27 | 33 | 34 | <% end %> 35 |
WorkerQueueCountAll ArgsExecution timeActions
<%= merge.worker_class %><%= merge.queue %><%= merge.size %><%= merge.all_args %><%= merge.execution_time || "–"%> 28 |
" method="post"> 29 | <%= csrf_tag %> 30 | 31 |
32 |
36 | <% else %> 37 |
No recurring jobs found.
38 | <% end %> 39 |
40 |
41 |
42 | -------------------------------------------------------------------------------- /lib/sidekiq/merger/web.rb: -------------------------------------------------------------------------------- 1 | require "sidekiq/web" 2 | 3 | module Sidekiq::Merger::Web 4 | VIEWS = File.expand_path("views", File.dirname(__FILE__)) 5 | 6 | def self.registered(app) 7 | app.get "/merges" do 8 | @merges = Sidekiq::Merger::Merge.all 9 | @merges.select! { |m| m.queue == params[:queue] } unless params[:queue].nil? 10 | erb File.read(File.join(VIEWS, "index.erb")), locals: { view_path: VIEWS } 11 | end 12 | 13 | app.post "/merges/:full_merge_key/delete" do 14 | full_merge_key = URI.decode_www_form_component params[:full_merge_key] 15 | merge = Sidekiq::Merger::Merge.initialize_with_full_merge_key(full_merge_key) 16 | merge.delete_all 17 | redirect "#{root_path}/merges" 18 | end 19 | end 20 | end 21 | 22 | Sidekiq::Web.register(Sidekiq::Merger::Web) 23 | Sidekiq::Web.tabs["Merges"] = "merges" 24 | -------------------------------------------------------------------------------- /misc/bulk_notification_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtaniwaki/sidekiq-merger/b7f67d386dca9631bdc263652412c965b671150d/misc/bulk_notification_flow.png -------------------------------------------------------------------------------- /misc/cancel_task_flow.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtaniwaki/sidekiq-merger/b7f67d386dca9631bdc263652412c965b671150d/misc/cancel_task_flow.png -------------------------------------------------------------------------------- /misc/web_ui.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtaniwaki/sidekiq-merger/b7f67d386dca9631bdc263652412c965b671150d/misc/web_ui.png -------------------------------------------------------------------------------- /sidekiq-merger.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path("../lib", __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require "sidekiq/merger/version" 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "sidekiq-merger" 8 | spec.version = Sidekiq::Merger::VERSION 9 | spec.platform = Gem::Platform::RUBY 10 | spec.authors = ["dtaniwaki"] 11 | spec.email = ["daisuketaniwaki@gmail.com"] 12 | 13 | spec.summary = "Sidekiq merger plugin" 14 | spec.description = "Merge sidekiq jobs." 15 | spec.homepage = "https://github.com/dtaniwaki/sidekiq-merger" 16 | spec.license = "MIT" 17 | 18 | spec.files = `git ls-files -z`.split("\x0").reject do |f| 19 | f.match(%r{^(test|spec|features)/}) 20 | end 21 | spec.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 22 | spec.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } 23 | spec.require_paths = ["lib"] 24 | 25 | spec.required_ruby_version = [">= 2.5.0"] 26 | 27 | spec.add_development_dependency "rake", ">= 10.0", "< 13" 28 | spec.add_development_dependency "rspec", ">= 3.0", "< 4" 29 | spec.add_development_dependency "simplecov", "~> 0.12" 30 | spec.add_development_dependency "timecop", "~> 0.8" 31 | spec.add_development_dependency "rubocop", "~> 0.93.1" 32 | spec.add_development_dependency "coveralls", "~> 0.8" 33 | spec.add_development_dependency "appraisal" 34 | 35 | spec.add_runtime_dependency "sidekiq", ">= 4.0", "< 6" 36 | spec.add_runtime_dependency "concurrent-ruby", "~> 1.0" 37 | spec.add_runtime_dependency "activesupport", ">= 3.2", "< 6" 38 | end 39 | -------------------------------------------------------------------------------- /spec/sidekiq/merger/flusher_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Sidekiq::Merger::Flusher do 4 | subject { described_class.new(Sidekiq.logger) } 5 | 6 | describe "#call" do 7 | let(:active_merge) { double(full_merge_key: "active", can_flush?: true, flush: nil) } 8 | let(:inactive_merge) { double(full_merge_key: "inactive", can_flush?: false, flush: nil) } 9 | let(:merges) { [active_merge, inactive_merge] } 10 | it "adds the args to the merge" do 11 | allow(Sidekiq::Merger::Merge).to receive(:all).and_return merges 12 | expect(active_merge).to receive(:flush) 13 | expect(inactive_merge).not_to receive(:flush) 14 | 15 | subject.flush 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/sidekiq/merger/logging_observer_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Sidekiq::Merger::LoggingObserver do 4 | subject { described_class.new(logger) } 5 | let(:logger) { Logger.new("/dev/null") } 6 | let(:now) { Time.now } 7 | before { Timecop.freeze(now) } 8 | 9 | describe "#update" do 10 | it "logs a timeout" do 11 | expect(logger).to receive(:error).with("[Sidekiq::Merger] Execution timed out\n") 12 | subject.update(now, nil, Concurrent::TimeoutError.new) 13 | end 14 | it "logs an error" do 15 | expect(logger).to receive(:error).with("[Sidekiq::Merger] Execution failed with error foo\n") 16 | subject.update(now, nil, "foo") 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /spec/sidekiq/merger/merge_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Sidekiq::Merger::Merge, worker_class: true do 4 | subject { described_class.new(worker_class, queue, args, redis: redis) } 5 | let(:args) { "foo" } 6 | let(:redis) { Sidekiq::Merger::Redis.new } 7 | let(:queue) { "queue" } 8 | let(:now) { Time.now } 9 | let(:execution_time) { now + 10.seconds } 10 | let(:worker_options) { { key: -> (args) { args.to_json } } } 11 | before { Timecop.freeze(now) } 12 | 13 | describe ".all" do 14 | it "returns all the keys" do 15 | redis.redis do |conn| 16 | conn.sadd("sidekiq-merger:merges", "string:foo:xxx") 17 | conn.sadd("sidekiq-merger:merges", "numeric:bar:yyy") 18 | end 19 | 20 | expect(described_class.all).to contain_exactly( 21 | described_class.new(String, "foo", "xxx"), 22 | described_class.new(Numeric, "bar", "yyy") 23 | ) 24 | end 25 | 26 | context "including invalid key" do 27 | it "raises an error" do 28 | redis.redis do |conn| 29 | conn.sadd("sidekiq-merger:merges", "string:foo:xxx") 30 | conn.sadd("sidekiq-merger:merges", "invalid") 31 | end 32 | expect { 33 | described_class.all 34 | }.to raise_error RuntimeError, "Invalid merge key" 35 | end 36 | end 37 | end 38 | 39 | describe ".initialize_with_args" do 40 | it "provides merge_key from args" do 41 | expect(described_class).to receive(:new).with(worker_class, queue, "[1,2,3]", anything) 42 | described_class.initialize_with_args(worker_class, queue, [1, 2, 3]) 43 | end 44 | it "passes options" do 45 | expect(described_class).to receive(:new).with(worker_class, queue, anything, { redis: 1 }) 46 | described_class.initialize_with_args(worker_class, queue, anything, redis: 1) 47 | end 48 | end 49 | 50 | describe ".merge_key" do 51 | let(:args) { "foo" } 52 | let(:worker_options) { {} } 53 | it "returns an empty string" do 54 | expect(described_class.merge_key(worker_class, args)).to eq "" 55 | end 56 | context "string key" do 57 | let(:worker_options) { { key: "bar" } } 58 | it "returns the string" do 59 | expect(described_class.merge_key(worker_class, args)).to eq "bar" 60 | end 61 | end 62 | context "other type key" do 63 | let(:worker_options) { { key: [1, 2, 3] } } 64 | it "returns nil" do 65 | expect(described_class.merge_key(worker_class, args)).to eq "[1,2,3]" 66 | end 67 | end 68 | context "proc key" do 69 | let(:args) { [1, 2, 3] } 70 | let(:worker_options) { { key: -> (args) { args[0].to_s } } } 71 | it "returns the result of the proc" do 72 | expect(described_class.merge_key(worker_class, args)).to eq "1" 73 | end 74 | context "non-string result" do 75 | let(:worker_options) { { key: -> (args) { args[0] } } } 76 | it "returns nil" do 77 | expect(described_class.merge_key(worker_class, args)).to eq "1" 78 | end 79 | end 80 | end 81 | end 82 | 83 | describe "#add" do 84 | it "adds the args in lazy merge" do 85 | expect(redis).to receive(:push_message).with("some_worker:queue:foo", [1, 2, 3], execution_time) 86 | subject.add([1, 2, 3], execution_time) 87 | end 88 | context "with unique option" do 89 | let(:worker_options) { { key: -> (args) { args.to_json }, unique: true } } 90 | it "adds the args in lazy merge" do 91 | expect(redis).to receive(:push_message).with("some_worker:queue:foo", [1, 2, 3], execution_time) 92 | subject.add([1, 2, 3], execution_time) 93 | end 94 | context "the args has alredy been added" do 95 | before { subject.add([1, 2, 3], execution_time) } 96 | it "adds the args in lazy merge" do 97 | expect(redis).not_to receive(:push_message) 98 | subject.add([1, 2, 3], execution_time) 99 | end 100 | end 101 | end 102 | end 103 | 104 | describe "#delete" do 105 | it "adds the args in lazy merge" do 106 | expect(redis).to receive(:delete_message).with("some_worker:queue:foo", [1, 2, 3]) 107 | subject.delete([1, 2, 3]) 108 | end 109 | end 110 | 111 | describe "#delete_all" do 112 | before do 113 | subject.add([1, 2, 3], execution_time) 114 | subject.add([2, 3, 4], execution_time) 115 | end 116 | it "deletes all" do 117 | expect { 118 | subject.delete_all 119 | }.to change { subject.size }.from(2).to(0) 120 | end 121 | end 122 | 123 | describe "#size" do 124 | before do 125 | subject.add([1, 2, 3], execution_time) 126 | subject.add([2, 3, 4], execution_time) 127 | end 128 | it "returns the size" do 129 | expect(subject.size).to eq 2 130 | end 131 | end 132 | 133 | describe "#all_args" do 134 | before do 135 | subject.add([1, 2, 3], execution_time) 136 | subject.add([2, 3, 4], execution_time) 137 | end 138 | it "returns all args" do 139 | expect(subject.all_args).to contain_exactly [1, 2, 3], [2, 3, 4] 140 | end 141 | end 142 | 143 | describe "#flush" do 144 | context "when no batch_size is configured" do 145 | before do 146 | subject.add([1, 2, 3], execution_time) 147 | subject.add([2, 3, 4], execution_time) 148 | end 149 | it "flushes all the args" do 150 | expect(Sidekiq::Client).to receive(:push).with( 151 | "class" => worker_class, 152 | "queue" => queue, 153 | "args" => a_collection_containing_exactly([1, 2, 3], [2, 3, 4]), 154 | "merged" => true 155 | ) 156 | 157 | subject.flush 158 | end 159 | end 160 | 161 | context "when batch_size is configured to 2" do 162 | let(:worker_options) { { key: -> (args) { args.to_json }, batch_size: 2 } } 163 | before do 164 | subject.add([1, 2, 3], execution_time) 165 | subject.add([2, 3, 4], execution_time) 166 | subject.add([3, 4, 5], execution_time) 167 | subject.add([4, 5, 6], execution_time) 168 | end 169 | it "flushes all the args" do 170 | expect(Sidekiq::Client).to receive(:push).with( 171 | "class" => worker_class, 172 | "queue" => queue, 173 | "args" => a_collection_containing_exactly([1, 2, 3], [2, 3, 4]), 174 | "merged" => true 175 | ) 176 | 177 | expect(Sidekiq::Client).to receive(:push).with( 178 | "class" => worker_class, 179 | "queue" => queue, 180 | "args" => a_collection_containing_exactly([3, 4, 5], [4, 5, 6]), 181 | "merged" => true 182 | ) 183 | 184 | subject.flush 185 | end 186 | end 187 | end 188 | 189 | describe "#can_flush?" do 190 | context "it has not get anything in merge" do 191 | it "returns false" do 192 | expect(subject.can_flush?).to eq false 193 | end 194 | end 195 | context "it has not passed the execution time" do 196 | it "returns false" do 197 | subject.add([], execution_time) 198 | expect(subject.can_flush?).to eq false 199 | end 200 | end 201 | context "it has passed the execution time" do 202 | it "returns true" do 203 | subject.add([], execution_time) 204 | Timecop.travel(10.seconds) 205 | expect(subject.can_flush?).to eq true 206 | end 207 | end 208 | end 209 | 210 | describe "#full_merge_key" do 211 | it "returns full merge key" do 212 | expect(subject.full_merge_key).to eq "some_worker:queue:foo" 213 | end 214 | end 215 | end 216 | -------------------------------------------------------------------------------- /spec/sidekiq/merger/middleware_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Sidekiq::Merger::Middleware, worker_class: true do 4 | subject { described_class.new } 5 | let(:flusher) { Sidekiq::Merger::Flusher.new(Sidekiq.logger) } 6 | let(:queue) { "queue" } 7 | let(:now) { Time.now } 8 | before :example do 9 | Timecop.freeze(now) 10 | end 11 | 12 | describe "#call" do 13 | context "non-merger worker" do 14 | it "leaves args alone" do 15 | msg = { "args" => [1, 2, 3] } 16 | expect { |b| subject.call(non_merge_worker_class, msg, queue, &b) }.to yield_with_no_args 17 | expect(msg).to eq({ "args" => [1, 2, 3] }) #unmodified 18 | flusher.flush 19 | expect(worker_class.jobs.size).to eq 0 20 | end 21 | end 22 | it "adds the args to the merge" do 23 | subject.call(worker_class, { "args" => [1, 2, 3], "at" => (now + 10.seconds).to_f }, queue) {} 24 | subject.call(worker_class, { "args" => [2, 3, 4], "at" => (now + 15.seconds).to_f }, queue) {} 25 | flusher.flush 26 | expect(worker_class.jobs.size).to eq 0 27 | Timecop.travel(10.seconds) 28 | flusher.flush 29 | expect(worker_class.jobs.size).to eq 1 30 | job = worker_class.jobs[0] 31 | expect(job["queue"]).to eq queue 32 | expect(job["args"]).to contain_exactly [1, 2, 3], [2, 3, 4] 33 | end 34 | context "without at msg" do 35 | it "peforms now with brackets" do 36 | msg = { "args" => [1, 2, 3] } 37 | expect { |b| subject.call(worker_class, msg, queue, &b) }.to yield_with_no_args 38 | expect(msg).to eq({ "args" => [[1, 2, 3]] }) 39 | flusher.flush 40 | expect(worker_class.jobs.size).to eq 0 41 | end 42 | context "merged msgs" do 43 | it "performs now" do 44 | msg = { "args" => [[1, 2, 3]], "merged" => true } 45 | expect { |b| subject.call(worker_class, msg, queue, &b) }.to yield_with_no_args 46 | expect(msg).to eq({ "args" => [[1, 2, 3]] }) 47 | flusher.flush 48 | expect(worker_class.jobs.size).to eq 0 49 | end 50 | end 51 | end 52 | context "at is before current time" do 53 | it "peforms now" do 54 | msg = { "args" => [1, 2, 3], "at" => now.to_f } 55 | expect { |b| subject.call(worker_class, msg, queue, &b) }.to yield_with_no_args 56 | expect(msg).to eq({ "args" => [[1, 2, 3]], "at" => now.to_f }) 57 | flusher.flush 58 | expect(worker_class.jobs.size).to eq 0 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /spec/sidekiq/merger/redis_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Sidekiq::Merger::Redis do 4 | subject { described_class.new } 5 | let(:now) { Time.now } 6 | let(:execution_time) { now + 10.seconds } 7 | before { Timecop.freeze(now) } 8 | 9 | describe ".purge" do 10 | it "cleans up all the keys" do 11 | described_class.redis do |conn| 12 | conn.sadd("sidekiq-merger:merges", "test") 13 | conn.set("sidekiq-merger:unique_msg:foo", "test") 14 | conn.set("sidekiq-merger:msg:foo", "test") 15 | conn.set("sidekiq-merger:lock:foo", "test") 16 | end 17 | 18 | described_class.purge! 19 | 20 | described_class.redis do |conn| 21 | expect(conn.smembers("sidekiq-merger:merges")).to be_empty 22 | expect(conn.keys("sidekiq-merger:unique_msg:*")).to be_empty 23 | expect(conn.keys("sidekiq-merger:msg:*")).to be_empty 24 | expect(conn.keys("sidekiq-merger:lock:*")).to be_empty 25 | end 26 | end 27 | end 28 | 29 | describe "#push_message" do 30 | shared_examples_for "push_message spec" do 31 | it "pushes the msg" do 32 | subject.push_message(pushing_key, pushing_msg, pushing_execution_time) 33 | described_class.redis do |conn| 34 | expect(conn.smembers("sidekiq-merger:merges")).to contain_exactly(*merge_keys) 35 | expect(conn.keys("sidekiq-merger:time:*")).to contain_exactly(*times.keys) 36 | expect(conn.keys("sidekiq-merger:unique_msg:*")).to contain_exactly(*unique_msgs_h.keys) 37 | unique_msgs_h.each do |key, msgs| 38 | expect(conn.smembers(key)).to contain_exactly(*msgs) 39 | end 40 | expect(conn.keys("sidekiq-merger:msg:*")).to contain_exactly(*msgs_h.keys) 41 | msgs_h.each do |key, msgs| 42 | expect(conn.lrange(key, 0, -1)).to contain_exactly(*msgs) 43 | end 44 | end 45 | end 46 | it "sets the execution time" do 47 | subject.push_message(pushing_key, pushing_msg, pushing_execution_time) 48 | described_class.redis do |conn| 49 | merge_keys.each do |key, time| 50 | expect(conn.get(key)).to eq time 51 | end 52 | end 53 | end 54 | end 55 | 56 | let(:pushing_key) { "foo" } 57 | let(:pushing_msg) { [1, 2, 3] } 58 | let(:pushing_execution_time) { execution_time } 59 | 60 | include_examples "push_message spec" do 61 | let(:merge_keys) { ["foo"] } 62 | let(:times) { { 63 | "sidekiq-merger:time:foo" => execution_time.to_i.to_s, 64 | } } 65 | let(:unique_msgs_h) { { 66 | "sidekiq-merger:unique_msg:foo" => ["[1,2,3]"] 67 | } } 68 | let(:msgs_h) { { 69 | "sidekiq-merger:msg:foo" => ["[1,2,3]"] 70 | } } 71 | end 72 | 73 | context "the merge key already exists" do 74 | let(:pushing_msg) { [2, 3, 4] } 75 | before { subject.push_message("foo", [1, 2, 3], execution_time) } 76 | include_examples "push_message spec" do 77 | let(:merge_keys) { ["foo"] } 78 | let(:times) { { 79 | "sidekiq-merger:time:foo" => execution_time.to_i.to_s, 80 | } } 81 | let(:unique_msgs_h) { { 82 | "sidekiq-merger:unique_msg:foo" => ["[1,2,3]", "[2,3,4]"] 83 | } } 84 | let(:msgs_h) { { 85 | "sidekiq-merger:msg:foo" => ["[1,2,3]", "[2,3,4]"] 86 | } } 87 | end 88 | end 89 | 90 | context "the msg has already ben pushed" do 91 | before { subject.push_message("foo", [1, 2, 3], execution_time) } 92 | include_examples "push_message spec" do 93 | let(:merge_keys) { ["foo"] } 94 | let(:times) { { 95 | "sidekiq-merger:time:foo" => execution_time.to_i.to_s, 96 | } } 97 | let(:unique_msgs_h) { { 98 | "sidekiq-merger:unique_msg:foo" => ["[1,2,3]"] 99 | } } 100 | let(:msgs_h) { { 101 | "sidekiq-merger:msg:foo" => ["[1,2,3]", "[1,2,3]"] 102 | } } 103 | end 104 | end 105 | 106 | context "other merge key already exists" do 107 | let(:pushing_key) { "bar" } 108 | let(:pushing_msg) { [2, 3, 4] } 109 | let(:pushing_execution_time) { execution_time + 1.hour } 110 | before { subject.push_message("foo", [1, 2, 3], execution_time) } 111 | include_examples "push_message spec" do 112 | let(:merge_keys) { ["foo", "bar"] } 113 | let(:times) { { 114 | "sidekiq-merger:time:foo" => execution_time.to_i.to_s, 115 | "sidekiq-merger:time:bar" => (execution_time + 1.hour).to_i.to_s, 116 | } } 117 | let(:unique_msgs_h) { { 118 | "sidekiq-merger:unique_msg:foo" => ["[1,2,3]"], 119 | "sidekiq-merger:unique_msg:bar" => ["[2,3,4]"], 120 | } } 121 | let(:msgs_h) { { 122 | "sidekiq-merger:msg:foo" => ["[1,2,3]"], 123 | "sidekiq-merger:msg:bar" => ["[2,3,4]"], 124 | } } 125 | end 126 | end 127 | end 128 | 129 | describe "#delete_message" do 130 | before do 131 | subject.redis do |conn| 132 | conn.sadd("sidekiq-merger:unique_msg:foo", "[1,2,3]") 133 | conn.sadd("sidekiq-merger:unique_msg:foo", "[2,3,4]") 134 | conn.lpush("sidekiq-merger:msg:foo", "[1,2,3]") 135 | conn.lpush("sidekiq-merger:msg:foo", "[2,3,4]") 136 | end 137 | end 138 | it "deletes the msg" do 139 | subject.delete_message("foo", [1, 2, 3]) 140 | subject.redis do |conn| 141 | expect(conn.smembers("sidekiq-merger:unique_msg:foo")).to contain_exactly "[2,3,4]" 142 | expect(conn.lrange("sidekiq-merger:msg:foo", 0, -1)).to contain_exactly "[2,3,4]" 143 | end 144 | end 145 | context "with duplicate msgs" do 146 | it "deletes the msg" do 147 | subject.redis do |conn| 148 | conn.lpush("sidekiq-merger:msg:foo", "[1,2,3]") 149 | end 150 | subject.delete_message("foo", [1, 2, 3]) 151 | subject.redis do |conn| 152 | expect(conn.smembers("sidekiq-merger:unique_msg:foo")).to contain_exactly "[2,3,4]" 153 | expect(conn.lrange("sidekiq-merger:msg:foo", 0, -1)).to contain_exactly "[2,3,4]" 154 | end 155 | end 156 | end 157 | end 158 | 159 | describe "#merge_size" do 160 | before do 161 | subject.redis do |conn| 162 | conn.lpush("sidekiq-merger:msg:foo", "[1,2,3]") 163 | conn.lpush("sidekiq-merger:msg:foo", "[2,3,4]") 164 | end 165 | end 166 | it "returns the size" do 167 | expect(subject.merge_size("foo")).to eq 2 168 | end 169 | end 170 | 171 | describe "#merge_exists?" do 172 | context "unique key exists" do 173 | it "returns true" do 174 | described_class.redis { |conn| conn.sadd("sidekiq-merger:unique_msg:foo", "\"test\"") } 175 | expect(subject.merge_exists?("foo", "test")).to eq true 176 | end 177 | end 178 | context "unique key does not exists" do 179 | it "returns false" do 180 | expect(subject.merge_exists?("foo", "test")).to eq false 181 | end 182 | end 183 | end 184 | 185 | describe "#all_merges" do 186 | before do 187 | subject.push_message("foo", [1, 2, 3], execution_time) 188 | subject.push_message("bar", [2, 3, 4], execution_time) 189 | end 190 | it "gets all the merges" do 191 | expect(subject.all_merges).to contain_exactly "foo", "bar" 192 | end 193 | end 194 | 195 | describe "#lock_merge" do 196 | it "locks the key" do 197 | subject.lock_merge("foo", 3) 198 | subject.redis do |conn| 199 | expect(conn.exists("sidekiq-merger:lock:foo")).to eq true 200 | end 201 | end 202 | end 203 | 204 | describe "#get_merge" do 205 | before do 206 | subject.push_message("bar", [1, 2, 3], execution_time) 207 | subject.push_message("bar", [2, 3, 4], execution_time) 208 | end 209 | it "gets all the msg" do 210 | expect(subject.get_merge("bar")).to contain_exactly [1, 2, 3], [2, 3, 4] 211 | expect(subject.merge_size("bar")).to eq 2 212 | end 213 | end 214 | 215 | describe "#pluck_merge" do 216 | before do 217 | subject.push_message("bar", [1, 2, 3], execution_time) 218 | subject.push_message("bar", [2, 3, 4], execution_time) 219 | end 220 | it "plucks all the msg" do 221 | expect(subject.pluck_merge("bar")).to contain_exactly [1, 2, 3], [2, 3, 4] 222 | expect(subject.merge_size("bar")).to eq 0 223 | end 224 | end 225 | 226 | describe "#delete_merge" do 227 | before do 228 | subject.push_message("foo", [1, 2, 3], execution_time) 229 | subject.push_message("foo", [1, 2, 3], execution_time) 230 | end 231 | it "deletes the merge" do 232 | expect { 233 | subject.delete_merge("foo") 234 | }.to change { subject.merge_size("foo") }.from(2).to(0) 235 | end 236 | end 237 | end 238 | -------------------------------------------------------------------------------- /spec/sidekiq/merger_spec.rb: -------------------------------------------------------------------------------- 1 | require "spec_helper" 2 | 3 | describe Sidekiq::Merger do 4 | it "has a version number" do 5 | expect(described_class::VERSION).not_to be nil 6 | end 7 | describe ".create_task" do 8 | it "starts a monitoring task" do 9 | task = described_class.create_task 10 | expect(task).to be_a Concurrent::TimerTask 11 | task.shutdown 12 | end 13 | end 14 | describe ".configure" do 15 | it "yields to the config" do 16 | expect { |b| described_class.configure(&b) }.to yield_with_args(described_class.config) 17 | end 18 | end 19 | describe ".config" do 20 | it "returns a config" do 21 | expect(described_class.config).to be_a Sidekiq::Merger::Config 22 | end 23 | context "called twice" do 24 | it "returns the same config instance" do 25 | expect(described_class.config).to be described_class.config 26 | end 27 | end 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift File.expand_path("../../lib", __FILE__) 2 | require "rubygems" 3 | require "sidekiq/testing" 4 | require "active_support/core_ext/numeric/time" 5 | require "timecop" 6 | require "simplecov" 7 | require "coveralls" 8 | 9 | pid = Process.pid 10 | SimpleCov.at_exit do 11 | SimpleCov.result.format! if Process.pid == pid 12 | end 13 | SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter[ 14 | SimpleCov::Formatter::HTMLFormatter, 15 | Coveralls::SimpleCov::Formatter 16 | ] 17 | SimpleCov.start 18 | 19 | require "sidekiq/merger" 20 | 21 | Dir[File.join(__dir__, "support", "**", "*.rb")].each { |f| require f } 22 | 23 | RSpec.configure do |config| 24 | config.expect_with :rspec do |expectations| 25 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 26 | end 27 | 28 | config.mock_with :rspec do |mocks| 29 | mocks.verify_partial_doubles = true 30 | end 31 | 32 | if config.files_to_run.one? 33 | config.default_formatter = "doc" 34 | end 35 | 36 | config.order = :random 37 | 38 | Kernel.srand config.seed 39 | 40 | config.before :suite do 41 | Sidekiq::Testing.fake! 42 | Sidekiq::Merger.logger = nil 43 | Sidekiq.logger = nil 44 | if Redis.respond_to?(:exists_returns_integer) 45 | Redis.exists_returns_integer = false 46 | end 47 | end 48 | 49 | config.around :example do |example| 50 | Sidekiq::Merger::Redis.redis { |conn| conn.flushall } 51 | begin 52 | example.run 53 | ensure 54 | Sidekiq::Merger::Redis.redis { |conn| conn.flushall } 55 | end 56 | end 57 | 58 | config.after :example do 59 | Timecop.return 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /spec/support/matchers.rb: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dtaniwaki/sidekiq-merger/b7f67d386dca9631bdc263652412c965b671150d/spec/support/matchers.rb -------------------------------------------------------------------------------- /spec/support/worker_class.rb: -------------------------------------------------------------------------------- 1 | RSpec.shared_context "worker class", worker_class: true do 2 | let(:worker_options) { { key: -> (args) { "key" } } } 3 | let(:worker_class) do 4 | local_options = worker_options 5 | Class.new do 6 | include Sidekiq::Worker 7 | 8 | sidekiq_options merger: local_options 9 | 10 | def self.name 11 | "SomeWorker" 12 | end 13 | 14 | def self.to_s 15 | "SomeWorker" 16 | end 17 | 18 | def perform(*args) 19 | end 20 | end 21 | end 22 | let(:non_merge_worker_class) do 23 | Class.new do 24 | include Sidekiq::Worker 25 | 26 | def self.to_s 27 | "NonMergeWorker" 28 | end 29 | 30 | def perform(*args) 31 | end 32 | end 33 | end 34 | before :example do 35 | allow(Object).to receive(:const_get).with(anything).and_call_original 36 | allow(Object).to receive(:const_get).with("SomeWorker").and_return worker_class 37 | allow(Object).to receive(:const_get).with("NonMergeWorker").and_return non_merge_worker_class 38 | end 39 | around :example do |example| 40 | worker_class.jobs.clear 41 | non_merge_worker_class.jobs.clear 42 | begin 43 | example.run 44 | ensure 45 | worker_class.jobs.clear 46 | non_merge_worker_class.jobs.clear 47 | end 48 | end 49 | end 50 | --------------------------------------------------------------------------------