├── .github ├── dependabot.yml └── workflows │ └── ci.yml ├── .gitignore ├── ChangeLog.md ├── Dockerfile ├── Dockerfile.slim ├── Gemfile ├── Guardfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── bin └── sneakers ├── docker-compose.yml ├── examples ├── benchmark_worker.rb ├── max_retry_handler.rb ├── metrics_worker.rb ├── middleware_worker.rb ├── newrelic_metrics_worker.rb ├── profiling_worker.rb ├── sneakers.conf.rb.example ├── title_scraper.rb └── workflow_worker.rb ├── kicks.gemspec ├── lib ├── active_job │ └── queue_adapters │ │ └── sneakers_adapter.rb ├── sneakers.rb └── sneakers │ ├── cli.rb │ ├── concerns │ ├── logging.rb │ └── metrics.rb │ ├── configuration.rb │ ├── content_encoding.rb │ ├── content_type.rb │ ├── error_reporter.rb │ ├── errors.rb │ ├── handlers │ ├── maxretry.rb │ └── oneshot.rb │ ├── metrics │ ├── logging_metrics.rb │ ├── newrelic_metrics.rb │ ├── null_metrics.rb │ └── statsd_metrics.rb │ ├── middleware │ └── config.rb │ ├── publisher.rb │ ├── queue.rb │ ├── runner.rb │ ├── spawner.rb │ ├── support │ ├── production_formatter.rb │ └── utils.rb │ ├── tasks.rb │ ├── version.rb │ ├── worker.rb │ └── workergroup.rb ├── log └── .gitkeep ├── scripts ├── local_integration └── local_worker └── spec ├── fixtures ├── integration_worker.rb ├── require_worker.rb └── test_job.rb ├── gzip_helper.rb ├── sneakers ├── active_job_integration_spec.rb ├── cli_spec.rb ├── concerns │ ├── logging_spec.rb │ └── metrics_spec.rb ├── configuration_spec.rb ├── content_encoding_spec.rb ├── content_type_spec.rb ├── integration_spec.rb ├── publisher_spec.rb ├── queue_spec.rb ├── runner_spec.rb ├── sneakers_spec.rb ├── support │ └── utils_spec.rb ├── tasks │ └── sneakers_run_spec.rb ├── worker_handlers_spec.rb ├── worker_spec.rb └── workergroup_spec.rb └── spec_helper.rb /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | tests: 7 | name: Ruby ${{ matrix.ruby-version }} 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | ruby-version: 12 | - "3.4.2" 13 | - "3.3.7" 14 | - "3.2.7" 15 | - "3.1.6" 16 | steps: 17 | - uses: actions/checkout@v4 18 | 19 | - name: Install Ruby ${{ matrix.ruby-version }} 20 | uses: ruby/setup-ruby@v1 21 | with: 22 | ruby-version: ${{ matrix.ruby-version }} 23 | bundler-cache: true # 'bundle install', cache gems 24 | 25 | - name: Run tests 26 | run: bundle exec rake 27 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | sneakers.yaml 2 | Gemfile.lock 3 | *.log 4 | sneakers.pid 5 | sneakers.log.* 6 | pkg/ 7 | coverage/ 8 | tmp/ 9 | .ruby-version 10 | .ruby-gemset 11 | .bundle/* 12 | *.gem 13 | -------------------------------------------------------------------------------- /ChangeLog.md: -------------------------------------------------------------------------------- 1 | # Change Log 2 | 3 | ## Changes Between 3.2.0 and 3.3.0 (in development) 4 | 5 | ### Improved Bunny Exception Handling for Consumers 6 | 7 | Contributed by @shashankmehra. 8 | 9 | GitHub issue: [#35](https://github.com/ruby-amqp/kicks/pull/35) 10 | 11 | ### Improved Logger Configuration 12 | 13 | Contributed by @cdhagmann. 14 | 15 | GitHub issue: [#34](https://github.com/ruby-amqp/kicks/pull/34) 16 | 17 | ### Bunny Version Bump 18 | 19 | Kicks now requires the latest (at the time of writing) Bunny `2.24.x`. 20 | 21 | ### Support Rails 7.2 22 | 23 | Contributed by @sekrett. 24 | 25 | GitHub issue: [#32](https://github.com/ruby-amqp/kicks/pull/32) 26 | 27 | 28 | ## Changes Between 3.1.0 and 3.2.0 (Jan 26, 2025) 29 | 30 | ### Improved Support for Bring-Your-Own-Connection (BYOC) 31 | 32 | Kicks now supports passing in a callable (e.g. a proc) instead of an externally-initialized 33 | and managed Bunny connection. 34 | 35 | In this case, it is entirely up to the caller 36 | to configure the connection and call `Bunny::Session#start` on it 37 | at the right moment. 38 | 39 | Contributed by @tie. 40 | 41 | GitHub issue: [#29](https://github.com/ruby-amqp/kicks/pull/29) 42 | 43 | 44 | ### ActiveJob Adapter Compatibility with Ruby on Rails Older Than 7.2 45 | 46 | Contributed by @dixpac. 47 | 48 | GitHub issues: [#19](https://github.com/ruby-amqp/kicks/pull/19), [#28](https://github.com/ruby-amqp/kicks/pull/28) 49 | 50 | 51 | ## Changes Between 3.0.0 and 3.1.0 (Oct 20, 2024) 52 | 53 | ### ActiveJob Adapter 54 | 55 | Kicks now ships with an ActiveJob adapter for Ruby on Rails. 56 | 57 | Contributed by dixpac. 58 | 59 | GitHub issue: [#12](https://github.com/ruby-amqp/kicks/pull/12) 60 | 61 | ### Make Queue Binding Optional 62 | 63 | It is now possible to opt out of binding of the Kicks-declared exchange and queue. 64 | 65 | Contributed by @texpert. 66 | 67 | GitHub issue: [#13](https://github.com/ruby-amqp/kicks/pull/13) 68 | 69 | 70 | ## Changes Between 2.12.0 and 3.0.0 (Oct 19, 2024) 71 | 72 | ### New Project Name and Major Version 73 | 74 | Kicks was originally developed by @jondot under the [name of Sneakers](https://github.com/jondot/sneakers). 75 | 76 | After a group of users and a RabbitMQ core team member have taken over maintenance, it was renamed 77 | to Kicks and the version was bumped to 3.0 to clearly indicate the split. 78 | 79 | ### Minimum Required Ruby Version 80 | 81 | Kicks now requires Ruby 2.5 or later. 82 | 83 | ### Content Encoding Support 84 | 85 | Similar to already supported content type. 86 | 87 | Contributed by @ansoncat. 88 | 89 | GitHub issue: [#449](https://github.com/jondot/sneakers/pull/449) 90 | 91 | 92 | ## Changes Between 2.10.0 and 2.11.0 93 | 94 | This releases includes bug fixes, support for more queue-binding options, better 95 | management of the Bunny dependency, and improved documentation. Following is a 96 | list of the notable changes: 97 | 98 | ### Rescue from ScriptError 99 | 100 | Fixes a bug that would cause Sneakers workers to freeze if an exception 101 | descending from `ScriptError`, such as `NotImplementedError`, is raised 102 | 103 | Contributed by @sharshenov 104 | 105 | GitHub Pull Request: [373](https://github.com/jondot/sneakers/pull/373) 106 | 107 | ### Loosen Bunny dependency to minor version 108 | 109 | The dependency on Bunny is now pinned to the minor version instead of patch, 110 | allowing users to benefit from non-breaking updates to Bunny without having to 111 | wait for a Sneakers release. 112 | 113 | Contributed by @olivierlacan 114 | 115 | GitHub Pull Request: [#372](https://github.com/jondot/sneakers/pull/372) 116 | 117 | ### Support `:bind_arguments` on bind 118 | 119 | It is now possible to set arguments on a queue when connecting to a headers 120 | exchange 121 | 122 | Contributed by @nerikj 123 | 124 | GitHub Pull Request: [#358](https://github.com/jondot/sneakers/pull/358) 125 | 126 | ### Other contributions 127 | 128 | This release also contains contributions from @ivan-kolmychek (bumping up Bunny 129 | dependency), @michaelklishin (improving code style), and @darren987469 (adding 130 | examples to the README) 131 | 132 | ## Changes Between 2.8.0 and 2.10.0 133 | 134 | This release contains **minor breaking API changes**. 135 | 136 | ### Worker Timeouts are No Longer Enforced 137 | 138 | This is a **breaking change** for `Sneakers::Worker` implementations. 139 | 140 | Timeouts can be disruptive and dangerous depending on what the workers do but not having them can also 141 | lead to operational headaches. 142 | 143 | The outcome of [a lengthy discussion](https://github.com/jondot/sneakers/issues/343) on possible 144 | alternatives to the timeout mechanisms is that only applications 145 | can know where it is safe to enforce a timeout (and how). 146 | 147 | `Sneakers::Worker` implementations are now expected to enforce timeouts 148 | in a way that makes sense (and is safe) to them. 149 | 150 | GitHub issues: [#343](https://github.com/jondot/sneakers/issues/343). 151 | 152 | 153 | ## Changes Between 2.6.0 and 2.7.0 154 | 155 | This release requires Ruby 2.2 and has **breaking API changes** 156 | around custom error handlers. 157 | 158 | ### Use Provided Connections in WorkerGroup 159 | 160 | It is now possible to use a custom connection instance in worker groups. 161 | 162 | Contributed by @pomnikita. 163 | 164 | GitHub issue: [#322](https://github.com/jondot/sneakers/pull/322) 165 | 166 | 167 | ### Worker Context Available to Worker Instances 168 | 169 | Contributed by Jason Lombardozzi. 170 | 171 | GitHub issue: [#307](https://github.com/jondot/sneakers/pull/307) 172 | 173 | 174 | ### Ruby 2.2 Requirement 175 | 176 | Sneakers now [requires Ruby 2.2](https://github.com/jondot/sneakers/commit/f33246a1bd3b5fe53ee662253dc5bac7864eec97). 177 | 178 | 179 | ### Bunny 2.9.x 180 | 181 | Bunny was [upgraded](https://github.com/jondot/sneakers/commit/c7fb0bd23280082e43065d7199668486db005c13) to 2.9.x. 182 | 183 | 184 | 185 | ### Server Engine 2.0.5 186 | 187 | Server Engine dependency was [upgraded to 2.0.5](https://github.com/jondot/sneakers/commit/3f60fd5e88822169fb04088f0ce5d2f94f803339). 188 | 189 | 190 | ### Refactored Publisher Connection 191 | 192 | Contributed by Christoph Wagner. 193 | 194 | GitHub issue: [#325](https://github.com/jondot/sneakers/pull/325) 195 | 196 | 197 | ### New Relic Reporter Migrated to the Modern API 198 | 199 | Contributed by @adamors. 200 | 201 | GitHub issue: [#324](https://github.com/jondot/sneakers/pull/324) 202 | 203 | 204 | ### Configuration Logged at Debug Level 205 | 206 | To avoid potentially leaking credentials in the log. 207 | 208 | Contributed by Kimmo Lehto. 209 | 210 | GitHub issue: [#301](https://github.com/jondot/sneakers/pull/301). 211 | 212 | 213 | ### Comment Corrections 214 | 215 | Contributed by Andrew Babichev 216 | 217 | GitHub issue: [#346](https://github.com/jondot/sneakers/pull/346) 218 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ruby:2.4-alpine 2 | 3 | RUN apk add --no-cache git 4 | 5 | RUN apk --update add --virtual build_deps \ 6 | build-base \ 7 | ruby-dev \ 8 | libc-dev \ 9 | linux-headers \ 10 | openssl-dev 11 | 12 | WORKDIR /sneakers 13 | 14 | COPY lib/sneakers/version.rb lib/sneakers/version.rb 15 | 16 | COPY sneakers.gemspec . 17 | 18 | COPY Gemfile* ./ 19 | 20 | RUN bundle install --retry=3 21 | 22 | COPY . . 23 | 24 | CMD rake test 25 | -------------------------------------------------------------------------------- /Dockerfile.slim: -------------------------------------------------------------------------------- 1 | FROM ruby:2.3-alpine 2 | 3 | RUN apk add --no-cache git 4 | 5 | ADD . /sneakers 6 | 7 | WORKDIR /sneakers 8 | 9 | RUN apk --update add --virtual build_deps \ 10 | build-base \ 11 | ruby-dev \ 12 | libc-dev \ 13 | linux-headers \ 14 | openssl-dev && \ 15 | 16 | bundle --jobs=4 --retry=3 && \ 17 | 18 | apk del build_deps 19 | 20 | CMD rake test 21 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'ruby-prof', platforms: [:ruby_22, :ruby_23, :ruby_24] 4 | 5 | gem 'rake', '>= 12.3', '< 14.0' 6 | gem 'metric_fu', git: 'https://github.com/metricfu/metric_fu', branch: 'main' 7 | 8 | gemspec 9 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | 2 | guard :minitest do 3 | watch(%r{^spec/(.*)_spec\.rb}) 4 | watch(%r{^lib/(.+)\.rb}) { |m| "spec/#{m[1]}_spec.rb" } 5 | watch(%r{^spec/spec_helper\.rb}) { 'spec' } 6 | end 7 | 8 | 9 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2023 Dotan Nahum 2 | Copyright (c) 2023-2024 Kicks Contributors 3 | 4 | MIT License 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining 7 | a copy of this software and associated documentation files (the 8 | "Software"), to deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, merge, publish, 10 | distribute, sublicense, and/or sell copies of the Software, and to 11 | permit persons to whom the Software is furnished to do so, subject to 12 | the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 21 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 22 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 23 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kicks, né Sneakers 2 | 3 | [![CI](https://github.com/ruby-amqp/kicks/actions/workflows/ci.yml/badge.svg)](https://github.com/ruby-amqp/kicks/actions/workflows/ci.yml) 4 | 5 | ``` 6 | __ 7 | ,--' > 8 | `===== 9 | 10 | ``` 11 | 12 | ## What is Kicks? 13 | 14 | Kicks is a high-performance RabbitMQ background processing framework for 15 | Ruby, originally developed by @jondot 👏 under the name of [Sneakers](https://github.com/jondot/sneakers). 16 | The original repo was abandoned by the person who has exclusive control 17 | over the RubyGems project, so the community of users has decided to move it 18 | to this "fork" (continuation) under the new name. 19 | 20 | By virtue of its Sneakers lineage, Kicks is a mature project that has been around since 2016. 21 | 22 | 23 | ## Installation 24 | 25 | Add this line to your application's Gemfile: 26 | 27 | ```ruby 28 | gem 'kicks' 29 | ``` 30 | 31 | And then execute: 32 | 33 | ```shell-session 34 | bundle 35 | ``` 36 | 37 | Or install it yourself as: 38 | 39 | ```shell-session 40 | gem install kicks 41 | ``` 42 | 43 | ## Documentation 44 | 45 | A quick start guide is available in the section below. 46 | 47 | Visit the [wiki](https://github.com/jondot/sneakers/wiki) for more detailed 48 | documentation and [GitHub releases](https://github.com/jondot/sneakers/releases) for release 49 | notes. 50 | 51 | A [change log](./ChangeLog.md) is also available. 52 | 53 | ## Quick start 54 | 55 | Set up a Gemfile 56 | 57 | ```ruby 58 | source 'https://rubygems.org' 59 | gem 'kicks' 60 | gem 'json' 61 | gem 'redis' 62 | ``` 63 | 64 | How do we add a worker? Firstly create a file and name it as `boot.rb` 65 | then create a worker named as `Processor`. 66 | 67 | > touch boot.rb 68 | 69 | ```ruby 70 | # Note: the files still use "sneakers" for the name 71 | require 'sneakers' 72 | require 'redis' 73 | require 'json' 74 | 75 | $redis = Redis.new 76 | 77 | class Processor 78 | include Sneakers::Worker 79 | from_queue :logs 80 | 81 | def work(msg) 82 | err = JSON.parse(msg) 83 | if err["type"] == "error" 84 | $redis.incr "processor:#{err["error"]}" 85 | end 86 | 87 | ack! 88 | end 89 | end 90 | ``` 91 | 92 | Let's test it out quickly from the command line: 93 | 94 | ```shell-session 95 | $ sneakers work Processor --require boot.rb 96 | ``` 97 | 98 | We just told Sneakers to spawn a worker named `Processor`, but first `--require` a file that we dedicate to setting up environment, including workers and what-not. 99 | 100 | If you go to your RabbitMQ admin now, you'll see a new queue named `logs` was created. Push a couple messages like below: 101 | 102 | ```javascript 103 | { 104 | "type": "error", 105 | "message": "HALP!", 106 | "error": "CODE001" 107 | } 108 | ``` 109 | 110 | Publish a message with the [bunny](https://github.com/ruby-amqp/bunny) gem RabbitMQ client: 111 | 112 | ```ruby 113 | require 'bunny' 114 | 115 | conn = Bunny.new 116 | conn.start 117 | 118 | ch = conn.create_channel 119 | ch.default_exchange.publish({ type: 'error', message: 'HALP!', error: 'CODE001' }.to_json, routing_key: 'logs') 120 | 121 | conn.close 122 | ``` 123 | 124 | And this is the output you should see at your terminal. 125 | 126 | ``` 127 | 2013-10-11T19:26:36Z p-4718 t-ovqgyb31o DEBUG: [worker-logs:1:213mmy][#][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log 128 | 2013-10-11T19:26:36Z p-4718 t-ovqgyrxu4 INFO: log log 129 | 2013-10-11T19:26:40Z p-4719 t-ovqgyb364 DEBUG: [worker-logs:1:h23iit][#][logs][{:prefetch=>10, :durable=>true, :ack=>true, :heartbeat_interval=>2, :exchange=>"sneakers"}] Working off: log log 130 | 2013-10-11T19:26:40Z p-4719 t-ovqgyrx8g INFO: log log 131 | ``` 132 | 133 | We'll count errors and error types with Redis. 134 | 135 | ``` shell-session 136 | $ redis-cli monitor 137 | 1381520329.888581 [0 127.0.0.1:49182] "incr" "processor:CODE001" 138 | ``` 139 | 140 | We're basically done with the ceremonies and all is left is to do some real work. 141 | 142 | ### Looking at metrics 143 | 144 | Let's use the `logging_metrics` provider just for the sake of fun of seeing the metrics as they happen. 145 | 146 | ```ruby 147 | # boot.rb 148 | require 'sneakers' 149 | require 'redis' 150 | require 'json' 151 | require 'sneakers/metrics/logging_metrics' 152 | Sneakers.configure(metrics: Sneakers::Metrics::LoggingMetrics.new) 153 | 154 | # ... rest of code 155 | ``` 156 | 157 | Now push a message again and you'll see: 158 | 159 | ``` 160 | 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.started 161 | 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: TIME: work.Processor.time 0.00242 162 | 2013-10-11T19:44:37Z p-9219 t-oxh8owywg INFO: INC: work.Processor.handled.ack 163 | ``` 164 | 165 | Which increments `started` and `handled.ack`, and times the work unit. 166 | 167 | From here, you can continue over to the 168 | [Wiki](https://github.com/jondot/sneakers/wiki) 169 | 170 | # Docker 171 | 172 | If you use Docker, there's some benefits to be had and you can use both 173 | `docker` and `docker-compose` with this project, in order to run tests, 174 | integration tests or a sample worker without setting up RabbitMQ or the 175 | environment needed locally on your development box. 176 | 177 | * To build a container run `docker build . -t sneakers_sneakers` 178 | * To run non-integration tests within a docker container, run `docker run --rm 179 | sneakers_sneakers:latest` 180 | * To run full integration tests within a docker topology including RabbitMQ, 181 | Redis (for integration worker) run `scripts/local_integration`, which will 182 | use docker-compose to orchestrate the topology and the sneakers Docker image 183 | to run the tests 184 | * To run a sample worker within Docker, try the `TitleScraper` example by 185 | running `script/local_worker`. This will use docker-compose as well. It will 186 | also help you get a feeling for how to run Sneakers in a Docker based 187 | production environment 188 | * Use `Dockerfile.slim` instead of `Dockerfile` for production docker builds. 189 | It generates a more compact image, while the "regular" `Dockerfile` generates 190 | a fatter image - yet faster to iterate when developing 191 | 192 | # Compatibility 193 | 194 | * Kicks main branch: Ruby 3.0+ 195 | * Kicks 3.1.x branch: Ruby 3.0+ 196 | * Kicks 3.0.x: Ruby 2.5+ 197 | * Sneakers 2.7.x and later (using Bunny 2.9): Ruby 2.2.x 198 | * Sneakers 1.1.x and later (using Bunny 2.x): Ruby 2.x 199 | * Sneakers 1.x.x and earlier: Ruby 1.9.x, 2.x 200 | 201 | # Contributing 202 | 203 | Fork, implement, add tests, pull request, get my everlasting thanks and a respectable place here :). 204 | 205 | ### Thanks: 206 | 207 | To all Sneakers [Contributors](https://github.com/jondot/sneakers/graphs/contributors) - you make this happen, thanks! 208 | 209 | ## License 210 | 211 | See [LICENSE](LICENSE.txt) for further details. 212 | 213 | ## Copyright 214 | 215 | Copyright (c) 2023-2024 Kicks contributors. 216 | 217 | Copyright (c) 2015-2023 [Dotan Nahum](http://gplus.to/dotan) [@jondot](http://twitter.com/jondot). 218 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require 'metric_fu' 3 | require 'rake/testtask' 4 | 5 | Rake::TestTask.new do |t| 6 | t.libs << "spec" 7 | t.test_files = FileList['spec/**/*_spec.rb'] 8 | t.warning = false 9 | end 10 | 11 | task default: :test 12 | 13 | -------------------------------------------------------------------------------- /bin/sneakers: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'sneakers' 3 | require 'sneakers/cli' 4 | 5 | Sneakers.server = true 6 | Sneakers::CLI.start 7 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3' 2 | 3 | services: 4 | sneakers: 5 | build: . 6 | volumes: 7 | - .:/sneakers 8 | depends_on: 9 | - rabbitmq 10 | - redis 11 | environment: 12 | - RABBITMQ_URL=amqp://guest:guest@rabbitmq:5672 13 | 14 | rabbitmq: 15 | image: rabbitmq:management-alpine 16 | ports: 17 | - "5672:5672" 18 | - "15672:15672" 19 | 20 | redis: 21 | image: redis:alpine 22 | ports: 23 | - "6379:6379" 24 | 25 | -------------------------------------------------------------------------------- /examples/benchmark_worker.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | 4 | class BenchmarkWorker 5 | include Sneakers::Worker 6 | from_queue 'downloads', 7 | exchange_options: { durable: false }, 8 | queue_options: { durable: false }, 9 | :ack => true, 10 | :threads => 50, 11 | :prefetch => 50, 12 | :timeout_job_after => 1, 13 | :exchange => 'dummy', 14 | :heartbeat => 5, 15 | :amqp_heartbeat => 10 16 | def work(msg) 17 | ack! 18 | end 19 | end 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /examples/max_retry_handler.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | require 'sneakers/handlers/maxretry' 5 | require 'logger' 6 | 7 | Sneakers.configure(:handler => Sneakers::Handlers::Maxretry, 8 | :workers => 1, 9 | :threads => 1, 10 | :prefetch => 1, 11 | :exchange => 'sneakers', 12 | :exchange_options => { :type => 'topic', durable: true }, 13 | :routing_key => ['#', 'something'] 14 | ) 15 | Sneakers.logger.level = Logger::DEBUG 16 | 17 | WORKER_OPTIONS = { 18 | :ack => true, 19 | :threads => 1, 20 | :prefetch => 1, 21 | :timeout_job_after => 60, 22 | :heartbeat => 5, 23 | :amqp_heartbeat => 10, 24 | :retry_timeout => 5000 25 | } 26 | 27 | # Example of how to write a retry worker. If your rabbit system is empty, then 28 | # you must run this twice. Once to setup the exchanges, queues and bindings a 29 | # second time to have the sent message end up on the downloads queue. 30 | # 31 | # Run this via: 32 | # bundle exec ruby examples/max_retry_handler.rb 33 | # 34 | class MaxRetryWorker 35 | include Sneakers::Worker 36 | from_queue 'downloads', WORKER_OPTIONS 37 | 38 | def work(msg) 39 | logger.info("MaxRetryWorker rejecting msg: #{msg.inspect}") 40 | 41 | # We always want to reject to see if we do the proper timeout 42 | reject! 43 | end 44 | end 45 | 46 | # Example of a worker on the same exchange that does not fail, so it should only 47 | # see the message once. 48 | class SucceedingWorker 49 | include Sneakers::Worker 50 | from_queue 'uploads', WORKER_OPTIONS 51 | 52 | def work(msg) 53 | logger.info("SucceedingWorker succeeding on msg: #{msg.inspect}") 54 | ack! 55 | end 56 | end 57 | 58 | messages = 1 59 | puts "feeding messages in" 60 | messages.times { 61 | Sneakers.publish(" -- message -- ", 62 | :to_queue => 'anywhere', 63 | :persistence => true) 64 | } 65 | puts "done" 66 | 67 | r = Sneakers::Runner.new([MaxRetryWorker, SucceedingWorker]) 68 | r.run 69 | -------------------------------------------------------------------------------- /examples/metrics_worker.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | require 'sneakers/metrics/logging_metrics' 5 | require 'open-uri' 6 | 7 | 8 | class MetricsWorker 9 | include Sneakers::Worker 10 | 11 | from_queue 'downloads' 12 | 13 | def work(msg) 14 | title = extract_title(open(msg)) 15 | logger.info "FOUND <#{title}>" 16 | ack! 17 | end 18 | 19 | private 20 | 21 | def extract_title(html) 22 | html =~ /(.*?)<\/title>/ 23 | $1 24 | end 25 | end 26 | 27 | 28 | Sneakers.configure(:metrics => Sneakers::Metrics::LoggingMetrics.new) 29 | r = Sneakers::Runner.new([ MetricsWorker ]) 30 | r.run 31 | 32 | 33 | 34 | 35 | -------------------------------------------------------------------------------- /examples/middleware_worker.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | 5 | class MiddlewareWorker 6 | include Sneakers::Worker 7 | 8 | from_queue 'middleware-demo', 9 | ack: false 10 | 11 | def work(message) 12 | puts "******** MiddlewareWorker -> #{message}" 13 | end 14 | end 15 | 16 | class DemoMiddleware 17 | def initialize(app, *args) 18 | @app = app 19 | @args = args 20 | end 21 | 22 | def call(deserialized_msg, delivery_info, metadata, handler) 23 | puts "******** DemoMiddleware - before; args #{@args}" 24 | res = @app.call(deserialized_msg, delivery_info, metadata, handler) 25 | puts "******** DemoMiddleware - after" 26 | 27 | res 28 | end 29 | end 30 | 31 | Sneakers.configure 32 | Sneakers.middleware.use(DemoMiddleware, foo: :bar) 33 | 34 | Sneakers.publish("{}", :to_queue => 'middleware-demo') 35 | r = Sneakers::Runner.new([MiddlewareWorker]) 36 | r.run 37 | -------------------------------------------------------------------------------- /examples/newrelic_metrics_worker.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | require 'sneakers/metrics/newrelic_metrics' 5 | require 'open-uri' 6 | require 'newrelic_rpm' 7 | 8 | # With this configuration will send two types of data to newrelic server: 9 | # 1. Transaction data which you would see under 'Applications' 10 | # 2. Metrics where you will be able to see by configuring a dashboardi, available for enterprise accounts 11 | # 12 | # You should have newrelic.yml in the 'config' folder with the proper account settings 13 | 14 | Sneakers::Metrics::NewrelicMetrics.eagent ::NewRelic 15 | Sneakers.configure metrics: Sneakers::Metrics::NewrelicMetrics.new 16 | 17 | class MetricsWorker 18 | include Sneakers::Worker 19 | include ::NewRelic::Agent::Instrumentation::ControllerInstrumentation 20 | 21 | from_queue 'downloads' 22 | 23 | def work(msg) 24 | title = extract_title(open(msg)) 25 | logger.info "FOUND <#{title}>" 26 | ack! 27 | end 28 | 29 | add_transaction_tracer :work, name: 'MetricsWorker', params: 'args[0]', category: :task 30 | 31 | private 32 | 33 | def extract_title(html) 34 | html =~ /<title>(.*?)<\/title>/ 35 | $1 36 | end 37 | end 38 | 39 | r = Sneakers::Runner.new([ MetricsWorker ]) 40 | r.run 41 | -------------------------------------------------------------------------------- /examples/profiling_worker.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | require 'logger' 5 | 6 | 7 | profiling = ARGV[0] 8 | messages = 100_000 9 | 10 | 11 | if profiling 12 | require 'ruby-prof' 13 | messages /= 100 # profiling makes everything much slower (around 300req/s) 14 | end 15 | 16 | Sneakers.configure 17 | Sneakers.logger.level = Logger::ERROR 18 | 19 | Sneakers::Worker.configure_logger(Logger.new('/dev/null')) 20 | 21 | puts "feeding messages in" 22 | messages.times { 23 | Sneakers.publish("{}", :to_queue => 'downloads') 24 | } 25 | puts "done" 26 | 27 | 28 | class ProfilingWorker 29 | include Sneakers::Worker 30 | from_queue 'downloads', 31 | :ack => true, 32 | :threads => 50, 33 | :prefetch => 50, 34 | :timeout_job_after => 1, 35 | :exchange => 'sneakers', 36 | :heartbeat => 5, 37 | :amqp_heartbeat => 10 38 | def work(msg) 39 | ack! 40 | end 41 | end 42 | 43 | 44 | 45 | r = Sneakers::Runner.new([ProfilingWorker]) 46 | 47 | # ctrl-c and Ruby 2.0 breaks signal handling 48 | # Sidekiq has same issues 49 | # https://github.com/mperham/sidekiq/issues/728 50 | # 51 | # so we use a timeout and a thread that kills profiling 52 | if profiling 53 | puts "profiling start" 54 | RubyProf.start 55 | 56 | 57 | Thread.new do 58 | sleep 10 59 | puts "stopping profiler" 60 | result = RubyProf.stop 61 | 62 | # Print a flat profile to text 63 | printer = RubyProf::FlatPrinter.new(result) 64 | printer.print(STDOUT) 65 | exit(0) 66 | end 67 | end 68 | 69 | r.run 70 | -------------------------------------------------------------------------------- /examples/sneakers.conf.rb.example: -------------------------------------------------------------------------------- 1 | workers 2 2 | amqp "amqp://guest:guest@localhost:55672" 3 | 4 | before_fork do 5 | Sneakers::logger.info " ** im before-fork'en ** " 6 | end 7 | 8 | 9 | after_fork do 10 | Sneakers::logger.info " !! im after forke'n !! " 11 | end 12 | -------------------------------------------------------------------------------- /examples/title_scraper.rb: -------------------------------------------------------------------------------- 1 | require "sneakers" 2 | require 'open-uri' 3 | require 'logger' 4 | 5 | def compose_or_localhost(key) 6 | Resolv::DNS.new.getaddress(key) 7 | rescue 8 | "localhost" 9 | end 10 | 11 | rmq_addr = compose_or_localhost("rabbitmq") 12 | 13 | Sneakers.configure :log => STDOUT, :amqp => "amqp://guest:guest@#{rmq_addr}:5672" 14 | Sneakers.logger.level = Logger::INFO 15 | 16 | class TitleScraper 17 | include Sneakers::Worker 18 | 19 | from_queue 'downloads' 20 | 21 | def work(msg) 22 | title = extract_title(open(msg)) 23 | logger.info "FOUND <#{title}>" 24 | ack! 25 | end 26 | 27 | private 28 | 29 | def extract_title(html) 30 | html =~ /<title>(.*?)<\/title>/ 31 | $1 32 | end 33 | end 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /examples/workflow_worker.rb: -------------------------------------------------------------------------------- 1 | $: << File.expand_path('../lib', File.dirname(__FILE__)) 2 | require 'sneakers' 3 | 4 | class WorkflowWorker 5 | include Sneakers::Worker 6 | from_queue 'downloads', 7 | exchange_options: { durable: false }, 8 | queue_options: { durable: false }, 9 | :ack => true, 10 | :threads => 50, 11 | :prefetch => 50, 12 | :timeout_job_after => 1, 13 | :exchange => 'dummy', 14 | :heartbeat => 5, 15 | :amqp_heartbeat => 10 16 | 17 | def work(msg) 18 | logger.info("Seriously, i'm DONE.") 19 | publish "cleaned up", :to_queue => "foobar" 20 | logger.info("Published to 'foobar'") 21 | ack! 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /kicks.gemspec: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env gem build 2 | # encoding: utf-8 3 | 4 | lib = File.expand_path('../lib', __FILE__) 5 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 6 | require 'sneakers/version' 7 | 8 | Gem::Specification.new do |gem| 9 | gem.name = 'kicks' 10 | gem.version = Sneakers::VERSION 11 | gem.authors = ['Dotan Nahum', 'Michael Klishin'] 12 | gem.email = ['michael@clojurewerkz.org'] 13 | gem.description = %q( Fast background processing framework for Ruby and RabbitMQ ) 14 | gem.summary = %q( Fast background processing framework for Ruby and RabbitMQ ) 15 | gem.homepage = 'https://github.com/ruby-amqp/kicks' 16 | gem.metadata = { "source_code_uri" => "https://github.com/ruby-amqp/kicks" } 17 | gem.license = 'MIT' 18 | gem.required_ruby_version = Gem::Requirement.new(">= 2.5") 19 | 20 | gem.files = `git ls-files`.split($/).reject { |f| f == 'Gemfile.lock' } 21 | gem.executables = gem.files.grep(/^bin/). 22 | reject { |f| f =~ /^bin\/ci/ }. 23 | map { |f| File.basename(f) } 24 | gem.test_files = gem.files.grep(/^(test|spec|features)\//) 25 | gem.require_paths = ['lib'] 26 | 27 | gem.add_dependency 'serverengine', '~> 2.1' 28 | gem.add_dependency 'bunny', '~> 2.24' 29 | gem.add_dependency 'concurrent-ruby', '~> 1.0' 30 | gem.add_dependency 'thor' 31 | gem.add_dependency 'rake', '>= 12.3', '< 14.0' 32 | 33 | # for integration environment (see .travis.yml and integration_spec) 34 | gem.add_development_dependency 'activejob', '>= 7.2' 35 | gem.add_development_dependency 'activesupport', '>= 7.2' 36 | gem.add_development_dependency 'rabbitmq_http_api_client' 37 | gem.add_development_dependency 'redis' 38 | 39 | gem.add_development_dependency 'minitest', '~> 5.15' 40 | gem.add_development_dependency 'rr', '~> 3.0' 41 | gem.add_development_dependency 'unparser', '~> 0.2' 42 | gem.add_development_dependency 'simplecov', '~> 0.21' 43 | gem.add_development_dependency 'simplecov-rcov-text' 44 | gem.add_development_dependency 'guard', '~> 2.18' 45 | gem.add_development_dependency 'guard-minitest', '~> 2.4' 46 | gem.add_development_dependency 'pry-byebug', '~> 3.9' 47 | end 48 | -------------------------------------------------------------------------------- /lib/active_job/queue_adapters/sneakers_adapter.rb: -------------------------------------------------------------------------------- 1 | module ActiveJob 2 | module QueueAdapters 3 | # Explicitly remove the implementation existing in older Rails versions'. 4 | remove_const(:SneakersAdapter) if const_defined?(:SneakersAdapter) 5 | 6 | # = Sneakers adapter for Active Job 7 | # 8 | # To use Sneakers set the queue_adapter config to +:sneakers+. 9 | # 10 | # Rails.application.config.active_job.queue_adapter = :sneakers 11 | class SneakersAdapter < (const_defined?(:AbstractAdapter) ? AbstractAdapter : Object) 12 | def initialize 13 | @monitor = Monitor.new 14 | end 15 | 16 | def enqueue(job) 17 | @monitor.synchronize do 18 | JobWrapper.from_queue job.queue_name 19 | JobWrapper.enqueue ActiveSupport::JSON.encode(job.serialize) 20 | end 21 | end 22 | 23 | def enqueue_at(job, timestamp) 24 | raise NotImplementedError, 'This queueing backend does not support scheduling jobs.' 25 | end 26 | 27 | class JobWrapper 28 | include Sneakers::Worker 29 | from_queue 'default' 30 | 31 | def work(msg) 32 | job_data = ActiveSupport::JSON.decode(msg) 33 | Base.execute job_data 34 | ack! 35 | end 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/sneakers.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers/version' 2 | require 'concurrent/executors' 3 | require 'bunny' 4 | require 'logger' 5 | require 'serverengine' 6 | 7 | module Sneakers 8 | module Handlers 9 | end 10 | module Concerns 11 | end 12 | end 13 | 14 | require 'sneakers/configuration' 15 | require 'sneakers/errors' 16 | require 'sneakers/support/production_formatter' 17 | require 'sneakers/concerns/logging' 18 | require 'sneakers/concerns/metrics' 19 | require 'sneakers/handlers/oneshot' 20 | require 'sneakers/content_type' 21 | require 'sneakers/content_encoding' 22 | require 'sneakers/middleware/config' 23 | require 'sneakers/worker' 24 | require 'sneakers/publisher' 25 | require 'active_job/queue_adapters/sneakers_adapter' if defined?(ActiveJob) 26 | 27 | 28 | module Sneakers 29 | extend self 30 | 31 | CONFIG = Configuration.new 32 | 33 | def configure(opts={}) 34 | # worker > userland > defaults 35 | CONFIG.merge!(opts) 36 | setup_general_logger! 37 | setup_worker_concerns! 38 | setup_general_publisher! 39 | @configured = true 40 | end 41 | 42 | def clear! 43 | CONFIG.clear 44 | @logger = nil 45 | @publisher = nil 46 | @configured = false 47 | end 48 | 49 | def daemonize!(loglevel=nil) 50 | CONFIG[:log] = 'sneakers.log' 51 | CONFIG[:log_level] = loglevel || Logger::INFO 52 | CONFIG[:daemonize] = true 53 | setup_general_logger! 54 | end 55 | 56 | def rake_worker_classes=(worker_classes) 57 | @rake_worker_classes = worker_classes 58 | end 59 | 60 | def rake_worker_classes 61 | @rake_worker_classes 62 | end 63 | 64 | def logger=(logger) 65 | @logger = logger 66 | end 67 | 68 | def logger 69 | @logger 70 | end 71 | 72 | def publish(msg, routing) 73 | @publisher.publish(msg, routing) 74 | end 75 | 76 | def configured? 77 | @configured 78 | end 79 | 80 | def server=(server) 81 | @server = server 82 | end 83 | 84 | def server? 85 | @server 86 | end 87 | 88 | def configure_server 89 | yield self if server? 90 | end 91 | 92 | # Register a proc to handle any error which occurs within the Sneakers process. 93 | # 94 | # Sneakers.error_reporters << proc { |exception, worker, context_hash| MyErrorService.notify(exception, context_hash) } 95 | # 96 | # The default error handler logs errors to Sneakers.logger. 97 | # Ripped off from https://github.com/mperham/sidekiq/blob/6ad6a3aa330deebd76c6cf0d353f66abd3bef93b/lib/sidekiq.rb#L165-L174 98 | def error_reporters 99 | CONFIG[:error_reporters] 100 | end 101 | 102 | def middleware 103 | @middleware ||= Sneakers::Middleware::Config 104 | end 105 | 106 | private 107 | 108 | def setup_general_logger! 109 | if [:info, :debug, :error, :warn].all?{ |meth| CONFIG[:log].respond_to?(meth) } 110 | @logger = CONFIG[:log] 111 | else 112 | @logger = ServerEngine::DaemonLogger.new(CONFIG[:log], CONFIG.slice(:log_level, :log_rotate_age, :log_rotate_size)) 113 | @logger.formatter = Sneakers::Support::ProductionFormatter 114 | end 115 | end 116 | 117 | def setup_worker_concerns! 118 | Worker.configure_logger(Sneakers::logger) 119 | Worker.configure_metrics(CONFIG[:metrics]) 120 | CONFIG[:handler] ||= Sneakers::Handlers::Oneshot 121 | end 122 | 123 | def setup_general_publisher! 124 | @publisher = Sneakers::Publisher.new 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /lib/sneakers/cli.rb: -------------------------------------------------------------------------------- 1 | require 'thor' 2 | require 'sneakers/runner' 3 | 4 | 5 | # 6 | # $ sneakers work TitleWorker,FooWorker 7 | # $ sneakers stop 8 | # $ sneakers recycle 9 | # $ sneakers reload 10 | # $ sneakers init 11 | # 12 | # 13 | module Sneakers 14 | class CLI < Thor 15 | 16 | SNEAKERS=<<-EOF 17 | 18 | __ 19 | ,--' > Sneakers 20 | `===== 21 | 22 | EOF 23 | 24 | BANNER = SNEAKERS 25 | 26 | method_option :debug 27 | method_option :daemonize 28 | method_option :log 29 | method_option :pid_path 30 | method_option :require 31 | 32 | desc "work FirstWorker,SecondWorker ... ,NthWorker", "Run workers" 33 | def work(workers = "") 34 | opts = { 35 | :daemonize => !!options[:daemonize] 36 | } 37 | 38 | opts[:log] = options[:log] || (opts[:daemonize] ? 'sneakers.log' : STDOUT) 39 | opts[:pid_path] = options[:pid_path] if options[:pid_path] 40 | 41 | if opts[:daemonize] 42 | puts "*** DEPRACATED: self-daemonization '--daemonize' is considered a bad practice, which is why this feature will be removed in future versions. Please run Sneakers in front, and use things like upstart, systemd, or supervisor to manage it as a daemon." 43 | end 44 | 45 | 46 | Sneakers.configure(opts) 47 | 48 | require_boot File.expand_path(options[:require]) if options[:require] 49 | 50 | if workers.empty? 51 | workers = Sneakers::Worker::Classes 52 | else 53 | workers, missing_workers = Sneakers::Utils.parse_workers(workers) 54 | end 55 | 56 | unless missing_workers.nil? || missing_workers.empty? 57 | say "Missing workers: #{missing_workers.join(', ')}" if missing_workers 58 | say "Did you `require` properly?" 59 | return 60 | end 61 | 62 | if workers.empty? 63 | say <<-EOF 64 | Error: No workers found. 65 | Please require your worker classes before specifying in CLI 66 | 67 | $ sneakers work FooWorker 68 | ^- require this in your code 69 | 70 | EOF 71 | return 72 | end 73 | 74 | r = Sneakers::Runner.new(workers) 75 | 76 | pid = Sneakers::CONFIG[:pid_path] 77 | 78 | say SNEAKERS 79 | say "Workers ....: #{em workers.join(', ')}" 80 | say "Log ........: #{em (Sneakers::CONFIG[:log] == STDOUT ? 'Console' : Sneakers::CONFIG[:log]) }" 81 | say "PID ........: #{em pid}" 82 | say "" 83 | say (" "*31)+"Process control" 84 | say "="*80 85 | say "Stop (nicely) ..............: kill -SIGTERM `cat #{pid}`" 86 | say "Stop (immediate) ...........: kill -SIGQUIT `cat #{pid}`" 87 | say "Restart (nicely) ...........: kill -SIGUSR1 `cat #{pid}`" 88 | say "Restart (immediate) ........: kill -SIGHUP `cat #{pid}`" 89 | say "Reconfigure ................: kill -SIGUSR2 `cat #{pid}`" 90 | say "Scale workers ..............: reconfigure, then restart" 91 | say "="*80 92 | say "" 93 | 94 | if options[:debug] 95 | say "==== configuration ===" 96 | say Sneakers::CONFIG.inspect 97 | say "======================" 98 | end 99 | 100 | r.run 101 | end 102 | 103 | 104 | private 105 | def require_boot(file) 106 | load file 107 | end 108 | 109 | def em(text) 110 | shell.set_color(text, nil, true) 111 | end 112 | 113 | def ok(detail=nil) 114 | text = detail ? "OK, #{detail}." : "OK." 115 | say text, :green 116 | end 117 | 118 | def error(detail) 119 | say detail, :red 120 | end 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /lib/sneakers/concerns/logging.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Concerns 3 | module Logging 4 | def self.included(base) 5 | base.extend ClassMethods 6 | base.send :define_method, :logger do 7 | base.logger 8 | end 9 | end 10 | 11 | module ClassMethods 12 | def logger 13 | @logger 14 | end 15 | 16 | def logger=(logger) 17 | @logger = logger 18 | end 19 | 20 | def configure_logger(log=nil) 21 | if log 22 | @logger = log 23 | else 24 | @logger = Logger.new(STDOUT) 25 | @logger.level = Logger::INFO 26 | @logger.formatter = Sneakers::Support::ProductionFormatter 27 | end 28 | end 29 | end 30 | end 31 | end 32 | end 33 | 34 | 35 | -------------------------------------------------------------------------------- /lib/sneakers/concerns/metrics.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers/metrics/null_metrics' 2 | 3 | module Sneakers 4 | module Concerns 5 | module Metrics 6 | def self.included(base) 7 | base.extend ClassMethods 8 | base.send :define_method, :metrics do 9 | base.metrics 10 | end 11 | end 12 | 13 | module ClassMethods 14 | def metrics 15 | @metrics 16 | end 17 | 18 | def metrics=(metrics) 19 | @metrics = metrics 20 | end 21 | 22 | def configure_metrics(metrics=nil) 23 | if metrics 24 | @metrics = metrics 25 | else 26 | @metrics = Sneakers::Metrics::NullMetrics.new 27 | end 28 | end 29 | end 30 | end 31 | end 32 | end 33 | 34 | 35 | -------------------------------------------------------------------------------- /lib/sneakers/configuration.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers/error_reporter' 2 | require 'forwardable' 3 | 4 | module Sneakers 5 | class Configuration 6 | 7 | extend Forwardable 8 | def_delegators :@hash, :to_hash, :[], :[]=, :==, :fetch, :delete, :has_key?, :dig, :slice 9 | 10 | EXCHANGE_OPTION_DEFAULTS = { 11 | :type => :direct, 12 | :durable => true, 13 | :auto_delete => false, 14 | :arguments => {} # Passed as :arguments to Bunny::Channel#exchange 15 | }.freeze 16 | 17 | QUEUE_OPTION_DEFAULTS = { 18 | :durable => true, 19 | :auto_delete => false, 20 | :exclusive => false, 21 | :arguments => {} 22 | }.freeze 23 | 24 | DEFAULTS = { 25 | # Set up default handler which just logs the error. 26 | # Remove this in production if you don't want sensitive data logged. 27 | :error_reporters => [Sneakers::ErrorReporter::DefaultLogger.new], 28 | 29 | # runner 30 | :runner_config_file => nil, 31 | :metrics => nil, 32 | :daemonize => false, 33 | :start_worker_delay => 0.2, 34 | :workers => 4, 35 | :log => STDOUT, 36 | :pid_path => 'sneakers.pid', 37 | :amqp_heartbeat => 30, 38 | 39 | # Default values from serverengine 40 | :log_rotate_age => 5, 41 | :log_rotate_size => 1048576, 42 | :log_level => 'debug', 43 | 44 | # workers 45 | :prefetch => 10, 46 | :threads => 10, 47 | :share_threads => false, 48 | :ack => true, 49 | :heartbeat => 30, 50 | :hooks => {}, 51 | :exchange => 'sneakers', 52 | :exchange_options => EXCHANGE_OPTION_DEFAULTS, 53 | :queue_options => QUEUE_OPTION_DEFAULTS 54 | }.freeze 55 | 56 | 57 | def initialize 58 | clear 59 | end 60 | 61 | def clear 62 | @hash = DEFAULTS.dup 63 | @hash[:amqp] = ENV.fetch('RABBITMQ_URL', 'amqp://guest:guest@localhost:5672') 64 | @hash[:vhost] = AMQ::Settings.parse_amqp_url(@hash[:amqp]).fetch(:vhost, '/') 65 | end 66 | 67 | def merge!(hash) 68 | hash = hash.dup 69 | hash = map_all_deprecated_options(hash) 70 | 71 | # parse vhost from amqp if vhost is not specified explicitly, only 72 | # if we're not given a connection to use. 73 | if hash[:connection].nil? 74 | if hash[:vhost].nil? && !hash[:amqp].nil? 75 | hash[:vhost] = AMQ::Settings.parse_amqp_url(hash[:amqp]).fetch(:vhost, '/') 76 | end 77 | else 78 | # If we are given a Bunny object, ignore params we'd otherwise use to 79 | # create one. This removes any question about where config params are 80 | # coming from, and makes it more likely that downstream code that needs 81 | # this info gets it from the right place. 82 | [:vhost, :amqp, :heartbeat].each do |k| 83 | hash.delete(k) 84 | @hash.delete(k) 85 | end 86 | end 87 | 88 | @hash = deep_merge(@hash, hash) 89 | end 90 | 91 | def merge(hash) 92 | instance = self.class.new 93 | instance.merge! to_hash 94 | instance.merge! hash 95 | instance 96 | end 97 | 98 | def inspect_with_redaction 99 | redacted = self.class.new 100 | redacted.merge! to_hash 101 | 102 | # redact passwords 103 | redacted[:amqp] = redacted[:amqp].sub(/(?<=\Aamqp:\/)[^@]+(?=@)/, "<redacted>") if redacted.has_key?(:amqp) 104 | return redacted.inspect_without_redaction 105 | end 106 | alias_method :inspect_without_redaction, :inspect 107 | alias_method :inspect, :inspect_with_redaction 108 | 109 | def map_all_deprecated_options(hash) 110 | hash = map_deprecated_options_key(:exchange_options, :exchange_type, :type, true, hash) 111 | hash = map_deprecated_options_key(:exchange_options, :exchange_arguments, :arguments, true, hash) 112 | hash = map_deprecated_options_key(:exchange_options, :durable, :durable, false, hash) 113 | hash = map_deprecated_options_key(:queue_options, :durable, :durable, true, hash) 114 | hash = map_deprecated_options_key(:queue_options, :arguments, :arguments, true, hash) 115 | hash 116 | end 117 | 118 | def map_deprecated_options_key(target_key, deprecated_key, key, delete_deprecated_key, hash = {}) 119 | return hash if hash[deprecated_key].nil? 120 | hash = deep_merge({ target_key => { key => hash[deprecated_key] } }, hash) 121 | hash.delete(deprecated_key) if delete_deprecated_key 122 | hash 123 | end 124 | 125 | def deep_merge(first, second) 126 | merger = proc { |_, v1, v2| Hash === v1 && Hash === v2 ? v1.merge(v2, &merger) : v2 } 127 | first.merge(second, &merger) 128 | end 129 | end 130 | end 131 | -------------------------------------------------------------------------------- /lib/sneakers/content_encoding.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | class ContentEncoding 3 | def self.register(content_encoding: nil, encoder: nil, decoder: nil) 4 | # This can be removed when support is dropped for ruby 2.0 and replaced 5 | # by a keyword arg with no default value 6 | fail ArgumentError, 'missing keyword: content_encoding' if content_encoding.nil? 7 | fail ArgumentError, 'missing keyword: encoder' if encoder.nil? 8 | fail ArgumentError, 'missing keyword: decoder' if decoder.nil? 9 | 10 | fail ArgumentError, "#{content_encoding} encoder must be a proc" unless encoder.is_a? Proc 11 | fail ArgumentError, "#{content_encoding} decoder must be a proc" unless decoder.is_a? Proc 12 | 13 | fail ArgumentError, "#{content_encoding} encoder must accept one argument, the payload" unless encoder.arity == 1 14 | fail ArgumentError, "#{content_encoding} decoder must accept one argument, the payload" unless decoder.arity == 1 15 | @_encodings[content_encoding] = new(encoder, decoder) 16 | end 17 | 18 | def self.encode(payload, content_encoding) 19 | return payload unless content_encoding 20 | @_encodings[content_encoding].encoder.(payload) 21 | end 22 | 23 | def self.decode(payload, content_encoding) 24 | return payload unless content_encoding 25 | @_encodings[content_encoding].decoder.(payload) 26 | end 27 | 28 | def self.reset! 29 | @_encodings = Hash.new( 30 | new(passthrough, passthrough) 31 | ) 32 | end 33 | 34 | def self.passthrough 35 | ->(payload) { payload } 36 | end 37 | 38 | def initialize(encoder, decoder) 39 | @encoder = encoder 40 | @decoder = decoder 41 | end 42 | 43 | attr_reader :encoder, :decoder 44 | 45 | reset! 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/sneakers/content_type.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | class ContentType 3 | def self.register(content_type: nil, serializer: nil, deserializer: nil) 4 | # This can be removed when support is dropped for ruby 2.0 and replaced 5 | # by a keyword arg with no default value 6 | fail ArgumentError, 'missing keyword: content_type' if content_type.nil? 7 | fail ArgumentError, 'missing keyword: serializer' if serializer.nil? 8 | fail ArgumentError, 'missing keyword: deserializer' if deserializer.nil? 9 | 10 | fail ArgumentError, "#{content_type} serializer must be a proc" unless serializer.is_a? Proc 11 | fail ArgumentError, "#{content_type} deserializer must be a proc" unless deserializer.is_a? Proc 12 | 13 | fail ArgumentError, "#{content_type} serializer must accept one argument, the payload" unless serializer.arity == 1 14 | fail ArgumentError, "#{content_type} deserializer must accept one argument, the payload" unless deserializer.arity == 1 15 | @_types[content_type] = new(serializer, deserializer) 16 | end 17 | 18 | def self.serialize(payload, content_type) 19 | return payload unless content_type 20 | @_types[content_type].serializer.(payload) 21 | end 22 | 23 | def self.deserialize(payload, content_type) 24 | return payload unless content_type 25 | @_types[content_type].deserializer.(payload) 26 | end 27 | 28 | def self.reset! 29 | @_types = Hash.new( 30 | new(passthrough, passthrough) 31 | ) 32 | end 33 | 34 | def self.passthrough 35 | ->(payload) { payload } 36 | end 37 | 38 | def initialize(serializer, deserializer) 39 | @serializer = serializer 40 | @deserializer = deserializer 41 | end 42 | 43 | attr_reader :serializer, :deserializer 44 | 45 | reset! 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/sneakers/error_reporter.rb: -------------------------------------------------------------------------------- 1 | # Ripped off from https://github.com/mperham/sidekiq/blob/master/lib/sidekiq/exception_handler.rb 2 | module Sneakers 3 | module ErrorReporter 4 | class DefaultLogger 5 | def call(exception, worker, context_hash) 6 | Sneakers.logger.warn(context_hash) unless context_hash.empty? 7 | log_string = '' 8 | log_string += "[Exception error=#{exception.message.inspect} error_class=#{exception.class} worker_class=#{worker.class}" unless exception.nil? 9 | log_string += " backtrace=#{exception.backtrace.take(50).join(',')}" unless exception.nil? || exception.backtrace.nil? 10 | log_string += ']' 11 | Sneakers.logger.error log_string 12 | end 13 | end 14 | 15 | def worker_error(exception, context_hash = {}) 16 | Sneakers.error_reporters.each do |handler| 17 | begin 18 | handler.call(exception, self, context_hash) 19 | rescue SignalException, SystemExit 20 | # ServerEngine handles these exceptions, so they are not expected 21 | # to be raised within the error reporter. 22 | # Nevertheless, they are listed here to ensure that they are not 23 | # caught by the rescue block below. 24 | raise 25 | rescue Exception => inner_exception 26 | Sneakers.logger.error '!!! ERROR REPORTER THREW AN ERROR !!!' 27 | Sneakers.logger.error inner_exception 28 | Sneakers.logger.error inner_exception.backtrace.join("\n") unless inner_exception.backtrace.nil? 29 | end 30 | end 31 | end 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/sneakers/errors.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | end 3 | -------------------------------------------------------------------------------- /lib/sneakers/handlers/maxretry.rb: -------------------------------------------------------------------------------- 1 | require 'base64' 2 | require 'json' 3 | 4 | module Sneakers 5 | module Handlers 6 | # 7 | # Maxretry uses dead letter policies on Rabbitmq to requeue and retry 8 | # messages after failure (rejections and errors). When the maximum 9 | # number of retries is reached it will put the message on an error queue. 10 | # This handler will only retry at the queue level. To accomplish that, the 11 | # setup is a bit complex. 12 | # 13 | # Input: 14 | # worker_exchange (eXchange) 15 | # worker_queue (Queue) 16 | # We create: 17 | # worker_queue-retry - (X) where we setup the worker queue to dead-letter. 18 | # worker_queue-retry - (Q) queue bound to ^ exchange, dead-letters to 19 | # worker_queue-retry-requeue. 20 | # worker_queue-error - (X) where to send max-retry failures 21 | # worker_queue-error - (Q) bound to worker_queue-error. 22 | # worker_queue-retry-requeue - (X) exchange to bind worker_queue to for 23 | # requeuing directly to the worker_queue. 24 | # 25 | # This requires that you setup arguments to the worker queue to line up the 26 | # dead letter queue. See the example for more information. 27 | # 28 | # Many of these can be override with options: 29 | # - retry_exchange - sets retry exchange & queue 30 | # - retry_error_exchange - sets error exchange and queue 31 | # - retry_requeue_exchange - sets the exchange created to re-queue things 32 | # back to the worker queue. 33 | # 34 | class Maxretry 35 | 36 | def initialize(channel, queue, opts) 37 | @worker_queue_name = queue.name 38 | Sneakers.logger.debug do 39 | "#{log_prefix} creating handler, opts=#{opts}" 40 | end 41 | 42 | @channel = channel 43 | @opts = opts 44 | 45 | # Construct names, defaulting where suitable 46 | retry_name = @opts[:retry_exchange] || "#{@worker_queue_name}-retry" 47 | error_name = @opts[:retry_error_exchange] || "#{@worker_queue_name}-error" 48 | requeue_name = @opts[:retry_requeue_exchange] || "#{@worker_queue_name}-retry-requeue" 49 | retry_routing_key = @opts[:retry_routing_key] || "#" 50 | 51 | # Create the exchanges 52 | @retry_exchange, @error_exchange, @requeue_exchange = [retry_name, error_name, requeue_name].map do |name| 53 | Sneakers.logger.debug { "#{log_prefix} creating exchange=#{name}" } 54 | @channel.exchange(name, 55 | :type => 'topic', 56 | :durable => exchange_durable?) 57 | end 58 | 59 | # Create the queues and bindings 60 | Sneakers.logger.debug do 61 | "#{log_prefix} creating queue=#{retry_name} x-dead-letter-exchange=#{requeue_name}" 62 | end 63 | @retry_queue = @channel.queue(retry_name, 64 | :durable => queue_durable?, 65 | :arguments => { 66 | :'x-dead-letter-exchange' => requeue_name, 67 | :'x-message-ttl' => @opts[:retry_timeout] || 60000 68 | }) 69 | @retry_queue.bind(@retry_exchange, :routing_key => '#') 70 | 71 | Sneakers.logger.debug do 72 | "#{log_prefix} creating queue=#{error_name}" 73 | end 74 | @error_queue = @channel.queue(error_name, 75 | :durable => queue_durable?) 76 | @error_queue.bind(@error_exchange, :routing_key => '#') 77 | 78 | # Finally, bind the worker queue to our requeue exchange 79 | queue.bind(@requeue_exchange, :routing_key => retry_routing_key) 80 | 81 | @max_retries = @opts[:retry_max_times] || 5 82 | 83 | end 84 | 85 | def self.configure_queue(name, opts) 86 | retry_name = opts.fetch(:retry_exchange, "#{name}-retry") 87 | opt_args = if opts.dig(:queue_options, :arguments).blank? 88 | {} 89 | else 90 | opts.dig(:queue_options, :arguments).transform_keys(&:to_sym) 91 | end 92 | opts[:queue_options][:arguments] = { :'x-dead-letter-exchange' => retry_name }.merge!(opt_args) 93 | opts[:queue_options] 94 | end 95 | 96 | def acknowledge(hdr, props, msg) 97 | @channel.acknowledge(hdr.delivery_tag, false) 98 | end 99 | 100 | def reject(hdr, props, msg, requeue = false) 101 | if requeue 102 | # This was explicitly rejected specifying it be requeued so we do not 103 | # want it to pass through our retry logic. 104 | @channel.reject(hdr.delivery_tag, requeue) 105 | else 106 | handle_retry(hdr, props, msg, :reject) 107 | end 108 | end 109 | 110 | 111 | def error(hdr, props, msg, err) 112 | handle_retry(hdr, props, msg, err) 113 | end 114 | 115 | def noop(hdr, props, msg) 116 | 117 | end 118 | 119 | # Helper logic for retry handling. This will reject the message if there 120 | # are remaining retries left on it, otherwise it will publish it to the 121 | # error exchange along with the reason. 122 | # @param hdr [Bunny::DeliveryInfo] 123 | # @param props [Bunny::MessageProperties] 124 | # @param msg [String] The message 125 | # @param reason [String, Symbol, Exception] Reason for the retry, included 126 | # in the JSON we put on the error exchange. 127 | def handle_retry(hdr, props, msg, reason) 128 | # +1 for the current attempt 129 | num_attempts = failure_count(props[:headers]) + 1 130 | if num_attempts <= @max_retries 131 | # We call reject which will route the message to the 132 | # x-dead-letter-exchange (ie. retry exchange) on the queue 133 | Sneakers.logger.info do 134 | "#{log_prefix} msg=retrying, count=#{num_attempts}, headers=#{props[:headers]}" 135 | end 136 | @channel.reject(hdr.delivery_tag, false) 137 | # TODO: metrics 138 | else 139 | # Retried more than the max times 140 | # Publish the original message with the routing_key to the error exchange 141 | Sneakers.logger.info do 142 | "#{log_prefix} msg=failing, retry_count=#{num_attempts}, reason=#{reason}" 143 | end 144 | data = { 145 | error: reason.to_s, 146 | num_attempts: num_attempts, 147 | failed_at: Time.now.iso8601, 148 | properties: props.to_hash 149 | }.tap do |hash| 150 | if reason.is_a?(Exception) 151 | hash[:error_class] = reason.class.to_s 152 | hash[:error_message] = "#{reason}" 153 | if reason.backtrace 154 | hash[:backtrace] = reason.backtrace.take(10) 155 | end 156 | end 157 | end 158 | 159 | # Preserve retry log in a list 160 | if retry_info = props[:headers]['retry_info'] 161 | old_retry0 = JSON.parse(retry_info) rescue {error: "Failed to parse retry info"} 162 | old_retry = Array(old_retry0) 163 | # Prevent old retry from nesting 164 | data[:properties][:headers].delete('retry_info') 165 | data = old_retry.unshift(data) 166 | end 167 | 168 | @error_exchange.publish(msg, { 169 | routing_key: hdr.routing_key, 170 | headers: { 171 | retry_info: data.to_json 172 | } 173 | }) 174 | @channel.acknowledge(hdr.delivery_tag, false) 175 | # TODO: metrics 176 | end 177 | end 178 | private :handle_retry 179 | 180 | # Uses the x-death header to determine the number of failures this job has 181 | # seen in the past. This does not count the current failure. So for 182 | # instance, the first time the job fails, this will return 0, the second 183 | # time, 1, etc. 184 | # @param headers [Hash] Hash of headers that Rabbit delivers as part of 185 | # the message 186 | # @return [Integer] Count of number of failures. 187 | def failure_count(headers) 188 | if headers.nil? || headers['x-death'].nil? 189 | 0 190 | else 191 | x_death_array = headers['x-death'].select do |x_death| 192 | x_death['queue'] == @worker_queue_name 193 | end 194 | if x_death_array.count > 0 && x_death_array.first['count'] 195 | # Newer versions of RabbitMQ return headers with a count key 196 | x_death_array.inject(0) {|sum, x_death| sum + x_death['count']} 197 | else 198 | # Older versions return a separate x-death header for each failure 199 | x_death_array.count 200 | end 201 | end 202 | end 203 | private :failure_count 204 | 205 | # Prefix all of our log messages so they are easier to find. We don't have 206 | # the worker, so the next best thing is the queue name. 207 | def log_prefix 208 | "Maxretry handler [queue=#{@worker_queue_name}]" 209 | end 210 | private :log_prefix 211 | 212 | private 213 | 214 | def queue_durable? 215 | @opts.fetch(:queue_options, {}).fetch(:durable, false) 216 | end 217 | 218 | def exchange_durable? 219 | queue_durable? 220 | end 221 | end 222 | end 223 | end 224 | -------------------------------------------------------------------------------- /lib/sneakers/handlers/oneshot.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Handlers 3 | class Oneshot 4 | def initialize(channel, queue, opts) 5 | @channel = channel 6 | @opts = opts 7 | end 8 | 9 | def acknowledge(hdr, props, msg) 10 | @channel.acknowledge(hdr.delivery_tag, false) 11 | end 12 | 13 | def reject(hdr, props, msg, requeue=false) 14 | @channel.reject(hdr.delivery_tag, requeue) 15 | end 16 | 17 | def error(hdr, props, msg, err) 18 | reject(hdr, props, msg) 19 | end 20 | 21 | def noop(hdr, props, msg) 22 | 23 | end 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/sneakers/metrics/logging_metrics.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Metrics 3 | class LoggingMetrics 4 | def increment(metric) 5 | Sneakers.logger.info("INC: #{metric}") 6 | end 7 | 8 | def timing(metric, &block) 9 | start = Time.now 10 | block.call 11 | Sneakers.logger.info("TIME: #{metric} #{Time.now - start}") 12 | end 13 | end 14 | end 15 | end 16 | 17 | -------------------------------------------------------------------------------- /lib/sneakers/metrics/newrelic_metrics.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Metrics 3 | class NewrelicMetrics 4 | 5 | def self.eagent(eagent = nil) 6 | @eagent = eagent || @eagent 7 | end 8 | 9 | def initialize() 10 | #@connection = conn 11 | end 12 | 13 | def increment(metric) 14 | record_stat metric, 1 15 | end 16 | 17 | def record_stat(metric, num) 18 | metric_name = "Custom/#{metric.gsub("\.", "\/")}" 19 | NewrelicMetrics.eagent::Agent.record_metric(metric_name, num) 20 | rescue Exception => e 21 | puts "NewrelicMetrics#record_stat: #{e}" 22 | end 23 | 24 | def timing(metric, &block) 25 | start = Time.now 26 | block.call 27 | record_stat(metric, ((Time.now - start)*1000).floor) 28 | end 29 | end 30 | end 31 | end 32 | 33 | -------------------------------------------------------------------------------- /lib/sneakers/metrics/null_metrics.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Metrics 3 | class NullMetrics 4 | def increment(metric) 5 | end 6 | 7 | def timing(metric, &block) 8 | block.call 9 | end 10 | end 11 | end 12 | end 13 | 14 | -------------------------------------------------------------------------------- /lib/sneakers/metrics/statsd_metrics.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Metrics 3 | class StatsdMetrics 4 | def initialize(conn) 5 | @connection = conn 6 | end 7 | 8 | def increment(metric) 9 | @connection.increment(metric) 10 | end 11 | 12 | def timing(metric, &block) 13 | start = Time.now 14 | block.call 15 | @connection.timing(metric, ((Time.now - start)*1000).floor) 16 | end 17 | 18 | end 19 | end 20 | end 21 | 22 | -------------------------------------------------------------------------------- /lib/sneakers/middleware/config.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module Middleware 3 | class Config 4 | def self.use(klass, args) 5 | middlewares << { class: klass, args: args } 6 | end 7 | 8 | def self.delete(klass) 9 | middlewares.reject! { |el| el[:class] == klass } 10 | end 11 | 12 | def self.to_a 13 | middlewares 14 | end 15 | 16 | def self.middlewares 17 | @middlewares ||= [] 18 | end 19 | 20 | private_class_method :middlewares 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/sneakers/publisher.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | class Publisher 3 | 4 | attr_reader :exchange, :channel 5 | 6 | def initialize(opts = {}) 7 | @mutex = Mutex.new 8 | @opts = Sneakers::CONFIG.merge(opts) 9 | end 10 | 11 | def publish(msg, options = {}) 12 | ensure_connection! 13 | to_queue = options.delete(:to_queue) 14 | options[:routing_key] ||= to_queue 15 | Sneakers.logger.info {"publishing <#{msg}> to [#{options[:routing_key]}]"} 16 | serialized_msg = Sneakers::ContentType.serialize(msg, options[:content_type]) 17 | encoded_msg = Sneakers::ContentEncoding.encode(serialized_msg, options[:content_encoding]) 18 | @exchange.publish(encoded_msg, options) 19 | end 20 | 21 | def ensure_connection! 22 | @mutex.synchronize do 23 | connect! unless connected? 24 | end 25 | end 26 | 27 | private 28 | def connect! 29 | # If we've already got a bunny object, use it. This allows people to 30 | # specify all kinds of options we don't need to know about (e.g. for ssl). 31 | @bunny = @opts[:connection] 32 | if @bunny.respond_to?(:call) 33 | @bunny = @bunny.call 34 | else 35 | @bunny ||= create_bunny_connection 36 | @bunny.start 37 | end 38 | @channel = @bunny.create_channel 39 | @exchange = @channel.exchange(@opts[:exchange], **@opts[:exchange_options]) 40 | end 41 | 42 | def connected? 43 | @bunny && @bunny.connected? && channel 44 | end 45 | 46 | def create_bunny_connection 47 | Bunny.new(@opts[:amqp], :vhost => @opts[:vhost], 48 | :heartbeat => @opts[:heartbeat], 49 | :properties => @opts.fetch(:properties, {}), 50 | :logger => Sneakers::logger) 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/sneakers/queue.rb: -------------------------------------------------------------------------------- 1 | 2 | class Sneakers::Queue 3 | attr_reader :name, :opts, :exchange, :channel 4 | 5 | def initialize(name, opts) 6 | @name = name 7 | @opts = opts 8 | @handler_klass = Sneakers::CONFIG[:handler] 9 | end 10 | 11 | # 12 | # :exchange 13 | # :heartbeat_interval 14 | # :prefetch 15 | # :durable 16 | # :ack 17 | # 18 | def subscribe(worker) 19 | # If we've already got a bunny object, use it. This allows people to 20 | # specify all kinds of options we don't need to know about (e.g. for ssl). 21 | @bunny = @opts[:connection] 22 | if @bunny.respond_to?(:call) 23 | @bunny = @bunny.call 24 | else 25 | @bunny ||= create_bunny_connection 26 | @bunny.start 27 | end 28 | 29 | @channel = @bunny.create_channel 30 | @channel.prefetch(@opts[:prefetch]) 31 | 32 | exchange_name = @opts[:exchange] 33 | @exchange = @channel.exchange(exchange_name, **@opts[:exchange_options]) 34 | 35 | routing_key = @opts[:routing_key] || @name 36 | routing_keys = [*routing_key] 37 | 38 | handler_klass = worker.opts[:handler] || Sneakers::CONFIG.fetch(:handler) 39 | # Configure options if needed 40 | if handler_klass.respond_to?(:configure_queue) 41 | @opts[:queue_options] = handler_klass.configure_queue(@name, @opts) 42 | end 43 | 44 | queue = @channel.queue(@name, **@opts[:queue_options]) 45 | 46 | should_bind = @opts.fetch(:bind, true) 47 | if should_bind && exchange_name.length > 0 48 | routing_keys.each do |key| 49 | if @opts[:bind_arguments] 50 | queue.bind(@exchange, routing_key: key, arguments: @opts[:bind_arguments]) 51 | else 52 | queue.bind(@exchange, routing_key: key) 53 | end 54 | end 55 | end 56 | 57 | # NOTE: we are using the worker's options. This is necessary so the handler 58 | # has the same configuration as the worker. Also pass along the exchange and 59 | # queue in case the handler requires access to them (for things like binding 60 | # retry queues, etc). 61 | handler = handler_klass.new(@channel, queue, worker.opts) 62 | 63 | @consumer = queue.subscribe(block: false, manual_ack: @opts[:ack]) do | delivery_info, metadata, msg | 64 | worker.do_work(delivery_info, metadata, msg, handler) 65 | end 66 | nil 67 | end 68 | 69 | def unsubscribe 70 | return unless @consumer 71 | 72 | # TODO: should we simply close the channel here? 73 | Sneakers.logger.info("Queue: will try to cancel consumer #{@consumer.inspect}") 74 | cancel_ok = @consumer.cancel 75 | if cancel_ok 76 | Sneakers.logger.info "Queue: consumer #{cancel_ok.consumer_tag} cancelled" 77 | @consumer = nil 78 | else 79 | Sneakers.logger.warn "Queue: could not cancel consumer #{@consumer.inspect}" 80 | sleep(1) 81 | unsubscribe 82 | end 83 | end 84 | 85 | def create_bunny_connection 86 | Bunny.new(@opts[:amqp], { vhost: @opts[:vhost], 87 | heartbeat: @opts[:heartbeat], 88 | properties: @opts.fetch(:properties, {}), 89 | logger: Sneakers::logger }) 90 | end 91 | private :create_bunny_connection 92 | end 93 | -------------------------------------------------------------------------------- /lib/sneakers/runner.rb: -------------------------------------------------------------------------------- 1 | require 'serverengine' 2 | require 'sneakers/workergroup' 3 | 4 | module Sneakers 5 | class Runner 6 | def initialize(worker_classes, opts={}) 7 | @runnerconfig = RunnerConfig.new(worker_classes, opts) 8 | end 9 | 10 | def run 11 | @se = ServerEngine.create(nil, WorkerGroup) { @runnerconfig.reload_config! } 12 | @se.run 13 | end 14 | 15 | def stop(stop_graceful=true) 16 | @se.stop(stop_graceful) 17 | end 18 | end 19 | 20 | 21 | class RunnerConfig 22 | def method_missing(meth, *args, &block) 23 | if %w{ before_fork after_fork }.include? meth.to_s 24 | @conf[meth] = block 25 | elsif %w{ workers start_worker_delay amqp }.include? meth.to_s 26 | @conf[meth] = args.first 27 | else 28 | super 29 | end 30 | end 31 | 32 | def initialize(worker_classes, opts) 33 | @worker_classes = worker_classes 34 | @conf = opts 35 | end 36 | 37 | def to_h 38 | @conf 39 | end 40 | 41 | 42 | def reload_config! 43 | Sneakers.logger.info("Loading runner configuration...") 44 | config_file = Sneakers::CONFIG[:runner_config_file] 45 | 46 | if config_file 47 | begin 48 | instance_eval(File.read(config_file), config_file) 49 | Sneakers.logger.info("Loading config with file: #{config_file}") 50 | rescue 51 | Sneakers.logger.error("Cannot load from file '#{config_file}', #{$!}") 52 | end 53 | end 54 | 55 | config = make_serverengine_config 56 | 57 | [:before_fork, :after_fork].each do | hook | 58 | Sneakers::CONFIG[:hooks][hook] = config.delete(hook) if config[hook] 59 | end 60 | 61 | Sneakers.logger.debug("New configuration: #{config.inspect}") 62 | config 63 | end 64 | 65 | private 66 | 67 | def make_serverengine_config 68 | # From Sneakers#setup_general_logger, there's support for a Logger object 69 | # in CONFIG[:log]. However, serverengine takes an object in :logger. 70 | # Pass our logger object so there's no issue about sometimes passing a 71 | # file and sometimes an object. 72 | serverengine_config = Sneakers::CONFIG.merge(@conf) 73 | serverengine_config.merge!( 74 | :logger => Sneakers.logger, 75 | :log_level => Sneakers.logger.level, 76 | :worker_type => 'process', 77 | :worker_classes => @worker_classes, 78 | 79 | # Turning off serverengine internal logging infra, causes 80 | # livelock and hang. 81 | # see https://github.com/jondot/sneakers/issues/153 82 | :log_stdout => false, 83 | :log_stderr => false 84 | ) 85 | serverengine_config.delete(:log) 86 | 87 | serverengine_config 88 | end 89 | end 90 | 91 | end 92 | -------------------------------------------------------------------------------- /lib/sneakers/spawner.rb: -------------------------------------------------------------------------------- 1 | require 'yaml' 2 | require 'erb' 3 | 4 | module Sneakers 5 | class Spawner 6 | def self.spawn 7 | worker_group_config_file = ENV['WORKER_GROUP_CONFIG'] || './config/sneaker_worker_groups.yml' 8 | unless File.exist?(worker_group_config_file) 9 | puts 'No worker group file found.' 10 | puts "Specify via ENV 'WORKER_GROUP_CONFIG' or by convention ./config/sneaker_worker_groups.yml" 11 | Kernel.exit(1) 12 | end 13 | @pids = [] 14 | @exec_string = 'bundle exec rake sneakers:run' 15 | worker_config = YAML.load(ERB.new(File.read(worker_group_config_file)).result) 16 | worker_config.keys.each do |group_name| 17 | workers = worker_config[group_name]['classes'] 18 | workers = workers.join ',' if workers.is_a?(Array) 19 | @pids << fork do 20 | @exec_hash = { 'WORKERS' => workers, 'WORKER_COUNT' => worker_config[group_name]['workers'].to_s } 21 | Kernel.exec(@exec_hash, @exec_string) 22 | end 23 | end 24 | %w[TERM USR1 HUP USR2].each do |signal| 25 | Signal.trap(signal) { @pids.each { |pid| Process.kill(signal, pid) } } 26 | end 27 | Process.waitall 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /lib/sneakers/support/production_formatter.rb: -------------------------------------------------------------------------------- 1 | require 'time' 2 | module Sneakers 3 | module Support 4 | class ProductionFormatter < Logger::Formatter 5 | def self.call(severity, time, program_name, message) 6 | "#{time.utc.iso8601} p-#{Process.pid} t-#{Thread.current.object_id.to_s(36)} #{severity}: #{message}\n" 7 | end 8 | end 9 | end 10 | end 11 | 12 | -------------------------------------------------------------------------------- /lib/sneakers/support/utils.rb: -------------------------------------------------------------------------------- 1 | class Sneakers::Utils 2 | def self.make_worker_id(namespace) 3 | "worker-#{namespace}:#{'1'}:#{rand(36**6).floor.to_s(36)}" # jid, worker id. include date. 4 | end 5 | def self.parse_workers(workerstring) 6 | missing_workers = [] 7 | workers = (workerstring || '').split(',').map do |k| 8 | begin 9 | w = Kernel.const_get(k) 10 | rescue 11 | missing_workers << k 12 | end 13 | w 14 | end.compact 15 | 16 | [workers, missing_workers] 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/sneakers/tasks.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers' 2 | require 'sneakers/runner' 3 | 4 | task :environment 5 | 6 | namespace :sneakers do 7 | desc "Start work (set $WORKERS=Klass1,Klass2)" 8 | task :run do 9 | Sneakers.server = true 10 | Rake::Task['environment'].invoke 11 | 12 | if defined?(::Rails) 13 | if Rails.autoloaders.zeitwerk_enabled? 14 | ::Zeitwerk::Loader.eager_load_all 15 | else 16 | ::Rails.application.eager_load! 17 | end 18 | end 19 | 20 | workers, missing_workers = get_worker_classes 21 | 22 | unless missing_workers.nil? || missing_workers.empty? 23 | puts "Missing workers: #{missing_workers.join(', ')}" if missing_workers 24 | puts "Did you `require` properly?" 25 | exit(1) 26 | end 27 | 28 | if workers.empty? 29 | puts <<EOF 30 | Error: No workers found. 31 | Please set the classes of the workers you want to run like so: 32 | 33 | $ export WORKERS=MyWorker,FooWorker 34 | $ rake sneakers:run 35 | 36 | You can also configure them with 37 | $ Sneakers.rake_worker_classes 38 | 39 | If you use something that responds to :call it will execute that 40 | 41 | Eventually, if nothing before applied, every class is used where you directly included the Sneakers::Worker 42 | EOF 43 | exit(1) 44 | end 45 | opts = (!ENV['WORKER_COUNT'].nil? ? {:workers => ENV['WORKER_COUNT'].to_i} : {}) 46 | r = Sneakers::Runner.new(workers, opts) 47 | 48 | r.run 49 | end 50 | 51 | private 52 | 53 | def get_worker_classes 54 | if ENV["WORKERS"] 55 | Sneakers::Utils.parse_workers(ENV['WORKERS']) 56 | elsif Sneakers.rake_worker_classes 57 | if Sneakers.rake_worker_classes.respond_to?(:call) 58 | [Sneakers.rake_worker_classes.call] 59 | else 60 | [Sneakers.rake_worker_classes] 61 | end 62 | else 63 | [Sneakers::Worker::Classes] 64 | end || [[]] 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/sneakers/version.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | VERSION = "3.3.0.pre" 3 | end 4 | -------------------------------------------------------------------------------- /lib/sneakers/worker.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers/queue' 2 | require 'sneakers/support/utils' 3 | 4 | module Sneakers 5 | module Worker 6 | attr_reader :queue, :id, :opts 7 | 8 | # For now, a worker is hardly dependant on these concerns 9 | # (because it uses methods from them directly.) 10 | include Concerns::Logging 11 | include Concerns::Metrics 12 | include Sneakers::ErrorReporter 13 | 14 | def initialize(queue = nil, pool = nil, opts = {}) 15 | opts = opts.merge(self.class.queue_opts || {}) 16 | queue_name = self.class.queue_name 17 | opts = Sneakers::CONFIG.merge(opts) 18 | 19 | @should_ack = opts[:ack] 20 | @pool = pool || Concurrent::FixedThreadPool.new(opts[:threads] || Sneakers::Configuration::DEFAULTS[:threads]) 21 | @call_with_params = respond_to?(:work_with_params) 22 | @content_type = opts[:content_type] 23 | @content_encoding = opts[:content_encoding] 24 | 25 | @queue = queue || Sneakers::Queue.new( 26 | queue_name, 27 | opts 28 | ) 29 | 30 | @opts = opts 31 | @id = Utils.make_worker_id(queue_name) 32 | end 33 | 34 | def ack!; :ack end 35 | def reject!; :reject; end 36 | def requeue!; :requeue; end 37 | 38 | def publish(msg, opts) 39 | to_queue = opts.delete(:to_queue) 40 | opts[:routing_key] ||= to_queue 41 | return unless opts[:routing_key] 42 | serialized_msg = Sneakers::ContentType.serialize(msg, opts[:content_type]) 43 | encoded_msg = Sneakers::ContentEncoding.encode(serialized_msg, opts[:content_encoding]) 44 | @queue.exchange.publish(encoded_msg, **opts) 45 | end 46 | 47 | def do_work(delivery_info, metadata, msg, handler) 48 | worker_trace "Working off: #{msg.inspect}" 49 | 50 | @pool.post do 51 | process_work(delivery_info, metadata, msg, handler) 52 | end 53 | end 54 | 55 | def process_work(delivery_info, metadata, msg, handler) 56 | res = nil 57 | error = nil 58 | 59 | begin 60 | metrics.increment("work.#{self.class.name}.started") 61 | metrics.timing("work.#{self.class.name}.time") do 62 | decoded_msg = ContentEncoding.decode(msg, @content_encoding || metadata && metadata[:content_encoding]) 63 | deserialized_msg = ContentType.deserialize(decoded_msg, @content_type || metadata && metadata[:content_type]) 64 | 65 | app = -> (deserialized_msg, delivery_info, metadata, handler) do 66 | if @call_with_params 67 | work_with_params(deserialized_msg, delivery_info, metadata) 68 | else 69 | work(deserialized_msg) 70 | end 71 | end 72 | 73 | middlewares = Sneakers.middleware.to_a 74 | block_to_call = middlewares.reverse.reduce(app) do |mem, h| 75 | h[:class].new(mem, *h[:args]) 76 | end 77 | res = block_to_call.call(deserialized_msg, delivery_info, metadata, handler) 78 | end 79 | rescue SignalException, SystemExit, Bunny::Exception 80 | # ServerEngine handles these exceptions, so they are not expected to be raised within the worker. 81 | # Nevertheless, they are listed here to ensure that they are not caught by the rescue block below. 82 | raise 83 | rescue Exception => ex 84 | res = :error 85 | error = ex 86 | worker_error(ex, log_msg: log_msg(msg), class: self.class.name, 87 | message: msg, delivery_info: delivery_info, metadata: metadata) 88 | ensure 89 | if @should_ack 90 | case res 91 | # note to future-self. never acknowledge multiple (multiple=true) messages under threads. 92 | when :ack then handler.acknowledge(delivery_info, metadata, msg) 93 | when :error then handler.error(delivery_info, metadata, msg, error) 94 | when :reject then handler.reject(delivery_info, metadata, msg) 95 | when :requeue then handler.reject(delivery_info, metadata, msg, true) 96 | else 97 | handler.noop(delivery_info, metadata, msg) 98 | end 99 | metrics.increment("work.#{self.class.name}.handled.#{res || 'noop'}") 100 | end 101 | 102 | metrics.increment("work.#{self.class.name}.ended") 103 | end 104 | end 105 | 106 | def stop 107 | worker_trace "Stopping worker: unsubscribing." 108 | @queue.unsubscribe 109 | worker_trace "Stopping worker: shutting down thread pool." 110 | @pool.shutdown 111 | @pool.wait_for_termination 112 | worker_trace "Stopping worker: I'm gone." 113 | end 114 | 115 | def run 116 | worker_trace "New worker: subscribing." 117 | @queue.subscribe(self) 118 | worker_trace "New worker: I'm alive." 119 | end 120 | 121 | # Construct a log message with some standard prefix for this worker 122 | def log_msg(msg) 123 | "[#{@id}][#{Thread.current}][#{@queue.name}][#{@queue.opts}] #{msg}" 124 | end 125 | 126 | def worker_trace(msg) 127 | logger.debug(log_msg(msg)) 128 | end 129 | 130 | Classes = [] 131 | 132 | def self.included(base) 133 | base.extend ClassMethods 134 | Classes << base if base.is_a? Class 135 | end 136 | 137 | module ClassMethods 138 | attr_reader :queue_opts 139 | attr_reader :queue_name 140 | 141 | def from_queue(q, opts={}) 142 | @queue_name = q.to_s 143 | @queue_opts = opts 144 | end 145 | 146 | def enqueue(msg, opts={}) 147 | opts[:routing_key] ||= @queue_opts[:routing_key] 148 | opts[:content_type] ||= @queue_opts[:content_type] 149 | opts[:content_encoding] ||= @queue_opts[:content_encoding] 150 | opts[:to_queue] ||= @queue_name 151 | 152 | publisher.publish(msg, opts) 153 | end 154 | 155 | private 156 | 157 | def publisher 158 | @publisher ||= Sneakers::Publisher.new(queue_opts) 159 | end 160 | end 161 | end 162 | end 163 | -------------------------------------------------------------------------------- /lib/sneakers/workergroup.rb: -------------------------------------------------------------------------------- 1 | module Sneakers 2 | module WorkerGroup 3 | @workers = [] 4 | 5 | def initialize 6 | @stop_flag = ServerEngine::BlockingFlag.new 7 | end 8 | 9 | def before_fork 10 | fbefore = Sneakers::CONFIG[:hooks][:before_fork] 11 | fbefore.call if fbefore 12 | end 13 | 14 | def after_fork # note! this is not Serverengine#after_start, this is ours! 15 | fafter = Sneakers::CONFIG[:hooks][:after_fork] 16 | fafter.call if fafter 17 | end 18 | 19 | def run 20 | after_fork 21 | 22 | # Allocate single thread pool if share_threads is set. This improves load balancing 23 | # when used with many workers. 24 | pool = config[:share_threads] ? Concurrent::FixedThreadPool.new(config[:threads]) : nil 25 | 26 | worker_classes = config[:worker_classes] 27 | 28 | if worker_classes.respond_to? :call 29 | worker_classes = worker_classes.call 30 | end 31 | 32 | # If we don't provide a connection to a worker, 33 | # the queue used in the worker will create a new one 34 | 35 | @workers = worker_classes.map do |worker_class| 36 | worker_class.new(nil, pool, { connection: config[:connection] }) 37 | end 38 | 39 | # if more than one worker this should be per worker 40 | # accumulate clients and consumers as well 41 | @workers.each do |worker| 42 | worker.run 43 | end 44 | # end per worker 45 | # 46 | until @stop_flag.wait_for_set(Sneakers::CONFIG[:amqp_heartbeat]) 47 | Sneakers.logger.debug("Heartbeat: running threads [#{Thread.list.count}]") 48 | # report aggregated stats? 49 | end 50 | end 51 | 52 | def stop 53 | Sneakers.logger.info("Shutting down workers") 54 | @workers.each do |worker| 55 | worker.stop 56 | end 57 | @stop_flag.set! 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /log/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/ruby-amqp/kicks/b24caca8c4a69ea2e69ba40191550ee7422f57eb/log/.gitkeep -------------------------------------------------------------------------------- /scripts/local_integration: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker-compose run -e INTEGRATION=1 -e INTEGRATION_LOG=1 sneakers rake test 3 | -------------------------------------------------------------------------------- /scripts/local_worker: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | docker-compose run sneakers sneakers work TitleScraper --require examples/title_scraper.rb 3 | 4 | -------------------------------------------------------------------------------- /spec/fixtures/integration_worker.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers' 2 | require 'thread' 3 | require 'redis' 4 | 5 | 6 | redis_addr = compose_or_localhost("redis") 7 | puts "REDIS is at #{redis_addr}" 8 | $redis = Redis.new(:host => redis_addr) 9 | 10 | class IntegrationWorker 11 | include Sneakers::Worker 12 | 13 | def work(msg) 14 | $redis.incr(self.class.queue_name) 15 | ack! 16 | end 17 | end 18 | 19 | -------------------------------------------------------------------------------- /spec/fixtures/require_worker.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers' 2 | require 'open-uri' 3 | 4 | 5 | class TitleScraper 6 | include Sneakers::Worker 7 | 8 | from_queue 'downloads' 9 | 10 | def work(msg) 11 | title = extract_title(open(msg)) 12 | logger.info "FOUND <#{title}>" 13 | ack! 14 | end 15 | 16 | private 17 | 18 | def extract_title(html) 19 | html =~ /<title>(.*?)<\/title>/ 20 | $1 21 | end 22 | end 23 | 24 | -------------------------------------------------------------------------------- /spec/fixtures/test_job.rb: -------------------------------------------------------------------------------- 1 | require 'sneakers' 2 | require 'redis' 3 | 4 | redis_addr = compose_or_localhost('redis') 5 | puts "REDIS is at #{redis_addr}" 6 | $redis = Redis.new(host: redis_addr) 7 | 8 | 9 | class TestJob < ActiveJob::Base 10 | def perform(message) 11 | $redis.incr('rails_active_job') 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/gzip_helper.rb: -------------------------------------------------------------------------------- 1 | require 'stringio' 2 | require 'zlib' 3 | 4 | # Simple gzip encoder/decoder for testing 5 | def gzip_compress(s) 6 | io = StringIO.new('w') 7 | w = Zlib::GzipWriter.new(io) 8 | w.write(s) 9 | w.close 10 | io.string 11 | end 12 | 13 | def gzip_decompress(s) 14 | Zlib::GzipReader.new(StringIO.new(s, 'rb')).read 15 | end 16 | -------------------------------------------------------------------------------- /spec/sneakers/active_job_integration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | require 'rabbitmq/http/client' 5 | require 'active_job' 6 | require 'active_job/queue_adapters/sneakers_adapter' 7 | require 'fixtures/test_job' 8 | 9 | describe 'ActiveJob integration' do 10 | before :each do 11 | skip unless ENV['INTEGRATION'] 12 | prepare 13 | end 14 | 15 | def integration_log(msg) 16 | puts msg if ENV['INTEGRATION_LOG'] 17 | end 18 | 19 | def rmq_addr 20 | @rmq_addr ||= compose_or_localhost('rabbitmq') 21 | end 22 | 23 | def prepare 24 | ActiveJob::Base.queue_adapter = :sneakers 25 | 26 | Sneakers.clear! 27 | Sneakers.configure(amqp: "amqp://guest:guest@#{rmq_addr}:5672") 28 | Sneakers.logger.level = Logger::ERROR 29 | 30 | redis_addr = compose_or_localhost('redis') 31 | @redis = Redis.new(host: redis_addr) 32 | @redis.del('rails_active_job') 33 | end 34 | 35 | def wait_for_jobs_to_finish 36 | sleep 5 37 | end 38 | 39 | def start_active_job_workers 40 | integration_log 'starting ActiveJob workers.' 41 | runner = Sneakers::Runner.new([ActiveJob::QueueAdapters::SneakersAdapter::JobWrapper], {}) 42 | 43 | pid = fork { runner.run } 44 | 45 | integration_log 'waiting for workers to stabilize (5s).' 46 | sleep 5 47 | 48 | yield if block_given? 49 | ensure 50 | Process.kill('TERM', pid) rescue nil 51 | end 52 | 53 | it 'runs jobs enqueued on a listening queue' do 54 | start_active_job_workers do 55 | TestJob.perform_later('Hello Rails!') 56 | wait_for_jobs_to_finish 57 | assert_equal @redis.get('rails_active_job').to_i, 1 58 | end 59 | end 60 | 61 | it 'scheduling jobs are not supported' do 62 | assert_raises NotImplementedError, 'This queueing backend does not support scheduling jobs.' do 63 | TestJob.set(wait: 1.second).perform_later('Say Hello to Rails later!') 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /spec/sneakers/cli_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'sneakers/cli' 4 | require 'sneakers/runner' 5 | 6 | describe Sneakers::CLI do 7 | describe "#work" do 8 | before do 9 | any_instance_of(Sneakers::Runner) do |runner| 10 | stub(runner).run{ true } 11 | end 12 | end 13 | 14 | after do 15 | # require cleanup 16 | Object.send(:remove_const, :TitleScraper) if Object.constants.include?(:TitleScraper) 17 | end 18 | 19 | describe 'with dirty class loading' do 20 | it "should perform a run" do 21 | any_instance_of(Sneakers::Runner) do |runner| 22 | mock(runner).run{ true } 23 | end 24 | out = capture_io{ Sneakers::CLI.start [ 25 | 'work', 26 | "TitleScraper", 27 | "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}" 28 | ]}.join '' 29 | 30 | _(out).must_match(/Workers.*:.*TitleScraper.*/) 31 | 32 | end 33 | 34 | it "should be able to run as front-running process" do 35 | out = capture_io{ Sneakers::CLI.start [ 36 | 'work', 37 | "TitleScraper", 38 | "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}" 39 | ]}.join '' 40 | 41 | _(out).must_match(/Log.*Console/) 42 | end 43 | 44 | it "should be able to run as daemonized process" do 45 | out = capture_io{ Sneakers::CLI.start [ 46 | 'work', 47 | "TitleScraper", 48 | "--daemonize", 49 | "--require=#{File.expand_path('../fixtures/require_worker.rb', File.dirname(__FILE__))}" 50 | ]}.join '' 51 | 52 | _(out).must_match(/sneakers.log/) 53 | end 54 | end 55 | 56 | it "should fail when no workers found" do 57 | out = capture_io{ Sneakers::CLI.start ['work', 'TitleScraper'] }.join '' 58 | _(out).must_match(/Missing workers: TitleScraper/) 59 | end 60 | 61 | it "should run all workers when run without specifying any" do 62 | out = capture_io{ Sneakers::CLI.start [ 63 | "work", 64 | "--require=#{File.expand_path("../fixtures/require_worker.rb", File.dirname(__FILE__))}" 65 | ]}.join '' 66 | 67 | _(out).must_match(/Workers.*:.*TitleScraper.*/) 68 | end 69 | 70 | after do 71 | # require cleanup 72 | Object.send(:remove_const, :TitleScraper) if Object.constants.include?(:TitleScraper) 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /spec/sneakers/concerns/logging_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'logger' 4 | 5 | 6 | class Foobar 7 | include Sneakers::Concerns::Logging 8 | end 9 | 10 | describe Sneakers::Concerns::Logging do 11 | describe ".configure" do 12 | before do 13 | Foobar.logger = nil 14 | end 15 | 16 | it "should configure a default logger when included" do 17 | _(Foobar.logger).must_be_nil 18 | Foobar.configure_logger 19 | _(Foobar.logger).wont_be_nil 20 | _(Foobar.logger.formatter).must_equal Sneakers::Support::ProductionFormatter 21 | end 22 | 23 | it "should supply accessible instance logger" do 24 | _(Foobar.logger).must_be_nil 25 | Foobar.configure_logger 26 | f = Foobar.new 27 | _(f.logger).must_equal Foobar.logger 28 | _(f.logger).wont_be_nil 29 | end 30 | 31 | it "should configure a given logger when specified" do 32 | _(Foobar.logger).must_be_nil 33 | log = Logger.new(STDOUT) 34 | Foobar.configure_logger(log) 35 | _(Foobar.logger).must_equal log 36 | end 37 | end 38 | end 39 | 40 | -------------------------------------------------------------------------------- /spec/sneakers/concerns/metrics_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'logger' 4 | 5 | 6 | class Foometrics 7 | include Sneakers::Concerns::Metrics 8 | end 9 | 10 | describe Sneakers::Concerns::Metrics do 11 | describe ".configure" do 12 | before do 13 | Foometrics.metrics = nil 14 | end 15 | 16 | it "should configure a default logger when included" do 17 | _(Foometrics.metrics).must_be_nil 18 | Foometrics.configure_metrics 19 | _(Foometrics.metrics).wont_be_nil 20 | end 21 | 22 | it "should supply accessible instance logger" do 23 | _(Foometrics.metrics).must_be_nil 24 | Foometrics.configure_metrics 25 | f = Foometrics.new 26 | _(f.metrics).must_equal Foometrics.metrics 27 | _(f.metrics).wont_be_nil 28 | end 29 | 30 | it "should configure a given metrics when specified" do 31 | _(Foometrics.metrics).must_be_nil 32 | o = Object.new 33 | Foometrics.configure_metrics(o) 34 | _(Foometrics.metrics).must_equal o 35 | end 36 | end 37 | end 38 | 39 | -------------------------------------------------------------------------------- /spec/sneakers/configuration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe Sneakers::Configuration do 4 | 5 | describe 'with a connection' do 6 | let(:connection) { Object.new } 7 | 8 | it 'does not use vhost option if it is specified' do 9 | url = 'amqp://foo:bar@localhost:5672/foobarvhost' 10 | with_env('RABBITMQ_URL', url) do 11 | config = Sneakers::Configuration.new 12 | config.merge!({ :vhost => 'test_host', :connection => connection }) 13 | _(config.has_key?(:vhost)).must_equal false 14 | end 15 | end 16 | 17 | it 'does not amqp option if it is specified' do 18 | url = 'amqp://foo:bar@localhost:5672' 19 | config = Sneakers::Configuration.new 20 | config.merge!({ :amqp => url, :connection => connection }) 21 | _(config.has_key?(:vhost)).must_equal false 22 | end 23 | end 24 | 25 | describe 'without a connection' do 26 | it 'should assign a default value for :amqp' do 27 | with_env('RABBITMQ_URL', nil) do 28 | config = Sneakers::Configuration.new 29 | _(config[:amqp]).must_equal 'amqp://guest:guest@localhost:5672' 30 | end 31 | end 32 | 33 | it 'should assign a default value for :vhost' do 34 | with_env('RABBITMQ_URL', nil) do 35 | config = Sneakers::Configuration.new 36 | _(config[:vhost]).must_equal '/' 37 | end 38 | end 39 | 40 | it 'should read the value for amqp from RABBITMQ_URL' do 41 | url = 'amqp://foo:bar@localhost:5672' 42 | with_env('RABBITMQ_URL', url) do 43 | config = Sneakers::Configuration.new 44 | _(config[:amqp]).must_equal url 45 | end 46 | end 47 | 48 | it 'should read the value for vhost from RABBITMQ_URL' do 49 | url = 'amqp://foo:bar@localhost:5672/foobarvhost' 50 | with_env('RABBITMQ_URL', url) do 51 | config = Sneakers::Configuration.new 52 | _(config[:vhost]).must_equal 'foobarvhost' 53 | end 54 | end 55 | 56 | it 'should parse vhost from amqp option' do 57 | env_url = 'amqp://foo:bar@localhost:5672/foobarvhost' 58 | with_env('RABBITMQ_URL', env_url) do 59 | url = 'amqp://foo:bar@localhost:5672/testvhost' 60 | config = Sneakers::Configuration.new 61 | config.merge!({ :amqp => url }) 62 | _(config[:vhost]).must_equal 'testvhost' 63 | end 64 | end 65 | 66 | it 'should not parse vhost from amqp option if vhost is specified explicitly' do 67 | url = 'amqp://foo:bar@localhost:5672/foobarvhost' 68 | config = Sneakers::Configuration.new 69 | config.merge!({ :amqp => url, :vhost => 'test_host' }) 70 | _(config[:vhost]).must_equal 'test_host' 71 | end 72 | 73 | it 'should use vhost option if it is specified' do 74 | url = 'amqp://foo:bar@localhost:5672/foobarvhost' 75 | with_env('RABBITMQ_URL', url) do 76 | config = Sneakers::Configuration.new 77 | config.merge!({ :vhost => 'test_host' }) 78 | _(config[:vhost]).must_equal 'test_host' 79 | end 80 | end 81 | 82 | it 'should use default vhost if vhost is not specified in amqp option' do 83 | url = 'amqp://foo:bar@localhost:5672' 84 | config = Sneakers::Configuration.new 85 | config.merge!({ :amqp => url }) 86 | _(config[:vhost]).must_equal '/' 87 | end 88 | end 89 | 90 | def with_env(key, value) 91 | old_value = ENV[key] 92 | ENV[key] = value 93 | yield 94 | ensure 95 | ENV[key] = old_value 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/sneakers/content_encoding_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'gzip_helper' 3 | require 'sneakers/content_encoding' 4 | 5 | describe Sneakers::ContentEncoding do 6 | after do 7 | Sneakers::ContentEncoding.reset! 8 | end 9 | 10 | describe '.decode' do 11 | it 'uses the given decoder' do 12 | Sneakers::ContentEncoding.register( 13 | content_encoding: 'gzip', 14 | encoder: ->(_) {}, 15 | decoder: ->(payload) { gzip_decompress(payload) }, 16 | ) 17 | 18 | _(Sneakers::ContentEncoding.decode(gzip_compress('foobar'), 'gzip')).must_equal('foobar') 19 | end 20 | end 21 | 22 | describe '.encode' do 23 | it 'uses the given encoder' do 24 | Sneakers::ContentEncoding.register( 25 | content_encoding: 'gzip', 26 | encoder: ->(payload) { gzip_compress(payload) }, 27 | decoder: ->(_) {}, 28 | ) 29 | 30 | _(gzip_decompress(Sneakers::ContentEncoding.encode('foobar', 'gzip'))).must_equal('foobar') 31 | end 32 | 33 | it 'passes the payload through by default' do 34 | payload = "just some text" 35 | _(Sneakers::ContentEncoding.encode(payload, 'unknown/encoding')).must_equal(payload) 36 | _(Sneakers::ContentEncoding.decode(payload, 'unknown/encoding')).must_equal(payload) 37 | _(Sneakers::ContentEncoding.encode(payload, nil)).must_equal(payload) 38 | _(Sneakers::ContentEncoding.decode(payload, nil)).must_equal(payload) 39 | end 40 | 41 | it 'passes the payload through if type not found' do 42 | Sneakers::ContentEncoding.register(content_encoding: 'found/encoding', encoder: ->(_) {}, decoder: ->(_) {}) 43 | payload = "just some text" 44 | 45 | _(Sneakers::ContentEncoding.encode(payload, 'unknown/encoding')).must_equal(payload) 46 | _(Sneakers::ContentEncoding.decode(payload, 'unknown/encoding')).must_equal(payload) 47 | end 48 | end 49 | 50 | describe '.register' do 51 | it 'provides a mechnism to register a given encoding' do 52 | Sneakers::ContentEncoding.register( 53 | content_encoding: 'gzip', 54 | encoder: ->(payload) { gzip_compress(payload) }, 55 | decoder: ->(payload) { gzip_decompress(payload) }, 56 | ) 57 | 58 | ce = Sneakers::ContentEncoding 59 | _(ce.decode(ce.encode('hello world', 'gzip'), 'gzip')).must_equal('hello world') 60 | end 61 | 62 | it 'requires a content encoding' do 63 | _(proc { Sneakers::ContentEncoding.register(encoder: -> { }, decoder: -> { }) }).must_raise ArgumentError 64 | end 65 | 66 | it 'expects encoder and decoder to be present' do 67 | _(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', decoder: -> { }) }).must_raise ArgumentError 68 | _(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: -> { }) }).must_raise ArgumentError 69 | end 70 | 71 | it 'expects encoder and decoder to be a proc' do 72 | _(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: 'not a proc', decoder: ->(_) { }) }).must_raise ArgumentError 73 | _(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: ->(_) {}, decoder: 'not a proc' ) }).must_raise ArgumentError 74 | end 75 | 76 | it 'expects encoder and deseridecoderalizer to have the correct arity' do 77 | _(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: ->(_,_) {}, decoder: ->(_) {}) }).must_raise ArgumentError 78 | _(proc { Sneakers::ContentEncoding.register(content_encoding: 'foo', encoder: ->(_) {}, decoder: ->() {} ) }).must_raise ArgumentError 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/sneakers/content_type_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers/content_type' 3 | require 'base64' 4 | 5 | describe Sneakers::ContentType do 6 | after do 7 | Sneakers::ContentType.reset! 8 | end 9 | 10 | describe '.deserialize' do 11 | it 'uses the given deserializer' do 12 | Sneakers::ContentType.register( 13 | content_type: 'application/json', 14 | serializer: ->(_) {}, 15 | deserializer: ->(payload) { JSON.parse(payload) }, 16 | ) 17 | 18 | _(Sneakers::ContentType.deserialize('{"foo":"bar"}', 'application/json')).must_equal('foo' => 'bar') 19 | end 20 | end 21 | 22 | describe '.serialize' do 23 | it 'uses the given serializer' do 24 | Sneakers::ContentType.register( 25 | content_type: 'application/json', 26 | serializer: ->(payload) { JSON.dump(payload) }, 27 | deserializer: ->(_) {}, 28 | ) 29 | 30 | _(Sneakers::ContentType.serialize({ 'foo' => 'bar' }, 'application/json')).must_equal('{"foo":"bar"}') 31 | end 32 | 33 | it 'passes the payload through by default' do 34 | payload = "just some text" 35 | _(Sneakers::ContentType.serialize(payload, 'unknown/type')).must_equal(payload) 36 | _(Sneakers::ContentType.deserialize(payload, 'unknown/type')).must_equal(payload) 37 | _(Sneakers::ContentType.serialize(payload, nil)).must_equal(payload) 38 | _(Sneakers::ContentType.deserialize(payload, nil)).must_equal(payload) 39 | end 40 | 41 | it 'passes the payload through if type not found' do 42 | Sneakers::ContentType.register(content_type: 'found/type', serializer: ->(_) {}, deserializer: ->(_) {}) 43 | payload = "just some text" 44 | 45 | _(Sneakers::ContentType.serialize(payload, 'unknown/type')).must_equal(payload) 46 | _(Sneakers::ContentType.deserialize(payload, 'unknown/type')).must_equal(payload) 47 | end 48 | end 49 | 50 | describe '.register' do 51 | it 'provides a mechnism to register a given type' do 52 | Sneakers::ContentType.register( 53 | content_type: 'text/base64', 54 | serializer: ->(payload) { Base64.encode64(payload) }, 55 | deserializer: ->(payload) { Base64.decode64(payload) }, 56 | ) 57 | 58 | ct = Sneakers::ContentType 59 | _(ct.deserialize(ct.serialize('hello world', 'text/base64'), 'text/base64')).must_equal('hello world') 60 | end 61 | 62 | it 'requires a content type' do 63 | _(proc { Sneakers::ContentType.register(serializer: -> { }, deserializer: -> { }) }).must_raise ArgumentError 64 | end 65 | 66 | it 'expects serializer and deserializer to be present' do 67 | _(proc { Sneakers::ContentType.register(content_type: 'foo', deserializer: -> { }) }).must_raise ArgumentError 68 | _(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: -> { }) }).must_raise ArgumentError 69 | end 70 | 71 | it 'expects serializer and deserializer to be a proc' do 72 | _(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: 'not a proc', deserializer: ->(_) { }) }).must_raise ArgumentError 73 | _(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: 'not a proc' ) }).must_raise ArgumentError 74 | end 75 | 76 | it 'expects serializer and deserializer to have the correct arity' do 77 | _(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_,_) {}, deserializer: ->(_) {}) }).must_raise ArgumentError 78 | _(proc { Sneakers::ContentType.register(content_type: 'foo', serializer: ->(_) {}, deserializer: ->() {} ) }).must_raise ArgumentError 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/sneakers/integration_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'sneakers/runner' 4 | require 'fixtures/integration_worker' 5 | 6 | require "rabbitmq/http/client" 7 | require 'timeout' 8 | 9 | 10 | describe "integration" do 11 | describe 'first' do 12 | before :each do 13 | skip unless ENV['INTEGRATION'] 14 | prepare 15 | end 16 | 17 | def integration_log(msg) 18 | puts msg if ENV['INTEGRATION_LOG'] 19 | end 20 | 21 | def rmq_addr 22 | @rmq_addr ||= compose_or_localhost("rabbitmq") 23 | end 24 | 25 | def admin 26 | @admin ||= 27 | begin 28 | puts "RABBITMQ is at #{rmq_addr}" 29 | RabbitMQ::HTTP::Client.new("http://#{rmq_addr}:15672/", username: "guest", password: "guest") 30 | rescue 31 | fail "Rabbitmq admin seems to not exist? you better be running this on Travis or Docker. proceeding.\n#{$!}" 32 | end 33 | end 34 | 35 | def prepare 36 | # clean up all integration queues; admin interface must be installed 37 | # in integration env 38 | qs = admin.list_queues 39 | qs.each do |q| 40 | name = q.name 41 | if name.start_with? 'integration_' 42 | admin.delete_queue('/', name) 43 | integration_log "cleaning up #{name}." 44 | end 45 | end 46 | 47 | Sneakers.clear! 48 | Sneakers.configure(:amqp => "amqp://guest:guest@#{rmq_addr}:5672") 49 | Sneakers.logger.level = Logger::ERROR 50 | 51 | # configure integration worker on a random generated queue 52 | random_queue = "integration_#{rand(10**36).to_s(36)}" 53 | 54 | redis_addr = compose_or_localhost("redis") 55 | @redis = Redis.new(:host => redis_addr) 56 | @redis.del(random_queue) 57 | IntegrationWorker.from_queue(random_queue) 58 | end 59 | 60 | def assert_all_accounted_for(opts) 61 | integration_log 'waiting for publishes to stabilize (5s).' 62 | sleep 5 63 | 64 | integration_log "polling for changes (max #{opts[:within_sec]}s)." 65 | pid = opts[:pid] 66 | opts[:within_sec].times do 67 | sleep 1 68 | count = @redis.get(opts[:queue]).to_i 69 | if count == opts[:jobs] 70 | integration_log "#{count} jobs accounted for successfully." 71 | Process.kill("TERM", pid) 72 | sleep 1 73 | return 74 | end 75 | end 76 | 77 | integration_log "failed test. killing off workers." 78 | Process.kill("TERM", pid) 79 | sleep 1 80 | fail "incomplete!" 81 | end 82 | 83 | def start_worker(w) 84 | integration_log "starting workers." 85 | r = Sneakers::Runner.new([w]) 86 | pid = fork { 87 | r.run 88 | } 89 | 90 | integration_log "waiting for workers to stabilize (5s)." 91 | sleep 5 92 | 93 | pid 94 | end 95 | 96 | def consumers_count 97 | qs = admin.list_queues 98 | qs.each do |q| 99 | if q.name.start_with? 'integration_' 100 | return [q.consumers, q.name] 101 | end 102 | end 103 | return [0, nil] 104 | end 105 | 106 | def assert_any_consumers(consumers_should_be_there, maximum_wait_time = 15) 107 | Timeout::timeout(maximum_wait_time) do 108 | loop do 109 | consumers, queue = consumers_count 110 | fail 'no queues so no consumers' if consumers_should_be_there && !queue 111 | puts "We see #{consumers} consumers on #{queue}" 112 | (consumers_should_be_there == consumers.zero?) ? sleep(1) : return 113 | end 114 | end 115 | rescue Timeout::Error 116 | fail "Consumers should #{'not' unless consumers_should_be_there} be here but #{consumers} consumers were after #{maximum_wait_time}s waiting." 117 | end 118 | 119 | it 'should be possible to terminate when queue is full' do 120 | job_count = 40000 121 | 122 | pid = start_worker(IntegrationWorker) 123 | Process.kill("TERM", pid) 124 | 125 | integration_log "publishing #{job_count} messages..." 126 | p = Sneakers::Publisher.new 127 | job_count.times do |i| 128 | p.publish("m #{i}", to_queue: IntegrationWorker.queue_name) 129 | end 130 | 131 | pid = start_worker(IntegrationWorker) 132 | assert_any_consumers true 133 | integration_log "Killing #{pid} now!" 134 | Process.kill("TERM", pid) 135 | assert_any_consumers false 136 | end 137 | 138 | it 'should pull down 100 jobs from a real queue' do 139 | job_count = 100 140 | 141 | pid = start_worker(IntegrationWorker) 142 | 143 | integration_log "publishing..." 144 | p = Sneakers::Publisher.new 145 | job_count.times do |i| 146 | p.publish("m #{i}", to_queue: IntegrationWorker.queue_name) 147 | end 148 | 149 | assert_all_accounted_for( 150 | queue: IntegrationWorker.queue_name, 151 | pid: pid, 152 | within_sec: 15, 153 | jobs: job_count, 154 | ) 155 | end 156 | 157 | end 158 | end 159 | -------------------------------------------------------------------------------- /spec/sneakers/publisher_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'gzip_helper' 3 | require 'sneakers' 4 | require 'serverengine' 5 | 6 | describe Sneakers::Publisher do 7 | let :pub_vars do 8 | { 9 | :prefetch => 25, 10 | :durable => true, 11 | :ack => true, 12 | :heartbeat => 2, 13 | :vhost => '/', 14 | :exchange => "sneakers", 15 | :exchange_type => :direct, 16 | :exchange_arguments => { 'x-arg' => 'value' } 17 | } 18 | end 19 | 20 | describe '#publish' do 21 | before do 22 | Sneakers.clear! 23 | Sneakers.configure(:log => 'sneakers.log') 24 | end 25 | 26 | it 'should publish a message to an exchange' do 27 | xchg = Object.new 28 | mock(xchg).publish('test msg', { routing_key: 'downloads' }) 29 | 30 | p = Sneakers::Publisher.new 31 | p.instance_variable_set(:@exchange, xchg) 32 | 33 | mock(p).ensure_connection! {} 34 | p.publish('test msg', to_queue: 'downloads') 35 | end 36 | 37 | it 'should publish with the persistence specified' do 38 | xchg = Object.new 39 | mock(xchg).publish('test msg', { routing_key: 'downloads', persistence: true }) 40 | 41 | p = Sneakers::Publisher.new 42 | p.instance_variable_set(:@exchange, xchg) 43 | 44 | mock(p).ensure_connection! {} 45 | p.publish('test msg', to_queue: 'downloads', persistence: true) 46 | end 47 | 48 | it 'should publish with arbitrary metadata specified' do 49 | xchg = Object.new 50 | mock(xchg).publish('test msg', { routing_key: 'downloads', expiration: 1, headers: {foo: 'bar'} }) 51 | 52 | p = Sneakers::Publisher.new 53 | p.instance_variable_set(:@exchange, xchg) 54 | 55 | mock(p).ensure_connection! {} 56 | p.publish('test msg', to_queue: 'downloads', expiration: 1, headers: {foo: 'bar'}) 57 | end 58 | 59 | it 'should not reconnect if already connected' do 60 | xchg = Object.new 61 | mock(xchg).publish('test msg', { routing_key: 'downloads' }) 62 | 63 | p = Sneakers::Publisher.new 64 | p.instance_variable_set(:@exchange, xchg) 65 | 66 | mock(p).connected? { true } 67 | mock(p).connect!.times(0) 68 | 69 | p.publish('test msg', to_queue: 'downloads') 70 | end 71 | 72 | it 'should connect to rabbitmq configured on Sneakers.configure' do 73 | logger = Logger.new('/dev/null') 74 | Sneakers.configure( 75 | amqp: 'amqp://someuser:somepassword@somehost:5672', 76 | heartbeat: 1, 77 | exchange: 'another_exchange', 78 | exchange_options: { :type => :topic, :arguments => { 'x-arg' => 'value' } }, 79 | log: logger, 80 | properties: { key: "value" }, 81 | durable: false) 82 | 83 | channel = Object.new 84 | mock(channel).exchange('another_exchange', type: :topic, durable: false, :auto_delete => false, arguments: { 'x-arg' => 'value' }) do 85 | mock(Object.new).publish('test msg', { routing_key: 'downloads' }) 86 | end 87 | 88 | bunny = Object.new 89 | mock(bunny).start 90 | mock(bunny).create_channel { channel } 91 | 92 | mock(Bunny).new('amqp://someuser:somepassword@somehost:5672', heartbeat: 1, vhost: '/', logger: logger, properties: { key: "value" }) { bunny } 93 | 94 | p = Sneakers::Publisher.new 95 | 96 | p.publish('test msg', to_queue: 'downloads') 97 | end 98 | 99 | describe 'externally instantiated bunny session' do 100 | let(:my_vars) { pub_vars.merge(to_queue: 'downloads') } 101 | before do 102 | logger = Logger.new('/dev/null') 103 | channel = Object.new 104 | exchange = Object.new 105 | existing_session = Bunny.new 106 | 107 | mock(existing_session).start 108 | mock(existing_session).create_channel { channel } 109 | 110 | mock(channel).exchange('another_exchange', type: :topic, durable: false, :auto_delete => false, arguments: { 'x-arg' => 'value' }) do 111 | exchange 112 | end 113 | 114 | mock(exchange).publish('test msg', my_vars) 115 | 116 | Sneakers.configure( 117 | connection: existing_session, 118 | heartbeat: 1, exchange: 'another_exchange', 119 | exchange_type: :topic, 120 | exchange_arguments: { 'x-arg' => 'value' }, 121 | log: logger, 122 | durable: false 123 | ) 124 | @existing_session = existing_session 125 | @exchange = exchange 126 | @channel = channel 127 | end 128 | 129 | it 'can handle an existing connection object' do 130 | p = Sneakers::Publisher.new 131 | p.publish('test msg', my_vars) 132 | _(p.instance_variable_get(:@bunny)).must_equal @existing_session 133 | end 134 | 135 | it 'can handle an existing connection function' do 136 | @existing_session.start 137 | p = Sneakers::Publisher.new(connection: ->() { @existing_session }) 138 | p.publish('test msg', my_vars) 139 | _(p.instance_variable_get(:@bunny)).must_equal @existing_session 140 | end 141 | 142 | it 'can handle an existing connection that is online' do 143 | p = Sneakers::Publisher.new 144 | p.publish('test msg', my_vars) 145 | _(p.instance_variable_get(:@bunny)).must_equal @existing_session 146 | mock(@existing_session).connected? { true } 147 | mock(@exchange).publish('test msg 2', my_vars) 148 | p.publish('test msg 2', my_vars) 149 | end 150 | 151 | it 'can handle an existing connection that goes offline' do 152 | p = Sneakers::Publisher.new 153 | p.publish('test msg', my_vars) 154 | _(p.instance_variable_get(:@bunny)).must_equal @existing_session 155 | mock(@existing_session).connected? { false } 156 | mock(@existing_session).start 157 | mock(@existing_session).create_channel { @channel } 158 | mock(@channel).exchange('another_exchange', type: :topic, durable: false, :auto_delete => false, arguments: { 'x-arg' => 'value' }) do 159 | @exchange 160 | end 161 | mock(@exchange).publish('test msg 2', my_vars) 162 | p.publish('test msg 2', my_vars) 163 | end 164 | end 165 | 166 | it 'should publish using the content type serializer' do 167 | Sneakers::ContentType.register( 168 | content_type: 'application/json', 169 | serializer: ->(payload) { JSON.dump(payload) }, 170 | deserializer: ->(_) {}, 171 | ) 172 | 173 | xchg = Object.new 174 | mock(xchg).publish('{"foo":"bar"}', { routing_key: 'downloads', content_type: 'application/json' }) 175 | 176 | p = Sneakers::Publisher.new 177 | p.instance_variable_set(:@exchange, xchg) 178 | 179 | mock(p).ensure_connection! {} 180 | p.publish({ 'foo' => 'bar' }, to_queue: 'downloads', content_type: 'application/json') 181 | 182 | Sneakers::ContentType.reset! 183 | end 184 | 185 | it 'should publish using the content encoding encoder' do 186 | Sneakers::ContentEncoding.register( 187 | content_encoding: 'gzip', 188 | encoder: ->(payload) { gzip_compress(payload) }, 189 | decoder: ->(_) {}, 190 | ) 191 | 192 | xchg = Object.new 193 | mock(xchg).publish(gzip_compress('foobar'), { routing_key: 'downloads', content_encoding: 'gzip' }) 194 | 195 | p = Sneakers::Publisher.new 196 | p.instance_variable_set(:@exchange, xchg) 197 | 198 | mock(p).ensure_connection! {} 199 | p.publish('foobar', to_queue: 'downloads', content_encoding: 'gzip') 200 | 201 | Sneakers::ContentEncoding.reset! 202 | end 203 | end 204 | end 205 | -------------------------------------------------------------------------------- /spec/sneakers/queue_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | 4 | describe Sneakers::Queue do 5 | let :queue_vars do 6 | { 7 | :prefetch => 25, 8 | :ack => true, 9 | :heartbeat => 2, 10 | :vhost => '/', 11 | :exchange => "sneakers", 12 | :exchange_options => { 13 | :type => :direct, 14 | durable: true, 15 | :arguments => { 'x-arg' => 'value' } 16 | }, 17 | queue_options: { 18 | durable: true 19 | } 20 | } 21 | end 22 | 23 | before do 24 | Sneakers.clear! 25 | Sneakers.configure 26 | 27 | @mkworker = Object.new 28 | stub(@mkworker).opts { { :exchange => 'test-exchange' } } 29 | @mkchan = Object.new 30 | mock(@mkchan).prefetch(25) 31 | @mkex = Object.new 32 | @mkqueue = Object.new 33 | end 34 | 35 | describe 'with our own Bunny object' do 36 | before do 37 | @mkbunny = Object.new 38 | @mkqueue_nondurable = Object.new 39 | 40 | mock(@mkbunny).start {} 41 | mock(@mkbunny).create_channel{ @mkchan } 42 | mock(Bunny).new( 43 | anything, 44 | hash_including(:vhost => '/', :heartbeat => 2) 45 | ){ @mkbunny } 46 | end 47 | 48 | describe "#subscribe with sneakers exchange" do 49 | before do 50 | mock(@mkchan).exchange("sneakers", 51 | :type => :direct, 52 | :durable => true, 53 | :arguments => { 'x-arg' => 'value' }){ @mkex } 54 | end 55 | 56 | it "should setup a bunny queue according to configuration values" do 57 | mock(@mkchan).queue("downloads", :durable => true) { @mkqueue } 58 | q = Sneakers::Queue.new("downloads", queue_vars) 59 | 60 | mock(@mkqueue).bind(@mkex, :routing_key => "downloads") 61 | mock(@mkqueue).subscribe(:block => false, :manual_ack => true) 62 | 63 | q.subscribe(@mkworker) 64 | end 65 | 66 | it "supports multiple routing_keys" do 67 | mock(@mkchan).queue("downloads", :durable => true) { @mkqueue } 68 | q = Sneakers::Queue.new("downloads", 69 | queue_vars.merge(:routing_key => ["alpha", "beta"])) 70 | 71 | mock(@mkqueue).bind(@mkex, :routing_key => "alpha") 72 | mock(@mkqueue).bind(@mkex, :routing_key => "beta") 73 | mock(@mkqueue).subscribe(:block => false, :manual_ack => true) 74 | 75 | q.subscribe(@mkworker) 76 | end 77 | 78 | it "supports setting arguments when binding" do 79 | mock(@mkchan).queue("downloads", :durable => true) { @mkqueue } 80 | q = Sneakers::Queue.new("downloads", 81 | queue_vars.merge(:bind_arguments => { "os" => "linux", "cores" => 8 })) 82 | 83 | mock(@mkqueue).bind(@mkex, :routing_key => "downloads", :arguments => { "os" => "linux", "cores" => 8 }) 84 | mock(@mkqueue).subscribe(:block => false, :manual_ack => true) 85 | 86 | q.subscribe(@mkworker) 87 | end 88 | 89 | it "will use whatever handler the worker specifies" do 90 | mock(@mkchan).queue("downloads", :durable => true) { @mkqueue } 91 | @handler = Object.new 92 | worker_opts = { :handler => @handler } 93 | stub(@mkworker).opts { worker_opts } 94 | mock(@handler).new(@mkchan, @mkqueue, worker_opts).once 95 | 96 | stub(@mkqueue).bind 97 | stub(@mkqueue).subscribe 98 | q = Sneakers::Queue.new("downloads", queue_vars) 99 | q.subscribe(@mkworker) 100 | end 101 | 102 | it "creates a non-durable queue if :queue_durable => false" do 103 | mock(@mkchan).queue("test_nondurable", :durable => false) { @mkqueue_nondurable } 104 | queue_vars[:queue_options][:durable] = false 105 | q = Sneakers::Queue.new("test_nondurable", queue_vars) 106 | 107 | mock(@mkqueue_nondurable).bind(@mkex, :routing_key => "test_nondurable") 108 | mock(@mkqueue_nondurable).subscribe(:block => false, :manual_ack => true) 109 | 110 | q.subscribe(@mkworker) 111 | myqueue = q.instance_variable_get(:@queue) 112 | end 113 | end 114 | 115 | describe "#subscribe with default exchange" do 116 | before do 117 | # expect default exchange 118 | queue_vars[:exchange] = "" 119 | mock(@mkchan).exchange("", 120 | :type => :direct, 121 | :durable => true, 122 | :arguments => {"x-arg" => "value"}){ @mkex } 123 | end 124 | 125 | it "does not bind to exchange" do 126 | mock(@mkchan).queue("downloads", :durable => true) { @mkqueue } 127 | @handler = Object.new 128 | worker_opts = { :handler => @handler } 129 | stub(@mkworker).opts { worker_opts } 130 | mock(@handler).new(@mkchan, @mkqueue, worker_opts).once 131 | 132 | stub(@mkqueue).bind do 133 | raise "bind should not be called" 134 | end 135 | 136 | stub(@mkqueue).subscribe 137 | q = Sneakers::Queue.new("downloads", queue_vars) 138 | q.subscribe(@mkworker) 139 | end 140 | end 141 | end 142 | 143 | describe 'with an externally-provided connection' do 144 | describe '#subscribe' do 145 | before do 146 | @external_connection = Bunny.new 147 | mock(@external_connection).start {} 148 | mock(@external_connection).create_channel{ @mkchan } 149 | mock(@mkchan).exchange("sneakers", 150 | :type => :direct, 151 | :durable => true, 152 | :arguments => { 'x-arg' => 'value' }){ @mkex } 153 | mock(@mkchan).queue('foo', :durable => true) { @mkqueue } 154 | mock(@mkqueue).bind(@mkex, :routing_key => 'foo') 155 | mock(@mkqueue).subscribe(:block => false, :manual_ack => true) 156 | end 157 | 158 | it 'uses that object' do 159 | q = Sneakers::Queue.new('foo', 160 | queue_vars.merge(:connection => @external_connection)) 161 | q.subscribe(@mkworker) 162 | _(q.instance_variable_get(:@bunny)).must_equal @external_connection 163 | end 164 | 165 | it 'uses that function' do 166 | @external_connection.start 167 | q = Sneakers::Queue.new('foo', 168 | queue_vars.merge(:connection => ->() { @external_connection })) 169 | q.subscribe(@mkworker) 170 | _(q.instance_variable_get(:@bunny)).must_equal @external_connection 171 | end 172 | end 173 | end 174 | end 175 | -------------------------------------------------------------------------------- /spec/sneakers/runner_spec.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'spec_helper' 3 | require 'sneakers' 4 | require 'sneakers/runner' 5 | 6 | describe Sneakers::Runner do 7 | let(:logger) { Logger.new('log/logtest.log') } 8 | 9 | describe "with configuration that specifies a logger object" do 10 | before do 11 | Sneakers.configure(log: logger) 12 | @runner = Sneakers::Runner.new([]) 13 | end 14 | 15 | it 'passes the logger to serverengine' do 16 | # Stub out ServerEngine::Daemon.run so we only exercise the way we invoke 17 | # ServerEngine.create 18 | any_instance_of(ServerEngine::Daemon) do |daemon| 19 | stub(daemon).main{ return 0 } 20 | end 21 | 22 | @runner.run 23 | # look at @runner's @se instance variable (actually of type Daemon)...and 24 | # figure out what it's logger is... 25 | end 26 | end 27 | end 28 | 29 | describe Sneakers::RunnerConfig do 30 | let(:logger) { Logger.new("log/logtest.log") } 31 | let(:runner_config) { Sneakers::Runner.new([]).instance_variable_get("@runnerconfig") } 32 | 33 | describe "with a connection" do 34 | let(:connection) { Object.new } 35 | 36 | before { Sneakers.configure(log: logger, connection: connection) } 37 | 38 | describe "#reload_config!" do 39 | it "does not throw exception" do 40 | runner_config.reload_config! 41 | end 42 | 43 | it "must not have :log key" do 44 | _(runner_config.reload_config!.has_key?(:log)).must_equal false 45 | end 46 | 47 | it "must have :logger key as an instance of Logger" do 48 | _(runner_config.reload_config![:logger].is_a?(Logger)).must_equal true 49 | end 50 | 51 | it "must have :connection" do 52 | _(runner_config.reload_config![:connection].is_a?(Object)).must_equal true 53 | end 54 | end 55 | end 56 | 57 | describe "without a connection" do 58 | before { Sneakers.configure(log: logger) } 59 | 60 | describe "#reload_config!" do 61 | it "must not have :log key" do 62 | _(runner_config.reload_config!.has_key?(:log)).must_equal false 63 | end 64 | 65 | it "must have :logger key as an instance of Logger" do 66 | _(runner_config.reload_config![:logger].is_a?(Logger)).must_equal true 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/sneakers/sneakers_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | 4 | class EnvWorker 5 | include Sneakers::Worker 6 | from_queue 'defaults' 7 | 8 | def work(msg) 9 | end 10 | end 11 | 12 | 13 | describe Sneakers do 14 | before do 15 | Sneakers.clear! 16 | end 17 | 18 | describe 'self' do 19 | it 'should have defaults set up' do 20 | _(Sneakers::CONFIG[:log]).must_equal(STDOUT) 21 | end 22 | 23 | it 'should configure itself' do 24 | Sneakers.configure 25 | _(Sneakers.logger).wont_be_nil 26 | _(Sneakers.configured?).must_equal(true) 27 | end 28 | end 29 | 30 | describe '.daemonize!' do 31 | it 'should set a logger to a default info level and not daemonize' do 32 | Sneakers.daemonize! 33 | _(Sneakers::CONFIG[:log]).must_equal('sneakers.log') 34 | _(Sneakers::CONFIG[:daemonize]).must_equal(true) 35 | _(Sneakers.logger.level).must_equal(Logger::INFO) 36 | end 37 | 38 | it 'should set a logger to a level given that level' do 39 | Sneakers.daemonize!(Logger::DEBUG) 40 | _(Sneakers.logger.level).must_equal(Logger::DEBUG) 41 | end 42 | end 43 | 44 | 45 | describe '.clear!' do 46 | it 'must reset dirty configuration to default' do 47 | _(Sneakers::CONFIG[:log]).must_equal(STDOUT) 48 | Sneakers.configure(:log => 'foobar.log') 49 | _(Sneakers::CONFIG[:log]).must_equal('foobar.log') 50 | Sneakers.clear! 51 | _(Sneakers::CONFIG[:log]).must_equal(STDOUT) 52 | end 53 | end 54 | 55 | 56 | describe '#setup_general_logger' do 57 | let(:logger_class) { ServerEngine::DaemonLogger } 58 | 59 | it 'should detect a string and configure a logger' do 60 | Sneakers.configure(:log => 'sneakers.log') 61 | _(Sneakers.logger.kind_of?(logger_class)).must_equal(true) 62 | end 63 | 64 | it 'should detect a file-like thing and configure a logger' do 65 | Sneakers.configure(:log => STDOUT) 66 | _(Sneakers.logger.kind_of?(logger_class)).must_equal(true) 67 | end 68 | 69 | it 'should detect an actual logger and configure it' do 70 | logger = Logger.new(STDOUT) 71 | Sneakers.configure(:log => logger) 72 | _(Sneakers.logger).must_equal(logger) 73 | end 74 | end 75 | 76 | end 77 | 78 | -------------------------------------------------------------------------------- /spec/sneakers/support/utils_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | 4 | describe Sneakers::Utils do 5 | describe '::parse_workers' do 6 | before(:all) do 7 | class Foo; end 8 | class Bar; end 9 | class Baz 10 | class Quux; end 11 | class Corge; end 12 | end 13 | end 14 | 15 | describe 'given a single class name' do 16 | describe 'without namespace' do 17 | it 'returns the worker class name' do 18 | _(Sneakers::Utils.parse_workers('Foo')).must_equal([[Foo],[]]) 19 | end 20 | end 21 | 22 | describe 'with namespace' do 23 | it 'returns the worker class name' do 24 | _(Sneakers::Utils.parse_workers('Baz::Quux')).must_equal([[Baz::Quux],[]]) 25 | end 26 | end 27 | end 28 | 29 | describe 'given a list of class names' do 30 | describe 'without namespaces' do 31 | it 'returns all worker class names' do 32 | _(Sneakers::Utils.parse_workers('Foo,Bar')).must_equal([[Foo,Bar],[]]) 33 | end 34 | end 35 | 36 | describe 'with namespaces' do 37 | it 'returns all worker class names' do 38 | workers = Sneakers::Utils.parse_workers('Baz::Quux,Baz::Corge') 39 | _(workers).must_equal([[Baz::Quux,Baz::Corge],[]]) 40 | end 41 | end 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/sneakers/tasks/sneakers_run_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'rake' 4 | require 'sneakers/tasks' 5 | 6 | describe 'Worker classes run by rake sneakers:run' do 7 | class TestWorker 8 | include Sneakers::Worker 9 | end 10 | class TestClass1 < TestWorker; end 11 | class TestClass2 < TestWorker; end 12 | 13 | def with_workers_env(workers) 14 | undefine, restore = if ENV.key?('WORKERS') 15 | [false, ENV['WORKERS']] 16 | else 17 | true 18 | end 19 | ENV['WORKERS'] = workers 20 | yield 21 | ensure 22 | undefine ? ENV.delete('WORKERS') : ENV['WORKERS'] = restore 23 | end 24 | 25 | def with_rake_worker_classes(workers) 26 | restore = Sneakers.rake_worker_classes 27 | Sneakers.rake_worker_classes = workers 28 | yield 29 | ensure 30 | Sneakers.rake_worker_classes = restore 31 | end 32 | 33 | def with_sneakers_worker_classes_reset 34 | restore = Sneakers::Worker::Classes.clone 35 | Sneakers::Worker::Classes.replace([]) 36 | yield 37 | ensure 38 | Sneakers::Worker::Classes.replace(restore) 39 | end 40 | 41 | let(:opts) { {} } 42 | 43 | let :runner do 44 | mock = Minitest::Mock.new 45 | mock.expect(:run, nil) 46 | mock.expect(:call, mock, [expected_workers, opts]) 47 | mock 48 | end 49 | 50 | let :run_rake_task do 51 | Rake::Task['sneakers:run'].reenable 52 | Rake.application.invoke_task 'sneakers:run' 53 | end 54 | 55 | describe 'without any settings' do 56 | let(:expected_workers) { [worker_class] } 57 | let(:worker_class) { Class.new.tap { |klass| klass.send(:include, Sneakers::Worker) } } 58 | 59 | it 'runs classes directly including the Worker' do 60 | with_workers_env(nil) do 61 | with_sneakers_worker_classes_reset do 62 | Sneakers::Runner.stub :new, runner do 63 | run_rake_task 64 | end 65 | runner.verify 66 | end 67 | end 68 | end 69 | end 70 | 71 | describe 'with rake_worker_classes set' do 72 | let(:expected_workers) { [TestClass1, TestClass2] } 73 | 74 | it 'runs the classes from the setting' do 75 | with_workers_env(nil) do 76 | with_rake_worker_classes([TestClass1, TestClass2]) do 77 | Sneakers::Runner.stub :new, runner do 78 | run_rake_task 79 | end 80 | runner.verify 81 | end 82 | end 83 | end 84 | end 85 | 86 | describe 'with rake_worker_classes set, overriden by WORKERS env' do 87 | let(:expected_workers) { [TestClass2] } 88 | 89 | it 'runs the classes from the setting' do 90 | with_rake_worker_classes([TestClass1, TestClass2]) do 91 | with_workers_env('TestClass2') do 92 | Sneakers::Runner.stub :new, runner do 93 | run_rake_task 94 | end 95 | runner.verify 96 | end 97 | end 98 | end 99 | end 100 | 101 | describe 'with rake_worker_classes responding to call' do 102 | let(:expected_workers) { [TestClass1] } 103 | 104 | it 'runs the classes from the setting' do 105 | with_workers_env(nil) do 106 | with_rake_worker_classes(-> { [TestClass1] }) do 107 | Sneakers::Runner.stub :new, runner do 108 | run_rake_task 109 | end 110 | runner.verify 111 | end 112 | end 113 | end 114 | end 115 | end -------------------------------------------------------------------------------- /spec/sneakers/worker_handlers_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'sneakers' 3 | require 'sneakers/handlers/oneshot' 4 | require 'sneakers/handlers/maxretry' 5 | require 'json' 6 | 7 | 8 | # Specific tests of the Handler implementations you can use to deal with job 9 | # results. These tests only make sense with a worker that requires acking. 10 | 11 | class HandlerTestWorker 12 | include Sneakers::Worker 13 | from_queue 'defaults', 14 | :ack => true 15 | 16 | def work(msg) 17 | if msg.is_a?(StandardError) 18 | raise msg 19 | elsif msg.is_a?(String) 20 | hash = maybe_json(msg) 21 | if hash.is_a?(Hash) 22 | hash['response'].to_sym 23 | else 24 | hash 25 | end 26 | else 27 | msg 28 | end 29 | end 30 | 31 | def maybe_json(string) 32 | JSON.parse(string) 33 | rescue 34 | string 35 | end 36 | end 37 | 38 | TestPool ||= Concurrent::ImmediateExecutor 39 | 40 | describe 'Handlers' do 41 | let(:channel) { Object.new } 42 | let(:queue) { Object.new } 43 | let(:worker) { HandlerTestWorker.new(@queue, TestPool.new) } 44 | 45 | before(:each) do 46 | Sneakers.configure(:daemonize => true, :log => 'sneakers.log') 47 | Sneakers::Worker.configure_logger(Logger.new('/dev/null')) 48 | Sneakers::Worker.configure_metrics 49 | end 50 | 51 | describe 'Oneshot' do 52 | before(:each) do 53 | @opts = Object.new 54 | @handler = Sneakers::Handlers::Oneshot.new(channel, queue, @opts) 55 | 56 | @header = Object.new 57 | stub(@header).delivery_tag { 37 } 58 | end 59 | 60 | describe '#do_work' do 61 | it 'should work and handle acks' do 62 | mock(channel).acknowledge(37, false) 63 | 64 | worker.do_work(@header, nil, :ack, @handler) 65 | end 66 | 67 | it 'should work and handle rejects' do 68 | mock(channel).reject(37, false) 69 | 70 | worker.do_work(@header, nil, :reject, @handler) 71 | end 72 | 73 | it 'should work and handle requeues' do 74 | mock(channel).reject(37, true) 75 | 76 | worker.do_work(@header, nil, :requeue, @handler) 77 | end 78 | 79 | it 'should work and handle user code error' do 80 | mock(channel).reject(37, false) 81 | 82 | worker.do_work(@header, nil, StandardError.new('boom!'), @handler) 83 | end 84 | 85 | it 'should work and handle noops' do 86 | worker.do_work(@header, nil, :wait, @handler) 87 | end 88 | end 89 | 90 | end 91 | 92 | describe 'Maxretry' do 93 | let(:max_retries) { nil } 94 | let(:props_with_x_death_count) { 95 | { 96 | :headers => { 97 | "x-death" => [ 98 | { 99 | "count" => 3, 100 | "reason" => "expired", 101 | "queue" => "downloads-retry", 102 | "time" => Time.now, 103 | "exchange" => "RawMail-retry", 104 | "routing-keys" => ["RawMail"] 105 | }, 106 | { 107 | "count" => 3, 108 | "reason" => "rejected", 109 | "queue" => "downloads", 110 | "time" => Time.now, 111 | "exchange" => "", 112 | "routing-keys" => ["RawMail"] 113 | } 114 | ] 115 | }, 116 | :delivery_mode => 1 117 | } 118 | } 119 | 120 | before(:each) do 121 | @opts = { 122 | :exchange => 'sneakers', 123 | :queue_options => { 124 | :durable => 'true', 125 | } 126 | }.tap do |opts| 127 | opts[:retry_max_times] = max_retries unless max_retries.nil? 128 | end 129 | 130 | mock(queue).name { 'downloads' } 131 | 132 | @retry_exchange = Object.new 133 | @error_exchange = Object.new 134 | @requeue_exchange = Object.new 135 | 136 | @retry_queue = Object.new 137 | @error_queue = Object.new 138 | 139 | mock(channel).exchange('downloads-retry', 140 | :type => 'topic', 141 | :durable => 'true').once { @retry_exchange } 142 | mock(channel).exchange('downloads-error', 143 | :type => 'topic', 144 | :durable => 'true').once { @error_exchange } 145 | mock(channel).exchange('downloads-retry-requeue', 146 | :type => 'topic', 147 | :durable => 'true').once { @requeue_exchange } 148 | 149 | mock(channel).queue('downloads-retry', 150 | :durable => 'true', 151 | :arguments => { 152 | :'x-dead-letter-exchange' => 'downloads-retry-requeue', 153 | :'x-message-ttl' => 60000 154 | } 155 | ).once { @retry_queue } 156 | mock(@retry_queue).bind(@retry_exchange, :routing_key => '#') 157 | 158 | mock(channel).queue('downloads-error', 159 | :durable => 'true').once { @error_queue } 160 | mock(@error_queue).bind(@error_exchange, :routing_key => '#') 161 | 162 | @header = Object.new 163 | stub(@header).delivery_tag { 37 } 164 | 165 | @props = {} 166 | @props_with_x_death = { 167 | :headers => { 168 | "x-death" => [ 169 | { 170 | "reason" => "expired", 171 | "queue" => "downloads-retry", 172 | "time" => Time.now, 173 | "exchange" => "RawMail-retry", 174 | "routing-keys" => ["RawMail"] 175 | }, 176 | { 177 | "reason" => "rejected", 178 | "queue" => "downloads", 179 | "time" => Time.now, 180 | "exchange" => "", 181 | "routing-keys" => ["RawMail"] 182 | } 183 | ] 184 | }, 185 | :delivery_mode => 1} 186 | end 187 | 188 | # it 'allows overriding the retry exchange name' 189 | # it 'allows overriding the error exchange name' 190 | 191 | describe '#do_work' do 192 | before do 193 | @now = Time.now 194 | 195 | mock(queue).bind(@requeue_exchange, :routing_key => '#') 196 | 197 | @handler = Sneakers::Handlers::Maxretry.new(channel, queue, @opts) 198 | end 199 | 200 | # Used to stub out the publish method args. Sadly RR doesn't support 201 | # this, only proxying existing methods. 202 | module MockPublish 203 | attr_reader :data, :opts, :called 204 | 205 | def publish(data, opts) 206 | @data = data 207 | @opts = opts 208 | @called = true 209 | end 210 | end 211 | 212 | it 'should work and handle acks' do 213 | mock(channel).acknowledge(37, false) 214 | 215 | worker.do_work(@header, @props, :ack, @handler) 216 | end 217 | 218 | describe 'rejects' do 219 | describe 'more retries ahead' do 220 | it 'should work and handle rejects' do 221 | mock(channel).reject(37, false) 222 | 223 | worker.do_work(@header, @props_with_x_death, :reject, @handler) 224 | end 225 | end 226 | 227 | describe 'no more retries' do 228 | let(:max_retries) { 1 } 229 | 230 | it 'sends the rejection to the error queue' do 231 | mock(@header).routing_key { '#' } 232 | mock(channel).acknowledge(37, false) 233 | 234 | @error_exchange.extend MockPublish 235 | worker.do_work(@header, @props_with_x_death, :reject, @handler) 236 | _(@error_exchange.called).must_equal(true) 237 | _(@error_exchange.opts[:routing_key]).must_equal('#') 238 | data = JSON.parse(@error_exchange.opts[:headers][:retry_info]) rescue nil 239 | _(data).wont_be_nil 240 | _(data['error']).must_equal('reject') 241 | _(data['num_attempts']).must_equal(2) 242 | _(@error_exchange.data).must_equal(:reject) 243 | _(data['properties'].to_json).must_equal(@props_with_x_death.to_json) 244 | _(Time.parse(data['failed_at'])).wont_be_nil 245 | end 246 | 247 | it 'counts the number of attempts using the count key' do 248 | mock(@header).routing_key { '#' } 249 | mock(channel).acknowledge(37, false) 250 | 251 | @error_exchange.extend MockPublish 252 | worker.do_work(@header, props_with_x_death_count, :reject, @handler) 253 | _(@error_exchange.called).must_equal(true) 254 | _(@error_exchange.opts[:routing_key]).must_equal('#') 255 | data = JSON.parse(@error_exchange.opts[:headers][:retry_info]) rescue nil 256 | _(data).wont_be_nil 257 | _(data['error']).must_equal('reject') 258 | _(data['num_attempts']).must_equal(4) 259 | _(@error_exchange.data).must_equal(:reject) 260 | _(data['properties'].to_json).must_equal(props_with_x_death_count.to_json) 261 | _(Time.parse(data['failed_at'])).wont_be_nil 262 | end 263 | 264 | end 265 | end 266 | 267 | describe 'requeues' do 268 | it 'should work and handle requeues' do 269 | mock(channel).reject(37, true) 270 | 271 | worker.do_work(@header, @props_with_x_death, :requeue, @handler) 272 | end 273 | 274 | describe 'no more retries left' do 275 | let(:max_retries) { 1 } 276 | 277 | it 'continues to reject with requeue' do 278 | mock(channel).reject(37, true) 279 | 280 | worker.do_work(@header, @props_with_x_death, :requeue, @handler) 281 | end 282 | end 283 | 284 | end 285 | 286 | describe 'exceptions' do 287 | describe 'more retries ahead' do 288 | it 'should reject the message' do 289 | mock(channel).reject(37, false) 290 | 291 | worker.do_work(@header, @props_with_x_death, StandardError.new('boom!'), @handler) 292 | end 293 | end 294 | 295 | describe 'no more retries left' do 296 | let(:max_retries) { 1 } 297 | 298 | it 'sends the rejection to the error queue' do 299 | mock(@header).routing_key { '#' } 300 | mock(channel).acknowledge(37, false) 301 | @error_exchange.extend MockPublish 302 | 303 | worker.do_work(@header, @props_with_x_death, StandardError.new('boom!'), @handler) 304 | _(@error_exchange.called).must_equal(true) 305 | _(@error_exchange.opts[:routing_key]).must_equal('#') 306 | data = JSON.parse(@error_exchange.opts[:headers][:retry_info]) rescue nil 307 | _(data).wont_be_nil 308 | _(data['error']).must_equal('boom!') 309 | _(data['error_class']).must_equal(StandardError.to_s) 310 | _(data['backtrace']).wont_be_nil 311 | _(data['num_attempts']).must_equal(2) 312 | _(@error_exchange.data.to_s).must_equal('boom!') 313 | _(data['properties'].to_json).must_equal(@props_with_x_death.to_json) 314 | _(Time.parse(data['failed_at'])).wont_be_nil 315 | end 316 | end 317 | end 318 | 319 | it 'should work and handle user-land error' do 320 | mock(channel).reject(37, false) 321 | 322 | worker.do_work(@header, @props, StandardError.new('boom!'), @handler) 323 | end 324 | 325 | it 'should work and handle noops' do 326 | worker.do_work(@header, @props, :wait, @handler) 327 | end 328 | end 329 | 330 | describe '.configure_queue' do 331 | before do 332 | mock(channel).prefetch(10) 333 | @mkbunny = Object.new 334 | @mkex = Object.new 335 | @mkworker = Object.new 336 | 337 | mock(@mkbunny).start {} 338 | mock(@mkbunny).create_channel{ channel } 339 | mock(Bunny).new( 340 | anything, 341 | hash_including(:vhost => '/', :heartbeat => 2) 342 | ){ @mkbunny } 343 | 344 | mock(channel).exchange("sneakers", 345 | :type => :direct, 346 | :durable => 'true', 347 | :auto_delete => false, 348 | :arguments => {}).once { @mkex } 349 | end 350 | 351 | describe 'use queue name for retry exchange' do 352 | before do 353 | Sneakers.clear! 354 | Sneakers.configure({ 355 | :connection => nil, 356 | :ack => true, 357 | :heartbeat => 2, 358 | :vhost => '/', 359 | :exchange => "sneakers", 360 | :exchange_options => { 361 | :type => :direct, 362 | durable: 'true' 363 | }, 364 | :queue_options => { 365 | :durable => 'true' 366 | }, 367 | :handler => Sneakers::Handlers::Maxretry 368 | }) 369 | end 370 | 371 | describe 'default settings' do 372 | before do 373 | mock(queue).bind(@requeue_exchange, :routing_key => '#') 374 | @worker_opts = Sneakers::CONFIG.merge({}) 375 | stub(@mkworker).opts { @worker_opts } 376 | end 377 | 378 | let(:q) { Sneakers::Queue.new("downloads", @worker_opts) } 379 | 380 | it 'should configure queue with x-dead-letter-exchange' do 381 | mock(channel).queue("downloads", :durable => 'true', :auto_delete => false, :exclusive => false, :arguments => { :"x-dead-letter-exchange" => "downloads-retry" }).once { queue } 382 | mock(queue).bind(@mkex, :routing_key => "downloads") 383 | mock(queue).subscribe(:block => false, :manual_ack => true) 384 | 385 | q.subscribe(@mkworker) 386 | end 387 | end 388 | 389 | describe 'preserve other worker arguments' do 390 | before do 391 | mock(queue).bind(@requeue_exchange, :routing_key => '#') 392 | @worker_opts = Sneakers::CONFIG.merge({ :arguments => { 'x-arg' => 'value' } }) 393 | stub(@mkworker).opts { @worker_opts } 394 | end 395 | 396 | let(:q) { Sneakers::Queue.new("downloads", @worker_opts) } 397 | 398 | it 'should configure queue with x-dead-letter-exchange and other args' do 399 | mock(channel).queue("downloads", :durable => 'true', :auto_delete => false, :exclusive => false, :arguments => { :"x-dead-letter-exchange" => "downloads-retry", :"x-arg" => 'value' }).once { queue } 400 | mock(queue).bind(@mkex, :routing_key => "downloads") 401 | mock(queue).subscribe(:block => false, :manual_ack => true) 402 | 403 | q.subscribe(@mkworker) 404 | end 405 | end 406 | end 407 | 408 | describe 'use globally configured retry exchange name' do 409 | before do 410 | Sneakers.clear! 411 | Sneakers.configure({ 412 | :connection => nil, 413 | :ack => true, 414 | :heartbeat => 2, 415 | :vhost => '/', 416 | :exchange => "sneakers", 417 | :exchange_options => { 418 | :type => :direct, 419 | durable: 'true' 420 | }, 421 | :queue_options => { 422 | :durable => 'true' 423 | }, 424 | :handler => Sneakers::Handlers::Maxretry, 425 | :retry_exchange => "downloads-retry", 426 | :retry_error_exchange => "downloads-error", 427 | :retry_requeue_exchange => "downloads-retry-requeue" 428 | }) 429 | end 430 | 431 | describe 'use global setup for worker' do 432 | before do 433 | mock(queue).bind(@requeue_exchange, :routing_key => 'uploads') 434 | @worker_opts = Sneakers::CONFIG.merge({ :retry_routing_key => "uploads" }) 435 | stub(@mkworker).opts { @worker_opts } 436 | end 437 | 438 | let(:q) { Sneakers::Queue.new("uploads", @worker_opts) } 439 | 440 | it 'should configure queue with x-dead-letter-exchange (not use queue name)' do 441 | mock(channel).queue("uploads", :durable => 'true', :auto_delete => false, :exclusive => false, :arguments => { :"x-dead-letter-exchange" => "downloads-retry" }).once { queue } 442 | mock(queue).bind(@mkex, :routing_key => "uploads") 443 | mock(queue).subscribe(:block => false, :manual_ack => true) 444 | 445 | q.subscribe(@mkworker) 446 | end 447 | end 448 | 449 | describe 'skip retry and go to error queue' do 450 | before do 451 | mock(queue).bind(@requeue_exchange, :routing_key => 'uploads') 452 | @worker_opts = Sneakers::CONFIG.merge({ :retry_routing_key => "uploads", :arguments => { :"x-dead-letter-exchange" => "downloads-error" } }) 453 | stub(@mkworker).opts { @worker_opts } 454 | end 455 | 456 | let(:q) { Sneakers::Queue.new("uploads", @worker_opts) } 457 | 458 | it 'should configure queue with x-dead-letter-exchange (not use queue name)' do 459 | mock(channel).queue("uploads", :durable => 'true', :auto_delete => false, :exclusive => false, :arguments => { :"x-dead-letter-exchange" => "downloads-error" }).once { queue } 460 | mock(queue).bind(@mkex, :routing_key => "uploads") 461 | mock(queue).subscribe(:block => false, :manual_ack => true) 462 | 463 | q.subscribe(@mkworker) 464 | end 465 | end 466 | end 467 | end 468 | end 469 | end 470 | -------------------------------------------------------------------------------- /spec/sneakers/worker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'gzip_helper' 3 | require 'sneakers' 4 | require 'serverengine' 5 | 6 | class DummyWorker 7 | include Sneakers::Worker 8 | from_queue 'downloads', 9 | :exchange_options => { 10 | :type => :topic, 11 | :durable => false, 12 | :auto_delete => true, 13 | :arguments => { 'x-arg' => 'value' } 14 | }, 15 | :queue_options => { 16 | :durable => false, 17 | :auto_delete => true, 18 | :exclusive => true, 19 | :arguments => { 'x-arg' => 'value' } 20 | }, 21 | :ack => false, 22 | :threads => 50, 23 | :prefetch => 40, 24 | :exchange => 'dummy', 25 | :heartbeat => 5 26 | 27 | def work(msg) 28 | end 29 | end 30 | 31 | class DefaultsWorker 32 | include Sneakers::Worker 33 | from_queue 'defaults' 34 | 35 | def work(msg) 36 | end 37 | end 38 | 39 | class AcksWorker 40 | include Sneakers::Worker 41 | from_queue 'defaults', 42 | :ack => true 43 | 44 | def work(msg) 45 | if msg == :ack 46 | ack! 47 | elsif msg == :nack 48 | nack! 49 | elsif msg == :reject 50 | reject! 51 | else 52 | msg 53 | end 54 | end 55 | end 56 | 57 | class PublishingWorker 58 | include Sneakers::Worker 59 | from_queue 'defaults', 60 | :ack => false, 61 | :exchange => 'foochange' 62 | 63 | def work(msg) 64 | publish msg, :to_queue => 'target' 65 | end 66 | end 67 | 68 | class JSONPublishingWorker 69 | include Sneakers::Worker 70 | from_queue 'defaults', 71 | :ack => false, 72 | :exchange => 'foochange' 73 | 74 | def work(msg) 75 | publish msg, :to_queue => 'target', :content_type => 'application/json' 76 | end 77 | end 78 | 79 | class GzipPublishingWorker 80 | include Sneakers::Worker 81 | from_queue 'defaults', 82 | :ack => false, 83 | :exchange => 'foochange' 84 | 85 | def work(msg) 86 | publish msg, :to_queue => 'target', :content_encoding => 'gzip' 87 | end 88 | end 89 | 90 | class LoggingWorker 91 | include Sneakers::Worker 92 | from_queue 'defaults', 93 | :ack => false 94 | 95 | def work(msg) 96 | logger.info "hello" 97 | end 98 | end 99 | 100 | class JSONWorker 101 | include Sneakers::Worker 102 | from_queue 'defaults', 103 | :ack => false, 104 | :content_type => 'application/json' 105 | 106 | def work(msg) 107 | end 108 | end 109 | 110 | class GzipWorker 111 | include Sneakers::Worker 112 | from_queue 'defaults', 113 | :ack => false, 114 | :content_encoding => 'gzip' 115 | 116 | def work(msg) 117 | end 118 | end 119 | 120 | class MetricsWorker 121 | include Sneakers::Worker 122 | from_queue 'defaults', 123 | :ack => true 124 | 125 | def work(msg) 126 | metrics.increment "foobar" 127 | msg 128 | end 129 | end 130 | 131 | class WithParamsWorker 132 | include Sneakers::Worker 133 | from_queue 'defaults', 134 | :ack => true 135 | 136 | def work_with_params(msg, delivery_info, metadata) 137 | msg 138 | end 139 | end 140 | 141 | class WithDeprecatedExchangeOptionsWorker 142 | include Sneakers::Worker 143 | from_queue 'defaults', 144 | :durable => false, 145 | :exchange_type => :topic, 146 | :exchange_arguments => { 'x-arg' => 'value' }, 147 | :arguments => { 'x-arg2' => 'value2' } 148 | 149 | def work(msg) 150 | end 151 | end 152 | 153 | TestPool ||= Concurrent::ImmediateExecutor 154 | 155 | describe Sneakers::Worker do 156 | before do 157 | @queue = Object.new 158 | @exchange = Object.new 159 | stub(@queue).name { 'test-queue' } 160 | stub(@queue).opts { {} } 161 | stub(@queue).exchange { @exchange } 162 | 163 | Sneakers.clear! 164 | Sneakers.configure(daemonize: true, log: 'sneakers.log') 165 | Sneakers::Worker.configure_metrics 166 | end 167 | 168 | describe ".enqueue" do 169 | it "publishes a message to the class queue" do 170 | message = 'test message' 171 | 172 | mock(Sneakers::Publisher).new(DummyWorker.queue_opts) do 173 | mock(Object.new).publish(message, { 174 | :routing_key => 'test.routing.key', 175 | :to_queue => 'downloads', 176 | :content_type => nil, 177 | :content_encoding => nil, 178 | }) 179 | end 180 | 181 | DummyWorker.enqueue(message, :routing_key => 'test.routing.key') 182 | end 183 | end 184 | 185 | describe "#initialize" do 186 | describe "builds an internal queue" do 187 | it "should build a queue with correct configuration given defaults" do 188 | @defaults_q = DefaultsWorker.new.queue 189 | _(@defaults_q.name).must_equal('defaults') 190 | _(@defaults_q.opts.to_hash).must_equal( 191 | :error_reporters => [Sneakers.error_reporters.last], 192 | :runner_config_file => nil, 193 | :metrics => nil, 194 | :daemonize => true, 195 | :start_worker_delay => 0.2, 196 | :workers => 4, 197 | :log => "sneakers.log", 198 | :pid_path => "sneakers.pid", 199 | :prefetch => 10, 200 | :threads => 10, 201 | :share_threads => false, 202 | :ack => true, 203 | :amqp => "amqp://guest:guest@localhost:5672", 204 | :vhost => "/", 205 | :exchange => "sneakers", 206 | :exchange_options => { 207 | :type => :direct, 208 | :durable => true, 209 | :auto_delete => false, 210 | :arguments => {} 211 | }, 212 | :queue_options => { 213 | :durable => true, 214 | :auto_delete => false, 215 | :exclusive => false, 216 | :arguments => {} 217 | }, 218 | :hooks => {}, 219 | :handler => Sneakers::Handlers::Oneshot, 220 | :heartbeat => 30, 221 | :amqp_heartbeat => 30, 222 | :log_rotate_age => 5, 223 | :log_rotate_size => 1048576, 224 | :log_level => "debug" 225 | ) 226 | end 227 | 228 | it "should build a queue with given configuration" do 229 | @dummy_q = DummyWorker.new.queue 230 | _(@dummy_q.name).must_equal('downloads') 231 | _(@dummy_q.opts.to_hash).must_equal( 232 | :error_reporters => [Sneakers.error_reporters.last], 233 | :runner_config_file => nil, 234 | :metrics => nil, 235 | :daemonize => true, 236 | :start_worker_delay => 0.2, 237 | :workers => 4, 238 | :log => "sneakers.log", 239 | :pid_path => "sneakers.pid", 240 | :prefetch => 40, 241 | :threads => 50, 242 | :share_threads => false, 243 | :ack => false, 244 | :amqp => "amqp://guest:guest@localhost:5672", 245 | :vhost => "/", 246 | :exchange => "dummy", 247 | :exchange_options => { 248 | :type => :topic, 249 | :durable => false, 250 | :auto_delete => true, 251 | :arguments => { 'x-arg' => 'value' } 252 | }, 253 | :queue_options => { 254 | :durable => false, 255 | :auto_delete => true, 256 | :exclusive => true, 257 | :arguments => { 'x-arg' => 'value' } 258 | }, 259 | :hooks => {}, 260 | :handler => Sneakers::Handlers::Oneshot, 261 | :heartbeat => 5, 262 | :amqp_heartbeat => 30, 263 | :log_rotate_age => 5, 264 | :log_rotate_size => 1048576, 265 | :log_level => "debug" 266 | ) 267 | end 268 | 269 | it "should build a queue with correct configuration given deprecated exchange options" do 270 | @deprecated_exchange_opts_q = WithDeprecatedExchangeOptionsWorker.new.queue 271 | _(@deprecated_exchange_opts_q.name).must_equal('defaults') 272 | _(@deprecated_exchange_opts_q.opts.to_hash).must_equal( 273 | :error_reporters => [Sneakers.error_reporters.last], 274 | :runner_config_file => nil, 275 | :metrics => nil, 276 | :daemonize => true, 277 | :start_worker_delay => 0.2, 278 | :workers => 4, 279 | :log => "sneakers.log", 280 | :pid_path => "sneakers.pid", 281 | :prefetch => 10, 282 | :threads => 10, 283 | :share_threads => false, 284 | :ack => true, 285 | :amqp => "amqp://guest:guest@localhost:5672", 286 | :vhost => "/", 287 | :exchange => "sneakers", 288 | :exchange_options => { 289 | :type => :topic, 290 | :durable => false, 291 | :auto_delete => false, 292 | :arguments => { 'x-arg' => 'value' } 293 | }, 294 | :queue_options => { 295 | :durable => false, 296 | :auto_delete => false, 297 | :exclusive => false, 298 | :arguments => { 'x-arg2' => 'value2' } 299 | }, 300 | :hooks => {}, 301 | :handler => Sneakers::Handlers::Oneshot, 302 | :heartbeat => 30, 303 | :amqp_heartbeat => 30, 304 | :log_rotate_age => 5, 305 | :log_rotate_size => 1048576, 306 | :log_level => "debug" 307 | ) 308 | end 309 | end 310 | 311 | describe "initializes worker" do 312 | it "should generate a worker id" do 313 | _(DummyWorker.new.id).must_match(/^worker-/) 314 | end 315 | end 316 | 317 | describe 'when connection provided' do 318 | before do 319 | @connection = Bunny.new(host: 'any-host.local') 320 | Sneakers.configure( 321 | exchange: 'some-exch', 322 | exchange_options: { type: :direct }, 323 | connection: @connection, 324 | ) 325 | end 326 | 327 | it "should build a queue with given connection" do 328 | @dummy_q = DummyWorker.new.queue 329 | _(@dummy_q.opts[:connection]).must_equal(@connection) 330 | end 331 | end 332 | end 333 | 334 | 335 | describe "#run" do 336 | it "should subscribe on internal queue" do 337 | q = Object.new 338 | w = DummyWorker.new(q) 339 | mock(q).subscribe(w).once #XXX once? 340 | stub(q).name{ "test" } 341 | stub(q).opts { nil } 342 | w.run 343 | end 344 | end 345 | 346 | describe "#stop" do 347 | it "should unsubscribe from internal queue" do 348 | q = Object.new 349 | mock(q).unsubscribe.once #XXX once? 350 | stub(q).name { 'test-queue' } 351 | stub(q).opts {nil} 352 | w = DummyWorker.new(q) 353 | w.stop 354 | end 355 | end 356 | 357 | 358 | describe "#do_work" do 359 | it "should perform worker's work" do 360 | w = DummyWorker.new(@queue, TestPool.new) 361 | mock(w).work("msg").once 362 | w.do_work(nil, nil, "msg", nil) 363 | end 364 | 365 | describe 'content type based deserialization' do 366 | before do 367 | Sneakers::ContentType.register( 368 | content_type: 'application/json', 369 | serializer: ->(_) {}, 370 | deserializer: ->(payload) { JSON.parse(payload) }, 371 | ) 372 | end 373 | 374 | after do 375 | Sneakers::ContentType.reset! 376 | end 377 | 378 | it 'should use the registered deserializer if the content type is in the metadata' do 379 | w = DummyWorker.new(@queue, TestPool.new) 380 | mock(w).work({'foo' => 'bar'}).once 381 | w.do_work(nil, { content_type: 'application/json' }, '{"foo":"bar"}', nil) 382 | end 383 | 384 | it 'should use the registered deserializer if the content type is in the queue options' do 385 | w = JSONWorker.new(@queue, TestPool.new) 386 | mock(w).work({'foo' => 'bar'}).once 387 | w.do_work(nil, {}, '{"foo":"bar"}', nil) 388 | end 389 | 390 | it 'should use the deserializer from the queue options even if the metadata has a different content type' do 391 | w = JSONWorker.new(@queue, TestPool.new) 392 | mock(w).work({'foo' => 'bar'}).once 393 | w.do_work(nil, { content_type: 'not/real' }, '{"foo":"bar"}', nil) 394 | end 395 | end 396 | 397 | describe 'content encoding based decoding' do 398 | before do 399 | Sneakers::ContentEncoding.register( 400 | content_encoding: 'gzip', 401 | encoder: ->(_) {}, 402 | decoder: ->(payload) { gzip_decompress(payload) }, 403 | ) 404 | end 405 | 406 | after do 407 | Sneakers::ContentEncoding.reset! 408 | end 409 | 410 | it 'should use the registered decoder if the content encoding is in the metadata' do 411 | w = DummyWorker.new(@queue, TestPool.new) 412 | mock(w).work('foobar').once 413 | w.do_work(nil, { content_encoding: 'gzip' }, gzip_compress('foobar'), nil) 414 | end 415 | 416 | it 'should use the registered decoder if the content encoding is in the queue options' do 417 | w = GzipWorker.new(@queue, TestPool.new) 418 | mock(w).work('foobar').once 419 | w.do_work(nil, {}, gzip_compress('foobar'), nil) 420 | end 421 | 422 | it 'should use the decoder from the queue options even if the metadata has a different content encoding' do 423 | w = GzipWorker.new(@queue, TestPool.new) 424 | mock(w).work('foobar').once 425 | w.do_work(nil, { content_encoding: 'not/real' }, gzip_compress('foobar'), nil) 426 | end 427 | end 428 | 429 | it "should catch runtime exceptions from a bad work" do 430 | w = AcksWorker.new(@queue, TestPool.new) 431 | mock(w).work("msg").once{ raise "foo" } 432 | handler = Object.new 433 | header = Object.new 434 | mock(handler).error(header, nil, "msg", anything) 435 | mock(w.logger).error(/\[Exception error="foo" error_class=RuntimeError worker_class=AcksWorker backtrace=.*/) 436 | w.do_work(header, nil, "msg", handler) 437 | end 438 | 439 | it "should catch script exceptions from a bad work" do 440 | w = AcksWorker.new(@queue, TestPool.new) 441 | mock(w).work("msg").once{ raise ScriptError } 442 | handler = Object.new 443 | header = Object.new 444 | mock(handler).error(header, nil, "msg", anything) 445 | mock(w.logger).error(/\[Exception error="ScriptError" error_class=ScriptError worker_class=AcksWorker backtrace=.*/) 446 | w.do_work(header, nil, "msg", handler) 447 | end 448 | 449 | it "should not catch bunny exceptions" do 450 | w = DummyWorker.new(@queue, TestPool.new) 451 | mock(w).work("msg").once{ raise Bunny::Exception } 452 | assert_raises(Bunny::Exception) do 453 | w.do_work(nil, nil, "msg", nil) 454 | end 455 | end 456 | 457 | it "should log exceptions from workers" do 458 | handler = Object.new 459 | header = Object.new 460 | w = AcksWorker.new(@queue, TestPool.new) 461 | mock(w).work("msg").once{ raise "foo" } 462 | mock(w.logger).error(/error="foo" error_class=RuntimeError worker_class=AcksWorker backtrace=/) 463 | mock(handler).error(header, nil, "msg", anything) 464 | w.do_work(header, nil, "msg", handler) 465 | end 466 | 467 | describe 'middleware' do 468 | let(:middleware) do 469 | Class.new do 470 | def initialize(app, *args) 471 | @app = app 472 | end 473 | 474 | def call(deserialized_msg, delivery_info, metadata, handler) 475 | @app.call(deserialized_msg, delivery_info, metadata, handler) 476 | end 477 | end 478 | end 479 | 480 | let(:worker) do 481 | Class.new do 482 | include Sneakers::Worker 483 | from_queue 'defaults', ack: false 484 | 485 | def work_with_params(msg, delivery_info, metadata) 486 | msg 487 | end 488 | end 489 | end 490 | 491 | before do 492 | Sneakers.middleware.use(middleware, 'args') 493 | 494 | @delivery_info = Object.new 495 | @metadata = Object.new 496 | stub(@metadata).[](:content_type) { 'some/fake' } 497 | stub(@metadata).[](:content_encoding) { 'some/fake' } 498 | @message = Object.new 499 | @handler = Object.new 500 | end 501 | 502 | after do 503 | Sneakers.middleware.delete(middleware) 504 | end 505 | 506 | it 'should process job and call #work_with_params/#work' do 507 | w = worker.new(@queue, TestPool.new) 508 | mock(w).work_with_params(@message, @delivery_info, @metadata).once 509 | 510 | w.do_work(@delivery_info, @metadata, @message, @handler) 511 | end 512 | 513 | it "should call registered middleware" do 514 | mock.proxy(middleware).new(instance_of(Proc), 'args').once do |res| 515 | mock.proxy(res).call(@message, @delivery_info, @metadata, @handler).once 516 | end 517 | 518 | w = worker.new(@queue, TestPool.new) 519 | w.do_work(@delivery_info, @metadata, @message, @handler) 520 | end 521 | end 522 | 523 | describe "with ack" do 524 | before do 525 | @delivery_info = Object.new 526 | stub(@delivery_info).delivery_tag{ "tag" } 527 | 528 | @worker = AcksWorker.new(@queue, TestPool.new) 529 | end 530 | 531 | it "should work and handle acks" do 532 | handler = Object.new 533 | mock(handler).acknowledge(@delivery_info, nil, :ack) 534 | 535 | @worker.do_work(@delivery_info, nil, :ack, handler) 536 | end 537 | 538 | it "should work and handle rejects" do 539 | handler = Object.new 540 | mock(handler).reject(@delivery_info, nil, :reject) 541 | 542 | @worker.do_work(@delivery_info, nil, :reject, handler) 543 | end 544 | 545 | it "should work and handle requeues" do 546 | handler = Object.new 547 | mock(handler).reject(@delivery_info, nil, :requeue, true) 548 | 549 | @worker.do_work(@delivery_info, nil, :requeue, handler) 550 | end 551 | 552 | it "should work and handle user code errors" do 553 | handler = Object.new 554 | mock(handler).error(@delivery_info, nil, :error, anything) 555 | 556 | @worker.do_work(@delivery_info, nil, :error, handler) 557 | end 558 | end 559 | 560 | describe "without ack" do 561 | it "should work and not care about acking if not ack" do 562 | handler = Object.new 563 | mock(handler).reject(anything).never 564 | mock(handler).acknowledge(anything).never 565 | 566 | w = DummyWorker.new(@queue, TestPool.new) 567 | w.do_work(nil, nil, 'msg', handler) 568 | end 569 | end 570 | end 571 | 572 | 573 | describe 'publish' do 574 | it 'should be able to publish a message from working context' do 575 | w = PublishingWorker.new(@queue, TestPool.new) 576 | mock(@exchange).publish('msg', :routing_key => 'target').once 577 | w.do_work(nil, nil, 'msg', nil) 578 | end 579 | 580 | it 'should be able to publish arbitrary metadata' do 581 | w = PublishingWorker.new(@queue, TestPool.new) 582 | mock(@exchange).publish('msg', :routing_key => 'target', :expiration => 1).once 583 | w.publish 'msg', :to_queue => 'target', :expiration => 1 584 | end 585 | 586 | describe 'content_type based serialization' do 587 | before do 588 | Sneakers::ContentType.register( 589 | content_type: 'application/json', 590 | serializer: ->(payload) { JSON.dump(payload) }, 591 | deserializer: ->(_) {}, 592 | ) 593 | end 594 | 595 | after do 596 | Sneakers::ContentType.reset! 597 | end 598 | 599 | it 'should be able to publish a message from working context' do 600 | w = JSONPublishingWorker.new(@queue, TestPool.new) 601 | mock(@exchange).publish('{"foo":"bar"}', :routing_key => 'target', :content_type => 'application/json').once 602 | w.do_work(nil, {}, {'foo' => 'bar'}, nil) 603 | end 604 | end 605 | 606 | describe 'content_encoding based encoding' do 607 | before do 608 | Sneakers::ContentEncoding.register( 609 | content_encoding: 'gzip', 610 | encoder: ->(payload) { gzip_compress(payload) }, 611 | decoder: ->(_) {}, 612 | ) 613 | end 614 | 615 | after do 616 | Sneakers::ContentEncoding.reset! 617 | end 618 | 619 | it 'should be able to publish a message from working context' do 620 | w = GzipPublishingWorker.new(@queue, TestPool.new) 621 | mock(@exchange).publish(gzip_compress('foobar'), :routing_key => 'target', :content_encoding => 'gzip').once 622 | w.do_work(nil, {}, 'foobar', nil) 623 | end 624 | end 625 | end 626 | 627 | 628 | describe 'Logging' do 629 | it 'should be able to use the logging facilities' do 630 | log = Logger.new('/dev/null') 631 | mock(log).debug(anything).once 632 | mock(log).info("hello").once 633 | Sneakers::Worker.configure_logger(log) 634 | 635 | w = LoggingWorker.new(@queue, TestPool.new) 636 | w.do_work(nil,nil,'msg',nil) 637 | end 638 | 639 | it 'has a helper to constuct log prefix values' do 640 | w = DummyWorker.new(@queue, TestPool.new) 641 | w.instance_variable_set(:@id, 'worker-id') 642 | m = w.log_msg('foo') 643 | _(w.log_msg('foo')).must_match(/\[worker-id\]\[#<Thread:.*>\]\[test-queue\]\[\{\}\] foo/) 644 | end 645 | 646 | describe '#worker_error' do 647 | it 'only logs backtraces if present' do 648 | w = DummyWorker.new(@queue, TestPool.new) 649 | mock(w.logger).warn('cuz') 650 | mock(w.logger).error(/\[Exception error="boom!" error_class=RuntimeError worker_class=DummyWorker\]/) 651 | w.worker_error(RuntimeError.new('boom!'), 'cuz') 652 | end 653 | end 654 | end 655 | 656 | 657 | describe 'Metrics' do 658 | before do 659 | @handler = Object.new 660 | @header = Object.new 661 | 662 | # We don't care how these are called, we're focusing on metrics here. 663 | stub(@handler).acknowledge 664 | stub(@handler).reject 665 | stub(@handler).error 666 | stub(@handler).noop 667 | 668 | @delivery_info = Object.new 669 | stub(@delivery_info).delivery_tag { "tag" } 670 | 671 | @w = MetricsWorker.new(@queue, TestPool.new) 672 | mock(@w.metrics).increment("work.MetricsWorker.started").once 673 | mock(@w.metrics).increment("work.MetricsWorker.ended").once 674 | mock(@w.metrics).timing("work.MetricsWorker.time").yields.once 675 | end 676 | 677 | it 'should be able to meter acks' do 678 | mock(@w.metrics).increment("foobar").once 679 | mock(@w.metrics).increment("work.MetricsWorker.handled.ack").once 680 | @w.do_work(@delivery_info, nil, :ack, @handler) 681 | end 682 | 683 | it 'should be able to meter rejects' do 684 | mock(@w.metrics).increment("foobar").once 685 | mock(@w.metrics).increment("work.MetricsWorker.handled.reject").once 686 | @w.do_work(@header, nil, :reject, @handler) 687 | end 688 | 689 | it 'should be able to meter requeue' do 690 | mock(@w.metrics).increment("foobar").once 691 | mock(@w.metrics).increment("work.MetricsWorker.handled.requeue").once 692 | @w.do_work(@header, nil, :requeue, @handler) 693 | end 694 | 695 | it 'should be able to meter errors' do 696 | mock(@w.metrics).increment("work.MetricsWorker.handled.error").once 697 | mock(@w).work('msg'){ raise :error } 698 | @w.do_work(@delivery_info, nil, 'msg', @handler) 699 | end 700 | 701 | it 'defaults to noop when no response is specified' do 702 | mock(@w.metrics).increment("foobar").once 703 | mock(@w.metrics).increment("work.MetricsWorker.handled.noop").once 704 | @w.do_work(@header, nil, nil, @handler) 705 | end 706 | end 707 | 708 | 709 | 710 | describe 'With Params' do 711 | before do 712 | @props = { :foo => 1 } 713 | @handler = Object.new 714 | @header = Object.new 715 | 716 | @delivery_info = Object.new 717 | 718 | stub(@handler).noop(@delivery_info, {:foo => 1}, :ack) 719 | 720 | @w = WithParamsWorker.new(@queue, TestPool.new) 721 | mock(@w.metrics).timing("work.WithParamsWorker.time").yields.once 722 | end 723 | 724 | it 'should call work_with_params and not work' do 725 | mock(@w).work_with_params(:ack, @delivery_info, {:foo => 1}).once 726 | @w.do_work(@delivery_info, {:foo => 1 }, :ack, @handler) 727 | end 728 | end 729 | end 730 | -------------------------------------------------------------------------------- /spec/sneakers/workergroup_spec.rb: -------------------------------------------------------------------------------- 1 | require 'logger' 2 | require 'spec_helper' 3 | require 'sneakers' 4 | require 'sneakers/runner' 5 | 6 | class DummyFlag 7 | def wait_for_set(*) 8 | true 9 | end 10 | end 11 | 12 | class DummyEngine 13 | include Sneakers::WorkerGroup 14 | 15 | attr_reader :config 16 | 17 | def initialize(config) 18 | @config = config 19 | @stop_flag = DummyFlag.new 20 | end 21 | end 22 | 23 | class DefaultsWorker 24 | include Sneakers::Worker 25 | from_queue 'defaults' 26 | 27 | def work(msg); end 28 | end 29 | 30 | class StubbedWorker 31 | attr_reader :opts 32 | 33 | def initialize(_, _, opts) 34 | @opts = opts 35 | end 36 | 37 | def run 38 | true 39 | end 40 | end 41 | 42 | describe Sneakers::WorkerGroup do 43 | let(:logger) { Logger.new('log/logtest.log') } 44 | let(:connection) { Bunny.new(host: 'any-host.local') } 45 | let(:runner) { Sneakers::Runner.new([DefaultsWorker]) } 46 | let(:runner_config) { runner.instance_variable_get('@runnerconfig') } 47 | let(:config) { runner_config.reload_config! } 48 | let(:engine) { DummyEngine.new(config) } 49 | 50 | describe '#run' do 51 | describe 'with connecion provided' do 52 | before do 53 | Sneakers.clear! 54 | Sneakers.configure(connection: connection, log: logger) 55 | end 56 | 57 | it 'creates workers with connection: connection' do 58 | DefaultsWorker.stub(:new, ->(*args) { StubbedWorker.new(*args) }) do 59 | engine.run 60 | 61 | workers = engine.instance_variable_get('@workers') 62 | _(workers.first.opts[:connection]).must_equal(connection) 63 | end 64 | end 65 | end 66 | 67 | describe 'without connecion provided' do 68 | before do 69 | Sneakers.clear! 70 | Sneakers.configure(log: logger) 71 | end 72 | 73 | it 'creates workers with connection: nil' do 74 | DefaultsWorker.stub(:new, ->(*args) { StubbedWorker.new(*args) }) do 75 | engine.run 76 | 77 | workers = engine.instance_variable_get('@workers') 78 | assert_nil(workers.first.opts[:connection]) 79 | end 80 | end 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'bundler/setup' 2 | require 'simplecov' 3 | require 'resolv' 4 | require 'pry-byebug' 5 | 6 | SimpleCov.start do 7 | add_filter "/spec/" 8 | end 9 | 10 | require 'minitest/autorun' 11 | 12 | require 'rr' 13 | 14 | def compose_or_localhost(key) 15 | Resolv::DNS.new.getaddress(key) 16 | rescue 17 | "localhost" 18 | end 19 | 20 | 21 | 22 | --------------------------------------------------------------------------------