├── .github ├── dependabot.yml └── workflows │ ├── check_changelog.yml │ └── ci.yml ├── .gitignore ├── CHANGELOG.md ├── Gemfile ├── MIT-LICENSE ├── README.md ├── Rakefile ├── UPGRADING.md ├── doc ├── exceptions.md ├── logging.md ├── observers.md ├── request-lifecycle.md ├── risks.md ├── rollbar.md └── settings.md ├── lib ├── rack-timeout.rb └── rack │ └── timeout │ ├── base.rb │ ├── core.rb │ ├── logger.rb │ ├── logging-observer.rb │ ├── rails.rb │ ├── rollbar.rb │ └── support │ ├── monotonic_time.rb │ ├── namespace.rb │ ├── scheduler.rb │ └── timeout.rb ├── rack-timeout.gemspec └── test ├── basic_test.rb ├── env_settings_test.rb └── test_helper.rb /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "weekly" 7 | -------------------------------------------------------------------------------- /.github/workflows/check_changelog.yml: -------------------------------------------------------------------------------- 1 | name: Check Changelog 2 | 3 | on: 4 | pull_request: 5 | types: [opened, reopened, edited, labeled, unlabeled, synchronize] 6 | 7 | jobs: 8 | check-changelog: 9 | runs-on: ubuntu-latest 10 | if: | 11 | !contains(github.event.pull_request.body, '[skip changelog]') && 12 | !contains(github.event.pull_request.body, '[changelog skip]') && 13 | !contains(github.event.pull_request.body, '[skip ci]') && 14 | !contains(github.event.pull_request.labels.*.name, 'skip changelog') && 15 | !contains(github.event.pull_request.labels.*.name, 'dependencies') && 16 | !contains(github.event.pull_request.labels.*.name, 'automation') 17 | steps: 18 | - uses: actions/checkout@v4 19 | - name: Check that CHANGELOG is touched 20 | run: | 21 | git fetch origin ${{ github.base_ref }} --depth 1 && \ 22 | git diff remotes/origin/${{ github.base_ref }} --name-only | grep CHANGELOG.md 23 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | - push 5 | - pull_request 6 | 7 | jobs: 8 | test: 9 | strategy: 10 | fail-fast: false 11 | matrix: 12 | os: [ ubuntu-latest, windows-latest ] 13 | rack-version: ['~> 2.0', '~> 3.0'] 14 | ruby: 15 | - '2.3' 16 | - '2.4' 17 | - '2.5' 18 | - '2.6' 19 | - '2.7' 20 | - '3.0' 21 | - '3.1' 22 | - '3.2' 23 | - '3.3' 24 | - '3.4' 25 | exclude: 26 | - ruby: '2.3' 27 | rack-version: '~> 3.0' 28 | runs-on: ${{ matrix.os }} 29 | env: 30 | RACK_VERSION: ${{ matrix.rack-version }} 31 | steps: 32 | - name: Checkout code 33 | uses: actions/checkout@v4 34 | - name: Set up Ruby 35 | uses: ruby/setup-ruby@v1 36 | with: 37 | ruby-version: ${{ matrix.ruby }} 38 | bundler-cache: true 39 | - name: Run test 40 | run: bundle exec rake test 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | Gemfile.lock 2 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.7.0 2 | 3 | - Honor an `X-Request-Start` header with the `t=` format, to allow using `wait_timeout` functionality with Apache (https://github.com/zombocom/rack-timeout/pull/210) 4 | - Improve message when Terminate on Timeout is used on a platform that does not support it (eg. Windows or JVM) (https://github.com/zombocom/rack-timeout/pull/192) 5 | - Fix a thread safety issue for forks that are not on the main thread (https://github.com/zombocom/rack-timeout/pull/212) 6 | - Add compatibility with frozen_string_literal: true (https://github.com/zombocom/rack-timeout/pull/196) 7 | - Fix if Rails is defined but Rails::VERSION is not defined (https://github.com/zombocom/rack-timeout/pull/191) 8 | 9 | ## 0.6.3 10 | 11 | - Fix `NoMethodError: undefined method 'logger' for Rails:Module` when Rails is defined as a Module, but is not a full Rails app (https://github.com/zombocom/rack-timeout/pull/180) 12 | 13 | ## 0.6.2 14 | 15 | - Migrate CI from Travis CI to GitHub Actions (https://github.com/zombocom/rack-timeout/pull/182) 16 | - Rails 7+ support (https://github.com/zombocom/rack-timeout/pull/184) 17 | 18 | ## 0.6.1 19 | 20 | - RACK_TIMEOUT_TERM_ON_TIMEOUT can be set to zero to disable (https://github.com/sharpstone/rack-timeout/pull/161) 21 | - Update the gemspec's homepage to the current repo URL(https://github.com/zombocom/rack-timeout/pull/183) 22 | 23 | ## 0.6.0 24 | 25 | - Allow sending SIGTERM to workers on timeout (https://github.com/sharpstone/rack-timeout/pull/157) 26 | 27 | 0.5.2 28 | ===== 29 | - Rails 6 support (#147) 30 | 31 | 0.5.1 32 | ===== 33 | - Fixes setting ENV vars to false or 0 would not disable a timeout 34 | (#133) 35 | 36 | 0.5.0.1 37 | ======= 38 | - Fix 0600 permissions in gem pushed to rubygems 39 | 40 | 0.5.0 41 | ===== 42 | 43 | Breaking Changes 44 | 45 | - Remove Rollbar module (#124) 46 | - Remove legacy class setters (#125) 47 | 48 | Other 49 | 50 | - Add support to configure via environment variables (#105) 51 | - Adds support for ActionDispatch::RequestId generated request ids (#115) 52 | - Changes uuid format to proper uuid (#115) 53 | 54 | 0.4.2 55 | ===== 56 | - Ruby 2.0 compatible 57 | 58 | 0.4.1 59 | ===== 60 | - Rails 5 support 61 | - Remove deprecation warning on timeout setter for Rails apps 62 | 63 | 0.4.0 64 | ===== 65 | - Using monotonic time instead of Time.now where available (/ht concurrent-ruby) 66 | - Settings are now passable to the middleware initializer instead of class-level 67 | - Rollbar module may take a custom fingerprint block 68 | - Rollbar module considered final 69 | - Fixed an issue where some heartbeats would live on forever (#103, /ht @0x0badc0de) 70 | 71 | 0.3.2 72 | ===== 73 | - Fixes calling timeout with a value of 0 (issue #90) 74 | 75 | 0.3.1 76 | ===== 77 | - Rollbar module improvements 78 | 79 | 0.3.0 80 | ===== 81 | - use a single scheduler thread to manage timeouts, instead of one timeout thread per request 82 | - instead of inserting middleware at position 0 for rails, insert before Rack::Runtime (which is right after Rack::Lock and the static file stuff) 83 | - reshuffle error types: RequestExpiryError is again a RuntimeError, and timeouts raise a RequestTimeoutException, an Exception, and not descending from Rack::Timeout::Error (see README for more) 84 | - don't insert middleware for rails in test environment 85 | - add convenience module Rack::Timeout::Logger (see README for more) 86 | - StageChangeLoggingObserver renamed to StateChangeLoggingObserver, works slightly differently too 87 | - file layout reorganization (see 6e82c276 for details) 88 | - CHANGELOG file is now in the gem (@dbackeus) 89 | - add optional and experimental support for grouping errors by url under rollbar. see "rack/timeout/rollbar" for usage 90 | 91 | 0.2.4 92 | ===== 93 | - Previous fix was borked. 94 | 95 | 0.2.3 96 | ===== 97 | - Ignore Rack::NullLogger when picking a logger 98 | 99 | 0.2.1 100 | ===== 101 | - Fix raised error messages 102 | 103 | 0.2.0 104 | ===== 105 | - Added CHANGELOG 106 | - Rack::Timeout::Error now inherits from Exception instead of StandardError, with the hope users won't rescue from it accidentally 107 | 108 | 0.1.2 109 | ===== 110 | - improve RequestTimeoutError error string so @watsonian is happy 111 | 112 | 0.1.1 113 | ===== 114 | - README updates 115 | - fix that setting properties to false resulted in an error 116 | 117 | 0.1.0 118 | ===== 119 | - Rewrote README 120 | 121 | 0.1.0beta4 122 | ========== 123 | - Renamed `timeout` setting to `service_timeout`; `timeout=` still works for backwards compatibility 124 | – `MAX_REQUEST_AGE` is gone, the `wait_timeout` setting more or less replaces it 125 | - Renamed `overtime` setting to `wait_overtime` 126 | - overtime setting should actually work (It had never made it to beta3) 127 | - In the request info struct, renamed `age` to `wait`, `duration` to `service` 128 | - Rack::Timeout::StageChangeLogger is gone, replaced by Rack::Timeout::StageChangeLoggingObserver, which is an observer class that composites with a logger, instead of inheriting from Logger. Anything logging related will likely be incompatible with previous beta release. 129 | - Log level can no longer be set with env vars, has to be set in the logger being used. (Which can now be custom / user-provided.) 130 | 131 | 0.1.0beta1,2,3 132 | ============== 133 | - Dropped ruby 1.8.x support 134 | - Dropped rails 2 support 135 | - Added rails 4 support 136 | - Added much logging 137 | – Added support for dropping requests that waited too long in the queue without ever handling them 138 | - Other things I can't remember, see git logs :P 139 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gemspec 4 | 5 | gem 'rack', ENV['RACK_VERSION'] if ENV['RACK_VERSION'] 6 | -------------------------------------------------------------------------------- /MIT-LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2010 Caio Chassot 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Rack::Timeout 2 | ============= 3 | 4 | Abort requests that are taking too long; an exception is raised. 5 | 6 | A timeout of 15s is the default. It's recommended to set the timeout as 7 | low as realistically viable for your application. You can modify this by 8 | setting the `RACK_TIMEOUT_SERVICE_TIMEOUT` environment variable. 9 | 10 | There's a handful of other settings, read on for details. 11 | 12 | Rack::Timeout is not a solution to the problem of long-running requests, 13 | it's a debug and remediation tool. App developers should track 14 | rack-timeout's data and address recurring instances of particular 15 | timeouts, for example by refactoring code so it runs faster or 16 | offsetting lengthy work to happen asynchronously. 17 | 18 | Upgrading 19 | --------- 20 | 21 | For fixing issues when upgrading, please see [UPGRADING](UPGRADING.md). 22 | 23 | Basic Usage 24 | ----------- 25 | 26 | The following covers currently supported versions of Rails, Rack, Ruby, 27 | and Bundler. See the Compatibility section at the end for legacy 28 | versions. 29 | 30 | ### Rails apps 31 | 32 | ```ruby 33 | # Gemfile 34 | gem "rack-timeout" 35 | ``` 36 | 37 | This will load rack-timeout and set it up as a Rails middleware using 38 | the default timeout of 15s. The middleware is not inserted for the test 39 | environment. You can modify the timeout by setting a 40 | `RACK_TIMEOUT_SERVICE_TIMEOUT` environment variable. 41 | 42 | ### Rails apps, manually 43 | 44 | You'll need to do this if you removed `Rack::Runtime` from the 45 | middleware stack, or if you want to determine yourself where in the 46 | stack `Rack::Timeout` gets inserted. 47 | 48 | ```ruby 49 | # Gemfile 50 | gem "rack-timeout", require: "rack/timeout/base" 51 | ``` 52 | 53 | ```ruby 54 | # config/initializers/rack_timeout.rb 55 | 56 | # insert middleware wherever you want in the stack, optionally pass 57 | # initialization arguments, or use environment variables 58 | Rails.application.config.middleware.insert_before Rack::Runtime, Rack::Timeout, service_timeout: 15 59 | ``` 60 | 61 | ### Sinatra and other Rack apps 62 | 63 | ```ruby 64 | # config.ru 65 | require "rack-timeout" 66 | 67 | # Call as early as possible so rack-timeout runs before all other middleware. 68 | # Setting service_timeout or `RACK_TIMEOUT_SERVICE_TIMEOUT` environment 69 | # variable is recommended. If omitted, defaults to 15 seconds. 70 | use Rack::Timeout, service_timeout: 15 71 | ``` 72 | 73 | Configuring 74 | ----------- 75 | 76 | Rack::Timeout takes the following settings, shown here with their 77 | default values and associated environment variables. 78 | 79 | ``` 80 | service_timeout: 15 # RACK_TIMEOUT_SERVICE_TIMEOUT 81 | wait_timeout: 30 # RACK_TIMEOUT_WAIT_TIMEOUT 82 | wait_overtime: 60 # RACK_TIMEOUT_WAIT_OVERTIME 83 | service_past_wait: false # RACK_TIMEOUT_SERVICE_PAST_WAIT 84 | term_on_timeout: false # RACK_TIMEOUT_TERM_ON_TIMEOUT 85 | ``` 86 | 87 | These settings can be overridden during middleware initialization or 88 | environment variables `RACK_TIMEOUT_*` mentioned above. Middleware 89 | parameters take precedence: 90 | 91 | ```ruby 92 | use Rack::Timeout, service_timeout: 15, wait_timeout: 30 93 | ``` 94 | 95 | For more on these settings, please see [doc/settings](doc/settings.md). 96 | 97 | Further Documentation 98 | --------------------- 99 | 100 | Please see the [doc](doc) folder for further documentation on: 101 | 102 | * [Risks and shortcomings of using Rack::Timeout](doc/risks.md) 103 | * [Understanding the request lifecycle](doc/request-lifecycle.md) 104 | * [Exceptions raised by Rack::Timeout](doc/exceptions.md) 105 | * [Rollbar fingerprinting](doc/rollbar.md) 106 | * [Observers](doc/observers.md) 107 | * [Settings](doc/settings.md) 108 | * [Logging](doc/logging.md) 109 | 110 | Additionally there is a [demo app](https://github.com/zombocom/rack_timeout_demos) 111 | that shows the impact of changing settings and how the library behaves 112 | when a timeout is hit. 113 | 114 | Contributing 115 | ------------ 116 | 117 | Run the test suite: 118 | 119 | ```console 120 | bundle 121 | bundle exec rake test 122 | ``` 123 | 124 | Compatibility 125 | ------------- 126 | 127 | This version of Rack::Timeout is compatible with Ruby 2.3 and up, and, 128 | for Rails apps, Rails 3.x and up. 129 | 130 | 131 | --- 132 | Copyright © 2010-2020 Caio Chassot, released under the MIT license 133 | 134 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'rake/testtask' 2 | require 'bundler/gem_tasks' 3 | 4 | Rake::TestTask.new do |t| 5 | t.libs << "test" 6 | t.test_files = FileList['test/**/*_test.rb'] 7 | t.verbose = true 8 | end 9 | 10 | task :fix_permissions do 11 | FileUtils.chmod_R("a+rX", File.dirname(__FILE__)) 12 | end 13 | 14 | task(:build).enhance([:fix_permissions]) 15 | 16 | task :default => :test 17 | -------------------------------------------------------------------------------- /UPGRADING.md: -------------------------------------------------------------------------------- 1 | Upgrading 2 | ========= 3 | 4 | From 0.4 or older 5 | ----------------- 6 | 7 | - Removal of the class setters, such as `Rack::Timeout.timeout = 5`, may 8 | lead to an error when upgrading. To fix this, remove these setters and 9 | instead use either the [environment variables][config-env], 10 | `RACK_TIMEOUT_*`, or [insert the middleware manually][config-insert] 11 | and configure the middleware as desired when inserting. 12 | 13 | [config-env]: README.md#configuring 14 | [config-insert]: README.md#rails-apps-manually 15 | 16 | - The Rollbar support was removed; a deprecation warning will be emitted 17 | if you are using this module. The recommendation is to use Rollbar's 18 | custom fingerprinting. A recommendation is provided in 19 | [doc/rollbar.md](doc/rollbar.md). 20 | -------------------------------------------------------------------------------- /doc/exceptions.md: -------------------------------------------------------------------------------- 1 | Exceptions 2 | ---------- 3 | 4 | Rack::Timeout can raise three types of exceptions. They are: 5 | 6 | Two descend from `Rack::Timeout::Error`, which itself descends from `RuntimeError` and is generally caught by an unqualified `rescue`. The third, `RequestTimeoutException`, is more complicated and the most important. 7 | 8 | * `Rack::Timeout::RequestTimeoutException`: this is raised when a request has run for longer than the specified timeout. This descends from `Exception`, not from `Rack::Timeout::Error` (it has to be rescued from explicitly). It's raised by the rack-timeout timer thread in the application thread, at the point in the stack the app happens to be in when the timeout is triggered. This exception could be caught explicitly within the application, but in doing so you're working past the timeout. This is ok for quick cleanup work but shouldn't be abused as Rack::Timeout will not kick in twice for the same request. 9 | 10 | Rails will generally intercept `Exception`s, but in plain Rack apps, this exception will be caught by rack-timeout and re-raised as a `Rack::Timeout::RequestTimeoutError`. This is to prevent an `Exception` from bubbling up beyond rack-timeout and to the server. 11 | 12 | * `Rack::Timeout::RequestTimeoutError` descends from `Rack::Timeout::Error`, but it's only really seen in the case described above. It'll not be seen in a standard Rails app, and will only be seen in Sinatra if rescuing from exceptions is disabled. 13 | 14 | * `Rack::Timeout::RequestExpiryError`: this is raised when a request is skipped for being too old (see Wait Timeout section). This error cannot generally be rescued from inside a Rails controller action as it happens before the request has a chance to enter Rails. 15 | 16 | This shouldn't be different for other frameworks, unless you have something above Rack::Timeout in the middleware stack, which you generally shouldn't. 17 | 18 | You shouldn't rescue from these errors for reporting purposes. Instead, you can subscribe for state change notifications with observers. 19 | 20 | If you're trying to test that a `Rack::Timeout::RequestTimeoutException` is raised in an action in your Rails application, you **must do so in integration tests**. Please note that Rack::Timeout will not kick in for functional tests as they bypass the rack middleware stack. 21 | 22 | [More details about testing middleware with Rails here][pablobm]. 23 | 24 | [pablobm]: http://stackoverflow.com/a/8681208/13989 25 | -------------------------------------------------------------------------------- /doc/logging.md: -------------------------------------------------------------------------------- 1 | Logging 2 | ------- 3 | 4 | Rack::Timeout logs a line every time there's a change in state in a request's lifetime. 5 | 6 | Request state changes into `timed_out` and `expired` are logged at the `ERROR` level, most other things are logged as `INFO`. The `active` state is logged as `DEBUG`, every ~1s while the request is still active. 7 | 8 | Rack::Timeout will try to use `Rails.logger` if present, otherwise it'll look for a logger in `env['rack.logger']`, and if neither are present, it'll create its own logger, either writing to `env['rack.errors']`, or to `$stderr` if the former is not set. 9 | 10 | When creating its own logger, rack-timeout will use a log level of `INFO`. Otherwise whatever log level is already set on the logger being used continues in effect. 11 | 12 | A custom logger can be set via `Rack::Timeout::Logger.logger`. This takes priority over the automatic logger detection: 13 | 14 | ```ruby 15 | Rack::Timeout::Logger.logger = Logger.new 16 | ``` 17 | 18 | There are helper setters that replace the logger: 19 | 20 | ```ruby 21 | Rack::Timeout::Logger.device = $stderr 22 | Rack::Timeout::Logger.level = Logger::INFO 23 | ``` 24 | 25 | Although each call replaces the logger, these can be use together and the final logger will retain both properties. (If only one is called, the defaults used above apply.) 26 | 27 | Logging is enabled by default, but can be removed with: 28 | 29 | ```ruby 30 | Rack::Timeout::Logger.disable 31 | ``` 32 | 33 | Each log line is a set of `key=value` pairs, containing the entries from the `env["rack-timeout.info"]` struct that are not `nil`. See the Request Lifetime section above for a description of each field. Note that while the values for `wait`, `timeout`, and `service` are stored internally as seconds, they are logged as milliseconds for readability. 34 | 35 | A sample log excerpt might look like: 36 | 37 | ``` 38 | source=rack-timeout id=13793c wait=369ms timeout=10000ms state=ready at=info 39 | source=rack-timeout id=13793c wait=369ms timeout=10000ms service=15ms state=completed at=info 40 | source=rack-timeout id=ea7bd3 wait=371ms timeout=10000ms state=timed_out at=error 41 | ``` 42 | -------------------------------------------------------------------------------- /doc/observers.md: -------------------------------------------------------------------------------- 1 | Observers 2 | --------- 3 | 4 | Observers are blocks that are notified about state changes during a request's lifetime. Keep in mind that the `active` state is set every ~1s, so you'll be notified every time. 5 | 6 | You can register an observer with: 7 | 8 | ```ruby 9 | Rack::Timeout.register_state_change_observer(:a_unique_name) { |env| do_things env } 10 | ``` 11 | 12 | There's currently no way to subscribe to changes into or out of a particular state. To check the actual state we're moving into, read `env['rack-timeout.info'].state`. Handling going out of a state would require some additional logic in the observer. 13 | 14 | You can remove an observer with `unregister_state_change_observer`: 15 | 16 | ```ruby 17 | Rack::Timeout.unregister_state_change_observer(:a_unique_name) 18 | ``` 19 | 20 | rack-timeout's logging is implemented using an observer; see `Rack::Timeout::StateChangeLoggingObserver` in logging-observer.rb for the implementation. 21 | 22 | Custom observers might be used to do cleanup, store statistics on request length, timeouts, etc., and potentially do performance tuning on the fly. 23 | -------------------------------------------------------------------------------- /doc/request-lifecycle.md: -------------------------------------------------------------------------------- 1 | Request Lifetime 2 | ---------------- 3 | 4 | Throughout a request's lifetime, Rack::Timeout keeps details about the request in `env[Rack::Timeout::ENV_INFO_KEY]`, or, more explicitly, `env["rack-timeout.info"]`. 5 | 6 | The value of that entry is an instance of `Rack::Timeout::RequestDetails`, which is a `Struct` consisting of the following fields: 7 | 8 | * `id`: a unique ID per request. Either the value of the `X-Request-ID` header or a random ID 9 | generated internally. 10 | 11 | * `wait`: time in seconds since `X-Request-Start` at the time the request was initially seen by Rack::Timeout. Only set if `X-Request-Start` is present. 12 | 13 | * `timeout`: the final timeout value that was used or to be used, in seconds. For `expired` requests, that would be the `wait_timeout`, possibly with `wait_overtime` applied. In all other cases it's the `service_timeout`, potentially reduced to make up for time lost waiting. (See discussion regarding `service_past_wait` above, under the Wait Timeout section.) 14 | 15 | * `service`: set after a request completes (or times out). The time in seconds it took being processed. This is also updated while a request is still active, around every second, with the time taken so far. 16 | 17 | * `state`: the possible states, and their log level, are: 18 | 19 | * `expired` (`ERROR`): the request is considered too old and is skipped entirely. This happens when `X-Request-Start` is present and older than `wait_timeout`. When in this state, `Rack::Timeout::RequestExpiryError` is raised. See earlier discussion about the `wait_overtime` setting, too. 20 | 21 | * `ready` (`INFO`): this is the state a request is in right before it's passed down the middleware chain. Once it's being processed, it'll move on to `active`, and then on to `timed_out` and/or `completed`. 22 | 23 | * `active` (`DEBUG`): the request is being actively processed in the application thread. This is signaled repeatedly every ~1s until the request completes or times out. 24 | 25 | * `timed_out` (`ERROR`): the request ran for longer than the determined timeout and was aborted. `Rack::Timeout::RequestTimeoutException` is raised in the application when this occurs. This state is not the final one, `completed` will be set after the framework is done with it. (If the exception does bubble up, it's caught by rack-timeout and re-raised as `Rack::Timeout::RequestTimeoutError`, which descends from RuntimeError.) 26 | 27 | * `completed` (`INFO`): the request completed and Rack::Timeout is done with it. This does not mean the request completed *successfully*. Rack::Timeout does not concern itself with that. As mentioned just above, a timed out request will still end up with a `completed` state. 28 | -------------------------------------------------------------------------------- /doc/risks.md: -------------------------------------------------------------------------------- 1 | Risks and shortcomings of using Rack::Timeout 2 | --------------------------------------------- 3 | 4 | ### Timing Out During IO Blocks 5 | 6 | Sometimes a request is taking too long to complete because it's blocked waiting on synchronous IO. Such IO does not need to be file operations, it could be, say, network or database operations. If said IO is happening in a C library that's unaware of ruby's interrupt system (i.e. anything written without ruby in mind), calling `Thread#raise` (that's what rack-timeout uses) will not have effect until after the IO block is gone. 7 | 8 | As a fail-safe against these cases, a blunter solution that kills the entire process is recommended, such as unicorn's timeouts. You can enable this process killing behavior by enabling `term_on_timeout` for more info see [setting][term-on-timeout]. 9 | 10 | More detailed explanations of the issues surrounding timing out in ruby during IO blocks can be found at: 11 | 12 | - http://redgetan.cc/understanding-timeouts-in-cruby/ 13 | 14 | ### Timing Out is Inherently Unsafe 15 | 16 | Raising mid-flight in stateful applications is inherently unsafe. A request can be aborted at any moment in the code flow, and the application can be left in an inconsistent state. There's little way rack-timeout could be aware of ongoing state changes. Applications that rely on a set of globals (like class variables) or any other state that lives beyond a single request may find those left in an unexpected/inconsistent state after an aborted request. Some cleanup code might not have run, or only half of a set of related changes may have been applied. 17 | 18 | A lot more can go wrong. An intricate explanation of the issue by JRuby's Charles Nutter can be found [ 19 | Ruby's Thread#raise, Thread#kill, timeout.rb, and net/protocol.rb libraries are broken][broken-timeout]. In addition Richard Schneeman talked about this issue in [The Oldest Bug In Ruby - Why Rack::Timeout Might Hose your Server][oldest-bug]. One solution from having `rack-timeout` corrupt process state is to restart the entire process on timeout. You can enable this behavior by setting [term_on_timeout][term-on-timeout]. 20 | 21 | Ruby 2.1+ provides a way to defer the result of raising exceptions through the [Thread.handle_interrupt][handle-interrupt] method. This low level interface is meant more for library authors than higher level application developers. This interface could be used in critical areas of your application code to prevent Rack::Timeout from accidentally wreaking havoc by raising just in the wrong moment. That said, `handle_interrupt` and threads in general are hard to reason about, and detecting all cases where it would be needed in an application is a tall order, and the added code complexity is probably not worth the trouble. 22 | 23 | Your time is better spent ensuring requests run fast and don't need to timeout. 24 | 25 | That said, it's something to be aware of, and may explain some eerie wonkiness seen in logs. 26 | 27 | [oldest-bug]: https://www.schneems.com/2017/02/21/the-oldest-bug-in-ruby-why-racktimeout-might-hose-your-server/ 28 | [broken-timeout]: http://headius.blogspot.de/2008/02/rubys-threadraise-threadkill-timeoutrb.html 29 | [handle-interrupt]: http://www.ruby-doc.org/core-2.1.3/Thread.html#method-c-handle_interrupt 30 | 31 | ### Time Out Early and Often 32 | 33 | Because of the aforementioned issues, it's recommended you set library-specific timeouts and leave Rack::Timeout as a last resort measure. Library timeouts will generally take care of IO issues and abort the operation safely. See [The Ultimate Guide to Ruby Timeouts][ruby-timeouts]. 34 | 35 | You'll want to set all relevant timeouts to something lower than Rack::Timeout's `service_timeout`. Generally you want them to be at least 1s lower, so as to account for time spent elsewhere during the request's lifetime while still giving libraries a chance to time out before Rack::Timeout. 36 | 37 | [ruby-timeouts]: https://github.com/ankane/the-ultimate-guide-to-ruby-timeouts 38 | [term-on-timeout]: https://github.com/zombocom/rack-timeout/blob/main/doc/settings.md#term-on-timeout 39 | 40 | -------------------------------------------------------------------------------- /doc/rollbar.md: -------------------------------------------------------------------------------- 1 | ### Rollbar 2 | 3 | Because rack-timeout may raise at any point in the codepath of a timed-out request, the stack traces for similar requests may differ, causing rollbar to create separate entries for each timeout. 4 | 5 | The recommended practice is to configure [Custom Fingerprints][rollbar-customfingerprint] on Rollbar. 6 | 7 | [rollbar-customfingerprint]: https://docs.rollbar.com/docs/custom-grouping/ 8 | 9 | Example: 10 | 11 | ```json 12 | [ 13 | { 14 | "condition": { 15 | "eq": "Rack::Timeout::RequestTimeoutException", 16 | "path": "body.trace.exception.class" 17 | }, 18 | "fingerprint": "Rack::Timeout::RequestTimeoutException {{context}}", 19 | "title": "Rack::Timeout::RequestTimeoutException {{context}}" 20 | } 21 | ] 22 | 23 | ``` 24 | 25 | This configuration will generate exceptions following the pattern: `Rack::Timeout::RequestTimeoutException controller#action 26 | ` 27 | 28 | On previous versions this configuration was made using `Rack::Timeout::Rollbar` which was removed. [More details on the Issue #122][rollbar-removal-issue]. 29 | 30 | [rollbar-removal-issue]: https://github.com/heroku/rack-timeout/issues/122 31 | -------------------------------------------------------------------------------- /doc/settings.md: -------------------------------------------------------------------------------- 1 | # Settings 2 | 3 | Rack::Timeout has 4 settings, each of which impacts when Rack::Timeout 4 | will raise an exception, and which type of exception will be raised. 5 | 6 | 7 | Additionally there is a [demo app](https://github.com/zombocom/rack_timeout_demos) that shows the impact of changing settings and how the library behaves when a timeout is hit. 8 | 9 | ### Service Timeout 10 | 11 | `service_timeout` is the most important setting. 12 | 13 | *Service time* is the time taken from when a request first enters rack to when its response is sent back. When the application takes longer than `service_timeout` to process a request, the request's status is logged as `timed_out` and `Rack::Timeout::RequestTimeoutException` or `Rack::Timeout::RequestTimeoutError` is raised on the application thread. This may be automatically caught by the framework or plugins, so beware. Also, the exception is not guaranteed to be raised in a timely fashion, see section below about IO blocks. 14 | 15 | Service timeout can be disabled entirely by setting the property to `0` or `false`, at which point the request skips Rack::Timeout's machinery (so no logging will be present). 16 | 17 | ### Wait Timeout 18 | 19 | Before a request reaches the rack application, it may have spent some time being received by the web server, or waiting in the application server's queue before being dispatched to rack. The time between when a request is received by the web server and when rack starts handling it is called the *wait time*. 20 | 21 | On Heroku, a request will be dropped when the routing layer sees no data being transferred for over 30 seconds. (You can read more about the specifics of Heroku routing's timeout [here][heroku-routing] and [here][heroku-timeout].) In this case, it makes no sense to process a request that reaches the application after having waited more than 30 seconds. That's where the `wait_timeout` setting comes in. When a request has a wait time greater than `wait_timeout`, it'll be dropped without ever being sent down to the application, and a `Rack::Timeout::RequestExpiryError` is raised. Such requests are logged as `expired`. 22 | 23 | [heroku-routing]: https://devcenter.heroku.com/articles/http-routing#timeouts 24 | [heroku-timeout]: https://devcenter.heroku.com/articles/request-timeout 25 | 26 | `wait_timeout` is set at a default of 30 seconds, matching Heroku's router's timeout. 27 | 28 | Wait timeout can be disabled entirely by setting the property to `0` or `false`. 29 | 30 | A request's computed wait time may affect the service timeout used for it. Basically, a request's wait time plus service time may not exceed the wait timeout. The reasoning for that is based on Heroku router's behavior, that the request would be dropped anyway after the wait timeout. So, for example, with the default settings of `service_timeout=15`, `wait_timeout=30`, a request that had 20 seconds of wait time will not have a service timeout of 15, but instead of 10, as there are only 10 seconds left before `wait_timeout` is reached. This behavior can be disabled by setting `service_past_wait` to `true`. When set, the `service_timeout` setting will always be honored. Please note that if you're using the `RACK_TIMEOUT_SERVICE_PAST_WAIT` environment variable, any value different than `"false"` will be considered `true`. 31 | 32 | The way we're able to infer a request's start time, and from that its wait time, is through the availability of the `X-Request-Start` HTTP header, which is expected to contain the time since UNIX epoch in milliseconds or microseconds. 33 | 34 | Compatible header string formats are: 35 | 36 | - `seconds.milliseconds`, e.g. `1700173924.763` - 10.3 digits (nginx format) 37 | - `t=seconds.milliseconds`, e.g. `t=1700173924.763` - 10.3 digits, nginx format with [New Relic recommended][new-relic-recommended-format] `t=` prefix 38 | - `milliseconds`, e.g. `1700173924763` - 13 digits (Heroku format) 39 | - `t=microseconds`, e.g. `t=1700173924763384` - 16 digits with `t=` prefix (Apache format) 40 | 41 | [new-relic-recommended-format]: https://docs.newrelic.com/docs/apm/applications-menu/features/request-queue-server-configuration-examples/ 42 | 43 | If the `X-Request-Start` header is not present, or does not match one of these formats, `wait_timeout` handling is skipped entirely. 44 | 45 | ### Wait Overtime 46 | 47 | Relying on `X-Request-Start` is less than ideal, as it computes the time since the request *started* being received by the web server, rather than the time the request *finished* being received by the web server. That poses a problem for lengthy requests. 48 | 49 | Lengthy requests are requests with a body, such as POST requests. These take time to complete being received by the application server, especially when the client has a slow upload speed, as is common for example with mobile clients or asymmetric connections. 50 | 51 | While we can infer the time since a request started being received, we can't tell when it completed being received, which would be preferable. We're also unable to tell the time since the last byte was sent in the request, which would be relevant in tracking Heroku's router timeout appropriately. 52 | 53 | A request that took longer than 30s to be fully received, but that had been uploading data all that while, would be dropped immediately by Rack::Timeout because it'd be considered too old. Heroku's router, however, would not have dropped this request because data was being transmitted all along. 54 | 55 | As a concession to these shortcomings, for requests that have a body present, we allow some additional wait time on top of `wait_timeout`. This aims to make up for time lost to long uploads. 56 | 57 | This extra time is called *wait overtime* and can be set via `wait_overtime`. It defaults to 60 seconds. This can be disabled as usual by setting the property to `0` or `false`. When disabled, there's no overtime. If you want lengthy requests to never get expired, set `wait_overtime` to a very high number. 58 | 59 | Keep in mind that Heroku [recommends][uploads] uploading large files directly to S3, so as to prevent the dyno from being blocked for too long and hence unable to handle further incoming requests. 60 | 61 | [uploads]: https://devcenter.heroku.com/articles/s3#file-uploads 62 | 63 | ### Term on Timeout 64 | 65 | If your application timeouts fire frequently then [they can cause your application to enter a corrupt state](https://www.schneems.com/2017/02/21/the-oldest-bug-in-ruby-why-racktimeout-might-hose-your-server/). One option for resetting that bad state is to restart the entire process. If you are running in an environment with multiple processes (such as `puma -w 2`) then when a process is sent a `SIGTERM` it will exit. The webserver then knows how to restart the process. For more information on process restart behavior see: 66 | 67 | - [Ruby Application Restart Behavior](https://devcenter.heroku.com/articles/what-happens-to-ruby-apps-when-they-are-restarted) 68 | - [License to SIGKILL](https://www.sitepoint.com/license-to-sigkill/) 69 | 70 | **Puma SIGTERM behavior** When a Puma worker receives a `SIGTERM` it will begin to shut down, but not exit right away. It stops accepting new requests and waits for any existing requests to finish before fully shutting down. This means that only the request that experiences a timeout will be interrupted, all other in-flight requests will be allowed to run until they return or also are timed out. 71 | 72 | After the worker process exits, Puma's parent process will boot a replacement worker. While one process is restarting, another can still serve requests (if you have more than 1 worker process per server/dyno). Between when a process exits and when a new process boots, there will be a reduction in throughput. If all processes are restarting, then incoming requests will be blocked while new processes boot. 73 | 74 | **How to enable** To enable this behavior you can set `term_on_timeout: 1` to an integer value. If you set it to one, then the first time the process encounters a timeout, it will receive a SIGTERM. 75 | 76 | To enable on Heroku run: 77 | 78 | ``` 79 | $ heroku config:set RACK_TIMEOUT_TERM_ON_TIMEOUT=1 80 | ``` 81 | 82 | **Caution** If you use this setting inside of a webserver without enabling multi-process mode, then it will exit the entire server when it fires: 83 | 84 | - ✅ `puma -w 2 -t 5` This is OKAY 85 | - ❌ `puma -t 5` This is NOT OKAY 86 | 87 | If you're using a `config/puma.rb` file then make sure you are calling `workers` configuration DSL. You should see multiple workers when the server boots: 88 | 89 | ``` 90 | [3922] Puma starting in cluster mode... 91 | [3922] * Version 4.3.0 (ruby 2.6.5-p114), codename: Mysterious Traveller 92 | [3922] * Min threads: 0, max threads: 16 93 | [3922] * Environment: development 94 | [3922] * Process workers: 2 95 | [3922] * Phased restart available 96 | [3922] * Listening on tcp://0.0.0.0:9292 97 | [3922] Use Ctrl-C to stop 98 | [3922] - Worker 0 (pid: 3924) booted, phase: 0 99 | [3922] - Worker 1 (pid: 3925) booted, phase: 0 100 | ``` 101 | 102 | > ✅ Notice how it says it is booting in "cluster mode" and how it gives PIDs for two worker processes at the bottom. 103 | 104 | **How to decide the term_on_timeout value** If you set to a higher value such as `5` then rack-timeout will wait until the process has experienced five timeouts before restarting the process. Setting this value to a higher number means the application restarts processes less frequently, so throughput will be less impacted. If you set it to too high of a number, then the underlying issue of the application being put into a bad state will not be effectively mitigated. 105 | 106 | 107 | **How do I know when a process is being restarted by rack-timeout?** This exception error should be visible in the logs: 108 | 109 | ``` 110 | Request ran for longer than 1000ms, sending SIGTERM to process 3925 111 | ``` 112 | 113 | > Note: Since the worker waits for all in-flight requests to finish (with puma) you may see multiple SIGTERMs to the same PID before it exits, this means that multiple requests timed out. 114 | -------------------------------------------------------------------------------- /lib/rack-timeout.rb: -------------------------------------------------------------------------------- 1 | require_relative "rack/timeout/base" 2 | require_relative "rack/timeout/rails" if defined?(Rails) && Rails.const_defined?(:VERSION) && Rails::VERSION::MAJOR >= 3 3 | -------------------------------------------------------------------------------- /lib/rack/timeout/base.rb: -------------------------------------------------------------------------------- 1 | require_relative "core" 2 | require_relative "logger" 3 | 4 | Rack::Timeout::Logger.init 5 | -------------------------------------------------------------------------------- /lib/rack/timeout/core.rb: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | require "securerandom" 3 | require_relative "support/monotonic_time" 4 | require_relative "support/scheduler" 5 | require_relative "support/timeout" 6 | 7 | module Rack 8 | class Timeout 9 | include Rack::Timeout::MonotonicTime # gets us the #fsecs method 10 | 11 | module ExceptionWithEnv # shared by the following exceptions, allows them to receive the current env 12 | attr :env 13 | def initialize(env) 14 | @env = env 15 | end 16 | end 17 | 18 | class Error < RuntimeError 19 | include ExceptionWithEnv 20 | end 21 | class RequestExpiryError < Error; end # raised when a request is dropped without being given a chance to run (because too old) 22 | class RequestTimeoutError < Error; end # raised when a request has run for too long 23 | class RequestTimeoutException < Exception # This is first raised to help prevent an application from inadvertently catching the above. It's then caught by rack-timeout and replaced with RequestTimeoutError to bubble up to wrapping middlewares and the web server 24 | include ExceptionWithEnv 25 | end 26 | 27 | RequestDetails = Struct.new( 28 | :id, # a unique identifier for the request. informative-only. 29 | :wait, # seconds the request spent in the web server before being serviced by rack 30 | :service, # time rack spent processing the request (updated ~ every second) 31 | :timeout, # the actual computed timeout to be used for this request 32 | :state, # the request's current state, see VALID_STATES below 33 | :term, 34 | ) { 35 | def ms(k) # helper method used for formatting values in milliseconds 36 | "%.fms" % (self[k] * 1000) if self[k] 37 | end 38 | } 39 | VALID_STATES = [ 40 | :expired, # The request was too old by the time it reached rack (see wait_timeout, wait_overtime) 41 | :ready, # We're about to start processing this request 42 | :active, # This request is currently being handled 43 | :timed_out, # This request has run for too long and we're raising a timeout error in it 44 | :completed, # We're done with this request (also set after having timed out a request) 45 | ] 46 | ENV_INFO_KEY = "rack-timeout.info".freeze # key under which each request's RequestDetails instance is stored in its env. 47 | HTTP_X_REQUEST_ID = "HTTP_X_REQUEST_ID".freeze # key where request id is stored if generated by upstream client/proxy 48 | ACTION_DISPATCH_REQUEST_ID = "action_dispatch.request_id".freeze # key where request id is stored if generated by action dispatch 49 | 50 | # helper methods to read timeout properties. Ensure they're always positive numbers or false. When set to false (or 0), their behaviour is disabled. 51 | def read_timeout_property value, default 52 | case value 53 | when nil ; read_timeout_property default, default 54 | when false ; false 55 | when 0 ; false 56 | else 57 | value.is_a?(Numeric) && value > 0 or raise ArgumentError, "value #{value.inspect} should be false, zero, or a positive number." 58 | value 59 | end 60 | end 61 | 62 | attr_reader \ 63 | :service_timeout, # How long the application can take to complete handling the request once it's passed down to it. 64 | :wait_timeout, # How long the request is allowed to have waited before reaching rack. If exceeded, the request is 'expired', i.e. dropped entirely without being passed down to the application. 65 | :wait_overtime, # Additional time over @wait_timeout for requests with a body, like POST requests. These may take longer to be received by the server before being passed down to the application, but should not be expired. 66 | :service_past_wait, # when false, reduces the request's computed timeout from the service_timeout value if the complete request lifetime (wait + service) would have been longer than wait_timeout (+ wait_overtime when applicable). When true, always uses the service_timeout value. we default to false under the assumption that the router would drop a request that's not responded within wait_timeout, thus being there no point in servicing beyond seconds_service_left (see code further down) up until service_timeout. 67 | :term_on_timeout 68 | 69 | def initialize(app, service_timeout:nil, wait_timeout:nil, wait_overtime:nil, service_past_wait:"not_specified", term_on_timeout: nil) 70 | @term_on_timeout = read_timeout_property term_on_timeout, ENV.fetch("RACK_TIMEOUT_TERM_ON_TIMEOUT", 0).to_i 71 | @service_timeout = read_timeout_property service_timeout, ENV.fetch("RACK_TIMEOUT_SERVICE_TIMEOUT", 15).to_i 72 | @wait_timeout = read_timeout_property wait_timeout, ENV.fetch("RACK_TIMEOUT_WAIT_TIMEOUT", 30).to_i 73 | @wait_overtime = read_timeout_property wait_overtime, ENV.fetch("RACK_TIMEOUT_WAIT_OVERTIME", 60).to_i 74 | @service_past_wait = service_past_wait == "not_specified" ? ENV.fetch("RACK_TIMEOUT_SERVICE_PAST_WAIT", false).to_s != "false" : service_past_wait 75 | 76 | if @term_on_timeout && !::Process.respond_to?(:fork) 77 | raise(NotImplementedError, <<-MSG) 78 | The platform running your application does not support forking (i.e. Windows, JVM, etc). 79 | 80 | To avoid this error, either specify RACK_TIMEOUT_TERM_ON_TIMEOUT=0 or 81 | leave it as default (which will have the same result). 82 | 83 | MSG 84 | end 85 | @app = app 86 | end 87 | 88 | 89 | RT = self # shorthand reference 90 | def call(env) 91 | info = (env[ENV_INFO_KEY] ||= RequestDetails.new) 92 | info.id ||= env[HTTP_X_REQUEST_ID] || env[ACTION_DISPATCH_REQUEST_ID] || SecureRandom.uuid 93 | 94 | time_started_service = Time.now # The wall time the request started being processed by rack 95 | ts_started_service = fsecs # The monotonic time the request started being processed by rack 96 | time_started_wait = RT._read_x_request_start(env) # The time the request was initially received by the web server (if available) 97 | effective_overtime = (wait_overtime && RT._request_has_body?(env)) ? wait_overtime : 0 # additional wait timeout (if set and applicable) 98 | seconds_service_left = nil 99 | 100 | # if X-Request-Start is present and wait_timeout is set, expire requests older than wait_timeout (+wait_overtime when applicable) 101 | if time_started_wait && wait_timeout 102 | seconds_waited = time_started_service - time_started_wait # how long it took between the web server first receiving the request and rack being able to handle it 103 | seconds_waited = 0 if seconds_waited < 0 # make up for potential time drift between the routing server and the application server 104 | final_wait_timeout = wait_timeout + effective_overtime # how long the request will be allowed to have waited 105 | seconds_service_left = final_wait_timeout - seconds_waited # first calculation of service timeout (relevant if request doesn't get expired, may be overridden later) 106 | info.wait = seconds_waited # updating the info properties; info.timeout will be the wait timeout at this point 107 | info.timeout = final_wait_timeout 108 | 109 | if seconds_service_left <= 0 # expire requests that have waited for too long in the queue (as they are assumed to have been dropped by the web server / routing layer at this point) 110 | RT._set_state! env, :expired 111 | raise RequestExpiryError.new(env), "Request older than #{info.ms(:timeout)}." 112 | end 113 | end 114 | 115 | # pass request through if service_timeout is false (i.e., don't time it out at all.) 116 | return @app.call(env) unless service_timeout 117 | 118 | # compute actual timeout to be used for this request; if service_past_wait is true, this is just service_timeout. If false (the default), and wait time was determined, we'll use the shortest value between seconds_service_left and service_timeout. See comment above at service_past_wait for justification. 119 | info.timeout = service_timeout # nice and simple, when service_past_wait is true, not so much otherwise: 120 | info.timeout = seconds_service_left if !service_past_wait && seconds_service_left && seconds_service_left > 0 && seconds_service_left < service_timeout 121 | info.term = term_on_timeout 122 | RT._set_state! env, :ready # we're good to go, but have done nothing yet 123 | 124 | heartbeat_event = nil # init var so it's in scope for following proc 125 | register_state_change = ->(status = :active) { # updates service time and state; will run every second 126 | heartbeat_event.cancel! if status != :active # if the request is no longer active we should stop updating every second 127 | info.service = fsecs - ts_started_service # update service time 128 | RT._set_state! env, status # update status 129 | } 130 | heartbeat_event = RT::Scheduler.run_every(1) { register_state_change.call :active } # start updating every second while active; if log level is debug, this will log every sec 131 | 132 | timeout = RT::Scheduler::Timeout.new do |app_thread| # creates a timeout instance responsible for timing out the request. the given block runs if timed out 133 | register_state_change.call :timed_out 134 | 135 | message = +"Request " 136 | message << "waited #{info.ms(:wait)}, then " if info.wait 137 | message << "ran for longer than #{info.ms(:timeout)} " 138 | if term_on_timeout 139 | Thread.main['RACK_TIMEOUT_COUNT'] ||= 0 140 | Thread.main['RACK_TIMEOUT_COUNT'] += 1 141 | 142 | if Thread.main['RACK_TIMEOUT_COUNT'] >= term_on_timeout 143 | message << ", sending SIGTERM to process #{Process.pid}" 144 | Process.kill("SIGTERM", Process.pid) 145 | else 146 | message << ", #{Thread.main['RACK_TIMEOUT_COUNT']}/#{term_on_timeout} timeouts allowed before SIGTERM for process #{Process.pid}" 147 | end 148 | end 149 | 150 | app_thread.raise(RequestTimeoutException.new(env), message) 151 | end 152 | 153 | response = timeout.timeout(info.timeout) do # perform request with timeout 154 | begin @app.call(env) # boom, send request down the middleware chain 155 | rescue RequestTimeoutException => e # will actually hardly ever get to this point because frameworks tend to catch this. see README for more 156 | raise RequestTimeoutError.new(env), e.message, e.backtrace # but in case it does get here, re-raise RequestTimeoutException as RequestTimeoutError 157 | ensure 158 | register_state_change.call :completed 159 | end 160 | end 161 | 162 | response 163 | end 164 | 165 | ### following methods are used internally (called by instances, so can't be private. _ marker should discourage people from calling them) 166 | 167 | # X-Request-Start contains the time the request was first seen by the server. Format varies wildly amongst servers, yay! 168 | # - nginx gives the time since epoch as seconds.milliseconds[1]. New Relic documentation recommends preceding it with t=[2], so might as well detect it. 169 | # - Heroku gives the time since epoch in milliseconds. [3] 170 | # - Apache uses t=microseconds[4], so 16 digits (until November 2286). 171 | # 172 | # The sane way to handle this would be by knowing the server being used, instead let's just hack around with regular expressions. 173 | # [1]: http://nginx.org/en/docs/http/ngx_http_log_module.html#var_msec 174 | # [2]: https://docs.newrelic.com/docs/apm/other-features/request-queueing/request-queue-server-configuration-examples#nginx 175 | # [3]: https://devcenter.heroku.com/articles/http-routing#heroku-headers 176 | # [4]: http://httpd.apache.org/docs/current/mod/mod_headers.html#header 177 | # 178 | # This is a code extraction for readability, this method is only called from a single point. 179 | RX_NGINX_X_REQUEST_START = /^(?:t=)?(\d+)\.(\d{3})$/ 180 | RX_HEROKU_X_REQUEST_START = /^(\d+)$/ 181 | RX_APACHE_X_REQUEST_START = /^t=(\d{16})$/ 182 | HTTP_X_REQUEST_START = "HTTP_X_REQUEST_START".freeze 183 | def self._read_x_request_start(env) 184 | return unless s = env[HTTP_X_REQUEST_START] 185 | if m = s.match(RX_HEROKU_X_REQUEST_START) || s.match(RX_NGINX_X_REQUEST_START) 186 | Time.at(m[1,2].join.to_f / 1000) 187 | elsif m = s.match(RX_APACHE_X_REQUEST_START) 188 | Time.at(m[1].to_f / 1_000_000) 189 | end 190 | end 191 | 192 | # This method determines if a body is present. requests with a body (generally POST, PUT) can have a lengthy body which may have taken a while to be received by the web server, inflating their computed wait time. This in turn could lead to unwanted expirations. See wait_overtime property as a way to overcome those. 193 | # This is a code extraction for readability, this method is only called from a single point. 194 | def self._request_has_body?(env) 195 | return true if env["HTTP_TRANSFER_ENCODING"] == "chunked" 196 | return false if env["CONTENT_LENGTH"].nil? 197 | return false if env["CONTENT_LENGTH"].to_i.zero? 198 | true 199 | end 200 | 201 | def self._set_state!(env, state) 202 | raise "Invalid state: #{state.inspect}" unless VALID_STATES.include? state 203 | env[ENV_INFO_KEY].state = state 204 | notify_state_change_observers(env) 205 | end 206 | 207 | ### state change notification-related methods 208 | @state_change_observers = {} 209 | 210 | # Registers a block to be called back when a request changes state in rack-timeout. The block will receive the request's env. 211 | # 212 | # `id` is anything that uniquely identifies this particular callback, mostly so it may be removed via `unregister_state_change_observer`. 213 | def self.register_state_change_observer(id, &callback) 214 | raise RuntimeError, "An observer with the id #{id.inspect} is already set." if @state_change_observers.key? id 215 | raise ArgumentError, "A callback block is required." unless callback 216 | @state_change_observers[id] = callback 217 | end 218 | 219 | # Removes the observer with the given id 220 | def self.unregister_state_change_observer(id) 221 | @state_change_observers.delete(id) 222 | end 223 | 224 | private 225 | # Sends out the notifications. Called internally at the end of `_set_state!` 226 | def self.notify_state_change_observers(env) 227 | @state_change_observers.values.each { |observer| observer.call(env) } 228 | end 229 | end 230 | end 231 | -------------------------------------------------------------------------------- /lib/rack/timeout/logger.rb: -------------------------------------------------------------------------------- 1 | require "logger" 2 | require_relative "core" 3 | require_relative "logging-observer" 4 | 5 | module Rack::Timeout::Logger 6 | extend self 7 | attr :device, :level, :logger 8 | 9 | def device=(new_device) 10 | update(new_device, level) 11 | end 12 | 13 | def level=(new_level) 14 | update(device, new_level) 15 | end 16 | 17 | def logger=(new_logger) 18 | @logger = @observer.logger = new_logger 19 | end 20 | 21 | def init 22 | @observer = ::Rack::Timeout::StateChangeLoggingObserver.new 23 | ::Rack::Timeout.register_state_change_observer(:logger, &@observer.callback) 24 | @inited = true 25 | end 26 | 27 | def disable 28 | @observer, @logger, @level, @device, @inited = nil 29 | ::Rack::Timeout.unregister_state_change_observer(:logger) 30 | end 31 | 32 | def update(new_device, new_level) 33 | init unless @inited 34 | @device = new_device || $stderr 35 | @level = new_level || ::Logger::INFO 36 | self.logger = ::Rack::Timeout::StateChangeLoggingObserver.mk_logger(device, level) 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/rack/timeout/logging-observer.rb: -------------------------------------------------------------------------------- 1 | require "logger" 2 | require_relative "core" 3 | 4 | class Rack::Timeout::StateChangeLoggingObserver 5 | STATE_LOG_LEVEL = { :expired => :error, 6 | :ready => :info, 7 | :active => :debug, 8 | :timed_out => :error, 9 | :completed => :info, 10 | } 11 | def initialize 12 | @logger = nil 13 | end 14 | 15 | # returns the Proc to be used as the observer callback block 16 | def callback 17 | method(:log_state_change) 18 | end 19 | 20 | SIMPLE_FORMATTER = ->(severity, timestamp, progname, msg) { "#{msg} at=#{severity.downcase}\n" } 21 | def self.mk_logger(device, level = ::Logger::INFO) 22 | ::Logger.new(device).tap do |logger| 23 | logger.level = level 24 | logger.formatter = SIMPLE_FORMATTER 25 | end 26 | end 27 | 28 | 29 | attr_writer :logger 30 | 31 | private 32 | 33 | def logger(env = nil) 34 | @logger || 35 | (defined?(::Rails) && ::Rails.respond_to?(:logger) && ::Rails.logger) || 36 | (env && !env["rack.logger"].is_a?(::Rack::NullLogger) && env["rack.logger"]) || 37 | (env && env["rack.errors"] && self.class.mk_logger(env["rack.errors"])) || 38 | (@fallback_logger ||= self.class.mk_logger($stderr)) 39 | end 40 | 41 | # generates the actual log string 42 | def log_state_change(env) 43 | info = env[::Rack::Timeout::ENV_INFO_KEY] 44 | level = STATE_LOG_LEVEL[info.state] 45 | logger(env).send(level) do 46 | s = +"source=rack-timeout" 47 | s << " id=" << info.id if info.id 48 | s << " wait=" << info.ms(:wait) if info.wait 49 | s << " timeout=" << info.ms(:timeout) if info.timeout 50 | s << " service=" << info.ms(:service) if info.service 51 | s << " term_on_timeout=" << info.term.to_s if info.term 52 | s << " state=" << info.state.to_s if info.state 53 | s 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /lib/rack/timeout/rails.rb: -------------------------------------------------------------------------------- 1 | require_relative "base" 2 | 3 | class Rack::Timeout::Railtie < Rails::Railtie 4 | initializer("rack-timeout.prepend") do |app| 5 | next if Rails.env.test? 6 | 7 | if defined?(ActionDispatch::RequestId) 8 | app.config.middleware.insert_after(ActionDispatch::RequestId, Rack::Timeout) 9 | else 10 | app.config.middleware.insert_before(Rack::Runtime, Rack::Timeout) 11 | end 12 | end 13 | end -------------------------------------------------------------------------------- /lib/rack/timeout/rollbar.rb: -------------------------------------------------------------------------------- 1 | warn 'DEPRECATION WARNING: The Rollbar module was removed from rack-timeout. For more details check the README on heroku/rack-timeout' -------------------------------------------------------------------------------- /lib/rack/timeout/support/monotonic_time.rb: -------------------------------------------------------------------------------- 1 | require_relative "namespace" 2 | 3 | # lifted from https://github.com/ruby-concurrency/concurrent-ruby/blob/master/lib/concurrent/utility/monotonic_time.rb 4 | 5 | module Rack::Timeout::MonotonicTime 6 | extend self 7 | 8 | def fsecs_mono 9 | Process.clock_gettime Process::CLOCK_MONOTONIC 10 | end 11 | 12 | def fsecs_java 13 | java.lang.System.nanoTime() / 1_000_000_000.0 14 | end 15 | 16 | mutex = Mutex.new 17 | last_time = Time.now.to_f 18 | define_method(:fsecs_ruby) do 19 | now = Time.now.to_f 20 | mutex.synchronize { last_time = last_time < now ? now : last_time + 1e-6 } 21 | end 22 | 23 | case 24 | when defined? Process::CLOCK_MONOTONIC ; alias fsecs fsecs_mono 25 | when RUBY_PLATFORM == "java" ; alias fsecs fsecs_java 26 | else ; alias fsecs fsecs_ruby 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/rack/timeout/support/namespace.rb: -------------------------------------------------------------------------------- 1 | # can be required by other files to prevent them from having to open and nest Rack and Timeout 2 | module Rack 3 | class Timeout 4 | end 5 | end 6 | -------------------------------------------------------------------------------- /lib/rack/timeout/support/scheduler.rb: -------------------------------------------------------------------------------- 1 | require_relative "namespace" 2 | require_relative "monotonic_time" 3 | 4 | # Runs code at a later time 5 | # 6 | # Basic usage: 7 | # 8 | # Scheduler.run_in(5) { do_stuff } # <- calls do_stuff 5 seconds from now 9 | # 10 | # Scheduled events run in sequence in a separate thread, the main thread continues on. 11 | # That means you may need to #join the scheduler if the main thread is only waiting on scheduled events to run. 12 | # 13 | # Scheduler.join 14 | # 15 | # Basic usage is through a singleton instance, its methods are available as class methods, as shown above. 16 | # One could also instantiate separate instances which would get you separate run threads, but generally there's no point in it. 17 | class Rack::Timeout::Scheduler 18 | MAX_IDLE_SECS = 30 # how long the runner thread is allowed to live doing nothing 19 | include Rack::Timeout::MonotonicTime # gets us the #fsecs method 20 | 21 | # stores a proc to run later, and the time it should run at 22 | class RunEvent < Struct.new(:monotime, :proc) 23 | def initialize(*args) 24 | @cancelled = false 25 | super(*args) 26 | end 27 | 28 | def cancel! 29 | @cancelled = true 30 | end 31 | 32 | def cancelled? 33 | !!@cancelled 34 | end 35 | 36 | def run! 37 | return if @cancelled 38 | proc.call(self) 39 | end 40 | end 41 | 42 | class RepeatEvent < RunEvent 43 | def initialize(monotime, proc, every) 44 | @start = monotime 45 | @every = every 46 | @iter = 0 47 | super(monotime, proc) 48 | end 49 | 50 | def run! 51 | super 52 | ensure 53 | self.monotime = @start + @every * (@iter += 1) until monotime >= Rack::Timeout::MonotonicTime.fsecs 54 | end 55 | end 56 | 57 | def initialize 58 | @runner = nil 59 | @events = [] # array of `RunEvent`s 60 | @mx_events = Mutex.new # mutex to change said array 61 | @mx_runner = Mutex.new # mutex for creating a runner thread 62 | end 63 | 64 | 65 | private 66 | 67 | # returns the runner thread, creating it if needed 68 | def runner 69 | @mx_runner.synchronize { 70 | return @runner unless @runner.nil? || !@runner.alive? 71 | @joined = false 72 | @runner = Thread.new { run_loop! } 73 | } 74 | end 75 | 76 | # the actual runner thread loop 77 | def run_loop! 78 | Thread.current.abort_on_exception = true # always be aborting 79 | sleep_for, run, last_run = nil, nil, fsecs # sleep_for: how long to sleep before next run; last_run: time of last run; run: just initializing it outside of the synchronize scope, will contain events to run now 80 | loop do # begin event reader loop 81 | @mx_events.synchronize { # 82 | @events.reject!(&:cancelled?) # get rid of cancelled events 83 | if @events.empty? # if there are no further events … 84 | return if @joined # exit the run loop if this runner thread has been joined (the thread will die and the join will return) 85 | return if fsecs - last_run > MAX_IDLE_SECS # exit the run loop if done nothing for the past MAX_IDLE_SECS seconds 86 | sleep_for = MAX_IDLE_SECS # sleep for MAX_IDLE_SECS (mind it that we get awaken when new events are scheduled) 87 | else # 88 | sleep_for = [@events.map(&:monotime).min - fsecs, 0].max # if we have events, set to sleep until it's time for the next one to run. (the max bit ensure we don't have negative sleep times) 89 | end # 90 | @mx_events.sleep sleep_for # do sleep 91 | # 92 | now = fsecs # 93 | run, defer = @events.partition { |ev| ev.monotime <= now } # separate events to run now and events to run later 94 | defer += run.select { |ev| ev.is_a? RepeatEvent } # repeat events both run and are deferred 95 | @events.replace(defer) # keep only events to run later 96 | } # 97 | # 98 | next if run.empty? # done here if there's nothing to run now 99 | run.sort_by(&:monotime).each { |ev| ev.run! } # run the events scheduled to run now 100 | last_run = fsecs # store that we did run things at this time, go immediately on to the next loop iteration as it may be time to run more things 101 | end 102 | end 103 | 104 | 105 | public 106 | 107 | # waits on the runner thread to finish 108 | def join 109 | @joined = true 110 | runner.join 111 | end 112 | 113 | # adds a RunEvent struct to the run schedule 114 | def schedule(event) 115 | @mx_events.synchronize { @events << event } 116 | runner.run # wakes up the runner thread so it can recalculate sleep length taking this new event into consideration 117 | return event 118 | end 119 | 120 | # reschedules an event by the given number of seconds. can be negative to run sooner. 121 | # returns nil and does nothing if the event is not already in the queue (might've run already), otherwise updates the event time in-place; returns the updated event. 122 | def delay(event, secs) 123 | @mx_events.synchronize { 124 | return unless @events.include? event 125 | event.monotime += secs 126 | runner.run 127 | return event 128 | } 129 | end 130 | 131 | # schedules a block to run in the given number of seconds; returns the created event object 132 | def run_in(secs, &block) 133 | schedule RunEvent.new(fsecs + secs, block) 134 | end 135 | 136 | # schedules a block to run every x seconds; returns the created event object 137 | def run_every(seconds, &block) 138 | schedule RepeatEvent.new(fsecs, block, seconds) 139 | end 140 | 141 | 142 | ### Singleton access 143 | 144 | # accessor to the singleton instance 145 | def self.singleton 146 | @singleton ||= new 147 | end 148 | 149 | # define public instance methods as class methods that delegate to the singleton instance 150 | instance_methods(false).each do |m| 151 | define_singleton_method(m) { |*a, &b| singleton.send(m, *a, &b) } 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/rack/timeout/support/timeout.rb: -------------------------------------------------------------------------------- 1 | require_relative "namespace" 2 | require_relative "scheduler" 3 | 4 | class Rack::Timeout::Scheduler::Timeout 5 | class Error < RuntimeError; end 6 | ON_TIMEOUT = ->thr { thr.raise Error, "execution expired" } # default action to take when a timeout happens 7 | 8 | # initializes a timeout object with an optional block to handle the timeout differently. the block is passed the thread that's gone overtime. 9 | def initialize(&on_timeout) 10 | @on_timeout = on_timeout || ON_TIMEOUT 11 | @scheduler = Rack::Timeout::Scheduler.singleton 12 | end 13 | 14 | # takes number of seconds to wait before timing out, and code block subject to time out 15 | def timeout(secs, &block) 16 | return block.call if secs.nil? || secs.zero? # skip timeout flow entirely for zero or nil 17 | thr = Thread.current # reference to current thread to be used in timeout thread 18 | job = @scheduler.run_in(secs) { @on_timeout.call thr } # schedule this thread to be timed out; should get cancelled if block completes on time 19 | return block.call # do what you gotta do 20 | ensure # 21 | job.cancel! if job # cancel the scheduled timeout job; if the block completed on time, this 22 | end # will get called before the timeout code's had a chance to run. 23 | 24 | # timeout method on singleton instance for when a custom on_timeout is not required 25 | def self.timeout(secs, &block) 26 | (@singleton ||= new).timeout(secs, &block) 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /rack-timeout.gemspec: -------------------------------------------------------------------------------- 1 | RACK_TIMEOUT_VERSION = "0.7.0" 2 | 3 | Gem::Specification.new do |spec| 4 | spec.name = "rack-timeout" 5 | spec.summary = "Abort requests that are taking too long" 6 | spec.description = "Rack middleware which aborts requests that have been running for longer than a specified timeout." 7 | spec.version = RACK_TIMEOUT_VERSION 8 | spec.homepage = "https://github.com/zombocom/rack-timeout" 9 | spec.author = "Caio Chassot" 10 | spec.email = "caio@heroku.com" 11 | spec.files = Dir[*%w( MIT-LICENSE CHANGELOG.md UPGRADING.md README.md lib/**/* doc/**/* )] 12 | spec.license = "MIT" 13 | spec.metadata = { 14 | "bug_tracker_uri" => "#{spec.homepage}/issues", 15 | "changelog_uri" => "#{spec.homepage}/blob/v#{RACK_TIMEOUT_VERSION}/CHANGELOG.md", 16 | "documentation_uri" => "https://rubydoc.info/gems/rack-timeout/#{RACK_TIMEOUT_VERSION}/", 17 | "source_code_uri" => spec.homepage 18 | } 19 | 20 | spec.test_files = Dir.glob("test/**/*").concat([ 21 | "Gemfile", 22 | "Rakefile" 23 | ]) 24 | 25 | spec.add_development_dependency("rake") 26 | spec.add_development_dependency("rack-test") 27 | spec.add_development_dependency("test-unit") 28 | end 29 | -------------------------------------------------------------------------------- /test/basic_test.rb: -------------------------------------------------------------------------------- 1 | require "test_helper" 2 | 3 | class BasicTest < RackTimeoutTest 4 | def test_ok 5 | self.settings = { service_timeout: 1 } 6 | get "/" 7 | assert last_response.ok? 8 | end 9 | 10 | def test_timeout 11 | self.settings = { service_timeout: 1 } 12 | assert_raises(Rack::Timeout::RequestTimeoutError) do 13 | get "/sleep" 14 | end 15 | end 16 | 17 | def test_wait_timeout 18 | self.settings = { service_timeout: 1, wait_timeout: 15 } 19 | assert_raises(Rack::Timeout::RequestExpiryError) do 20 | get "/", "", 'HTTP_X_REQUEST_START' => time_in_msec(Time.now - 100) 21 | end 22 | end 23 | 24 | def test_apache_formatted_header_wait_timeout 25 | self.settings = { service_timeout: 1, wait_timeout: 15 } 26 | assert_raises(Rack::Timeout::RequestExpiryError) do 27 | get "/", "", 'HTTP_X_REQUEST_START' => "t=#{time_in_usec(Time.now - 100)}" 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /test/env_settings_test.rb: -------------------------------------------------------------------------------- 1 | require 'test_helper' 2 | 3 | class EnvSettingsTest < RackTimeoutTest 4 | 5 | 6 | def test_zero_wait_timeout 7 | with_env(RACK_TIMEOUT_WAIT_TIMEOUT: 0) do 8 | get "/", "", 'HTTP_X_REQUEST_START' => time_in_msec(Time.now - 100) 9 | assert last_response.ok? 10 | end 11 | end 12 | 13 | 14 | if Process.respond_to?(:fork) # This functionality does not work on windows, so we cannot test it there. 15 | def test_service_timeout 16 | with_env(RACK_TIMEOUT_SERVICE_TIMEOUT: 1) do 17 | assert_raises(Rack::Timeout::RequestTimeoutError) do 18 | get "/sleep" 19 | end 20 | end 21 | end 22 | 23 | def test_term 24 | with_env(RACK_TIMEOUT_TERM_ON_TIMEOUT: 1) do 25 | assert_raises(SignalException) do 26 | get "/sleep" 27 | end 28 | end 29 | end 30 | else 31 | def test_service_timeout # Confirm that on Windows we raise an exception when someone attempts to use term on timeout 32 | with_env(RACK_TIMEOUT_TERM_ON_TIMEOUT: 1) do 33 | assert_raises(NotImplementedError) do 34 | get "/" 35 | end 36 | end 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /test/test_helper.rb: -------------------------------------------------------------------------------- 1 | require "test/unit" 2 | require "rack" 3 | require "rack/test" 4 | require "rack/builder" 5 | require "rack/null_logger" 6 | require "rack-timeout" 7 | 8 | class RackTimeoutTest < Test::Unit::TestCase 9 | include Rack::Test::Methods 10 | 11 | attr_accessor :settings 12 | 13 | def initialize(*args) 14 | self.settings ||= {} 15 | super(*args) 16 | end 17 | 18 | def app 19 | settings = self.settings 20 | Rack::Builder.new do 21 | use Rack::Timeout, **settings 22 | 23 | map "/" do 24 | run lambda { |env| [200, {'Content-Type' => 'text/plain'}, ['OK']] } 25 | end 26 | 27 | map "/sleep" do 28 | run lambda { |env| sleep } 29 | end 30 | end 31 | end 32 | 33 | # runs the test with the given environment, but doesn't restore the original 34 | # environment afterwards. This should be sufficient for rack-timeout testing. 35 | def with_env(hash) 36 | hash.each_pair do |k, v| 37 | ENV[k.to_s] = v.to_s 38 | end 39 | yield 40 | hash.each_key do |k| 41 | ENV[k.to_s] = nil 42 | end 43 | end 44 | 45 | def time_in_msec(t = Time.now) 46 | "#{t.tv_sec}#{t.tv_usec/1000}" 47 | end 48 | 49 | def time_in_usec(t = Time.now) 50 | # time in microseconds, currently 16 digits 51 | "%d%06d" % [t.tv_sec, t.tv_usec] 52 | end 53 | end 54 | --------------------------------------------------------------------------------