├── .github └── workflows │ └── ci.yml ├── .gitignore ├── .rspec ├── .rubocop.yml ├── .rubocop ├── layout.yml ├── metrics.yml ├── rspec.yml └── style.yml ├── .rubocop_todo.yml ├── .yardopts ├── CHANGELOG.md ├── CHANGES_OLD.md ├── CONTRIBUTING.md ├── Gemfile ├── Guardfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── SECURITY.md ├── http.gemspec ├── lib ├── http.rb └── http │ ├── base64.rb │ ├── chainable.rb │ ├── client.rb │ ├── connection.rb │ ├── content_type.rb │ ├── errors.rb │ ├── feature.rb │ ├── features │ ├── auto_deflate.rb │ ├── auto_inflate.rb │ ├── instrumentation.rb │ ├── logging.rb │ ├── normalize_uri.rb │ └── raise_error.rb │ ├── headers.rb │ ├── headers │ ├── known.rb │ ├── mixin.rb │ └── normalizer.rb │ ├── mime_type.rb │ ├── mime_type │ ├── adapter.rb │ └── json.rb │ ├── options.rb │ ├── redirector.rb │ ├── request.rb │ ├── request │ ├── body.rb │ └── writer.rb │ ├── response.rb │ ├── response │ ├── body.rb │ ├── inflater.rb │ ├── parser.rb │ ├── status.rb │ └── status │ │ └── reasons.rb │ ├── retriable │ ├── client.rb │ ├── delay_calculator.rb │ ├── errors.rb │ └── performer.rb │ ├── timeout │ ├── global.rb │ ├── null.rb │ └── per_operation.rb │ ├── uri.rb │ └── version.rb ├── logo.png └── spec ├── lib ├── http │ ├── client_spec.rb │ ├── connection_spec.rb │ ├── content_type_spec.rb │ ├── features │ │ ├── auto_deflate_spec.rb │ │ ├── auto_inflate_spec.rb │ │ ├── instrumentation_spec.rb │ │ ├── logging_spec.rb │ │ └── raise_error_spec.rb │ ├── headers │ │ ├── mixin_spec.rb │ │ └── normalizer_spec.rb │ ├── headers_spec.rb │ ├── options │ │ ├── body_spec.rb │ │ ├── features_spec.rb │ │ ├── form_spec.rb │ │ ├── headers_spec.rb │ │ ├── json_spec.rb │ │ ├── merge_spec.rb │ │ ├── new_spec.rb │ │ └── proxy_spec.rb │ ├── options_spec.rb │ ├── redirector_spec.rb │ ├── request │ │ ├── body_spec.rb │ │ └── writer_spec.rb │ ├── request_spec.rb │ ├── response │ │ ├── body_spec.rb │ │ ├── parser_spec.rb │ │ └── status_spec.rb │ ├── response_spec.rb │ ├── retriable │ │ ├── delay_calculator_spec.rb │ │ └── performer_spec.rb │ ├── uri │ │ └── normalizer_spec.rb │ └── uri_spec.rb └── http_spec.rb ├── regression_specs.rb ├── spec_helper.rb └── support ├── black_hole.rb ├── capture_warning.rb ├── dummy_server.rb ├── dummy_server └── servlet.rb ├── fakeio.rb ├── fuubar.rb ├── http_handling_shared.rb ├── proxy_server.rb ├── servers ├── config.rb └── runner.rb ├── simplecov.rb └── ssl_helper.rb /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | BUNDLE_WITHOUT: "development" 11 | JRUBY_OPTS: "--dev --debug" 12 | 13 | jobs: 14 | test: 15 | runs-on: ${{ matrix.os }} 16 | 17 | strategy: 18 | matrix: 19 | ruby: [ ruby-3.0, ruby-3.1, ruby-3.2, ruby-3.3 ] 20 | os: [ ubuntu-latest ] 21 | 22 | steps: 23 | - uses: actions/checkout@v4 24 | 25 | - uses: ruby/setup-ruby@v1 26 | with: 27 | ruby-version: ${{ matrix.ruby }} 28 | bundler-cache: true 29 | 30 | - name: bundle exec rspec 31 | run: bundle exec rspec --format progress --force-colour 32 | 33 | test-flaky: 34 | runs-on: ${{ matrix.os }} 35 | 36 | strategy: 37 | matrix: 38 | ruby: [ jruby-9.4 ] 39 | os: [ ubuntu-latest ] 40 | 41 | steps: 42 | - uses: actions/checkout@v4 43 | 44 | - uses: ruby/setup-ruby@v1 45 | with: 46 | ruby-version: ${{ matrix.ruby }} 47 | bundler-cache: true 48 | 49 | - name: bundle exec rspec 50 | continue-on-error: true 51 | run: bundle exec rspec --format progress --force-colour 52 | 53 | lint: 54 | runs-on: ubuntu-latest 55 | 56 | steps: 57 | - uses: actions/checkout@v4 58 | 59 | - uses: ruby/setup-ruby@v1 60 | with: 61 | ruby-version: ruby-3.0 62 | bundler-cache: true 63 | 64 | - name: bundle exec rubocop 65 | run: bundle exec rubocop --format progress --color 66 | 67 | - run: bundle exec rake verify_measurements 68 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .config 3 | .rvmrc 4 | .yardoc 5 | InstalledFiles 6 | _yardoc 7 | 8 | .bundle 9 | .ruby-version 10 | doc 11 | coverage 12 | pkg 13 | spec/examples.txt 14 | tmp 15 | Gemfile.lock 16 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | require: 2 | - rubocop-performance 3 | - rubocop-rake 4 | - rubocop-rspec 5 | 6 | inherit_from: 7 | - .rubocop_todo.yml 8 | - .rubocop/layout.yml 9 | - .rubocop/metrics.yml 10 | - .rubocop/rspec.yml 11 | - .rubocop/style.yml 12 | 13 | AllCops: 14 | DefaultFormatter: fuubar 15 | DisplayCopNames: true 16 | DisplayStyleGuide: true 17 | NewCops: enable 18 | TargetRubyVersion: 3.0 19 | -------------------------------------------------------------------------------- /.rubocop/layout.yml: -------------------------------------------------------------------------------- 1 | Layout/DotPosition: 2 | Enabled: true 3 | EnforcedStyle: leading 4 | 5 | Layout/FirstHashElementIndentation: 6 | Enabled: true 7 | EnforcedStyle: consistent 8 | 9 | Layout/HashAlignment: 10 | Enabled: true 11 | EnforcedColonStyle: table 12 | EnforcedHashRocketStyle: table 13 | -------------------------------------------------------------------------------- /.rubocop/metrics.yml: -------------------------------------------------------------------------------- 1 | Metrics/BlockLength: 2 | CountAsOne: 3 | - array 4 | - heredoc 5 | - method_call 6 | Exclude: 7 | - 'spec/**/*.rb' 8 | - '*.gemspec' 9 | 10 | Metrics/ClassLength: 11 | CountAsOne: 12 | - array 13 | - heredoc 14 | - method_call 15 | 16 | Metrics/MethodLength: 17 | CountAsOne: 18 | - array 19 | - heredoc 20 | - method_call 21 | 22 | Metrics/ModuleLength: 23 | CountAsOne: 24 | - array 25 | - heredoc 26 | - method_call 27 | -------------------------------------------------------------------------------- /.rubocop/rspec.yml: -------------------------------------------------------------------------------- 1 | RSpec/ExampleLength: 2 | CountAsOne: 3 | - array 4 | - hash 5 | - heredoc 6 | - method_call 7 | 8 | RSpec/MultipleExpectations: 9 | Max: 5 10 | -------------------------------------------------------------------------------- /.rubocop/style.yml: -------------------------------------------------------------------------------- 1 | Style/Documentation: 2 | Enabled: false 3 | 4 | Style/DocumentDynamicEvalDefinition: 5 | Enabled: true 6 | Exclude: 7 | - 'spec/**/*.rb' 8 | 9 | Style/FormatStringToken: 10 | Enabled: true 11 | EnforcedStyle: unannotated 12 | 13 | Style/HashSyntax: 14 | Enabled: true 15 | EnforcedStyle: ruby19_no_mixed_keys 16 | 17 | Style/OptionHash: 18 | Enabled: true 19 | 20 | # Using explicit `./` makes code more consistent 21 | Style/RedundantCurrentDirectoryInPath: 22 | Enabled: false 23 | 24 | Style/RescueStandardError: 25 | Enabled: true 26 | EnforcedStyle: implicit 27 | 28 | Style/StringLiterals: 29 | Enabled: true 30 | EnforcedStyle: double_quotes 31 | 32 | Style/WordArray: 33 | Enabled: true 34 | 35 | Style/YodaCondition: 36 | Enabled: false 37 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --markup-provider=kramdown 2 | --markup=markdown 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | All notable changes to this project will be documented in this file. 4 | 5 | The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), 6 | and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). 7 | 8 | ## [Unreleased] 9 | 10 | ### Changed 11 | 12 | - Use native llhttp gem for MRI Ruby and llhttp-ffi for other interpreters for better performance 13 | 14 | ### Removed 15 | 16 | - **BREAKING** Drop Ruby 2.x support 17 | 18 | [unreleased]: https://github.com/httprb/http/compare/v5.2.0...main 19 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Help and Discussion 2 | 3 | If you need help or just want to talk about the http.rb, 4 | visit the http.rb Google Group: 5 | 6 | https://groups.google.com/forum/#!forum/httprb 7 | 8 | You can join by email by sending a message to: 9 | 10 | [httprb+subscribe@googlegroups.com](mailto:httprb+subscribe@googlegroups.com) 11 | 12 | 13 | # Reporting bugs 14 | 15 | The best way to report a bug is by providing a reproduction script. A half 16 | working script with comments for the parts you were unable to automate is still 17 | appreciated. 18 | 19 | In any case, specify following info in description of your issue: 20 | 21 | - What you're trying to accomplish 22 | - What you expected to happen 23 | - What actually happened 24 | - The exception backtrace(s), if any 25 | - Version of gem or commit ref you are using 26 | - Version of ruby you are using 27 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | ruby RUBY_VERSION 5 | 6 | gem "rake" 7 | 8 | # Ruby 3.0 does not ship it anymore. 9 | # TODO: We should probably refactor specs to avoid need for it. 10 | gem "webrick" 11 | 12 | group :development do 13 | gem "guard-rspec", require: false 14 | gem "nokogiri", require: false 15 | gem "pry", require: false 16 | 17 | # RSpec formatter 18 | gem "fuubar", require: false 19 | 20 | platform :mri do 21 | gem "pry-byebug" 22 | end 23 | end 24 | 25 | group :test do 26 | gem "certificate_authority", "~> 1.0", require: false 27 | 28 | gem "backports" 29 | 30 | gem "rubocop", "~> 1.61.0" 31 | gem "rubocop-performance", "~> 1.19.1" 32 | gem "rubocop-rake", "~> 0.6.0" 33 | gem "rubocop-rspec", "~> 2.24.1" 34 | 35 | gem "simplecov", require: false 36 | gem "simplecov-lcov", require: false 37 | 38 | gem "rspec", "~> 3.10" 39 | gem "rspec-its" 40 | gem "rspec-memory" 41 | 42 | gem "yardstick" 43 | end 44 | 45 | group :doc do 46 | gem "kramdown" 47 | gem "yard" 48 | end 49 | 50 | # Specify your gem's dependencies in http.gemspec 51 | gemspec 52 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # More info at https://github.com/guard/guard#readme 4 | 5 | guard :rspec, cmd: "GUARD_RSPEC=1 bundle exec rspec --no-profile" do 6 | require "guard/rspec/dsl" 7 | dsl = Guard::RSpec::Dsl.new(self) 8 | 9 | # RSpec files 10 | rspec = dsl.rspec 11 | watch(rspec.spec_helper) { rspec.spec_dir } 12 | watch(rspec.spec_support) { rspec.spec_dir } 13 | watch(rspec.spec_files) 14 | 15 | # Ruby files 16 | ruby = dsl.ruby 17 | dsl.watch_spec_files_for(ruby.lib_files) 18 | end 19 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011-2022 Tony Arcieri, Erik Michaels-Ober, Alexey V. Zapparov, Zachary Anker 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ![http.rb](https://raw.github.com/httprb/http.rb/main/logo.png) 2 | 3 | [![Gem Version][gem-image]][gem-link] 4 | [![MIT licensed][license-image]][license-link] 5 | [![Build Status][build-image]][build-link] 6 | 7 | [Documentation] 8 | 9 | ## About 10 | 11 | HTTP (The Gem! a.k.a. http.rb) is an easy-to-use client library for making requests 12 | from Ruby. It uses a simple method chaining system for building requests, similar to 13 | Python's [Requests]. 14 | 15 | Under the hood, http.rb uses the [llhttp] parser, a fast HTTP parsing native extension. 16 | This library isn't just yet another wrapper around `Net::HTTP`. It implements the HTTP 17 | protocol natively and outsources the parsing to native extensions. 18 | 19 | ### Why http.rb? 20 | 21 | - **Clean API**: http.rb offers an easy-to-use API that should be a 22 | breath of fresh air after using something like Net::HTTP. 23 | 24 | - **Maturity**: http.rb is one of the most mature Ruby HTTP clients, supporting 25 | features like persistent connections and fine-grained timeouts. 26 | 27 | - **Performance**: using native parsers and a clean, lightweight implementation, 28 | http.rb achieves high performance while implementing HTTP in Ruby instead of C. 29 | 30 | 31 | ## Installation 32 | 33 | Add this line to your application's Gemfile: 34 | ```ruby 35 | gem "http" 36 | ``` 37 | 38 | And then execute: 39 | ```bash 40 | $ bundle 41 | ``` 42 | 43 | Or install it yourself as: 44 | ```bash 45 | $ gem install http 46 | ``` 47 | 48 | Inside of your Ruby program do: 49 | ```ruby 50 | require "http" 51 | ``` 52 | 53 | ...to pull it in as a dependency. 54 | 55 | 56 | ## Documentation 57 | 58 | [Please see the http.rb wiki][documentation] 59 | for more detailed documentation and usage notes. 60 | 61 | The following API documentation is also available: 62 | 63 | - [YARD API documentation](https://www.rubydoc.info/github/httprb/http) 64 | - [Chainable module (all chainable methods)](https://www.rubydoc.info/github/httprb/http/HTTP/Chainable) 65 | 66 | 67 | ### Basic Usage 68 | 69 | Here's some simple examples to get you started: 70 | 71 | ```ruby 72 | >> HTTP.get("https://github.com").to_s 73 | => "\n\n\n\n\n > HTTP.get("https://github.com") 81 | => #"GitHub.com", "Date"=>"Tue, 10 May...> 82 | ``` 83 | 84 | We can also obtain an `HTTP::Response::Body` object for this response: 85 | 86 | ```ruby 87 | >> HTTP.get("https://github.com").body 88 | => # 89 | ``` 90 | 91 | The response body can be streamed with `HTTP::Response::Body#readpartial`. 92 | In practice, you'll want to bind the `HTTP::Response::Body` to a local variable 93 | and call `#readpartial` on it repeatedly until it returns `nil`: 94 | 95 | ```ruby 96 | >> body = HTTP.get("https://github.com").body 97 | => # 98 | >> body.readpartial 99 | => "\n\n\n\n\n > body.readpartial 101 | => "\" href=\"/apple-touch-icon-72x72.png\">\n > body.readpartial 104 | => nil 105 | ``` 106 | 107 | ## Supported Ruby Versions 108 | 109 | This library aims to support and is [tested against][build-link] 110 | the following Ruby versions: 111 | 112 | - Ruby 3.0 113 | - Ruby 3.1 114 | - Ruby 3.2 115 | - Ruby 3.3 116 | - JRuby 9.4 117 | 118 | If something doesn't work on one of these versions, it's a bug. 119 | 120 | This library may inadvertently work (or seem to work) on other Ruby versions, 121 | however support will only be provided for the versions listed above. 122 | 123 | If you would like this library to support another Ruby version or 124 | implementation, you may volunteer to be a maintainer. Being a maintainer 125 | entails making sure all tests run and pass on that implementation. When 126 | something breaks on your implementation, you will be responsible for providing 127 | patches in a timely fashion. If critical issues for a particular implementation 128 | exist at the time of a major release, support for that Ruby version may be 129 | dropped. 130 | 131 | 132 | ## Contributing to http.rb 133 | 134 | - Fork http.rb on GitHub 135 | - Make your changes 136 | - Ensure all tests pass (`bundle exec rake`) 137 | - Send a pull request 138 | - If we like them we'll merge them 139 | - If we've accepted a patch, feel free to ask for commit access! 140 | 141 | 142 | ## Copyright 143 | 144 | Copyright © 2011-2022 Tony Arcieri, Alexey V. Zapparov, Erik Michaels-Ober, Zachary Anker. 145 | See LICENSE.txt for further details. 146 | 147 | 148 | [//]: # (badges) 149 | 150 | [gem-image]: https://img.shields.io/gem/v/http?logo=ruby 151 | [gem-link]: https://rubygems.org/gems/http 152 | [license-image]: https://img.shields.io/badge/license-MIT-blue.svg 153 | [license-link]: https://github.com/httprb/http/blob/main/LICENSE.txt 154 | [build-image]: https://github.com/httprb/http/workflows/CI/badge.svg 155 | [build-link]: https://github.com/httprb/http/actions/workflows/ci.yml 156 | 157 | [//]: # (links) 158 | 159 | [documentation]: https://github.com/httprb/http/wiki 160 | [requests]: https://docs.python-requests.org/en/latest/ 161 | [llhttp]: https://llhttp.org/ 162 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | 5 | require "rspec/core/rake_task" 6 | RSpec::Core::RakeTask.new 7 | 8 | require "rubocop/rake_task" 9 | RuboCop::RakeTask.new 10 | 11 | require "yardstick/rake/measurement" 12 | Yardstick::Rake::Measurement.new do |measurement| 13 | measurement.output = "measurement/report.txt" 14 | end 15 | 16 | require "yardstick/rake/verify" 17 | Yardstick::Rake::Verify.new do |verify| 18 | verify.require_exact_threshold = false 19 | verify.threshold = 55 20 | end 21 | 22 | task :generate_status_codes do 23 | require "http" 24 | require "nokogiri" 25 | 26 | url = "http://www.iana.org/assignments/http-status-codes/http-status-codes.xml" 27 | xml = Nokogiri::XML HTTP.get url 28 | arr = xml.xpath("//xmlns:record").reduce([]) do |a, e| 29 | code = e.xpath("xmlns:value").text.to_s 30 | desc = e.xpath("xmlns:description").text.to_s 31 | 32 | next a if %w[Unassigned (Unused)].include?(desc) 33 | 34 | a << "#{code} => #{desc.inspect}" 35 | end 36 | 37 | File.open("./lib/http/response/status/reasons.rb", "w") do |io| 38 | io.puts <<~TPL 39 | # AUTO-GENERATED FILE, DO NOT CHANGE IT MANUALLY 40 | 41 | require "delegate" 42 | 43 | module HTTP 44 | class Response 45 | class Status < ::Delegator 46 | # Code to Reason map 47 | # 48 | # @example Usage 49 | # 50 | # REASONS[400] # => "Bad Request" 51 | # REASONS[414] # => "Request-URI Too Long" 52 | # 53 | # @return [Hash String>] 54 | REASONS = { 55 | #{arr.join ",\n "} 56 | }.each { |_, v| v.freeze }.freeze 57 | end 58 | end 59 | end 60 | TPL 61 | end 62 | end 63 | 64 | task default: %i[spec rubocop verify_measurements] 65 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | Security updates are applied only to the most recent release. 6 | 7 | ## Reporting a Vulnerability 8 | 9 | If you have discovered a security vulnerability in this project, please report 10 | it privately. **Do not disclose it as a public issue.** This gives us time to 11 | work with you to fix the issue before public exposure, reducing the chance that 12 | the exploit will be used before a patch is released. 13 | 14 | Please disclose it at [security advisory](https://github.com/httprb/http/security/advisories/new). 15 | 16 | This project is maintained by a team of volunteers on a reasonable-effort basis. 17 | As such, please give us at least 90 days to work on a fix before public exposure. 18 | -------------------------------------------------------------------------------- /http.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path("lib", __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require "http/version" 6 | 7 | Gem::Specification.new do |gem| 8 | gem.authors = ["Tony Arcieri", "Erik Michaels-Ober", "Alexey V. Zapparov", "Zachary Anker"] 9 | gem.email = ["bascule@gmail.com"] 10 | 11 | gem.description = <<-DESCRIPTION.strip.gsub(/\s+/, " ") 12 | An easy-to-use client library for making requests from Ruby. 13 | It uses a simple method chaining system for building requests, 14 | similar to Python's Requests. 15 | DESCRIPTION 16 | 17 | gem.summary = "HTTP should be easy" 18 | gem.homepage = "https://github.com/httprb/http" 19 | gem.licenses = ["MIT"] 20 | 21 | gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } 22 | gem.files = `git ls-files`.split("\n") 23 | gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 24 | gem.name = "http" 25 | gem.require_paths = ["lib"] 26 | gem.version = HTTP::VERSION 27 | 28 | gem.required_ruby_version = ">= 3.0" 29 | 30 | gem.add_runtime_dependency "addressable", "~> 2.8" 31 | gem.add_runtime_dependency "http-cookie", "~> 1.0" 32 | gem.add_runtime_dependency "http-form_data", "~> 2.2" 33 | 34 | # Use native llhttp for MRI (more performant) and llhttp-ffi for other interpreters (better compatibility) 35 | if RUBY_ENGINE == "ruby" 36 | gem.add_runtime_dependency "llhttp", "~> 0.5.0" 37 | else 38 | gem.add_runtime_dependency "llhttp-ffi", "~> 0.5.0" 39 | end 40 | 41 | gem.metadata = { 42 | "source_code_uri" => "https://github.com/httprb/http", 43 | "wiki_uri" => "https://github.com/httprb/http/wiki", 44 | "bug_tracker_uri" => "https://github.com/httprb/http/issues", 45 | "changelog_uri" => "https://github.com/httprb/http/blob/v#{HTTP::VERSION}/CHANGELOG.md", 46 | "rubygems_mfa_required" => "true" 47 | } 48 | end 49 | -------------------------------------------------------------------------------- /lib/http.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/errors" 4 | require "http/timeout/null" 5 | require "http/timeout/per_operation" 6 | require "http/timeout/global" 7 | require "http/chainable" 8 | require "http/client" 9 | require "http/retriable/client" 10 | require "http/connection" 11 | require "http/options" 12 | require "http/feature" 13 | require "http/request" 14 | require "http/request/writer" 15 | require "http/response" 16 | require "http/response/body" 17 | require "http/response/parser" 18 | 19 | # HTTP should be easy 20 | module HTTP 21 | extend Chainable 22 | 23 | class << self 24 | # HTTP[:accept => 'text/html'].get(...) 25 | alias [] headers 26 | end 27 | end 28 | -------------------------------------------------------------------------------- /lib/http/base64.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Base64 5 | module_function 6 | 7 | # Equivalent to Base64.strict_encode64 8 | def encode64(input) 9 | [input].pack("m0") 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/http/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/form_data" 6 | require "http/options" 7 | require "http/feature" 8 | require "http/headers" 9 | require "http/connection" 10 | require "http/redirector" 11 | require "http/uri" 12 | 13 | module HTTP 14 | # Clients make requests and receive responses 15 | class Client 16 | extend Forwardable 17 | include Chainable 18 | 19 | HTTP_OR_HTTPS_RE = %r{^https?://}i 20 | 21 | def initialize(default_options = {}) 22 | @default_options = HTTP::Options.new(default_options) 23 | @connection = nil 24 | @state = :clean 25 | end 26 | 27 | # Make an HTTP request 28 | def request(verb, uri, opts = {}) 29 | opts = @default_options.merge(opts) 30 | req = build_request(verb, uri, opts) 31 | res = perform(req, opts) 32 | return res unless opts.follow 33 | 34 | Redirector.new(opts.follow).perform(req, res) do |request| 35 | perform(wrap_request(request, opts), opts) 36 | end 37 | end 38 | 39 | # Prepare an HTTP request 40 | def build_request(verb, uri, opts = {}) 41 | opts = @default_options.merge(opts) 42 | uri = make_request_uri(uri, opts) 43 | headers = make_request_headers(opts) 44 | body = make_request_body(opts, headers) 45 | 46 | req = HTTP::Request.new({ 47 | verb: verb, 48 | uri: uri, 49 | uri_normalizer: opts.feature(:normalize_uri)&.normalizer, 50 | proxy: opts.proxy, 51 | headers: headers, 52 | body: body 53 | }) 54 | 55 | wrap_request(req, opts) 56 | end 57 | 58 | # @!method persistent? 59 | # @see Options#persistent? 60 | # @return [Boolean] whenever client is persistent 61 | def_delegator :default_options, :persistent? 62 | 63 | # Perform a single (no follow) HTTP request 64 | def perform(req, options) 65 | verify_connection!(req.uri) 66 | 67 | @state = :dirty 68 | 69 | begin 70 | @connection ||= HTTP::Connection.new(req, options) 71 | 72 | unless @connection.failed_proxy_connect? 73 | @connection.send_request(req) 74 | @connection.read_headers! 75 | end 76 | rescue Error => e 77 | options.features.each_value do |feature| 78 | feature.on_error(req, e) 79 | end 80 | raise 81 | end 82 | res = build_response(req, options) 83 | 84 | res = options.features.values.reverse.inject(res) do |response, feature| 85 | feature.wrap_response(response) 86 | end 87 | 88 | @connection.finish_response if req.verb == :head 89 | @state = :clean 90 | 91 | res 92 | rescue 93 | close 94 | raise 95 | end 96 | 97 | def close 98 | @connection&.close 99 | @connection = nil 100 | @state = :clean 101 | end 102 | 103 | private 104 | 105 | def wrap_request(req, opts) 106 | opts.features.inject(req) do |request, (_name, feature)| 107 | feature.wrap_request(request) 108 | end 109 | end 110 | 111 | def build_response(req, options) 112 | Response.new( 113 | status: @connection.status_code, 114 | version: @connection.http_version, 115 | headers: @connection.headers, 116 | proxy_headers: @connection.proxy_response_headers, 117 | connection: @connection, 118 | encoding: options.encoding, 119 | request: req 120 | ) 121 | end 122 | 123 | # Verify our request isn't going to be made against another URI 124 | def verify_connection!(uri) 125 | if default_options.persistent? && uri.origin != default_options.persistent 126 | raise StateError, "Persistence is enabled for #{default_options.persistent}, but we got #{uri.origin}" 127 | end 128 | 129 | # We re-create the connection object because we want to let prior requests 130 | # lazily load the body as long as possible, and this mimics prior functionality. 131 | return close if @connection && (!@connection.keep_alive? || @connection.expired?) 132 | 133 | # If we get into a bad state (eg, Timeout.timeout ensure being killed) 134 | # close the connection to prevent potential for mixed responses. 135 | close if @state == :dirty 136 | end 137 | 138 | # Merges query params if needed 139 | # 140 | # @param [#to_s] uri 141 | # @return [URI] 142 | def make_request_uri(uri, opts) 143 | uri = uri.to_s 144 | 145 | uri = "#{default_options.persistent}#{uri}" if default_options.persistent? && uri !~ HTTP_OR_HTTPS_RE 146 | 147 | uri = HTTP::URI.parse uri 148 | 149 | uri.query_values = uri.query_values(Array).to_a.concat(opts.params.to_a) if opts.params && !opts.params.empty? 150 | 151 | # Some proxies (seen on WEBRick) fail if URL has 152 | # empty path (e.g. `http://example.com`) while it's RFC-complaint: 153 | # http://tools.ietf.org/html/rfc1738#section-3.1 154 | uri.path = "/" if uri.path.empty? 155 | 156 | uri 157 | end 158 | 159 | # Creates request headers with cookies (if any) merged in 160 | def make_request_headers(opts) 161 | headers = opts.headers 162 | 163 | # Tell the server to keep the conn open 164 | headers[Headers::CONNECTION] = default_options.persistent? ? Connection::KEEP_ALIVE : Connection::CLOSE 165 | 166 | cookies = opts.cookies.values 167 | 168 | unless cookies.empty? 169 | cookies = opts.headers.get(Headers::COOKIE).concat(cookies).join("; ") 170 | headers[Headers::COOKIE] = cookies 171 | end 172 | 173 | headers 174 | end 175 | 176 | # Create the request body object to send 177 | def make_request_body(opts, headers) 178 | case 179 | when opts.body 180 | opts.body 181 | when opts.form 182 | form = make_form_data(opts.form) 183 | headers[Headers::CONTENT_TYPE] ||= form.content_type 184 | form 185 | when opts.json 186 | body = MimeType[:json].encode opts.json 187 | headers[Headers::CONTENT_TYPE] ||= "application/json; charset=#{body.encoding.name.downcase}" 188 | body 189 | end 190 | end 191 | 192 | def make_form_data(form) 193 | return form if form.is_a? HTTP::FormData::Multipart 194 | return form if form.is_a? HTTP::FormData::Urlencoded 195 | 196 | HTTP::FormData.create(form) 197 | end 198 | end 199 | end 200 | -------------------------------------------------------------------------------- /lib/http/connection.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/headers" 6 | 7 | module HTTP 8 | # A connection to the HTTP server 9 | class Connection 10 | extend Forwardable 11 | 12 | # Allowed values for CONNECTION header 13 | KEEP_ALIVE = "Keep-Alive" 14 | CLOSE = "close" 15 | 16 | # Attempt to read this much data 17 | BUFFER_SIZE = 16_384 18 | 19 | # HTTP/1.0 20 | HTTP_1_0 = "1.0" 21 | 22 | # HTTP/1.1 23 | HTTP_1_1 = "1.1" 24 | 25 | # Returned after HTTP CONNECT (via proxy) 26 | attr_reader :proxy_response_headers 27 | 28 | # @param [HTTP::Request] req 29 | # @param [HTTP::Options] options 30 | # @raise [HTTP::ConnectionError] when failed to connect 31 | def initialize(req, options) 32 | @persistent = options.persistent? 33 | @keep_alive_timeout = options.keep_alive_timeout.to_f 34 | @pending_request = false 35 | @pending_response = false 36 | @failed_proxy_connect = false 37 | @buffer = "".b 38 | 39 | @parser = Response::Parser.new 40 | 41 | @socket = options.timeout_class.new(options.timeout_options) 42 | @socket.connect(options.socket_class, req.socket_host, req.socket_port, options.nodelay) 43 | 44 | send_proxy_connect_request(req) 45 | start_tls(req, options) 46 | reset_timer 47 | rescue IOError, SocketError, SystemCallError => e 48 | raise ConnectionError, "failed to connect: #{e}", e.backtrace 49 | rescue TimeoutError 50 | close 51 | raise 52 | end 53 | 54 | # @see (HTTP::Response::Parser#status_code) 55 | def_delegator :@parser, :status_code 56 | 57 | # @see (HTTP::Response::Parser#http_version) 58 | def_delegator :@parser, :http_version 59 | 60 | # @see (HTTP::Response::Parser#headers) 61 | def_delegator :@parser, :headers 62 | 63 | # @return [Boolean] whenever proxy connect failed 64 | def failed_proxy_connect? 65 | @failed_proxy_connect 66 | end 67 | 68 | # Send a request to the server 69 | # 70 | # @param [Request] req Request to send to the server 71 | # @return [nil] 72 | def send_request(req) 73 | if @pending_response 74 | raise StateError, "Tried to send a request while one is pending already. Make sure you read off the body." 75 | end 76 | 77 | if @pending_request 78 | raise StateError, "Tried to send a request while a response is pending. Make sure you read off the body." 79 | end 80 | 81 | @pending_request = true 82 | 83 | req.stream @socket 84 | 85 | @pending_response = true 86 | @pending_request = false 87 | end 88 | 89 | # Read a chunk of the body 90 | # 91 | # @return [String] data chunk 92 | # @return [nil] when no more data left 93 | def readpartial(size = BUFFER_SIZE) 94 | return unless @pending_response 95 | 96 | chunk = @parser.read(size) 97 | return chunk if chunk 98 | 99 | finished = (read_more(size) == :eof) || @parser.finished? 100 | chunk = @parser.read(size) 101 | finish_response if finished 102 | 103 | chunk || "".b 104 | end 105 | 106 | # Reads data from socket up until headers are loaded 107 | # @return [void] 108 | # @raise [ResponseHeaderError] when unable to read response headers 109 | def read_headers! 110 | until @parser.headers? 111 | result = read_more(BUFFER_SIZE) 112 | raise ResponseHeaderError, "couldn't read response headers" if result == :eof 113 | end 114 | 115 | set_keep_alive 116 | end 117 | 118 | # Callback for when we've reached the end of a response 119 | # @return [void] 120 | def finish_response 121 | close unless keep_alive? 122 | 123 | @parser.reset 124 | @socket.reset_counter if @socket.respond_to?(:reset_counter) 125 | reset_timer 126 | 127 | @pending_response = false 128 | end 129 | 130 | # Close the connection 131 | # @return [void] 132 | def close 133 | @socket.close unless @socket&.closed? 134 | 135 | @pending_response = false 136 | @pending_request = false 137 | end 138 | 139 | def finished_request? 140 | !@pending_request && !@pending_response 141 | end 142 | 143 | # Whether we're keeping the conn alive 144 | # @return [Boolean] 145 | def keep_alive? 146 | !!@keep_alive && !@socket.closed? 147 | end 148 | 149 | # Whether our connection has expired 150 | # @return [Boolean] 151 | def expired? 152 | !@conn_expires_at || @conn_expires_at < Time.now 153 | end 154 | 155 | private 156 | 157 | # Sets up SSL context and starts TLS if needed. 158 | # @param (see #initialize) 159 | # @return [void] 160 | def start_tls(req, options) 161 | return unless req.uri.https? && !failed_proxy_connect? 162 | 163 | ssl_context = options.ssl_context 164 | 165 | unless ssl_context 166 | ssl_context = OpenSSL::SSL::SSLContext.new 167 | ssl_context.set_params(options.ssl || {}) 168 | end 169 | 170 | @socket.start_tls(req.uri.host, options.ssl_socket_class, ssl_context) 171 | end 172 | 173 | # Open tunnel through proxy 174 | def send_proxy_connect_request(req) 175 | return unless req.uri.https? && req.using_proxy? 176 | 177 | @pending_request = true 178 | 179 | req.connect_using_proxy @socket 180 | 181 | @pending_request = false 182 | @pending_response = true 183 | 184 | read_headers! 185 | @proxy_response_headers = @parser.headers 186 | 187 | if @parser.status_code != 200 188 | @failed_proxy_connect = true 189 | return 190 | end 191 | 192 | @parser.reset 193 | @pending_response = false 194 | end 195 | 196 | # Resets expiration of persistent connection. 197 | # @return [void] 198 | def reset_timer 199 | @conn_expires_at = Time.now + @keep_alive_timeout if @persistent 200 | end 201 | 202 | # Store whether the connection should be kept alive. 203 | # Once we reset the parser, we lose all of this state. 204 | # @return [void] 205 | def set_keep_alive 206 | return @keep_alive = false unless @persistent 207 | 208 | @keep_alive = 209 | case @parser.http_version 210 | when HTTP_1_0 # HTTP/1.0 requires opt in for Keep Alive 211 | @parser.headers[Headers::CONNECTION] == KEEP_ALIVE 212 | when HTTP_1_1 # HTTP/1.1 is opt-out 213 | @parser.headers[Headers::CONNECTION] != CLOSE 214 | else # Anything else we assume doesn't supportit 215 | false 216 | end 217 | end 218 | 219 | # Feeds some more data into parser 220 | # @return [void] 221 | # @raise [SocketReadError] when unable to read from socket 222 | def read_more(size) 223 | return if @parser.finished? 224 | 225 | value = @socket.readpartial(size, @buffer) 226 | if value == :eof 227 | @parser << "" 228 | :eof 229 | elsif value 230 | @parser << value 231 | end 232 | rescue IOError, SocketError, SystemCallError => e 233 | raise SocketReadError, "error reading from socket: #{e}", e.backtrace 234 | end 235 | end 236 | end 237 | -------------------------------------------------------------------------------- /lib/http/content_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class ContentType 5 | MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)} 6 | CHARSET_RE = /;\s*charset=([^;]+)/i 7 | 8 | attr_accessor :mime_type, :charset 9 | 10 | class << self 11 | # Parse string and return ContentType struct 12 | def parse(str) 13 | new mime_type(str), charset(str) 14 | end 15 | 16 | private 17 | 18 | # :nodoc: 19 | def mime_type(str) 20 | str.to_s[MIME_TYPE_RE, 1]&.strip&.downcase 21 | end 22 | 23 | # :nodoc: 24 | def charset(str) 25 | str.to_s[CHARSET_RE, 1]&.strip&.delete('"') 26 | end 27 | end 28 | 29 | def initialize(mime_type = nil, charset = nil) 30 | @mime_type = mime_type 31 | @charset = charset 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/http/errors.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | # Generic error 5 | class Error < StandardError; end 6 | 7 | # Generic Connection error 8 | class ConnectionError < Error; end 9 | 10 | # Types of Connection errors 11 | class ResponseHeaderError < ConnectionError; end 12 | class SocketReadError < ConnectionError; end 13 | class SocketWriteError < ConnectionError; end 14 | 15 | # Generic Request error 16 | class RequestError < Error; end 17 | 18 | # Generic Response error 19 | class ResponseError < Error; end 20 | 21 | # Requested to do something when we're in the wrong state 22 | class StateError < ResponseError; end 23 | 24 | # When status code indicates an error 25 | class StatusError < ResponseError 26 | attr_reader :response 27 | 28 | def initialize(response) 29 | @response = response 30 | 31 | super("Unexpected status code #{response.code}") 32 | end 33 | end 34 | 35 | # Generic Timeout error 36 | class TimeoutError < Error; end 37 | 38 | # Timeout when first establishing the connection 39 | class ConnectTimeoutError < TimeoutError; end 40 | 41 | # Header value is of unexpected format (similar to Net::HTTPHeaderSyntaxError) 42 | class HeaderError < Error; end 43 | end 44 | -------------------------------------------------------------------------------- /lib/http/feature.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Feature 5 | def wrap_request(request) 6 | request 7 | end 8 | 9 | def wrap_response(response) 10 | response 11 | end 12 | 13 | def on_error(_request, _error); end 14 | end 15 | end 16 | 17 | require "http/features/auto_inflate" 18 | require "http/features/auto_deflate" 19 | require "http/features/instrumentation" 20 | require "http/features/logging" 21 | require "http/features/normalize_uri" 22 | require "http/features/raise_error" 23 | -------------------------------------------------------------------------------- /lib/http/features/auto_deflate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "set" 4 | require "tempfile" 5 | require "zlib" 6 | 7 | require "http/request/body" 8 | 9 | module HTTP 10 | module Features 11 | class AutoDeflate < Feature 12 | VALID_METHODS = Set.new(%w[gzip deflate]).freeze 13 | 14 | attr_reader :method 15 | 16 | def initialize(method: "gzip") 17 | super() 18 | 19 | @method = method.to_s 20 | 21 | raise Error, "Only gzip and deflate methods are supported" unless VALID_METHODS.include?(@method) 22 | end 23 | 24 | def wrap_request(request) 25 | return request unless method 26 | return request if request.body.size.zero? 27 | 28 | # We need to delete Content-Length header. It will be set automatically by HTTP::Request::Writer 29 | request.headers.delete(Headers::CONTENT_LENGTH) 30 | request.headers[Headers::CONTENT_ENCODING] = method 31 | 32 | Request.new( 33 | version: request.version, 34 | verb: request.verb, 35 | uri: request.uri, 36 | headers: request.headers, 37 | proxy: request.proxy, 38 | body: deflated_body(request.body), 39 | uri_normalizer: request.uri_normalizer 40 | ) 41 | end 42 | 43 | def deflated_body(body) 44 | case method 45 | when "gzip" 46 | GzippedBody.new(body) 47 | when "deflate" 48 | DeflatedBody.new(body) 49 | end 50 | end 51 | 52 | HTTP::Options.register_feature(:auto_deflate, self) 53 | 54 | class CompressedBody < HTTP::Request::Body 55 | def initialize(uncompressed_body) 56 | @body = uncompressed_body 57 | @compressed = nil 58 | end 59 | 60 | def size 61 | compress_all! unless @compressed 62 | @compressed.size 63 | end 64 | 65 | def each(&block) 66 | return to_enum __method__ unless block 67 | 68 | if @compressed 69 | compressed_each(&block) 70 | else 71 | compress(&block) 72 | end 73 | 74 | self 75 | end 76 | 77 | private 78 | 79 | def compressed_each 80 | while (data = @compressed.read(Connection::BUFFER_SIZE)) 81 | yield data 82 | end 83 | ensure 84 | @compressed.close! 85 | end 86 | 87 | def compress_all! 88 | @compressed = Tempfile.new("http-compressed_body", binmode: true) 89 | compress { |data| @compressed.write(data) } 90 | @compressed.rewind 91 | end 92 | end 93 | 94 | class GzippedBody < CompressedBody 95 | def compress(&block) 96 | gzip = Zlib::GzipWriter.new(BlockIO.new(block)) 97 | @body.each { |chunk| gzip.write(chunk) } 98 | ensure 99 | gzip.finish 100 | end 101 | 102 | class BlockIO 103 | def initialize(block) 104 | @block = block 105 | end 106 | 107 | def write(data) 108 | @block.call(data) 109 | end 110 | end 111 | end 112 | 113 | class DeflatedBody < CompressedBody 114 | def compress 115 | deflater = Zlib::Deflate.new 116 | 117 | @body.each { |chunk| yield deflater.deflate(chunk) } 118 | 119 | yield deflater.finish 120 | ensure 121 | deflater.close 122 | end 123 | end 124 | end 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /lib/http/features/auto_inflate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "set" 4 | 5 | module HTTP 6 | module Features 7 | class AutoInflate < Feature 8 | SUPPORTED_ENCODING = Set.new(%w[deflate gzip x-gzip]).freeze 9 | private_constant :SUPPORTED_ENCODING 10 | 11 | def wrap_response(response) 12 | return response unless supported_encoding?(response) 13 | 14 | options = { 15 | status: response.status, 16 | version: response.version, 17 | headers: response.headers, 18 | proxy_headers: response.proxy_headers, 19 | connection: response.connection, 20 | body: stream_for(response.connection), 21 | request: response.request 22 | } 23 | 24 | Response.new(options) 25 | end 26 | 27 | def stream_for(connection) 28 | Response::Body.new(Response::Inflater.new(connection)) 29 | end 30 | 31 | private 32 | 33 | def supported_encoding?(response) 34 | content_encoding = response.headers.get(Headers::CONTENT_ENCODING).first 35 | content_encoding && SUPPORTED_ENCODING.include?(content_encoding) 36 | end 37 | 38 | HTTP::Options.register_feature(:auto_inflate, self) 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/http/features/instrumentation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Features 5 | # Instrument requests and responses. Expects an 6 | # ActiveSupport::Notifications-compatible instrumenter. Defaults to use a 7 | # namespace of 'http' which may be overridden with a `:namespace` param. 8 | # Emits a single event like `"request.{namespace}"`, eg `"request.http"`. 9 | # Be sure to specify the instrumenter when enabling the feature: 10 | # 11 | # HTTP 12 | # .use(instrumentation: {instrumenter: ActiveSupport::Notifications.instrumenter}) 13 | # .get("https://example.com/") 14 | # 15 | # Emits two events on every request: 16 | # 17 | # * `start_request.http` before the request is made, so you can log the reqest being started 18 | # * `request.http` after the response is recieved, and contains `start` 19 | # and `finish` so the duration of the request can be calculated. 20 | # 21 | class Instrumentation < Feature 22 | attr_reader :instrumenter, :name, :error_name 23 | 24 | def initialize(instrumenter: NullInstrumenter.new, namespace: "http") 25 | super() 26 | @instrumenter = instrumenter 27 | @name = "request.#{namespace}" 28 | @error_name = "error.#{namespace}" 29 | end 30 | 31 | def wrap_request(request) 32 | # Emit a separate "start" event, so a logger can print the request 33 | # being run without waiting for a response 34 | instrumenter.instrument("start_#{name}", request: request) 35 | instrumenter.start(name, request: request) 36 | request 37 | end 38 | 39 | def wrap_response(response) 40 | instrumenter.finish(name, response: response) 41 | response 42 | end 43 | 44 | def on_error(request, error) 45 | instrumenter.instrument(error_name, request: request, error: error) 46 | end 47 | 48 | HTTP::Options.register_feature(:instrumentation, self) 49 | 50 | class NullInstrumenter 51 | def instrument(name, payload = {}) 52 | start(name, payload) 53 | begin 54 | yield payload if block_given? 55 | ensure 56 | finish name, payload 57 | end 58 | end 59 | 60 | def start(_name, _payload) 61 | true 62 | end 63 | 64 | def finish(_name, _payload) 65 | true 66 | end 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/http/features/logging.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Features 5 | # Log requests and responses. Request verb and uri, and Response status are 6 | # logged at `info`, and the headers and bodies of both are logged at 7 | # `debug`. Be sure to specify the logger when enabling the feature: 8 | # 9 | # HTTP.use(logging: {logger: Logger.new(STDOUT)}).get("https://example.com/") 10 | # 11 | class Logging < Feature 12 | HTTP::Options.register_feature(:logging, self) 13 | 14 | class NullLogger 15 | %w[fatal error warn info debug].each do |level| 16 | define_method(level.to_sym) do |*_args| 17 | nil 18 | end 19 | 20 | define_method(:"#{level}?") do 21 | true 22 | end 23 | end 24 | end 25 | 26 | attr_reader :logger 27 | 28 | def initialize(logger: NullLogger.new) 29 | super() 30 | @logger = logger 31 | end 32 | 33 | def wrap_request(request) 34 | logger.info { "> #{request.verb.to_s.upcase} #{request.uri}" } 35 | logger.debug { "#{stringify_headers(request.headers)}\n\n#{request.body.source}" } 36 | 37 | request 38 | end 39 | 40 | def wrap_response(response) 41 | logger.info { "< #{response.status}" } 42 | logger.debug { "#{stringify_headers(response.headers)}\n\n#{response.body}" } 43 | 44 | response 45 | end 46 | 47 | private 48 | 49 | def stringify_headers(headers) 50 | headers.map { |name, value| "#{name}: #{value}" }.join("\n") 51 | end 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/http/features/normalize_uri.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/uri" 4 | 5 | module HTTP 6 | module Features 7 | class NormalizeUri < Feature 8 | attr_reader :normalizer 9 | 10 | def initialize(normalizer: HTTP::URI::NORMALIZER) 11 | super() 12 | @normalizer = normalizer 13 | end 14 | 15 | HTTP::Options.register_feature(:normalize_uri, self) 16 | end 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /lib/http/features/raise_error.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Features 5 | class RaiseError < Feature 6 | def initialize(ignore: []) 7 | super() 8 | 9 | @ignore = ignore 10 | end 11 | 12 | def wrap_response(response) 13 | return response if response.code < 400 14 | return response if @ignore.include?(response.code) 15 | 16 | raise HTTP::StatusError, response 17 | end 18 | 19 | HTTP::Options.register_feature(:raise_error, self) 20 | end 21 | end 22 | end 23 | -------------------------------------------------------------------------------- /lib/http/headers.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/errors" 6 | require "http/headers/mixin" 7 | require "http/headers/normalizer" 8 | require "http/headers/known" 9 | 10 | module HTTP 11 | # HTTP Headers container. 12 | class Headers 13 | extend Forwardable 14 | include Enumerable 15 | 16 | class << self 17 | # Coerces given `object` into Headers. 18 | # 19 | # @raise [Error] if object can't be coerced 20 | # @param [#to_hash, #to_h, #to_a] object 21 | # @return [Headers] 22 | def coerce(object) 23 | unless object.is_a? self 24 | object = case 25 | when object.respond_to?(:to_hash) then object.to_hash 26 | when object.respond_to?(:to_h) then object.to_h 27 | when object.respond_to?(:to_a) then object.to_a 28 | else raise Error, "Can't coerce #{object.inspect} to Headers" 29 | end 30 | end 31 | 32 | headers = new 33 | object.each { |k, v| headers.add k, v } 34 | headers 35 | end 36 | alias [] coerce 37 | 38 | def normalizer 39 | @normalizer ||= Headers::Normalizer.new 40 | end 41 | end 42 | 43 | # Class constructor. 44 | def initialize 45 | # The @pile stores each header value using a three element array: 46 | # 0 - the normalized header key, used for lookup 47 | # 1 - the header key as it will be sent with a request 48 | # 2 - the value 49 | @pile = [] 50 | end 51 | 52 | # Sets header. 53 | # 54 | # @param (see #add) 55 | # @return [void] 56 | def set(name, value) 57 | delete(name) 58 | add(name, value) 59 | end 60 | alias []= set 61 | 62 | # Removes header. 63 | # 64 | # @param [#to_s] name header name 65 | # @return [void] 66 | def delete(name) 67 | name = normalize_header name.to_s 68 | @pile.delete_if { |k, _| k == name } 69 | end 70 | 71 | # Appends header. 72 | # 73 | # @param [String, Symbol] name header name. When specified as a string, the 74 | # name is sent as-is. When specified as a symbol, the name is converted 75 | # to a string of capitalized words separated by a dash. Word boundaries 76 | # are determined by an underscore (`_`) or a dash (`-`). 77 | # Ex: `:content_type` is sent as `"Content-Type"`, and `"auth_key"` (string) 78 | # is sent as `"auth_key"`. 79 | # @param [Array<#to_s>, #to_s] value header value(s) to be appended 80 | # @return [void] 81 | def add(name, value) 82 | lookup_name = normalize_header(name.to_s) 83 | wire_name = case name 84 | when String 85 | name 86 | when Symbol 87 | lookup_name 88 | else 89 | raise HTTP::HeaderError, "HTTP header must be a String or Symbol: #{name.inspect}" 90 | end 91 | Array(value).each do |v| 92 | @pile << [ 93 | lookup_name, 94 | wire_name, 95 | validate_value(v) 96 | ] 97 | end 98 | end 99 | 100 | # Returns list of header values if any. 101 | # 102 | # @return [Array] 103 | def get(name) 104 | name = normalize_header name.to_s 105 | @pile.select { |k, _| k == name }.map { |_, _, v| v } 106 | end 107 | 108 | # Smart version of {#get}. 109 | # 110 | # @return [nil] if header was not set 111 | # @return [String] if header has exactly one value 112 | # @return [Array] if header has more than one value 113 | def [](name) 114 | values = get(name) 115 | 116 | case values.count 117 | when 0 then nil 118 | when 1 then values.first 119 | else values 120 | end 121 | end 122 | 123 | # Tells whenever header with given `name` is set or not. 124 | # 125 | # @return [Boolean] 126 | def include?(name) 127 | name = normalize_header name.to_s 128 | @pile.any? { |k, _| k == name } 129 | end 130 | 131 | # Returns Rack-compatible headers Hash 132 | # 133 | # @return [Hash] 134 | def to_h 135 | keys.to_h { |k| [k, self[k]] } 136 | end 137 | alias to_hash to_h 138 | 139 | # Returns headers key/value pairs. 140 | # 141 | # @return [Array<[String, String]>] 142 | def to_a 143 | @pile.map { |item| item[1..2] } 144 | end 145 | 146 | # Returns human-readable representation of `self` instance. 147 | # 148 | # @return [String] 149 | def inspect 150 | "#<#{self.class} #{to_h.inspect}>" 151 | end 152 | 153 | # Returns list of header names. 154 | # 155 | # @return [Array] 156 | def keys 157 | @pile.map { |_, k, _| k }.uniq 158 | end 159 | 160 | # Compares headers to another Headers or Array of key/value pairs 161 | # 162 | # @return [Boolean] 163 | def ==(other) 164 | return false unless other.respond_to? :to_a 165 | 166 | to_a == other.to_a 167 | end 168 | 169 | # Calls the given block once for each key/value pair in headers container. 170 | # 171 | # @return [Enumerator] if no block given 172 | # @return [Headers] self-reference 173 | def each 174 | return to_enum(__method__) unless block_given? 175 | 176 | @pile.each { |item| yield(item[1..2]) } 177 | self 178 | end 179 | 180 | # @!method empty? 181 | # Returns `true` if `self` has no key/value pairs 182 | # 183 | # @return [Boolean] 184 | def_delegator :@pile, :empty? 185 | 186 | # @!method hash 187 | # Compute a hash-code for this headers container. 188 | # Two containers with the same content will have the same hash code. 189 | # 190 | # @see http://www.ruby-doc.org/core/Object.html#method-i-hash 191 | # @return [Fixnum] 192 | def_delegator :@pile, :hash 193 | 194 | # Properly clones internal key/value storage. 195 | # 196 | # @api private 197 | def initialize_copy(orig) 198 | super 199 | @pile = @pile.map(&:dup) 200 | end 201 | 202 | # Merges `other` headers into `self`. 203 | # 204 | # @see #merge 205 | # @return [void] 206 | def merge!(other) 207 | self.class.coerce(other).to_h.each { |name, values| set name, values } 208 | end 209 | 210 | # Returns new instance with `other` headers merged in. 211 | # 212 | # @see #merge! 213 | # @return [Headers] 214 | def merge(other) 215 | dup.tap { |dupped| dupped.merge! other } 216 | end 217 | 218 | private 219 | 220 | # Transforms `name` to canonical HTTP header capitalization 221 | def normalize_header(name) 222 | self.class.normalizer.call(name) 223 | end 224 | 225 | # Ensures there is no new line character in the header value 226 | # 227 | # @param [String] value 228 | # @raise [HeaderError] if value includes new line character 229 | # @return [String] stringified header value 230 | def validate_value(value) 231 | v = value.to_s 232 | return v unless v.include?("\n") 233 | 234 | raise HeaderError, "Invalid HTTP header field value: #{v.inspect}" 235 | end 236 | end 237 | end 238 | -------------------------------------------------------------------------------- /lib/http/headers/known.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Headers 5 | # Content-Types that are acceptable for the response. 6 | ACCEPT = "Accept" 7 | 8 | # Content-codings that are acceptable in the response. 9 | ACCEPT_ENCODING = "Accept-Encoding" 10 | 11 | # The age the object has been in a proxy cache in seconds. 12 | AGE = "Age" 13 | 14 | # Authentication credentials for HTTP authentication. 15 | AUTHORIZATION = "Authorization" 16 | 17 | # Used to specify directives that must be obeyed by all caching mechanisms 18 | # along the request-response chain. 19 | CACHE_CONTROL = "Cache-Control" 20 | 21 | # An HTTP cookie previously sent by the server with Set-Cookie. 22 | COOKIE = "Cookie" 23 | 24 | # Control options for the current connection and list 25 | # of hop-by-hop request fields. 26 | CONNECTION = "Connection" 27 | 28 | # The length of the request body in octets (8-bit bytes). 29 | CONTENT_LENGTH = "Content-Length" 30 | 31 | # The MIME type of the body of the request 32 | # (used with POST and PUT requests). 33 | CONTENT_TYPE = "Content-Type" 34 | 35 | # The date and time that the message was sent (in "HTTP-date" format as 36 | # defined by RFC 7231 Date/Time Formats). 37 | DATE = "Date" 38 | 39 | # An identifier for a specific version of a resource, 40 | # often a message digest. 41 | ETAG = "ETag" 42 | 43 | # Gives the date/time after which the response is considered stale (in 44 | # "HTTP-date" format as defined by RFC 7231). 45 | EXPIRES = "Expires" 46 | 47 | # The domain name of the server (for virtual hosting), and the TCP port 48 | # number on which the server is listening. The port number may be omitted 49 | # if the port is the standard port for the service requested. 50 | HOST = "Host" 51 | 52 | # Allows a 304 Not Modified to be returned if content is unchanged. 53 | IF_MODIFIED_SINCE = "If-Modified-Since" 54 | 55 | # Allows a 304 Not Modified to be returned if content is unchanged. 56 | IF_NONE_MATCH = "If-None-Match" 57 | 58 | # The last modified date for the requested object (in "HTTP-date" format as 59 | # defined by RFC 7231). 60 | LAST_MODIFIED = "Last-Modified" 61 | 62 | # Used in redirection, or when a new resource has been created. 63 | LOCATION = "Location" 64 | 65 | # Authorization credentials for connecting to a proxy. 66 | PROXY_AUTHORIZATION = "Proxy-Authorization" 67 | 68 | # An HTTP cookie. 69 | SET_COOKIE = "Set-Cookie" 70 | 71 | # The form of encoding used to safely transfer the entity to the user. 72 | # Currently defined methods are: chunked, compress, deflate, gzip, identity. 73 | TRANSFER_ENCODING = "Transfer-Encoding" 74 | 75 | # Indicates what additional content codings have been applied to the 76 | # entity-body. 77 | CONTENT_ENCODING = "Content-Encoding" 78 | 79 | # The user agent string of the user agent. 80 | USER_AGENT = "User-Agent" 81 | 82 | # Tells downstream proxies how to match future request headers to decide 83 | # whether the cached response can be used rather than requesting a fresh 84 | # one from the origin server. 85 | VARY = "Vary" 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/http/headers/mixin.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | module HTTP 6 | class Headers 7 | # Provides shared behavior for {HTTP::Request} and {HTTP::Response}. 8 | # Expects `@headers` to be an instance of {HTTP::Headers}. 9 | # 10 | # @example Usage 11 | # 12 | # class MyHttpRequest 13 | # include HTTP::Headers::Mixin 14 | # 15 | # def initialize 16 | # @headers = HTTP::Headers.new 17 | # end 18 | # end 19 | module Mixin 20 | extend Forwardable 21 | 22 | # @return [HTTP::Headers] 23 | attr_reader :headers 24 | 25 | # @!method [] 26 | # (see HTTP::Headers#[]) 27 | def_delegator :headers, :[] 28 | 29 | # @!method []= 30 | # (see HTTP::Headers#[]=) 31 | def_delegator :headers, :[]= 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/http/headers/normalizer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Headers 5 | class Normalizer 6 | # Matches HTTP header names when in "Canonical-Http-Format" 7 | CANONICAL_NAME_RE = /\A[A-Z][a-z]*(?:-[A-Z][a-z]*)*\z/ 8 | 9 | # Matches valid header field name according to RFC. 10 | # @see http://tools.ietf.org/html/rfc7230#section-3.2 11 | COMPLIANT_NAME_RE = /\A[A-Za-z0-9!#$%&'*+\-.^_`|~]+\z/ 12 | 13 | NAME_PARTS_SEPARATOR_RE = /[\-_]/ 14 | 15 | # @private 16 | # Normalized header names cache 17 | class Cache 18 | MAX_SIZE = 200 19 | 20 | def initialize 21 | @store = {} 22 | end 23 | 24 | def get(key) 25 | @store[key] 26 | end 27 | alias [] get 28 | 29 | def set(key, value) 30 | # Maintain cache size 31 | @store.delete(@store.each_key.first) while MAX_SIZE <= @store.size 32 | 33 | @store[key] = value 34 | end 35 | alias []= set 36 | end 37 | 38 | def initialize 39 | @cache = Cache.new 40 | end 41 | 42 | # Transforms `name` to canonical HTTP header capitalization 43 | def call(name) 44 | name = -name.to_s 45 | value = (@cache[name] ||= -normalize_header(name)) 46 | 47 | value.dup 48 | end 49 | 50 | private 51 | 52 | # Transforms `name` to canonical HTTP header capitalization 53 | # 54 | # @param [String] name 55 | # @raise [HeaderError] if normalized name does not 56 | # match {COMPLIANT_NAME_RE} 57 | # @return [String] canonical HTTP header name 58 | def normalize_header(name) 59 | return name if CANONICAL_NAME_RE.match?(name) 60 | 61 | normalized = name.split(NAME_PARTS_SEPARATOR_RE).each(&:capitalize!).join("-") 62 | 63 | return normalized if COMPLIANT_NAME_RE.match?(normalized) 64 | 65 | raise HeaderError, "Invalid HTTP header field name: #{name.inspect}" 66 | end 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /lib/http/mime_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | # MIME type encode/decode adapters 5 | module MimeType 6 | class << self 7 | # Associate MIME type with adapter 8 | # 9 | # @example 10 | # 11 | # module JsonAdapter 12 | # class << self 13 | # def encode(obj) 14 | # # encode logic here 15 | # end 16 | # 17 | # def decode(str) 18 | # # decode logic here 19 | # end 20 | # end 21 | # end 22 | # 23 | # HTTP::MimeType.register_adapter 'application/json', MyJsonAdapter 24 | # 25 | # @param [#to_s] type 26 | # @param [#encode, #decode] adapter 27 | # @return [void] 28 | def register_adapter(type, adapter) 29 | adapters[type.to_s] = adapter 30 | end 31 | 32 | # Returns adapter associated with MIME type 33 | # 34 | # @param [#to_s] type 35 | # @raise [Error] if no adapter found 36 | # @return [Class] 37 | def [](type) 38 | adapters[normalize type] || raise(Error, "Unknown MIME type: #{type}") 39 | end 40 | 41 | # Register a shortcut for MIME type 42 | # 43 | # @example 44 | # 45 | # HTTP::MimeType.register_alias 'application/json', :json 46 | # 47 | # @param [#to_s] type 48 | # @param [#to_sym] shortcut 49 | # @return [void] 50 | def register_alias(type, shortcut) 51 | aliases[shortcut.to_sym] = type.to_s 52 | end 53 | 54 | # Resolves type by shortcut if possible 55 | # 56 | # @param [#to_s] type 57 | # @return [String] 58 | def normalize(type) 59 | aliases.fetch type, type.to_s 60 | end 61 | 62 | private 63 | 64 | # :nodoc: 65 | def adapters 66 | @adapters ||= {} 67 | end 68 | 69 | # :nodoc: 70 | def aliases 71 | @aliases ||= {} 72 | end 73 | end 74 | end 75 | end 76 | 77 | # built-in mime types 78 | require "http/mime_type/json" 79 | -------------------------------------------------------------------------------- /lib/http/mime_type/adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | require "singleton" 5 | 6 | module HTTP 7 | module MimeType 8 | # Base encode/decode MIME type adapter 9 | class Adapter 10 | include Singleton 11 | 12 | class << self 13 | extend Forwardable 14 | def_delegators :instance, :encode, :decode 15 | end 16 | 17 | # rubocop:disable Style/DocumentDynamicEvalDefinition 18 | %w[encode decode].each do |operation| 19 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 20 | def #{operation}(*) 21 | fail Error, "\#{self.class} does not supports ##{operation}" 22 | end 23 | RUBY 24 | end 25 | # rubocop:enable Style/DocumentDynamicEvalDefinition 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/http/mime_type/json.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "json" 4 | require "http/mime_type/adapter" 5 | 6 | module HTTP 7 | module MimeType 8 | # JSON encode/decode MIME type adapter 9 | class JSON < Adapter 10 | # Encodes object to JSON 11 | def encode(obj) 12 | return obj.to_json if obj.respond_to?(:to_json) 13 | 14 | ::JSON.dump obj 15 | end 16 | 17 | # Decodes JSON 18 | def decode(str) 19 | ::JSON.parse str 20 | end 21 | end 22 | 23 | register_adapter "application/json", JSON 24 | register_alias "application/json", :json 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /lib/http/options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/headers" 4 | require "openssl" 5 | require "socket" 6 | require "http/uri" 7 | 8 | module HTTP 9 | class Options # rubocop:disable Metrics/ClassLength 10 | @default_socket_class = TCPSocket 11 | @default_ssl_socket_class = OpenSSL::SSL::SSLSocket 12 | @default_timeout_class = HTTP::Timeout::Null 13 | @available_features = {} 14 | 15 | class << self 16 | attr_accessor :default_socket_class, :default_ssl_socket_class, :default_timeout_class 17 | attr_reader :available_features 18 | 19 | def new(options = {}) 20 | options.is_a?(self) ? options : super 21 | end 22 | 23 | def defined_options 24 | @defined_options ||= [] 25 | end 26 | 27 | def register_feature(name, impl) 28 | @available_features[name] = impl 29 | end 30 | 31 | protected 32 | 33 | def def_option(name, reader_only: false, &interpreter) 34 | defined_options << name.to_sym 35 | interpreter ||= ->(v) { v } 36 | 37 | if reader_only 38 | attr_reader name 39 | else 40 | attr_accessor name 41 | protected :"#{name}=" 42 | end 43 | 44 | define_method(:"with_#{name}") do |value| 45 | dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) } 46 | end 47 | end 48 | end 49 | 50 | def initialize(options = {}) 51 | defaults = { 52 | response: :auto, 53 | proxy: {}, 54 | timeout_class: self.class.default_timeout_class, 55 | timeout_options: {}, 56 | socket_class: self.class.default_socket_class, 57 | nodelay: false, 58 | ssl_socket_class: self.class.default_ssl_socket_class, 59 | ssl: {}, 60 | keep_alive_timeout: 5, 61 | headers: {}, 62 | cookies: {}, 63 | encoding: nil, 64 | features: {} 65 | } 66 | 67 | opts_w_defaults = defaults.merge(options) 68 | opts_w_defaults[:headers] = HTTP::Headers.coerce(opts_w_defaults[:headers]) 69 | opts_w_defaults.each { |(k, v)| self[k] = v } 70 | end 71 | 72 | def_option :headers do |new_headers| 73 | headers.merge(new_headers) 74 | end 75 | 76 | def_option :cookies do |new_cookies| 77 | new_cookies.each_with_object cookies.dup do |(k, v), jar| 78 | cookie = k.is_a?(Cookie) ? k : Cookie.new(k.to_s, v.to_s) 79 | jar[cookie.name] = cookie.cookie_value 80 | end 81 | end 82 | 83 | def_option :encoding do |encoding| 84 | self.encoding = Encoding.find(encoding) 85 | end 86 | 87 | def_option :features, reader_only: true do |new_features| 88 | # Normalize features from: 89 | # 90 | # [{feature_one: {opt: 'val'}}, :feature_two] 91 | # 92 | # into: 93 | # 94 | # {feature_one: {opt: 'val'}, feature_two: {}} 95 | normalized_features = new_features.each_with_object({}) do |feature, h| 96 | if feature.is_a?(Hash) 97 | h.merge!(feature) 98 | else 99 | h[feature] = {} 100 | end 101 | end 102 | 103 | features.merge(normalized_features) 104 | end 105 | 106 | def features=(features) 107 | @features = features.each_with_object({}) do |(name, opts_or_feature), h| 108 | h[name] = if opts_or_feature.is_a?(Feature) 109 | opts_or_feature 110 | else 111 | unless (feature = self.class.available_features[name]) 112 | argument_error! "Unsupported feature: #{name}" 113 | end 114 | feature.new(**opts_or_feature) 115 | end 116 | end 117 | end 118 | 119 | %w[ 120 | proxy params form json body response 121 | socket_class nodelay ssl_socket_class ssl_context ssl 122 | keep_alive_timeout timeout_class timeout_options 123 | ].each do |method_name| 124 | def_option method_name 125 | end 126 | 127 | def_option :follow, reader_only: true 128 | 129 | def follow=(value) 130 | @follow = 131 | case 132 | when !value then nil 133 | when true == value then {} 134 | when value.respond_to?(:fetch) then value 135 | else argument_error! "Unsupported follow options: #{value}" 136 | end 137 | end 138 | 139 | def_option :persistent, reader_only: true 140 | 141 | def persistent=(value) 142 | @persistent = value ? HTTP::URI.parse(value).origin : nil 143 | end 144 | 145 | def persistent? 146 | !persistent.nil? 147 | end 148 | 149 | def merge(other) 150 | h1 = to_hash 151 | h2 = other.to_hash 152 | 153 | merged = h1.merge(h2) do |k, v1, v2| 154 | case k 155 | when :headers 156 | v1.merge(v2) 157 | else 158 | v2 159 | end 160 | end 161 | 162 | self.class.new(merged) 163 | end 164 | 165 | def to_hash 166 | hash_pairs = self.class. 167 | defined_options. 168 | flat_map { |opt_name| [opt_name, send(opt_name)] } 169 | Hash[*hash_pairs] 170 | end 171 | 172 | def dup 173 | dupped = super 174 | yield(dupped) if block_given? 175 | dupped 176 | end 177 | 178 | def feature(name) 179 | features[name] 180 | end 181 | 182 | protected 183 | 184 | def []=(option, val) 185 | send(:"#{option}=", val) 186 | end 187 | 188 | private 189 | 190 | def argument_error!(message) 191 | raise(Error, message, caller(1..-1)) 192 | end 193 | end 194 | end 195 | -------------------------------------------------------------------------------- /lib/http/redirector.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "set" 4 | 5 | require "http/headers" 6 | 7 | module HTTP 8 | class Redirector 9 | # Notifies that we reached max allowed redirect hops 10 | class TooManyRedirectsError < ResponseError; end 11 | 12 | # Notifies that following redirects got into an endless loop 13 | class EndlessRedirectError < TooManyRedirectsError; end 14 | 15 | # HTTP status codes which indicate redirects 16 | REDIRECT_CODES = [300, 301, 302, 303, 307, 308].to_set.freeze 17 | 18 | # Codes which which should raise StateError in strict mode if original 19 | # request was any of {UNSAFE_VERBS} 20 | STRICT_SENSITIVE_CODES = [300, 301, 302].to_set.freeze 21 | 22 | # Insecure http verbs, which should trigger StateError in strict mode 23 | # upon {STRICT_SENSITIVE_CODES} 24 | UNSAFE_VERBS = %i[put delete post].to_set.freeze 25 | 26 | # Verbs which will remain unchanged upon See Other response. 27 | SEE_OTHER_ALLOWED_VERBS = %i[get head].to_set.freeze 28 | 29 | # @!attribute [r] strict 30 | # Returns redirector policy. 31 | # @return [Boolean] 32 | attr_reader :strict 33 | 34 | # @!attribute [r] max_hops 35 | # Returns maximum allowed hops. 36 | # @return [Fixnum] 37 | attr_reader :max_hops 38 | 39 | # @param [Hash] opts 40 | # @option opts [Boolean] :strict (true) redirector hops policy 41 | # @option opts [#to_i] :max_hops (5) maximum allowed amount of hops 42 | def initialize(opts = {}) 43 | @strict = opts.fetch(:strict, true) 44 | @max_hops = opts.fetch(:max_hops, 5).to_i 45 | @on_redirect = opts.fetch(:on_redirect, nil) 46 | end 47 | 48 | # Follows redirects until non-redirect response found 49 | def perform(request, response) 50 | @request = request 51 | @response = response 52 | @visited = [] 53 | collect_cookies_from_request 54 | collect_cookies_from_response 55 | 56 | while REDIRECT_CODES.include? @response.status.code 57 | @visited << "#{@request.verb} #{@request.uri}" 58 | 59 | raise TooManyRedirectsError if too_many_hops? 60 | raise EndlessRedirectError if endless_loop? 61 | 62 | @response.flush 63 | 64 | # XXX(ixti): using `Array#inject` to return `nil` if no Location header. 65 | @request = redirect_to(@response.headers.get(Headers::LOCATION).inject(:+)) 66 | unless cookie_jar.empty? 67 | @request.headers.set(Headers::COOKIE, cookie_jar.cookies.map { |c| "#{c.name}=#{c.value}" }.join("; ")) 68 | end 69 | @on_redirect.call @response, @request if @on_redirect.respond_to?(:call) 70 | @response = yield @request 71 | collect_cookies_from_response 72 | end 73 | 74 | @response 75 | end 76 | 77 | private 78 | 79 | # All known cookies. On the original request, this is only the original cookies, but after that, 80 | # Set-Cookie headers can add, set or delete cookies. 81 | def cookie_jar 82 | # it seems that @response.cookies instance is reused between responses, so we have to "clone" 83 | @cookie_jar ||= HTTP::CookieJar.new 84 | end 85 | 86 | def collect_cookies_from_request 87 | request_cookie_header = @request.headers["Cookie"] 88 | cookies = 89 | if request_cookie_header 90 | HTTP::Cookie.cookie_value_to_hash(request_cookie_header) 91 | else 92 | {} 93 | end 94 | 95 | cookies.each do |key, value| 96 | cookie_jar.add(HTTP::Cookie.new(key, value, path: @request.uri.path, domain: @request.host)) 97 | end 98 | end 99 | 100 | # Carry cookies from one response to the next. Carrying cookies to the next response ends up 101 | # carrying them to the next request as well. 102 | # 103 | # Note that this isn't part of the IETF standard, but all major browsers support setting cookies 104 | # on redirect: https://blog.dubbelboer.com/2012/11/25/302-cookie.html 105 | def collect_cookies_from_response 106 | # Overwrite previous cookies 107 | @response.cookies.each do |cookie| 108 | if cookie.value == "" 109 | cookie_jar.delete(cookie) 110 | else 111 | cookie_jar.add(cookie) 112 | end 113 | end 114 | 115 | # I wish we could just do @response.cookes = cookie_jar 116 | cookie_jar.each do |cookie| 117 | @response.cookies.add(cookie) 118 | end 119 | end 120 | 121 | # Check if we reached max amount of redirect hops 122 | # @return [Boolean] 123 | def too_many_hops? 124 | 1 <= @max_hops && @max_hops < @visited.count 125 | end 126 | 127 | # Check if we got into an endless loop 128 | # @return [Boolean] 129 | def endless_loop? 130 | 2 <= @visited.count(@visited.last) 131 | end 132 | 133 | # Redirect policy for follow 134 | # @return [Request] 135 | def redirect_to(uri) 136 | raise StateError, "no Location header in redirect" unless uri 137 | 138 | verb = @request.verb 139 | code = @response.status.code 140 | 141 | if UNSAFE_VERBS.include?(verb) && STRICT_SENSITIVE_CODES.include?(code) 142 | raise StateError, "can't follow #{@response.status} redirect" if @strict 143 | 144 | verb = :get 145 | end 146 | 147 | verb = :get if !SEE_OTHER_ALLOWED_VERBS.include?(verb) && 303 == code 148 | 149 | @request.redirect(uri, verb) 150 | end 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /lib/http/request/body.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Request 5 | class Body 6 | attr_reader :source 7 | 8 | def initialize(source) 9 | @source = source 10 | 11 | validate_source_type! 12 | end 13 | 14 | # Returns size which should be used for the "Content-Length" header. 15 | # 16 | # @return [Integer] 17 | def size 18 | if @source.is_a?(String) 19 | @source.bytesize 20 | elsif @source.respond_to?(:read) 21 | raise RequestError, "IO object must respond to #size" unless @source.respond_to?(:size) 22 | 23 | @source.size 24 | elsif @source.nil? 25 | 0 26 | else 27 | raise RequestError, "cannot determine size of body: #{@source.inspect}" 28 | end 29 | end 30 | 31 | # Yields chunks of content to be streamed to the request body. 32 | # 33 | # @yieldparam [String] 34 | def each(&block) 35 | if @source.is_a?(String) 36 | yield @source 37 | elsif @source.respond_to?(:read) 38 | IO.copy_stream(@source, ProcIO.new(block)) 39 | rewind(@source) 40 | elsif @source.is_a?(Enumerable) 41 | @source.each(&block) 42 | end 43 | 44 | self 45 | end 46 | 47 | # Request bodies are equivalent when they have the same source. 48 | def ==(other) 49 | self.class == other.class && self.source == other.source # rubocop:disable Style/RedundantSelf 50 | end 51 | 52 | private 53 | 54 | def rewind(io) 55 | io.rewind if io.respond_to? :rewind 56 | rescue Errno::ESPIPE, Errno::EPIPE 57 | # Pipe IOs respond to `:rewind` but fail when you call it. 58 | # 59 | # Calling `IO#rewind` on a pipe, fails with *ESPIPE* on MRI, 60 | # but *EPIPE* on jRuby. 61 | # 62 | # - **ESPIPE** -- "Illegal seek." 63 | # Invalid seek operation (such as on a pipe). 64 | # 65 | # - **EPIPE** -- "Broken pipe." 66 | # There is no process reading from the other end of a pipe. Every 67 | # library function that returns this error code also generates 68 | # a SIGPIPE signal; this signal terminates the program if not handled 69 | # or blocked. Thus, your program will never actually see EPIPE unless 70 | # it has handled or blocked SIGPIPE. 71 | # 72 | # See: https://www.gnu.org/software/libc/manual/html_node/Error-Codes.html 73 | nil 74 | end 75 | 76 | def validate_source_type! 77 | return if @source.is_a?(String) 78 | return if @source.respond_to?(:read) 79 | return if @source.is_a?(Enumerable) 80 | return if @source.nil? 81 | 82 | raise RequestError, "body of wrong type: #{@source.class}" 83 | end 84 | 85 | # This class provides a "writable IO" wrapper around a proc object, with 86 | # #write simply calling the proc, which we can pass in as the 87 | # "destination IO" in IO.copy_stream. 88 | class ProcIO 89 | def initialize(block) 90 | @block = block 91 | end 92 | 93 | def write(data) 94 | @block.call(data) 95 | data.bytesize 96 | end 97 | end 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/http/request/writer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/headers" 4 | 5 | module HTTP 6 | class Request 7 | class Writer 8 | # CRLF is the universal HTTP delimiter 9 | CRLF = "\r\n" 10 | 11 | # Chunked data termintaor. 12 | ZERO = "0" 13 | 14 | # Chunked transfer encoding 15 | CHUNKED = "chunked" 16 | 17 | # End of a chunked transfer 18 | CHUNKED_END = "#{ZERO}#{CRLF}#{CRLF}".freeze 19 | 20 | def initialize(socket, body, headers, headline) 21 | @body = body 22 | @socket = socket 23 | @headers = headers 24 | @request_header = [headline] 25 | end 26 | 27 | # Adds headers to the request header from the headers array 28 | def add_headers 29 | @headers.each do |field, value| 30 | @request_header << "#{field}: #{value}" 31 | end 32 | end 33 | 34 | # Stream the request to a socket 35 | def stream 36 | add_headers 37 | add_body_type_headers 38 | send_request 39 | end 40 | 41 | # Send headers needed to connect through proxy 42 | def connect_through_proxy 43 | add_headers 44 | write(join_headers) 45 | end 46 | 47 | # Adds the headers to the header array for the given request body we are working 48 | # with 49 | def add_body_type_headers 50 | return if @headers[Headers::CONTENT_LENGTH] || chunked? || ( 51 | @body.source.nil? && %w[GET HEAD DELETE CONNECT].any? do |method| 52 | @request_header[0].start_with?("#{method} ") 53 | end 54 | ) 55 | 56 | @request_header << "#{Headers::CONTENT_LENGTH}: #{@body.size}" 57 | end 58 | 59 | # Joins the headers specified in the request into a correctly formatted 60 | # http request header string 61 | def join_headers 62 | # join the headers array with crlfs, stick two on the end because 63 | # that ends the request header 64 | @request_header.join(CRLF) + (CRLF * 2) 65 | end 66 | 67 | # Writes HTTP request data into the socket. 68 | def send_request 69 | each_chunk { |chunk| write chunk } 70 | rescue Errno::EPIPE 71 | # server doesn't need any more data 72 | nil 73 | end 74 | 75 | # Yields chunks of request data that should be sent to the socket. 76 | # 77 | # It's important to send the request in a single write call when possible 78 | # in order to play nicely with Nagle's algorithm. Making two writes in a 79 | # row triggers a pathological case where Nagle is expecting a third write 80 | # that never happens. 81 | def each_chunk 82 | data = join_headers 83 | 84 | @body.each do |chunk| 85 | data << encode_chunk(chunk) 86 | yield data 87 | data.clear 88 | end 89 | 90 | yield data unless data.empty? 91 | 92 | yield CHUNKED_END if chunked? 93 | end 94 | 95 | # Returns the chunk encoded for to the specified "Transfer-Encoding" header. 96 | def encode_chunk(chunk) 97 | if chunked? 98 | chunk.bytesize.to_s(16) << CRLF << chunk << CRLF 99 | else 100 | chunk 101 | end 102 | end 103 | 104 | # Returns true if the request should be sent in chunked encoding. 105 | def chunked? 106 | @headers[Headers::TRANSFER_ENCODING] == CHUNKED 107 | end 108 | 109 | private 110 | 111 | # @raise [SocketWriteError] when unable to write to socket 112 | def write(data) 113 | until data.empty? 114 | length = @socket.write(data) 115 | break unless data.bytesize > length 116 | 117 | data = data.byteslice(length..-1) 118 | end 119 | rescue Errno::EPIPE 120 | raise 121 | rescue IOError, SocketError, SystemCallError => e 122 | raise SocketWriteError, "error writing to socket: #{e}", e.backtrace 123 | end 124 | end 125 | end 126 | end 127 | -------------------------------------------------------------------------------- /lib/http/response.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/headers" 6 | require "http/content_type" 7 | require "http/mime_type" 8 | require "http/response/status" 9 | require "http/response/inflater" 10 | require "http/cookie_jar" 11 | require "time" 12 | 13 | module HTTP 14 | class Response 15 | extend Forwardable 16 | 17 | include HTTP::Headers::Mixin 18 | 19 | # @return [Status] 20 | attr_reader :status 21 | 22 | # @return [String] 23 | attr_reader :version 24 | 25 | # @return [Body] 26 | attr_reader :body 27 | 28 | # @return [Request] 29 | attr_reader :request 30 | 31 | # @return [Hash] 32 | attr_reader :proxy_headers 33 | 34 | # Inits a new instance 35 | # 36 | # @option opts [Integer] :status Status code 37 | # @option opts [String] :version HTTP version 38 | # @option opts [Hash] :headers 39 | # @option opts [Hash] :proxy_headers 40 | # @option opts [HTTP::Connection] :connection 41 | # @option opts [String] :encoding Encoding to use when reading body 42 | # @option opts [String] :body 43 | # @option opts [HTTP::Request] request The request this is in response to. 44 | # @option opts [String] :uri (DEPRECATED) used to populate a missing request 45 | def initialize(opts) 46 | @version = opts.fetch(:version) 47 | @request = init_request(opts) 48 | @status = HTTP::Response::Status.new(opts.fetch(:status)) 49 | @headers = HTTP::Headers.coerce(opts[:headers] || {}) 50 | @proxy_headers = HTTP::Headers.coerce(opts[:proxy_headers] || {}) 51 | 52 | if opts.include?(:body) 53 | @body = opts.fetch(:body) 54 | else 55 | connection = opts.fetch(:connection) 56 | encoding = opts[:encoding] || charset || default_encoding 57 | 58 | @body = Response::Body.new(connection, encoding: encoding) 59 | end 60 | end 61 | 62 | # @!method reason 63 | # @return (see HTTP::Response::Status#reason) 64 | def_delegator :@status, :reason 65 | 66 | # @!method code 67 | # @return (see HTTP::Response::Status#code) 68 | def_delegator :@status, :code 69 | 70 | # @!method to_s 71 | # (see HTTP::Response::Body#to_s) 72 | def_delegator :@body, :to_s 73 | alias to_str to_s 74 | 75 | # @!method readpartial 76 | # (see HTTP::Response::Body#readpartial) 77 | def_delegator :@body, :readpartial 78 | 79 | # @!method connection 80 | # (see HTTP::Response::Body#connection) 81 | def_delegator :@body, :connection 82 | 83 | # @!method uri 84 | # @return (see HTTP::Request#uri) 85 | def_delegator :@request, :uri 86 | 87 | # Returns an Array ala Rack: `[status, headers, body]` 88 | # 89 | # @return [Array(Fixnum, Hash, String)] 90 | def to_a 91 | [status.to_i, headers.to_h, body.to_s] 92 | end 93 | 94 | # Flushes body and returns self-reference 95 | # 96 | # @return [Response] 97 | def flush 98 | body.to_s 99 | self 100 | end 101 | 102 | # Value of the Content-Length header. 103 | # 104 | # @return [nil] if Content-Length was not given, or it's value was invalid 105 | # (not an integer, e.g. empty string or string with non-digits). 106 | # @return [Integer] otherwise 107 | def content_length 108 | # http://greenbytes.de/tech/webdav/rfc7230.html#rfc.section.3.3.3 109 | # Clause 3: "If a message is received with both a Transfer-Encoding 110 | # and a Content-Length header field, the Transfer-Encoding overrides the Content-Length. 111 | return nil if @headers.include?(Headers::TRANSFER_ENCODING) 112 | 113 | value = @headers[Headers::CONTENT_LENGTH] 114 | return nil unless value 115 | 116 | begin 117 | Integer(value) 118 | rescue ArgumentError 119 | nil 120 | end 121 | end 122 | 123 | # Parsed Content-Type header 124 | # 125 | # @return [HTTP::ContentType] 126 | def content_type 127 | @content_type ||= ContentType.parse headers[Headers::CONTENT_TYPE] 128 | end 129 | 130 | # @!method mime_type 131 | # MIME type of response (if any) 132 | # @return [String, nil] 133 | def_delegator :content_type, :mime_type 134 | 135 | # @!method charset 136 | # Charset of response (if any) 137 | # @return [String, nil] 138 | def_delegator :content_type, :charset 139 | 140 | def cookies 141 | @cookies ||= headers.get(Headers::SET_COOKIE).each_with_object CookieJar.new do |v, jar| 142 | jar.parse(v, uri) 143 | end 144 | end 145 | 146 | def chunked? 147 | return false unless @headers.include?(Headers::TRANSFER_ENCODING) 148 | 149 | encoding = @headers.get(Headers::TRANSFER_ENCODING) 150 | 151 | # TODO: "chunked" is frozen in the request writer. How about making it accessible? 152 | encoding.last == "chunked" 153 | end 154 | 155 | # Parse response body with corresponding MIME type adapter. 156 | # 157 | # @param type [#to_s] Parse as given MIME type. 158 | # @raise (see MimeType.[]) 159 | # @return [Object] 160 | def parse(type = nil) 161 | MimeType[type || mime_type].decode to_s 162 | end 163 | 164 | # Inspect a response 165 | def inspect 166 | "#<#{self.class}/#{@version} #{code} #{reason} #{headers.to_h.inspect}>" 167 | end 168 | 169 | private 170 | 171 | def default_encoding 172 | return Encoding::UTF_8 if mime_type == "application/json" 173 | 174 | Encoding::BINARY 175 | end 176 | 177 | # Initialize an HTTP::Request from options. 178 | # 179 | # @return [HTTP::Request] 180 | def init_request(opts) 181 | raise ArgumentError, ":uri is for backwards compatibilty and conflicts with :request" \ 182 | if opts[:request] && opts[:uri] 183 | 184 | # For backwards compatibilty 185 | if opts[:uri] 186 | HTTP::Request.new(uri: opts[:uri], verb: :get) 187 | else 188 | opts.fetch(:request) 189 | end 190 | end 191 | end 192 | end 193 | -------------------------------------------------------------------------------- /lib/http/response/body.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | require "http/client" 5 | 6 | module HTTP 7 | class Response 8 | # A streamable response body, also easily converted into a string 9 | class Body 10 | extend Forwardable 11 | include Enumerable 12 | def_delegator :to_s, :empty? 13 | 14 | # The connection object used to make the corresponding request. 15 | # 16 | # @return [HTTP::Connection] 17 | attr_reader :connection 18 | 19 | def initialize(stream, encoding: Encoding::BINARY) 20 | @stream = stream 21 | @connection = stream.is_a?(Inflater) ? stream.connection : stream 22 | @streaming = nil 23 | @contents = nil 24 | @encoding = find_encoding(encoding) 25 | end 26 | 27 | # (see HTTP::Client#readpartial) 28 | def readpartial(*args) 29 | stream! 30 | chunk = @stream.readpartial(*args) 31 | 32 | String.new(chunk, encoding: @encoding) if chunk 33 | end 34 | 35 | # Iterate over the body, allowing it to be enumerable 36 | def each 37 | while (chunk = readpartial) 38 | yield chunk 39 | end 40 | end 41 | 42 | # @return [String] eagerly consume the entire body as a string 43 | def to_s 44 | return @contents if @contents 45 | 46 | raise StateError, "body is being streamed" unless @streaming.nil? 47 | 48 | begin 49 | @streaming = false 50 | @contents = String.new("", encoding: @encoding) 51 | 52 | while (chunk = @stream.readpartial) 53 | @contents << String.new(chunk, encoding: @encoding) 54 | chunk = nil # deallocate string 55 | end 56 | rescue 57 | @contents = nil 58 | raise 59 | end 60 | 61 | @contents 62 | end 63 | alias to_str to_s 64 | 65 | # Assert that the body is actively being streamed 66 | def stream! 67 | raise StateError, "body has already been consumed" if @streaming == false 68 | 69 | @streaming = true 70 | end 71 | 72 | # Easier to interpret string inspect 73 | def inspect 74 | "#<#{self.class}:#{object_id.to_s(16)} @streaming=#{!!@streaming}>" 75 | end 76 | 77 | private 78 | 79 | # Retrieve encoding by name. If encoding cannot be found, default to binary. 80 | def find_encoding(encoding) 81 | Encoding.find encoding 82 | rescue ArgumentError 83 | Encoding::BINARY 84 | end 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/http/response/inflater.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "zlib" 4 | 5 | module HTTP 6 | class Response 7 | class Inflater 8 | attr_reader :connection 9 | 10 | def initialize(connection) 11 | @connection = connection 12 | end 13 | 14 | def readpartial(*args) 15 | chunk = @connection.readpartial(*args) 16 | if chunk 17 | chunk = zstream.inflate(chunk) 18 | elsif !zstream.closed? 19 | zstream.finish if zstream.total_in.positive? 20 | zstream.close 21 | end 22 | chunk 23 | end 24 | 25 | private 26 | 27 | def zstream 28 | @zstream ||= Zlib::Inflate.new(32 + Zlib::MAX_WBITS) 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/http/response/parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "llhttp" 4 | 5 | module HTTP 6 | class Response 7 | # @api private 8 | class Parser 9 | attr_reader :parser, :headers, :status_code, :http_version 10 | 11 | def initialize 12 | @handler = Handler.new(self) 13 | @parser = LLHttp::Parser.new(@handler, type: :response) 14 | reset 15 | end 16 | 17 | def reset 18 | @parser.reset 19 | @handler.reset 20 | @header_finished = false 21 | @message_finished = false 22 | @headers = Headers.new 23 | @chunk = nil 24 | @status_code = nil 25 | @http_version = nil 26 | end 27 | 28 | def add(data) 29 | parser << data 30 | 31 | self 32 | rescue LLHttp::Error => e 33 | raise IOError, e.message 34 | end 35 | 36 | alias << add 37 | 38 | def mark_header_finished 39 | @header_finished = true 40 | @status_code = @parser.status_code 41 | @http_version = "#{@parser.http_major}.#{@parser.http_minor}" 42 | end 43 | 44 | def headers? 45 | @header_finished 46 | end 47 | 48 | def add_header(name, value) 49 | @headers.add(name, value) 50 | end 51 | 52 | def mark_message_finished 53 | @message_finished = true 54 | end 55 | 56 | def finished? 57 | @message_finished 58 | end 59 | 60 | def add_body(chunk) 61 | if @chunk 62 | @chunk << chunk 63 | else 64 | @chunk = chunk 65 | end 66 | end 67 | 68 | def read(size) 69 | return if @chunk.nil? 70 | 71 | if @chunk.bytesize <= size 72 | chunk = @chunk 73 | @chunk = nil 74 | else 75 | chunk = @chunk.byteslice(0, size) 76 | @chunk[0, size] = "" 77 | end 78 | 79 | chunk 80 | end 81 | 82 | class Handler < LLHttp::Delegate 83 | def initialize(target) 84 | @target = target 85 | super() 86 | reset 87 | end 88 | 89 | def reset 90 | @reading_header_value = false 91 | @field_value = +"" 92 | @field = +"" 93 | end 94 | 95 | def on_header_field(field) 96 | append_header if @reading_header_value 97 | @field << field 98 | end 99 | 100 | def on_header_value(value) 101 | @reading_header_value = true 102 | @field_value << value 103 | end 104 | 105 | def on_headers_complete 106 | append_header if @reading_header_value 107 | @target.mark_header_finished 108 | end 109 | 110 | def on_body(body) 111 | @target.add_body(body) 112 | end 113 | 114 | def on_message_complete 115 | @target.mark_message_finished 116 | end 117 | 118 | private 119 | 120 | def append_header 121 | @target.add_header(@field, @field_value) 122 | @reading_header_value = false 123 | @field_value = +"" 124 | @field = +"" 125 | end 126 | end 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/http/response/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "delegate" 4 | 5 | require "http/response/status/reasons" 6 | 7 | module HTTP 8 | class Response 9 | class Status < ::Delegator 10 | class << self 11 | # Coerces given value to Status. 12 | # 13 | # @example 14 | # 15 | # Status.coerce(:bad_request) # => Status.new(400) 16 | # Status.coerce("400") # => Status.new(400) 17 | # Status.coerce(true) # => raises HTTP::Error 18 | # 19 | # @raise [Error] if coercion is impossible 20 | # @param [Symbol, #to_i] object 21 | # @return [Status] 22 | def coerce(object) 23 | code = case 24 | when object.is_a?(String) then SYMBOL_CODES[symbolize object] 25 | when object.is_a?(Symbol) then SYMBOL_CODES[object] 26 | when object.is_a?(Numeric) then object.to_i 27 | end 28 | 29 | return new code if code 30 | 31 | raise Error, "Can't coerce #{object.class}(#{object}) to #{self}" 32 | end 33 | alias [] coerce 34 | 35 | private 36 | 37 | # Symbolizes given string 38 | # 39 | # @example 40 | # 41 | # symbolize "Bad Request" # => :bad_request 42 | # symbolize "Request-URI Too Long" # => :request_uri_too_long 43 | # symbolize "I'm a Teapot" # => :im_a_teapot 44 | # 45 | # @param [#to_s] str 46 | # @return [Symbol] 47 | def symbolize(str) 48 | str.to_s.downcase.tr("-", " ").gsub(/[^a-z ]/, "").gsub(/\s+/, "_").to_sym 49 | end 50 | end 51 | 52 | # Code to Symbol map 53 | # 54 | # @example Usage 55 | # 56 | # SYMBOLS[400] # => :bad_request 57 | # SYMBOLS[414] # => :request_uri_too_long 58 | # SYMBOLS[418] # => :im_a_teapot 59 | # 60 | # @return [Hash Symbol>] 61 | SYMBOLS = REASONS.transform_values { |v| symbolize(v) }.freeze 62 | 63 | # Reversed {SYMBOLS} map. 64 | # 65 | # @example Usage 66 | # 67 | # SYMBOL_CODES[:bad_request] # => 400 68 | # SYMBOL_CODES[:request_uri_too_long] # => 414 69 | # SYMBOL_CODES[:im_a_teapot] # => 418 70 | # 71 | # @return [Hash Fixnum>] 72 | SYMBOL_CODES = SYMBOLS.to_h { |k, v| [v, k] }.freeze 73 | 74 | # @return [Fixnum] status code 75 | attr_reader :code 76 | 77 | # @see REASONS 78 | # @return [String, nil] status message 79 | def reason 80 | REASONS[code] 81 | end 82 | 83 | # @return [String] string representation of HTTP status 84 | def to_s 85 | "#{code} #{reason}".strip 86 | end 87 | 88 | # Check if status code is informational (1XX) 89 | # @return [Boolean] 90 | def informational? 91 | 100 <= code && code < 200 92 | end 93 | 94 | # Check if status code is successful (2XX) 95 | # @return [Boolean] 96 | def success? 97 | 200 <= code && code < 300 98 | end 99 | 100 | # Check if status code is redirection (3XX) 101 | # @return [Boolean] 102 | def redirect? 103 | 300 <= code && code < 400 104 | end 105 | 106 | # Check if status code is client error (4XX) 107 | # @return [Boolean] 108 | def client_error? 109 | 400 <= code && code < 500 110 | end 111 | 112 | # Check if status code is server error (5XX) 113 | # @return [Boolean] 114 | def server_error? 115 | 500 <= code && code < 600 116 | end 117 | 118 | # Symbolized {#reason} 119 | # 120 | # @return [nil] unless code is well-known (see REASONS) 121 | # @return [Symbol] 122 | def to_sym 123 | SYMBOLS[code] 124 | end 125 | 126 | # Printable version of HTTP Status, surrounded by quote marks, 127 | # with special characters escaped. 128 | # 129 | # (see String#inspect) 130 | def inspect 131 | "#<#{self.class} #{self}>" 132 | end 133 | 134 | SYMBOLS.each do |code, symbol| 135 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 136 | def #{symbol}? # def bad_request? 137 | #{code} == code # 400 == code 138 | end # end 139 | RUBY 140 | end 141 | 142 | def __setobj__(obj) 143 | raise TypeError, "Expected #{obj.inspect} to respond to #to_i" unless obj.respond_to? :to_i 144 | 145 | @code = obj.to_i 146 | end 147 | 148 | def __getobj__ 149 | @code 150 | end 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/http/response/status/reasons.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # AUTO-GENERATED FILE, DO NOT CHANGE IT MANUALLY 4 | 5 | require "delegate" 6 | 7 | module HTTP 8 | class Response 9 | class Status < ::Delegator 10 | # Code to Reason map 11 | # 12 | # @example Usage 13 | # 14 | # REASONS[400] # => "Bad Request" 15 | # REASONS[414] # => "Request-URI Too Long" 16 | # 17 | # @return [Hash String>] 18 | REASONS = { 19 | 100 => "Continue", 20 | 101 => "Switching Protocols", 21 | 102 => "Processing", 22 | 200 => "OK", 23 | 201 => "Created", 24 | 202 => "Accepted", 25 | 203 => "Non-Authoritative Information", 26 | 204 => "No Content", 27 | 205 => "Reset Content", 28 | 206 => "Partial Content", 29 | 207 => "Multi-Status", 30 | 208 => "Already Reported", 31 | 226 => "IM Used", 32 | 300 => "Multiple Choices", 33 | 301 => "Moved Permanently", 34 | 302 => "Found", 35 | 303 => "See Other", 36 | 304 => "Not Modified", 37 | 305 => "Use Proxy", 38 | 307 => "Temporary Redirect", 39 | 308 => "Permanent Redirect", 40 | 400 => "Bad Request", 41 | 401 => "Unauthorized", 42 | 402 => "Payment Required", 43 | 403 => "Forbidden", 44 | 404 => "Not Found", 45 | 405 => "Method Not Allowed", 46 | 406 => "Not Acceptable", 47 | 407 => "Proxy Authentication Required", 48 | 408 => "Request Timeout", 49 | 409 => "Conflict", 50 | 410 => "Gone", 51 | 411 => "Length Required", 52 | 412 => "Precondition Failed", 53 | 413 => "Payload Too Large", 54 | 414 => "URI Too Long", 55 | 415 => "Unsupported Media Type", 56 | 416 => "Range Not Satisfiable", 57 | 417 => "Expectation Failed", 58 | 421 => "Misdirected Request", 59 | 422 => "Unprocessable Entity", 60 | 423 => "Locked", 61 | 424 => "Failed Dependency", 62 | 426 => "Upgrade Required", 63 | 428 => "Precondition Required", 64 | 429 => "Too Many Requests", 65 | 431 => "Request Header Fields Too Large", 66 | 451 => "Unavailable For Legal Reasons", 67 | 500 => "Internal Server Error", 68 | 501 => "Not Implemented", 69 | 502 => "Bad Gateway", 70 | 503 => "Service Unavailable", 71 | 504 => "Gateway Timeout", 72 | 505 => "HTTP Version Not Supported", 73 | 506 => "Variant Also Negotiates", 74 | 507 => "Insufficient Storage", 75 | 508 => "Loop Detected", 76 | 510 => "Not Extended", 77 | 511 => "Network Authentication Required" 78 | }.each_value(&:freeze).freeze 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /lib/http/retriable/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/retriable/performer" 4 | 5 | module HTTP 6 | module Retriable 7 | # Retriable version of HTTP::Client. 8 | # 9 | # @see http://www.rubydoc.info/gems/http/HTTP/Client 10 | class Client < HTTP::Client 11 | # @param [Performer] performer 12 | # @param [HTTP::Options, Hash] options 13 | def initialize(performer, options) 14 | @performer = performer 15 | super(options) 16 | end 17 | 18 | # Overriden version of `HTTP::Client#make_request`. 19 | # 20 | # Monitors request/response phase with performer. 21 | # 22 | # @see http://www.rubydoc.info/gems/http/HTTP/Client:perform 23 | def perform(req, options) 24 | @performer.perform(self, req) { super(req, options) } 25 | end 26 | 27 | private 28 | 29 | # Overriden version of `HTTP::Chainable#branch`. 30 | # 31 | # @return [HTTP::Retriable::Client] 32 | def branch(options) 33 | Retriable::Client.new(@performer, options) 34 | end 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/http/retriable/delay_calculator.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Retriable 5 | # @api private 6 | class DelayCalculator 7 | def initialize(opts) 8 | @max_delay = opts.fetch(:max_delay, Float::MAX).to_f 9 | if (delay = opts[:delay]).respond_to?(:call) 10 | @delay_proc = opts.fetch(:delay) 11 | else 12 | @delay = delay 13 | end 14 | end 15 | 16 | def call(iteration, response) 17 | delay = if response && (retry_header = response.headers["Retry-After"]) 18 | delay_from_retry_header(retry_header) 19 | else 20 | calculate_delay_from_iteration(iteration) 21 | end 22 | 23 | ensure_dealy_in_bounds(delay) 24 | end 25 | 26 | RFC2822_DATE_REGEX = /^ 27 | (?:Sun|Mon|Tue|Wed|Thu|Fri|Sat),\s+ 28 | (?:0[1-9]|[1-2]?[0-9]|3[01])\s+ 29 | (?:Jan|Feb|Mar|Apr|May|Jun|Jul|Aug|Sep|Oct|Nov|Dec)\s+ 30 | (?:19[0-9]{2}|[2-9][0-9]{3})\s+ 31 | (?:2[0-3]|[0-1][0-9]):(?:[0-5][0-9]):(?:60|[0-5][0-9])\s+ 32 | GMT 33 | $/x 34 | 35 | # Spec for Retry-After header 36 | # https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After 37 | def delay_from_retry_header(value) 38 | value = value.to_s.strip 39 | 40 | case value 41 | when RFC2822_DATE_REGEX then DateTime.rfc2822(value).to_time - Time.now.utc 42 | when /^\d+$/ then value.to_i 43 | else 0 44 | end 45 | end 46 | 47 | def calculate_delay_from_iteration(iteration) 48 | if @delay_proc 49 | @delay_proc.call(iteration) 50 | elsif @delay 51 | @delay 52 | else 53 | delay = (2**(iteration - 1)) - 1 54 | delay_noise = rand 55 | delay + delay_noise 56 | end 57 | end 58 | 59 | def ensure_dealy_in_bounds(delay) 60 | delay.clamp(0, @max_delay) 61 | end 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /lib/http/retriable/errors.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | # Retriable performance ran out of attempts 5 | class OutOfRetriesError < Error 6 | attr_accessor :response 7 | 8 | attr_writer :cause 9 | 10 | def cause 11 | @cause || super 12 | end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /lib/http/retriable/performer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "date" 4 | require "http/retriable/errors" 5 | require "http/retriable/delay_calculator" 6 | require "openssl" 7 | 8 | module HTTP 9 | module Retriable 10 | # Request performing watchdog. 11 | # @api private 12 | class Performer 13 | # Exceptions we should retry 14 | RETRIABLE_ERRORS = [ 15 | HTTP::TimeoutError, 16 | HTTP::ConnectionError, 17 | IO::EAGAINWaitReadable, 18 | Errno::ECONNRESET, 19 | Errno::ECONNREFUSED, 20 | Errno::EHOSTUNREACH, 21 | OpenSSL::SSL::SSLError, 22 | EOFError, 23 | IOError 24 | ].freeze 25 | 26 | # @param [Hash] opts 27 | # @option opts [#to_i] :tries (5) 28 | # @option opts [#call, #to_i] :delay (DELAY_PROC) 29 | # @option opts [Array(Exception)] :exceptions (RETRIABLE_ERRORS) 30 | # @option opts [Array(#to_i)] :retry_statuses 31 | # @option opts [#call] :on_retry 32 | # @option opts [#to_f] :max_delay (Float::MAX) 33 | # @option opts [#call] :should_retry 34 | def initialize(opts) 35 | @exception_classes = opts.fetch(:exceptions, RETRIABLE_ERRORS) 36 | @retry_statuses = opts[:retry_statuses] 37 | @tries = opts.fetch(:tries, 5).to_i 38 | @on_retry = opts.fetch(:on_retry, ->(*) {}) 39 | @should_retry_proc = opts[:should_retry] 40 | @delay_calculator = DelayCalculator.new(opts) 41 | end 42 | 43 | # Watches request/response execution. 44 | # 45 | # If any of {RETRIABLE_ERRORS} occur or response status is `5xx`, retries 46 | # up to `:tries` amount of times. Sleeps for amount of seconds calculated 47 | # with `:delay` proc before each retry. 48 | # 49 | # @see #initialize 50 | # @api private 51 | def perform(client, req, &block) 52 | 1.upto(Float::INFINITY) do |attempt| # infinite loop with index 53 | err, res = try_request(&block) 54 | 55 | if retry_request?(req, err, res, attempt) 56 | begin 57 | wait_for_retry_or_raise(req, err, res, attempt) 58 | ensure 59 | # Some servers support Keep-Alive on any response. Thus we should 60 | # flush response before retry, to avoid state error (when socket 61 | # has pending response data and we try to write new request). 62 | # Alternatively, as we don't need response body here at all, we 63 | # are going to close client, effectivle closing underlying socket 64 | # and resetting client's state. 65 | client.close 66 | end 67 | elsif err 68 | client.close 69 | raise err 70 | elsif res 71 | return res 72 | end 73 | end 74 | end 75 | 76 | def calculate_delay(iteration, response) 77 | @delay_calculator.call(iteration, response) 78 | end 79 | 80 | private 81 | 82 | # rubocop:disable Lint/RescueException 83 | def try_request 84 | err, res = nil 85 | 86 | begin 87 | res = yield 88 | rescue Exception => e 89 | err = e 90 | end 91 | 92 | [err, res] 93 | end 94 | # rubocop:enable Lint/RescueException 95 | 96 | def retry_request?(req, err, res, attempt) 97 | if @should_retry_proc 98 | @should_retry_proc.call(req, err, res, attempt) 99 | elsif err 100 | retry_exception?(err) 101 | else 102 | retry_response?(res) 103 | end 104 | end 105 | 106 | def retry_exception?(err) 107 | @exception_classes.any? { |e| err.is_a?(e) } 108 | end 109 | 110 | def retry_response?(res) 111 | return false unless @retry_statuses 112 | 113 | response_status = res.status.to_i 114 | retry_matchers = [@retry_statuses].flatten 115 | 116 | retry_matchers.any? do |matcher| 117 | case matcher 118 | when Range then matcher.cover?(response_status) 119 | when Numeric then matcher == response_status 120 | else matcher.call(response_status) 121 | end 122 | end 123 | end 124 | 125 | def wait_for_retry_or_raise(req, err, res, attempt) 126 | if attempt < @tries 127 | @on_retry.call(req, err, res) 128 | sleep calculate_delay(attempt, res) 129 | else 130 | res&.flush 131 | raise out_of_retries_error(req, res, err) 132 | end 133 | end 134 | 135 | # Builds OutOfRetriesError 136 | # 137 | # @param request [HTTP::Request] 138 | # @param status [HTTP::Response, nil] 139 | # @param exception [Exception, nil] 140 | def out_of_retries_error(request, response, exception) 141 | message = "#{request.verb.to_s.upcase} <#{request.uri}> failed" 142 | 143 | message += " with #{response.status}" if response 144 | message += ":#{exception}" if exception 145 | 146 | HTTP::OutOfRetriesError.new(message).tap do |ex| 147 | ex.cause = exception 148 | ex.response = response 149 | end 150 | end 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/http/timeout/global.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "timeout" 4 | require "io/wait" 5 | 6 | require "http/timeout/null" 7 | 8 | module HTTP 9 | module Timeout 10 | class Global < Null 11 | def initialize(*args) 12 | super 13 | 14 | @timeout = @time_left = options.fetch(:global_timeout) 15 | end 16 | 17 | # To future me: Don't remove this again, past you was smarter. 18 | def reset_counter 19 | @time_left = @timeout 20 | end 21 | 22 | def connect(socket_class, host, port, nodelay = false) 23 | reset_timer 24 | ::Timeout.timeout(@time_left, ConnectTimeoutError) do 25 | @socket = socket_class.open(host, port) 26 | @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay 27 | end 28 | 29 | log_time 30 | end 31 | 32 | def connect_ssl 33 | reset_timer 34 | 35 | begin 36 | @socket.connect_nonblock 37 | rescue IO::WaitReadable 38 | wait_readable_or_timeout 39 | retry 40 | rescue IO::WaitWritable 41 | wait_writable_or_timeout 42 | retry 43 | end 44 | end 45 | 46 | # Read from the socket 47 | def readpartial(size, buffer = nil) 48 | perform_io { read_nonblock(size, buffer) } 49 | end 50 | 51 | # Write to the socket 52 | def write(data) 53 | perform_io { write_nonblock(data) } 54 | end 55 | 56 | alias << write 57 | 58 | private 59 | 60 | def read_nonblock(size, buffer = nil) 61 | @socket.read_nonblock(size, buffer, exception: false) 62 | end 63 | 64 | def write_nonblock(data) 65 | @socket.write_nonblock(data, exception: false) 66 | end 67 | 68 | # Perform the given I/O operation with the given argument 69 | def perform_io 70 | reset_timer 71 | 72 | loop do 73 | result = yield 74 | 75 | case result 76 | when :wait_readable then wait_readable_or_timeout 77 | when :wait_writable then wait_writable_or_timeout 78 | when NilClass then return :eof 79 | else return result 80 | end 81 | rescue IO::WaitReadable 82 | wait_readable_or_timeout 83 | rescue IO::WaitWritable 84 | wait_writable_or_timeout 85 | end 86 | rescue EOFError 87 | :eof 88 | end 89 | 90 | # Wait for a socket to become readable 91 | def wait_readable_or_timeout 92 | @socket.to_io.wait_readable(@time_left) 93 | log_time 94 | end 95 | 96 | # Wait for a socket to become writable 97 | def wait_writable_or_timeout 98 | @socket.to_io.wait_writable(@time_left) 99 | log_time 100 | end 101 | 102 | # Due to the run/retry nature of nonblocking I/O, it's easier to keep track of time 103 | # via method calls instead of a block to monitor. 104 | def reset_timer 105 | @started = Time.now 106 | end 107 | 108 | def log_time 109 | @time_left -= (Time.now - @started) 110 | raise TimeoutError, "Timed out after using the allocated #{@timeout} seconds" if @time_left <= 0 111 | 112 | reset_timer 113 | end 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /lib/http/timeout/null.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "io/wait" 4 | 5 | module HTTP 6 | module Timeout 7 | class Null 8 | attr_reader :options, :socket 9 | 10 | def initialize(options = {}) 11 | @options = options 12 | end 13 | 14 | # Connects to a socket 15 | def connect(socket_class, host, port, nodelay = false) 16 | @socket = socket_class.open(host, port) 17 | @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay 18 | end 19 | 20 | # Starts a SSL connection on a socket 21 | def connect_ssl 22 | @socket.connect 23 | end 24 | 25 | def close 26 | @socket&.close 27 | end 28 | 29 | def closed? 30 | @socket&.closed? 31 | end 32 | 33 | # Configures the SSL connection and starts the connection 34 | def start_tls(host, ssl_socket_class, ssl_context) 35 | @socket = ssl_socket_class.new(socket, ssl_context) 36 | @socket.hostname = host if @socket.respond_to? :hostname= 37 | @socket.sync_close = true if @socket.respond_to? :sync_close= 38 | 39 | connect_ssl 40 | 41 | return unless ssl_context.verify_mode == OpenSSL::SSL::VERIFY_PEER 42 | return if ssl_context.respond_to?(:verify_hostname) && !ssl_context.verify_hostname 43 | 44 | @socket.post_connection_check(host) 45 | end 46 | 47 | # Read from the socket 48 | def readpartial(size, buffer = nil) 49 | @socket.readpartial(size, buffer) 50 | rescue EOFError 51 | :eof 52 | end 53 | 54 | # Write to the socket 55 | def write(data) 56 | @socket.write(data) 57 | end 58 | alias << write 59 | 60 | private 61 | 62 | # Retry reading 63 | def rescue_readable(timeout = read_timeout) 64 | yield 65 | rescue IO::WaitReadable 66 | retry if @socket.to_io.wait_readable(timeout) 67 | raise TimeoutError, "Read timed out after #{timeout} seconds" 68 | end 69 | 70 | # Retry writing 71 | def rescue_writable(timeout = write_timeout) 72 | yield 73 | rescue IO::WaitWritable 74 | retry if @socket.to_io.wait_writable(timeout) 75 | raise TimeoutError, "Write timed out after #{timeout} seconds" 76 | end 77 | end 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/http/timeout/per_operation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "timeout" 4 | 5 | require "http/timeout/null" 6 | 7 | module HTTP 8 | module Timeout 9 | class PerOperation < Null 10 | CONNECT_TIMEOUT = 0.25 11 | WRITE_TIMEOUT = 0.25 12 | READ_TIMEOUT = 0.25 13 | 14 | def initialize(*args) 15 | super 16 | 17 | @read_timeout = options.fetch(:read_timeout, READ_TIMEOUT) 18 | @write_timeout = options.fetch(:write_timeout, WRITE_TIMEOUT) 19 | @connect_timeout = options.fetch(:connect_timeout, CONNECT_TIMEOUT) 20 | end 21 | 22 | def connect(socket_class, host, port, nodelay = false) 23 | ::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do 24 | @socket = socket_class.open(host, port) 25 | @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay 26 | end 27 | end 28 | 29 | def connect_ssl 30 | rescue_readable(@connect_timeout) do 31 | rescue_writable(@connect_timeout) do 32 | @socket.connect_nonblock 33 | end 34 | end 35 | end 36 | 37 | # Read data from the socket 38 | def readpartial(size, buffer = nil) 39 | timeout = false 40 | loop do 41 | result = @socket.read_nonblock(size, buffer, exception: false) 42 | 43 | return :eof if result.nil? 44 | return result if result != :wait_readable 45 | 46 | raise TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout 47 | 48 | # marking the socket for timeout. Why is this not being raised immediately? 49 | # it seems there is some race-condition on the network level between calling 50 | # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting 51 | # for reads, and when waiting for x seconds, it returns nil suddenly without completing 52 | # the x seconds. In a normal case this would be a timeout on wait/read, but it can 53 | # also mean that the socket has been closed by the server. Therefore we "mark" the 54 | # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no 55 | # timeout. Else, the first timeout was a proper timeout. 56 | # This hack has to be done because io/wait#wait_readable doesn't provide a value for when 57 | # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks. 58 | timeout = true unless @socket.to_io.wait_readable(@read_timeout) 59 | end 60 | end 61 | 62 | # Write data to the socket 63 | def write(data) 64 | timeout = false 65 | loop do 66 | result = @socket.write_nonblock(data, exception: false) 67 | return result unless result == :wait_writable 68 | 69 | raise TimeoutError, "Write timed out after #{@write_timeout} seconds" if timeout 70 | 71 | timeout = true unless @socket.to_io.wait_writable(@write_timeout) 72 | end 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/http/uri.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "addressable/uri" 4 | 5 | module HTTP 6 | class URI 7 | extend Forwardable 8 | 9 | def_delegators :@uri, :scheme, :normalized_scheme, :scheme= 10 | def_delegators :@uri, :user, :normalized_user, :user= 11 | def_delegators :@uri, :password, :normalized_password, :password= 12 | def_delegators :@uri, :authority, :normalized_authority, :authority= 13 | def_delegators :@uri, :origin, :origin= 14 | def_delegators :@uri, :normalized_port, :port= 15 | def_delegators :@uri, :path, :normalized_path, :path= 16 | def_delegators :@uri, :query, :normalized_query, :query= 17 | def_delegators :@uri, :query_values, :query_values= 18 | def_delegators :@uri, :request_uri, :request_uri= 19 | def_delegators :@uri, :fragment, :normalized_fragment, :fragment= 20 | def_delegators :@uri, :omit, :join, :normalize 21 | 22 | # Host, either a domain name or IP address. If the host is an IPv6 address, it will be returned 23 | # without brackets surrounding it. 24 | # 25 | # @return [String] The host of the URI 26 | attr_reader :host 27 | 28 | # Normalized host, either a domain name or IP address. If the host is an IPv6 address, it will 29 | # be returned without brackets surrounding it. 30 | # 31 | # @return [String] The normalized host of the URI 32 | attr_reader :normalized_host 33 | 34 | # @private 35 | HTTP_SCHEME = "http" 36 | 37 | # @private 38 | HTTPS_SCHEME = "https" 39 | 40 | # @private 41 | PERCENT_ENCODE = /[^\x21-\x7E]+/ 42 | 43 | # @private 44 | NORMALIZER = lambda do |uri| 45 | uri = HTTP::URI.parse uri 46 | 47 | HTTP::URI.new( 48 | scheme: uri.normalized_scheme, 49 | authority: uri.normalized_authority, 50 | path: uri.path.empty? ? "/" : percent_encode(Addressable::URI.normalize_path(uri.path)), 51 | query: percent_encode(uri.query), 52 | fragment: uri.normalized_fragment 53 | ) 54 | end 55 | 56 | # Parse the given URI string, returning an HTTP::URI object 57 | # 58 | # @param [HTTP::URI, String, #to_str] uri to parse 59 | # 60 | # @return [HTTP::URI] new URI instance 61 | def self.parse(uri) 62 | return uri if uri.is_a?(self) 63 | 64 | new(Addressable::URI.parse(uri)) 65 | end 66 | 67 | # Encodes key/value pairs as application/x-www-form-urlencoded 68 | # 69 | # @param [#to_hash, #to_ary] form_values to encode 70 | # @param [TrueClass, FalseClass] sort should key/value pairs be sorted first? 71 | # 72 | # @return [String] encoded value 73 | def self.form_encode(form_values, sort = false) 74 | Addressable::URI.form_encode(form_values, sort) 75 | end 76 | 77 | # Percent-encode all characters matching a regular expression. 78 | # 79 | # @param [String] string raw string 80 | # 81 | # @return [String] encoded value 82 | # 83 | # @private 84 | def self.percent_encode(string) 85 | string&.gsub(PERCENT_ENCODE) do |substr| 86 | substr.encode(Encoding::UTF_8).bytes.map { |c| format("%%%02X", c) }.join 87 | end 88 | end 89 | 90 | # Creates an HTTP::URI instance from the given options 91 | # 92 | # @param [Hash, Addressable::URI] options_or_uri 93 | # 94 | # @option options_or_uri [String, #to_str] :scheme URI scheme 95 | # @option options_or_uri [String, #to_str] :user for basic authentication 96 | # @option options_or_uri [String, #to_str] :password for basic authentication 97 | # @option options_or_uri [String, #to_str] :host name component 98 | # @option options_or_uri [String, #to_str] :port network port to connect to 99 | # @option options_or_uri [String, #to_str] :path component to request 100 | # @option options_or_uri [String, #to_str] :query component distinct from path 101 | # @option options_or_uri [String, #to_str] :fragment component at the end of the URI 102 | # 103 | # @return [HTTP::URI] new URI instance 104 | def initialize(options_or_uri = {}) 105 | case options_or_uri 106 | when Hash 107 | @uri = Addressable::URI.new(options_or_uri) 108 | when Addressable::URI 109 | @uri = options_or_uri 110 | else 111 | raise TypeError, "expected Hash for options, got #{options_or_uri.class}" 112 | end 113 | 114 | @host = process_ipv6_brackets(@uri.host) 115 | @normalized_host = process_ipv6_brackets(@uri.normalized_host) 116 | end 117 | 118 | # Are these URI objects equal? Normalizes both URIs prior to comparison 119 | # 120 | # @param [Object] other URI to compare this one with 121 | # 122 | # @return [TrueClass, FalseClass] are the URIs equivalent (after normalization)? 123 | def ==(other) 124 | other.is_a?(URI) && normalize.to_s == other.normalize.to_s 125 | end 126 | 127 | # Are these URI objects equal? Does NOT normalizes both URIs prior to comparison 128 | # 129 | # @param [Object] other URI to compare this one with 130 | # 131 | # @return [TrueClass, FalseClass] are the URIs equivalent? 132 | def eql?(other) 133 | other.is_a?(URI) && to_s == other.to_s 134 | end 135 | 136 | # Hash value based off the normalized form of a URI 137 | # 138 | # @return [Integer] A hash of the URI 139 | def hash 140 | @hash ||= to_s.hash * -1 141 | end 142 | 143 | # Sets the host component for the URI. 144 | # 145 | # @param [String, #to_str] new_host The new host component. 146 | # @return [void] 147 | def host=(new_host) 148 | @uri.host = process_ipv6_brackets(new_host, brackets: true) 149 | 150 | @host = process_ipv6_brackets(@uri.host) 151 | @normalized_host = process_ipv6_brackets(@uri.normalized_host) 152 | end 153 | 154 | # Port number, either as specified or the default if unspecified 155 | # 156 | # @return [Integer] port number 157 | def port 158 | @uri.port || @uri.default_port 159 | end 160 | 161 | # @return [True] if URI is HTTP 162 | # @return [False] otherwise 163 | def http? 164 | HTTP_SCHEME == scheme 165 | end 166 | 167 | # @return [True] if URI is HTTPS 168 | # @return [False] otherwise 169 | def https? 170 | HTTPS_SCHEME == scheme 171 | end 172 | 173 | # @return [Object] duplicated URI 174 | def dup 175 | self.class.new @uri.dup 176 | end 177 | 178 | # Convert an HTTP::URI to a String 179 | # 180 | # @return [String] URI serialized as a String 181 | def to_s 182 | @uri.to_s 183 | end 184 | alias to_str to_s 185 | 186 | # @return [String] human-readable representation of URI 187 | def inspect 188 | format("#<%s:0x%014x URI:%s>", self.class.name, object_id << 1, to_s) 189 | end 190 | 191 | private 192 | 193 | # Process a URI host, adding or removing surrounding brackets if the host is an IPv6 address. 194 | # 195 | # @param [Boolean] brackets When true, brackets will be added to IPv6 addresses if missing. When 196 | # false, they will be removed if present. 197 | # 198 | # @return [String] Host with IPv6 address brackets added or removed 199 | def process_ipv6_brackets(raw_host, brackets: false) 200 | ip = IPAddr.new(raw_host) 201 | 202 | if ip.ipv6? 203 | brackets ? "[#{ip}]" : ip.to_s 204 | else 205 | raw_host 206 | end 207 | rescue IPAddr::Error 208 | raw_host 209 | end 210 | end 211 | end 212 | -------------------------------------------------------------------------------- /lib/http/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | VERSION = "5.2.0" 5 | end 6 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/httprb/http/11e237f34fac08ad42d08b425ece7cbd25da2751/logo.png -------------------------------------------------------------------------------- /spec/lib/http/connection_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Connection do 4 | let(:req) do 5 | HTTP::Request.new( 6 | verb: :get, 7 | uri: "http://example.com/", 8 | headers: {} 9 | ) 10 | end 11 | let(:socket) { double(connect: nil, close: nil) } 12 | let(:timeout_class) { double(new: socket) } 13 | let(:opts) { HTTP::Options.new(timeout_class: timeout_class) } 14 | let(:connection) { HTTP::Connection.new(req, opts) } 15 | 16 | describe "#initialize times out" do 17 | let(:req) do 18 | HTTP::Request.new( 19 | verb: :get, 20 | uri: "https://example.com/", 21 | headers: {} 22 | ) 23 | end 24 | 25 | before do 26 | expect(socket).to receive(:start_tls).and_raise(HTTP::TimeoutError) 27 | expect(socket).to receive(:closed?).and_return(false) 28 | expect(socket).to receive(:close) 29 | end 30 | 31 | it "closes the connection" do 32 | expect { connection }.to raise_error(HTTP::TimeoutError) 33 | end 34 | end 35 | 36 | describe "#read_headers!" do 37 | before do 38 | connection.instance_variable_set(:@pending_response, true) 39 | expect(socket).to receive(:readpartial) do 40 | <<-RESPONSE.gsub(/^\s*\| */, "").gsub("\n", "\r\n") 41 | | HTTP/1.1 200 OK 42 | | Content-Type: text 43 | | foo_bar: 123 44 | | 45 | RESPONSE 46 | end 47 | end 48 | 49 | it "populates headers collection, preserving casing" do 50 | connection.read_headers! 51 | expect(connection.headers).to eq("Content-Type" => "text", "foo_bar" => "123") 52 | expect(connection.headers["Foo-Bar"]).to eq("123") 53 | expect(connection.headers["foo_bar"]).to eq("123") 54 | end 55 | end 56 | 57 | describe "#readpartial" do 58 | before do 59 | connection.instance_variable_set(:@pending_response, true) 60 | expect(socket).to receive(:readpartial) do 61 | <<-RESPONSE.gsub(/^\s*\| */, "").gsub("\n", "\r\n") 62 | | HTTP/1.1 200 OK 63 | | Content-Type: text 64 | | 65 | RESPONSE 66 | end 67 | expect(socket).to receive(:readpartial).and_return("1") 68 | expect(socket).to receive(:readpartial).and_return("23") 69 | expect(socket).to receive(:readpartial).and_return("456") 70 | expect(socket).to receive(:readpartial).and_return("78") 71 | expect(socket).to receive(:readpartial).and_return("9") 72 | expect(socket).to receive(:readpartial).and_return("0") 73 | expect(socket).to receive(:readpartial).and_return(:eof) 74 | expect(socket).to receive(:closed?).and_return(true) 75 | end 76 | 77 | it "reads data in parts" do 78 | connection.read_headers! 79 | buffer = String.new 80 | while (s = connection.readpartial(3)) 81 | expect(connection.finished_request?).to be false if s != "" 82 | buffer << s 83 | end 84 | expect(buffer).to eq "1234567890" 85 | expect(connection.finished_request?).to be true 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /spec/lib/http/content_type_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::ContentType do 4 | describe ".parse" do 5 | context "with text/plain" do 6 | subject { described_class.parse "text/plain" } 7 | 8 | its(:mime_type) { is_expected.to eq "text/plain" } 9 | its(:charset) { is_expected.to be_nil } 10 | end 11 | 12 | context "with tEXT/plaIN" do 13 | subject { described_class.parse "tEXT/plaIN" } 14 | 15 | its(:mime_type) { is_expected.to eq "text/plain" } 16 | its(:charset) { is_expected.to be_nil } 17 | end 18 | 19 | context "with text/plain; charset=utf-8" do 20 | subject { described_class.parse "text/plain; charset=utf-8" } 21 | 22 | its(:mime_type) { is_expected.to eq "text/plain" } 23 | its(:charset) { is_expected.to eq "utf-8" } 24 | end 25 | 26 | context 'with text/plain; charset="utf-8"' do 27 | subject { described_class.parse 'text/plain; charset="utf-8"' } 28 | 29 | its(:mime_type) { is_expected.to eq "text/plain" } 30 | its(:charset) { is_expected.to eq "utf-8" } 31 | end 32 | 33 | context "with text/plain; charSET=utf-8" do 34 | subject { described_class.parse "text/plain; charSET=utf-8" } 35 | 36 | its(:mime_type) { is_expected.to eq "text/plain" } 37 | its(:charset) { is_expected.to eq "utf-8" } 38 | end 39 | 40 | context "with text/plain; foo=bar; charset=utf-8" do 41 | subject { described_class.parse "text/plain; foo=bar; charset=utf-8" } 42 | 43 | its(:mime_type) { is_expected.to eq "text/plain" } 44 | its(:charset) { is_expected.to eq "utf-8" } 45 | end 46 | 47 | context "with text/plain;charset=utf-8;foo=bar" do 48 | subject { described_class.parse "text/plain;charset=utf-8;foo=bar" } 49 | 50 | its(:mime_type) { is_expected.to eq "text/plain" } 51 | its(:charset) { is_expected.to eq "utf-8" } 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /spec/lib/http/features/auto_deflate_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::AutoDeflate do 4 | subject { HTTP::Features::AutoDeflate.new } 5 | 6 | it "raises error for wrong type" do 7 | expect { HTTP::Features::AutoDeflate.new(method: :wrong) }. 8 | to raise_error(HTTP::Error) { |error| 9 | expect(error.message).to eq("Only gzip and deflate methods are supported") 10 | } 11 | end 12 | 13 | it "accepts gzip method" do 14 | expect(HTTP::Features::AutoDeflate.new(method: :gzip).method).to eq "gzip" 15 | end 16 | 17 | it "accepts deflate method" do 18 | expect(HTTP::Features::AutoDeflate.new(method: :deflate).method).to eq "deflate" 19 | end 20 | 21 | it "accepts string as method" do 22 | expect(HTTP::Features::AutoDeflate.new(method: "gzip").method).to eq "gzip" 23 | end 24 | 25 | it "uses gzip by default" do 26 | expect(subject.method).to eq("gzip") 27 | end 28 | 29 | describe "#deflated_body" do 30 | let(:body) { %w[bees cows] } 31 | let(:deflated_body) { subject.deflated_body(body) } 32 | 33 | context "when method is gzip" do 34 | subject { HTTP::Features::AutoDeflate.new(method: :gzip) } 35 | 36 | it "returns object which yields gzipped content of the given body" do 37 | io = StringIO.new 38 | io.set_encoding(Encoding::BINARY) 39 | gzip = Zlib::GzipWriter.new(io) 40 | gzip.write("beescows") 41 | gzip.close 42 | gzipped = io.string 43 | 44 | expect(deflated_body.each.to_a.join).to eq gzipped 45 | end 46 | 47 | it "caches compressed content when size is called" do 48 | io = StringIO.new 49 | io.set_encoding(Encoding::BINARY) 50 | gzip = Zlib::GzipWriter.new(io) 51 | gzip.write("beescows") 52 | gzip.close 53 | gzipped = io.string 54 | 55 | expect(deflated_body.size).to eq gzipped.bytesize 56 | expect(deflated_body.each.to_a.join).to eq gzipped 57 | end 58 | end 59 | 60 | context "when method is deflate" do 61 | subject { HTTP::Features::AutoDeflate.new(method: :deflate) } 62 | 63 | it "returns object which yields deflated content of the given body" do 64 | deflated = Zlib::Deflate.deflate("beescows") 65 | 66 | expect(deflated_body.each.to_a.join).to eq deflated 67 | end 68 | 69 | it "caches compressed content when size is called" do 70 | deflated = Zlib::Deflate.deflate("beescows") 71 | 72 | expect(deflated_body.size).to eq deflated.bytesize 73 | expect(deflated_body.each.to_a.join).to eq deflated 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/lib/http/features/auto_inflate_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::AutoInflate do 4 | subject(:feature) { HTTP::Features::AutoInflate.new } 5 | 6 | let(:connection) { double } 7 | let(:headers) { {} } 8 | 9 | let(:response) do 10 | HTTP::Response.new( 11 | version: "1.1", 12 | status: 200, 13 | headers: headers, 14 | connection: connection, 15 | request: HTTP::Request.new(verb: :get, uri: "http://example.com") 16 | ) 17 | end 18 | 19 | describe "#wrap_response" do 20 | subject(:result) { feature.wrap_response(response) } 21 | 22 | context "when there is no Content-Encoding header" do 23 | it "returns original request" do 24 | expect(result).to be response 25 | end 26 | end 27 | 28 | context "for identity Content-Encoding header" do 29 | let(:headers) { {content_encoding: "identity"} } 30 | 31 | it "returns original request" do 32 | expect(result).to be response 33 | end 34 | end 35 | 36 | context "for unknown Content-Encoding header" do 37 | let(:headers) { {content_encoding: "not-supported"} } 38 | 39 | it "returns original request" do 40 | expect(result).to be response 41 | end 42 | end 43 | 44 | context "for deflate Content-Encoding header" do 45 | let(:headers) { {content_encoding: "deflate"} } 46 | 47 | it "returns a HTTP::Response wrapping the inflated response body" do 48 | expect(result.body).to be_instance_of HTTP::Response::Body 49 | end 50 | end 51 | 52 | context "for gzip Content-Encoding header" do 53 | let(:headers) { {content_encoding: "gzip"} } 54 | 55 | it "returns a HTTP::Response wrapping the inflated response body" do 56 | expect(result.body).to be_instance_of HTTP::Response::Body 57 | end 58 | end 59 | 60 | context "for x-gzip Content-Encoding header" do 61 | let(:headers) { {content_encoding: "x-gzip"} } 62 | 63 | it "returns a HTTP::Response wrapping the inflated response body" do 64 | expect(result.body).to be_instance_of HTTP::Response::Body 65 | end 66 | end 67 | 68 | # TODO(ixti): We should refactor API to either make uri non-optional, 69 | # or add reference to request into response object (better). 70 | context "when response has uri" do 71 | let(:response) do 72 | HTTP::Response.new( 73 | version: "1.1", 74 | status: 200, 75 | headers: {content_encoding: "gzip"}, 76 | connection: connection, 77 | request: HTTP::Request.new(verb: :get, uri: "https://example.com") 78 | ) 79 | end 80 | 81 | it "preserves uri in wrapped response" do 82 | expect(result.uri).to eq HTTP::URI.parse("https://example.com") 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/lib/http/features/instrumentation_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::Instrumentation do 4 | subject(:feature) { HTTP::Features::Instrumentation.new(instrumenter: instrumenter) } 5 | 6 | let(:instrumenter) { TestInstrumenter.new } 7 | 8 | before do 9 | test_instrumenter = Class.new(HTTP::Features::Instrumentation::NullInstrumenter) do 10 | attr_reader :output 11 | 12 | def initialize 13 | @output = {} 14 | end 15 | 16 | def start(_name, payload) 17 | output[:start] = payload 18 | end 19 | 20 | def finish(_name, payload) 21 | output[:finish] = payload 22 | end 23 | end 24 | 25 | stub_const("TestInstrumenter", test_instrumenter) 26 | end 27 | 28 | describe "logging the request" do 29 | let(:request) do 30 | HTTP::Request.new( 31 | verb: :post, 32 | uri: "https://example.com/", 33 | headers: {accept: "application/json"}, 34 | body: '{"hello": "world!"}' 35 | ) 36 | end 37 | 38 | it "logs the request" do 39 | feature.wrap_request(request) 40 | 41 | expect(instrumenter.output[:start]).to eq(request: request) 42 | end 43 | end 44 | 45 | describe "logging the response" do 46 | let(:response) do 47 | HTTP::Response.new( 48 | version: "1.1", 49 | status: 200, 50 | headers: {content_type: "application/json"}, 51 | body: '{"success": true}', 52 | request: HTTP::Request.new(verb: :get, uri: "https://example.com") 53 | ) 54 | end 55 | 56 | it "logs the response" do 57 | feature.wrap_response(response) 58 | 59 | expect(instrumenter.output[:finish]).to eq(response: response) 60 | end 61 | end 62 | 63 | describe "logging errors" do 64 | let(:request) do 65 | HTTP::Request.new( 66 | verb: :post, 67 | uri: "https://example.com/", 68 | headers: {accept: "application/json"}, 69 | body: '{"hello": "world!"}' 70 | ) 71 | end 72 | 73 | let(:error) { HTTP::TimeoutError.new } 74 | 75 | it "logs the error" do 76 | feature.on_error(request, error) 77 | 78 | expect(instrumenter.output[:finish]).to eq(request: request, error: error) 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/lib/http/features/logging_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "logger" 4 | 5 | RSpec.describe HTTP::Features::Logging do 6 | subject(:feature) do 7 | logger = Logger.new(logdev) 8 | logger.formatter = ->(severity, _, _, message) { format("** %s **\n%s\n", severity, message) } 9 | 10 | described_class.new(logger: logger) 11 | end 12 | 13 | let(:logdev) { StringIO.new } 14 | 15 | describe "logging the request" do 16 | let(:request) do 17 | HTTP::Request.new( 18 | verb: :post, 19 | uri: "https://example.com/", 20 | headers: {accept: "application/json"}, 21 | body: '{"hello": "world!"}' 22 | ) 23 | end 24 | 25 | it "logs the request" do 26 | feature.wrap_request(request) 27 | 28 | expect(logdev.string).to eq <<~OUTPUT 29 | ** INFO ** 30 | > POST https://example.com/ 31 | ** DEBUG ** 32 | Accept: application/json 33 | Host: example.com 34 | User-Agent: http.rb/#{HTTP::VERSION} 35 | 36 | {"hello": "world!"} 37 | OUTPUT 38 | end 39 | end 40 | 41 | describe "logging the response" do 42 | let(:response) do 43 | HTTP::Response.new( 44 | version: "1.1", 45 | status: 200, 46 | headers: {content_type: "application/json"}, 47 | body: '{"success": true}', 48 | request: HTTP::Request.new(verb: :get, uri: "https://example.com") 49 | ) 50 | end 51 | 52 | it "logs the response" do 53 | feature.wrap_response(response) 54 | 55 | expect(logdev.string).to eq <<~OUTPUT 56 | ** INFO ** 57 | < 200 OK 58 | ** DEBUG ** 59 | Content-Type: application/json 60 | 61 | {"success": true} 62 | OUTPUT 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /spec/lib/http/features/raise_error_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::RaiseError do 4 | subject(:feature) { described_class.new(ignore: ignore) } 5 | 6 | let(:connection) { double } 7 | let(:status) { 200 } 8 | let(:ignore) { [] } 9 | 10 | describe "#wrap_response" do 11 | subject(:result) { feature.wrap_response(response) } 12 | 13 | let(:response) do 14 | HTTP::Response.new( 15 | version: "1.1", 16 | status: status, 17 | headers: {}, 18 | connection: connection, 19 | request: HTTP::Request.new(verb: :get, uri: "https://example.com") 20 | ) 21 | end 22 | 23 | context "when status is 200" do 24 | it "returns original request" do 25 | expect(result).to be response 26 | end 27 | end 28 | 29 | context "when status is 399" do 30 | let(:status) { 399 } 31 | 32 | it "returns original request" do 33 | expect(result).to be response 34 | end 35 | end 36 | 37 | context "when status is 400" do 38 | let(:status) { 400 } 39 | 40 | it "raises" do 41 | expect { result }.to raise_error(HTTP::StatusError, "Unexpected status code 400") 42 | end 43 | end 44 | 45 | context "when status is 599" do 46 | let(:status) { 599 } 47 | 48 | it "raises" do 49 | expect { result }.to raise_error(HTTP::StatusError, "Unexpected status code 599") 50 | end 51 | end 52 | 53 | context "when error status is ignored" do 54 | let(:status) { 500 } 55 | let(:ignore) { [500] } 56 | 57 | it "returns original request" do 58 | expect(result).to be response 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /spec/lib/http/headers/mixin_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Headers::Mixin do 4 | let :dummy_class do 5 | Class.new do 6 | include HTTP::Headers::Mixin 7 | 8 | def initialize(headers) 9 | @headers = headers 10 | end 11 | end 12 | end 13 | 14 | let(:headers) { HTTP::Headers.new } 15 | let(:dummy) { dummy_class.new headers } 16 | 17 | describe "#headers" do 18 | it "returns @headers instance variable" do 19 | expect(dummy.headers).to be headers 20 | end 21 | end 22 | 23 | describe "#[]" do 24 | it "proxies to headers#[]" do 25 | expect(headers).to receive(:[]).with(:accept) 26 | dummy[:accept] 27 | end 28 | end 29 | 30 | describe "#[]=" do 31 | it "proxies to headers#[]" do 32 | expect(headers).to receive(:[]=).with(:accept, "text/plain") 33 | dummy[:accept] = "text/plain" 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/lib/http/headers/normalizer_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Headers::Normalizer do 4 | subject(:normalizer) { described_class.new } 5 | 6 | include_context RSpec::Memory 7 | 8 | describe "#call" do 9 | it "normalizes the header" do 10 | expect(normalizer.call("content_type")).to eq "Content-Type" 11 | end 12 | 13 | it "returns a non-frozen string" do 14 | expect(normalizer.call("content_type")).not_to be_frozen 15 | end 16 | 17 | it "evicts the oldest item when cache is full" do 18 | max_headers = (1..described_class::Cache::MAX_SIZE).map { |i| "Header#{i}" } 19 | max_headers.each { |header| normalizer.call(header) } 20 | normalizer.call("New-Header") 21 | cache_store = normalizer.instance_variable_get(:@cache).instance_variable_get(:@store) 22 | expect(cache_store.keys).to eq(max_headers[1..] + ["New-Header"]) 23 | end 24 | 25 | it "retuns mutable strings" do 26 | normalized_headers = Array.new(3) { normalizer.call("content_type") } 27 | 28 | expect(normalized_headers) 29 | .to satisfy { |arr| arr.uniq.size == 1 } 30 | .and(satisfy { |arr| arr.map(&:object_id).uniq.size == normalized_headers.size }) 31 | .and(satisfy { |arr| arr.none?(&:frozen?) }) 32 | end 33 | 34 | it "allocates minimal memory for normalization of the same header" do 35 | normalizer.call("accept") # XXX: Ensure normalizer is pre-allocated 36 | 37 | # On first call it is expected to allocate during normalization 38 | expect { normalizer.call("content_type") }.to limit_allocations( 39 | Array => 1, 40 | MatchData => 1, 41 | String => 6 42 | ) 43 | 44 | # On subsequent call it is expected to only allocate copy of a cached string 45 | expect { normalizer.call("content_type") }.to limit_allocations( 46 | Array => 0, 47 | MatchData => 0, 48 | String => 1 49 | ) 50 | end 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /spec/lib/http/options/body_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "body" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to nil" do 7 | expect(opts.body).to be_nil 8 | end 9 | 10 | it "may be specified with with_body" do 11 | opts2 = opts.with_body("foo") 12 | expect(opts.body).to be_nil 13 | expect(opts2.body).to eq("foo") 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/lib/http/options/features_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "features" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to be empty" do 7 | expect(opts.features).to be_empty 8 | end 9 | 10 | it "accepts plain symbols in array" do 11 | opts2 = opts.with_features([:auto_inflate]) 12 | expect(opts.features).to be_empty 13 | expect(opts2.features.keys).to eq([:auto_inflate]) 14 | expect(opts2.features[:auto_inflate]). 15 | to be_instance_of(HTTP::Features::AutoInflate) 16 | end 17 | 18 | it "accepts feature name with its options in array" do 19 | opts2 = opts.with_features([{auto_deflate: {method: :deflate}}]) 20 | expect(opts.features).to be_empty 21 | expect(opts2.features.keys).to eq([:auto_deflate]) 22 | expect(opts2.features[:auto_deflate]). 23 | to be_instance_of(HTTP::Features::AutoDeflate) 24 | expect(opts2.features[:auto_deflate].method).to eq("deflate") 25 | end 26 | 27 | it "raises error for not supported features" do 28 | expect { opts.with_features([:wrong_feature]) }. 29 | to raise_error(HTTP::Error) { |error| 30 | expect(error.message).to eq("Unsupported feature: wrong_feature") 31 | } 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /spec/lib/http/options/form_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "form" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to nil" do 7 | expect(opts.form).to be_nil 8 | end 9 | 10 | it "may be specified with with_form_data" do 11 | opts2 = opts.with_form(foo: 42) 12 | expect(opts.form).to be_nil 13 | expect(opts2.form).to eq(foo: 42) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/lib/http/options/headers_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "headers" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to be empty" do 7 | expect(opts.headers).to be_empty 8 | end 9 | 10 | it "may be specified with with_headers" do 11 | opts2 = opts.with_headers(accept: "json") 12 | expect(opts.headers).to be_empty 13 | expect(opts2.headers).to eq([%w[Accept json]]) 14 | end 15 | 16 | it "accepts any object that respond to :to_hash" do 17 | x = if RUBY_VERSION >= "3.2.0" 18 | Data.define(:to_hash).new(to_hash: { "accept" => "json" }) 19 | else 20 | Struct.new(:to_hash).new({ "accept" => "json" }) 21 | end 22 | expect(opts.with_headers(x).headers["accept"]).to eq("json") 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /spec/lib/http/options/json_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "json" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to nil" do 7 | expect(opts.json).to be_nil 8 | end 9 | 10 | it "may be specified with with_json data" do 11 | opts2 = opts.with_json(foo: 42) 12 | expect(opts.json).to be_nil 13 | expect(opts2.json).to eq(foo: 42) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/lib/http/options/merge_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "merge" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "supports a Hash" do 7 | old_response = opts.response 8 | expect(opts.merge(response: :body).response).to eq(:body) 9 | expect(opts.response).to eq(old_response) 10 | end 11 | 12 | it "supports another Options" do 13 | merged = opts.merge(HTTP::Options.new(response: :body)) 14 | expect(merged.response).to eq(:body) 15 | end 16 | 17 | it "merges as excepted in complex cases" do 18 | # FIXME: yuck :( 19 | 20 | foo = HTTP::Options.new( 21 | response: :body, 22 | params: {baz: "bar"}, 23 | form: {foo: "foo"}, 24 | body: "body-foo", 25 | json: {foo: "foo"}, 26 | headers: {accept: "json", foo: "foo"}, 27 | proxy: {}, 28 | features: {} 29 | ) 30 | 31 | bar = HTTP::Options.new( 32 | response: :parsed_body, 33 | persistent: "https://www.googe.com", 34 | params: {plop: "plip"}, 35 | form: {bar: "bar"}, 36 | body: "body-bar", 37 | json: {bar: "bar"}, 38 | keep_alive_timeout: 10, 39 | headers: {accept: "xml", bar: "bar"}, 40 | timeout_options: {foo: :bar}, 41 | ssl: {foo: "bar"}, 42 | proxy: {proxy_address: "127.0.0.1", proxy_port: 8080} 43 | ) 44 | 45 | expect(foo.merge(bar).to_hash).to eq( 46 | response: :parsed_body, 47 | timeout_class: described_class.default_timeout_class, 48 | timeout_options: {foo: :bar}, 49 | params: {plop: "plip"}, 50 | form: {bar: "bar"}, 51 | body: "body-bar", 52 | json: {bar: "bar"}, 53 | persistent: "https://www.googe.com", 54 | keep_alive_timeout: 10, 55 | ssl: {foo: "bar"}, 56 | headers: {"Foo" => "foo", "Accept" => "xml", "Bar" => "bar"}, 57 | proxy: {proxy_address: "127.0.0.1", proxy_port: 8080}, 58 | follow: nil, 59 | socket_class: described_class.default_socket_class, 60 | nodelay: false, 61 | ssl_socket_class: described_class.default_ssl_socket_class, 62 | ssl_context: nil, 63 | cookies: {}, 64 | encoding: nil, 65 | features: {} 66 | ) 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/lib/http/options/new_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "new" do 4 | it "supports a Options instance" do 5 | opts = HTTP::Options.new 6 | expect(HTTP::Options.new(opts)).to eq(opts) 7 | end 8 | 9 | context "with a Hash" do 10 | it "coerces :response correctly" do 11 | opts = HTTP::Options.new(response: :object) 12 | expect(opts.response).to eq(:object) 13 | end 14 | 15 | it "coerces :headers correctly" do 16 | opts = HTTP::Options.new(headers: {accept: "json"}) 17 | expect(opts.headers).to eq([%w[Accept json]]) 18 | end 19 | 20 | it "coerces :proxy correctly" do 21 | opts = HTTP::Options.new(proxy: {proxy_address: "127.0.0.1", proxy_port: 8080}) 22 | expect(opts.proxy).to eq(proxy_address: "127.0.0.1", proxy_port: 8080) 23 | end 24 | 25 | it "coerces :form correctly" do 26 | opts = HTTP::Options.new(form: {foo: 42}) 27 | expect(opts.form).to eq(foo: 42) 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /spec/lib/http/options/proxy_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "proxy" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to {}" do 7 | expect(opts.proxy).to eq({}) 8 | end 9 | 10 | it "may be specified with with_proxy" do 11 | opts2 = opts.with_proxy(proxy_address: "127.0.0.1", proxy_port: 8080) 12 | expect(opts.proxy).to eq({}) 13 | expect(opts2.proxy).to eq(proxy_address: "127.0.0.1", proxy_port: 8080) 14 | end 15 | 16 | it "accepts proxy address, port, username, and password" do 17 | opts2 = opts.with_proxy(proxy_address: "127.0.0.1", proxy_port: 8080, proxy_username: "username", proxy_password: "password") 18 | expect(opts2.proxy).to eq(proxy_address: "127.0.0.1", proxy_port: 8080, proxy_username: "username", proxy_password: "password") 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/lib/http/options_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options do 4 | subject { described_class.new(response: :body) } 5 | 6 | it "has reader methods for attributes" do 7 | expect(subject.response).to eq(:body) 8 | end 9 | 10 | it "coerces to a Hash" do 11 | expect(subject.to_hash).to be_a(Hash) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/lib/http/request/body_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Request::Body do 4 | subject { HTTP::Request::Body.new(body) } 5 | 6 | let(:body) { "" } 7 | 8 | describe "#initialize" do 9 | context "when body is nil" do 10 | let(:body) { nil } 11 | 12 | it "does not raise an error" do 13 | expect { subject }.not_to raise_error 14 | end 15 | end 16 | 17 | context "when body is a string" do 18 | let(:body) { "string body" } 19 | 20 | it "does not raise an error" do 21 | expect { subject }.not_to raise_error 22 | end 23 | end 24 | 25 | context "when body is an IO" do 26 | let(:body) { FakeIO.new("IO body") } 27 | 28 | it "does not raise an error" do 29 | expect { subject }.not_to raise_error 30 | end 31 | end 32 | 33 | context "when body is an Enumerable" do 34 | let(:body) { %w[bees cows] } 35 | 36 | it "does not raise an error" do 37 | expect { subject }.not_to raise_error 38 | end 39 | end 40 | 41 | context "when body is of unrecognized type" do 42 | let(:body) { 123 } 43 | 44 | it "raises an error" do 45 | expect { subject }.to raise_error(HTTP::RequestError) 46 | end 47 | end 48 | end 49 | 50 | describe "#source" do 51 | it "returns the original object" do 52 | expect(subject.source).to eq "" 53 | end 54 | end 55 | 56 | describe "#size" do 57 | context "when body is nil" do 58 | let(:body) { nil } 59 | 60 | it "returns zero" do 61 | expect(subject.size).to eq 0 62 | end 63 | end 64 | 65 | context "when body is a string" do 66 | let(:body) { "Привет, мир!" } 67 | 68 | it "returns string bytesize" do 69 | expect(subject.size).to eq 21 70 | end 71 | end 72 | 73 | context "when body is an IO with size" do 74 | let(:body) { FakeIO.new("content") } 75 | 76 | it "returns IO size" do 77 | expect(subject.size).to eq 7 78 | end 79 | end 80 | 81 | context "when body is an IO without size" do 82 | let(:body) { IO.pipe[0] } 83 | 84 | it "raises a RequestError" do 85 | expect { subject.size }.to raise_error(HTTP::RequestError) 86 | end 87 | end 88 | 89 | context "when body is an Enumerable" do 90 | let(:body) { %w[bees cows] } 91 | 92 | it "raises a RequestError" do 93 | expect { subject.size }.to raise_error(HTTP::RequestError) 94 | end 95 | end 96 | end 97 | 98 | describe "#each" do 99 | let(:chunks) do 100 | chunks = [] 101 | subject.each { |chunk| chunks << chunk.dup } 102 | chunks 103 | end 104 | 105 | context "when body is nil" do 106 | let(:body) { nil } 107 | 108 | it "yields nothing" do 109 | expect(chunks).to eq [] 110 | end 111 | end 112 | 113 | context "when body is a string" do 114 | let(:body) { "content" } 115 | 116 | it "yields the string" do 117 | expect(chunks).to eq %w[content] 118 | end 119 | end 120 | 121 | context "when body is a non-Enumerable IO" do 122 | let(:body) { FakeIO.new(("a" * 16 * 1024) + ("b" * 10 * 1024)) } 123 | 124 | it "yields chunks of content" do 125 | expect(chunks.inject("", :+)).to eq ("a" * 16 * 1024) + ("b" * 10 * 1024) 126 | end 127 | end 128 | 129 | context "when body is a pipe" do 130 | let(:ios) { IO.pipe } 131 | let(:body) { ios[0] } 132 | 133 | around do |example| 134 | writer = Thread.new(ios[1]) do |io| 135 | io << "abcdef" 136 | io.close 137 | end 138 | 139 | begin 140 | example.run 141 | ensure 142 | writer.join 143 | end 144 | end 145 | 146 | it "yields chunks of content" do 147 | expect(chunks.inject("", :+)).to eq("abcdef") 148 | end 149 | end 150 | 151 | context "when body is an Enumerable IO" do 152 | let(:data) { ("a" * 16 * 1024) + ("b" * 10 * 1024) } 153 | let(:body) { StringIO.new data } 154 | 155 | it "yields chunks of content" do 156 | expect(chunks.inject("", :+)).to eq data 157 | end 158 | 159 | it "allows to enumerate multiple times" do 160 | results = [] 161 | 162 | 2.times do 163 | result = "" 164 | subject.each { |chunk| result += chunk } 165 | results << result 166 | end 167 | 168 | aggregate_failures do 169 | expect(results.count).to eq 2 170 | expect(results).to all eq data 171 | end 172 | end 173 | end 174 | 175 | context "when body is an Enumerable" do 176 | let(:body) { %w[bees cows] } 177 | 178 | it "yields elements" do 179 | expect(chunks).to eq %w[bees cows] 180 | end 181 | end 182 | end 183 | 184 | describe "#==" do 185 | context "when sources are equivalent" do 186 | let(:body1) { HTTP::Request::Body.new("content") } 187 | let(:body2) { HTTP::Request::Body.new("content") } 188 | 189 | it "returns true" do 190 | expect(body1).to eq body2 191 | end 192 | end 193 | 194 | context "when sources are not equivalent" do 195 | let(:body1) { HTTP::Request::Body.new("content") } 196 | let(:body2) { HTTP::Request::Body.new(nil) } 197 | 198 | it "returns false" do 199 | expect(body1).not_to eq body2 200 | end 201 | end 202 | 203 | context "when objects are not of the same class" do 204 | let(:body1) { HTTP::Request::Body.new("content") } 205 | let(:body2) { "content" } 206 | 207 | it "returns false" do 208 | expect(body1).not_to eq body2 209 | end 210 | end 211 | end 212 | end 213 | -------------------------------------------------------------------------------- /spec/lib/http/request/writer_spec.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # frozen_string_literal: true 3 | 4 | RSpec.describe HTTP::Request::Writer do 5 | subject(:writer) { described_class.new(io, body, headers, headerstart) } 6 | 7 | let(:io) { StringIO.new } 8 | let(:body) { HTTP::Request::Body.new("") } 9 | let(:headers) { HTTP::Headers.new } 10 | let(:headerstart) { "GET /test HTTP/1.1" } 11 | 12 | describe "#stream" do 13 | context "when multiple headers are set" do 14 | let(:headers) { HTTP::Headers.coerce "Host" => "example.org" } 15 | 16 | it "separates headers with carriage return and line feed" do 17 | writer.stream 18 | expect(io.string).to eq [ 19 | "#{headerstart}\r\n", 20 | "Host: example.org\r\nContent-Length: 0\r\n\r\n" 21 | ].join 22 | end 23 | end 24 | 25 | context "when headers are specified as strings with mixed case" do 26 | let(:headers) { HTTP::Headers.coerce "content-Type" => "text", "X_MAX" => "200" } 27 | 28 | it "writes the headers with the same casing" do 29 | writer.stream 30 | expect(io.string).to eq [ 31 | "#{headerstart}\r\n", 32 | "content-Type: text\r\nX_MAX: 200\r\nContent-Length: 0\r\n\r\n" 33 | ].join 34 | end 35 | end 36 | 37 | context "when body is nonempty" do 38 | let(:body) { HTTP::Request::Body.new("content") } 39 | 40 | it "writes it to the socket and sets Content-Length" do 41 | writer.stream 42 | expect(io.string).to eq [ 43 | "#{headerstart}\r\n", 44 | "Content-Length: 7\r\n\r\n", 45 | "content" 46 | ].join 47 | end 48 | end 49 | 50 | context "when body is not set" do 51 | let(:body) { HTTP::Request::Body.new(nil) } 52 | 53 | it "doesn't write anything to the socket and doesn't set Content-Length" do 54 | writer.stream 55 | expect(io.string).to eq [ 56 | "#{headerstart}\r\n\r\n" 57 | ].join 58 | end 59 | end 60 | 61 | context "when body is empty" do 62 | let(:body) { HTTP::Request::Body.new("") } 63 | 64 | it "doesn't write anything to the socket and sets Content-Length" do 65 | writer.stream 66 | expect(io.string).to eq [ 67 | "#{headerstart}\r\n", 68 | "Content-Length: 0\r\n\r\n" 69 | ].join 70 | end 71 | end 72 | 73 | context "when Content-Length header is set" do 74 | let(:headers) { HTTP::Headers.coerce "Content-Length" => "12" } 75 | let(:body) { HTTP::Request::Body.new("content") } 76 | 77 | it "keeps the given value" do 78 | writer.stream 79 | expect(io.string).to eq [ 80 | "#{headerstart}\r\n", 81 | "Content-Length: 12\r\n\r\n", 82 | "content" 83 | ].join 84 | end 85 | end 86 | 87 | context "when Transfer-Encoding is chunked" do 88 | let(:headers) { HTTP::Headers.coerce "Transfer-Encoding" => "chunked" } 89 | let(:body) { HTTP::Request::Body.new(%w[request body]) } 90 | 91 | it "writes encoded content and omits Content-Length" do 92 | writer.stream 93 | expect(io.string).to eq [ 94 | "#{headerstart}\r\n", 95 | "Transfer-Encoding: chunked\r\n\r\n", 96 | "7\r\nrequest\r\n4\r\nbody\r\n0\r\n\r\n" 97 | ].join 98 | end 99 | end 100 | 101 | context "when server won't accept any more data" do 102 | before do 103 | expect(io).to receive(:write).and_raise(Errno::EPIPE) 104 | end 105 | 106 | it "aborts silently" do 107 | writer.stream 108 | end 109 | end 110 | 111 | context "when writing to socket raises an exception" do 112 | before do 113 | expect(io).to receive(:write).and_raise(Errno::ECONNRESET) 114 | end 115 | 116 | it "raises a ConnectionError" do 117 | expect { writer.stream }.to raise_error(HTTP::ConnectionError) 118 | end 119 | end 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /spec/lib/http/request_spec.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # frozen_string_literal: true 3 | 4 | RSpec.describe HTTP::Request do 5 | subject :request do 6 | HTTP::Request.new( 7 | verb: :get, 8 | uri: request_uri, 9 | headers: headers, 10 | proxy: proxy 11 | ) 12 | end 13 | 14 | let(:proxy) { {} } 15 | let(:headers) { {accept: "text/html"} } 16 | let(:request_uri) { "http://example.com/foo?bar=baz" } 17 | 18 | it "includes HTTP::Headers::Mixin" do 19 | expect(described_class).to include HTTP::Headers::Mixin 20 | end 21 | 22 | it "requires URI to have scheme part" do 23 | expect { HTTP::Request.new(verb: :get, uri: "example.com/") }.to \ 24 | raise_error(HTTP::Request::UnsupportedSchemeError) 25 | end 26 | 27 | it "provides a #scheme accessor" do 28 | expect(request.scheme).to eq(:http) 29 | end 30 | 31 | it "provides a #verb accessor" do 32 | expect(subject.verb).to eq(:get) 33 | end 34 | 35 | it "sets given headers" do 36 | expect(subject["Accept"]).to eq("text/html") 37 | end 38 | 39 | describe "Host header" do 40 | subject { request["Host"] } 41 | 42 | context "was not given" do 43 | it { is_expected.to eq "example.com" } 44 | 45 | context "and request URI has non-standard port" do 46 | let(:request_uri) { "http://example.com:3000/" } 47 | 48 | it { is_expected.to eq "example.com:3000" } 49 | end 50 | end 51 | 52 | context "was explicitly given" do 53 | before { headers[:host] = "github.com" } 54 | 55 | it { is_expected.to eq "github.com" } 56 | end 57 | end 58 | 59 | describe "User-Agent header" do 60 | subject { request["User-Agent"] } 61 | 62 | context "was not given" do 63 | it { is_expected.to eq HTTP::Request::USER_AGENT } 64 | end 65 | 66 | context "was explicitly given" do 67 | before { headers[:user_agent] = "MrCrawly/123" } 68 | 69 | it { is_expected.to eq "MrCrawly/123" } 70 | end 71 | end 72 | 73 | describe "#redirect" do 74 | subject(:redirected) { request.redirect "http://blog.example.com/" } 75 | 76 | let(:headers) { {accept: "text/html"} } 77 | let(:proxy) { {proxy_username: "douglas", proxy_password: "adams"} } 78 | let(:body) { "The Ultimate Question" } 79 | 80 | let :request do 81 | HTTP::Request.new( 82 | verb: :post, 83 | uri: "http://example.com/", 84 | headers: headers, 85 | proxy: proxy, 86 | body: body 87 | ) 88 | end 89 | 90 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://blog.example.com/" } 91 | 92 | its(:verb) { is_expected.to eq request.verb } 93 | its(:body) { is_expected.to eq request.body } 94 | its(:proxy) { is_expected.to eq request.proxy } 95 | 96 | it "presets new Host header" do 97 | expect(redirected["Host"]).to eq "blog.example.com" 98 | end 99 | 100 | context "with URL with non-standard port given" do 101 | subject(:redirected) { request.redirect "http://example.com:8080" } 102 | 103 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com:8080" } 104 | 105 | its(:verb) { is_expected.to eq request.verb } 106 | its(:body) { is_expected.to eq request.body } 107 | its(:proxy) { is_expected.to eq request.proxy } 108 | 109 | it "presets new Host header" do 110 | expect(redirected["Host"]).to eq "example.com:8080" 111 | end 112 | end 113 | 114 | context "with schema-less absolute URL given" do 115 | subject(:redirected) { request.redirect "//another.example.com/blog" } 116 | 117 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://another.example.com/blog" } 118 | 119 | its(:verb) { is_expected.to eq request.verb } 120 | its(:body) { is_expected.to eq request.body } 121 | its(:proxy) { is_expected.to eq request.proxy } 122 | 123 | it "presets new Host header" do 124 | expect(redirected["Host"]).to eq "another.example.com" 125 | end 126 | end 127 | 128 | context "with relative URL given" do 129 | subject(:redirected) { request.redirect "/blog" } 130 | 131 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com/blog" } 132 | 133 | its(:verb) { is_expected.to eq request.verb } 134 | its(:body) { is_expected.to eq request.body } 135 | its(:proxy) { is_expected.to eq request.proxy } 136 | 137 | it "keeps Host header" do 138 | expect(redirected["Host"]).to eq "example.com" 139 | end 140 | 141 | context "with original URI having non-standard port" do 142 | let :request do 143 | HTTP::Request.new( 144 | verb: :post, 145 | uri: "http://example.com:8080/", 146 | headers: headers, 147 | proxy: proxy, 148 | body: body 149 | ) 150 | end 151 | 152 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com:8080/blog" } 153 | end 154 | end 155 | 156 | context "with relative URL that misses leading slash given" do 157 | subject(:redirected) { request.redirect "blog" } 158 | 159 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com/blog" } 160 | 161 | its(:verb) { is_expected.to eq request.verb } 162 | its(:body) { is_expected.to eq request.body } 163 | its(:proxy) { is_expected.to eq request.proxy } 164 | 165 | it "keeps Host header" do 166 | expect(redirected["Host"]).to eq "example.com" 167 | end 168 | 169 | context "with original URI having non-standard port" do 170 | let :request do 171 | HTTP::Request.new( 172 | verb: :post, 173 | uri: "http://example.com:8080/", 174 | headers: headers, 175 | proxy: proxy, 176 | body: body 177 | ) 178 | end 179 | 180 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com:8080/blog" } 181 | end 182 | end 183 | 184 | context "with new verb given" do 185 | subject { request.redirect "http://blog.example.com/", :get } 186 | 187 | its(:verb) { is_expected.to be :get } 188 | end 189 | end 190 | 191 | describe "#headline" do 192 | subject(:headline) { request.headline } 193 | 194 | it { is_expected.to eq "GET /foo?bar=baz HTTP/1.1" } 195 | 196 | context "when URI contains encoded query" do 197 | let(:encoded_query) { "t=1970-01-01T01%3A00%3A00%2B01%3A00" } 198 | let(:request_uri) { "http://example.com/foo/?#{encoded_query}" } 199 | 200 | it "does not unencodes query part" do 201 | expect(headline).to eq "GET /foo/?#{encoded_query} HTTP/1.1" 202 | end 203 | end 204 | 205 | context "when URI contains non-ASCII path" do 206 | let(:request_uri) { "http://example.com/キョ" } 207 | 208 | it "encodes non-ASCII path part" do 209 | expect(headline).to eq "GET /%E3%82%AD%E3%83%A7 HTTP/1.1" 210 | end 211 | end 212 | 213 | context "when URI contains fragment" do 214 | let(:request_uri) { "http://example.com/foo#bar" } 215 | 216 | it "omits fragment part" do 217 | expect(headline).to eq "GET /foo HTTP/1.1" 218 | end 219 | end 220 | 221 | context "with proxy" do 222 | let(:proxy) { {user: "user", pass: "pass"} } 223 | 224 | it { is_expected.to eq "GET http://example.com/foo?bar=baz HTTP/1.1" } 225 | 226 | context "and HTTPS uri" do 227 | let(:request_uri) { "https://example.com/foo?bar=baz" } 228 | 229 | it { is_expected.to eq "GET /foo?bar=baz HTTP/1.1" } 230 | end 231 | end 232 | end 233 | 234 | describe "#inspect" do 235 | subject { request.inspect } 236 | 237 | it { is_expected.to eq "#" } 238 | end 239 | end 240 | -------------------------------------------------------------------------------- /spec/lib/http/response/body_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response::Body do 4 | subject(:body) { described_class.new(connection, encoding: Encoding::UTF_8) } 5 | 6 | let(:connection) { double(sequence_id: 0) } 7 | let(:chunks) { ["Hello, ", "World!"] } 8 | 9 | before do 10 | allow(connection).to receive(:readpartial) { chunks.shift } 11 | allow(connection).to receive(:body_completed?) { chunks.empty? } 12 | end 13 | 14 | it "streams bodies from responses" do 15 | expect(subject.to_s).to eq("Hello, World!") 16 | end 17 | 18 | context "when body empty" do 19 | let(:chunks) { [""] } 20 | 21 | it "returns responds to empty? with true" do 22 | expect(subject).to be_empty 23 | end 24 | end 25 | 26 | describe "#readpartial" do 27 | context "with size given" do 28 | it "passes value to underlying connection" do 29 | expect(connection).to receive(:readpartial).with(42) 30 | body.readpartial 42 31 | end 32 | end 33 | 34 | context "without size given" do 35 | it "does not blows up" do 36 | expect { body.readpartial }.not_to raise_error 37 | end 38 | 39 | it "calls underlying connection readpartial without specific size" do 40 | expect(connection).to receive(:readpartial).with no_args 41 | body.readpartial 42 | end 43 | end 44 | 45 | it "returns content in specified encoding" do 46 | body = described_class.new(connection) 47 | expect(connection).to receive(:readpartial). 48 | and_return(String.new("content", encoding: Encoding::UTF_8)) 49 | expect(body.readpartial.encoding).to eq Encoding::BINARY 50 | 51 | body = described_class.new(connection, encoding: Encoding::UTF_8) 52 | expect(connection).to receive(:readpartial). 53 | and_return(String.new("content", encoding: Encoding::BINARY)) 54 | expect(body.readpartial.encoding).to eq Encoding::UTF_8 55 | end 56 | end 57 | 58 | context "when body is gzipped" do 59 | subject(:body) do 60 | inflater = HTTP::Response::Inflater.new(connection) 61 | described_class.new(inflater, encoding: Encoding::UTF_8) 62 | end 63 | 64 | let(:chunks) do 65 | body = Zlib::Deflate.deflate("Hi, HTTP here ☺") 66 | len = body.length 67 | [body[0, len / 2], body[(len / 2)..]] 68 | end 69 | 70 | it "decodes body" do 71 | expect(subject.to_s).to eq("Hi, HTTP here ☺") 72 | end 73 | 74 | describe "#readpartial" do 75 | it "streams decoded body" do 76 | [ 77 | "Hi, HTTP ", 78 | "here ☺", 79 | nil 80 | ].each do |part| 81 | expect(subject.readpartial).to eq(part) 82 | end 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/lib/http/response/parser_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response::Parser do 4 | subject(:parser) { described_class.new } 5 | 6 | let(:raw_response) do 7 | "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: application/json\r\nMyHeader: val\r\nEmptyHeader: \r\n\r\n{}" 8 | end 9 | let(:expected_headers) do 10 | { 11 | "Content-Length" => "2", 12 | "Content-Type" => "application/json", 13 | "MyHeader" => "val", 14 | "EmptyHeader" => "" 15 | } 16 | end 17 | let(:expected_body) { "{}" } 18 | 19 | before do 20 | parts.each { |part| subject.add(part) } 21 | end 22 | 23 | context "whole response in one part" do 24 | let(:parts) { [raw_response] } 25 | 26 | it "parses headers" do 27 | expect(subject.headers.to_h).to eq(expected_headers) 28 | end 29 | 30 | it "parses body" do 31 | expect(subject.read(expected_body.size)).to eq(expected_body) 32 | end 33 | end 34 | 35 | context "response in many parts" do 36 | let(:parts) { raw_response.chars } 37 | 38 | it "parses headers" do 39 | expect(subject.headers.to_h).to eq(expected_headers) 40 | end 41 | 42 | it "parses body" do 43 | expect(subject.read(expected_body.size)).to eq(expected_body) 44 | end 45 | end 46 | 47 | context "when got 100 Continue response" do 48 | let :raw_response do 49 | "HTTP/1.1 100 Continue\r\n\r\n" \ 50 | "HTTP/1.1 200 OK\r\n" \ 51 | "Content-Length: 12\r\n\r\n" \ 52 | "Hello World!" 53 | end 54 | 55 | context "when response is feeded in one part" do 56 | let(:parts) { [raw_response] } 57 | 58 | it "skips to next non-info response" do 59 | expect(subject.status_code).to eq(200) 60 | expect(subject.headers).to eq("Content-Length" => "12") 61 | expect(subject.read(12)).to eq("Hello World!") 62 | end 63 | end 64 | 65 | context "when response is feeded in many parts" do 66 | let(:parts) { raw_response.chars } 67 | 68 | it "skips to next non-info response" do 69 | expect(subject.status_code).to eq(200) 70 | expect(subject.headers).to eq("Content-Length" => "12") 71 | expect(subject.read(12)).to eq("Hello World!") 72 | end 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /spec/lib/http/response/status_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response::Status do 4 | describe ".new" do 5 | it "fails if given value does not respond to #to_i" do 6 | expect { described_class.new double }.to raise_error TypeError 7 | end 8 | 9 | it "accepts any object that responds to #to_i" do 10 | expect { described_class.new double to_i: 200 }.not_to raise_error 11 | end 12 | end 13 | 14 | describe "#code" do 15 | subject { described_class.new("200.0").code } 16 | 17 | it { is_expected.to eq 200 } 18 | it { is_expected.to be_a Integer } 19 | end 20 | 21 | describe "#reason" do 22 | subject { described_class.new(code).reason } 23 | 24 | context "with unknown code" do 25 | let(:code) { 1024 } 26 | 27 | it { is_expected.to be_nil } 28 | end 29 | 30 | described_class::REASONS.each do |code, reason| 31 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 32 | context 'with well-known code: #{code}' do 33 | let(:code) { #{code} } 34 | it { is_expected.to eq #{reason.inspect} } 35 | it { is_expected.to be_frozen } 36 | end 37 | RUBY 38 | end 39 | end 40 | 41 | context "with 1xx codes" do 42 | subject { (100...200).map { |code| described_class.new code } } 43 | 44 | it "is #informational?" do 45 | expect(subject).to all(satisfy(&:informational?)) 46 | end 47 | 48 | it "is not #success?" do 49 | expect(subject).to all(satisfy { |status| !status.success? }) 50 | end 51 | 52 | it "is not #redirect?" do 53 | expect(subject).to all(satisfy { |status| !status.redirect? }) 54 | end 55 | 56 | it "is not #client_error?" do 57 | expect(subject).to all(satisfy { |status| !status.client_error? }) 58 | end 59 | 60 | it "is not #server_error?" do 61 | expect(subject).to all(satisfy { |status| !status.server_error? }) 62 | end 63 | end 64 | 65 | context "with 2xx codes" do 66 | subject { (200...300).map { |code| described_class.new code } } 67 | 68 | it "is not #informational?" do 69 | expect(subject).to all(satisfy { |status| !status.informational? }) 70 | end 71 | 72 | it "is #success?" do 73 | expect(subject).to all(satisfy(&:success?)) 74 | end 75 | 76 | it "is not #redirect?" do 77 | expect(subject).to all(satisfy { |status| !status.redirect? }) 78 | end 79 | 80 | it "is not #client_error?" do 81 | expect(subject).to all(satisfy { |status| !status.client_error? }) 82 | end 83 | 84 | it "is not #server_error?" do 85 | expect(subject).to all(satisfy { |status| !status.server_error? }) 86 | end 87 | end 88 | 89 | context "with 3xx codes" do 90 | subject { (300...400).map { |code| described_class.new code } } 91 | 92 | it "is not #informational?" do 93 | expect(subject).to all(satisfy { |status| !status.informational? }) 94 | end 95 | 96 | it "is not #success?" do 97 | expect(subject).to all(satisfy { |status| !status.success? }) 98 | end 99 | 100 | it "is #redirect?" do 101 | expect(subject).to all(satisfy(&:redirect?)) 102 | end 103 | 104 | it "is not #client_error?" do 105 | expect(subject).to all(satisfy { |status| !status.client_error? }) 106 | end 107 | 108 | it "is not #server_error?" do 109 | expect(subject).to all(satisfy { |status| !status.server_error? }) 110 | end 111 | end 112 | 113 | context "with 4xx codes" do 114 | subject { (400...500).map { |code| described_class.new code } } 115 | 116 | it "is not #informational?" do 117 | expect(subject).to all(satisfy { |status| !status.informational? }) 118 | end 119 | 120 | it "is not #success?" do 121 | expect(subject).to all(satisfy { |status| !status.success? }) 122 | end 123 | 124 | it "is not #redirect?" do 125 | expect(subject).to all(satisfy { |status| !status.redirect? }) 126 | end 127 | 128 | it "is #client_error?" do 129 | expect(subject).to all(satisfy(&:client_error?)) 130 | end 131 | 132 | it "is not #server_error?" do 133 | expect(subject).to all(satisfy { |status| !status.server_error? }) 134 | end 135 | end 136 | 137 | context "with 5xx codes" do 138 | subject { (500...600).map { |code| described_class.new code } } 139 | 140 | it "is not #informational?" do 141 | expect(subject).to all(satisfy { |status| !status.informational? }) 142 | end 143 | 144 | it "is not #success?" do 145 | expect(subject).to all(satisfy { |status| !status.success? }) 146 | end 147 | 148 | it "is not #redirect?" do 149 | expect(subject).to all(satisfy { |status| !status.redirect? }) 150 | end 151 | 152 | it "is not #client_error?" do 153 | expect(subject).to all(satisfy { |status| !status.client_error? }) 154 | end 155 | 156 | it "is #server_error?" do 157 | expect(subject).to all(satisfy(&:server_error?)) 158 | end 159 | end 160 | 161 | describe "#to_sym" do 162 | subject { described_class.new(code).to_sym } 163 | 164 | context "with unknown code" do 165 | let(:code) { 1024 } 166 | 167 | it { is_expected.to be_nil } 168 | end 169 | 170 | described_class::SYMBOLS.each do |code, symbol| 171 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 172 | context 'with well-known code: #{code}' do 173 | let(:code) { #{code} } 174 | it { is_expected.to be #{symbol.inspect} } 175 | end 176 | RUBY 177 | end 178 | end 179 | 180 | describe "#inspect" do 181 | it "returns quoted code and reason phrase" do 182 | status = described_class.new 200 183 | expect(status.inspect).to eq "#" 184 | end 185 | end 186 | 187 | # testing edge cases only 188 | describe "::SYMBOLS" do 189 | subject { described_class::SYMBOLS } 190 | 191 | # "OK" 192 | its([200]) { is_expected.to be :ok } 193 | 194 | # "Bad Request" 195 | its([400]) { is_expected.to be :bad_request } 196 | end 197 | 198 | described_class::SYMBOLS.each do |code, symbol| 199 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 200 | describe '##{symbol}?' do 201 | subject { status.#{symbol}? } 202 | 203 | context 'when code is #{code}' do 204 | let(:status) { described_class.new #{code} } 205 | it { is_expected.to be true } 206 | end 207 | 208 | context 'when code is higher than #{code}' do 209 | let(:status) { described_class.new #{code + 1} } 210 | it { is_expected.to be false } 211 | end 212 | 213 | context 'when code is lower than #{code}' do 214 | let(:status) { described_class.new #{code - 1} } 215 | it { is_expected.to be false } 216 | end 217 | end 218 | RUBY 219 | end 220 | 221 | describe ".coerce" do 222 | context "with String" do 223 | it "coerces reasons" do 224 | expect(described_class.coerce("Bad request")).to eq described_class.new 400 225 | end 226 | 227 | it "fails when reason is unknown" do 228 | expect { described_class.coerce "foobar" }.to raise_error HTTP::Error 229 | end 230 | end 231 | 232 | context "with Symbol" do 233 | it "coerces symbolized reasons" do 234 | expect(described_class.coerce(:bad_request)).to eq described_class.new 400 235 | end 236 | 237 | it "fails when symbolized reason is unknown" do 238 | expect { described_class.coerce(:foobar) }.to raise_error HTTP::Error 239 | end 240 | end 241 | 242 | context "with Numeric" do 243 | it "coerces as Fixnum code" do 244 | expect(described_class.coerce(200.1)).to eq described_class.new 200 245 | end 246 | end 247 | 248 | it "fails if coercion failed" do 249 | expect { described_class.coerce(true) }.to raise_error HTTP::Error 250 | end 251 | 252 | it "is aliased as `.[]`" do 253 | expect(described_class.method(:coerce)).to eq described_class.method :[] 254 | end 255 | end 256 | end 257 | -------------------------------------------------------------------------------- /spec/lib/http/retriable/delay_calculator_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Retriable::DelayCalculator do 4 | let(:response) do 5 | HTTP::Response.new( 6 | status: 200, 7 | version: "1.1", 8 | headers: {}, 9 | body: "Hello world!", 10 | request: HTTP::Request.new(verb: :get, uri: "http://example.com") 11 | ) 12 | end 13 | 14 | def call_delay(iterations, **options) 15 | described_class.new(options).call(iterations, response) 16 | end 17 | 18 | def call_retry_header(value, **options) 19 | response.headers["Retry-After"] = value 20 | described_class.new(options).call(rand(1...100), response) 21 | end 22 | 23 | it "prevents negative sleep time" do 24 | expect(call_delay(20, delay: -20)).to eq 0 25 | end 26 | 27 | it "backs off exponentially" do 28 | expect(call_delay(1)).to be_between 0, 1 29 | expect(call_delay(2)).to be_between 1, 2 30 | expect(call_delay(3)).to be_between 3, 4 31 | expect(call_delay(4)).to be_between 7, 8 32 | expect(call_delay(5)).to be_between 15, 16 33 | end 34 | 35 | it "can have a maximum wait time" do 36 | expect(call_delay(1, max_delay: 5)).to be_between 0, 1 37 | expect(call_delay(5, max_delay: 5)).to eq 5 38 | end 39 | 40 | it "respects Retry-After headers as integer" do 41 | delay_time = rand(6...2500) 42 | header_value = delay_time.to_s 43 | expect(call_retry_header(header_value)).to eq delay_time 44 | expect(call_retry_header(header_value, max_delay: 5)).to eq 5 45 | end 46 | 47 | it "respects Retry-After headers as rfc2822 timestamp" do 48 | delay_time = rand(6...2500) 49 | header_value = (Time.now.gmtime + delay_time).to_datetime.rfc2822.sub("+0000", "GMT") 50 | expect(call_retry_header(header_value)).to be_within(1).of(delay_time) 51 | expect(call_retry_header(header_value, max_delay: 5)).to eq 5 52 | end 53 | 54 | it "respects Retry-After headers as rfc2822 timestamp in the past" do 55 | delay_time = rand(6...2500) 56 | header_value = (Time.now.gmtime - delay_time).to_datetime.rfc2822.sub("+0000", "GMT") 57 | expect(call_retry_header(header_value)).to eq 0 58 | end 59 | 60 | it "does not error on invalid Retry-After header" do 61 | [ # invalid strings 62 | "This is a string with a number 5 in it", 63 | "8 Eight is the first digit in this string", 64 | "This is a string with a #{Time.now.gmtime.to_datetime.rfc2822} timestamp in it" 65 | ].each do |header_value| 66 | expect(call_retry_header(header_value)).to eq 0 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /spec/lib/http/uri/normalizer_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::URI::NORMALIZER do 4 | describe "scheme" do 5 | it "lower-cases scheme" do 6 | expect(HTTP::URI::NORMALIZER.call("HttP://example.com").scheme).to eq "http" 7 | end 8 | end 9 | 10 | describe "hostname" do 11 | it "lower-cases hostname" do 12 | expect(HTTP::URI::NORMALIZER.call("http://EXAMPLE.com").host).to eq "example.com" 13 | end 14 | 15 | it "decodes percent-encoded hostname" do 16 | expect(HTTP::URI::NORMALIZER.call("http://ex%61mple.com").host).to eq "example.com" 17 | end 18 | 19 | it "removes trailing period in hostname" do 20 | expect(HTTP::URI::NORMALIZER.call("http://example.com.").host).to eq "example.com" 21 | end 22 | 23 | it "IDN-encodes non-ASCII hostname" do 24 | expect(HTTP::URI::NORMALIZER.call("http://exämple.com").host).to eq "xn--exmple-cua.com" 25 | end 26 | end 27 | 28 | describe "path" do 29 | it "ensures path is not empty" do 30 | expect(HTTP::URI::NORMALIZER.call("http://example.com").path).to eq "/" 31 | end 32 | 33 | it "preserves double slashes in path" do 34 | expect(HTTP::URI::NORMALIZER.call("http://example.com//a///b").path).to eq "//a///b" 35 | end 36 | 37 | it "resolves single-dot segments in path" do 38 | expect(HTTP::URI::NORMALIZER.call("http://example.com/a/./b").path).to eq "/a/b" 39 | end 40 | 41 | it "resolves double-dot segments in path" do 42 | expect(HTTP::URI::NORMALIZER.call("http://example.com/a/b/../c").path).to eq "/a/c" 43 | end 44 | 45 | it "resolves leading double-dot segments in path" do 46 | expect(HTTP::URI::NORMALIZER.call("http://example.com/../a/b").path).to eq "/a/b" 47 | end 48 | 49 | it "percent-encodes control characters in path" do 50 | expect(HTTP::URI::NORMALIZER.call("http://example.com/\x00\x7F\n").path).to eq "/%00%7F%0A" 51 | end 52 | 53 | it "percent-encodes space in path" do 54 | expect(HTTP::URI::NORMALIZER.call("http://example.com/a b").path).to eq "/a%20b" 55 | end 56 | 57 | it "percent-encodes non-ASCII characters in path" do 58 | expect(HTTP::URI::NORMALIZER.call("http://example.com/キョ").path).to eq "/%E3%82%AD%E3%83%A7" 59 | end 60 | 61 | it "does not percent-encode non-special characters in path" do 62 | expect(HTTP::URI::NORMALIZER.call("http://example.com/~.-_!$&()*,;=:@{}").path).to eq "/~.-_!$&()*,;=:@{}" 63 | end 64 | 65 | it "preserves escape sequences in path" do 66 | expect(HTTP::URI::NORMALIZER.call("http://example.com/%41").path).to eq "/%41" 67 | end 68 | end 69 | 70 | describe "query" do 71 | it "allows no query" do 72 | expect(HTTP::URI::NORMALIZER.call("http://example.com").query).to be_nil 73 | end 74 | 75 | it "percent-encodes control characters in query" do 76 | expect(HTTP::URI::NORMALIZER.call("http://example.com/?\x00\x7F\n").query).to eq "%00%7F%0A" 77 | end 78 | 79 | it "percent-encodes space in query" do 80 | expect(HTTP::URI::NORMALIZER.call("http://example.com/?a b").query).to eq "a%20b" 81 | end 82 | 83 | it "percent-encodes non-ASCII characters in query" do 84 | expect(HTTP::URI::NORMALIZER.call("http://example.com?キョ").query).to eq "%E3%82%AD%E3%83%A7" 85 | end 86 | 87 | it "does not percent-encode non-special characters in query" do 88 | expect(HTTP::URI::NORMALIZER.call("http://example.com/?~.-_!$&()*,;=:@{}?").query).to eq "~.-_!$&()*,;=:@{}?" 89 | end 90 | 91 | it "preserves escape sequences in query" do 92 | expect(HTTP::URI::NORMALIZER.call("http://example.com/?%41").query).to eq "%41" 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /spec/lib/http/uri_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::URI do 4 | subject(:ipv6_uri) { described_class.parse(example_ipv6_uri_string) } 5 | 6 | let(:example_ipv6_address) { "2606:2800:220:1:248:1893:25c8:1946" } 7 | 8 | let(:example_http_uri_string) { "http://example.com" } 9 | let(:example_https_uri_string) { "https://example.com" } 10 | let(:example_ipv6_uri_string) { "https://[#{example_ipv6_address}]" } 11 | 12 | let(:http_uri) { described_class.parse(example_http_uri_string) } 13 | 14 | let(:https_uri) { described_class.parse(example_https_uri_string) } 15 | 16 | it "knows URI schemes" do 17 | expect(http_uri.scheme).to eq "http" 18 | expect(https_uri.scheme).to eq "https" 19 | end 20 | 21 | it "sets default ports for HTTP URIs" do 22 | expect(http_uri.port).to eq 80 23 | end 24 | 25 | it "sets default ports for HTTPS URIs" do 26 | expect(https_uri.port).to eq 443 27 | end 28 | 29 | describe "#host" do 30 | it "strips brackets from IPv6 addresses" do 31 | expect(ipv6_uri.host).to eq("2606:2800:220:1:248:1893:25c8:1946") 32 | end 33 | end 34 | 35 | describe "#normalized_host" do 36 | it "strips brackets from IPv6 addresses" do 37 | expect(ipv6_uri.normalized_host).to eq("2606:2800:220:1:248:1893:25c8:1946") 38 | end 39 | end 40 | 41 | describe "#host=" do 42 | it "updates cached values for #host and #normalized_host" do 43 | expect(http_uri.host).to eq("example.com") 44 | expect(http_uri.normalized_host).to eq("example.com") 45 | 46 | http_uri.host = "[#{example_ipv6_address}]" 47 | 48 | expect(http_uri.host).to eq(example_ipv6_address) 49 | expect(http_uri.normalized_host).to eq(example_ipv6_address) 50 | end 51 | 52 | it "ensures IPv6 addresses are bracketed in the inner Addressable::URI" do 53 | expect(http_uri.host).to eq("example.com") 54 | expect(http_uri.normalized_host).to eq("example.com") 55 | 56 | http_uri.host = example_ipv6_address 57 | 58 | expect(http_uri.host).to eq(example_ipv6_address) 59 | expect(http_uri.normalized_host).to eq(example_ipv6_address) 60 | expect(http_uri.instance_variable_get(:@uri).host).to eq("[#{example_ipv6_address}]") 61 | end 62 | end 63 | 64 | describe "#dup" do 65 | it "doesn't share internal value between duplicates" do 66 | duplicated_uri = http_uri.dup 67 | duplicated_uri.host = "example.org" 68 | 69 | expect(duplicated_uri.to_s).to eq("http://example.org") 70 | expect(http_uri.to_s).to eq("http://example.com") 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/regression_specs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe "Regression testing" do 6 | describe "#248" do 7 | it "does not fail with github" do 8 | github_uri = "http://github.com/" 9 | expect { HTTP.get(github_uri).to_s }.not_to raise_error 10 | end 11 | 12 | it "does not fail with googleapis" do 13 | google_uri = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" 14 | expect { HTTP.get(google_uri).to_s }.not_to raise_error 15 | end 16 | end 17 | 18 | describe "#422" do 19 | it "reads body when 200 OK response contains Upgrade header" do 20 | res = HTTP.get("https://httpbin.org/response-headers?Upgrade=h2,h2c") 21 | expect(res.parse(:json)).to include("Upgrade" => "h2,h2c") 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "./support/simplecov" 4 | require_relative "./support/fuubar" unless ENV["CI"] 5 | 6 | require "http" 7 | require "rspec/its" 8 | require "rspec/memory" 9 | require "support/capture_warning" 10 | require "support/fakeio" 11 | 12 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 13 | RSpec.configure do |config| 14 | config.expect_with :rspec do |expectations| 15 | # This option will default to `true` in RSpec 4. It makes the `description` 16 | # and `failure_message` of custom matchers include text for helper methods 17 | # defined using `chain`, e.g.: 18 | # be_bigger_than(2).and_smaller_than(4).description 19 | # # => "be bigger than 2 and smaller than 4" 20 | # ...rather than: 21 | # # => "be bigger than 2" 22 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 23 | end 24 | 25 | config.mock_with :rspec do |mocks| 26 | # Prevents you from mocking or stubbing a method that does not exist on 27 | # a real object. This is generally recommended, and will default to 28 | # `true` in RSpec 4. 29 | mocks.verify_partial_doubles = true 30 | end 31 | 32 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 33 | # have no way to turn it off -- the option exists only for backwards 34 | # compatibility in RSpec 3). It causes shared context metadata to be 35 | # inherited by the metadata hash of host groups and examples, rather than 36 | # triggering implicit auto-inclusion in groups with matching metadata. 37 | config.shared_context_metadata_behavior = :apply_to_host_groups 38 | 39 | # These two settings work together to allow you to limit a spec run 40 | # to individual examples or groups you care about by tagging them with 41 | # `:focus` metadata. When nothing is tagged with `:focus`, all examples 42 | # get run. 43 | config.filter_run :focus 44 | config.filter_run_excluding :flaky if defined?(JRUBY_VERSION) && ENV["CI"] 45 | config.run_all_when_everything_filtered = true 46 | 47 | # This setting enables warnings. It's recommended, but in some cases may 48 | # be too noisy due to issues in dependencies. 49 | config.warnings = 0 == ENV["GUARD_RSPEC"].to_i 50 | 51 | # Allows RSpec to persist some state between runs in order to support 52 | # the `--only-failures` and `--next-failure` CLI options. We recommend 53 | # you configure your source control system to ignore this file. 54 | config.example_status_persistence_file_path = "spec/examples.txt" 55 | 56 | # Limits the available syntax to the non-monkey patched syntax that is 57 | # recommended. For more details, see: 58 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 59 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 60 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 61 | config.disable_monkey_patching! 62 | 63 | # Many RSpec users commonly either run the entire suite or an individual 64 | # file, and it's useful to allow more verbose output when running an 65 | # individual spec file. 66 | if config.files_to_run.one? 67 | # Use the documentation formatter for detailed output, 68 | # unless a formatter has already been configured 69 | # (e.g. via a command-line flag). 70 | config.default_formatter = "doc" 71 | end 72 | 73 | # Print the 10 slowest examples and example groups at the 74 | # end of the spec run, to help surface which specs are running 75 | # particularly slow. 76 | config.profile_examples = 10 77 | 78 | # Run specs in random order to surface order dependencies. If you find an 79 | # order dependency and want to debug it, you can fix the order by providing 80 | # the seed, which is printed after each run. 81 | # --seed 1234 82 | config.order = :random 83 | 84 | # Seed global randomization in this process using the `--seed` CLI option. 85 | # Setting this allows you to use `--seed` to deterministically reproduce 86 | # test failures related to randomization by passing the same `--seed` value 87 | # as the one that triggered the failure. 88 | Kernel.srand config.seed 89 | end 90 | -------------------------------------------------------------------------------- /spec/support/black_hole.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module BlackHole 4 | class << self 5 | def method_missing(*) 6 | self 7 | end 8 | 9 | def respond_to_missing?(*) 10 | true 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/support/capture_warning.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | def capture_warning 4 | old_stderr = $stderr 5 | $stderr = StringIO.new 6 | yield 7 | $stderr.string 8 | ensure 9 | $stderr = old_stderr 10 | end 11 | -------------------------------------------------------------------------------- /spec/support/dummy_server.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "webrick" 4 | require "webrick/ssl" 5 | 6 | require "support/black_hole" 7 | require "support/dummy_server/servlet" 8 | require "support/servers/config" 9 | require "support/servers/runner" 10 | require "support/ssl_helper" 11 | 12 | class DummyServer < WEBrick::HTTPServer 13 | include ServerConfig 14 | 15 | CONFIG = { 16 | BindAddress: "127.0.0.1", 17 | Port: 0, 18 | AccessLog: BlackHole, 19 | Logger: BlackHole 20 | }.freeze 21 | 22 | SSL_CONFIG = CONFIG.merge( 23 | SSLEnable: true, 24 | SSLStartImmediately: true 25 | ).freeze 26 | 27 | def initialize(options = {}) 28 | super(options[:ssl] ? SSL_CONFIG : CONFIG) 29 | @memo = {} 30 | mount("/", Servlet, @memo) 31 | end 32 | 33 | def endpoint 34 | "#{scheme}://#{addr}:#{port}" 35 | end 36 | 37 | def scheme 38 | config[:SSLEnable] ? "https" : "http" 39 | end 40 | 41 | def ssl_context 42 | @ssl_context ||= SSLHelper.server_context 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /spec/support/dummy_server/servlet.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # frozen_string_literal: true 3 | 4 | require "cgi" 5 | 6 | class DummyServer < WEBrick::HTTPServer 7 | class Servlet < WEBrick::HTTPServlet::AbstractServlet # rubocop:disable Metrics/ClassLength 8 | def self.sockets 9 | @sockets ||= [] 10 | end 11 | 12 | def not_found(req, res) 13 | res.status = 404 14 | res.body = "#{req.unparsed_uri} not found" 15 | end 16 | 17 | def self.handlers 18 | @handlers ||= {} 19 | end 20 | 21 | def initialize(server, memo) 22 | super(server) 23 | @memo = memo 24 | end 25 | 26 | %w[get post head].each do |method| 27 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 28 | def self.#{method}(path, &block) 29 | handlers["#{method}:\#{path}"] = block 30 | end 31 | 32 | def do_#{method.upcase}(req, res) 33 | handler = self.class.handlers["#{method}:\#{req.path}"] 34 | return instance_exec(req, res, &handler) if handler 35 | not_found(req, res) 36 | end 37 | RUBY 38 | end 39 | 40 | get "/" do |req, res| 41 | res.status = 200 42 | 43 | case req["Accept"] 44 | when "application/json" 45 | res["Content-Type"] = "application/json" 46 | res.body = '{"json": true}' 47 | else 48 | res["Content-Type"] = "text/html" 49 | res.body = "" 50 | end 51 | end 52 | 53 | get "/sleep" do |_, res| 54 | sleep 2 55 | 56 | res.status = 200 57 | res.body = "hello" 58 | end 59 | 60 | post "/sleep" do |_, res| 61 | sleep 2 62 | 63 | res.status = 200 64 | res.body = "hello" 65 | end 66 | 67 | ["", "/1", "/2"].each do |path| 68 | get "/socket#{path}" do |req, res| 69 | self.class.sockets << req.instance_variable_get(:@socket) 70 | res.status = 200 71 | res.body = req.instance_variable_get(:@socket).object_id.to_s 72 | end 73 | end 74 | 75 | get "/params" do |req, res| 76 | next not_found(req, res) unless "foo=bar" == req.query_string 77 | 78 | res.status = 200 79 | res.body = "Params!" 80 | end 81 | 82 | get "/multiple-params" do |req, res| 83 | params = CGI.parse req.query_string 84 | 85 | next not_found(req, res) unless {"foo" => ["bar"], "baz" => ["quux"]} == params 86 | 87 | res.status = 200 88 | res.body = "More Params!" 89 | end 90 | 91 | get "/proxy" do |_req, res| 92 | res.status = 200 93 | res.body = "Proxy!" 94 | end 95 | 96 | get "/not-found" do |_req, res| 97 | res.status = 404 98 | res.body = "not found" 99 | end 100 | 101 | get "/redirect-301" do |_req, res| 102 | res.status = 301 103 | res["Location"] = "http://#{@server.config[:BindAddress]}:#{@server.config[:Port]}/" 104 | end 105 | 106 | get "/redirect-302" do |_req, res| 107 | res.status = 302 108 | res["Location"] = "http://#{@server.config[:BindAddress]}:#{@server.config[:Port]}/" 109 | end 110 | 111 | post "/form" do |req, res| 112 | if "testing-form" == req.query["example"] 113 | res.status = 200 114 | res.body = "passed :)" 115 | else 116 | res.status = 400 117 | res.body = "invalid! >:E" 118 | end 119 | end 120 | 121 | post "/body" do |req, res| 122 | if "testing-body" == req.body 123 | res.status = 200 124 | res.body = "passed :)" 125 | else 126 | res.status = 400 127 | res.body = "invalid! >:E" 128 | end 129 | end 130 | 131 | head "/" do |_req, res| 132 | res.status = 200 133 | res["Content-Type"] = "text/html" 134 | end 135 | 136 | get "/bytes" do |_req, res| 137 | bytes = [80, 75, 3, 4, 20, 0, 0, 0, 8, 0, 123, 104, 169, 70, 99, 243, 243] 138 | res["Content-Type"] = "application/octet-stream" 139 | res.body = bytes.pack("c*") 140 | end 141 | 142 | get "/iso-8859-1" do |_req, res| 143 | res["Content-Type"] = "text/plain; charset=ISO-8859-1" 144 | res.body = "testæ".encode(Encoding::ISO8859_1) 145 | end 146 | 147 | get "/cookies" do |req, res| 148 | res["Set-Cookie"] = "foo=bar" 149 | res.body = req.cookies.map { |c| [c.name, c.value].join ": " }.join("\n") 150 | end 151 | 152 | post "/echo-body" do |req, res| 153 | res.status = 200 154 | res.body = req.body 155 | end 156 | 157 | get "/héllö-wörld".b do |_req, res| 158 | res.status = 200 159 | res.body = "hello world" 160 | end 161 | 162 | post "/encoded-body" do |req, res| 163 | res.status = 200 164 | 165 | res.body = case req["Accept-Encoding"] 166 | when "gzip" 167 | res["Content-Encoding"] = "gzip" 168 | StringIO.open do |out| 169 | Zlib::GzipWriter.wrap(out) do |gz| 170 | gz.write "#{req.body}-gzipped" 171 | gz.finish 172 | out.tap(&:rewind).read 173 | end 174 | end 175 | when "deflate" 176 | res["Content-Encoding"] = "deflate" 177 | Zlib::Deflate.deflate("#{req.body}-deflated") 178 | else 179 | "#{req.body}-raw" 180 | end 181 | end 182 | 183 | post "/no-content-204" do |req, res| 184 | res.status = 204 185 | res.body = "" 186 | 187 | case req["Accept-Encoding"] 188 | when "gzip" 189 | res["Content-Encoding"] = "gzip" 190 | when "deflate" 191 | res["Content-Encoding"] = "deflate" 192 | end 193 | end 194 | 195 | get "/retry-2" do |_req, res| 196 | @memo[:attempts] ||= 0 197 | @memo[:attempts] += 1 198 | 199 | res.body = "retried #{@memo[:attempts]}x" 200 | res.status = @memo[:attempts] == 2 ? 200 : 500 201 | end 202 | end 203 | end 204 | -------------------------------------------------------------------------------- /spec/support/fakeio.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "stringio" 4 | 5 | class FakeIO 6 | def initialize(content) 7 | @io = StringIO.new(content) 8 | end 9 | 10 | def string 11 | @io.string 12 | end 13 | 14 | def read(*args) 15 | @io.read(*args) 16 | end 17 | 18 | def size 19 | @io.size 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/support/fuubar.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "fuubar" 4 | 5 | RSpec.configure do |config| 6 | # Use Fuubar instafail-alike formatter, unless a formatter has already been 7 | # configured (e.g. via a command-line flag). 8 | config.default_formatter = "Fuubar" 9 | 10 | # Disable auto-refresh of the fuubar progress bar to avoid surprises during 11 | # debugiing. And simply because there's next to absolutely no point in having 12 | # this turned on. 13 | # 14 | # > By default fuubar will automatically refresh the bar (and therefore 15 | # > the ETA) every second. Unfortunately this doesn't play well with things 16 | # > like debuggers. When you're debugging, having a bar show up every second 17 | # > is undesireable. 18 | # 19 | # See: https://github.com/thekompanee/fuubar#disabling-auto-refresh 20 | config.fuubar_auto_refresh = false 21 | end 22 | -------------------------------------------------------------------------------- /spec/support/http_handling_shared.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.shared_context "HTTP handling" do 4 | context "without timeouts" do 5 | let(:options) { {timeout_class: HTTP::Timeout::Null, timeout_options: {}} } 6 | 7 | it "works" do 8 | expect(client.get(server.endpoint).body.to_s).to eq("") 9 | end 10 | end 11 | 12 | context "with a per operation timeout" do 13 | let(:response) { client.get(server.endpoint).body.to_s } 14 | 15 | let(:options) do 16 | { 17 | timeout_class: HTTP::Timeout::PerOperation, 18 | timeout_options: { 19 | connect_timeout: conn_timeout, 20 | read_timeout: read_timeout, 21 | write_timeout: write_timeout 22 | } 23 | } 24 | end 25 | let(:conn_timeout) { 1 } 26 | let(:read_timeout) { 1 } 27 | let(:write_timeout) { 1 } 28 | 29 | it "works" do 30 | expect(response).to eq("") 31 | end 32 | 33 | context "connection" do 34 | context "of 1" do 35 | let(:conn_timeout) { 1 } 36 | 37 | it "does not time out" do 38 | expect { response }.not_to raise_error 39 | end 40 | end 41 | end 42 | 43 | context "read" do 44 | context "of 0" do 45 | let(:read_timeout) { 0 } 46 | 47 | it "times out", :flaky do 48 | expect { response }.to raise_error(HTTP::TimeoutError, /Read/i) 49 | end 50 | end 51 | 52 | context "of 2.5" do 53 | let(:read_timeout) { 2.5 } 54 | 55 | it "does not time out", :flaky do 56 | expect { client.get("#{server.endpoint}/sleep").body.to_s }.not_to raise_error 57 | end 58 | end 59 | end 60 | end 61 | 62 | context "with a global timeout" do 63 | let(:options) do 64 | { 65 | timeout_class: HTTP::Timeout::Global, 66 | timeout_options: { 67 | global_timeout: global_timeout 68 | } 69 | } 70 | end 71 | let(:global_timeout) { 1 } 72 | 73 | let(:response) { client.get(server.endpoint).body.to_s } 74 | 75 | it "errors if connecting takes too long" do 76 | expect(TCPSocket).to receive(:open) do 77 | sleep 1.25 78 | end 79 | 80 | expect { response }.to raise_error(HTTP::ConnectTimeoutError, /execution/) 81 | end 82 | 83 | it "errors if reading takes too long" do 84 | expect { client.get("#{server.endpoint}/sleep").body.to_s }. 85 | to raise_error(HTTP::TimeoutError, /Timed out/) 86 | end 87 | 88 | context "it resets state when reusing connections" do 89 | let(:extra_options) { {persistent: server.endpoint} } 90 | 91 | let(:global_timeout) { 2.5 } 92 | 93 | it "does not timeout", :flaky do 94 | client.get("#{server.endpoint}/sleep").body.to_s 95 | client.get("#{server.endpoint}/sleep").body.to_s 96 | end 97 | end 98 | end 99 | 100 | describe "connection reuse" do 101 | let(:sockets_used) do 102 | [ 103 | client.get("#{server.endpoint}/socket/1").body.to_s, 104 | client.get("#{server.endpoint}/socket/2").body.to_s 105 | ] 106 | end 107 | 108 | context "when enabled" do 109 | let(:options) { {persistent: server.endpoint} } 110 | 111 | context "without a host" do 112 | it "infers host from persistent config" do 113 | expect(client.get("/").body.to_s).to eq("") 114 | end 115 | end 116 | 117 | it "re-uses the socket" do 118 | expect(sockets_used).not_to include("") 119 | expect(sockets_used.uniq.length).to eq(1) 120 | end 121 | 122 | context "on a mixed state" do 123 | it "re-opens the connection", :flaky do 124 | first_socket_id = client.get("#{server.endpoint}/socket/1").body.to_s 125 | 126 | client.instance_variable_set(:@state, :dirty) 127 | 128 | second_socket_id = client.get("#{server.endpoint}/socket/2").body.to_s 129 | 130 | expect(first_socket_id).not_to eq(second_socket_id) 131 | end 132 | end 133 | 134 | context "when trying to read a stale body" do 135 | it "errors" do 136 | client.get("#{server.endpoint}/not-found") 137 | expect { client.get(server.endpoint) }.to raise_error(HTTP::StateError, /Tried to send a request/) 138 | end 139 | end 140 | 141 | context "when reading a cached body" do 142 | it "succeeds" do 143 | first_res = client.get(server.endpoint) 144 | first_res.body.to_s 145 | 146 | second_res = client.get(server.endpoint) 147 | 148 | expect(first_res.body.to_s).to eq("") 149 | expect(second_res.body.to_s).to eq("") 150 | end 151 | end 152 | 153 | context "with a socket issue" do 154 | it "transparently reopens", :flaky do 155 | first_socket_id = client.get("#{server.endpoint}/socket").body.to_s 156 | expect(first_socket_id).not_to eq("") 157 | # Kill off the sockets we used 158 | # rubocop:disable Style/RescueModifier 159 | DummyServer::Servlet.sockets.each do |socket| 160 | socket.close rescue nil 161 | end 162 | DummyServer::Servlet.sockets.clear 163 | # rubocop:enable Style/RescueModifier 164 | 165 | # Should error because we tried to use a bad socket 166 | expect { client.get("#{server.endpoint}/socket").body.to_s }.to raise_error HTTP::ConnectionError 167 | 168 | # Should succeed since we create a new socket 169 | second_socket_id = client.get("#{server.endpoint}/socket").body.to_s 170 | expect(second_socket_id).not_to eq(first_socket_id) 171 | end 172 | end 173 | 174 | context "with a change in host" do 175 | it "errors" do 176 | expect { client.get("https://invalid.com/socket") }.to raise_error(/Persistence is enabled/i) 177 | end 178 | end 179 | end 180 | 181 | context "when disabled" do 182 | let(:options) { {} } 183 | 184 | it "opens new sockets", :flaky do 185 | expect(sockets_used).not_to include("") 186 | expect(sockets_used.uniq.length).to eq(2) 187 | end 188 | end 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /spec/support/proxy_server.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "webrick/httpproxy" 4 | 5 | require "support/black_hole" 6 | require "support/servers/config" 7 | require "support/servers/runner" 8 | 9 | class ProxyServer < WEBrick::HTTPProxyServer 10 | include ServerConfig 11 | 12 | CONFIG = { 13 | BindAddress: "127.0.0.1", 14 | Port: 0, 15 | AccessLog: BlackHole, 16 | Logger: BlackHole, 17 | RequestCallback: proc { |_, res| res["X-PROXIED"] = true } 18 | }.freeze 19 | 20 | def initialize 21 | super(CONFIG) 22 | end 23 | end 24 | 25 | class AuthProxyServer < WEBrick::HTTPProxyServer 26 | include ServerConfig 27 | 28 | AUTHENTICATOR = proc do |req, res| 29 | WEBrick::HTTPAuth.proxy_basic_auth(req, res, "proxy") do |user, pass| 30 | user == "username" && pass == "password" 31 | end 32 | end 33 | 34 | CONFIG = ProxyServer::CONFIG.merge ProxyAuthProc: AUTHENTICATOR 35 | 36 | def initialize 37 | super(CONFIG) 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/support/servers/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ServerConfig 4 | def addr 5 | config[:BindAddress] 6 | end 7 | 8 | def port 9 | config[:Port] 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/support/servers/runner.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ServerRunner 4 | def run_server(name) 5 | let! name do 6 | server = yield 7 | 8 | Thread.new { server.start } 9 | 10 | server 11 | end 12 | 13 | after do 14 | send(name).shutdown 15 | end 16 | end 17 | end 18 | 19 | RSpec.configure { |c| c.extend ServerRunner } 20 | -------------------------------------------------------------------------------- /spec/support/simplecov.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "simplecov" 4 | 5 | if ENV["CI"] 6 | require "simplecov-lcov" 7 | 8 | SimpleCov::Formatter::LcovFormatter.config do |config| 9 | config.report_with_single_file = true 10 | config.lcov_file_name = "lcov.info" 11 | end 12 | 13 | SimpleCov.formatter = SimpleCov::Formatter::LcovFormatter 14 | end 15 | 16 | SimpleCov.start do 17 | add_filter "/spec/" 18 | minimum_coverage 80 19 | end 20 | -------------------------------------------------------------------------------- /spec/support/ssl_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "pathname" 4 | 5 | require "certificate_authority" 6 | 7 | module SSLHelper 8 | CERTS_PATH = Pathname.new File.expand_path("../../tmp/certs", __dir__) 9 | 10 | class RootCertificate < ::CertificateAuthority::Certificate 11 | EXTENSIONS = {"keyUsage" => {"usage" => %w[critical keyCertSign]}}.freeze 12 | 13 | def initialize 14 | super() 15 | 16 | subject.common_name = "honestachmed.com" 17 | serial_number.number = 1 18 | key_material.generate_key 19 | 20 | self.signing_entity = true 21 | 22 | sign!("extensions" => EXTENSIONS) 23 | end 24 | 25 | def file 26 | return @file if defined? @file 27 | 28 | CERTS_PATH.mkpath 29 | 30 | cert_file = CERTS_PATH.join("ca.crt") 31 | cert_file.open("w") { |io| io << to_pem } 32 | 33 | @file = cert_file.to_s 34 | end 35 | end 36 | 37 | class ChildCertificate < ::CertificateAuthority::Certificate 38 | def initialize(parent) 39 | super() 40 | 41 | subject.common_name = "127.0.0.1" 42 | serial_number.number = 1 43 | 44 | key_material.generate_key 45 | 46 | self.parent = parent 47 | 48 | sign! 49 | end 50 | 51 | def cert 52 | OpenSSL::X509::Certificate.new to_pem 53 | end 54 | 55 | def key 56 | OpenSSL::PKey::RSA.new key_material.private_key.to_pem 57 | end 58 | end 59 | 60 | class << self 61 | def server_context 62 | context = OpenSSL::SSL::SSLContext.new 63 | 64 | context.verify_mode = OpenSSL::SSL::VERIFY_PEER 65 | context.key = server_cert.key 66 | context.cert = server_cert.cert 67 | context.ca_file = ca.file 68 | 69 | context 70 | end 71 | 72 | def client_context 73 | context = OpenSSL::SSL::SSLContext.new 74 | 75 | context.options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options] 76 | context.verify_mode = OpenSSL::SSL::VERIFY_PEER 77 | context.key = client_cert.key 78 | context.cert = client_cert.cert 79 | context.ca_file = ca.file 80 | 81 | context 82 | end 83 | 84 | def client_params 85 | { 86 | key: client_cert.key, 87 | cert: client_cert.cert, 88 | ca_file: ca.file 89 | } 90 | end 91 | 92 | %w[server client].each do |side| 93 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 94 | def #{side}_cert 95 | @#{side}_cert ||= ChildCertificate.new ca 96 | end 97 | RUBY 98 | end 99 | 100 | def ca 101 | @ca ||= RootCertificate.new 102 | end 103 | end 104 | end 105 | --------------------------------------------------------------------------------