├── .rspec ├── .yardopts ├── logo.png ├── lib ├── http │ ├── version.rb │ ├── features │ │ ├── normalize_uri.rb │ │ ├── auto_inflate.rb │ │ ├── logging.rb │ │ ├── instrumentation.rb │ │ └── auto_deflate.rb │ ├── feature.rb │ ├── mime_type │ │ ├── json.rb │ │ └── adapter.rb │ ├── response │ │ ├── inflater.rb │ │ ├── body.rb │ │ ├── status │ │ │ └── reasons.rb │ │ ├── parser.rb │ │ └── status.rb │ ├── errors.rb │ ├── content_type.rb │ ├── headers │ │ ├── mixin.rb │ │ └── known.rb │ ├── mime_type.rb │ ├── timeout │ │ ├── null.rb │ │ ├── per_operation.rb │ │ └── global.rb │ ├── request │ │ ├── body.rb │ │ └── writer.rb │ ├── uri.rb │ ├── redirector.rb │ ├── options.rb │ ├── response.rb │ ├── client.rb │ ├── connection.rb │ ├── headers.rb │ ├── chainable.rb │ └── request.rb └── http.rb ├── SECURITY.md ├── .gitignore ├── .rubocop ├── layout.yml └── style.yml ├── spec ├── support │ ├── servers │ │ ├── config.rb │ │ └── runner.rb │ ├── capture_warning.rb │ ├── black_hole.rb │ ├── fakeio.rb │ ├── simplecov.rb │ ├── fuubar.rb │ ├── proxy_server.rb │ ├── dummy_server.rb │ ├── ssl_helper.rb │ ├── dummy_server │ │ └── servlet.rb │ └── http_handling_shared.rb ├── lib │ └── http │ │ ├── options_spec.rb │ │ ├── options │ │ ├── body_spec.rb │ │ ├── form_spec.rb │ │ ├── json_spec.rb │ │ ├── headers_spec.rb │ │ ├── proxy_spec.rb │ │ ├── new_spec.rb │ │ ├── features_spec.rb │ │ └── merge_spec.rb │ │ ├── headers │ │ └── mixin_spec.rb │ │ ├── uri_spec.rb │ │ ├── features │ │ ├── instrumentation_spec.rb │ │ ├── logging_spec.rb │ │ ├── auto_deflate_spec.rb │ │ └── auto_inflate_spec.rb │ │ ├── content_type_spec.rb │ │ ├── response │ │ ├── parser_spec.rb │ │ ├── body_spec.rb │ │ └── status_spec.rb │ │ ├── connection_spec.rb │ │ ├── request │ │ ├── writer_spec.rb │ │ └── body_spec.rb │ │ ├── request_spec.rb │ │ └── response_spec.rb ├── regression_specs.rb └── spec_helper.rb ├── .rubocop.yml ├── Guardfile ├── CONTRIBUTING.md ├── Gemfile ├── LICENSE.txt ├── http.gemspec ├── Rakefile ├── .github └── workflows │ └── ci.yml ├── README.md └── .rubocop_todo.yml /.rspec: -------------------------------------------------------------------------------- 1 | --require spec_helper 2 | -------------------------------------------------------------------------------- /.yardopts: -------------------------------------------------------------------------------- 1 | --markup-provider=kramdown 2 | --markup=markdown 3 | -------------------------------------------------------------------------------- /logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/feedbin/http/main/logo.png -------------------------------------------------------------------------------- /lib/http/version.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | VERSION = "5.1.0" 5 | end 6 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Reporting a Vulnerability 4 | 5 | Please report security issues to `bascule@gmail.com` -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | .config 3 | .rvmrc 4 | .yardoc 5 | InstalledFiles 6 | _yardoc 7 | 8 | .bundle 9 | .ruby-version 10 | doc 11 | coverage 12 | pkg 13 | spec/examples.txt 14 | tmp 15 | Gemfile.lock 16 | -------------------------------------------------------------------------------- /.rubocop/layout.yml: -------------------------------------------------------------------------------- 1 | Layout/DotPosition: 2 | Enabled: true 3 | EnforcedStyle: leading 4 | 5 | Layout/HashAlignment: 6 | Enabled: true 7 | EnforcedColonStyle: table 8 | EnforcedHashRocketStyle: table 9 | -------------------------------------------------------------------------------- /spec/support/servers/config.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ServerConfig 4 | def addr 5 | config[:BindAddress] 6 | end 7 | 8 | def port 9 | config[:Port] 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/support/capture_warning.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | def capture_warning 4 | old_stderr = $stderr 5 | $stderr = StringIO.new 6 | yield 7 | $stderr.string 8 | ensure 9 | $stderr = old_stderr 10 | end 11 | -------------------------------------------------------------------------------- /.rubocop.yml: -------------------------------------------------------------------------------- 1 | inherit_from: 2 | - .rubocop_todo.yml 3 | - .rubocop/layout.yml 4 | - .rubocop/style.yml 5 | 6 | AllCops: 7 | DefaultFormatter: fuubar 8 | DisplayCopNames: true 9 | NewCops: enable 10 | TargetRubyVersion: 2.6 11 | -------------------------------------------------------------------------------- /spec/support/black_hole.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module BlackHole 4 | class << self 5 | def method_missing(*) 6 | self 7 | end 8 | 9 | def respond_to_missing?(*) 10 | true 11 | end 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/support/fakeio.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "stringio" 4 | 5 | class FakeIO 6 | def initialize(content) 7 | @io = StringIO.new(content) 8 | end 9 | 10 | def string 11 | @io.string 12 | end 13 | 14 | def read(*args) 15 | @io.read(*args) 16 | end 17 | 18 | def size 19 | @io.size 20 | end 21 | end 22 | -------------------------------------------------------------------------------- /spec/lib/http/options_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options do 4 | subject { described_class.new(:response => :body) } 5 | 6 | it "has reader methods for attributes" do 7 | expect(subject.response).to eq(:body) 8 | end 9 | 10 | it "coerces to a Hash" do 11 | expect(subject.to_hash).to be_a(Hash) 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /spec/support/servers/runner.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module ServerRunner 4 | def run_server(name) 5 | let! name do 6 | server = yield 7 | 8 | Thread.new { server.start } 9 | 10 | server 11 | end 12 | 13 | after do 14 | send(name).shutdown 15 | end 16 | end 17 | end 18 | 19 | RSpec.configure { |c| c.extend ServerRunner } 20 | -------------------------------------------------------------------------------- /spec/lib/http/options/body_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "body" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to nil" do 7 | expect(opts.body).to be nil 8 | end 9 | 10 | it "may be specified with with_body" do 11 | opts2 = opts.with_body("foo") 12 | expect(opts.body).to be nil 13 | expect(opts2.body).to eq("foo") 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /lib/http/features/normalize_uri.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/uri" 4 | 5 | module HTTP 6 | module Features 7 | class NormalizeUri < Feature 8 | attr_reader :normalizer 9 | 10 | def initialize(normalizer: HTTP::URI::NORMALIZER) 11 | @normalizer = normalizer 12 | end 13 | 14 | HTTP::Options.register_feature(:normalize_uri, self) 15 | end 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /spec/lib/http/options/form_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "form" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to nil" do 7 | expect(opts.form).to be nil 8 | end 9 | 10 | it "may be specified with with_form_data" do 11 | opts2 = opts.with_form(:foo => 42) 12 | expect(opts.form).to be nil 13 | expect(opts2.form).to eq(:foo => 42) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/lib/http/options/json_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "json" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to nil" do 7 | expect(opts.json).to be nil 8 | end 9 | 10 | it "may be specified with with_json data" do 11 | opts2 = opts.with_json(:foo => 42) 12 | expect(opts.json).to be nil 13 | expect(opts2.json).to eq(:foo => 42) 14 | end 15 | end 16 | -------------------------------------------------------------------------------- /spec/support/simplecov.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "simplecov" 4 | 5 | if ENV["CI"] 6 | require "simplecov-lcov" 7 | 8 | SimpleCov::Formatter::LcovFormatter.config do |config| 9 | config.report_with_single_file = true 10 | config.lcov_file_name = "lcov.info" 11 | end 12 | 13 | SimpleCov.formatter = SimpleCov::Formatter::LcovFormatter 14 | end 15 | 16 | SimpleCov.start do 17 | add_filter "/spec/" 18 | minimum_coverage 80 19 | end 20 | -------------------------------------------------------------------------------- /Guardfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # More info at https://github.com/guard/guard#readme 4 | 5 | guard :rspec, :cmd => "GUARD_RSPEC=1 bundle exec rspec --no-profile" do 6 | require "guard/rspec/dsl" 7 | dsl = Guard::RSpec::Dsl.new(self) 8 | 9 | # RSpec files 10 | rspec = dsl.rspec 11 | watch(rspec.spec_helper) { rspec.spec_dir } 12 | watch(rspec.spec_support) { rspec.spec_dir } 13 | watch(rspec.spec_files) 14 | 15 | # Ruby files 16 | ruby = dsl.ruby 17 | dsl.watch_spec_files_for(ruby.lib_files) 18 | end 19 | -------------------------------------------------------------------------------- /lib/http/feature.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Feature 5 | def initialize(opts = {}) 6 | @opts = opts 7 | end 8 | 9 | def wrap_request(request) 10 | request 11 | end 12 | 13 | def wrap_response(response) 14 | response 15 | end 16 | 17 | def on_error(request, error); end 18 | end 19 | end 20 | 21 | require "http/features/auto_inflate" 22 | require "http/features/auto_deflate" 23 | require "http/features/logging" 24 | require "http/features/instrumentation" 25 | require "http/features/normalize_uri" 26 | -------------------------------------------------------------------------------- /lib/http/mime_type/json.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "json" 4 | require "http/mime_type/adapter" 5 | 6 | module HTTP 7 | module MimeType 8 | # JSON encode/decode MIME type adapter 9 | class JSON < Adapter 10 | # Encodes object to JSON 11 | def encode(obj) 12 | return obj.to_json if obj.respond_to?(:to_json) 13 | 14 | ::JSON.dump obj 15 | end 16 | 17 | # Decodes JSON 18 | def decode(str) 19 | ::JSON.parse str 20 | end 21 | end 22 | 23 | register_adapter "application/json", JSON 24 | register_alias "application/json", :json 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /.rubocop/style.yml: -------------------------------------------------------------------------------- 1 | Style/Documentation: 2 | Enabled: false 3 | 4 | Style/DocumentDynamicEvalDefinition: 5 | Enabled: true 6 | Exclude: 7 | - 'spec/**/*.rb' 8 | 9 | Style/FormatStringToken: 10 | Enabled: true 11 | EnforcedStyle: unannotated 12 | 13 | Style/HashSyntax: 14 | Enabled: true 15 | EnforcedStyle: hash_rockets 16 | 17 | Style/OptionHash: 18 | Enabled: true 19 | 20 | Style/RescueStandardError: 21 | Enabled: true 22 | EnforcedStyle: implicit 23 | 24 | Style/StringLiterals: 25 | Enabled: true 26 | EnforcedStyle: double_quotes 27 | 28 | Style/WordArray: 29 | Enabled: true 30 | 31 | Style/YodaCondition: 32 | Enabled: false 33 | -------------------------------------------------------------------------------- /lib/http.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/errors" 4 | require "http/timeout/null" 5 | require "http/timeout/per_operation" 6 | require "http/timeout/global" 7 | require "http/chainable" 8 | require "http/client" 9 | require "http/connection" 10 | require "http/options" 11 | require "http/feature" 12 | require "http/request" 13 | require "http/request/writer" 14 | require "http/response" 15 | require "http/response/body" 16 | require "http/response/parser" 17 | 18 | # HTTP should be easy 19 | module HTTP 20 | extend Chainable 21 | 22 | class << self 23 | # HTTP[:accept => 'text/html'].get(...) 24 | alias [] headers 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /spec/lib/http/options/headers_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "headers" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to be empty" do 7 | expect(opts.headers).to be_empty 8 | end 9 | 10 | it "may be specified with with_headers" do 11 | opts2 = opts.with_headers(:accept => "json") 12 | expect(opts.headers).to be_empty 13 | expect(opts2.headers).to eq([%w[Accept json]]) 14 | end 15 | 16 | it "accepts any object that respond to :to_hash" do 17 | x = Struct.new(:to_hash).new("accept" => "json") 18 | expect(opts.with_headers(x).headers["accept"]).to eq("json") 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/http/response/inflater.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "zlib" 4 | 5 | module HTTP 6 | class Response 7 | class Inflater 8 | attr_reader :connection 9 | 10 | def initialize(connection) 11 | @connection = connection 12 | end 13 | 14 | def readpartial(*args) 15 | chunk = @connection.readpartial(*args) 16 | if chunk 17 | chunk = zstream.inflate(chunk) 18 | elsif !zstream.closed? 19 | zstream.finish if zstream.total_in.positive? 20 | zstream.close 21 | end 22 | chunk 23 | end 24 | 25 | private 26 | 27 | def zstream 28 | @zstream ||= Zlib::Inflate.new(32 + Zlib::MAX_WBITS) 29 | end 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /lib/http/errors.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | # Generic error 5 | class Error < StandardError; end 6 | 7 | # Generic Connection error 8 | class ConnectionError < Error; end 9 | 10 | # Generic Request error 11 | class RequestError < Error; end 12 | 13 | # Generic Response error 14 | class ResponseError < Error; end 15 | 16 | # Requested to do something when we're in the wrong state 17 | class StateError < ResponseError; end 18 | 19 | # Generic Timeout error 20 | class TimeoutError < Error; end 21 | 22 | # Timeout when first establishing the conncetion 23 | class ConnectTimeoutError < TimeoutError; end 24 | 25 | # Header value is of unexpected format (similar to Net::HTTPHeaderSyntaxError) 26 | class HeaderError < Error; end 27 | end 28 | -------------------------------------------------------------------------------- /lib/http/mime_type/adapter.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | require "singleton" 5 | 6 | module HTTP 7 | module MimeType 8 | # Base encode/decode MIME type adapter 9 | class Adapter 10 | include Singleton 11 | 12 | class << self 13 | extend Forwardable 14 | def_delegators :instance, :encode, :decode 15 | end 16 | 17 | # rubocop:disable Style/DocumentDynamicEvalDefinition 18 | %w[encode decode].each do |operation| 19 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 20 | def #{operation}(*) 21 | fail Error, "\#{self.class} does not supports ##{operation}" 22 | end 23 | RUBY 24 | end 25 | # rubocop:enable Style/DocumentDynamicEvalDefinition 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/regression_specs.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "spec_helper" 4 | 5 | RSpec.describe "Regression testing" do 6 | describe "#248" do 7 | it "does not fail with github" do 8 | github_uri = "http://github.com/" 9 | expect { HTTP.get(github_uri).to_s }.not_to raise_error 10 | end 11 | 12 | it "does not fail with googleapis" do 13 | google_uri = "https://www.googleapis.com/oauth2/v1/userinfo?alt=json" 14 | expect { HTTP.get(google_uri).to_s }.not_to raise_error 15 | end 16 | end 17 | 18 | describe "#422" do 19 | it "reads body when 200 OK response contains Upgrade header" do 20 | res = HTTP.get("https://httpbin.org/response-headers?Upgrade=h2,h2c") 21 | expect(res.parse(:json)).to include("Upgrade" => "h2,h2c") 22 | end 23 | end 24 | end 25 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Help and Discussion 2 | 3 | If you need help or just want to talk about the http.rb, 4 | visit the http.rb Google Group: 5 | 6 | https://groups.google.com/forum/#!forum/httprb 7 | 8 | You can join by email by sending a message to: 9 | 10 | [httprb+subscribe@googlegroups.com](mailto:httprb+subscribe@googlegroups.com) 11 | 12 | 13 | # Reporting bugs 14 | 15 | The best way to report a bug is by providing a reproduction script. A half 16 | working script with comments for the parts you were unable to automate is still 17 | appreciated. 18 | 19 | In any case, specify following info in description of your issue: 20 | 21 | - What you're trying to accomplish 22 | - What you expected to happen 23 | - What actually happened 24 | - The exception backtrace(s), if any 25 | - Version of gem or commit ref you are using 26 | - Version of ruby you are using 27 | -------------------------------------------------------------------------------- /lib/http/content_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class ContentType 5 | MIME_TYPE_RE = %r{^([^/]+/[^;]+)(?:$|;)}.freeze 6 | CHARSET_RE = /;\s*charset=([^;]+)/i.freeze 7 | 8 | attr_accessor :mime_type, :charset 9 | 10 | class << self 11 | # Parse string and return ContentType struct 12 | def parse(str) 13 | new mime_type(str), charset(str) 14 | end 15 | 16 | private 17 | 18 | # :nodoc: 19 | def mime_type(str) 20 | str.to_s[MIME_TYPE_RE, 1]&.strip&.downcase 21 | end 22 | 23 | # :nodoc: 24 | def charset(str) 25 | str.to_s[CHARSET_RE, 1]&.strip&.delete('"') 26 | end 27 | end 28 | 29 | def initialize(mime_type = nil, charset = nil) 30 | @mime_type = mime_type 31 | @charset = charset 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/http/headers/mixin.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | module HTTP 6 | class Headers 7 | # Provides shared behavior for {HTTP::Request} and {HTTP::Response}. 8 | # Expects `@headers` to be an instance of {HTTP::Headers}. 9 | # 10 | # @example Usage 11 | # 12 | # class MyHttpRequest 13 | # include HTTP::Headers::Mixin 14 | # 15 | # def initialize 16 | # @headers = HTTP::Headers.new 17 | # end 18 | # end 19 | module Mixin 20 | extend Forwardable 21 | 22 | # @return [HTTP::Headers] 23 | attr_reader :headers 24 | 25 | # @!method [] 26 | # (see HTTP::Headers#[]) 27 | def_delegator :headers, :[] 28 | 29 | # @!method []= 30 | # (see HTTP::Headers#[]=) 31 | def_delegator :headers, :[]= 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /spec/lib/http/options/proxy_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "proxy" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to {}" do 7 | expect(opts.proxy).to eq({}) 8 | end 9 | 10 | it "may be specified with with_proxy" do 11 | opts2 = opts.with_proxy(:proxy_address => "127.0.0.1", :proxy_port => 8080) 12 | expect(opts.proxy).to eq({}) 13 | expect(opts2.proxy).to eq(:proxy_address => "127.0.0.1", :proxy_port => 8080) 14 | end 15 | 16 | it "accepts proxy address, port, username, and password" do 17 | opts2 = opts.with_proxy(:proxy_address => "127.0.0.1", :proxy_port => 8080, :proxy_username => "username", :proxy_password => "password") 18 | expect(opts2.proxy).to eq(:proxy_address => "127.0.0.1", :proxy_port => 8080, :proxy_username => "username", :proxy_password => "password") 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /spec/support/fuubar.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "fuubar" 4 | 5 | RSpec.configure do |config| 6 | # Use Fuubar instafail-alike formatter, unless a formatter has already been 7 | # configured (e.g. via a command-line flag). 8 | config.default_formatter = "Fuubar" 9 | 10 | # Disable auto-refresh of the fuubar progress bar to avoid surprises during 11 | # debugiing. And simply because there's next to absolutely no point in having 12 | # this turned on. 13 | # 14 | # > By default fuubar will automatically refresh the bar (and therefore 15 | # > the ETA) every second. Unfortunately this doesn't play well with things 16 | # > like debuggers. When you're debugging, having a bar show up every second 17 | # > is undesireable. 18 | # 19 | # See: https://github.com/thekompanee/fuubar#disabling-auto-refresh 20 | config.fuubar_auto_refresh = false 21 | end 22 | -------------------------------------------------------------------------------- /spec/lib/http/headers/mixin_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Headers::Mixin do 4 | let :dummy_class do 5 | Class.new do 6 | include HTTP::Headers::Mixin 7 | 8 | def initialize(headers) 9 | @headers = headers 10 | end 11 | end 12 | end 13 | 14 | let(:headers) { HTTP::Headers.new } 15 | let(:dummy) { dummy_class.new headers } 16 | 17 | describe "#headers" do 18 | it "returns @headers instance variable" do 19 | expect(dummy.headers).to be headers 20 | end 21 | end 22 | 23 | describe "#[]" do 24 | it "proxies to headers#[]" do 25 | expect(headers).to receive(:[]).with(:accept) 26 | dummy[:accept] 27 | end 28 | end 29 | 30 | describe "#[]=" do 31 | it "proxies to headers#[]" do 32 | expect(headers).to receive(:[]=).with(:accept, "text/plain") 33 | dummy[:accept] = "text/plain" 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/support/proxy_server.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "webrick/httpproxy" 4 | 5 | require "support/black_hole" 6 | require "support/servers/config" 7 | require "support/servers/runner" 8 | 9 | class ProxyServer < WEBrick::HTTPProxyServer 10 | include ServerConfig 11 | 12 | CONFIG = { 13 | :BindAddress => "127.0.0.1", 14 | :Port => 0, 15 | :AccessLog => BlackHole, 16 | :Logger => BlackHole, 17 | :RequestCallback => proc { |_, res| res["X-PROXIED"] = true } 18 | }.freeze 19 | 20 | def initialize 21 | super CONFIG 22 | end 23 | end 24 | 25 | class AuthProxyServer < WEBrick::HTTPProxyServer 26 | include ServerConfig 27 | 28 | AUTHENTICATOR = proc do |req, res| 29 | WEBrick::HTTPAuth.proxy_basic_auth(req, res, "proxy") do |user, pass| 30 | user == "username" && pass == "password" 31 | end 32 | end 33 | 34 | CONFIG = ProxyServer::CONFIG.merge :ProxyAuthProc => AUTHENTICATOR 35 | 36 | def initialize 37 | super CONFIG 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/lib/http/options/new_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "new" do 4 | it "supports a Options instance" do 5 | opts = HTTP::Options.new 6 | expect(HTTP::Options.new(opts)).to eq(opts) 7 | end 8 | 9 | context "with a Hash" do 10 | it "coerces :response correctly" do 11 | opts = HTTP::Options.new(:response => :object) 12 | expect(opts.response).to eq(:object) 13 | end 14 | 15 | it "coerces :headers correctly" do 16 | opts = HTTP::Options.new(:headers => {:accept => "json"}) 17 | expect(opts.headers).to eq([%w[Accept json]]) 18 | end 19 | 20 | it "coerces :proxy correctly" do 21 | opts = HTTP::Options.new(:proxy => {:proxy_address => "127.0.0.1", :proxy_port => 8080}) 22 | expect(opts.proxy).to eq(:proxy_address => "127.0.0.1", :proxy_port => 8080) 23 | end 24 | 25 | it "coerces :form correctly" do 26 | opts = HTTP::Options.new(:form => {:foo => 42}) 27 | expect(opts.form).to eq(:foo => 42) 28 | end 29 | end 30 | end 31 | -------------------------------------------------------------------------------- /spec/support/dummy_server.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "webrick" 4 | require "webrick/ssl" 5 | 6 | require "support/black_hole" 7 | require "support/dummy_server/servlet" 8 | require "support/servers/config" 9 | require "support/servers/runner" 10 | require "support/ssl_helper" 11 | 12 | class DummyServer < WEBrick::HTTPServer 13 | include ServerConfig 14 | 15 | CONFIG = { 16 | :BindAddress => "127.0.0.1", 17 | :Port => 0, 18 | :AccessLog => BlackHole, 19 | :Logger => BlackHole 20 | }.freeze 21 | 22 | SSL_CONFIG = CONFIG.merge( 23 | :SSLEnable => true, 24 | :SSLStartImmediately => true 25 | ).freeze 26 | 27 | def initialize(options = {}) 28 | super(options[:ssl] ? SSL_CONFIG : CONFIG) 29 | mount("/", Servlet) 30 | end 31 | 32 | def endpoint 33 | "#{scheme}://#{addr}:#{port}" 34 | end 35 | 36 | def scheme 37 | config[:SSLEnable] ? "https" : "http" 38 | end 39 | 40 | def ssl_context 41 | @ssl_context ||= SSLHelper.server_context 42 | end 43 | end 44 | -------------------------------------------------------------------------------- /spec/lib/http/uri_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::URI do 4 | let(:example_http_uri_string) { "http://example.com" } 5 | let(:example_https_uri_string) { "https://example.com" } 6 | 7 | subject(:http_uri) { described_class.parse(example_http_uri_string) } 8 | subject(:https_uri) { described_class.parse(example_https_uri_string) } 9 | 10 | it "knows URI schemes" do 11 | expect(http_uri.scheme).to eq "http" 12 | expect(https_uri.scheme).to eq "https" 13 | end 14 | 15 | it "sets default ports for HTTP URIs" do 16 | expect(http_uri.port).to eq 80 17 | end 18 | 19 | it "sets default ports for HTTPS URIs" do 20 | expect(https_uri.port).to eq 443 21 | end 22 | 23 | describe "#dup" do 24 | it "doesn't share internal value between duplicates" do 25 | duplicated_uri = http_uri.dup 26 | duplicated_uri.host = "example.org" 27 | 28 | expect(duplicated_uri.to_s).to eq("http://example.org") 29 | expect(http_uri.to_s).to eq("http://example.com") 30 | end 31 | end 32 | end 33 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | source "https://rubygems.org" 4 | ruby RUBY_VERSION 5 | 6 | gem "rake" 7 | 8 | # Ruby 3.0 does not ship it anymore. 9 | # TODO: We should probably refactor specs to avoid need for it. 10 | gem "webrick" 11 | 12 | group :development do 13 | gem "guard-rspec", :require => false 14 | gem "nokogiri", :require => false 15 | gem "pry", :require => false 16 | 17 | # RSpec formatter 18 | gem "fuubar", :require => false 19 | 20 | platform :mri do 21 | gem "pry-byebug" 22 | end 23 | end 24 | 25 | group :test do 26 | gem "certificate_authority", "~> 1.0", :require => false 27 | 28 | gem "backports" 29 | 30 | gem "rubocop", "~> 1.30.0" 31 | gem "rubocop-performance" 32 | gem "rubocop-rake" 33 | gem "rubocop-rspec" 34 | 35 | gem "simplecov", :require => false 36 | gem "simplecov-lcov", :require => false 37 | 38 | gem "rspec", "~> 3.10" 39 | gem "rspec-its" 40 | 41 | gem "yardstick" 42 | end 43 | 44 | group :doc do 45 | gem "kramdown" 46 | gem "yard" 47 | end 48 | 49 | # Specify your gem's dependencies in http.gemspec 50 | gemspec 51 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011-2022 Tony Arcieri, Erik Michaels-Ober, Alexey V. Zapparov, Zachary Anker 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /spec/lib/http/options/features_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "features" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "defaults to be empty" do 7 | expect(opts.features).to be_empty 8 | end 9 | 10 | it "accepts plain symbols in array" do 11 | opts2 = opts.with_features([:auto_inflate]) 12 | expect(opts.features).to be_empty 13 | expect(opts2.features.keys).to eq([:auto_inflate]) 14 | expect(opts2.features[:auto_inflate]). 15 | to be_instance_of(HTTP::Features::AutoInflate) 16 | end 17 | 18 | it "accepts feature name with its options in array" do 19 | opts2 = opts.with_features([{:auto_deflate => {:method => :deflate}}]) 20 | expect(opts.features).to be_empty 21 | expect(opts2.features.keys).to eq([:auto_deflate]) 22 | expect(opts2.features[:auto_deflate]). 23 | to be_instance_of(HTTP::Features::AutoDeflate) 24 | expect(opts2.features[:auto_deflate].method).to eq("deflate") 25 | end 26 | 27 | it "raises error for not supported features" do 28 | expect { opts.with_features([:wrong_feature]) }. 29 | to raise_error(HTTP::Error) { |error| 30 | expect(error.message).to eq("Unsupported feature: wrong_feature") 31 | } 32 | end 33 | end 34 | -------------------------------------------------------------------------------- /lib/http/features/auto_inflate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "set" 4 | 5 | module HTTP 6 | module Features 7 | class AutoInflate < Feature 8 | SUPPORTED_ENCODING = Set.new(%w[deflate gzip x-gzip]).freeze 9 | private_constant :SUPPORTED_ENCODING 10 | 11 | def wrap_response(response) 12 | return response unless supported_encoding?(response) 13 | 14 | options = { 15 | :status => response.status, 16 | :version => response.version, 17 | :headers => response.headers, 18 | :proxy_headers => response.proxy_headers, 19 | :connection => response.connection, 20 | :body => stream_for(response.connection), 21 | :request => response.request 22 | } 23 | 24 | Response.new(options) 25 | end 26 | 27 | def stream_for(connection) 28 | Response::Body.new(Response::Inflater.new(connection)) 29 | end 30 | 31 | private 32 | 33 | def supported_encoding?(response) 34 | content_encoding = response.headers.get(Headers::CONTENT_ENCODING).first 35 | content_encoding && SUPPORTED_ENCODING.include?(content_encoding) 36 | end 37 | 38 | HTTP::Options.register_feature(:auto_inflate, self) 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/http/features/logging.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Features 5 | # Log requests and responses. Request verb and uri, and Response status are 6 | # logged at `info`, and the headers and bodies of both are logged at 7 | # `debug`. Be sure to specify the logger when enabling the feature: 8 | # 9 | # HTTP.use(logging: {logger: Logger.new(STDOUT)}).get("https://example.com/") 10 | # 11 | class Logging < Feature 12 | HTTP::Options.register_feature(:logging, self) 13 | 14 | class NullLogger 15 | %w[fatal error warn info debug].each do |level| 16 | define_method(level.to_sym) do |*_args| 17 | nil 18 | end 19 | 20 | define_method(:"#{level}?") do 21 | true 22 | end 23 | end 24 | end 25 | 26 | attr_reader :logger 27 | 28 | def initialize(logger: NullLogger.new) 29 | @logger = logger 30 | end 31 | 32 | def wrap_request(request) 33 | logger.info { "> #{request.verb.to_s.upcase} #{request.uri}" } 34 | logger.debug { "#{stringify_headers(request.headers)}\n\n#{request.body.source}" } 35 | 36 | request 37 | end 38 | 39 | def wrap_response(response) 40 | logger.info { "< #{response.status}" } 41 | logger.debug { "#{stringify_headers(response.headers)}\n\n#{response.body}" } 42 | 43 | response 44 | end 45 | 46 | private 47 | 48 | def stringify_headers(headers) 49 | headers.map { |name, value| "#{name}: #{value}" }.join("\n") 50 | end 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /spec/lib/http/features/instrumentation_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::Instrumentation do 4 | subject(:feature) { HTTP::Features::Instrumentation.new(:instrumenter => instrumenter) } 5 | 6 | let(:instrumenter) { TestInstrumenter.new } 7 | 8 | before do 9 | test_instrumenter = Class.new(HTTP::Features::Instrumentation::NullInstrumenter) do 10 | attr_reader :output 11 | 12 | def initialize 13 | @output = {} 14 | end 15 | 16 | def start(_name, payload) 17 | output[:start] = payload 18 | end 19 | 20 | def finish(_name, payload) 21 | output[:finish] = payload 22 | end 23 | end 24 | 25 | stub_const("TestInstrumenter", test_instrumenter) 26 | end 27 | 28 | describe "logging the request" do 29 | let(:request) do 30 | HTTP::Request.new( 31 | :verb => :post, 32 | :uri => "https://example.com/", 33 | :headers => {:accept => "application/json"}, 34 | :body => '{"hello": "world!"}' 35 | ) 36 | end 37 | 38 | it "should log the request" do 39 | feature.wrap_request(request) 40 | 41 | expect(instrumenter.output[:start]).to eq(:request => request) 42 | end 43 | end 44 | 45 | describe "logging the response" do 46 | let(:response) do 47 | HTTP::Response.new( 48 | :version => "1.1", 49 | :status => 200, 50 | :headers => {:content_type => "application/json"}, 51 | :body => '{"success": true}', 52 | :request => HTTP::Request.new(:verb => :get, :uri => "https://example.com") 53 | ) 54 | end 55 | 56 | it "should log the response" do 57 | feature.wrap_response(response) 58 | 59 | expect(instrumenter.output[:finish]).to eq(:response => response) 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /http.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | lib = File.expand_path("lib", __dir__) 4 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 5 | require "http/version" 6 | 7 | Gem::Specification.new do |gem| 8 | gem.authors = ["Tony Arcieri", "Erik Michaels-Ober", "Alexey V. Zapparov", "Zachary Anker"] 9 | gem.email = ["bascule@gmail.com"] 10 | 11 | gem.description = <<-DESCRIPTION.strip.gsub(/\s+/, " ") 12 | An easy-to-use client library for making requests from Ruby. 13 | It uses a simple method chaining system for building requests, 14 | similar to Python's Requests. 15 | DESCRIPTION 16 | 17 | gem.summary = "HTTP should be easy" 18 | gem.homepage = "https://github.com/httprb/http" 19 | gem.licenses = ["MIT"] 20 | 21 | gem.executables = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) } 22 | gem.files = `git ls-files`.split("\n") 23 | gem.test_files = `git ls-files -- {test,spec,features}/*`.split("\n") 24 | gem.name = "http" 25 | gem.require_paths = ["lib"] 26 | gem.version = HTTP::VERSION 27 | 28 | gem.required_ruby_version = ">= 2.6" 29 | 30 | gem.add_runtime_dependency "addressable", "~> 2.8" 31 | gem.add_runtime_dependency "http-cookie", "~> 1.0" 32 | gem.add_runtime_dependency "http-form_data", "~> 2.2" 33 | gem.add_runtime_dependency "llhttp-ffi", "~> 0.4.0" 34 | 35 | gem.add_development_dependency "bundler", "~> 2.0" 36 | 37 | gem.metadata = { 38 | "source_code_uri" => "https://github.com/httprb/http", 39 | "wiki_uri" => "https://github.com/httprb/http/wiki", 40 | "bug_tracker_uri" => "https://github.com/httprb/http/issues", 41 | "changelog_uri" => "https://github.com/httprb/http/blob/v#{HTTP::VERSION}/CHANGES.md", 42 | "rubygems_mfa_required" => "true" 43 | } 44 | end 45 | -------------------------------------------------------------------------------- /spec/lib/http/features/logging_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "logger" 4 | 5 | RSpec.describe HTTP::Features::Logging do 6 | subject(:feature) do 7 | logger = Logger.new(logdev) 8 | logger.formatter = ->(severity, _, _, message) { format("** %s **\n%s\n", severity, message) } 9 | 10 | described_class.new(:logger => logger) 11 | end 12 | 13 | let(:logdev) { StringIO.new } 14 | 15 | describe "logging the request" do 16 | let(:request) do 17 | HTTP::Request.new( 18 | :verb => :post, 19 | :uri => "https://example.com/", 20 | :headers => {:accept => "application/json"}, 21 | :body => '{"hello": "world!"}' 22 | ) 23 | end 24 | 25 | it "should log the request" do 26 | feature.wrap_request(request) 27 | 28 | expect(logdev.string).to eq <<~OUTPUT 29 | ** INFO ** 30 | > POST https://example.com/ 31 | ** DEBUG ** 32 | Accept: application/json 33 | Host: example.com 34 | User-Agent: http.rb/#{HTTP::VERSION} 35 | 36 | {"hello": "world!"} 37 | OUTPUT 38 | end 39 | end 40 | 41 | describe "logging the response" do 42 | let(:response) do 43 | HTTP::Response.new( 44 | :version => "1.1", 45 | :status => 200, 46 | :headers => {:content_type => "application/json"}, 47 | :body => '{"success": true}', 48 | :request => HTTP::Request.new(:verb => :get, :uri => "https://example.com") 49 | ) 50 | end 51 | 52 | it "should log the response" do 53 | feature.wrap_response(response) 54 | 55 | expect(logdev.string).to eq <<~OUTPUT 56 | ** INFO ** 57 | < 200 OK 58 | ** DEBUG ** 59 | Content-Type: application/json 60 | 61 | {"success": true} 62 | OUTPUT 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "bundler/gem_tasks" 4 | 5 | require "rspec/core/rake_task" 6 | RSpec::Core::RakeTask.new 7 | 8 | require "rubocop/rake_task" 9 | RuboCop::RakeTask.new 10 | 11 | require "yardstick/rake/measurement" 12 | Yardstick::Rake::Measurement.new do |measurement| 13 | measurement.output = "measurement/report.txt" 14 | end 15 | 16 | require "yardstick/rake/verify" 17 | Yardstick::Rake::Verify.new do |verify| 18 | verify.require_exact_threshold = false 19 | verify.threshold = 55 20 | end 21 | 22 | task :generate_status_codes do 23 | require "http" 24 | require "nokogiri" 25 | 26 | url = "http://www.iana.org/assignments/http-status-codes/http-status-codes.xml" 27 | xml = Nokogiri::XML HTTP.get url 28 | arr = xml.xpath("//xmlns:record").reduce([]) do |a, e| 29 | code = e.xpath("xmlns:value").text.to_s 30 | desc = e.xpath("xmlns:description").text.to_s 31 | 32 | next a if %w[Unassigned (Unused)].include?(desc) 33 | 34 | a << "#{code} => #{desc.inspect}" 35 | end 36 | 37 | File.open("./lib/http/response/status/reasons.rb", "w") do |io| 38 | io.puts <<~TPL 39 | # AUTO-GENERATED FILE, DO NOT CHANGE IT MANUALLY 40 | 41 | require "delegate" 42 | 43 | module HTTP 44 | class Response 45 | class Status < ::Delegator 46 | # Code to Reason map 47 | # 48 | # @example Usage 49 | # 50 | # REASONS[400] # => "Bad Request" 51 | # REASONS[414] # => "Request-URI Too Long" 52 | # 53 | # @return [Hash String>] 54 | REASONS = { 55 | #{arr.join ",\n "} 56 | }.each { |_, v| v.freeze }.freeze 57 | end 58 | end 59 | end 60 | TPL 61 | end 62 | end 63 | 64 | task :default => %i[spec rubocop verify_measurements] 65 | -------------------------------------------------------------------------------- /spec/lib/http/content_type_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::ContentType do 4 | describe ".parse" do 5 | context "with text/plain" do 6 | subject { described_class.parse "text/plain" } 7 | its(:mime_type) { is_expected.to eq "text/plain" } 8 | its(:charset) { is_expected.to be_nil } 9 | end 10 | 11 | context "with tEXT/plaIN" do 12 | subject { described_class.parse "tEXT/plaIN" } 13 | its(:mime_type) { is_expected.to eq "text/plain" } 14 | its(:charset) { is_expected.to be_nil } 15 | end 16 | 17 | context "with text/plain; charset=utf-8" do 18 | subject { described_class.parse "text/plain; charset=utf-8" } 19 | its(:mime_type) { is_expected.to eq "text/plain" } 20 | its(:charset) { is_expected.to eq "utf-8" } 21 | end 22 | 23 | context 'with text/plain; charset="utf-8"' do 24 | subject { described_class.parse 'text/plain; charset="utf-8"' } 25 | its(:mime_type) { is_expected.to eq "text/plain" } 26 | its(:charset) { is_expected.to eq "utf-8" } 27 | end 28 | 29 | context "with text/plain; charSET=utf-8" do 30 | subject { described_class.parse "text/plain; charSET=utf-8" } 31 | its(:mime_type) { is_expected.to eq "text/plain" } 32 | its(:charset) { is_expected.to eq "utf-8" } 33 | end 34 | 35 | context "with text/plain; foo=bar; charset=utf-8" do 36 | subject { described_class.parse "text/plain; foo=bar; charset=utf-8" } 37 | its(:mime_type) { is_expected.to eq "text/plain" } 38 | its(:charset) { is_expected.to eq "utf-8" } 39 | end 40 | 41 | context "with text/plain;charset=utf-8;foo=bar" do 42 | subject { described_class.parse "text/plain;charset=utf-8;foo=bar" } 43 | its(:mime_type) { is_expected.to eq "text/plain" } 44 | its(:charset) { is_expected.to eq "utf-8" } 45 | end 46 | end 47 | end 48 | -------------------------------------------------------------------------------- /lib/http/mime_type.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | # MIME type encode/decode adapters 5 | module MimeType 6 | class << self 7 | # Associate MIME type with adapter 8 | # 9 | # @example 10 | # 11 | # module JsonAdapter 12 | # class << self 13 | # def encode(obj) 14 | # # encode logic here 15 | # end 16 | # 17 | # def decode(str) 18 | # # decode logic here 19 | # end 20 | # end 21 | # end 22 | # 23 | # HTTP::MimeType.register_adapter 'application/json', MyJsonAdapter 24 | # 25 | # @param [#to_s] type 26 | # @param [#encode, #decode] adapter 27 | # @return [void] 28 | def register_adapter(type, adapter) 29 | adapters[type.to_s] = adapter 30 | end 31 | 32 | # Returns adapter associated with MIME type 33 | # 34 | # @param [#to_s] type 35 | # @raise [Error] if no adapter found 36 | # @return [Class] 37 | def [](type) 38 | adapters[normalize type] || raise(Error, "Unknown MIME type: #{type}") 39 | end 40 | 41 | # Register a shortcut for MIME type 42 | # 43 | # @example 44 | # 45 | # HTTP::MimeType.register_alias 'application/json', :json 46 | # 47 | # @param [#to_s] type 48 | # @param [#to_sym] shortcut 49 | # @return [void] 50 | def register_alias(type, shortcut) 51 | aliases[shortcut.to_sym] = type.to_s 52 | end 53 | 54 | # Resolves type by shortcut if possible 55 | # 56 | # @param [#to_s] type 57 | # @return [String] 58 | def normalize(type) 59 | aliases.fetch type, type.to_s 60 | end 61 | 62 | private 63 | 64 | # :nodoc: 65 | def adapters 66 | @adapters ||= {} 67 | end 68 | 69 | # :nodoc: 70 | def aliases 71 | @aliases ||= {} 72 | end 73 | end 74 | end 75 | end 76 | 77 | # built-in mime types 78 | require "http/mime_type/json" 79 | -------------------------------------------------------------------------------- /lib/http/features/instrumentation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | module Features 5 | # Instrument requests and responses. Expects an 6 | # ActiveSupport::Notifications-compatible instrumenter. Defaults to use a 7 | # namespace of 'http' which may be overridden with a `:namespace` param. 8 | # Emits a single event like `"request.{namespace}"`, eg `"request.http"`. 9 | # Be sure to specify the instrumenter when enabling the feature: 10 | # 11 | # HTTP 12 | # .use(instrumentation: {instrumenter: ActiveSupport::Notifications.instrumenter}) 13 | # .get("https://example.com/") 14 | # 15 | # Emits two events on every request: 16 | # 17 | # * `start_request.http` before the request is made, so you can log the reqest being started 18 | # * `request.http` after the response is recieved, and contains `start` 19 | # and `finish` so the duration of the request can be calculated. 20 | # 21 | class Instrumentation < Feature 22 | attr_reader :instrumenter, :name 23 | 24 | def initialize(instrumenter: NullInstrumenter.new, namespace: "http") 25 | @instrumenter = instrumenter 26 | @name = "request.#{namespace}" 27 | end 28 | 29 | def wrap_request(request) 30 | # Emit a separate "start" event, so a logger can print the request 31 | # being run without waiting for a response 32 | instrumenter.instrument("start_#{name}", :request => request) 33 | instrumenter.start(name, :request => request) 34 | request 35 | end 36 | 37 | def wrap_response(response) 38 | instrumenter.finish(name, :response => response) 39 | response 40 | end 41 | 42 | HTTP::Options.register_feature(:instrumentation, self) 43 | 44 | class NullInstrumenter 45 | def instrument(name, payload = {}) 46 | start(name, payload) 47 | begin 48 | yield payload if block_given? 49 | ensure 50 | finish name, payload 51 | end 52 | end 53 | 54 | def start(_name, _payload) 55 | true 56 | end 57 | 58 | def finish(_name, _payload) 59 | true 60 | end 61 | end 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /.github/workflows/ci.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: 4 | push: 5 | branches: [ main ] 6 | pull_request: 7 | branches: [ main ] 8 | 9 | env: 10 | BUNDLE_WITHOUT: "development" 11 | JRUBY_OPTS: "--dev --debug" 12 | 13 | jobs: 14 | test: 15 | runs-on: ${{ matrix.os }} 16 | 17 | strategy: 18 | matrix: 19 | ruby: [ ruby-2.6, ruby-2.7, ruby-3.0, ruby-3.1 ] 20 | os: [ ubuntu-latest ] 21 | 22 | steps: 23 | - uses: actions/checkout@v3 24 | 25 | - uses: ruby/setup-ruby@v1 26 | with: 27 | ruby-version: ${{ matrix.ruby }} 28 | bundler-cache: true 29 | 30 | - name: bundle exec rspec 31 | run: bundle exec rspec --format progress --force-colour 32 | 33 | - name: Prepare Coveralls test coverage report 34 | uses: coverallsapp/github-action@v1.1.2 35 | with: 36 | github-token: ${{ secrets.GITHUB_TOKEN }} 37 | flag-name: "${{ matrix.ruby }} @${{ matrix.os }}" 38 | path-to-lcov: ./coverage/lcov/lcov.info 39 | parallel: true 40 | 41 | test-flaky: 42 | runs-on: ${{ matrix.os }} 43 | 44 | strategy: 45 | matrix: 46 | ruby: [ jruby-9.3 ] 47 | os: [ ubuntu-latest ] 48 | 49 | steps: 50 | - uses: actions/checkout@v3 51 | 52 | - uses: ruby/setup-ruby@v1 53 | with: 54 | ruby-version: ${{ matrix.ruby }} 55 | bundler-cache: true 56 | 57 | - name: bundle exec rspec 58 | continue-on-error: true 59 | run: bundle exec rspec --format progress --force-colour 60 | 61 | coveralls: 62 | needs: test 63 | runs-on: ubuntu-latest 64 | steps: 65 | - name: Finalize Coveralls test coverage report 66 | uses: coverallsapp/github-action@master 67 | with: 68 | github-token: ${{ secrets.GITHUB_TOKEN }} 69 | parallel-finished: true 70 | 71 | lint: 72 | runs-on: ubuntu-latest 73 | 74 | steps: 75 | - uses: actions/checkout@v3 76 | 77 | - uses: ruby/setup-ruby@v1 78 | with: 79 | ruby-version: 2.6 80 | bundler-cache: true 81 | 82 | - name: bundle exec rubocop 83 | run: bundle exec rubocop --format progress --color 84 | 85 | - run: bundle exec rake verify_measurements 86 | -------------------------------------------------------------------------------- /spec/lib/http/response/parser_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response::Parser do 4 | subject(:parser) { described_class.new } 5 | let(:raw_response) do 6 | "HTTP/1.1 200 OK\r\nContent-Length: 2\r\nContent-Type: application/json\r\nMyHeader: val\r\nEmptyHeader: \r\n\r\n{}" 7 | end 8 | let(:expected_headers) do 9 | { 10 | "Content-Length" => "2", 11 | "Content-Type" => "application/json", 12 | "MyHeader" => "val", 13 | "EmptyHeader" => "" 14 | } 15 | end 16 | let(:expected_body) { "{}" } 17 | 18 | before do 19 | parts.each { |part| subject.add(part) } 20 | end 21 | 22 | context "whole response in one part" do 23 | let(:parts) { [raw_response] } 24 | 25 | it "parses headers" do 26 | expect(subject.headers.to_h).to eq(expected_headers) 27 | end 28 | 29 | it "parses body" do 30 | expect(subject.read(expected_body.size)).to eq(expected_body) 31 | end 32 | end 33 | 34 | context "response in many parts" do 35 | let(:parts) { raw_response.chars } 36 | 37 | it "parses headers" do 38 | expect(subject.headers.to_h).to eq(expected_headers) 39 | end 40 | 41 | it "parses body" do 42 | expect(subject.read(expected_body.size)).to eq(expected_body) 43 | end 44 | end 45 | 46 | context "when got 100 Continue response" do 47 | let :raw_response do 48 | "HTTP/1.1 100 Continue\r\n\r\n" \ 49 | "HTTP/1.1 200 OK\r\n" \ 50 | "Content-Length: 12\r\n\r\n" \ 51 | "Hello World!" 52 | end 53 | 54 | context "when response is feeded in one part" do 55 | let(:parts) { [raw_response] } 56 | 57 | it "skips to next non-info response" do 58 | expect(subject.status_code).to eq(200) 59 | expect(subject.headers).to eq("Content-Length" => "12") 60 | expect(subject.read(12)).to eq("Hello World!") 61 | end 62 | end 63 | 64 | context "when response is feeded in many parts" do 65 | let(:parts) { raw_response.chars } 66 | 67 | it "skips to next non-info response" do 68 | expect(subject.status_code).to eq(200) 69 | expect(subject.headers).to eq("Content-Length" => "12") 70 | expect(subject.read(12)).to eq("Hello World!") 71 | end 72 | end 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /spec/lib/http/connection_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Connection do 4 | let(:req) do 5 | HTTP::Request.new( 6 | :verb => :get, 7 | :uri => "http://example.com/", 8 | :headers => {} 9 | ) 10 | end 11 | let(:socket) { double(:connect => nil) } 12 | let(:timeout_class) { double(:new => socket) } 13 | let(:opts) { HTTP::Options.new(:timeout_class => timeout_class) } 14 | let(:connection) { HTTP::Connection.new(req, opts) } 15 | 16 | describe "#read_headers!" do 17 | before do 18 | connection.instance_variable_set(:@pending_response, true) 19 | expect(socket).to receive(:readpartial) do 20 | <<-RESPONSE.gsub(/^\s*\| */, "").gsub(/\n/, "\r\n") 21 | | HTTP/1.1 200 OK 22 | | Content-Type: text 23 | | foo_bar: 123 24 | | 25 | RESPONSE 26 | end 27 | end 28 | 29 | it "populates headers collection, preserving casing" do 30 | connection.read_headers! 31 | expect(connection.headers).to eq("Content-Type" => "text", "foo_bar" => "123") 32 | expect(connection.headers["Foo-Bar"]).to eq("123") 33 | expect(connection.headers["foo_bar"]).to eq("123") 34 | end 35 | end 36 | 37 | describe "#readpartial" do 38 | before do 39 | connection.instance_variable_set(:@pending_response, true) 40 | expect(socket).to receive(:readpartial) do 41 | <<-RESPONSE.gsub(/^\s*\| */, "").gsub(/\n/, "\r\n") 42 | | HTTP/1.1 200 OK 43 | | Content-Type: text 44 | | 45 | RESPONSE 46 | end 47 | expect(socket).to receive(:readpartial) { "1" } 48 | expect(socket).to receive(:readpartial) { "23" } 49 | expect(socket).to receive(:readpartial) { "456" } 50 | expect(socket).to receive(:readpartial) { "78" } 51 | expect(socket).to receive(:readpartial) { "9" } 52 | expect(socket).to receive(:readpartial) { "0" } 53 | expect(socket).to receive(:readpartial) { :eof } 54 | expect(socket).to receive(:closed?) { true } 55 | end 56 | 57 | it "reads data in parts" do 58 | connection.read_headers! 59 | buffer = String.new 60 | while (s = connection.readpartial(3)) 61 | buffer << s 62 | end 63 | expect(buffer).to eq "1234567890" 64 | end 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/http/timeout/null.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | require "io/wait" 5 | 6 | module HTTP 7 | module Timeout 8 | class Null 9 | extend Forwardable 10 | 11 | def_delegators :@socket, :close, :closed? 12 | 13 | attr_reader :options, :socket 14 | 15 | def initialize(options = {}) 16 | @options = options 17 | end 18 | 19 | # Connects to a socket 20 | def connect(socket_class, host, port, nodelay = false) 21 | @socket = socket_class.open(host, port) 22 | @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay 23 | end 24 | 25 | # Starts a SSL connection on a socket 26 | def connect_ssl 27 | @socket.connect 28 | end 29 | 30 | # Configures the SSL connection and starts the connection 31 | def start_tls(host, ssl_socket_class, ssl_context) 32 | @socket = ssl_socket_class.new(socket, ssl_context) 33 | @socket.hostname = host if @socket.respond_to? :hostname= 34 | @socket.sync_close = true if @socket.respond_to? :sync_close= 35 | 36 | connect_ssl 37 | 38 | return unless ssl_context.verify_mode == OpenSSL::SSL::VERIFY_PEER 39 | return if ssl_context.respond_to?(:verify_hostname) && !ssl_context.verify_hostname 40 | 41 | @socket.post_connection_check(host) 42 | end 43 | 44 | # Read from the socket 45 | def readpartial(size, buffer = nil) 46 | @socket.readpartial(size, buffer) 47 | rescue EOFError 48 | :eof 49 | end 50 | 51 | # Write to the socket 52 | def write(data) 53 | @socket.write(data) 54 | end 55 | alias << write 56 | 57 | private 58 | 59 | # Retry reading 60 | def rescue_readable(timeout = read_timeout) 61 | yield 62 | rescue IO::WaitReadable 63 | retry if @socket.to_io.wait_readable(timeout) 64 | raise TimeoutError, "Read timed out after #{timeout} seconds" 65 | end 66 | 67 | # Retry writing 68 | def rescue_writable(timeout = write_timeout) 69 | yield 70 | rescue IO::WaitWritable 71 | retry if @socket.to_io.wait_writable(timeout) 72 | raise TimeoutError, "Write timed out after #{timeout} seconds" 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/http/response/body.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | require "http/client" 5 | 6 | module HTTP 7 | class Response 8 | # A streamable response body, also easily converted into a string 9 | class Body 10 | extend Forwardable 11 | include Enumerable 12 | def_delegator :to_s, :empty? 13 | 14 | # The connection object used to make the corresponding request. 15 | # 16 | # @return [HTTP::Connection] 17 | attr_reader :connection 18 | 19 | def initialize(stream, encoding: Encoding::BINARY) 20 | @stream = stream 21 | @connection = stream.is_a?(Inflater) ? stream.connection : stream 22 | @streaming = nil 23 | @contents = nil 24 | @encoding = find_encoding(encoding) 25 | end 26 | 27 | # (see HTTP::Client#readpartial) 28 | def readpartial(*args) 29 | stream! 30 | chunk = @stream.readpartial(*args) 31 | 32 | String.new(chunk, :encoding => @encoding) if chunk 33 | end 34 | 35 | # Iterate over the body, allowing it to be enumerable 36 | def each 37 | while (chunk = readpartial) 38 | yield chunk 39 | end 40 | end 41 | 42 | # @return [String] eagerly consume the entire body as a string 43 | def to_s 44 | return @contents if @contents 45 | 46 | raise StateError, "body is being streamed" unless @streaming.nil? 47 | 48 | begin 49 | @streaming = false 50 | @contents = String.new("", :encoding => @encoding) 51 | 52 | while (chunk = @stream.readpartial) 53 | @contents << String.new(chunk, :encoding => @encoding) 54 | chunk = nil # deallocate string 55 | end 56 | rescue 57 | @contents = nil 58 | raise 59 | end 60 | 61 | @contents 62 | end 63 | alias to_str to_s 64 | 65 | # Assert that the body is actively being streamed 66 | def stream! 67 | raise StateError, "body has already been consumed" if @streaming == false 68 | 69 | @streaming = true 70 | end 71 | 72 | # Easier to interpret string inspect 73 | def inspect 74 | "#<#{self.class}:#{object_id.to_s(16)} @streaming=#{!!@streaming}>" 75 | end 76 | 77 | private 78 | 79 | # Retrieve encoding by name. If encoding cannot be found, default to binary. 80 | def find_encoding(encoding) 81 | Encoding.find encoding 82 | rescue ArgumentError 83 | Encoding::BINARY 84 | end 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /spec/lib/http/features/auto_deflate_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::AutoDeflate do 4 | subject { HTTP::Features::AutoDeflate.new } 5 | 6 | it "raises error for wrong type" do 7 | expect { HTTP::Features::AutoDeflate.new(:method => :wrong) }. 8 | to raise_error(HTTP::Error) { |error| 9 | expect(error.message).to eq("Only gzip and deflate methods are supported") 10 | } 11 | end 12 | 13 | it "accepts gzip method" do 14 | expect(HTTP::Features::AutoDeflate.new(:method => :gzip).method).to eq "gzip" 15 | end 16 | 17 | it "accepts deflate method" do 18 | expect(HTTP::Features::AutoDeflate.new(:method => :deflate).method).to eq "deflate" 19 | end 20 | 21 | it "accepts string as method" do 22 | expect(HTTP::Features::AutoDeflate.new(:method => "gzip").method).to eq "gzip" 23 | end 24 | 25 | it "uses gzip by default" do 26 | expect(subject.method).to eq("gzip") 27 | end 28 | 29 | describe "#deflated_body" do 30 | let(:body) { %w[bees cows] } 31 | let(:deflated_body) { subject.deflated_body(body) } 32 | 33 | context "when method is gzip" do 34 | subject { HTTP::Features::AutoDeflate.new(:method => :gzip) } 35 | 36 | it "returns object which yields gzipped content of the given body" do 37 | io = StringIO.new 38 | io.set_encoding(Encoding::BINARY) 39 | gzip = Zlib::GzipWriter.new(io) 40 | gzip.write("beescows") 41 | gzip.close 42 | gzipped = io.string 43 | 44 | expect(deflated_body.each.to_a.join).to eq gzipped 45 | end 46 | 47 | it "caches compressed content when size is called" do 48 | io = StringIO.new 49 | io.set_encoding(Encoding::BINARY) 50 | gzip = Zlib::GzipWriter.new(io) 51 | gzip.write("beescows") 52 | gzip.close 53 | gzipped = io.string 54 | 55 | expect(deflated_body.size).to eq gzipped.bytesize 56 | expect(deflated_body.each.to_a.join).to eq gzipped 57 | end 58 | end 59 | 60 | context "when method is deflate" do 61 | subject { HTTP::Features::AutoDeflate.new(:method => :deflate) } 62 | 63 | it "returns object which yields deflated content of the given body" do 64 | deflated = Zlib::Deflate.deflate("beescows") 65 | 66 | expect(deflated_body.each.to_a.join).to eq deflated 67 | end 68 | 69 | it "caches compressed content when size is called" do 70 | deflated = Zlib::Deflate.deflate("beescows") 71 | 72 | expect(deflated_body.size).to eq deflated.bytesize 73 | expect(deflated_body.each.to_a.join).to eq deflated 74 | end 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /spec/lib/http/options/merge_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Options, "merge" do 4 | let(:opts) { HTTP::Options.new } 5 | 6 | it "supports a Hash" do 7 | old_response = opts.response 8 | expect(opts.merge(:response => :body).response).to eq(:body) 9 | expect(opts.response).to eq(old_response) 10 | end 11 | 12 | it "supports another Options" do 13 | merged = opts.merge(HTTP::Options.new(:response => :body)) 14 | expect(merged.response).to eq(:body) 15 | end 16 | 17 | it "merges as excepted in complex cases" do 18 | # FIXME: yuck :( 19 | 20 | foo = HTTP::Options.new( 21 | :response => :body, 22 | :params => {:baz => "bar"}, 23 | :form => {:foo => "foo"}, 24 | :body => "body-foo", 25 | :json => {:foo => "foo"}, 26 | :headers => {:accept => "json", :foo => "foo"}, 27 | :proxy => {}, 28 | :features => {} 29 | ) 30 | 31 | bar = HTTP::Options.new( 32 | :response => :parsed_body, 33 | :persistent => "https://www.googe.com", 34 | :params => {:plop => "plip"}, 35 | :form => {:bar => "bar"}, 36 | :body => "body-bar", 37 | :json => {:bar => "bar"}, 38 | :keep_alive_timeout => 10, 39 | :headers => {:accept => "xml", :bar => "bar"}, 40 | :timeout_options => {:foo => :bar}, 41 | :ssl => {:foo => "bar"}, 42 | :proxy => {:proxy_address => "127.0.0.1", :proxy_port => 8080} 43 | ) 44 | 45 | expect(foo.merge(bar).to_hash).to eq( 46 | :response => :parsed_body, 47 | :timeout_class => described_class.default_timeout_class, 48 | :timeout_options => {:foo => :bar}, 49 | :params => {:plop => "plip"}, 50 | :form => {:bar => "bar"}, 51 | :body => "body-bar", 52 | :json => {:bar => "bar"}, 53 | :persistent => "https://www.googe.com", 54 | :keep_alive_timeout => 10, 55 | :ssl => {:foo => "bar"}, 56 | :headers => {"Foo" => "foo", "Accept" => "xml", "Bar" => "bar"}, 57 | :proxy => {:proxy_address => "127.0.0.1", :proxy_port => 8080}, 58 | :follow => nil, 59 | :socket_class => described_class.default_socket_class, 60 | :nodelay => false, 61 | :ssl_socket_class => described_class.default_ssl_socket_class, 62 | :ssl_context => nil, 63 | :cookies => {}, 64 | :encoding => nil, 65 | :features => {} 66 | ) 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /spec/support/ssl_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "pathname" 4 | 5 | require "certificate_authority" 6 | 7 | module SSLHelper 8 | CERTS_PATH = Pathname.new File.expand_path("../../tmp/certs", __dir__) 9 | 10 | class RootCertificate < ::CertificateAuthority::Certificate 11 | EXTENSIONS = {"keyUsage" => {"usage" => %w[critical keyCertSign]}}.freeze 12 | 13 | def initialize 14 | super() 15 | 16 | subject.common_name = "honestachmed.com" 17 | serial_number.number = 1 18 | key_material.generate_key 19 | 20 | self.signing_entity = true 21 | 22 | sign!("extensions" => EXTENSIONS) 23 | end 24 | 25 | def file 26 | return @file if defined? @file 27 | 28 | CERTS_PATH.mkpath 29 | 30 | cert_file = CERTS_PATH.join("ca.crt") 31 | cert_file.open("w") { |io| io << to_pem } 32 | 33 | @file = cert_file.to_s 34 | end 35 | end 36 | 37 | class ChildCertificate < ::CertificateAuthority::Certificate 38 | def initialize(parent) 39 | super() 40 | 41 | subject.common_name = "127.0.0.1" 42 | serial_number.number = 1 43 | 44 | key_material.generate_key 45 | 46 | self.parent = parent 47 | 48 | sign! 49 | end 50 | 51 | def cert 52 | OpenSSL::X509::Certificate.new to_pem 53 | end 54 | 55 | def key 56 | OpenSSL::PKey::RSA.new key_material.private_key.to_pem 57 | end 58 | end 59 | 60 | class << self 61 | def server_context 62 | context = OpenSSL::SSL::SSLContext.new 63 | 64 | context.verify_mode = OpenSSL::SSL::VERIFY_PEER 65 | context.key = server_cert.key 66 | context.cert = server_cert.cert 67 | context.ca_file = ca.file 68 | 69 | context 70 | end 71 | 72 | def client_context 73 | context = OpenSSL::SSL::SSLContext.new 74 | 75 | context.options = OpenSSL::SSL::SSLContext::DEFAULT_PARAMS[:options] 76 | context.verify_mode = OpenSSL::SSL::VERIFY_PEER 77 | context.key = client_cert.key 78 | context.cert = client_cert.cert 79 | context.ca_file = ca.file 80 | 81 | context 82 | end 83 | 84 | def client_params 85 | { 86 | :key => client_cert.key, 87 | :cert => client_cert.cert, 88 | :ca_file => ca.file 89 | } 90 | end 91 | 92 | %w[server client].each do |side| 93 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 94 | def #{side}_cert 95 | @#{side}_cert ||= ChildCertificate.new ca 96 | end 97 | RUBY 98 | end 99 | 100 | def ca 101 | @ca ||= RootCertificate.new 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /spec/lib/http/response/body_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response::Body do 4 | let(:connection) { double(:sequence_id => 0) } 5 | let(:chunks) { ["Hello, ", "World!"] } 6 | 7 | before do 8 | allow(connection).to receive(:readpartial) { chunks.shift } 9 | allow(connection).to receive(:body_completed?) { chunks.empty? } 10 | end 11 | 12 | subject(:body) { described_class.new(connection, :encoding => Encoding::UTF_8) } 13 | 14 | it "streams bodies from responses" do 15 | expect(subject.to_s).to eq("Hello, World!") 16 | end 17 | 18 | context "when body empty" do 19 | let(:chunks) { [""] } 20 | 21 | it "returns responds to empty? with true" do 22 | expect(subject).to be_empty 23 | end 24 | end 25 | 26 | describe "#readpartial" do 27 | context "with size given" do 28 | it "passes value to underlying connection" do 29 | expect(connection).to receive(:readpartial).with(42) 30 | body.readpartial 42 31 | end 32 | end 33 | 34 | context "without size given" do 35 | it "does not blows up" do 36 | expect { body.readpartial }.to_not raise_error 37 | end 38 | 39 | it "calls underlying connection readpartial without specific size" do 40 | expect(connection).to receive(:readpartial).with no_args 41 | body.readpartial 42 | end 43 | end 44 | 45 | it "returns content in specified encoding" do 46 | body = described_class.new(connection) 47 | expect(connection).to receive(:readpartial). 48 | and_return(String.new("content", :encoding => Encoding::UTF_8)) 49 | expect(body.readpartial.encoding).to eq Encoding::BINARY 50 | 51 | body = described_class.new(connection, :encoding => Encoding::UTF_8) 52 | expect(connection).to receive(:readpartial). 53 | and_return(String.new("content", :encoding => Encoding::BINARY)) 54 | expect(body.readpartial.encoding).to eq Encoding::UTF_8 55 | end 56 | end 57 | 58 | context "when body is gzipped" do 59 | let(:chunks) do 60 | body = Zlib::Deflate.deflate("Hi, HTTP here ☺") 61 | len = body.length 62 | [body[0, len / 2], body[(len / 2)..]] 63 | end 64 | subject(:body) do 65 | inflater = HTTP::Response::Inflater.new(connection) 66 | described_class.new(inflater, :encoding => Encoding::UTF_8) 67 | end 68 | 69 | it "decodes body" do 70 | expect(subject.to_s).to eq("Hi, HTTP here ☺") 71 | end 72 | 73 | describe "#readpartial" do 74 | it "streams decoded body" do 75 | [ 76 | "Hi, HTTP ", 77 | "here ☺", 78 | nil 79 | ].each do |part| 80 | expect(subject.readpartial).to eq(part) 81 | end 82 | end 83 | end 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /lib/http/response/status/reasons.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | # AUTO-GENERATED FILE, DO NOT CHANGE IT MANUALLY 4 | 5 | require "delegate" 6 | 7 | module HTTP 8 | class Response 9 | class Status < ::Delegator 10 | # Code to Reason map 11 | # 12 | # @example Usage 13 | # 14 | # REASONS[400] # => "Bad Request" 15 | # REASONS[414] # => "Request-URI Too Long" 16 | # 17 | # @return [Hash String>] 18 | REASONS = { 19 | 100 => "Continue", 20 | 101 => "Switching Protocols", 21 | 102 => "Processing", 22 | 200 => "OK", 23 | 201 => "Created", 24 | 202 => "Accepted", 25 | 203 => "Non-Authoritative Information", 26 | 204 => "No Content", 27 | 205 => "Reset Content", 28 | 206 => "Partial Content", 29 | 207 => "Multi-Status", 30 | 208 => "Already Reported", 31 | 226 => "IM Used", 32 | 300 => "Multiple Choices", 33 | 301 => "Moved Permanently", 34 | 302 => "Found", 35 | 303 => "See Other", 36 | 304 => "Not Modified", 37 | 305 => "Use Proxy", 38 | 307 => "Temporary Redirect", 39 | 308 => "Permanent Redirect", 40 | 400 => "Bad Request", 41 | 401 => "Unauthorized", 42 | 402 => "Payment Required", 43 | 403 => "Forbidden", 44 | 404 => "Not Found", 45 | 405 => "Method Not Allowed", 46 | 406 => "Not Acceptable", 47 | 407 => "Proxy Authentication Required", 48 | 408 => "Request Timeout", 49 | 409 => "Conflict", 50 | 410 => "Gone", 51 | 411 => "Length Required", 52 | 412 => "Precondition Failed", 53 | 413 => "Payload Too Large", 54 | 414 => "URI Too Long", 55 | 415 => "Unsupported Media Type", 56 | 416 => "Range Not Satisfiable", 57 | 417 => "Expectation Failed", 58 | 421 => "Misdirected Request", 59 | 422 => "Unprocessable Entity", 60 | 423 => "Locked", 61 | 424 => "Failed Dependency", 62 | 426 => "Upgrade Required", 63 | 428 => "Precondition Required", 64 | 429 => "Too Many Requests", 65 | 431 => "Request Header Fields Too Large", 66 | 451 => "Unavailable For Legal Reasons", 67 | 500 => "Internal Server Error", 68 | 501 => "Not Implemented", 69 | 502 => "Bad Gateway", 70 | 503 => "Service Unavailable", 71 | 504 => "Gateway Timeout", 72 | 505 => "HTTP Version Not Supported", 73 | 506 => "Variant Also Negotiates", 74 | 507 => "Insufficient Storage", 75 | 508 => "Loop Detected", 76 | 510 => "Not Extended", 77 | 511 => "Network Authentication Required" 78 | }.each { |_, v| v.freeze }.freeze 79 | end 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /spec/lib/http/features/auto_inflate_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Features::AutoInflate do 4 | subject(:feature) { HTTP::Features::AutoInflate.new } 5 | 6 | let(:connection) { double } 7 | let(:headers) { {} } 8 | 9 | let(:response) do 10 | HTTP::Response.new( 11 | :version => "1.1", 12 | :status => 200, 13 | :headers => headers, 14 | :connection => connection, 15 | :request => HTTP::Request.new(:verb => :get, :uri => "http://example.com") 16 | ) 17 | end 18 | 19 | describe "#wrap_response" do 20 | subject(:result) { feature.wrap_response(response) } 21 | 22 | context "when there is no Content-Encoding header" do 23 | it "returns original request" do 24 | expect(result).to be response 25 | end 26 | end 27 | 28 | context "for identity Content-Encoding header" do 29 | let(:headers) { {:content_encoding => "identity"} } 30 | 31 | it "returns original request" do 32 | expect(result).to be response 33 | end 34 | end 35 | 36 | context "for unknown Content-Encoding header" do 37 | let(:headers) { {:content_encoding => "not-supported"} } 38 | 39 | it "returns original request" do 40 | expect(result).to be response 41 | end 42 | end 43 | 44 | context "for deflate Content-Encoding header" do 45 | let(:headers) { {:content_encoding => "deflate"} } 46 | 47 | it "returns a HTTP::Response wrapping the inflated response body" do 48 | expect(result.body).to be_instance_of HTTP::Response::Body 49 | end 50 | end 51 | 52 | context "for gzip Content-Encoding header" do 53 | let(:headers) { {:content_encoding => "gzip"} } 54 | 55 | it "returns a HTTP::Response wrapping the inflated response body" do 56 | expect(result.body).to be_instance_of HTTP::Response::Body 57 | end 58 | end 59 | 60 | context "for x-gzip Content-Encoding header" do 61 | let(:headers) { {:content_encoding => "x-gzip"} } 62 | 63 | it "returns a HTTP::Response wrapping the inflated response body" do 64 | expect(result.body).to be_instance_of HTTP::Response::Body 65 | end 66 | end 67 | 68 | # TODO(ixti): We should refactor API to either make uri non-optional, 69 | # or add reference to request into response object (better). 70 | context "when response has uri" do 71 | let(:response) do 72 | HTTP::Response.new( 73 | :version => "1.1", 74 | :status => 200, 75 | :headers => {:content_encoding => "gzip"}, 76 | :connection => connection, 77 | :request => HTTP::Request.new(:verb => :get, :uri => "https://example.com") 78 | ) 79 | end 80 | 81 | it "preserves uri in wrapped response" do 82 | expect(result.uri).to eq HTTP::URI.parse("https://example.com") 83 | end 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /lib/http/timeout/per_operation.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "timeout" 4 | 5 | require "http/timeout/null" 6 | 7 | module HTTP 8 | module Timeout 9 | class PerOperation < Null 10 | CONNECT_TIMEOUT = 0.25 11 | WRITE_TIMEOUT = 0.25 12 | READ_TIMEOUT = 0.25 13 | 14 | def initialize(*args) 15 | super 16 | 17 | @read_timeout = options.fetch(:read_timeout, READ_TIMEOUT) 18 | @write_timeout = options.fetch(:write_timeout, WRITE_TIMEOUT) 19 | @connect_timeout = options.fetch(:connect_timeout, CONNECT_TIMEOUT) 20 | end 21 | 22 | def connect(socket_class, host, port, nodelay = false) 23 | ::Timeout.timeout(@connect_timeout, ConnectTimeoutError) do 24 | @socket = socket_class.open(host, port) 25 | @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay 26 | end 27 | end 28 | 29 | def connect_ssl 30 | rescue_readable(@connect_timeout) do 31 | rescue_writable(@connect_timeout) do 32 | @socket.connect_nonblock 33 | end 34 | end 35 | end 36 | 37 | # Read data from the socket 38 | def readpartial(size, buffer = nil) 39 | timeout = false 40 | loop do 41 | result = @socket.read_nonblock(size, buffer, :exception => false) 42 | 43 | return :eof if result.nil? 44 | return result if result != :wait_readable 45 | 46 | raise TimeoutError, "Read timed out after #{@read_timeout} seconds" if timeout 47 | 48 | # marking the socket for timeout. Why is this not being raised immediately? 49 | # it seems there is some race-condition on the network level between calling 50 | # #read_nonblock and #wait_readable, in which #read_nonblock signalizes waiting 51 | # for reads, and when waiting for x seconds, it returns nil suddenly without completing 52 | # the x seconds. In a normal case this would be a timeout on wait/read, but it can 53 | # also mean that the socket has been closed by the server. Therefore we "mark" the 54 | # socket for timeout and try to read more bytes. If it returns :eof, it's all good, no 55 | # timeout. Else, the first timeout was a proper timeout. 56 | # This hack has to be done because io/wait#wait_readable doesn't provide a value for when 57 | # the socket is closed by the server, and HTTP::Parser doesn't provide the limit for the chunks. 58 | timeout = true unless @socket.to_io.wait_readable(@read_timeout) 59 | end 60 | end 61 | 62 | # Write data to the socket 63 | def write(data) 64 | timeout = false 65 | loop do 66 | result = @socket.write_nonblock(data, :exception => false) 67 | return result unless result == :wait_writable 68 | 69 | raise TimeoutError, "Write timed out after #{@write_timeout} seconds" if timeout 70 | 71 | timeout = true unless @socket.to_io.wait_writable(@write_timeout) 72 | end 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/http/headers/known.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Headers 5 | # Content-Types that are acceptable for the response. 6 | ACCEPT = "Accept" 7 | 8 | # Content-codings that are acceptable in the response. 9 | ACCEPT_ENCODING = "Accept-Encoding" 10 | 11 | # The age the object has been in a proxy cache in seconds. 12 | AGE = "Age" 13 | 14 | # Authentication credentials for HTTP authentication. 15 | AUTHORIZATION = "Authorization" 16 | 17 | # Used to specify directives that must be obeyed by all caching mechanisms 18 | # along the request-response chain. 19 | CACHE_CONTROL = "Cache-Control" 20 | 21 | # An HTTP cookie previously sent by the server with Set-Cookie. 22 | COOKIE = "Cookie" 23 | 24 | # Control options for the current connection and list 25 | # of hop-by-hop request fields. 26 | CONNECTION = "Connection" 27 | 28 | # The length of the request body in octets (8-bit bytes). 29 | CONTENT_LENGTH = "Content-Length" 30 | 31 | # The MIME type of the body of the request 32 | # (used with POST and PUT requests). 33 | CONTENT_TYPE = "Content-Type" 34 | 35 | # The date and time that the message was sent (in "HTTP-date" format as 36 | # defined by RFC 7231 Date/Time Formats). 37 | DATE = "Date" 38 | 39 | # An identifier for a specific version of a resource, 40 | # often a message digest. 41 | ETAG = "ETag" 42 | 43 | # Gives the date/time after which the response is considered stale (in 44 | # "HTTP-date" format as defined by RFC 7231). 45 | EXPIRES = "Expires" 46 | 47 | # The domain name of the server (for virtual hosting), and the TCP port 48 | # number on which the server is listening. The port number may be omitted 49 | # if the port is the standard port for the service requested. 50 | HOST = "Host" 51 | 52 | # Allows a 304 Not Modified to be returned if content is unchanged. 53 | IF_MODIFIED_SINCE = "If-Modified-Since" 54 | 55 | # Allows a 304 Not Modified to be returned if content is unchanged. 56 | IF_NONE_MATCH = "If-None-Match" 57 | 58 | # The last modified date for the requested object (in "HTTP-date" format as 59 | # defined by RFC 7231). 60 | LAST_MODIFIED = "Last-Modified" 61 | 62 | # Used in redirection, or when a new resource has been created. 63 | LOCATION = "Location" 64 | 65 | # Authorization credentials for connecting to a proxy. 66 | PROXY_AUTHORIZATION = "Proxy-Authorization" 67 | 68 | # An HTTP cookie. 69 | SET_COOKIE = "Set-Cookie" 70 | 71 | # The form of encoding used to safely transfer the entity to the user. 72 | # Currently defined methods are: chunked, compress, deflate, gzip, identity. 73 | TRANSFER_ENCODING = "Transfer-Encoding" 74 | 75 | # Indicates what additional content codings have been applied to the 76 | # entity-body. 77 | CONTENT_ENCODING = "Content-Encoding" 78 | 79 | # The user agent string of the user agent. 80 | USER_AGENT = "User-Agent" 81 | 82 | # Tells downstream proxies how to match future request headers to decide 83 | # whether the cached response can be used rather than requesting a fresh 84 | # one from the origin server. 85 | VARY = "Vary" 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /lib/http/response/parser.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "llhttp" 4 | 5 | module HTTP 6 | class Response 7 | # @api private 8 | class Parser 9 | attr_reader :parser, :headers, :status_code, :http_version 10 | 11 | def initialize 12 | @handler = Handler.new(self) 13 | @parser = LLHttp::Parser.new(@handler, :type => :response) 14 | reset 15 | end 16 | 17 | def reset 18 | @parser.reset 19 | @handler.reset 20 | @header_finished = false 21 | @message_finished = false 22 | @headers = Headers.new 23 | @chunk = nil 24 | @status_code = nil 25 | @http_version = nil 26 | end 27 | 28 | def add(data) 29 | parser << data 30 | 31 | self 32 | rescue LLHttp::Error => e 33 | raise IOError, e.message 34 | end 35 | 36 | alias << add 37 | 38 | def mark_header_finished 39 | @header_finished = true 40 | @status_code = @parser.status_code 41 | @http_version = "#{@parser.http_major}.#{@parser.http_minor}" 42 | end 43 | 44 | def headers? 45 | @header_finished 46 | end 47 | 48 | def add_header(name, value) 49 | @headers.add(name, value) 50 | end 51 | 52 | def mark_message_finished 53 | @message_finished = true 54 | end 55 | 56 | def finished? 57 | @message_finished 58 | end 59 | 60 | def add_body(chunk) 61 | if @chunk 62 | @chunk << chunk 63 | else 64 | @chunk = chunk 65 | end 66 | end 67 | 68 | def read(size) 69 | return if @chunk.nil? 70 | 71 | if @chunk.bytesize <= size 72 | chunk = @chunk 73 | @chunk = nil 74 | else 75 | chunk = @chunk.byteslice(0, size) 76 | @chunk[0, size] = "" 77 | end 78 | 79 | chunk 80 | end 81 | 82 | class Handler < LLHttp::Delegate 83 | def initialize(target) 84 | @target = target 85 | super() 86 | reset 87 | end 88 | 89 | def reset 90 | @reading_header_value = false 91 | @field_value = +"" 92 | @field = +"" 93 | end 94 | 95 | def on_header_field(field) 96 | append_header if @reading_header_value 97 | @field << field 98 | end 99 | 100 | def on_header_value(value) 101 | @reading_header_value = true 102 | @field_value << value 103 | end 104 | 105 | def on_headers_complete 106 | append_header if @reading_header_value 107 | @target.mark_header_finished 108 | end 109 | 110 | def on_body(body) 111 | @target.add_body(body) 112 | end 113 | 114 | def on_message_complete 115 | @target.mark_message_finished 116 | end 117 | 118 | private 119 | 120 | def append_header 121 | @target.add_header(@field, @field_value) 122 | @reading_header_value = false 123 | @field_value = +"" 124 | @field = +"" 125 | end 126 | end 127 | end 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /lib/http/request/body.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module HTTP 4 | class Request 5 | class Body 6 | attr_reader :source 7 | 8 | def initialize(source) 9 | @source = source 10 | 11 | validate_source_type! 12 | end 13 | 14 | # Returns size which should be used for the "Content-Length" header. 15 | # 16 | # @return [Integer] 17 | def size 18 | if @source.is_a?(String) 19 | @source.bytesize 20 | elsif @source.respond_to?(:read) 21 | raise RequestError, "IO object must respond to #size" unless @source.respond_to?(:size) 22 | 23 | @source.size 24 | elsif @source.nil? 25 | 0 26 | else 27 | raise RequestError, "cannot determine size of body: #{@source.inspect}" 28 | end 29 | end 30 | 31 | # Yields chunks of content to be streamed to the request body. 32 | # 33 | # @yieldparam [String] 34 | def each(&block) 35 | if @source.is_a?(String) 36 | yield @source 37 | elsif @source.respond_to?(:read) 38 | IO.copy_stream(@source, ProcIO.new(block)) 39 | rewind(@source) 40 | elsif @source.is_a?(Enumerable) 41 | @source.each(&block) 42 | end 43 | 44 | self 45 | end 46 | 47 | # Request bodies are equivalent when they have the same source. 48 | def ==(other) 49 | self.class == other.class && self.source == other.source # rubocop:disable Style/RedundantSelf 50 | end 51 | 52 | private 53 | 54 | def rewind(io) 55 | io.rewind if io.respond_to? :rewind 56 | rescue Errno::ESPIPE, Errno::EPIPE 57 | # Pipe IOs respond to `:rewind` but fail when you call it. 58 | # 59 | # Calling `IO#rewind` on a pipe, fails with *ESPIPE* on MRI, 60 | # but *EPIPE* on jRuby. 61 | # 62 | # - **ESPIPE** -- "Illegal seek." 63 | # Invalid seek operation (such as on a pipe). 64 | # 65 | # - **EPIPE** -- "Broken pipe." 66 | # There is no process reading from the other end of a pipe. Every 67 | # library function that returns this error code also generates 68 | # a SIGPIPE signal; this signal terminates the program if not handled 69 | # or blocked. Thus, your program will never actually see EPIPE unless 70 | # it has handled or blocked SIGPIPE. 71 | # 72 | # See: https://www.gnu.org/software/libc/manual/html_node/Error-Codes.html 73 | nil 74 | end 75 | 76 | def validate_source_type! 77 | return if @source.is_a?(String) 78 | return if @source.respond_to?(:read) 79 | return if @source.is_a?(Enumerable) 80 | return if @source.nil? 81 | 82 | raise RequestError, "body of wrong type: #{@source.class}" 83 | end 84 | 85 | # This class provides a "writable IO" wrapper around a proc object, with 86 | # #write simply calling the proc, which we can pass in as the 87 | # "destination IO" in IO.copy_stream. 88 | class ProcIO 89 | def initialize(block) 90 | @block = block 91 | end 92 | 93 | def write(data) 94 | @block.call(data) 95 | data.bytesize 96 | end 97 | end 98 | end 99 | end 100 | end 101 | -------------------------------------------------------------------------------- /lib/http/timeout/global.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "timeout" 4 | require "io/wait" 5 | 6 | require "http/timeout/null" 7 | 8 | module HTTP 9 | module Timeout 10 | class Global < Null 11 | def initialize(*args) 12 | super 13 | 14 | @timeout = @time_left = options.fetch(:global_timeout) 15 | end 16 | 17 | # To future me: Don't remove this again, past you was smarter. 18 | def reset_counter 19 | @time_left = @timeout 20 | end 21 | 22 | def connect(socket_class, host, port, nodelay = false) 23 | reset_timer 24 | ::Timeout.timeout(@time_left, ConnectTimeoutError) do 25 | @socket = socket_class.open(host, port) 26 | @socket.setsockopt(Socket::IPPROTO_TCP, Socket::TCP_NODELAY, 1) if nodelay 27 | end 28 | 29 | log_time 30 | end 31 | 32 | def connect_ssl 33 | reset_timer 34 | 35 | begin 36 | @socket.connect_nonblock 37 | rescue IO::WaitReadable 38 | wait_readable_or_timeout 39 | retry 40 | rescue IO::WaitWritable 41 | wait_writable_or_timeout 42 | retry 43 | end 44 | end 45 | 46 | # Read from the socket 47 | def readpartial(size, buffer = nil) 48 | perform_io { read_nonblock(size, buffer) } 49 | end 50 | 51 | # Write to the socket 52 | def write(data) 53 | perform_io { write_nonblock(data) } 54 | end 55 | 56 | alias << write 57 | 58 | private 59 | 60 | def read_nonblock(size, buffer = nil) 61 | @socket.read_nonblock(size, buffer, :exception => false) 62 | end 63 | 64 | def write_nonblock(data) 65 | @socket.write_nonblock(data, :exception => false) 66 | end 67 | 68 | # Perform the given I/O operation with the given argument 69 | def perform_io 70 | reset_timer 71 | 72 | loop do 73 | result = yield 74 | 75 | case result 76 | when :wait_readable then wait_readable_or_timeout 77 | when :wait_writable then wait_writable_or_timeout 78 | when NilClass then return :eof 79 | else return result 80 | end 81 | rescue IO::WaitReadable 82 | wait_readable_or_timeout 83 | rescue IO::WaitWritable 84 | wait_writable_or_timeout 85 | end 86 | rescue EOFError 87 | :eof 88 | end 89 | 90 | # Wait for a socket to become readable 91 | def wait_readable_or_timeout 92 | @socket.to_io.wait_readable(@time_left) 93 | log_time 94 | end 95 | 96 | # Wait for a socket to become writable 97 | def wait_writable_or_timeout 98 | @socket.to_io.wait_writable(@time_left) 99 | log_time 100 | end 101 | 102 | # Due to the run/retry nature of nonblocking I/O, it's easier to keep track of time 103 | # via method calls instead of a block to monitor. 104 | def reset_timer 105 | @started = Time.now 106 | end 107 | 108 | def log_time 109 | @time_left -= (Time.now - @started) 110 | raise TimeoutError, "Timed out after using the allocated #{@timeout} seconds" if @time_left <= 0 111 | 112 | reset_timer 113 | end 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /lib/http/features/auto_deflate.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "zlib" 4 | require "tempfile" 5 | 6 | require "http/request/body" 7 | 8 | module HTTP 9 | module Features 10 | class AutoDeflate < Feature 11 | attr_reader :method 12 | 13 | def initialize(**) 14 | super 15 | 16 | @method = @opts.key?(:method) ? @opts[:method].to_s : "gzip" 17 | 18 | raise Error, "Only gzip and deflate methods are supported" unless %w[gzip deflate].include?(@method) 19 | end 20 | 21 | def wrap_request(request) 22 | return request unless method 23 | return request if request.body.size.zero? 24 | 25 | # We need to delete Content-Length header. It will be set automatically by HTTP::Request::Writer 26 | request.headers.delete(Headers::CONTENT_LENGTH) 27 | request.headers[Headers::CONTENT_ENCODING] = method 28 | 29 | Request.new( 30 | :version => request.version, 31 | :verb => request.verb, 32 | :uri => request.uri, 33 | :headers => request.headers, 34 | :proxy => request.proxy, 35 | :body => deflated_body(request.body), 36 | :uri_normalizer => request.uri_normalizer 37 | ) 38 | end 39 | 40 | def deflated_body(body) 41 | case method 42 | when "gzip" 43 | GzippedBody.new(body) 44 | when "deflate" 45 | DeflatedBody.new(body) 46 | end 47 | end 48 | 49 | HTTP::Options.register_feature(:auto_deflate, self) 50 | 51 | class CompressedBody < HTTP::Request::Body 52 | def initialize(uncompressed_body) 53 | @body = uncompressed_body 54 | @compressed = nil 55 | end 56 | 57 | def size 58 | compress_all! unless @compressed 59 | @compressed.size 60 | end 61 | 62 | def each(&block) 63 | return to_enum __method__ unless block 64 | 65 | if @compressed 66 | compressed_each(&block) 67 | else 68 | compress(&block) 69 | end 70 | 71 | self 72 | end 73 | 74 | private 75 | 76 | def compressed_each 77 | while (data = @compressed.read(Connection::BUFFER_SIZE)) 78 | yield data 79 | end 80 | ensure 81 | @compressed.close! 82 | end 83 | 84 | def compress_all! 85 | @compressed = Tempfile.new("http-compressed_body", :binmode => true) 86 | compress { |data| @compressed.write(data) } 87 | @compressed.rewind 88 | end 89 | end 90 | 91 | class GzippedBody < CompressedBody 92 | def compress(&block) 93 | gzip = Zlib::GzipWriter.new(BlockIO.new(block)) 94 | @body.each { |chunk| gzip.write(chunk) } 95 | ensure 96 | gzip.finish 97 | end 98 | 99 | class BlockIO 100 | def initialize(block) 101 | @block = block 102 | end 103 | 104 | def write(data) 105 | @block.call(data) 106 | end 107 | end 108 | end 109 | 110 | class DeflatedBody < CompressedBody 111 | def compress 112 | deflater = Zlib::Deflate.new 113 | 114 | @body.each { |chunk| yield deflater.deflate(chunk) } 115 | 116 | yield deflater.finish 117 | ensure 118 | deflater.close 119 | end 120 | end 121 | end 122 | end 123 | end 124 | -------------------------------------------------------------------------------- /lib/http/request/writer.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/headers" 4 | 5 | module HTTP 6 | class Request 7 | class Writer 8 | # CRLF is the universal HTTP delimiter 9 | CRLF = "\r\n" 10 | 11 | # Chunked data termintaor. 12 | ZERO = "0" 13 | 14 | # Chunked transfer encoding 15 | CHUNKED = "chunked" 16 | 17 | # End of a chunked transfer 18 | CHUNKED_END = "#{ZERO}#{CRLF}#{CRLF}" 19 | 20 | def initialize(socket, body, headers, headline) 21 | @body = body 22 | @socket = socket 23 | @headers = headers 24 | @request_header = [headline] 25 | end 26 | 27 | # Adds headers to the request header from the headers array 28 | def add_headers 29 | @headers.each do |field, value| 30 | @request_header << "#{field}: #{value}" 31 | end 32 | end 33 | 34 | # Stream the request to a socket 35 | def stream 36 | add_headers 37 | add_body_type_headers 38 | send_request 39 | end 40 | 41 | # Send headers needed to connect through proxy 42 | def connect_through_proxy 43 | add_headers 44 | write(join_headers) 45 | end 46 | 47 | # Adds the headers to the header array for the given request body we are working 48 | # with 49 | def add_body_type_headers 50 | return if @headers[Headers::CONTENT_LENGTH] || chunked? || ( 51 | @body.source.nil? && %w[GET HEAD DELETE CONNECT].any? do |method| 52 | @request_header[0].start_with?("#{method} ") 53 | end 54 | ) 55 | 56 | @request_header << "#{Headers::CONTENT_LENGTH}: #{@body.size}" 57 | end 58 | 59 | # Joins the headers specified in the request into a correctly formatted 60 | # http request header string 61 | def join_headers 62 | # join the headers array with crlfs, stick two on the end because 63 | # that ends the request header 64 | @request_header.join(CRLF) + (CRLF * 2) 65 | end 66 | 67 | # Writes HTTP request data into the socket. 68 | def send_request 69 | each_chunk { |chunk| write chunk } 70 | rescue Errno::EPIPE 71 | # server doesn't need any more data 72 | nil 73 | end 74 | 75 | # Yields chunks of request data that should be sent to the socket. 76 | # 77 | # It's important to send the request in a single write call when possible 78 | # in order to play nicely with Nagle's algorithm. Making two writes in a 79 | # row triggers a pathological case where Nagle is expecting a third write 80 | # that never happens. 81 | def each_chunk 82 | data = join_headers 83 | 84 | @body.each do |chunk| 85 | data << encode_chunk(chunk) 86 | yield data 87 | data.clear 88 | end 89 | 90 | yield data unless data.empty? 91 | 92 | yield CHUNKED_END if chunked? 93 | end 94 | 95 | # Returns the chunk encoded for to the specified "Transfer-Encoding" header. 96 | def encode_chunk(chunk) 97 | if chunked? 98 | chunk.bytesize.to_s(16) << CRLF << chunk << CRLF 99 | else 100 | chunk 101 | end 102 | end 103 | 104 | # Returns true if the request should be sent in chunked encoding. 105 | def chunked? 106 | @headers[Headers::TRANSFER_ENCODING] == CHUNKED 107 | end 108 | 109 | private 110 | 111 | def write(data) 112 | until data.empty? 113 | length = @socket.write(data) 114 | break unless data.bytesize > length 115 | 116 | data = data.byteslice(length..-1) 117 | end 118 | rescue Errno::EPIPE 119 | raise 120 | rescue IOError, SocketError, SystemCallError => e 121 | raise ConnectionError, "error writing to socket: #{e}", e.backtrace 122 | end 123 | end 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /spec/lib/http/request/writer_spec.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # frozen_string_literal: true 3 | 4 | RSpec.describe HTTP::Request::Writer do 5 | let(:io) { StringIO.new } 6 | let(:body) { HTTP::Request::Body.new("") } 7 | let(:headers) { HTTP::Headers.new } 8 | let(:headerstart) { "GET /test HTTP/1.1" } 9 | 10 | subject(:writer) { described_class.new(io, body, headers, headerstart) } 11 | 12 | describe "#stream" do 13 | context "when multiple headers are set" do 14 | let(:headers) { HTTP::Headers.coerce "Host" => "example.org" } 15 | 16 | it "separates headers with carriage return and line feed" do 17 | writer.stream 18 | expect(io.string).to eq [ 19 | "#{headerstart}\r\n", 20 | "Host: example.org\r\nContent-Length: 0\r\n\r\n" 21 | ].join 22 | end 23 | end 24 | 25 | context "when headers are specified as strings with mixed case" do 26 | let(:headers) { HTTP::Headers.coerce "content-Type" => "text", "X_MAX" => "200" } 27 | 28 | it "writes the headers with the same casing" do 29 | writer.stream 30 | expect(io.string).to eq [ 31 | "#{headerstart}\r\n", 32 | "content-Type: text\r\nX_MAX: 200\r\nContent-Length: 0\r\n\r\n" 33 | ].join 34 | end 35 | end 36 | 37 | context "when body is nonempty" do 38 | let(:body) { HTTP::Request::Body.new("content") } 39 | 40 | it "writes it to the socket and sets Content-Length" do 41 | writer.stream 42 | expect(io.string).to eq [ 43 | "#{headerstart}\r\n", 44 | "Content-Length: 7\r\n\r\n", 45 | "content" 46 | ].join 47 | end 48 | end 49 | 50 | context "when body is not set" do 51 | let(:body) { HTTP::Request::Body.new(nil) } 52 | 53 | it "doesn't write anything to the socket and doesn't set Content-Length" do 54 | writer.stream 55 | expect(io.string).to eq [ 56 | "#{headerstart}\r\n\r\n" 57 | ].join 58 | end 59 | end 60 | 61 | context "when body is empty" do 62 | let(:body) { HTTP::Request::Body.new("") } 63 | 64 | it "doesn't write anything to the socket and sets Content-Length" do 65 | writer.stream 66 | expect(io.string).to eq [ 67 | "#{headerstart}\r\n", 68 | "Content-Length: 0\r\n\r\n" 69 | ].join 70 | end 71 | end 72 | 73 | context "when Content-Length header is set" do 74 | let(:headers) { HTTP::Headers.coerce "Content-Length" => "12" } 75 | let(:body) { HTTP::Request::Body.new("content") } 76 | 77 | it "keeps the given value" do 78 | writer.stream 79 | expect(io.string).to eq [ 80 | "#{headerstart}\r\n", 81 | "Content-Length: 12\r\n\r\n", 82 | "content" 83 | ].join 84 | end 85 | end 86 | 87 | context "when Transfer-Encoding is chunked" do 88 | let(:headers) { HTTP::Headers.coerce "Transfer-Encoding" => "chunked" } 89 | let(:body) { HTTP::Request::Body.new(%w[request body]) } 90 | 91 | it "writes encoded content and omits Content-Length" do 92 | writer.stream 93 | expect(io.string).to eq [ 94 | "#{headerstart}\r\n", 95 | "Transfer-Encoding: chunked\r\n\r\n", 96 | "7\r\nrequest\r\n4\r\nbody\r\n0\r\n\r\n" 97 | ].join 98 | end 99 | end 100 | 101 | context "when server won't accept any more data" do 102 | before do 103 | expect(io).to receive(:write).and_raise(Errno::EPIPE) 104 | end 105 | 106 | it "aborts silently" do 107 | writer.stream 108 | end 109 | end 110 | 111 | context "when writing to socket raises an exception" do 112 | before do 113 | expect(io).to receive(:write).and_raise(Errno::ECONNRESET) 114 | end 115 | 116 | it "raises a ConnectionError" do 117 | expect { writer.stream }.to raise_error(HTTP::ConnectionError) 118 | end 119 | end 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "./support/simplecov" 4 | require_relative "./support/fuubar" unless ENV["CI"] 5 | 6 | require "http" 7 | require "rspec/its" 8 | require "support/capture_warning" 9 | require "support/fakeio" 10 | 11 | # See http://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration 12 | RSpec.configure do |config| 13 | config.expect_with :rspec do |expectations| 14 | # This option will default to `true` in RSpec 4. It makes the `description` 15 | # and `failure_message` of custom matchers include text for helper methods 16 | # defined using `chain`, e.g.: 17 | # be_bigger_than(2).and_smaller_than(4).description 18 | # # => "be bigger than 2 and smaller than 4" 19 | # ...rather than: 20 | # # => "be bigger than 2" 21 | expectations.include_chain_clauses_in_custom_matcher_descriptions = true 22 | end 23 | 24 | config.mock_with :rspec do |mocks| 25 | # Prevents you from mocking or stubbing a method that does not exist on 26 | # a real object. This is generally recommended, and will default to 27 | # `true` in RSpec 4. 28 | mocks.verify_partial_doubles = true 29 | end 30 | 31 | # This option will default to `:apply_to_host_groups` in RSpec 4 (and will 32 | # have no way to turn it off -- the option exists only for backwards 33 | # compatibility in RSpec 3). It causes shared context metadata to be 34 | # inherited by the metadata hash of host groups and examples, rather than 35 | # triggering implicit auto-inclusion in groups with matching metadata. 36 | config.shared_context_metadata_behavior = :apply_to_host_groups 37 | 38 | # These two settings work together to allow you to limit a spec run 39 | # to individual examples or groups you care about by tagging them with 40 | # `:focus` metadata. When nothing is tagged with `:focus`, all examples 41 | # get run. 42 | config.filter_run :focus 43 | config.filter_run_excluding :flaky if defined?(JRUBY_VERSION) && ENV["CI"] 44 | config.run_all_when_everything_filtered = true 45 | 46 | # This setting enables warnings. It's recommended, but in some cases may 47 | # be too noisy due to issues in dependencies. 48 | config.warnings = 0 == ENV["GUARD_RSPEC"].to_i 49 | 50 | # Allows RSpec to persist some state between runs in order to support 51 | # the `--only-failures` and `--next-failure` CLI options. We recommend 52 | # you configure your source control system to ignore this file. 53 | config.example_status_persistence_file_path = "spec/examples.txt" 54 | 55 | # Limits the available syntax to the non-monkey patched syntax that is 56 | # recommended. For more details, see: 57 | # - http://rspec.info/blog/2012/06/rspecs-new-expectation-syntax/ 58 | # - http://www.teaisaweso.me/blog/2013/05/27/rspecs-new-message-expectation-syntax/ 59 | # - http://rspec.info/blog/2014/05/notable-changes-in-rspec-3/#zero-monkey-patching-mode 60 | config.disable_monkey_patching! 61 | 62 | # Many RSpec users commonly either run the entire suite or an individual 63 | # file, and it's useful to allow more verbose output when running an 64 | # individual spec file. 65 | if config.files_to_run.one? 66 | # Use the documentation formatter for detailed output, 67 | # unless a formatter has already been configured 68 | # (e.g. via a command-line flag). 69 | config.default_formatter = "doc" 70 | end 71 | 72 | # Print the 10 slowest examples and example groups at the 73 | # end of the spec run, to help surface which specs are running 74 | # particularly slow. 75 | config.profile_examples = 10 76 | 77 | # Run specs in random order to surface order dependencies. If you find an 78 | # order dependency and want to debug it, you can fix the order by providing 79 | # the seed, which is printed after each run. 80 | # --seed 1234 81 | config.order = :random 82 | 83 | # Seed global randomization in this process using the `--seed` CLI option. 84 | # Setting this allows you to use `--seed` to deterministically reproduce 85 | # test failures related to randomization by passing the same `--seed` value 86 | # as the one that triggered the failure. 87 | Kernel.srand config.seed 88 | end 89 | -------------------------------------------------------------------------------- /lib/http/response/status.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "delegate" 4 | 5 | require "http/response/status/reasons" 6 | 7 | module HTTP 8 | class Response 9 | class Status < ::Delegator 10 | class << self 11 | # Coerces given value to Status. 12 | # 13 | # @example 14 | # 15 | # Status.coerce(:bad_request) # => Status.new(400) 16 | # Status.coerce("400") # => Status.new(400) 17 | # Status.coerce(true) # => raises HTTP::Error 18 | # 19 | # @raise [Error] if coercion is impossible 20 | # @param [Symbol, #to_i] object 21 | # @return [Status] 22 | def coerce(object) 23 | code = case 24 | when object.is_a?(String) then SYMBOL_CODES[symbolize object] 25 | when object.is_a?(Symbol) then SYMBOL_CODES[object] 26 | when object.is_a?(Numeric) then object.to_i 27 | end 28 | 29 | return new code if code 30 | 31 | raise Error, "Can't coerce #{object.class}(#{object}) to #{self}" 32 | end 33 | alias [] coerce 34 | 35 | private 36 | 37 | # Symbolizes given string 38 | # 39 | # @example 40 | # 41 | # symbolize "Bad Request" # => :bad_request 42 | # symbolize "Request-URI Too Long" # => :request_uri_too_long 43 | # symbolize "I'm a Teapot" # => :im_a_teapot 44 | # 45 | # @param [#to_s] str 46 | # @return [Symbol] 47 | def symbolize(str) 48 | str.to_s.downcase.tr("-", " ").gsub(/[^a-z ]/, "").gsub(/\s+/, "_").to_sym 49 | end 50 | end 51 | 52 | # Code to Symbol map 53 | # 54 | # @example Usage 55 | # 56 | # SYMBOLS[400] # => :bad_request 57 | # SYMBOLS[414] # => :request_uri_too_long 58 | # SYMBOLS[418] # => :im_a_teapot 59 | # 60 | # @return [Hash Symbol>] 61 | SYMBOLS = REASONS.transform_values { |v| symbolize(v) }.freeze 62 | 63 | # Reversed {SYMBOLS} map. 64 | # 65 | # @example Usage 66 | # 67 | # SYMBOL_CODES[:bad_request] # => 400 68 | # SYMBOL_CODES[:request_uri_too_long] # => 414 69 | # SYMBOL_CODES[:im_a_teapot] # => 418 70 | # 71 | # @return [Hash Fixnum>] 72 | SYMBOL_CODES = SYMBOLS.to_h { |k, v| [v, k] }.freeze 73 | 74 | # @return [Fixnum] status code 75 | attr_reader :code 76 | 77 | # @see REASONS 78 | # @return [String, nil] status message 79 | def reason 80 | REASONS[code] 81 | end 82 | 83 | # @return [String] string representation of HTTP status 84 | def to_s 85 | "#{code} #{reason}".strip 86 | end 87 | 88 | # Check if status code is informational (1XX) 89 | # @return [Boolean] 90 | def informational? 91 | 100 <= code && code < 200 92 | end 93 | 94 | # Check if status code is successful (2XX) 95 | # @return [Boolean] 96 | def success? 97 | 200 <= code && code < 300 98 | end 99 | 100 | # Check if status code is redirection (3XX) 101 | # @return [Boolean] 102 | def redirect? 103 | 300 <= code && code < 400 104 | end 105 | 106 | # Check if status code is client error (4XX) 107 | # @return [Boolean] 108 | def client_error? 109 | 400 <= code && code < 500 110 | end 111 | 112 | # Check if status code is server error (5XX) 113 | # @return [Boolean] 114 | def server_error? 115 | 500 <= code && code < 600 116 | end 117 | 118 | # Symbolized {#reason} 119 | # 120 | # @return [nil] unless code is well-known (see REASONS) 121 | # @return [Symbol] 122 | def to_sym 123 | SYMBOLS[code] 124 | end 125 | 126 | # Printable version of HTTP Status, surrounded by quote marks, 127 | # with special characters escaped. 128 | # 129 | # (see String#inspect) 130 | def inspect 131 | "#<#{self.class} #{self}>" 132 | end 133 | 134 | SYMBOLS.each do |code, symbol| 135 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 136 | def #{symbol}? # def bad_request? 137 | #{code} == code # 400 == code 138 | end # end 139 | RUBY 140 | end 141 | 142 | def __setobj__(obj) 143 | raise TypeError, "Expected #{obj.inspect} to respond to #to_i" unless obj.respond_to? :to_i 144 | 145 | @code = obj.to_i 146 | end 147 | 148 | def __getobj__ 149 | @code 150 | end 151 | end 152 | end 153 | end 154 | -------------------------------------------------------------------------------- /lib/http/uri.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "addressable/uri" 4 | 5 | module HTTP 6 | class URI 7 | extend Forwardable 8 | 9 | def_delegators :@uri, :scheme, :normalized_scheme, :scheme= 10 | def_delegators :@uri, :user, :normalized_user, :user= 11 | def_delegators :@uri, :password, :normalized_password, :password= 12 | def_delegators :@uri, :host, :normalized_host, :host= 13 | def_delegators :@uri, :authority, :normalized_authority, :authority= 14 | def_delegators :@uri, :origin, :origin= 15 | def_delegators :@uri, :normalized_port, :port= 16 | def_delegators :@uri, :path, :normalized_path, :path= 17 | def_delegators :@uri, :query, :normalized_query, :query= 18 | def_delegators :@uri, :query_values, :query_values= 19 | def_delegators :@uri, :request_uri, :request_uri= 20 | def_delegators :@uri, :fragment, :normalized_fragment, :fragment= 21 | def_delegators :@uri, :omit, :join, :normalize 22 | 23 | # @private 24 | HTTP_SCHEME = "http" 25 | 26 | # @private 27 | HTTPS_SCHEME = "https" 28 | 29 | # @private 30 | NORMALIZER = lambda do |uri| 31 | uri = HTTP::URI.parse uri 32 | 33 | HTTP::URI.new( 34 | :scheme => uri.normalized_scheme, 35 | :authority => uri.normalized_authority, 36 | :path => uri.normalized_path, 37 | :query => uri.query, 38 | :fragment => uri.normalized_fragment 39 | ) 40 | end 41 | 42 | # Parse the given URI string, returning an HTTP::URI object 43 | # 44 | # @param [HTTP::URI, String, #to_str] uri to parse 45 | # 46 | # @return [HTTP::URI] new URI instance 47 | def self.parse(uri) 48 | return uri if uri.is_a?(self) 49 | 50 | new(Addressable::URI.parse(uri)) 51 | end 52 | 53 | # Encodes key/value pairs as application/x-www-form-urlencoded 54 | # 55 | # @param [#to_hash, #to_ary] form_values to encode 56 | # @param [TrueClass, FalseClass] sort should key/value pairs be sorted first? 57 | # 58 | # @return [String] encoded value 59 | def self.form_encode(form_values, sort = false) 60 | Addressable::URI.form_encode(form_values, sort) 61 | end 62 | 63 | # Creates an HTTP::URI instance from the given options 64 | # 65 | # @param [Hash, Addressable::URI] options_or_uri 66 | # 67 | # @option options_or_uri [String, #to_str] :scheme URI scheme 68 | # @option options_or_uri [String, #to_str] :user for basic authentication 69 | # @option options_or_uri [String, #to_str] :password for basic authentication 70 | # @option options_or_uri [String, #to_str] :host name component 71 | # @option options_or_uri [String, #to_str] :port network port to connect to 72 | # @option options_or_uri [String, #to_str] :path component to request 73 | # @option options_or_uri [String, #to_str] :query component distinct from path 74 | # @option options_or_uri [String, #to_str] :fragment component at the end of the URI 75 | # 76 | # @return [HTTP::URI] new URI instance 77 | def initialize(options_or_uri = {}) 78 | case options_or_uri 79 | when Hash 80 | @uri = Addressable::URI.new(options_or_uri) 81 | when Addressable::URI 82 | @uri = options_or_uri 83 | else 84 | raise TypeError, "expected Hash for options, got #{options_or_uri.class}" 85 | end 86 | end 87 | 88 | # Are these URI objects equal? Normalizes both URIs prior to comparison 89 | # 90 | # @param [Object] other URI to compare this one with 91 | # 92 | # @return [TrueClass, FalseClass] are the URIs equivalent (after normalization)? 93 | def ==(other) 94 | other.is_a?(URI) && normalize.to_s == other.normalize.to_s 95 | end 96 | 97 | # Are these URI objects equal? Does NOT normalizes both URIs prior to comparison 98 | # 99 | # @param [Object] other URI to compare this one with 100 | # 101 | # @return [TrueClass, FalseClass] are the URIs equivalent? 102 | def eql?(other) 103 | other.is_a?(URI) && to_s == other.to_s 104 | end 105 | 106 | # Hash value based off the normalized form of a URI 107 | # 108 | # @return [Integer] A hash of the URI 109 | def hash 110 | @hash ||= to_s.hash * -1 111 | end 112 | 113 | # Port number, either as specified or the default if unspecified 114 | # 115 | # @return [Integer] port number 116 | def port 117 | @uri.port || @uri.default_port 118 | end 119 | 120 | # @return [True] if URI is HTTP 121 | # @return [False] otherwise 122 | def http? 123 | HTTP_SCHEME == scheme 124 | end 125 | 126 | # @return [True] if URI is HTTPS 127 | # @return [False] otherwise 128 | def https? 129 | HTTPS_SCHEME == scheme 130 | end 131 | 132 | # @return [Object] duplicated URI 133 | def dup 134 | self.class.new @uri.dup 135 | end 136 | 137 | # Convert an HTTP::URI to a String 138 | # 139 | # @return [String] URI serialized as a String 140 | def to_s 141 | @uri.to_s 142 | end 143 | alias to_str to_s 144 | 145 | # @return [String] human-readable representation of URI 146 | def inspect 147 | format("#<%s:0x%014x URI:%s>", self.class.name, object_id << 1, to_s) 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /lib/http/redirector.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "set" 4 | 5 | require "http/headers" 6 | 7 | module HTTP 8 | class Redirector 9 | # Notifies that we reached max allowed redirect hops 10 | class TooManyRedirectsError < ResponseError; end 11 | 12 | # Notifies that following redirects got into an endless loop 13 | class EndlessRedirectError < TooManyRedirectsError; end 14 | 15 | # HTTP status codes which indicate redirects 16 | REDIRECT_CODES = [300, 301, 302, 303, 307, 308].to_set.freeze 17 | 18 | # Codes which which should raise StateError in strict mode if original 19 | # request was any of {UNSAFE_VERBS} 20 | STRICT_SENSITIVE_CODES = [300, 301, 302].to_set.freeze 21 | 22 | # Insecure http verbs, which should trigger StateError in strict mode 23 | # upon {STRICT_SENSITIVE_CODES} 24 | UNSAFE_VERBS = %i[put delete post].to_set.freeze 25 | 26 | # Verbs which will remain unchanged upon See Other response. 27 | SEE_OTHER_ALLOWED_VERBS = %i[get head].to_set.freeze 28 | 29 | # @!attribute [r] strict 30 | # Returns redirector policy. 31 | # @return [Boolean] 32 | attr_reader :strict 33 | 34 | # @!attribute [r] max_hops 35 | # Returns maximum allowed hops. 36 | # @return [Fixnum] 37 | attr_reader :max_hops 38 | 39 | # @param [Hash] opts 40 | # @option opts [Boolean] :strict (true) redirector hops policy 41 | # @option opts [#to_i] :max_hops (5) maximum allowed amount of hops 42 | def initialize(opts = {}) 43 | @strict = opts.fetch(:strict, true) 44 | @max_hops = opts.fetch(:max_hops, 5).to_i 45 | end 46 | 47 | # Follows redirects until non-redirect response found 48 | def perform(request, response) 49 | @request = request 50 | @response = response 51 | @visited = [] 52 | collect_cookies_from_request 53 | collect_cookies_from_response 54 | 55 | while REDIRECT_CODES.include? @response.status.code 56 | @visited << "#{@request.verb} #{@request.uri}" 57 | 58 | raise TooManyRedirectsError if too_many_hops? 59 | raise EndlessRedirectError if endless_loop? 60 | 61 | @response.flush 62 | 63 | # XXX(ixti): using `Array#inject` to return `nil` if no Location header. 64 | @request = redirect_to(@response.headers.get(Headers::LOCATION).inject(:+)) 65 | unless cookie_jar.empty? 66 | @request.headers.set(Headers::COOKIE, cookie_jar.cookies.map { |c| "#{c.name}=#{c.value}" }.join("; ")) 67 | end 68 | @response = yield @request 69 | collect_cookies_from_response 70 | end 71 | 72 | @response 73 | end 74 | 75 | private 76 | 77 | # All known cookies. On the original request, this is only the original cookies, but after that, 78 | # Set-Cookie headers can add, set or delete cookies. 79 | def cookie_jar 80 | # it seems that @response.cookies instance is reused between responses, so we have to "clone" 81 | @cookie_jar ||= HTTP::CookieJar.new 82 | end 83 | 84 | def collect_cookies_from_request 85 | request_cookie_header = @request.headers["Cookie"] 86 | cookies = 87 | if request_cookie_header 88 | HTTP::Cookie.cookie_value_to_hash(request_cookie_header) 89 | else 90 | {} 91 | end 92 | 93 | cookies.each do |key, value| 94 | cookie_jar.add(HTTP::Cookie.new(key, value, :path => @request.uri.path, :domain => @request.host)) 95 | end 96 | end 97 | 98 | # Carry cookies from one response to the next. Carrying cookies to the next response ends up 99 | # carrying them to the next request as well. 100 | # 101 | # Note that this isn't part of the IETF standard, but all major browsers support setting cookies 102 | # on redirect: https://blog.dubbelboer.com/2012/11/25/302-cookie.html 103 | def collect_cookies_from_response 104 | # Overwrite previous cookies 105 | @response.cookies.each do |cookie| 106 | if cookie.value == "" 107 | cookie_jar.delete(cookie) 108 | else 109 | cookie_jar.add(cookie) 110 | end 111 | end 112 | 113 | # I wish we could just do @response.cookes = cookie_jar 114 | cookie_jar.each do |cookie| 115 | @response.cookies.add(cookie) 116 | end 117 | end 118 | 119 | # Check if we reached max amount of redirect hops 120 | # @return [Boolean] 121 | def too_many_hops? 122 | 1 <= @max_hops && @max_hops < @visited.count 123 | end 124 | 125 | # Check if we got into an endless loop 126 | # @return [Boolean] 127 | def endless_loop? 128 | 2 <= @visited.count(@visited.last) 129 | end 130 | 131 | # Redirect policy for follow 132 | # @return [Request] 133 | def redirect_to(uri) 134 | raise StateError, "no Location header in redirect" unless uri 135 | 136 | verb = @request.verb 137 | code = @response.status.code 138 | 139 | if UNSAFE_VERBS.include?(verb) && STRICT_SENSITIVE_CODES.include?(code) 140 | raise StateError, "can't follow #{@response.status} redirect" if @strict 141 | 142 | verb = :get 143 | end 144 | 145 | verb = :get if !SEE_OTHER_ALLOWED_VERBS.include?(verb) && 303 == code 146 | 147 | @request.redirect(uri, verb) 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /spec/support/dummy_server/servlet.rb: -------------------------------------------------------------------------------- 1 | # encoding: UTF-8 2 | # frozen_string_literal: true 3 | 4 | require "cgi" 5 | 6 | class DummyServer < WEBrick::HTTPServer 7 | class Servlet < WEBrick::HTTPServlet::AbstractServlet # rubocop:disable Metrics/ClassLength 8 | def self.sockets 9 | @sockets ||= [] 10 | end 11 | 12 | def not_found(_req, res) 13 | res.status = 404 14 | res.body = "Not Found" 15 | end 16 | 17 | def self.handlers 18 | @handlers ||= {} 19 | end 20 | 21 | %w[get post head].each do |method| 22 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 23 | def self.#{method}(path, &block) 24 | handlers["#{method}:\#{path}"] = block 25 | end 26 | 27 | def do_#{method.upcase}(req, res) 28 | handler = self.class.handlers["#{method}:\#{req.path}"] 29 | return instance_exec(req, res, &handler) if handler 30 | not_found 31 | end 32 | RUBY 33 | end 34 | 35 | get "/" do |req, res| 36 | res.status = 200 37 | 38 | case req["Accept"] 39 | when "application/json" 40 | res["Content-Type"] = "application/json" 41 | res.body = '{"json": true}' 42 | else 43 | res["Content-Type"] = "text/html" 44 | res.body = "" 45 | end 46 | end 47 | 48 | get "/sleep" do |_, res| 49 | sleep 2 50 | 51 | res.status = 200 52 | res.body = "hello" 53 | end 54 | 55 | post "/sleep" do |_, res| 56 | sleep 2 57 | 58 | res.status = 200 59 | res.body = "hello" 60 | end 61 | 62 | ["", "/1", "/2"].each do |path| 63 | get "/socket#{path}" do |req, res| 64 | self.class.sockets << req.instance_variable_get(:@socket) 65 | res.status = 200 66 | res.body = req.instance_variable_get(:@socket).object_id.to_s 67 | end 68 | end 69 | 70 | get "/params" do |req, res| 71 | next not_found unless "foo=bar" == req.query_string 72 | 73 | res.status = 200 74 | res.body = "Params!" 75 | end 76 | 77 | get "/multiple-params" do |req, res| 78 | params = CGI.parse req.query_string 79 | 80 | next not_found unless {"foo" => ["bar"], "baz" => ["quux"]} == params 81 | 82 | res.status = 200 83 | res.body = "More Params!" 84 | end 85 | 86 | get "/proxy" do |_req, res| 87 | res.status = 200 88 | res.body = "Proxy!" 89 | end 90 | 91 | get "/not-found" do |_req, res| 92 | res.status = 404 93 | res.body = "not found" 94 | end 95 | 96 | get "/redirect-301" do |_req, res| 97 | res.status = 301 98 | res["Location"] = "http://#{@server.config[:BindAddress]}:#{@server.config[:Port]}/" 99 | end 100 | 101 | get "/redirect-302" do |_req, res| 102 | res.status = 302 103 | res["Location"] = "http://#{@server.config[:BindAddress]}:#{@server.config[:Port]}/" 104 | end 105 | 106 | post "/form" do |req, res| 107 | if "testing-form" == req.query["example"] 108 | res.status = 200 109 | res.body = "passed :)" 110 | else 111 | res.status = 400 112 | res.body = "invalid! >:E" 113 | end 114 | end 115 | 116 | post "/body" do |req, res| 117 | if "testing-body" == req.body 118 | res.status = 200 119 | res.body = "passed :)" 120 | else 121 | res.status = 400 122 | res.body = "invalid! >:E" 123 | end 124 | end 125 | 126 | head "/" do |_req, res| 127 | res.status = 200 128 | res["Content-Type"] = "text/html" 129 | end 130 | 131 | get "/bytes" do |_req, res| 132 | bytes = [80, 75, 3, 4, 20, 0, 0, 0, 8, 0, 123, 104, 169, 70, 99, 243, 243] 133 | res["Content-Type"] = "application/octet-stream" 134 | res.body = bytes.pack("c*") 135 | end 136 | 137 | get "/iso-8859-1" do |_req, res| 138 | res["Content-Type"] = "text/plain; charset=ISO-8859-1" 139 | res.body = "testæ".encode(Encoding::ISO8859_1) 140 | end 141 | 142 | get "/cookies" do |req, res| 143 | res["Set-Cookie"] = "foo=bar" 144 | res.body = req.cookies.map { |c| [c.name, c.value].join ": " }.join("\n") 145 | end 146 | 147 | post "/echo-body" do |req, res| 148 | res.status = 200 149 | res.body = req.body 150 | end 151 | 152 | get "/hello world" do |_req, res| 153 | res.status = 200 154 | res.body = "hello world" 155 | end 156 | 157 | post "/encoded-body" do |req, res| 158 | res.status = 200 159 | 160 | res.body = case req["Accept-Encoding"] 161 | when "gzip" 162 | res["Content-Encoding"] = "gzip" 163 | StringIO.open do |out| 164 | Zlib::GzipWriter.wrap(out) do |gz| 165 | gz.write "#{req.body}-gzipped" 166 | gz.finish 167 | out.tap(&:rewind).read 168 | end 169 | end 170 | when "deflate" 171 | res["Content-Encoding"] = "deflate" 172 | Zlib::Deflate.deflate("#{req.body}-deflated") 173 | else 174 | "#{req.body}-raw" 175 | end 176 | end 177 | 178 | post "/no-content-204" do |req, res| 179 | res.status = 204 180 | res.body = "" 181 | 182 | case req["Accept-Encoding"] 183 | when "gzip" 184 | res["Content-Encoding"] = "gzip" 185 | when "deflate" 186 | res["Content-Encoding"] = "deflate" 187 | end 188 | end 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ![http.rb](https://raw.github.com/httprb/http.rb/main/logo.png) 2 | 3 | [![Gem Version][gem-image]][gem-link] 4 | [![MIT licensed][license-image]][license-link] 5 | [![Build Status][build-image]][build-link] 6 | [![Code Climate][codeclimate-image]][codeclimate-link] 7 | 8 | [Documentation] 9 | 10 | ## About 11 | 12 | HTTP (The Gem! a.k.a. http.rb) is an easy-to-use client library for making requests 13 | from Ruby. It uses a simple method chaining system for building requests, similar to 14 | Python's [Requests]. 15 | 16 | Under the hood, http.rb uses the [llhttp] parser, a fast HTTP parsing native extension. 17 | This library isn't just yet another wrapper around `Net::HTTP`. It implements the HTTP 18 | protocol natively and outsources the parsing to native extensions. 19 | 20 | ### Why http.rb? 21 | 22 | - **Clean API**: http.rb offers an easy-to-use API that should be a 23 | breath of fresh air after using something like Net::HTTP. 24 | 25 | - **Maturity**: http.rb is one of the most mature Ruby HTTP clients, supporting 26 | features like persistent connections and fine-grained timeouts. 27 | 28 | - **Performance**: using native parsers and a clean, lightweight implementation, 29 | http.rb achieves high performance while implementing HTTP in Ruby instead of C. 30 | 31 | 32 | ## Installation 33 | 34 | Add this line to your application's Gemfile: 35 | ```ruby 36 | gem "http" 37 | ``` 38 | 39 | And then execute: 40 | ```bash 41 | $ bundle 42 | ``` 43 | 44 | Or install it yourself as: 45 | ```bash 46 | $ gem install http 47 | ``` 48 | 49 | Inside of your Ruby program do: 50 | ```ruby 51 | require "http" 52 | ``` 53 | 54 | ...to pull it in as a dependency. 55 | 56 | 57 | ## Documentation 58 | 59 | [Please see the http.rb wiki][documentation] 60 | for more detailed documentation and usage notes. 61 | 62 | The following API documentation is also available: 63 | 64 | - [YARD API documentation](https://www.rubydoc.info/github/httprb/http) 65 | - [Chainable module (all chainable methods)](https://www.rubydoc.info/github/httprb/http/HTTP/Chainable) 66 | 67 | 68 | ### Basic Usage 69 | 70 | Here's some simple examples to get you started: 71 | 72 | ```ruby 73 | >> HTTP.get("https://github.com").to_s 74 | => "\n\n\n\n\n > HTTP.get("https://github.com") 82 | => #"GitHub.com", "Date"=>"Tue, 10 May...> 83 | ``` 84 | 85 | We can also obtain an `HTTP::Response::Body` object for this response: 86 | 87 | ```ruby 88 | >> HTTP.get("https://github.com").body 89 | => # 90 | ``` 91 | 92 | The response body can be streamed with `HTTP::Response::Body#readpartial`. 93 | In practice, you'll want to bind the `HTTP::Response::Body` to a local variable 94 | and call `#readpartial` on it repeatedly until it returns `nil`: 95 | 96 | ```ruby 97 | >> body = HTTP.get("https://github.com").body 98 | => # 99 | >> body.readpartial 100 | => "\n\n\n\n\n > body.readpartial 102 | => "\" href=\"/apple-touch-icon-72x72.png\">\n > body.readpartial 105 | => nil 106 | ``` 107 | 108 | ## Supported Ruby Versions 109 | 110 | This library aims to support and is [tested against][build-link] 111 | the following Ruby versions: 112 | 113 | - Ruby 2.6 114 | - Ruby 2.7 115 | - Ruby 3.0 116 | - Ruby 3.1 117 | - JRuby 9.3 118 | 119 | If something doesn't work on one of these versions, it's a bug. 120 | 121 | This library may inadvertently work (or seem to work) on other Ruby versions, 122 | however support will only be provided for the versions listed above. 123 | 124 | If you would like this library to support another Ruby version or 125 | implementation, you may volunteer to be a maintainer. Being a maintainer 126 | entails making sure all tests run and pass on that implementation. When 127 | something breaks on your implementation, you will be responsible for providing 128 | patches in a timely fashion. If critical issues for a particular implementation 129 | exist at the time of a major release, support for that Ruby version may be 130 | dropped. 131 | 132 | 133 | ## Contributing to http.rb 134 | 135 | - Fork http.rb on GitHub 136 | - Make your changes 137 | - Ensure all tests pass (`bundle exec rake`) 138 | - Send a pull request 139 | - If we like them we'll merge them 140 | - If we've accepted a patch, feel free to ask for commit access! 141 | 142 | 143 | ## Copyright 144 | 145 | Copyright © 2011-2022 Tony Arcieri, Alexey V. Zapparov, Erik Michaels-Ober, Zachary Anker. 146 | See LICENSE.txt for further details. 147 | 148 | 149 | [//]: # (badges) 150 | 151 | [gem-image]: https://img.shields.io/gem/v/http?logo=ruby 152 | [gem-link]: https://rubygems.org/gems/http 153 | [license-image]: https://img.shields.io/badge/license-MIT-blue.svg 154 | [license-link]: https://github.com/httprb/http/blob/main/LICENSE.txt 155 | [build-image]: https://github.com/httprb/http/workflows/CI/badge.svg 156 | [build-link]: https://github.com/httprb/http/actions/workflows/ci.yml 157 | [codeclimate-image]: https://codeclimate.com/github/httprb/http.svg?branch=main 158 | [codeclimate-link]: https://codeclimate.com/github/httprb/http 159 | 160 | [//]: # (links) 161 | 162 | [documentation]: https://github.com/httprb/http/wiki 163 | [requests]: http://docs.python-requests.org/en/latest/ 164 | [llhttp]: https://llhttp.org/ 165 | -------------------------------------------------------------------------------- /spec/lib/http/request/body_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Request::Body do 4 | let(:body) { "" } 5 | subject { HTTP::Request::Body.new(body) } 6 | 7 | describe "#initialize" do 8 | context "when body is nil" do 9 | let(:body) { nil } 10 | 11 | it "does not raise an error" do 12 | expect { subject }.not_to raise_error 13 | end 14 | end 15 | 16 | context "when body is a string" do 17 | let(:body) { "string body" } 18 | 19 | it "does not raise an error" do 20 | expect { subject }.not_to raise_error 21 | end 22 | end 23 | 24 | context "when body is an IO" do 25 | let(:body) { FakeIO.new("IO body") } 26 | 27 | it "does not raise an error" do 28 | expect { subject }.not_to raise_error 29 | end 30 | end 31 | 32 | context "when body is an Enumerable" do 33 | let(:body) { %w[bees cows] } 34 | 35 | it "does not raise an error" do 36 | expect { subject }.not_to raise_error 37 | end 38 | end 39 | 40 | context "when body is of unrecognized type" do 41 | let(:body) { 123 } 42 | 43 | it "raises an error" do 44 | expect { subject }.to raise_error(HTTP::RequestError) 45 | end 46 | end 47 | end 48 | 49 | describe "#source" do 50 | it "returns the original object" do 51 | expect(subject.source).to eq "" 52 | end 53 | end 54 | 55 | describe "#size" do 56 | context "when body is nil" do 57 | let(:body) { nil } 58 | 59 | it "returns zero" do 60 | expect(subject.size).to eq 0 61 | end 62 | end 63 | 64 | context "when body is a string" do 65 | let(:body) { "Привет, мир!" } 66 | 67 | it "returns string bytesize" do 68 | expect(subject.size).to eq 21 69 | end 70 | end 71 | 72 | context "when body is an IO with size" do 73 | let(:body) { FakeIO.new("content") } 74 | 75 | it "returns IO size" do 76 | expect(subject.size).to eq 7 77 | end 78 | end 79 | 80 | context "when body is an IO without size" do 81 | let(:body) { IO.pipe[0] } 82 | 83 | it "raises a RequestError" do 84 | expect { subject.size }.to raise_error(HTTP::RequestError) 85 | end 86 | end 87 | 88 | context "when body is an Enumerable" do 89 | let(:body) { %w[bees cows] } 90 | 91 | it "raises a RequestError" do 92 | expect { subject.size }.to raise_error(HTTP::RequestError) 93 | end 94 | end 95 | end 96 | 97 | describe "#each" do 98 | let(:chunks) do 99 | chunks = [] 100 | subject.each { |chunk| chunks << chunk.dup } 101 | chunks 102 | end 103 | 104 | context "when body is nil" do 105 | let(:body) { nil } 106 | 107 | it "yields nothing" do 108 | expect(chunks).to eq [] 109 | end 110 | end 111 | 112 | context "when body is a string" do 113 | let(:body) { "content" } 114 | 115 | it "yields the string" do 116 | expect(chunks).to eq %w[content] 117 | end 118 | end 119 | 120 | context "when body is a non-Enumerable IO" do 121 | let(:body) { FakeIO.new(("a" * 16 * 1024) + ("b" * 10 * 1024)) } 122 | 123 | it "yields chunks of content" do 124 | expect(chunks.inject("", :+)).to eq ("a" * 16 * 1024) + ("b" * 10 * 1024) 125 | end 126 | end 127 | 128 | context "when body is a pipe" do 129 | let(:ios) { IO.pipe } 130 | let(:body) { ios[0] } 131 | 132 | around do |example| 133 | writer = Thread.new(ios[1]) do |io| 134 | io << "abcdef" 135 | io.close 136 | end 137 | 138 | begin 139 | example.run 140 | ensure 141 | writer.join 142 | end 143 | end 144 | 145 | it "yields chunks of content" do 146 | expect(chunks.inject("", :+)).to eq("abcdef") 147 | end 148 | end 149 | 150 | context "when body is an Enumerable IO" do 151 | let(:data) { ("a" * 16 * 1024) + ("b" * 10 * 1024) } 152 | let(:body) { StringIO.new data } 153 | 154 | it "yields chunks of content" do 155 | expect(chunks.inject("", :+)).to eq data 156 | end 157 | 158 | it "allows to enumerate multiple times" do 159 | results = [] 160 | 161 | 2.times do 162 | result = "" 163 | subject.each { |chunk| result += chunk } 164 | results << result 165 | end 166 | 167 | aggregate_failures do 168 | expect(results.count).to eq 2 169 | expect(results).to all eq data 170 | end 171 | end 172 | end 173 | 174 | context "when body is an Enumerable" do 175 | let(:body) { %w[bees cows] } 176 | 177 | it "yields elements" do 178 | expect(chunks).to eq %w[bees cows] 179 | end 180 | end 181 | end 182 | 183 | describe "#==" do 184 | context "when sources are equivalent" do 185 | let(:body1) { HTTP::Request::Body.new("content") } 186 | let(:body2) { HTTP::Request::Body.new("content") } 187 | 188 | it "returns true" do 189 | expect(body1).to eq body2 190 | end 191 | end 192 | 193 | context "when sources are not equivalent" do 194 | let(:body1) { HTTP::Request::Body.new("content") } 195 | let(:body2) { HTTP::Request::Body.new(nil) } 196 | 197 | it "returns false" do 198 | expect(body1).not_to eq body2 199 | end 200 | end 201 | 202 | context "when objects are not of the same class" do 203 | let(:body1) { HTTP::Request::Body.new("content") } 204 | let(:body2) { "content" } 205 | 206 | it "returns false" do 207 | expect(body1).not_to eq body2 208 | end 209 | end 210 | end 211 | end 212 | -------------------------------------------------------------------------------- /lib/http/options.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "http/headers" 4 | require "openssl" 5 | require "socket" 6 | require "http/uri" 7 | 8 | module HTTP 9 | class Options # rubocop:disable Metrics/ClassLength 10 | @default_socket_class = TCPSocket 11 | @default_ssl_socket_class = OpenSSL::SSL::SSLSocket 12 | @default_timeout_class = HTTP::Timeout::Null 13 | @available_features = {} 14 | 15 | class << self 16 | attr_accessor :default_socket_class, :default_ssl_socket_class, :default_timeout_class 17 | attr_reader :available_features 18 | 19 | def new(options = {}) 20 | options.is_a?(self) ? options : super 21 | end 22 | 23 | def defined_options 24 | @defined_options ||= [] 25 | end 26 | 27 | def register_feature(name, impl) 28 | @available_features[name] = impl 29 | end 30 | 31 | protected 32 | 33 | def def_option(name, reader_only: false, &interpreter) 34 | defined_options << name.to_sym 35 | interpreter ||= ->(v) { v } 36 | 37 | if reader_only 38 | attr_reader name 39 | else 40 | attr_accessor name 41 | protected :"#{name}=" 42 | end 43 | 44 | define_method(:"with_#{name}") do |value| 45 | dup { |opts| opts.send(:"#{name}=", instance_exec(value, &interpreter)) } 46 | end 47 | end 48 | end 49 | 50 | def initialize(options = {}) 51 | defaults = { 52 | :response => :auto, 53 | :proxy => {}, 54 | :timeout_class => self.class.default_timeout_class, 55 | :timeout_options => {}, 56 | :socket_class => self.class.default_socket_class, 57 | :nodelay => false, 58 | :ssl_socket_class => self.class.default_ssl_socket_class, 59 | :ssl => {}, 60 | :keep_alive_timeout => 5, 61 | :headers => {}, 62 | :cookies => {}, 63 | :encoding => nil, 64 | :features => {} 65 | } 66 | 67 | opts_w_defaults = defaults.merge(options) 68 | opts_w_defaults[:headers] = HTTP::Headers.coerce(opts_w_defaults[:headers]) 69 | opts_w_defaults.each { |(k, v)| self[k] = v } 70 | end 71 | 72 | def_option :headers do |new_headers| 73 | headers.merge(new_headers) 74 | end 75 | 76 | def_option :cookies do |new_cookies| 77 | new_cookies.each_with_object cookies.dup do |(k, v), jar| 78 | cookie = k.is_a?(Cookie) ? k : Cookie.new(k.to_s, v.to_s) 79 | jar[cookie.name] = cookie.cookie_value 80 | end 81 | end 82 | 83 | def_option :encoding do |encoding| 84 | self.encoding = Encoding.find(encoding) 85 | end 86 | 87 | def_option :features, :reader_only => true do |new_features| 88 | # Normalize features from: 89 | # 90 | # [{feature_one: {opt: 'val'}}, :feature_two] 91 | # 92 | # into: 93 | # 94 | # {feature_one: {opt: 'val'}, feature_two: {}} 95 | normalized_features = new_features.each_with_object({}) do |feature, h| 96 | if feature.is_a?(Hash) 97 | h.merge!(feature) 98 | else 99 | h[feature] = {} 100 | end 101 | end 102 | 103 | features.merge(normalized_features) 104 | end 105 | 106 | def features=(features) 107 | @features = features.each_with_object({}) do |(name, opts_or_feature), h| 108 | h[name] = if opts_or_feature.is_a?(Feature) 109 | opts_or_feature 110 | else 111 | unless (feature = self.class.available_features[name]) 112 | argument_error! "Unsupported feature: #{name}" 113 | end 114 | feature.new(**opts_or_feature) 115 | end 116 | end 117 | end 118 | 119 | %w[ 120 | proxy params form json body response 121 | socket_class nodelay ssl_socket_class ssl_context ssl 122 | keep_alive_timeout timeout_class timeout_options 123 | ].each do |method_name| 124 | def_option method_name 125 | end 126 | 127 | def_option :follow, :reader_only => true 128 | 129 | def follow=(value) 130 | @follow = 131 | case 132 | when !value then nil 133 | when true == value then {} 134 | when value.respond_to?(:fetch) then value 135 | else argument_error! "Unsupported follow options: #{value}" 136 | end 137 | end 138 | 139 | def_option :persistent, :reader_only => true 140 | 141 | def persistent=(value) 142 | @persistent = value ? HTTP::URI.parse(value).origin : nil 143 | end 144 | 145 | def persistent? 146 | !persistent.nil? 147 | end 148 | 149 | def merge(other) 150 | h1 = to_hash 151 | h2 = other.to_hash 152 | 153 | merged = h1.merge(h2) do |k, v1, v2| 154 | case k 155 | when :headers 156 | v1.merge(v2) 157 | else 158 | v2 159 | end 160 | end 161 | 162 | self.class.new(merged) 163 | end 164 | 165 | def to_hash 166 | hash_pairs = self.class. 167 | defined_options. 168 | flat_map { |opt_name| [opt_name, send(opt_name)] } 169 | Hash[*hash_pairs] 170 | end 171 | 172 | def dup 173 | dupped = super 174 | yield(dupped) if block_given? 175 | dupped 176 | end 177 | 178 | def feature(name) 179 | features[name] 180 | end 181 | 182 | protected 183 | 184 | def []=(option, val) 185 | send(:"#{option}=", val) 186 | end 187 | 188 | private 189 | 190 | def argument_error!(message) 191 | raise(Error, message, caller(1..-1)) 192 | end 193 | end 194 | end 195 | -------------------------------------------------------------------------------- /lib/http/response.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/headers" 6 | require "http/content_type" 7 | require "http/mime_type" 8 | require "http/response/status" 9 | require "http/response/inflater" 10 | require "http/cookie_jar" 11 | require "time" 12 | 13 | module HTTP 14 | class Response 15 | extend Forwardable 16 | 17 | include HTTP::Headers::Mixin 18 | 19 | # @return [Status] 20 | attr_reader :status 21 | 22 | # @return [String] 23 | attr_reader :version 24 | 25 | # @return [Body] 26 | attr_reader :body 27 | 28 | # @return [Request] 29 | attr_reader :request 30 | 31 | # @return [Hash] 32 | attr_reader :proxy_headers 33 | 34 | # Inits a new instance 35 | # 36 | # @option opts [Integer] :status Status code 37 | # @option opts [String] :version HTTP version 38 | # @option opts [Hash] :headers 39 | # @option opts [Hash] :proxy_headers 40 | # @option opts [HTTP::Connection] :connection 41 | # @option opts [String] :encoding Encoding to use when reading body 42 | # @option opts [String] :body 43 | # @option opts [HTTP::Request] request The request this is in response to. 44 | # @option opts [String] :uri (DEPRECATED) used to populate a missing request 45 | def initialize(opts) 46 | @version = opts.fetch(:version) 47 | @request = init_request(opts) 48 | @status = HTTP::Response::Status.new(opts.fetch(:status)) 49 | @headers = HTTP::Headers.coerce(opts[:headers] || {}) 50 | @proxy_headers = HTTP::Headers.coerce(opts[:proxy_headers] || {}) 51 | 52 | if opts.include?(:body) 53 | @body = opts.fetch(:body) 54 | else 55 | connection = opts.fetch(:connection) 56 | encoding = opts[:encoding] || charset || default_encoding 57 | 58 | @body = Response::Body.new(connection, :encoding => encoding) 59 | end 60 | end 61 | 62 | # @!method reason 63 | # @return (see HTTP::Response::Status#reason) 64 | def_delegator :@status, :reason 65 | 66 | # @!method code 67 | # @return (see HTTP::Response::Status#code) 68 | def_delegator :@status, :code 69 | 70 | # @!method to_s 71 | # (see HTTP::Response::Body#to_s) 72 | def_delegator :@body, :to_s 73 | alias to_str to_s 74 | 75 | # @!method readpartial 76 | # (see HTTP::Response::Body#readpartial) 77 | def_delegator :@body, :readpartial 78 | 79 | # @!method connection 80 | # (see HTTP::Response::Body#connection) 81 | def_delegator :@body, :connection 82 | 83 | # @!method uri 84 | # @return (see HTTP::Request#uri) 85 | def_delegator :@request, :uri 86 | 87 | # Returns an Array ala Rack: `[status, headers, body]` 88 | # 89 | # @return [Array(Fixnum, Hash, String)] 90 | def to_a 91 | [status.to_i, headers.to_h, body.to_s] 92 | end 93 | 94 | # Flushes body and returns self-reference 95 | # 96 | # @return [Response] 97 | def flush 98 | body.to_s 99 | self 100 | end 101 | 102 | # Value of the Content-Length header. 103 | # 104 | # @return [nil] if Content-Length was not given, or it's value was invalid 105 | # (not an integer, e.g. empty string or string with non-digits). 106 | # @return [Integer] otherwise 107 | def content_length 108 | # http://greenbytes.de/tech/webdav/rfc7230.html#rfc.section.3.3.3 109 | # Clause 3: "If a message is received with both a Transfer-Encoding 110 | # and a Content-Length header field, the Transfer-Encoding overrides the Content-Length. 111 | return nil if @headers.include?(Headers::TRANSFER_ENCODING) 112 | 113 | value = @headers[Headers::CONTENT_LENGTH] 114 | return nil unless value 115 | 116 | begin 117 | Integer(value) 118 | rescue ArgumentError 119 | nil 120 | end 121 | end 122 | 123 | # Parsed Content-Type header 124 | # 125 | # @return [HTTP::ContentType] 126 | def content_type 127 | @content_type ||= ContentType.parse headers[Headers::CONTENT_TYPE] 128 | end 129 | 130 | # @!method mime_type 131 | # MIME type of response (if any) 132 | # @return [String, nil] 133 | def_delegator :content_type, :mime_type 134 | 135 | # @!method charset 136 | # Charset of response (if any) 137 | # @return [String, nil] 138 | def_delegator :content_type, :charset 139 | 140 | def cookies 141 | @cookies ||= headers.get(Headers::SET_COOKIE).each_with_object CookieJar.new do |v, jar| 142 | jar.parse(v, uri) 143 | end 144 | end 145 | 146 | def chunked? 147 | return false unless @headers.include?(Headers::TRANSFER_ENCODING) 148 | 149 | encoding = @headers.get(Headers::TRANSFER_ENCODING) 150 | 151 | # TODO: "chunked" is frozen in the request writer. How about making it accessible? 152 | encoding.last == "chunked" 153 | end 154 | 155 | # Parse response body with corresponding MIME type adapter. 156 | # 157 | # @param type [#to_s] Parse as given MIME type. 158 | # @raise (see MimeType.[]) 159 | # @return [Object] 160 | def parse(type = nil) 161 | MimeType[type || mime_type].decode to_s 162 | end 163 | 164 | # Inspect a response 165 | def inspect 166 | "#<#{self.class}/#{@version} #{code} #{reason} #{headers.to_h.inspect}>" 167 | end 168 | 169 | private 170 | 171 | def default_encoding 172 | return Encoding::UTF_8 if mime_type == "application/json" 173 | 174 | Encoding::BINARY 175 | end 176 | 177 | # Initialize an HTTP::Request from options. 178 | # 179 | # @return [HTTP::Request] 180 | def init_request(opts) 181 | raise ArgumentError, ":uri is for backwards compatibilty and conflicts with :request" \ 182 | if opts[:request] && opts[:uri] 183 | 184 | # For backwards compatibilty 185 | if opts[:uri] 186 | HTTP::Request.new(:uri => opts[:uri], :verb => :get) 187 | else 188 | opts.fetch(:request) 189 | end 190 | end 191 | end 192 | end 193 | -------------------------------------------------------------------------------- /spec/support/http_handling_shared.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.shared_context "HTTP handling" do 4 | context "without timeouts" do 5 | let(:options) { {:timeout_class => HTTP::Timeout::Null, :timeout_options => {}} } 6 | 7 | it "works" do 8 | expect(client.get(server.endpoint).body.to_s).to eq("") 9 | end 10 | end 11 | 12 | context "with a per operation timeout" do 13 | let(:response) { client.get(server.endpoint).body.to_s } 14 | 15 | let(:options) do 16 | { 17 | :timeout_class => HTTP::Timeout::PerOperation, 18 | :timeout_options => { 19 | :connect_timeout => conn_timeout, 20 | :read_timeout => read_timeout, 21 | :write_timeout => write_timeout 22 | } 23 | } 24 | end 25 | let(:conn_timeout) { 1 } 26 | let(:read_timeout) { 1 } 27 | let(:write_timeout) { 1 } 28 | 29 | it "works" do 30 | expect(response).to eq("") 31 | end 32 | 33 | context "connection" do 34 | context "of 1" do 35 | let(:conn_timeout) { 1 } 36 | 37 | it "does not time out" do 38 | expect { response }.to_not raise_error 39 | end 40 | end 41 | end 42 | 43 | context "read" do 44 | context "of 0" do 45 | let(:read_timeout) { 0 } 46 | 47 | it "times out", :flaky do 48 | expect { response }.to raise_error(HTTP::TimeoutError, /Read/i) 49 | end 50 | end 51 | 52 | context "of 2.5" do 53 | let(:read_timeout) { 2.5 } 54 | 55 | it "does not time out", :flaky do 56 | expect { client.get("#{server.endpoint}/sleep").body.to_s }.to_not raise_error 57 | end 58 | end 59 | end 60 | end 61 | 62 | context "with a global timeout" do 63 | let(:options) do 64 | { 65 | :timeout_class => HTTP::Timeout::Global, 66 | :timeout_options => { 67 | :global_timeout => global_timeout 68 | } 69 | } 70 | end 71 | let(:global_timeout) { 1 } 72 | 73 | let(:response) { client.get(server.endpoint).body.to_s } 74 | 75 | it "errors if connecting takes too long" do 76 | expect(TCPSocket).to receive(:open) do 77 | sleep 1.25 78 | end 79 | 80 | expect { response }.to raise_error(HTTP::ConnectTimeoutError, /execution/) 81 | end 82 | 83 | it "errors if reading takes too long" do 84 | expect { client.get("#{server.endpoint}/sleep").body.to_s }. 85 | to raise_error(HTTP::TimeoutError, /Timed out/) 86 | end 87 | 88 | context "it resets state when reusing connections" do 89 | let(:extra_options) { {:persistent => server.endpoint} } 90 | 91 | let(:global_timeout) { 2.5 } 92 | 93 | it "does not timeout", :flaky do 94 | client.get("#{server.endpoint}/sleep").body.to_s 95 | client.get("#{server.endpoint}/sleep").body.to_s 96 | end 97 | end 98 | end 99 | 100 | describe "connection reuse" do 101 | let(:sockets_used) do 102 | [ 103 | client.get("#{server.endpoint}/socket/1").body.to_s, 104 | client.get("#{server.endpoint}/socket/2").body.to_s 105 | ] 106 | end 107 | 108 | context "when enabled" do 109 | let(:options) { {:persistent => server.endpoint} } 110 | 111 | context "without a host" do 112 | it "infers host from persistent config" do 113 | expect(client.get("/").body.to_s).to eq("") 114 | end 115 | end 116 | 117 | it "re-uses the socket" do 118 | expect(sockets_used).to_not include("") 119 | expect(sockets_used.uniq.length).to eq(1) 120 | end 121 | 122 | context "on a mixed state" do 123 | it "re-opens the connection", :flaky do 124 | first_socket_id = client.get("#{server.endpoint}/socket/1").body.to_s 125 | 126 | client.instance_variable_set(:@state, :dirty) 127 | 128 | second_socket_id = client.get("#{server.endpoint}/socket/2").body.to_s 129 | 130 | expect(first_socket_id).to_not eq(second_socket_id) 131 | end 132 | end 133 | 134 | context "when trying to read a stale body" do 135 | it "errors" do 136 | client.get("#{server.endpoint}/not-found") 137 | expect { client.get(server.endpoint) }.to raise_error(HTTP::StateError, /Tried to send a request/) 138 | end 139 | end 140 | 141 | context "when reading a cached body" do 142 | it "succeeds" do 143 | first_res = client.get(server.endpoint) 144 | first_res.body.to_s 145 | 146 | second_res = client.get(server.endpoint) 147 | 148 | expect(first_res.body.to_s).to eq("") 149 | expect(second_res.body.to_s).to eq("") 150 | end 151 | end 152 | 153 | context "with a socket issue" do 154 | it "transparently reopens", :flaky do 155 | first_socket_id = client.get("#{server.endpoint}/socket").body.to_s 156 | expect(first_socket_id).to_not eq("") 157 | # Kill off the sockets we used 158 | # rubocop:disable Style/RescueModifier 159 | DummyServer::Servlet.sockets.each do |socket| 160 | socket.close rescue nil 161 | end 162 | DummyServer::Servlet.sockets.clear 163 | # rubocop:enable Style/RescueModifier 164 | 165 | # Should error because we tried to use a bad socket 166 | expect { client.get("#{server.endpoint}/socket").body.to_s }.to raise_error HTTP::ConnectionError 167 | 168 | # Should succeed since we create a new socket 169 | second_socket_id = client.get("#{server.endpoint}/socket").body.to_s 170 | expect(second_socket_id).to_not eq(first_socket_id) 171 | end 172 | end 173 | 174 | context "with a change in host" do 175 | it "errors" do 176 | expect { client.get("https://invalid.com/socket") }.to raise_error(/Persistence is enabled/i) 177 | end 178 | end 179 | end 180 | 181 | context "when disabled" do 182 | let(:options) { {} } 183 | 184 | it "opens new sockets", :flaky do 185 | expect(sockets_used).to_not include("") 186 | expect(sockets_used.uniq.length).to eq(2) 187 | end 188 | end 189 | end 190 | end 191 | -------------------------------------------------------------------------------- /lib/http/client.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/form_data" 6 | require "http/options" 7 | require "http/feature" 8 | require "http/headers" 9 | require "http/connection" 10 | require "http/redirector" 11 | require "http/uri" 12 | 13 | module HTTP 14 | # Clients make requests and receive responses 15 | class Client 16 | extend Forwardable 17 | include Chainable 18 | 19 | HTTP_OR_HTTPS_RE = %r{^https?://}i.freeze 20 | 21 | def initialize(default_options = {}) 22 | @default_options = HTTP::Options.new(default_options) 23 | @connection = nil 24 | @state = :clean 25 | end 26 | 27 | # Make an HTTP request 28 | def request(verb, uri, opts = {}) 29 | opts = @default_options.merge(opts) 30 | req = build_request(verb, uri, opts) 31 | res = perform(req, opts) 32 | return res unless opts.follow 33 | 34 | Redirector.new(opts.follow).perform(req, res) do |request| 35 | perform(wrap_request(request, opts), opts) 36 | end 37 | end 38 | 39 | # Prepare an HTTP request 40 | def build_request(verb, uri, opts = {}) 41 | opts = @default_options.merge(opts) 42 | uri = make_request_uri(uri, opts) 43 | headers = make_request_headers(opts) 44 | body = make_request_body(opts, headers) 45 | 46 | req = HTTP::Request.new( 47 | :verb => verb, 48 | :uri => uri, 49 | :uri_normalizer => opts.feature(:normalize_uri)&.normalizer, 50 | :proxy => opts.proxy, 51 | :headers => headers, 52 | :body => body 53 | ) 54 | 55 | wrap_request(req, opts) 56 | end 57 | 58 | # @!method persistent? 59 | # @see Options#persistent? 60 | # @return [Boolean] whenever client is persistent 61 | def_delegator :default_options, :persistent? 62 | 63 | # Perform a single (no follow) HTTP request 64 | def perform(req, options) 65 | verify_connection!(req.uri) 66 | 67 | @state = :dirty 68 | 69 | begin 70 | @connection ||= HTTP::Connection.new(req, options) 71 | 72 | unless @connection.failed_proxy_connect? 73 | @connection.send_request(req) 74 | @connection.read_headers! 75 | end 76 | rescue Error => e 77 | options.features.each_value do |feature| 78 | feature.on_error(req, e) 79 | end 80 | raise 81 | end 82 | res = build_response(req, options) 83 | 84 | res = options.features.inject(res) do |response, (_name, feature)| 85 | feature.wrap_response(response) 86 | end 87 | 88 | @connection.finish_response if req.verb == :head 89 | @state = :clean 90 | 91 | res 92 | rescue 93 | close 94 | raise 95 | end 96 | 97 | def close 98 | @connection&.close 99 | @connection = nil 100 | @state = :clean 101 | end 102 | 103 | private 104 | 105 | def wrap_request(req, opts) 106 | opts.features.inject(req) do |request, (_name, feature)| 107 | feature.wrap_request(request) 108 | end 109 | end 110 | 111 | def build_response(req, options) 112 | Response.new( 113 | :status => @connection.status_code, 114 | :version => @connection.http_version, 115 | :headers => @connection.headers, 116 | :proxy_headers => @connection.proxy_response_headers, 117 | :connection => @connection, 118 | :encoding => options.encoding, 119 | :request => req 120 | ) 121 | end 122 | 123 | # Verify our request isn't going to be made against another URI 124 | def verify_connection!(uri) 125 | if default_options.persistent? && uri.origin != default_options.persistent 126 | raise StateError, "Persistence is enabled for #{default_options.persistent}, but we got #{uri.origin}" 127 | end 128 | 129 | # We re-create the connection object because we want to let prior requests 130 | # lazily load the body as long as possible, and this mimics prior functionality. 131 | return close if @connection && (!@connection.keep_alive? || @connection.expired?) 132 | 133 | # If we get into a bad state (eg, Timeout.timeout ensure being killed) 134 | # close the connection to prevent potential for mixed responses. 135 | return close if @state == :dirty 136 | end 137 | 138 | # Merges query params if needed 139 | # 140 | # @param [#to_s] uri 141 | # @return [URI] 142 | def make_request_uri(uri, opts) 143 | uri = uri.to_s 144 | 145 | uri = "#{default_options.persistent}#{uri}" if default_options.persistent? && uri !~ HTTP_OR_HTTPS_RE 146 | 147 | uri = HTTP::URI.parse uri 148 | 149 | uri.query_values = uri.query_values(Array).to_a.concat(opts.params.to_a) if opts.params && !opts.params.empty? 150 | 151 | # Some proxies (seen on WEBRick) fail if URL has 152 | # empty path (e.g. `http://example.com`) while it's RFC-complaint: 153 | # http://tools.ietf.org/html/rfc1738#section-3.1 154 | uri.path = "/" if uri.path.empty? 155 | 156 | uri 157 | end 158 | 159 | # Creates request headers with cookies (if any) merged in 160 | def make_request_headers(opts) 161 | headers = opts.headers 162 | 163 | # Tell the server to keep the conn open 164 | headers[Headers::CONNECTION] = default_options.persistent? ? Connection::KEEP_ALIVE : Connection::CLOSE 165 | 166 | cookies = opts.cookies.values 167 | 168 | unless cookies.empty? 169 | cookies = opts.headers.get(Headers::COOKIE).concat(cookies).join("; ") 170 | headers[Headers::COOKIE] = cookies 171 | end 172 | 173 | headers 174 | end 175 | 176 | # Create the request body object to send 177 | def make_request_body(opts, headers) 178 | case 179 | when opts.body 180 | opts.body 181 | when opts.form 182 | form = make_form_data(opts.form) 183 | headers[Headers::CONTENT_TYPE] ||= form.content_type 184 | form 185 | when opts.json 186 | body = MimeType[:json].encode opts.json 187 | headers[Headers::CONTENT_TYPE] ||= "application/json; charset=#{body.encoding.name}" 188 | body 189 | end 190 | end 191 | 192 | def make_form_data(form) 193 | return form if form.is_a? HTTP::FormData::Multipart 194 | return form if form.is_a? HTTP::FormData::Urlencoded 195 | 196 | HTTP::FormData.create(form) 197 | end 198 | end 199 | end 200 | -------------------------------------------------------------------------------- /lib/http/connection.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/headers" 6 | 7 | module HTTP 8 | # A connection to the HTTP server 9 | class Connection 10 | extend Forwardable 11 | 12 | # Allowed values for CONNECTION header 13 | KEEP_ALIVE = "Keep-Alive" 14 | CLOSE = "close" 15 | 16 | # Attempt to read this much data 17 | BUFFER_SIZE = 16_384 18 | 19 | # HTTP/1.0 20 | HTTP_1_0 = "1.0" 21 | 22 | # HTTP/1.1 23 | HTTP_1_1 = "1.1" 24 | 25 | # Returned after HTTP CONNECT (via proxy) 26 | attr_reader :proxy_response_headers 27 | 28 | # @param [HTTP::Request] req 29 | # @param [HTTP::Options] options 30 | # @raise [HTTP::ConnectionError] when failed to connect 31 | def initialize(req, options) 32 | @persistent = options.persistent? 33 | @keep_alive_timeout = options.keep_alive_timeout.to_f 34 | @pending_request = false 35 | @pending_response = false 36 | @failed_proxy_connect = false 37 | @buffer = "".b 38 | 39 | @parser = Response::Parser.new 40 | 41 | @socket = options.timeout_class.new(options.timeout_options) 42 | @socket.connect(options.socket_class, req.socket_host, req.socket_port, options.nodelay) 43 | 44 | send_proxy_connect_request(req) 45 | start_tls(req, options) 46 | reset_timer 47 | rescue IOError, SocketError, SystemCallError => e 48 | raise ConnectionError, "failed to connect: #{e}", e.backtrace 49 | end 50 | 51 | # @see (HTTP::Response::Parser#status_code) 52 | def_delegator :@parser, :status_code 53 | 54 | # @see (HTTP::Response::Parser#http_version) 55 | def_delegator :@parser, :http_version 56 | 57 | # @see (HTTP::Response::Parser#headers) 58 | def_delegator :@parser, :headers 59 | 60 | # @return [Boolean] whenever proxy connect failed 61 | def failed_proxy_connect? 62 | @failed_proxy_connect 63 | end 64 | 65 | # Send a request to the server 66 | # 67 | # @param [Request] req Request to send to the server 68 | # @return [nil] 69 | def send_request(req) 70 | if @pending_response 71 | raise StateError, "Tried to send a request while one is pending already. Make sure you read off the body." 72 | end 73 | 74 | if @pending_request 75 | raise StateError, "Tried to send a request while a response is pending. Make sure you read off the body." 76 | end 77 | 78 | @pending_request = true 79 | 80 | req.stream @socket 81 | 82 | @pending_response = true 83 | @pending_request = false 84 | end 85 | 86 | # Read a chunk of the body 87 | # 88 | # @return [String] data chunk 89 | # @return [nil] when no more data left 90 | def readpartial(size = BUFFER_SIZE) 91 | return unless @pending_response 92 | 93 | chunk = @parser.read(size) 94 | return chunk if chunk 95 | 96 | finished = (read_more(size) == :eof) || @parser.finished? 97 | chunk = @parser.read(size) 98 | finish_response if finished 99 | 100 | chunk || "".b 101 | end 102 | 103 | # Reads data from socket up until headers are loaded 104 | # @return [void] 105 | def read_headers! 106 | until @parser.headers? 107 | result = read_more(BUFFER_SIZE) 108 | raise ConnectionError, "couldn't read response headers" if result == :eof 109 | end 110 | 111 | set_keep_alive 112 | end 113 | 114 | # Callback for when we've reached the end of a response 115 | # @return [void] 116 | def finish_response 117 | close unless keep_alive? 118 | 119 | @parser.reset 120 | @socket.reset_counter if @socket.respond_to?(:reset_counter) 121 | reset_timer 122 | 123 | @pending_response = false 124 | end 125 | 126 | # Close the connection 127 | # @return [void] 128 | def close 129 | @socket.close unless @socket.closed? 130 | 131 | @pending_response = false 132 | @pending_request = false 133 | end 134 | 135 | # Whether we're keeping the conn alive 136 | # @return [Boolean] 137 | def keep_alive? 138 | !!@keep_alive && !@socket.closed? 139 | end 140 | 141 | # Whether our connection has expired 142 | # @return [Boolean] 143 | def expired? 144 | !@conn_expires_at || @conn_expires_at < Time.now 145 | end 146 | 147 | private 148 | 149 | # Sets up SSL context and starts TLS if needed. 150 | # @param (see #initialize) 151 | # @return [void] 152 | def start_tls(req, options) 153 | return unless req.uri.https? && !failed_proxy_connect? 154 | 155 | ssl_context = options.ssl_context 156 | 157 | unless ssl_context 158 | ssl_context = OpenSSL::SSL::SSLContext.new 159 | ssl_context.set_params(options.ssl || {}) 160 | end 161 | 162 | @socket.start_tls(req.uri.host, options.ssl_socket_class, ssl_context) 163 | end 164 | 165 | # Open tunnel through proxy 166 | def send_proxy_connect_request(req) 167 | return unless req.uri.https? && req.using_proxy? 168 | 169 | @pending_request = true 170 | 171 | req.connect_using_proxy @socket 172 | 173 | @pending_request = false 174 | @pending_response = true 175 | 176 | read_headers! 177 | @proxy_response_headers = @parser.headers 178 | 179 | if @parser.status_code != 200 180 | @failed_proxy_connect = true 181 | return 182 | end 183 | 184 | @parser.reset 185 | @pending_response = false 186 | end 187 | 188 | # Resets expiration of persistent connection. 189 | # @return [void] 190 | def reset_timer 191 | @conn_expires_at = Time.now + @keep_alive_timeout if @persistent 192 | end 193 | 194 | # Store whether the connection should be kept alive. 195 | # Once we reset the parser, we lose all of this state. 196 | # @return [void] 197 | def set_keep_alive 198 | return @keep_alive = false unless @persistent 199 | 200 | @keep_alive = 201 | case @parser.http_version 202 | when HTTP_1_0 # HTTP/1.0 requires opt in for Keep Alive 203 | @parser.headers[Headers::CONNECTION] == KEEP_ALIVE 204 | when HTTP_1_1 # HTTP/1.1 is opt-out 205 | @parser.headers[Headers::CONNECTION] != CLOSE 206 | else # Anything else we assume doesn't supportit 207 | false 208 | end 209 | end 210 | 211 | # Feeds some more data into parser 212 | # @return [void] 213 | def read_more(size) 214 | return if @parser.finished? 215 | 216 | value = @socket.readpartial(size, @buffer) 217 | if value == :eof 218 | @parser << "" 219 | :eof 220 | elsif value 221 | @parser << value 222 | end 223 | rescue IOError, SocketError, SystemCallError => e 224 | raise ConnectionError, "error reading from socket: #{e}", e.backtrace 225 | end 226 | end 227 | end 228 | -------------------------------------------------------------------------------- /.rubocop_todo.yml: -------------------------------------------------------------------------------- 1 | # This configuration was generated by 2 | # `rubocop --auto-gen-config --auto-gen-only-exclude --exclude-limit 100` 3 | # on 2022-06-16 14:35:44 UTC using RuboCop version 1.30.1. 4 | # The point is for the user to remove these configuration records 5 | # one by one as the offenses are removed from the code base. 6 | # Note that changes in the inspected code, or installation of new 7 | # versions of RuboCop, may require this file to be generated again. 8 | 9 | # Offense count: 1 10 | # This cop supports safe autocorrection (--autocorrect). 11 | # Configuration parameters: Include. 12 | # Include: **/*.gemspec 13 | Gemspec/DeprecatedAttributeAssignment: 14 | Exclude: 15 | - 'http.gemspec' 16 | 17 | # Offense count: 53 18 | # This cop supports safe autocorrection (--autocorrect). 19 | # Configuration parameters: EnforcedStyle. 20 | # SupportedStyles: leading, trailing 21 | Layout/DotPosition: 22 | Exclude: 23 | - 'lib/http/options.rb' 24 | - 'spec/lib/http/client_spec.rb' 25 | - 'spec/lib/http/features/auto_deflate_spec.rb' 26 | - 'spec/lib/http/headers_spec.rb' 27 | - 'spec/lib/http/options/features_spec.rb' 28 | - 'spec/lib/http/redirector_spec.rb' 29 | - 'spec/lib/http/response/body_spec.rb' 30 | - 'spec/lib/http_spec.rb' 31 | - 'spec/support/http_handling_shared.rb' 32 | 33 | # Offense count: 176 34 | # This cop supports safe autocorrection (--autocorrect). 35 | # Configuration parameters: EnforcedStyle, EnforcedStyleForEmptyBraces. 36 | # SupportedStyles: space, no_space, compact 37 | # SupportedStylesForEmptyBraces: space, no_space 38 | Layout/SpaceInsideHashLiteralBraces: 39 | Exclude: 40 | - 'lib/http/chainable.rb' 41 | - 'spec/lib/http/client_spec.rb' 42 | - 'spec/lib/http/features/auto_inflate_spec.rb' 43 | - 'spec/lib/http/features/instrumentation_spec.rb' 44 | - 'spec/lib/http/features/logging_spec.rb' 45 | - 'spec/lib/http/headers_spec.rb' 46 | - 'spec/lib/http/options/features_spec.rb' 47 | - 'spec/lib/http/options/merge_spec.rb' 48 | - 'spec/lib/http/options/new_spec.rb' 49 | - 'spec/lib/http/redirector_spec.rb' 50 | - 'spec/lib/http/request_spec.rb' 51 | - 'spec/lib/http/response_spec.rb' 52 | - 'spec/lib/http_spec.rb' 53 | - 'spec/support/dummy_server/servlet.rb' 54 | - 'spec/support/http_handling_shared.rb' 55 | - 'spec/support/ssl_helper.rb' 56 | 57 | # Offense count: 4 58 | Lint/MissingSuper: 59 | Exclude: 60 | - 'lib/http/features/auto_deflate.rb' 61 | - 'lib/http/features/instrumentation.rb' 62 | - 'lib/http/features/logging.rb' 63 | - 'lib/http/features/normalize_uri.rb' 64 | 65 | # Offense count: 8 66 | # Configuration parameters: IgnoredMethods, CountRepeatedAttributes, Max. 67 | Metrics/AbcSize: 68 | Exclude: 69 | - 'lib/http/chainable.rb' 70 | - 'lib/http/client.rb' 71 | - 'lib/http/connection.rb' 72 | - 'lib/http/features/auto_deflate.rb' 73 | - 'lib/http/redirector.rb' 74 | - 'lib/http/request.rb' 75 | - 'lib/http/response.rb' 76 | 77 | # Offense count: 69 78 | # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods, IgnoredMethods. 79 | # IgnoredMethods: refine 80 | Metrics/BlockLength: 81 | Exclude: 82 | - '**/*.gemspec' 83 | - 'spec/lib/http/client_spec.rb' 84 | - 'spec/lib/http/connection_spec.rb' 85 | - 'spec/lib/http/content_type_spec.rb' 86 | - 'spec/lib/http/features/auto_deflate_spec.rb' 87 | - 'spec/lib/http/features/auto_inflate_spec.rb' 88 | - 'spec/lib/http/features/instrumentation_spec.rb' 89 | - 'spec/lib/http/features/logging_spec.rb' 90 | - 'spec/lib/http/headers/mixin_spec.rb' 91 | - 'spec/lib/http/headers_spec.rb' 92 | - 'spec/lib/http/options/merge_spec.rb' 93 | - 'spec/lib/http/redirector_spec.rb' 94 | - 'spec/lib/http/request/body_spec.rb' 95 | - 'spec/lib/http/request/writer_spec.rb' 96 | - 'spec/lib/http/request_spec.rb' 97 | - 'spec/lib/http/response/body_spec.rb' 98 | - 'spec/lib/http/response/parser_spec.rb' 99 | - 'spec/lib/http/response/status_spec.rb' 100 | - 'spec/lib/http/response_spec.rb' 101 | - 'spec/lib/http_spec.rb' 102 | - 'spec/support/http_handling_shared.rb' 103 | 104 | # Offense count: 4 105 | # Configuration parameters: CountComments, Max, CountAsOne. 106 | Metrics/ClassLength: 107 | Exclude: 108 | - 'lib/http/client.rb' 109 | - 'lib/http/connection.rb' 110 | - 'lib/http/headers.rb' 111 | - 'lib/http/request.rb' 112 | 113 | # Offense count: 2 114 | # Configuration parameters: IgnoredMethods, Max. 115 | Metrics/CyclomaticComplexity: 116 | Exclude: 117 | - 'lib/http/chainable.rb' 118 | - 'lib/http/client.rb' 119 | 120 | # Offense count: 18 121 | # Configuration parameters: CountComments, Max, CountAsOne, ExcludedMethods, IgnoredMethods. 122 | Metrics/MethodLength: 123 | Exclude: 124 | - 'lib/http/chainable.rb' 125 | - 'lib/http/client.rb' 126 | - 'lib/http/connection.rb' 127 | - 'lib/http/features/auto_deflate.rb' 128 | - 'lib/http/features/auto_inflate.rb' 129 | - 'lib/http/headers.rb' 130 | - 'lib/http/options.rb' 131 | - 'lib/http/redirector.rb' 132 | - 'lib/http/request.rb' 133 | - 'lib/http/response.rb' 134 | - 'lib/http/response/body.rb' 135 | - 'lib/http/timeout/global.rb' 136 | 137 | # Offense count: 1 138 | # Configuration parameters: CountComments, Max, CountAsOne. 139 | Metrics/ModuleLength: 140 | Exclude: 141 | - 'lib/http/chainable.rb' 142 | 143 | # Offense count: 1 144 | Security/CompoundHash: 145 | Exclude: 146 | - 'lib/http/uri.rb' 147 | 148 | # Offense count: 2 149 | # This cop supports safe autocorrection (--autocorrect). 150 | # Configuration parameters: EnforcedStyle. 151 | # SupportedStyles: separated, grouped 152 | Style/AccessorGrouping: 153 | Exclude: 154 | - 'lib/http/request.rb' 155 | 156 | # Offense count: 4 157 | # This cop supports safe autocorrection (--autocorrect). 158 | Style/EmptyCaseCondition: 159 | Exclude: 160 | - 'lib/http/client.rb' 161 | - 'lib/http/headers.rb' 162 | - 'lib/http/options.rb' 163 | - 'lib/http/response/status.rb' 164 | 165 | # Offense count: 5 166 | # This cop supports safe autocorrection (--autocorrect). 167 | Style/Encoding: 168 | Exclude: 169 | - 'spec/lib/http/client_spec.rb' 170 | - 'spec/lib/http/request/writer_spec.rb' 171 | - 'spec/lib/http/request_spec.rb' 172 | - 'spec/lib/http_spec.rb' 173 | - 'spec/support/dummy_server/servlet.rb' 174 | 175 | # Offense count: 17 176 | # Configuration parameters: SuspiciousParamNames, Allowlist. 177 | # SuspiciousParamNames: options, opts, args, params, parameters 178 | Style/OptionHash: 179 | Exclude: 180 | - 'lib/http/chainable.rb' 181 | - 'lib/http/client.rb' 182 | - 'lib/http/feature.rb' 183 | - 'lib/http/options.rb' 184 | - 'lib/http/redirector.rb' 185 | - 'lib/http/timeout/null.rb' 186 | - 'spec/support/dummy_server.rb' 187 | 188 | # Offense count: 4 189 | # Configuration parameters: AllowedMethods. 190 | # AllowedMethods: respond_to_missing? 191 | Style/OptionalBooleanParameter: 192 | Exclude: 193 | - 'lib/http/timeout/global.rb' 194 | - 'lib/http/timeout/null.rb' 195 | - 'lib/http/timeout/per_operation.rb' 196 | - 'lib/http/uri.rb' 197 | 198 | # Offense count: 3 199 | # This cop supports safe autocorrection (--autocorrect). 200 | # Configuration parameters: Max, AllowHeredoc, AllowURI, URISchemes, IgnoreCopDirectives, AllowedPatterns, IgnoredPatterns. 201 | # URISchemes: http, https 202 | Layout/LineLength: 203 | Exclude: 204 | - 'lib/http/chainable.rb' 205 | - 'spec/lib/http/options/proxy_spec.rb' 206 | -------------------------------------------------------------------------------- /lib/http/headers.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | 5 | require "http/errors" 6 | require "http/headers/mixin" 7 | require "http/headers/known" 8 | 9 | module HTTP 10 | # HTTP Headers container. 11 | class Headers 12 | extend Forwardable 13 | include Enumerable 14 | 15 | # Matches HTTP header names when in "Canonical-Http-Format" 16 | CANONICAL_NAME_RE = /\A[A-Z][a-z]*(?:-[A-Z][a-z]*)*\z/.freeze 17 | 18 | # Matches valid header field name according to RFC. 19 | # @see http://tools.ietf.org/html/rfc7230#section-3.2 20 | COMPLIANT_NAME_RE = /\A[A-Za-z0-9!#$%&'*+\-.^_`|~]+\z/.freeze 21 | 22 | # Class constructor. 23 | def initialize 24 | # The @pile stores each header value using a three element array: 25 | # 0 - the normalized header key, used for lookup 26 | # 1 - the header key as it will be sent with a request 27 | # 2 - the value 28 | @pile = [] 29 | end 30 | 31 | # Sets header. 32 | # 33 | # @param (see #add) 34 | # @return [void] 35 | def set(name, value) 36 | delete(name) 37 | add(name, value) 38 | end 39 | alias []= set 40 | 41 | # Removes header. 42 | # 43 | # @param [#to_s] name header name 44 | # @return [void] 45 | def delete(name) 46 | name = normalize_header name.to_s 47 | @pile.delete_if { |k, _| k == name } 48 | end 49 | 50 | # Appends header. 51 | # 52 | # @param [String, Symbol] name header name. When specified as a string, the 53 | # name is sent as-is. When specified as a symbol, the name is converted 54 | # to a string of capitalized words separated by a dash. Word boundaries 55 | # are determined by an underscore (`_`) or a dash (`-`). 56 | # Ex: `:content_type` is sent as `"Content-Type"`, and `"auth_key"` (string) 57 | # is sent as `"auth_key"`. 58 | # @param [Array<#to_s>, #to_s] value header value(s) to be appended 59 | # @return [void] 60 | def add(name, value) 61 | lookup_name = normalize_header(name.to_s) 62 | wire_name = case name 63 | when String 64 | name 65 | when Symbol 66 | lookup_name 67 | else 68 | raise HTTP::HeaderError, "HTTP header must be a String or Symbol: #{name.inspect}" 69 | end 70 | Array(value).each do |v| 71 | @pile << [ 72 | lookup_name, 73 | wire_name, 74 | validate_value(v) 75 | ] 76 | end 77 | end 78 | 79 | # Returns list of header values if any. 80 | # 81 | # @return [Array] 82 | def get(name) 83 | name = normalize_header name.to_s 84 | @pile.select { |k, _| k == name }.map { |_, _, v| v } 85 | end 86 | 87 | # Smart version of {#get}. 88 | # 89 | # @return [nil] if header was not set 90 | # @return [String] if header has exactly one value 91 | # @return [Array] if header has more than one value 92 | def [](name) 93 | values = get(name) 94 | 95 | case values.count 96 | when 0 then nil 97 | when 1 then values.first 98 | else values 99 | end 100 | end 101 | 102 | # Tells whenever header with given `name` is set or not. 103 | # 104 | # @return [Boolean] 105 | def include?(name) 106 | name = normalize_header name.to_s 107 | @pile.any? { |k, _| k == name } 108 | end 109 | 110 | # Returns Rack-compatible headers Hash 111 | # 112 | # @return [Hash] 113 | def to_h 114 | keys.to_h { |k| [k, self[k]] } 115 | end 116 | alias to_hash to_h 117 | 118 | # Returns headers key/value pairs. 119 | # 120 | # @return [Array<[String, String]>] 121 | def to_a 122 | @pile.map { |item| item[1..2] } 123 | end 124 | 125 | # Returns human-readable representation of `self` instance. 126 | # 127 | # @return [String] 128 | def inspect 129 | "#<#{self.class} #{to_h.inspect}>" 130 | end 131 | 132 | # Returns list of header names. 133 | # 134 | # @return [Array] 135 | def keys 136 | @pile.map { |_, k, _| k }.uniq 137 | end 138 | 139 | # Compares headers to another Headers or Array of key/value pairs 140 | # 141 | # @return [Boolean] 142 | def ==(other) 143 | return false unless other.respond_to? :to_a 144 | 145 | to_a == other.to_a 146 | end 147 | 148 | # Calls the given block once for each key/value pair in headers container. 149 | # 150 | # @return [Enumerator] if no block given 151 | # @return [Headers] self-reference 152 | def each 153 | return to_enum(__method__) unless block_given? 154 | 155 | @pile.each { |item| yield(item[1..2]) } 156 | self 157 | end 158 | 159 | # @!method empty? 160 | # Returns `true` if `self` has no key/value pairs 161 | # 162 | # @return [Boolean] 163 | def_delegator :@pile, :empty? 164 | 165 | # @!method hash 166 | # Compute a hash-code for this headers container. 167 | # Two containers with the same content will have the same hash code. 168 | # 169 | # @see http://www.ruby-doc.org/core/Object.html#method-i-hash 170 | # @return [Fixnum] 171 | def_delegator :@pile, :hash 172 | 173 | # Properly clones internal key/value storage. 174 | # 175 | # @api private 176 | def initialize_copy(orig) 177 | super 178 | @pile = @pile.map(&:dup) 179 | end 180 | 181 | # Merges `other` headers into `self`. 182 | # 183 | # @see #merge 184 | # @return [void] 185 | def merge!(other) 186 | self.class.coerce(other).to_h.each { |name, values| set name, values } 187 | end 188 | 189 | # Returns new instance with `other` headers merged in. 190 | # 191 | # @see #merge! 192 | # @return [Headers] 193 | def merge(other) 194 | dup.tap { |dupped| dupped.merge! other } 195 | end 196 | 197 | class << self 198 | # Coerces given `object` into Headers. 199 | # 200 | # @raise [Error] if object can't be coerced 201 | # @param [#to_hash, #to_h, #to_a] object 202 | # @return [Headers] 203 | def coerce(object) 204 | unless object.is_a? self 205 | object = case 206 | when object.respond_to?(:to_hash) then object.to_hash 207 | when object.respond_to?(:to_h) then object.to_h 208 | when object.respond_to?(:to_a) then object.to_a 209 | else raise Error, "Can't coerce #{object.inspect} to Headers" 210 | end 211 | end 212 | 213 | headers = new 214 | object.each { |k, v| headers.add k, v } 215 | headers 216 | end 217 | alias [] coerce 218 | end 219 | 220 | private 221 | 222 | # Transforms `name` to canonical HTTP header capitalization 223 | # 224 | # @param [String] name 225 | # @raise [HeaderError] if normalized name does not 226 | # match {HEADER_NAME_RE} 227 | # @return [String] canonical HTTP header name 228 | def normalize_header(name) 229 | return name if name =~ CANONICAL_NAME_RE 230 | 231 | normalized = name.split(/[\-_]/).each(&:capitalize!).join("-") 232 | 233 | return normalized if normalized =~ COMPLIANT_NAME_RE 234 | 235 | raise HeaderError, "Invalid HTTP header field name: #{name.inspect}" 236 | end 237 | 238 | # Ensures there is no new line character in the header value 239 | # 240 | # @param [String] value 241 | # @raise [HeaderError] if value includes new line character 242 | # @return [String] stringified header value 243 | def validate_value(value) 244 | v = value.to_s 245 | return v unless v.include?("\n") 246 | 247 | raise HeaderError, "Invalid HTTP header field value: #{v.inspect}" 248 | end 249 | end 250 | end 251 | -------------------------------------------------------------------------------- /spec/lib/http/request_spec.rb: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | # frozen_string_literal: true 3 | 4 | RSpec.describe HTTP::Request do 5 | let(:proxy) { {} } 6 | let(:headers) { {:accept => "text/html"} } 7 | let(:request_uri) { "http://example.com/foo?bar=baz" } 8 | 9 | subject :request do 10 | HTTP::Request.new( 11 | :verb => :get, 12 | :uri => request_uri, 13 | :headers => headers, 14 | :proxy => proxy 15 | ) 16 | end 17 | 18 | it "includes HTTP::Headers::Mixin" do 19 | expect(described_class).to include HTTP::Headers::Mixin 20 | end 21 | 22 | it "requires URI to have scheme part" do 23 | expect { HTTP::Request.new(:verb => :get, :uri => "example.com/") }.to \ 24 | raise_error(HTTP::Request::UnsupportedSchemeError) 25 | end 26 | 27 | it "provides a #scheme accessor" do 28 | expect(request.scheme).to eq(:http) 29 | end 30 | 31 | it "provides a #verb accessor" do 32 | expect(subject.verb).to eq(:get) 33 | end 34 | 35 | it "sets given headers" do 36 | expect(subject["Accept"]).to eq("text/html") 37 | end 38 | 39 | describe "Host header" do 40 | subject { request["Host"] } 41 | 42 | context "was not given" do 43 | it { is_expected.to eq "example.com" } 44 | 45 | context "and request URI has non-standard port" do 46 | let(:request_uri) { "http://example.com:3000/" } 47 | it { is_expected.to eq "example.com:3000" } 48 | end 49 | end 50 | 51 | context "was explicitly given" do 52 | before { headers[:host] = "github.com" } 53 | it { is_expected.to eq "github.com" } 54 | end 55 | end 56 | 57 | describe "User-Agent header" do 58 | subject { request["User-Agent"] } 59 | 60 | context "was not given" do 61 | it { is_expected.to eq HTTP::Request::USER_AGENT } 62 | end 63 | 64 | context "was explicitly given" do 65 | before { headers[:user_agent] = "MrCrawly/123" } 66 | it { is_expected.to eq "MrCrawly/123" } 67 | end 68 | end 69 | 70 | describe "#redirect" do 71 | let(:headers) { {:accept => "text/html"} } 72 | let(:proxy) { {:proxy_username => "douglas", :proxy_password => "adams"} } 73 | let(:body) { "The Ultimate Question" } 74 | 75 | let :request do 76 | HTTP::Request.new( 77 | :verb => :post, 78 | :uri => "http://example.com/", 79 | :headers => headers, 80 | :proxy => proxy, 81 | :body => body 82 | ) 83 | end 84 | 85 | subject(:redirected) { request.redirect "http://blog.example.com/" } 86 | 87 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://blog.example.com/" } 88 | 89 | its(:verb) { is_expected.to eq request.verb } 90 | its(:body) { is_expected.to eq request.body } 91 | its(:proxy) { is_expected.to eq request.proxy } 92 | 93 | it "presets new Host header" do 94 | expect(redirected["Host"]).to eq "blog.example.com" 95 | end 96 | 97 | context "with URL with non-standard port given" do 98 | subject(:redirected) { request.redirect "http://example.com:8080" } 99 | 100 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com:8080" } 101 | 102 | its(:verb) { is_expected.to eq request.verb } 103 | its(:body) { is_expected.to eq request.body } 104 | its(:proxy) { is_expected.to eq request.proxy } 105 | 106 | it "presets new Host header" do 107 | expect(redirected["Host"]).to eq "example.com:8080" 108 | end 109 | end 110 | 111 | context "with schema-less absolute URL given" do 112 | subject(:redirected) { request.redirect "//another.example.com/blog" } 113 | 114 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://another.example.com/blog" } 115 | 116 | its(:verb) { is_expected.to eq request.verb } 117 | its(:body) { is_expected.to eq request.body } 118 | its(:proxy) { is_expected.to eq request.proxy } 119 | 120 | it "presets new Host header" do 121 | expect(redirected["Host"]).to eq "another.example.com" 122 | end 123 | end 124 | 125 | context "with relative URL given" do 126 | subject(:redirected) { request.redirect "/blog" } 127 | 128 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com/blog" } 129 | 130 | its(:verb) { is_expected.to eq request.verb } 131 | its(:body) { is_expected.to eq request.body } 132 | its(:proxy) { is_expected.to eq request.proxy } 133 | 134 | it "keeps Host header" do 135 | expect(redirected["Host"]).to eq "example.com" 136 | end 137 | 138 | context "with original URI having non-standard port" do 139 | let :request do 140 | HTTP::Request.new( 141 | :verb => :post, 142 | :uri => "http://example.com:8080/", 143 | :headers => headers, 144 | :proxy => proxy, 145 | :body => body 146 | ) 147 | end 148 | 149 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com:8080/blog" } 150 | end 151 | end 152 | 153 | context "with relative URL that misses leading slash given" do 154 | subject(:redirected) { request.redirect "blog" } 155 | 156 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com/blog" } 157 | 158 | its(:verb) { is_expected.to eq request.verb } 159 | its(:body) { is_expected.to eq request.body } 160 | its(:proxy) { is_expected.to eq request.proxy } 161 | 162 | it "keeps Host header" do 163 | expect(redirected["Host"]).to eq "example.com" 164 | end 165 | 166 | context "with original URI having non-standard port" do 167 | let :request do 168 | HTTP::Request.new( 169 | :verb => :post, 170 | :uri => "http://example.com:8080/", 171 | :headers => headers, 172 | :proxy => proxy, 173 | :body => body 174 | ) 175 | end 176 | 177 | its(:uri) { is_expected.to eq HTTP::URI.parse "http://example.com:8080/blog" } 178 | end 179 | end 180 | 181 | context "with new verb given" do 182 | subject { request.redirect "http://blog.example.com/", :get } 183 | its(:verb) { is_expected.to be :get } 184 | end 185 | end 186 | 187 | describe "#headline" do 188 | subject(:headline) { request.headline } 189 | 190 | it { is_expected.to eq "GET /foo?bar=baz HTTP/1.1" } 191 | 192 | context "when URI contains encoded query" do 193 | let(:encoded_query) { "t=1970-01-01T01%3A00%3A00%2B01%3A00" } 194 | let(:request_uri) { "http://example.com/foo/?#{encoded_query}" } 195 | 196 | it "does not unencodes query part" do 197 | expect(headline).to eq "GET /foo/?#{encoded_query} HTTP/1.1" 198 | end 199 | end 200 | 201 | context "when URI contains non-ASCII path" do 202 | let(:request_uri) { "http://example.com/キョ" } 203 | 204 | it "encodes non-ASCII path part" do 205 | expect(headline).to eq "GET /%E3%82%AD%E3%83%A7 HTTP/1.1" 206 | end 207 | end 208 | 209 | context "when URI contains fragment" do 210 | let(:request_uri) { "http://example.com/foo#bar" } 211 | 212 | it "omits fragment part" do 213 | expect(headline).to eq "GET /foo HTTP/1.1" 214 | end 215 | end 216 | 217 | context "with proxy" do 218 | let(:proxy) { {:user => "user", :pass => "pass"} } 219 | it { is_expected.to eq "GET http://example.com/foo?bar=baz HTTP/1.1" } 220 | 221 | context "and HTTPS uri" do 222 | let(:request_uri) { "https://example.com/foo?bar=baz" } 223 | 224 | it { is_expected.to eq "GET /foo?bar=baz HTTP/1.1" } 225 | end 226 | end 227 | end 228 | 229 | describe "#inspect" do 230 | subject { request.inspect } 231 | 232 | it { is_expected.to eq "#" } 233 | end 234 | end 235 | -------------------------------------------------------------------------------- /spec/lib/http/response/status_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response::Status do 4 | describe ".new" do 5 | it "fails if given value does not respond to #to_i" do 6 | expect { described_class.new double }.to raise_error TypeError 7 | end 8 | 9 | it "accepts any object that responds to #to_i" do 10 | expect { described_class.new double :to_i => 200 }.to_not raise_error 11 | end 12 | end 13 | 14 | describe "#code" do 15 | subject { described_class.new("200.0").code } 16 | it { is_expected.to eq 200 } 17 | it { is_expected.to be_a Integer } 18 | end 19 | 20 | describe "#reason" do 21 | subject { described_class.new(code).reason } 22 | 23 | context "with unknown code" do 24 | let(:code) { 1024 } 25 | it { is_expected.to be_nil } 26 | end 27 | 28 | described_class::REASONS.each do |code, reason| 29 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 30 | context 'with well-known code: #{code}' do 31 | let(:code) { #{code} } 32 | it { is_expected.to eq #{reason.inspect} } 33 | it { is_expected.to be_frozen } 34 | end 35 | RUBY 36 | end 37 | end 38 | 39 | context "with 1xx codes" do 40 | subject { (100...200).map { |code| described_class.new code } } 41 | 42 | it "is #informational?" do 43 | expect(subject).to all(satisfy(&:informational?)) 44 | end 45 | 46 | it "is not #success?" do 47 | expect(subject).to all(satisfy { |status| !status.success? }) 48 | end 49 | 50 | it "is not #redirect?" do 51 | expect(subject).to all(satisfy { |status| !status.redirect? }) 52 | end 53 | 54 | it "is not #client_error?" do 55 | expect(subject).to all(satisfy { |status| !status.client_error? }) 56 | end 57 | 58 | it "is not #server_error?" do 59 | expect(subject).to all(satisfy { |status| !status.server_error? }) 60 | end 61 | end 62 | 63 | context "with 2xx codes" do 64 | subject { (200...300).map { |code| described_class.new code } } 65 | 66 | it "is not #informational?" do 67 | expect(subject).to all(satisfy { |status| !status.informational? }) 68 | end 69 | 70 | it "is #success?" do 71 | expect(subject).to all(satisfy(&:success?)) 72 | end 73 | 74 | it "is not #redirect?" do 75 | expect(subject).to all(satisfy { |status| !status.redirect? }) 76 | end 77 | 78 | it "is not #client_error?" do 79 | expect(subject).to all(satisfy { |status| !status.client_error? }) 80 | end 81 | 82 | it "is not #server_error?" do 83 | expect(subject).to all(satisfy { |status| !status.server_error? }) 84 | end 85 | end 86 | 87 | context "with 3xx codes" do 88 | subject { (300...400).map { |code| described_class.new code } } 89 | 90 | it "is not #informational?" do 91 | expect(subject).to all(satisfy { |status| !status.informational? }) 92 | end 93 | 94 | it "is not #success?" do 95 | expect(subject).to all(satisfy { |status| !status.success? }) 96 | end 97 | 98 | it "is #redirect?" do 99 | expect(subject).to all(satisfy(&:redirect?)) 100 | end 101 | 102 | it "is not #client_error?" do 103 | expect(subject).to all(satisfy { |status| !status.client_error? }) 104 | end 105 | 106 | it "is not #server_error?" do 107 | expect(subject).to all(satisfy { |status| !status.server_error? }) 108 | end 109 | end 110 | 111 | context "with 4xx codes" do 112 | subject { (400...500).map { |code| described_class.new code } } 113 | 114 | it "is not #informational?" do 115 | expect(subject).to all(satisfy { |status| !status.informational? }) 116 | end 117 | 118 | it "is not #success?" do 119 | expect(subject).to all(satisfy { |status| !status.success? }) 120 | end 121 | 122 | it "is not #redirect?" do 123 | expect(subject).to all(satisfy { |status| !status.redirect? }) 124 | end 125 | 126 | it "is #client_error?" do 127 | expect(subject).to all(satisfy(&:client_error?)) 128 | end 129 | 130 | it "is not #server_error?" do 131 | expect(subject).to all(satisfy { |status| !status.server_error? }) 132 | end 133 | end 134 | 135 | context "with 5xx codes" do 136 | subject { (500...600).map { |code| described_class.new code } } 137 | 138 | it "is not #informational?" do 139 | expect(subject).to all(satisfy { |status| !status.informational? }) 140 | end 141 | 142 | it "is not #success?" do 143 | expect(subject).to all(satisfy { |status| !status.success? }) 144 | end 145 | 146 | it "is not #redirect?" do 147 | expect(subject).to all(satisfy { |status| !status.redirect? }) 148 | end 149 | 150 | it "is not #client_error?" do 151 | expect(subject).to all(satisfy { |status| !status.client_error? }) 152 | end 153 | 154 | it "is #server_error?" do 155 | expect(subject).to all(satisfy(&:server_error?)) 156 | end 157 | end 158 | 159 | describe "#to_sym" do 160 | subject { described_class.new(code).to_sym } 161 | 162 | context "with unknown code" do 163 | let(:code) { 1024 } 164 | it { is_expected.to be_nil } 165 | end 166 | 167 | described_class::SYMBOLS.each do |code, symbol| 168 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 169 | context 'with well-known code: #{code}' do 170 | let(:code) { #{code} } 171 | it { is_expected.to be #{symbol.inspect} } 172 | end 173 | RUBY 174 | end 175 | end 176 | 177 | describe "#inspect" do 178 | it "returns quoted code and reason phrase" do 179 | status = described_class.new 200 180 | expect(status.inspect).to eq "#" 181 | end 182 | end 183 | 184 | # testing edge cases only 185 | describe "::SYMBOLS" do 186 | subject { described_class::SYMBOLS } 187 | 188 | # "OK" 189 | its([200]) { is_expected.to be :ok } 190 | 191 | # "Bad Request" 192 | its([400]) { is_expected.to be :bad_request } 193 | end 194 | 195 | described_class::SYMBOLS.each do |code, symbol| 196 | class_eval <<-RUBY, __FILE__, __LINE__ + 1 197 | describe '##{symbol}?' do 198 | subject { status.#{symbol}? } 199 | 200 | context 'when code is #{code}' do 201 | let(:status) { described_class.new #{code} } 202 | it { is_expected.to be true } 203 | end 204 | 205 | context 'when code is higher than #{code}' do 206 | let(:status) { described_class.new #{code + 1} } 207 | it { is_expected.to be false } 208 | end 209 | 210 | context 'when code is lower than #{code}' do 211 | let(:status) { described_class.new #{code - 1} } 212 | it { is_expected.to be false } 213 | end 214 | end 215 | RUBY 216 | end 217 | 218 | describe ".coerce" do 219 | context "with String" do 220 | it "coerces reasons" do 221 | expect(described_class.coerce("Bad request")).to eq described_class.new 400 222 | end 223 | 224 | it "fails when reason is unknown" do 225 | expect { described_class.coerce "foobar" }.to raise_error HTTP::Error 226 | end 227 | end 228 | 229 | context "with Symbol" do 230 | it "coerces symbolized reasons" do 231 | expect(described_class.coerce(:bad_request)).to eq described_class.new 400 232 | end 233 | 234 | it "fails when symbolized reason is unknown" do 235 | expect { described_class.coerce(:foobar) }.to raise_error HTTP::Error 236 | end 237 | end 238 | 239 | context "with Numeric" do 240 | it "coerces as Fixnum code" do 241 | expect(described_class.coerce(200.1)).to eq described_class.new 200 242 | end 243 | end 244 | 245 | it "fails if coercion failed" do 246 | expect { described_class.coerce(true) }.to raise_error HTTP::Error 247 | end 248 | 249 | it "is aliased as `.[]`" do 250 | expect(described_class.method(:coerce)).to eq described_class.method :[] 251 | end 252 | end 253 | end 254 | -------------------------------------------------------------------------------- /spec/lib/http/response_spec.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | RSpec.describe HTTP::Response do 4 | let(:body) { "Hello world!" } 5 | let(:uri) { "http://example.com/" } 6 | let(:headers) { {} } 7 | let(:request) { HTTP::Request.new(:verb => :get, :uri => uri) } 8 | 9 | subject(:response) do 10 | HTTP::Response.new( 11 | :status => 200, 12 | :version => "1.1", 13 | :headers => headers, 14 | :body => body, 15 | :request => request 16 | ) 17 | end 18 | 19 | it "includes HTTP::Headers::Mixin" do 20 | expect(described_class).to include HTTP::Headers::Mixin 21 | end 22 | 23 | describe "to_a" do 24 | let(:body) { "Hello world" } 25 | let(:content_type) { "text/plain" } 26 | let(:headers) { {"Content-Type" => content_type} } 27 | 28 | it "returns a Rack-like array" do 29 | expect(subject.to_a).to eq([200, headers, body]) 30 | end 31 | end 32 | 33 | describe "#content_length" do 34 | subject { response.content_length } 35 | 36 | context "without Content-Length header" do 37 | it { is_expected.to be_nil } 38 | end 39 | 40 | context "with Content-Length: 5" do 41 | let(:headers) { {"Content-Length" => "5"} } 42 | it { is_expected.to eq 5 } 43 | end 44 | 45 | context "with invalid Content-Length" do 46 | let(:headers) { {"Content-Length" => "foo"} } 47 | it { is_expected.to be_nil } 48 | end 49 | end 50 | 51 | describe "mime_type" do 52 | subject { response.mime_type } 53 | 54 | context "without Content-Type header" do 55 | let(:headers) { {} } 56 | it { is_expected.to be_nil } 57 | end 58 | 59 | context "with Content-Type: text/html" do 60 | let(:headers) { {"Content-Type" => "text/html"} } 61 | it { is_expected.to eq "text/html" } 62 | end 63 | 64 | context "with Content-Type: text/html; charset=utf-8" do 65 | let(:headers) { {"Content-Type" => "text/html; charset=utf-8"} } 66 | it { is_expected.to eq "text/html" } 67 | end 68 | end 69 | 70 | describe "charset" do 71 | subject { response.charset } 72 | 73 | context "without Content-Type header" do 74 | let(:headers) { {} } 75 | it { is_expected.to be_nil } 76 | end 77 | 78 | context "with Content-Type: text/html" do 79 | let(:headers) { {"Content-Type" => "text/html"} } 80 | it { is_expected.to be_nil } 81 | end 82 | 83 | context "with Content-Type: text/html; charset=utf-8" do 84 | let(:headers) { {"Content-Type" => "text/html; charset=utf-8"} } 85 | it { is_expected.to eq "utf-8" } 86 | end 87 | end 88 | 89 | describe "#parse" do 90 | let(:headers) { {"Content-Type" => content_type} } 91 | let(:body) { '{"foo":"bar"}' } 92 | 93 | context "with known content type" do 94 | let(:content_type) { "application/json" } 95 | it "returns parsed body" do 96 | expect(response.parse).to eq "foo" => "bar" 97 | end 98 | end 99 | 100 | context "with unknown content type" do 101 | let(:content_type) { "application/deadbeef" } 102 | it "raises HTTP::Error" do 103 | expect { response.parse }.to raise_error HTTP::Error 104 | end 105 | end 106 | 107 | context "with explicitly given mime type" do 108 | let(:content_type) { "application/deadbeef" } 109 | it "ignores mime_type of response" do 110 | expect(response.parse("application/json")).to eq "foo" => "bar" 111 | end 112 | 113 | it "supports mime type aliases" do 114 | expect(response.parse(:json)).to eq "foo" => "bar" 115 | end 116 | end 117 | end 118 | 119 | describe "#flush" do 120 | let(:body) { double :to_s => "" } 121 | 122 | it "returns response self-reference" do 123 | expect(response.flush).to be response 124 | end 125 | 126 | it "flushes body" do 127 | expect(body).to receive :to_s 128 | response.flush 129 | end 130 | end 131 | 132 | describe "#inspect" do 133 | subject { response.inspect } 134 | 135 | let(:headers) { {:content_type => "text/plain"} } 136 | let(:body) { double :to_s => "foobar" } 137 | 138 | it { is_expected.to eq '#"text/plain"}>' } 139 | end 140 | 141 | describe "#cookies" do 142 | let(:cookies) { ["a=1", "b=2; domain=example.com", "c=3; domain=bad.org"] } 143 | let(:headers) { {"Set-Cookie" => cookies} } 144 | 145 | subject(:jar) { response.cookies } 146 | 147 | it { is_expected.to be_an HTTP::CookieJar } 148 | 149 | it "contains cookies without domain restriction" do 150 | expect(jar.count { |c| "a" == c.name }).to eq 1 151 | end 152 | 153 | it "contains cookies limited to domain of request uri" do 154 | expect(jar.count { |c| "b" == c.name }).to eq 1 155 | end 156 | 157 | it "does not contains cookies limited to non-requeted uri" do 158 | expect(jar.count { |c| "c" == c.name }).to eq 0 159 | end 160 | end 161 | 162 | describe "#connection" do 163 | let(:connection) { double } 164 | 165 | subject(:response) do 166 | HTTP::Response.new( 167 | :version => "1.1", 168 | :status => 200, 169 | :connection => connection, 170 | :request => request 171 | ) 172 | end 173 | 174 | it "returns the connection object used to instantiate the response" do 175 | expect(response.connection).to eq connection 176 | end 177 | end 178 | 179 | describe "#chunked?" do 180 | subject { response } 181 | context "when encoding is set to chunked" do 182 | let(:headers) { {"Transfer-Encoding" => "chunked"} } 183 | it { is_expected.to be_chunked } 184 | end 185 | it { is_expected.not_to be_chunked } 186 | end 187 | 188 | describe "backwards compatibilty with :uri" do 189 | context "with no :verb" do 190 | subject(:response) do 191 | HTTP::Response.new( 192 | :status => 200, 193 | :version => "1.1", 194 | :headers => headers, 195 | :body => body, 196 | :uri => uri 197 | ) 198 | end 199 | 200 | it "defaults the uri to :uri" do 201 | expect(response.request.uri.to_s).to eq uri 202 | end 203 | 204 | it "defaults to the verb to :get" do 205 | expect(response.request.verb).to eq :get 206 | end 207 | end 208 | 209 | context "with both a :request and :uri" do 210 | subject(:response) do 211 | HTTP::Response.new( 212 | :status => 200, 213 | :version => "1.1", 214 | :headers => headers, 215 | :body => body, 216 | :uri => uri, 217 | :request => request 218 | ) 219 | end 220 | 221 | it "raises ArgumentError" do 222 | expect { response }.to raise_error(ArgumentError) 223 | end 224 | end 225 | end 226 | 227 | describe "#body" do 228 | let(:connection) { double(:sequence_id => 0) } 229 | let(:chunks) { ["Hello, ", "World!"] } 230 | 231 | subject(:response) do 232 | HTTP::Response.new( 233 | :status => 200, 234 | :version => "1.1", 235 | :headers => headers, 236 | :request => request, 237 | :connection => connection 238 | ) 239 | end 240 | 241 | before do 242 | allow(connection).to receive(:readpartial) { chunks.shift } 243 | allow(connection).to receive(:body_completed?) { chunks.empty? } 244 | end 245 | 246 | context "with no Content-Type" do 247 | let(:headers) { {} } 248 | 249 | it "returns a body with default binary encoding" do 250 | expect(response.body.to_s.encoding).to eq Encoding::BINARY 251 | end 252 | end 253 | 254 | context "with Content-Type: application/json" do 255 | let(:headers) { {"Content-Type" => "application/json"} } 256 | 257 | it "returns a body with a default UTF_8 encoding" do 258 | expect(response.body.to_s.encoding).to eq Encoding::UTF_8 259 | end 260 | end 261 | end 262 | end 263 | -------------------------------------------------------------------------------- /lib/http/chainable.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "base64" 4 | 5 | require "http/headers" 6 | 7 | module HTTP 8 | module Chainable 9 | # Request a get sans response body 10 | # @param uri 11 | # @option options [Hash] 12 | def head(uri, options = {}) 13 | request :head, uri, options 14 | end 15 | 16 | # Get a resource 17 | # @param uri 18 | # @option options [Hash] 19 | def get(uri, options = {}) 20 | request :get, uri, options 21 | end 22 | 23 | # Post to a resource 24 | # @param uri 25 | # @option options [Hash] 26 | def post(uri, options = {}) 27 | request :post, uri, options 28 | end 29 | 30 | # Put to a resource 31 | # @param uri 32 | # @option options [Hash] 33 | def put(uri, options = {}) 34 | request :put, uri, options 35 | end 36 | 37 | # Delete a resource 38 | # @param uri 39 | # @option options [Hash] 40 | def delete(uri, options = {}) 41 | request :delete, uri, options 42 | end 43 | 44 | # Echo the request back to the client 45 | # @param uri 46 | # @option options [Hash] 47 | def trace(uri, options = {}) 48 | request :trace, uri, options 49 | end 50 | 51 | # Return the methods supported on the given URI 52 | # @param uri 53 | # @option options [Hash] 54 | def options(uri, options = {}) 55 | request :options, uri, options 56 | end 57 | 58 | # Convert to a transparent TCP/IP tunnel 59 | # @param uri 60 | # @option options [Hash] 61 | def connect(uri, options = {}) 62 | request :connect, uri, options 63 | end 64 | 65 | # Apply partial modifications to a resource 66 | # @param uri 67 | # @option options [Hash] 68 | def patch(uri, options = {}) 69 | request :patch, uri, options 70 | end 71 | 72 | # Make an HTTP request with the given verb 73 | # @param (see Client#request) 74 | def request(*args) 75 | branch(default_options).request(*args) 76 | end 77 | 78 | # Prepare an HTTP request with the given verb 79 | # @param (see Client#build_request) 80 | def build_request(*args) 81 | branch(default_options).build_request(*args) 82 | end 83 | 84 | # @overload timeout(options = {}) 85 | # Adds per operation timeouts to the request 86 | # @param [Hash] options 87 | # @option options [Float] :read Read timeout 88 | # @option options [Float] :write Write timeout 89 | # @option options [Float] :connect Connect timeout 90 | # @overload timeout(global_timeout) 91 | # Adds a global timeout to the full request 92 | # @param [Numeric] global_timeout 93 | def timeout(options) 94 | klass, options = case options 95 | when Numeric then [HTTP::Timeout::Global, {:global => options}] 96 | when Hash then [HTTP::Timeout::PerOperation, options.dup] 97 | when :null then [HTTP::Timeout::Null, {}] 98 | else raise ArgumentError, "Use `.timeout(global_timeout_in_seconds)` or `.timeout(connect: x, write: y, read: z)`." 99 | 100 | end 101 | 102 | %i[global read write connect].each do |k| 103 | next unless options.key? k 104 | 105 | options["#{k}_timeout".to_sym] = options.delete k 106 | end 107 | 108 | branch default_options.merge( 109 | :timeout_class => klass, 110 | :timeout_options => options 111 | ) 112 | end 113 | 114 | # @overload persistent(host, timeout: 5) 115 | # Flags as persistent 116 | # @param [String] host 117 | # @option [Integer] timeout Keep alive timeout 118 | # @raise [Request::Error] if Host is invalid 119 | # @return [HTTP::Client] Persistent client 120 | # @overload persistent(host, timeout: 5, &block) 121 | # Executes given block with persistent client and automatically closes 122 | # connection at the end of execution. 123 | # 124 | # @example 125 | # 126 | # def keys(users) 127 | # HTTP.persistent("https://github.com") do |http| 128 | # users.map { |u| http.get("/#{u}.keys").to_s } 129 | # end 130 | # end 131 | # 132 | # # same as 133 | # 134 | # def keys(users) 135 | # http = HTTP.persistent "https://github.com" 136 | # users.map { |u| http.get("/#{u}.keys").to_s } 137 | # ensure 138 | # http.close if http 139 | # end 140 | # 141 | # 142 | # @yieldparam [HTTP::Client] client Persistent client 143 | # @return [Object] result of last expression in the block 144 | def persistent(host, timeout: 5) 145 | options = {:keep_alive_timeout => timeout} 146 | p_client = branch default_options.merge(options).with_persistent host 147 | return p_client unless block_given? 148 | 149 | yield p_client 150 | ensure 151 | p_client&.close 152 | end 153 | 154 | # Make a request through an HTTP proxy 155 | # @param [Array] proxy 156 | # @raise [Request::Error] if HTTP proxy is invalid 157 | def via(*proxy) 158 | proxy_hash = {} 159 | proxy_hash[:proxy_address] = proxy[0] if proxy[0].is_a?(String) 160 | proxy_hash[:proxy_port] = proxy[1] if proxy[1].is_a?(Integer) 161 | proxy_hash[:proxy_username] = proxy[2] if proxy[2].is_a?(String) 162 | proxy_hash[:proxy_password] = proxy[3] if proxy[3].is_a?(String) 163 | proxy_hash[:proxy_headers] = proxy[2] if proxy[2].is_a?(Hash) 164 | proxy_hash[:proxy_headers] = proxy[4] if proxy[4].is_a?(Hash) 165 | 166 | raise(RequestError, "invalid HTTP proxy: #{proxy_hash}") unless (2..5).cover?(proxy_hash.keys.size) 167 | 168 | branch default_options.with_proxy(proxy_hash) 169 | end 170 | alias through via 171 | 172 | # Make client follow redirects. 173 | # @param options 174 | # @return [HTTP::Client] 175 | # @see Redirector#initialize 176 | def follow(options = {}) 177 | branch default_options.with_follow options 178 | end 179 | 180 | # Make a request with the given headers 181 | # @param headers 182 | def headers(headers) 183 | branch default_options.with_headers(headers) 184 | end 185 | 186 | # Make a request with the given cookies 187 | def cookies(cookies) 188 | branch default_options.with_cookies(cookies) 189 | end 190 | 191 | # Force a specific encoding for response body 192 | def encoding(encoding) 193 | branch default_options.with_encoding(encoding) 194 | end 195 | 196 | # Accept the given MIME type(s) 197 | # @param type 198 | def accept(type) 199 | headers Headers::ACCEPT => MimeType.normalize(type) 200 | end 201 | 202 | # Make a request with the given Authorization header 203 | # @param [#to_s] value Authorization header value 204 | def auth(value) 205 | headers Headers::AUTHORIZATION => value.to_s 206 | end 207 | 208 | # Make a request with the given Basic authorization header 209 | # @see http://tools.ietf.org/html/rfc2617 210 | # @param [#fetch] opts 211 | # @option opts [#to_s] :user 212 | # @option opts [#to_s] :pass 213 | def basic_auth(opts) 214 | user = opts.fetch(:user) 215 | pass = opts.fetch(:pass) 216 | creds = "#{user}:#{pass}" 217 | 218 | auth("Basic #{Base64.strict_encode64(creds)}") 219 | end 220 | 221 | # Get options for HTTP 222 | # @return [HTTP::Options] 223 | def default_options 224 | @default_options ||= HTTP::Options.new 225 | end 226 | 227 | # Set options for HTTP 228 | # @param opts 229 | # @return [HTTP::Options] 230 | def default_options=(opts) 231 | @default_options = HTTP::Options.new(opts) 232 | end 233 | 234 | # Set TCP_NODELAY on the socket 235 | def nodelay 236 | branch default_options.with_nodelay(true) 237 | end 238 | 239 | # Turn on given features. Available features are: 240 | # * auto_inflate 241 | # * auto_deflate 242 | # * instrumentation 243 | # * logging 244 | # * normalize_uri 245 | # @param features 246 | def use(*features) 247 | branch default_options.with_features(features) 248 | end 249 | 250 | private 251 | 252 | # :nodoc: 253 | def branch(options) 254 | HTTP::Client.new(options) 255 | end 256 | end 257 | end 258 | -------------------------------------------------------------------------------- /lib/http/request.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require "forwardable" 4 | require "base64" 5 | require "time" 6 | 7 | require "http/errors" 8 | require "http/headers" 9 | require "http/request/body" 10 | require "http/request/writer" 11 | require "http/version" 12 | require "http/uri" 13 | 14 | module HTTP 15 | class Request 16 | extend Forwardable 17 | 18 | include HTTP::Headers::Mixin 19 | 20 | # The method given was not understood 21 | class UnsupportedMethodError < RequestError; end 22 | 23 | # The scheme of given URI was not understood 24 | class UnsupportedSchemeError < RequestError; end 25 | 26 | # Default User-Agent header value 27 | USER_AGENT = "http.rb/#{HTTP::VERSION}" 28 | 29 | METHODS = [ 30 | # RFC 2616: Hypertext Transfer Protocol -- HTTP/1.1 31 | :options, :get, :head, :post, :put, :delete, :trace, :connect, 32 | 33 | # RFC 2518: HTTP Extensions for Distributed Authoring -- WEBDAV 34 | :propfind, :proppatch, :mkcol, :copy, :move, :lock, :unlock, 35 | 36 | # RFC 3648: WebDAV Ordered Collections Protocol 37 | :orderpatch, 38 | 39 | # RFC 3744: WebDAV Access Control Protocol 40 | :acl, 41 | 42 | # RFC 6352: vCard Extensions to WebDAV -- CardDAV 43 | :report, 44 | 45 | # RFC 5789: PATCH Method for HTTP 46 | :patch, 47 | 48 | # draft-reschke-webdav-search: WebDAV Search 49 | :search, 50 | 51 | # RFC 4791: Calendaring Extensions to WebDAV -- CalDAV 52 | :mkcalendar 53 | ].freeze 54 | 55 | # Allowed schemes 56 | SCHEMES = %i[http https ws wss].freeze 57 | 58 | # Default ports of supported schemes 59 | PORTS = { 60 | :http => 80, 61 | :https => 443, 62 | :ws => 80, 63 | :wss => 443 64 | }.freeze 65 | 66 | # Method is given as a lowercase symbol e.g. :get, :post 67 | attr_reader :verb 68 | 69 | # Scheme is normalized to be a lowercase symbol e.g. :http, :https 70 | attr_reader :scheme 71 | 72 | attr_reader :uri_normalizer 73 | 74 | # "Request URI" as per RFC 2616 75 | # http://www.w3.org/Protocols/rfc2616/rfc2616-sec5.html 76 | attr_reader :uri 77 | attr_reader :proxy, :body, :version 78 | 79 | # @option opts [String] :version 80 | # @option opts [#to_s] :verb HTTP request method 81 | # @option opts [#call] :uri_normalizer (HTTP::URI::NORMALIZER) 82 | # @option opts [HTTP::URI, #to_s] :uri 83 | # @option opts [Hash] :headers 84 | # @option opts [Hash] :proxy 85 | # @option opts [String, Enumerable, IO, nil] :body 86 | def initialize(opts) 87 | @verb = opts.fetch(:verb).to_s.downcase.to_sym 88 | @uri_normalizer = opts[:uri_normalizer] || HTTP::URI::NORMALIZER 89 | 90 | @uri = @uri_normalizer.call(opts.fetch(:uri)) 91 | @scheme = @uri.scheme.to_s.downcase.to_sym if @uri.scheme 92 | 93 | raise(UnsupportedMethodError, "unknown method: #{verb}") unless METHODS.include?(@verb) 94 | raise(UnsupportedSchemeError, "unknown scheme: #{scheme}") unless SCHEMES.include?(@scheme) 95 | 96 | @proxy = opts[:proxy] || {} 97 | @version = opts[:version] || "1.1" 98 | @headers = prepare_headers(opts[:headers]) 99 | @body = prepare_body(opts[:body]) 100 | end 101 | 102 | # Returns new Request with updated uri 103 | def redirect(uri, verb = @verb) 104 | headers = self.headers.dup 105 | headers.delete(Headers::HOST) 106 | 107 | new_body = body.source 108 | if verb == :get 109 | # request bodies should not always be resubmitted when following a redirect 110 | # some servers will close the connection after receiving the request headers 111 | # which may cause Errno::ECONNRESET: Connection reset by peer 112 | # see https://github.com/httprb/http/issues/649 113 | # new_body = Request::Body.new(nil) 114 | new_body = nil 115 | # the CONTENT_TYPE header causes problems if set on a get request w/ an empty body 116 | # the server might assume that there should be content if it is set to multipart 117 | # rack raises EmptyContentError if this happens 118 | headers.delete(Headers::CONTENT_TYPE) 119 | end 120 | 121 | self.class.new( 122 | :verb => verb, 123 | :uri => @uri.join(uri), 124 | :headers => headers, 125 | :proxy => proxy, 126 | :body => new_body, 127 | :version => version, 128 | :uri_normalizer => uri_normalizer 129 | ) 130 | end 131 | 132 | # Stream the request to a socket 133 | def stream(socket) 134 | include_proxy_headers if using_proxy? && !@uri.https? 135 | Request::Writer.new(socket, body, headers, headline).stream 136 | end 137 | 138 | # Is this request using a proxy? 139 | def using_proxy? 140 | proxy && proxy.keys.size >= 2 141 | end 142 | 143 | # Is this request using an authenticated proxy? 144 | def using_authenticated_proxy? 145 | proxy && proxy.keys.size >= 4 146 | end 147 | 148 | def include_proxy_headers 149 | headers.merge!(proxy[:proxy_headers]) if proxy.key?(:proxy_headers) 150 | include_proxy_authorization_header if using_authenticated_proxy? 151 | end 152 | 153 | # Compute and add the Proxy-Authorization header 154 | def include_proxy_authorization_header 155 | headers[Headers::PROXY_AUTHORIZATION] = proxy_authorization_header 156 | end 157 | 158 | def proxy_authorization_header 159 | digest = Base64.strict_encode64("#{proxy[:proxy_username]}:#{proxy[:proxy_password]}") 160 | "Basic #{digest}" 161 | end 162 | 163 | # Setup tunnel through proxy for SSL request 164 | def connect_using_proxy(socket) 165 | Request::Writer.new(socket, nil, proxy_connect_headers, proxy_connect_header).connect_through_proxy 166 | end 167 | 168 | # Compute HTTP request header for direct or proxy request 169 | def headline 170 | request_uri = 171 | if using_proxy? && !uri.https? 172 | uri.omit(:fragment) 173 | else 174 | uri.request_uri 175 | end 176 | 177 | "#{verb.to_s.upcase} #{request_uri} HTTP/#{version}" 178 | end 179 | 180 | # Compute HTTP request header SSL proxy connection 181 | def proxy_connect_header 182 | "CONNECT #{host}:#{port} HTTP/#{version}" 183 | end 184 | 185 | # Headers to send with proxy connect request 186 | def proxy_connect_headers 187 | connect_headers = HTTP::Headers.coerce( 188 | Headers::HOST => headers[Headers::HOST], 189 | Headers::USER_AGENT => headers[Headers::USER_AGENT] 190 | ) 191 | 192 | connect_headers[Headers::PROXY_AUTHORIZATION] = proxy_authorization_header if using_authenticated_proxy? 193 | connect_headers.merge!(proxy[:proxy_headers]) if proxy.key?(:proxy_headers) 194 | connect_headers 195 | end 196 | 197 | # Host for tcp socket 198 | def socket_host 199 | using_proxy? ? proxy[:proxy_address] : host 200 | end 201 | 202 | # Port for tcp socket 203 | def socket_port 204 | using_proxy? ? proxy[:proxy_port] : port 205 | end 206 | 207 | # Human-readable representation of base request info. 208 | # 209 | # @example 210 | # 211 | # req.inspect 212 | # # => # 213 | # 214 | # @return [String] 215 | def inspect 216 | "#<#{self.class}/#{@version} #{verb.to_s.upcase} #{uri}>" 217 | end 218 | 219 | private 220 | 221 | # @!attribute [r] host 222 | # @return [String] 223 | def_delegator :@uri, :host 224 | 225 | # @!attribute [r] port 226 | # @return [Fixnum] 227 | def port 228 | @uri.port || @uri.default_port 229 | end 230 | 231 | # @return [String] Default host (with port if needed) header value. 232 | def default_host_header_value 233 | PORTS[@scheme] == port ? host : "#{host}:#{port}" 234 | end 235 | 236 | def prepare_body(body) 237 | body.is_a?(Request::Body) ? body : Request::Body.new(body) 238 | end 239 | 240 | def prepare_headers(headers) 241 | headers = HTTP::Headers.coerce(headers || {}) 242 | 243 | headers[Headers::HOST] ||= default_host_header_value 244 | headers[Headers::USER_AGENT] ||= USER_AGENT 245 | 246 | headers 247 | end 248 | end 249 | end 250 | --------------------------------------------------------------------------------