├── .github └── workflows │ └── ruby.yml ├── .gitignore ├── Gemfile ├── README.md ├── Rakefile ├── bin └── dagger ├── dagger.gemspec ├── lib ├── dagger.rb └── dagger │ ├── connection_manager.rb │ ├── ox_extension.rb │ ├── parsers.rb │ ├── response.rb │ ├── version.rb │ └── wrapper.rb └── spec ├── arguments_spec.rb ├── ip_connect_spec.rb ├── parsers_spec.rb ├── persistent_spec.rb ├── retries_spec.rb └── sending_data_spec.rb /.github/workflows/ruby.yml: -------------------------------------------------------------------------------- 1 | name: Ruby 2 | on: [push] 3 | 4 | jobs: 5 | build: 6 | runs-on: ubuntu-latest 7 | 8 | steps: 9 | - uses: actions/checkout@v1 10 | - name: Set up Ruby 2.6 11 | uses: actions/setup-ruby@v1 12 | with: 13 | ruby-version: 2.6.x 14 | - name: Build and test with Rake 15 | run: | 16 | gem install bundler 17 | bundle install --jobs 4 --retry 3 18 | bundle exec rspec 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pkg 2 | .ruby-version 3 | Gemfile.lock 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # specified in tuktuk.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Dagger 2 | 3 | Featherweight wrapper around Net::HTTP. 4 | 5 | Follows redirects if instructed, and comes with out-of-the-box parsing of JSON and XML, via [oj](https://github.com/ohler55/oj) and [ox](https://github.com/ohler55/ox), respectively. 6 | 7 | # Installation 8 | 9 | In your Gemfile: 10 | 11 | gem 'dagger' 12 | 13 | # Usage 14 | 15 | ## `get(url, [options])` 16 | 17 | ```rb 18 | require 'dagger' 19 | resp = Dagger.get('http://google.com') 20 | 21 | puts resp.body # => "" 22 | 23 | # you can also pass a query via the options hash, in which case is appended as a query string. 24 | Dagger.get('google.com/search', { query: { q: 'dagger' } }) # => requests '/search?q=dagger' 25 | ``` 26 | 27 | ## `post(url, params, [options])` 28 | 29 | ```rb 30 | resp = Dagger.post('http://api.server.com', { foo: 'bar' }) 31 | puts resp.code # => 200 32 | 33 | # if you want to send JSON to the server, you can pass the { json: true } option, 34 | # which converts your params object to JSON, and also sets Content-Type to 'application/json' 35 | resp = Dagger.put('http://server.com', { foo: 'bar' }, { json: true }) 36 | 37 | # now, if the endpoint returned a parseable content-type (e.g. 'application/json') 38 | # then `resp.data` will return the parsed result. `body` contains the raw bytes. 39 | puts resp.data # => { result: 'GREAT SUCCESS!' } 40 | ``` 41 | 42 | Same syntax applies for `put`, `patch` and `delete` requests. 43 | 44 | ## `request(method, url, [params], [options])` 45 | 46 | ```rb 47 | resp = Dagger.request(:put, 'https://api.server.com', { foo: 'bar' }, { follow: 10 }) 48 | puts resp.headers # { 'Content-Type' => 'application/json', ... } 49 | ``` 50 | In this case, if you want to include a query in your get request, simply pass it as 51 | the `params` argument. 52 | 53 | ## `open(url, [options]) # => &block` 54 | 55 | Oh yes. Dagger can open and hold a persistent connection so you can perform various 56 | requests without the overhead of establishing new TCP sessions. 57 | 58 | ```rb 59 | Dagger.open('https://api.server.com', { verify_ssl: 'false' }) do 60 | if post('/login', { email: 'foo@bar.com', pass: 'secret' }).success? 61 | resp = get('/something', { query: { items: 20 }, follow: 5 }) # follow 5 redirects max. 62 | File.open('something', 'wb') { |f| f.write(resp.body) } 63 | end 64 | end 65 | ``` 66 | 67 | Passing the block is optional, by the way. You can also open and call the request verb on the returned object: 68 | 69 | ```rb 70 | http = Dagger.open('https://api.server.com') 71 | resp = http.get('/foo') 72 | puts resp.code # => 200 73 | resp = http.post('/bar', { some: 'thing' }) 74 | puts resp.data.inspect # => { status: "success" } 75 | http.close # don't forget to! 76 | ``` 77 | 78 | # Options 79 | 80 | These are all the available options. 81 | 82 | ```rb 83 | opts = { 84 | json: true, # converts params object to JSON and sets Content-Type header. (POST/PUT/PATCH only) 85 | follow: true, # follow redirects (10 by default) 86 | headers: { 'Accept': 'text/xml' }, 87 | username: 'dagger', # for HTTP auth 88 | password: 'fidelio', 89 | verify_ssl: false, # true by default 90 | open_timeout: 30, 91 | read_timeout: 30 92 | } 93 | resp = Dagger.post('http://test.server.com', { payload: 1 }, opts) 94 | ``` 95 | 96 | # Credits 97 | 98 | Written by Tomás Pollak. 99 | 100 | # Copyright 101 | 102 | (c) Fork, Ltd. MIT Licensed. 103 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require 'bundler' 2 | Bundler::GemHelper.install_tasks 3 | -------------------------------------------------------------------------------- /bin/dagger: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'dagger' 3 | 4 | method = ARGV[0] or abort "Usage: dagger [method] [url] [data]" 5 | 6 | method = if ARGV[1].nil? 7 | 'get' 8 | else 9 | ARGV.shift 10 | end 11 | 12 | def parse_data(str) 13 | {} # not ready yet. TODO! 14 | end 15 | 16 | url = ARGV[0] 17 | data = parse_data(ARGV[1]) 18 | 19 | options = { 20 | :follow => true 21 | } 22 | 23 | # puts "#{method} #{url}" 24 | resp = Dagger.send(method, url, data, options) 25 | 26 | if ARGV.include?('-I') 27 | puts resp.status 28 | puts resp.headers 29 | else 30 | puts resp.body 31 | end 32 | -------------------------------------------------------------------------------- /dagger.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | require File.expand_path("../lib/dagger/version", __FILE__) 3 | 4 | Gem::Specification.new do |s| 5 | s.name = "dagger" 6 | s.version = Dagger::VERSION 7 | s.platform = Gem::Platform::RUBY 8 | s.authors = ['Tomás Pollak'] 9 | s.email = ['tomas@forkhq.com'] 10 | s.homepage = "https://github.com/tomas/dagger" 11 | s.summary = "Simplified Net::HTTP wrapper." 12 | s.description = "Dagger.post(url, params).body" 13 | 14 | s.required_rubygems_version = ">= 1.3.6" 15 | s.rubyforge_project = "dagger" 16 | 17 | s.add_development_dependency "bundler", ">= 1.0.0" 18 | s.add_development_dependency "rspec-core" 19 | s.add_development_dependency "rspec-mocks" 20 | s.add_development_dependency "rspec-expectations" 21 | 22 | s.add_runtime_dependency "net-http-persistent", ">= 3.0" 23 | s.add_runtime_dependency "oj", ">= 2.1" 24 | s.add_runtime_dependency "ox", ">= 2.4" 25 | 26 | s.files = `git ls-files`.split("\n") 27 | s.executables = `git ls-files`.split("\n").map{|f| f =~ /^bin\/(.*)/ ? $1 : nil}.compact 28 | s.require_path = 'lib' 29 | # s.bindir = 'bin' 30 | end 31 | -------------------------------------------------------------------------------- /lib/dagger.rb: -------------------------------------------------------------------------------- 1 | require 'dagger/version' 2 | require 'dagger/response' 3 | require 'dagger/parsers' 4 | require 'net/http/persistent' 5 | # require 'dagger/connection_manager' # unused 6 | require 'net/https' 7 | require 'base64' 8 | require 'erb' 9 | 10 | class URI::HTTP 11 | def scheme_and_host 12 | [scheme, host].join('://') 13 | end 14 | end 15 | 16 | module Dagger 17 | 18 | DAGGER_NAME = "Dagger/#{VERSION}".freeze 19 | REDIRECT_CODES = [301, 302, 303].freeze 20 | DEFAULT_RETRY_WAIT = 5.freeze # seconds 21 | DEFAULT_HEADERS = { 22 | 'Accept' => '*/*', 23 | 'User-Agent' => "#{DAGGER_NAME} (Ruby Net::HTTP wrapper, like curl)" 24 | }.freeze 25 | 26 | DEFAULTS = { 27 | open_timeout: 10, 28 | read_timeout: 10, 29 | keep_alive_timeout: 10 30 | }.freeze 31 | 32 | module Utils 33 | 34 | def self.parse_uri(uri) 35 | raise ArgumentError.new("Empty URI") if uri.to_s.strip == '' 36 | uri = 'http://' + uri unless uri.to_s['http'] 37 | uri = URI.parse(uri) 38 | raise ArgumentError.new("Invalid URI: #{uri}") unless uri.is_a?(URI::HTTP) 39 | uri.path = '/' if uri.path == '' 40 | uri 41 | end 42 | 43 | def self.resolve_uri(uri, host = nil, query = nil) 44 | uri = host + uri if uri.to_s[0] == '/' && host 45 | uri = parse_uri(uri.to_s) 46 | uri.path.sub!(/\?.*|$/, '?' + to_query_string(query)) if query and query.any? 47 | uri 48 | end 49 | 50 | def self.encode_body(obj, opts = {}) 51 | return if obj.nil? || obj.empty? 52 | if obj.is_a?(String) 53 | obj 54 | elsif opts[:json] 55 | Oj.dump(obj, mode: :compat) # compat ensures symbols are converted to strings 56 | else 57 | to_query_string(obj) 58 | end 59 | end 60 | 61 | def self.to_query_string(obj, key = nil) 62 | if key.nil? && obj.is_a?(String) # && obj['='] 63 | return obj 64 | end 65 | 66 | case obj 67 | when Hash then obj.map { |k, v| to_query_string(v, append_key(key, k)) }.join('&') 68 | when Array then obj.map { |v| to_query_string(v, "#{key}[]") }.join('&') 69 | when nil then '' 70 | else 71 | "#{key}=#{ERB::Util.url_encode(obj.to_s)}" 72 | end 73 | end 74 | 75 | def self.append_key(root_key, key) 76 | root_key.nil? ? key : "#{root_key}[#{key.to_s}]" 77 | end 78 | 79 | end 80 | 81 | class Client 82 | 83 | def self.init_connection(uri, opts = {}) 84 | http = if opts.delete(:persistent) 85 | pool_size = opts[:pool_size] || Net::HTTP::Persistent::DEFAULT_POOL_SIZE 86 | Net::HTTP::Persistent.new(name: DAGGER_NAME, pool_size: pool_size) 87 | else 88 | Net::HTTP.new(opts[:ip] || uri.host, uri.port) 89 | end 90 | 91 | if uri.port == 443 || uri.scheme == 'https' 92 | http.use_ssl = true if http.respond_to?(:use_ssl=) # persistent does it automatically 93 | http.verify_mode = opts[:verify_ssl] === false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER 94 | end 95 | 96 | [:open_timeout, :read_timeout, :ssl_version, :ciphers].each do |key| 97 | http.send("#{key}=", opts[key] || DEFAULTS[key]) if (opts.has_key?(key) || DEFAULTS.has_key?(key)) 98 | end 99 | 100 | http 101 | end 102 | 103 | def self.init(uri, opts) 104 | uri = Utils.parse_uri(uri) 105 | http = init_connection(uri, opts) 106 | 107 | new(http, uri.scheme_and_host) 108 | end 109 | 110 | def initialize(http, host = nil) 111 | @http, @host = http, host 112 | end 113 | 114 | def get(uri, opts = {}) 115 | uri = Utils.resolve_uri(uri, @host, opts[:query]) 116 | 117 | if @host != uri.scheme_and_host 118 | raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}") 119 | end 120 | 121 | opts[:follow] = 10 if opts[:follow] == true 122 | headers = opts[:headers] || {} 123 | headers['Accept'] = 'application/json' if opts[:json] && headers['Accept'].nil? 124 | headers['Content-Type'] = 'application/json' if opts[:json] && opts[:body] && opts[:body].size > 0 125 | 126 | if opts[:ip] 127 | headers['Host'] = uri.host 128 | uri = opts[:ip] 129 | end 130 | 131 | debug { "Sending GET request to #{uri.request_uri} with headers #{headers.inspect} -- #{opts[:body]}" } 132 | 133 | request = Net::HTTP::Get.new(uri, DEFAULT_HEADERS.merge(headers)) 134 | request.basic_auth(opts.delete(:username), opts.delete(:password)) if opts[:username] 135 | request.body = Utils.encode_body(opts[:body], opts) if opts[:body] && opts[:body].size > 0 136 | 137 | if @http.respond_to?(:started?) # regular Net::HTTP 138 | @http.start unless @http.started? 139 | resp, data = @http.request(request) 140 | else # persistent 141 | resp, data = @http.request(uri, request) 142 | end 143 | 144 | if REDIRECT_CODES.include?(resp.code.to_i) && resp['Location'] && (opts[:follow] && opts[:follow] > 0) 145 | opts[:follow] -= 1 146 | debug { "Following redirect to #{resp['Location']}" } 147 | return get(resp['Location'], opts) 148 | end 149 | 150 | @response = build_response(resp, data || resp.body) 151 | 152 | rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \ 153 | Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \ 154 | SocketError, EOFError, OpenSSL::SSL::SSLError => e 155 | 156 | if retries = opts[:retries] and retries.to_i > 0 157 | debug { "Got #{e.class}! Retrying in a sec (#{retries} retries left)" } 158 | sleep (opts[:retry_wait] || DEFAULT_RETRY_WAIT) 159 | get(uri, opts.merge(retries: retries - 1)) 160 | else 161 | raise 162 | end 163 | end 164 | 165 | def post(uri, data, options = {}) 166 | request(:post, uri, data, options) 167 | end 168 | 169 | def put(uri, data, options = {}) 170 | request(:put, uri, data, options) 171 | end 172 | 173 | def patch(uri, data, options = {}) 174 | request(:patch, uri, data, options) 175 | end 176 | 177 | def delete(uri, data, options = {}) 178 | request(:delete, uri, data, options) 179 | end 180 | 181 | def request(method, uri, data, opts = {}) 182 | if method.to_s.downcase == 'get' 183 | data ||= opts[:body] 184 | return get(uri, opts.merge(body: data)) 185 | end 186 | 187 | uri = Utils.resolve_uri(uri, @host, opts[:query]) 188 | if @host != uri.scheme_and_host 189 | raise ArgumentError.new("#{uri.scheme_and_host} does not match #{@host}") 190 | end 191 | 192 | headers = DEFAULT_HEADERS.merge(opts[:headers] || {}) 193 | body = Utils.encode_body(data, opts) 194 | 195 | if opts[:username] # opts[:password] is optional 196 | str = [opts[:username], opts[:password]].compact.join(':') 197 | headers['Authorization'] = 'Basic ' + Base64.strict_encode64(str) 198 | end 199 | 200 | if opts[:json] 201 | headers['Content-Type'] = 'application/json' 202 | headers['Accept'] = 'application/json' if headers['Accept'].nil? 203 | end 204 | 205 | start = Time.now 206 | debug { "Sending #{method} request to #{uri.request_uri} with headers #{headers.inspect} -- #{data}" } 207 | 208 | if @http.respond_to?(:started?) # regular Net::HTTP 209 | args = [method.to_s.downcase, uri.request_uri, body, headers] 210 | args.delete_at(2) if args[0] == 'delete' # Net::HTTP's delete does not accept data 211 | 212 | @http.start unless @http.started? 213 | resp, data = @http.send(*args) 214 | else # Net::HTTP::Persistent 215 | req = Kernel.const_get("Net::HTTP::#{method.capitalize}").new(uri.request_uri, headers) 216 | req.body = body 217 | resp, data = @http.request(uri, req) 218 | end 219 | 220 | debug { "Got response #{resp.code} in #{(Time.now - start).round(2)}s: #{data || resp.body}" } 221 | @response = build_response(resp, data || resp.body) 222 | 223 | rescue Errno::ECONNREFUSED, Errno::ECONNRESET, Errno::ETIMEDOUT, Errno::EINVAL, Timeout::Error, \ 224 | Net::OpenTimeout, Net::ReadTimeout, Net::HTTPBadResponse, Net::HTTPHeaderSyntaxError, Net::ProtocolError, \ 225 | SocketError, EOFError, OpenSSL::SSL::SSLError => e 226 | 227 | if method.to_s.downcase != 'get' && retries = opts[:retries] and retries.to_i > 0 228 | debug { "Got #{e.class}! Retrying in a sec (#{retries} retries left)" } 229 | sleep (opts[:retry_wait] || DEFAULT_RETRY_WAIT) 230 | request(method, uri, data, opts.merge(retries: retries - 1)) 231 | else 232 | raise 233 | end 234 | end 235 | 236 | def response 237 | @response or raise 'Request not sent!' 238 | end 239 | 240 | def open(&block) 241 | if @http.is_a?(Net::HTTP::Persistent) 242 | instance_eval(&block) 243 | else 244 | @http.start do 245 | instance_eval(&block) 246 | end 247 | end 248 | end 249 | 250 | def close 251 | if @http.is_a?(Net::HTTP::Persistent) 252 | @http.shutdown # calls finish on pool connections 253 | else 254 | @http.finish if @http.started? 255 | end 256 | end 257 | 258 | private 259 | 260 | def debug(&block) 261 | if ENV['DEBUGGING'] || ENV['DEBUG'] 262 | str = yield 263 | logger.info "[#{DAGGER_NAME}] #{str}" 264 | end 265 | end 266 | 267 | def logger 268 | require 'logger' 269 | @logger ||= Logger.new(@logfile || STDOUT) 270 | end 271 | 272 | def build_response(resp, body) 273 | resp.extend(Response) 274 | resp.set_body(body) unless resp.body 275 | resp 276 | end 277 | 278 | end 279 | 280 | class << self 281 | 282 | def open(uri, opts = {}, &block) 283 | client = Client.init(uri, opts.merge(persistent: true)) 284 | if block_given? 285 | client.open(&block) 286 | client.close 287 | end 288 | client 289 | end 290 | 291 | def get(uri, options = {}) 292 | request(:get, uri, nil, options) 293 | end 294 | 295 | def post(uri, data, options = {}) 296 | request(:post, uri, data, options) 297 | end 298 | 299 | def put(uri, data, options = {}) 300 | request(:put, uri, data, options) 301 | end 302 | 303 | def patch(uri, data, options = {}) 304 | request(:patch, uri, data, options) 305 | end 306 | 307 | def delete(uri, data, options = {}) 308 | request(:delete, uri, data, options) 309 | end 310 | 311 | def request(method, url, data = {}, options = {}) 312 | Client.init(url, options).request(method, url, data, options) 313 | end 314 | 315 | end 316 | 317 | end 318 | -------------------------------------------------------------------------------- /lib/dagger/connection_manager.rb: -------------------------------------------------------------------------------- 1 | # unused (we're using Net::HTTP::Persistent again) 2 | 3 | module Dagger 4 | 5 | class ConnectionManager 6 | 7 | def initialize(opts = {}) 8 | @opts = opts 9 | @active_connections = {} 10 | @mutex = Mutex.new 11 | end 12 | 13 | def shutdown 14 | @mutex.synchronize do 15 | # puts "Shutting down connections: #{@active_connections.count}" 16 | @active_connections.each do |_, connection| 17 | connection.finish 18 | end 19 | @active_connections = {} 20 | end 21 | end 22 | 23 | # Gets a connection for a given URI. This is for internal use only as it's 24 | # subject to change (we've moved between HTTP client schemes in the past 25 | # and may do it again). 26 | # 27 | # `uri` is expected to be a string. 28 | def connection_for(uri) 29 | @mutex.synchronize do 30 | connection = @active_connections[[uri.host, uri.port]] 31 | 32 | if connection.nil? 33 | connection = Dagger::Client.init_connection(uri, @opts) 34 | connection.start 35 | 36 | @active_connections[[uri.host, uri.port]] = connection 37 | # puts "#{@active_connections.count} connections" 38 | end 39 | 40 | connection 41 | end 42 | end 43 | 44 | # Executes an HTTP request to the given URI with the given method. Also 45 | # allows a request body, headers, and query string to be specified. 46 | def request(uri, request) 47 | connection = connection_for(uri) 48 | @mutex.synchronize do 49 | begin 50 | connection.request(request) 51 | rescue StandardError => err 52 | err 53 | end 54 | end.tap do |result| 55 | raise(result) if result.is_a?(StandardError) 56 | end 57 | end 58 | 59 | end 60 | 61 | end -------------------------------------------------------------------------------- /lib/dagger/ox_extension.rb: -------------------------------------------------------------------------------- 1 | require 'ox' 2 | 3 | XMLNode = Struct.new(:name, :text, :attributes, :children) do 4 | 5 | alias_method :to_s, :text 6 | alias_method :value, :text 7 | 8 | def to_node 9 | self 10 | end 11 | 12 | def count 13 | raise "Please call #children.count" 14 | end 15 | 16 | def keys 17 | @keys ||= children.collect(&:name) 18 | end 19 | 20 | def is_array? 21 | keys.count != keys.uniq.count 22 | end 23 | 24 | # this lets us traverse an parsed object like this: 25 | # doc[:child][:grandchild].value 26 | def [](key) 27 | found = children.select { |node| node.name.to_s == key.to_s } 28 | found.empty? ? nil : found.size == 1 ? found.first : found 29 | end 30 | 31 | # returns list of XMLNodes with matching names 32 | def slice(*arr) 33 | Array(arr).flatten.map { |key| self[key] } 34 | end 35 | 36 | def values(keys_arr = nil, include_empty: false) 37 | if keys_arr 38 | Array(keys_arr).flatten.each_with_object({}) do |key, memo| 39 | if found = self[key] and (found.to_s || include_empty) 40 | memo[key] = found.to_s 41 | end 42 | end 43 | elsif is_array? 44 | children.map(&:values) 45 | else 46 | children.each_with_object({}) do |child, memo| 47 | memo[child.name] = child.children.any? ? child.values : child.text 48 | end 49 | end 50 | end 51 | 52 | alias_method :to_hash, :values 53 | alias_method :to_h, :values 54 | 55 | def dig(*paths) 56 | list = Array(paths).flatten 57 | res = list.reduce([self]) do |parents, key| 58 | if parents 59 | found = parents.map do |parent| 60 | parent.children.select { |node| node.name.to_s == key.to_s } 61 | end.flatten 62 | 63 | found.any? ? found : nil 64 | end 65 | end 66 | 67 | res.nil? || res.empty? ? nil : res.size == 1 ? res.first : res 68 | end 69 | 70 | # returns first matching node 71 | def first(key) 72 | if found = self[key] 73 | found.is_a?(XMLNode) ? found : found.first 74 | else 75 | children.find do |ch| 76 | if res = ch.first(key) 77 | return res 78 | end 79 | end 80 | end 81 | end 82 | 83 | # returns all matching nodes 84 | def all(key) 85 | found = self[key] 86 | direct = found.is_a?(XMLNode) ? [found] : found || [] 87 | indirect = children.map { |ch| ch.all(key) }.flatten.compact 88 | direct + indirect 89 | end 90 | end 91 | 92 | class Ox::Document 93 | def to_node 94 | nodes.first.to_node 95 | end 96 | end 97 | 98 | class Ox::Element 99 | def to_node 100 | children = nodes.map { |n| n.class == self.class ? n.to_node : nil }.compact 101 | XMLNode.new(value, text, attributes, children) 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /lib/dagger/parsers.rb: -------------------------------------------------------------------------------- 1 | require 'oj' 2 | require 'ox' 3 | require 'dagger/ox_extension' 4 | 5 | class Parsers 6 | 7 | def initialize(response) 8 | if type = response.content_type 9 | @normalized_type = type.split(';').first.gsub(/[^a-z]/, '_') 10 | @body = response.body 11 | end 12 | end 13 | 14 | def process 15 | send(@normalized_type, @body) if @normalized_type && respond_to?(@normalized_type) 16 | end 17 | 18 | def application_json(body) 19 | Oj.load(body) 20 | rescue Oj::ParseError 21 | nil 22 | end 23 | 24 | alias_method :text_javascript, :application_json 25 | alias_method :application_x_javascript, :application_json 26 | 27 | def text_xml(body) 28 | if res = Ox.parse(body) 29 | res.to_node 30 | end 31 | rescue Ox::ParseError 32 | nil 33 | end 34 | 35 | alias_method :application_xml, :text_xml 36 | 37 | end -------------------------------------------------------------------------------- /lib/dagger/response.rb: -------------------------------------------------------------------------------- 1 | module Dagger 2 | 3 | module Response 4 | 5 | attr_reader :body 6 | 7 | def self.extended(base) 8 | # puts base.inspect 9 | end 10 | 11 | def set_body(string) 12 | raise "Body is set!" if body 13 | @body = string 14 | end 15 | 16 | def headers 17 | to_hash # from Net::HTTPHeader module 18 | end 19 | 20 | def code 21 | super.to_i 22 | end 23 | 24 | alias_method :status, :code 25 | 26 | def content_type 27 | self['Content-Type'] 28 | end 29 | 30 | def success? 31 | [200, 201].include?(code) 32 | end 33 | 34 | alias_method :ok?, :success? 35 | 36 | def redirect? 37 | [301, 302, 303, 307, 308].include?(code) 38 | end 39 | 40 | def to_s 41 | body.to_s 42 | end 43 | 44 | def to_a 45 | [code, headers, to_s] 46 | end 47 | 48 | def data 49 | @data ||= Parsers.new(self).process 50 | end 51 | 52 | end 53 | 54 | end -------------------------------------------------------------------------------- /lib/dagger/version.rb: -------------------------------------------------------------------------------- 1 | module Dagger 2 | MAJOR = 2 3 | MINOR = 3 4 | PATCH = 1 5 | 6 | VERSION = [MAJOR, MINOR, PATCH].join('.') 7 | end 8 | -------------------------------------------------------------------------------- /lib/dagger/wrapper.rb: -------------------------------------------------------------------------------- 1 | require_relative '../dagger' 2 | 3 | module Dagger 4 | 5 | module Wrapper 6 | 7 | def self.included(base) 8 | base.extend(ClassMethods) 9 | end 10 | 11 | module ClassMethods 12 | def base_url(str = nil) 13 | if str # set 14 | @base_url = str 15 | else 16 | @base_url or raise "base_url unset!" # get 17 | end 18 | end 19 | 20 | def base_options(opts = nil) 21 | if opts # set 22 | @base_options = opts 23 | else 24 | @base_options or raise "base_url unset!" # get 25 | end 26 | end 27 | end 28 | 29 | def initialize(opts = {}) 30 | @logger = opts.delete(:logger) 31 | @options = opts 32 | end 33 | 34 | def get(path, params = {}, opts = {}) 35 | request(:get, path, params, opts) 36 | end 37 | 38 | def post(path, params = {}, opts = {}) 39 | request(:post, path, params, opts) 40 | end 41 | 42 | def put(path, params = {}, opts = {}) 43 | request(:put, path, params, opts) 44 | end 45 | 46 | def patch(path, params = {}, opts = {}) 47 | request(:patch, path, params, opts) 48 | end 49 | 50 | def delete(path, params = {}, opts = {}) 51 | request(:delete, path, params, opts) 52 | end 53 | 54 | def request(method, path, params = {}, opts = nil) 55 | url = self.class.base_url + path 56 | resp = benchmark("#{method} #{path}") do 57 | http.request(method, url, params, base_options.merge(opts)) 58 | end 59 | 60 | handle_response(resp, method, path, params) 61 | end 62 | 63 | def connect(&block) 64 | open_http 65 | if block_given? 66 | yield 67 | close_http 68 | else 69 | at_exit { close_http } 70 | end 71 | end 72 | 73 | def disconnect 74 | close_http 75 | end 76 | 77 | private 78 | attr_reader :options 79 | 80 | def handle_response(resp, method, path, params) 81 | resp 82 | end 83 | 84 | def base_options 85 | {} 86 | end 87 | 88 | def request_options 89 | self.class.base_options.merge(base_options) 90 | end 91 | 92 | def benchmark(message, &block) 93 | log(message) 94 | start = Time.now 95 | resp = yield 96 | time = Time.now - start 97 | log("Got response in #{time.round(2)} secs") 98 | resp 99 | end 100 | 101 | def log(str) 102 | logger.info(str) 103 | end 104 | 105 | def logger 106 | @logger ||= begin 107 | require 'logger' 108 | Logger.new(@options[:logfile]) 109 | end 110 | end 111 | 112 | def http 113 | @http || Dagger 114 | end 115 | 116 | def open_http 117 | raise "Already open!" if @http 118 | @http = Dagger.open(self.class.base_url) 119 | end 120 | 121 | def close_http 122 | @http.close if @http 123 | @http = nil 124 | end 125 | 126 | # def wrap(hash) 127 | # Entity.new(hash) 128 | # end 129 | 130 | # class Entity 131 | # def initialize(props) 132 | # @props = props 133 | # end 134 | 135 | # def get(prop) 136 | # val = @props[name.to_s] 137 | # end 138 | 139 | # def method_missing(name, args, &block) 140 | # if @props.key?(name.to_s) 141 | # get(name) 142 | # else 143 | # # raise NoMethodError, "undefined method #{name}" 144 | # super 145 | # end 146 | # end 147 | # end 148 | 149 | end 150 | 151 | end -------------------------------------------------------------------------------- /spec/arguments_spec.rb: -------------------------------------------------------------------------------- 1 | require './lib/dagger' 2 | 3 | require 'rspec/mocks' 4 | require 'rspec/expectations' 5 | 6 | describe 'arguments' do 7 | 8 | describe 'URL' do 9 | 10 | def send_request(url) 11 | Dagger.get(url) 12 | end 13 | 14 | describe 'empty url' do 15 | 16 | it 'raises error' do 17 | # expect { send_request('') }.to raise_error(URI::InvalidURIError) 18 | expect { send_request('') }.to raise_error(ArgumentError) 19 | end 20 | 21 | end 22 | 23 | describe 'invalid URL' do 24 | 25 | it 'raises error' do 26 | expect { send_request('asd123.rewqw') }.to raise_error(SocketError) 27 | end 28 | 29 | end 30 | 31 | describe 'nonexisting host' do 32 | 33 | it 'raises error' do 34 | expect { send_request('http://www.foobar1234567890foobar.com/hello') }.to raise_error(SocketError) 35 | end 36 | 37 | end 38 | 39 | describe 'host without protocol' do 40 | 41 | it 'works' do 42 | expect(send_request('www.google.com')).to be_a(Net::HTTPResponse) 43 | end 44 | 45 | end 46 | 47 | 48 | describe 'valid host' do 49 | 50 | it 'works' do 51 | expect(send_request('http://www.google.com')).to be_a(Net::HTTPResponse) 52 | end 53 | 54 | end 55 | 56 | end 57 | 58 | end 59 | -------------------------------------------------------------------------------- /spec/ip_connect_spec.rb: -------------------------------------------------------------------------------- 1 | require './lib/dagger' 2 | 3 | require 'rspec/mocks' 4 | require 'rspec/expectations' 5 | 6 | describe 'IP Connection' do 7 | 8 | it 'works' do 9 | expect do 10 | Dagger.get('http://www.awiefjoawijfaowef.com') 11 | end.to raise_error(SocketError, /getaddrinfo/) 12 | 13 | resp = Dagger.get('http://www.awiefjoawijfaowef.com', { ip: '1.1.1.1'} ) 14 | expect(resp.body).to match('
cloudflare
') 15 | end 16 | 17 | end -------------------------------------------------------------------------------- /spec/parsers_spec.rb: -------------------------------------------------------------------------------- 1 | require './lib/dagger' 2 | 3 | require 'rspec/mocks' 4 | require 'rspec/expectations' 5 | 6 | describe 'Parsers' do 7 | 8 | def send_request 9 | Dagger.post('http://foobar.com/test', { foo: 'bar'}) 10 | end 11 | 12 | let(:fake_http) { double('Net::HTTP', started?: true) } 13 | let(:fake_resp) { double('Net::HTTPResponse', code: 200) } 14 | 15 | before do 16 | allow(Net::HTTP).to receive(:new).and_return(fake_http) 17 | allow(fake_http).to receive(:keep_alive_timeout=).and_return(true) 18 | allow(fake_http).to receive(:read_timeout=).and_return(true) 19 | allow(fake_http).to receive(:open_timeout=).and_return(true) 20 | allow(fake_http).to receive(:verify_mode=).and_return(true) 21 | allow(fake_http).to receive(:post).and_return(fake_resp) 22 | end 23 | 24 | describe 'json' do 25 | before do 26 | allow(fake_resp).to receive(:content_type).and_return('application/json') 27 | end 28 | 29 | describe 'non matching content-type' do 30 | before do 31 | allow(fake_resp).to receive(:content_type).and_return('text/html') 32 | allow(fake_resp).to receive(:body).and_return('foo') 33 | end 34 | 35 | it 'returns nil' do 36 | expect(send_request.data).to eql(nil) 37 | end 38 | end 39 | 40 | describe 'empty data' do 41 | before do 42 | allow(fake_resp).to receive(:body).and_return('') 43 | end 44 | 45 | it 'returns nil' do 46 | expect(send_request.data).to eql(nil) 47 | end 48 | end 49 | 50 | describe 'invalid data' do 51 | before do 52 | allow(fake_resp).to receive(:body).and_return('abcdef') 53 | end 54 | 55 | it 'returns nil' do 56 | expect(send_request.data).to eql(nil) 57 | end 58 | end 59 | 60 | describe 'valid data' do 61 | before do 62 | allow(fake_resp).to receive(:body).and_return('{"foo":123}') 63 | end 64 | 65 | it 'returns nil' do 66 | expect(send_request.data).to eql({'foo' => 123}) 67 | end 68 | end 69 | 70 | end 71 | 72 | describe 'XML' do 73 | 74 | before do 75 | allow(fake_resp).to receive(:content_type).and_return('text/xml') 76 | end 77 | 78 | describe 'empty data' do 79 | before do 80 | allow(fake_resp).to receive(:body).and_return('') 81 | end 82 | 83 | it 'returns nil' do 84 | expect(send_request.data).to eql(nil) 85 | end 86 | end 87 | 88 | describe 'invalid data' do 89 | before do 90 | allow(fake_resp).to receive(:body).and_return('abcdef') 91 | end 92 | 93 | it 'returns nil' do 94 | expect(send_request.data).to eql(nil) 95 | end 96 | end 97 | 98 | describe 'valid data' do 99 | before do 100 | allow(fake_resp).to receive(:body).and_return('123456') 101 | end 102 | 103 | it 'returns XMLNode obj' do 104 | res = send_request.data 105 | expect(res).to be_a(XMLNode) 106 | expect(res.to_node).to eql(res) 107 | expect(res['foo']).to be_a(XMLNode) 108 | expect(res['foo'].text).to eql('123') 109 | 110 | # test dig behaviour 111 | expect(res.dig('xxx', 'test', '111')).to be(nil) 112 | expect(res.dig('bar', 'test')).to be_a(XMLNode) 113 | expect(res.dig('bar', 'test').to_s).to eql('456') 114 | end 115 | end 116 | 117 | describe 'XMLNode extension' do 118 | 119 | xml = %( 120 | 121 | test 122 | 123 | 124 | foobar 125 | cat 126 | true 127 | 128 | 129 | 130 | ) 131 | 132 | describe '#to_node' do 133 | it 'works as expected' do 134 | doc = Ox.parse(xml) 135 | obj = doc.to_node 136 | expect(obj[:nested][:item][:title].text).to eql('foobar') 137 | end 138 | end 139 | 140 | describe '#values' do 141 | it 'works as expected' do 142 | doc = Ox.parse(xml) 143 | obj = doc.to_node.values 144 | expect(obj).to eql({ 145 | "foo" => "test", 146 | "nested" => {"item"=>{"category"=>"cat", "published"=>"true", "title"=>"foobar"}}, 147 | }) 148 | end 149 | end 150 | 151 | end 152 | 153 | end 154 | end 155 | -------------------------------------------------------------------------------- /spec/persistent_spec.rb: -------------------------------------------------------------------------------- 1 | require './lib/dagger' 2 | 3 | require 'rspec/mocks' 4 | require 'rspec/expectations' 5 | 6 | describe 'Persistent mode' do 7 | 8 | it 'works' do 9 | # fake_client = double('Client') 10 | # expect(Dagger::Client).to receive(:new).once.and_return(fake_client) 11 | # expect(fake_client).to receive(:open).once #.and_return(fake_resp) 12 | # expect(fake_client).to receive(:close).once #.and_return(fake_resp) 13 | 14 | res1, res2 = nil, nil 15 | obj = Dagger.open('https://www.google.com') do 16 | res1 = get('/search?q=dagger+http+client', { body: 'foo' }) 17 | res2 = get('https://www.google.com/search?q=thank+you+ruby') 18 | res3 = post('https://www.google.com/search?q=foobar', { foo: 'bar' }) 19 | end 20 | 21 | expect(res1.code).to eq(400) 22 | expect(res2.code).to eq(200) 23 | expect(res2.code).to eq(200) 24 | expect(obj).to be_a(Dagger::Client) 25 | end 26 | 27 | end 28 | 29 | describe 'using threads' do 30 | 31 | def connect(host) 32 | raise if @http 33 | @http = Dagger.open(host) 34 | end 35 | 36 | def disconnect 37 | raise if @http.nil? 38 | @http.close 39 | @http = nil 40 | end 41 | 42 | it 'works' do 43 | thread_count = 10 44 | urls_count = 100 45 | host = 'https://postman-echo.com' 46 | urls = urls_count.times.map { |i| "/get?page/#{i}" } 47 | result = [] 48 | 49 | mutex = Mutex.new 50 | thread_count.times.map do 51 | Thread.new(urls, result) do |urls, result| 52 | # mutex.synchronize { Dagger.open(host) } 53 | http = Dagger.open(host) 54 | while url = mutex.synchronize { urls.pop } 55 | # puts "Fetching #{url}" 56 | resp = http.get(url) 57 | mutex.synchronize do 58 | result.push(resp.code) 59 | end 60 | end 61 | # mutex.synchronize { http.close } 62 | http.close 63 | end 64 | end.each(&:join) 65 | 66 | expect(result.count).to eq(urls_count) 67 | end 68 | 69 | end -------------------------------------------------------------------------------- /spec/retries_spec.rb: -------------------------------------------------------------------------------- 1 | require './lib/dagger' 2 | 3 | require 'rspec/mocks' 4 | require 'rspec/expectations' 5 | 6 | describe 'Retries' do 7 | 8 | def send_request 9 | Dagger.get('http://foobar.com/test', opts) 10 | end 11 | 12 | let(:fake_http) { double('Net::HTTP', started?: true, "keep_alive_timeout=": true, "open_timeout=": true, "read_timeout=": true) } 13 | let(:fake_resp) { double('Net::HTTPResponse', code: 200, body: 'foo') } 14 | 15 | before do 16 | allow(Net::HTTP).to receive(:new).at_least(:once).and_return(fake_http) 17 | allow(fake_http).to receive(:verify_mode=).and_return(true) 18 | end 19 | 20 | describe 'on ECONNREFUSED' do 21 | 22 | context 'if no retries option passed' do 23 | 24 | let(:opts) { {} } 25 | 26 | it 'does not retry request, and raises error' do 27 | expect(fake_http).to receive(:request).once.and_raise(Errno::ECONNREFUSED) 28 | expect { send_request }.to raise_error(Errno::ECONNREFUSED) 29 | end 30 | 31 | end 32 | 33 | context 'if retries is 1' do 34 | 35 | let(:opts) { { retries: 1, retry_wait: 1 } } 36 | 37 | context 'and it still fails' do 38 | 39 | it 'sends a second request, and raises error' do 40 | expect(fake_http).to receive(:request).twice.and_raise(Errno::ECONNREFUSED) 41 | expect { send_request }.to raise_error(Errno::ECONNREFUSED) 42 | end 43 | 44 | end 45 | 46 | context 'and then it works' do 47 | 48 | it 'sends a second request, and does not raise error' do 49 | expect(fake_http).to receive(:request).once.and_raise(Errno::ECONNREFUSED) 50 | expect(fake_http).to receive(:request).once.and_return(fake_resp) 51 | allow(fake_resp).to receive(:[]).with('Content-Type').and_return('text/plain') 52 | 53 | expect(send_request.body).to eq('foo') 54 | end 55 | 56 | end 57 | 58 | end 59 | 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /spec/sending_data_spec.rb: -------------------------------------------------------------------------------- 1 | require './lib/dagger' 2 | 3 | require 'rspec/mocks' 4 | require 'rspec/expectations' 5 | 6 | describe 'sending data' do 7 | 8 | it 'works with get if using .request' do 9 | resp = Dagger.request('get', 'https://httpbingo.org/get?x=123', { foo: 'bar', testing: 1 }, { json: true }) 10 | expect(resp.ok?).to eq(true) 11 | end 12 | 13 | it 'works with get if passing body as option' do 14 | resp = Dagger.get('https://httpbingo.org/get?x=123', { body: { foo: 'bar', testing: 1 }, json: true }) 15 | expect(resp.ok?).to eq(true) 16 | end 17 | 18 | 19 | end --------------------------------------------------------------------------------