├── .rspec ├── Gemfile ├── lib ├── http_logger │ ├── version.rb │ └── configuration.rb └── http_logger.rb ├── .document ├── spec ├── image.webp ├── spec_helper.rb └── http_logger_spec.rb ├── screenshots ├── solr.png ├── hoptoad.png └── rails_console.png ├── .gitignore ├── Rakefile ├── LICENSE.txt ├── CHANGELOG.md ├── http_logger.gemspec └── Readme.md /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "http://rubygems.org" 2 | 3 | gemspec 4 | 5 | -------------------------------------------------------------------------------- /lib/http_logger/version.rb: -------------------------------------------------------------------------------- 1 | class HttpLogger 2 | VERSION = "1.0.1" 3 | end 4 | -------------------------------------------------------------------------------- /.document: -------------------------------------------------------------------------------- 1 | lib/**/*.rb 2 | bin/* 3 | - 4 | features/**/*.feature 5 | LICENSE.txt 6 | -------------------------------------------------------------------------------- /spec/image.webp: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/railsware/http_logger/HEAD/spec/image.webp -------------------------------------------------------------------------------- /screenshots/solr.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/railsware/http_logger/HEAD/screenshots/solr.png -------------------------------------------------------------------------------- /screenshots/hoptoad.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/railsware/http_logger/HEAD/screenshots/hoptoad.png -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | coverage 2 | rdoc 3 | doc 4 | .yardoc 5 | .bundle 6 | pkg 7 | http.log 8 | Gemfile.lock 9 | 10 | -------------------------------------------------------------------------------- /screenshots/rails_console.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/railsware/http_logger/HEAD/screenshots/rails_console.png -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | $LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib')) 2 | $LOAD_PATH.unshift(File.dirname(__FILE__)) 3 | require 'rspec' 4 | require 'webmock/rspec' 5 | require 'http_logger' 6 | require "logger" 7 | require "fileutils" 8 | 9 | # Requires supporting files with custom matchers and macros, etc, 10 | # in ./support/ and its subdirectories. 11 | Dir["#{File.dirname(__FILE__)}/support/**/*.rb"].each {|f| require f} 12 | 13 | LOGFILE = 'http.log' 14 | 15 | RSpec.configure do |config| 16 | config.expect_with(:rspec) { |c| c.syntax = :should } 17 | FileUtils.rm_f(LOGFILE) 18 | HttpLogger.configuration.logger = Logger.new(LOGFILE) 19 | end 20 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | # encoding: utf-8 2 | 3 | require 'rubygems' 4 | require 'bundler' 5 | begin 6 | Bundler.setup(:default, :development) 7 | rescue Bundler::BundlerError => e 8 | $stderr.puts e.message 9 | $stderr.puts "Run `bundle install` to install missing gems" 10 | exit e.status_code 11 | end 12 | 13 | require 'rake' 14 | require 'bundler/gem_tasks' 15 | 16 | require 'rspec/core' 17 | require 'rspec/core/rake_task' 18 | 19 | RSpec::Core::RakeTask.new(:spec) do |spec| 20 | spec.pattern = FileList['spec/**/*_spec.rb'] 21 | end 22 | 23 | RSpec::Core::RakeTask.new(:rcov) do |spec| 24 | spec.pattern = 'spec/**/*_spec.rb' 25 | spec.rcov = true 26 | end 27 | 28 | task :default => :spec 29 | 30 | -------------------------------------------------------------------------------- /lib/http_logger/configuration.rb: -------------------------------------------------------------------------------- 1 | class HttpLogger 2 | class Configuration 3 | attr_accessor :collapse_body_limit 4 | attr_accessor :log_headers 5 | attr_accessor :log_request_body 6 | attr_accessor :log_response_body 7 | attr_accessor :logger 8 | attr_accessor :colorize 9 | attr_accessor :ignore 10 | attr_accessor :level 11 | 12 | def initialize 13 | reset 14 | end 15 | 16 | def reset 17 | self.log_headers = false 18 | self.log_request_body = true 19 | self.log_response_body = true 20 | self.colorize = true 21 | self.collapse_body_limit = 5000 22 | self.ignore = [] 23 | self.level = :debug 24 | end 25 | end 26 | end 27 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2011 Bogdan Gusiev 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## [1.0.1] 4 | 5 | * Modernize gemspec 6 | 7 | ## [1.0.0] 8 | 9 | ### Added 10 | - Introduced a new `Configuration` class for `HttpLogger` to manage settings. 11 | - Added support for binary response detection and logging. 12 | - Added a new spec for binary response handling. 13 | 14 | ### Changed 15 | - Updated `Gemfile` to use `gemspec` instead of listing development dependencies directly. 16 | - Refactored `HttpLogger` to use a configuration object for managing settings. 17 | - Updated `Readme.md` with new configuration example using `HttpLogger.configure`. 18 | 19 | ### Removed 20 | - Removed `Gemfile.lock` from version control. 21 | 22 | ## Configuration Example 23 | 24 | ```ruby 25 | HttpLogger.configure do |c| 26 | c.logger = Logger.new('/path/to/logfile.log') 27 | c.colorize = true 28 | c.ignore = [/example\.com/] 29 | c.log_headers = true 30 | c.log_request_body = true 31 | c.log_response_body = true 32 | c.level = :info 33 | c.collapse_body_limit = 5000 34 | end 35 | ``` 36 | 37 | ## Output Example for Binary Body 38 | 39 | When a binary response is detected, the log will include a message indicating the binary content and its size: 40 | 41 | ``` 42 | Response body: 43 | ``` 44 | 45 | -------------------------------------------------------------------------------- /http_logger.gemspec: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require_relative "lib/http_logger/version" 4 | 5 | Gem::Specification.new do |spec| 6 | spec.name = "http_logger" 7 | spec.version = HttpLogger::VERSION 8 | spec.authors = ["Bogdan Gusiev"] 9 | spec.email = ["agresso@gmail.com"] 10 | 11 | spec.summary = "Log your http api calls just like SQL queries" 12 | spec.description = "This gem keeps an eye on every Net::HTTP library usage and dumps all request and response data to the log file." 13 | spec.homepage = "https://github.com/railsware/http_logger" 14 | spec.license = "MIT" 15 | spec.required_ruby_version = ">= 2.5" 16 | 17 | spec.metadata["source_code_uri"] = spec.homepage 18 | spec.metadata["changelog_uri"] = "#{spec.homepage}/CHANGELOG.md" 19 | 20 | spec.files = Dir.glob("{lib,spec}/**/*") + ["LICENSE.txt", "README.md"] 21 | spec.extra_rdoc_files = ["LICENSE.txt"] 22 | spec.require_paths = ["lib"] 23 | 24 | spec.add_development_dependency "bundler", ">= 2.0" 25 | spec.add_development_dependency "rake", ">= 13.0" 26 | spec.add_development_dependency "rspec", ">= 3.0" 27 | spec.add_development_dependency "webmock", ">= 3.0" 28 | spec.add_development_dependency "debug", ">= 1.0" 29 | spec.add_development_dependency "bump", ">= 0.10" 30 | end 31 | 32 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # Net::HTTP logger 2 | 3 | Simple gem that logs your HTTP api requests just like database queries 4 | 5 | 6 | ## Screenshot 7 | 8 | * [Hoptoad](https://github.com/railsware/http_logger/raw/master/screenshots/hoptoad.png) 9 | * [Simple get](https://github.com/railsware/http_logger/raw/master/screenshots/rails_console.png) 10 | * [Solr](https://github.com/railsware/http_logger/raw/master/screenshots/solr.png) 11 | 12 | ## Installation 13 | 14 | ``` sh 15 | gem install http_logger 16 | ``` 17 | 18 | ## Usage 19 | 20 | ``` ruby 21 | require 'http_logger' 22 | 23 | HttpLogger.configure do |c| 24 | # defaults to Rails.logger if Rails is defined 25 | c.logger = Logger.new(LOGFILE) 26 | 27 | # Default: true 28 | c.colorize = true 29 | 30 | # Ignore patterns (e.g., URLs to ignore) 31 | c.ignore = [/newrelic\.com/] 32 | 33 | # Default: false 34 | c.log_headers = false 35 | 36 | # Default: true 37 | c.log_request_body = false 38 | 39 | # Default: true 40 | c.log_response_body = false 41 | 42 | # Desired log level as a symbol. Default: :debug 43 | c.level = :info 44 | 45 | # Change default truncate limit. Default: 5000 46 | c.collapse_body_limit = 5000 47 | end 48 | ``` 49 | 50 | ## Alternative 51 | 52 | Net::HTTP has a builtin logger that can be set via \#set\_debug\_output. 53 | This method is only available at the instance level and it is not always accessible if used inside of a library. Also output of builtin debugger is not formed well for API debug purposes. 54 | 55 | ## Integration 56 | 57 | If you are using Net::HTTP#request hackers like FakeWeb make sure you require http\_logger after all others because http\_logger always calls "super", rather than others. 58 | -------------------------------------------------------------------------------- /spec/http_logger_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require "uri" 3 | require "base64" 4 | 5 | describe HttpLogger do 6 | 7 | before do 8 | # flush log 9 | f = File.open(LOGFILE, "w") 10 | f.close 11 | 12 | stub_request(:any, url).to_return( 13 | body: response_body, 14 | headers: {"X-Http-logger" => true, **response_headers}, 15 | ) 16 | end 17 | 18 | let(:response_body) { "Success" } 19 | let(:response_headers) { {} } 20 | let(:request_headers) { {} } 21 | 22 | let(:url) { "http://google.com/" } 23 | let(:uri) { URI.parse(url) } 24 | let(:request) do 25 | Net::HTTP.get_response(uri, **request_headers) 26 | end 27 | 28 | let(:long_body) do 29 | "12,Dodo case,dodo@case.com,tech@dodcase.com,single elimination\n" * 50 + 30 | "12,Bonobos,bono@bos.com,tech@bonobos.com,double elimination\n" * 50 31 | end 32 | 33 | subject do 34 | _context if defined?(_context) 35 | request 36 | File.read(LOGFILE) 37 | end 38 | 39 | it { should_not be_empty } 40 | 41 | context "when url has escaped chars" do 42 | 43 | let(:url) { "http://google.com?query=a%20b"} 44 | 45 | it { subject.should include("query=a b")} 46 | 47 | end 48 | 49 | context "when headers logging is on" do 50 | 51 | before(:each) do 52 | HttpLogger.configuration.log_headers = true 53 | end 54 | 55 | it { should include("HTTP response header") } 56 | it { should include("HTTP request header") } 57 | 58 | 59 | context "authorization header" do 60 | 61 | let(:request_headers) do 62 | {'Authorization' => "Basic #{Base64.encode64('hello:world')}".strip} 63 | end 64 | it { should include("Authorization: ") } 65 | end 66 | end 67 | 68 | describe "post request" do 69 | let(:body) {{:a => 'hello', :b => 1}} 70 | let(:request) do 71 | Net::HTTP.post_form(uri, body) 72 | end 73 | 74 | it {should include("Request body")} 75 | it {should include("a=hello&b=1")} 76 | context "with too long body" do 77 | let(:response_body) { long_body } 78 | let(:url) do 79 | "http://github.com/" 80 | end 81 | it { should include("12,Dodo case,dodo@case.com,tech@dodcase.com,single elimination\n")} 82 | it { should include("") } 83 | it { should include("12,Bonobos,bono@bos.com,tech@bonobos.com,double elimination\n")} 84 | end 85 | 86 | end 87 | 88 | describe "put request" do 89 | let(:request) do 90 | http = Net::HTTP.new(uri.host, uri.port) 91 | request = Net::HTTP::Put.new(uri.path) 92 | request.set_form_data(:a => 'hello', :b => 1) 93 | http.request(request) 94 | end 95 | 96 | it {should include("Request body")} 97 | it {should include("a=hello&b=1")} 98 | end 99 | 100 | describe "generic request" do 101 | let(:request) do 102 | http = Net::HTTP.new(uri.host, uri.port) 103 | request = Net::HTTPGenericRequest.new('PUT', true, true, uri.path) 104 | request.body = "a=hello&b=1" 105 | http.request(request) 106 | end 107 | 108 | it {should include("Request body")} 109 | it {should include("a=hello&b=1")} 110 | end 111 | 112 | context "when request body logging is off" do 113 | 114 | before(:each) do 115 | HttpLogger.configuration.log_request_body = false 116 | end 117 | 118 | let(:request) do 119 | Net::HTTP.post_form(uri, {}) 120 | end 121 | 122 | it { should_not include("Request body") } 123 | 124 | end 125 | 126 | context "with long response body" do 127 | 128 | let(:response_body) { long_body } 129 | let(:url) do 130 | stub_request(:get, "http://github.com/").to_return(body: long_body) 131 | "http://github.com" 132 | end 133 | 134 | it { should include("12,Dodo case,dodo@case.com,tech@dodcase.com,single elimination\n")} 135 | it { should include("") } 136 | it { should include("12,Bonobos,bono@bos.com,tech@bonobos.com,double elimination\n")} 137 | 138 | end 139 | 140 | context "when response body logging is off" do 141 | 142 | before(:each) do 143 | HttpLogger.configuration.log_response_body = false 144 | end 145 | 146 | let(:response_body) { long_body } 147 | let(:url) do 148 | "http://github.com" 149 | end 150 | 151 | it { should_not include("Response body") } 152 | end 153 | 154 | context "ignore option is set" do 155 | 156 | let(:url) do 157 | "http://rpm.newrelic.com/hello/world" 158 | end 159 | 160 | before(:each) do 161 | HttpLogger.configuration.ignore = [/rpm\.newrelic\.com/] 162 | end 163 | 164 | it { should be_empty} 165 | end 166 | 167 | context "when level is set" do 168 | 169 | let(:url) do 170 | stub_request(:get, "http://rpm.newrelic.com/hello/world").to_return(body: "") 171 | "http://rpm.newrelic.com/hello/world" 172 | end 173 | 174 | before(:each) do 175 | HttpLogger.configuration.level = :info 176 | end 177 | 178 | it { should_not be_empty } 179 | end 180 | 181 | context "when binary response" do 182 | let(:response_headers) do 183 | { 184 | 'Content-Type' => 'image/webp' 185 | } 186 | end 187 | let(:url) do 188 | "http://example.com/image.webp" 189 | end 190 | 191 | let(:response_body) do 192 | File.read("#{File.dirname(__FILE__)}/image.webp") 193 | end 194 | 195 | it { should include("") } 196 | end 197 | 198 | after(:each) do 199 | HttpLogger.configuration.reset 200 | end 201 | end 202 | -------------------------------------------------------------------------------- /lib/http_logger.rb: -------------------------------------------------------------------------------- 1 | require 'net/http' 2 | require 'uri' 3 | require 'set' 4 | require 'http_logger/configuration' 5 | require 'http_logger/version' 6 | 7 | # Usage: 8 | # 9 | # require 'http_logger' 10 | # 11 | # == Setup logger 12 | # 13 | # HttpLogger.configuration.logger = Logger.new('/tmp/all.log') 14 | # HttpLogger.configuration.log_headers = true 15 | # 16 | # == Do request 17 | # 18 | # res = Net::HTTP.start(url.host, url.port) { |http| 19 | # http.request(req) 20 | # } 21 | # ... 22 | # 23 | # == View the log 24 | # 25 | # cat /tmp/all.log 26 | class HttpLogger 27 | AUTHORIZATION_HEADER = 'Authorization' 28 | 29 | def self.configuration 30 | @configuration ||= Configuration.new 31 | end 32 | 33 | def self.configure(&block) 34 | block.call(configuration) 35 | end 36 | 37 | 38 | def self.perform(*args, &block) 39 | instance.perform(*args, &block) 40 | end 41 | 42 | def self.instance 43 | @instance ||= HttpLogger.new 44 | end 45 | 46 | def perform(http, request, request_body) 47 | start_time = Time.now 48 | response = yield 49 | ensure 50 | if require_logging?(http, request) 51 | log_request_url(http, request, start_time) 52 | log_request_body(request) 53 | log_request_headers(request) 54 | if defined?(response) && response 55 | log_response_code(response) 56 | log_response_headers(response) 57 | log_response_body(response.body, binary_response?(response)) 58 | end 59 | end 60 | end 61 | 62 | protected 63 | 64 | def binary_response?(response) 65 | content_type = response['Content-Type'] 66 | return false if content_type.nil? 67 | 68 | !content_type.start_with?('text/', 'application/json', 'application/xml', 'application/javascript', 'application/x-www-form-urlencoded', 'application/xhtml+xml', 'application/rss+xml', 'application/atom+xml', 'application/svg+xml', 'application/yaml') 69 | 70 | end 71 | 72 | def log_request_url(http, request, start_time) 73 | ofset = Time.now - start_time 74 | log("HTTP #{request.method} (%0.2fms)" % (ofset * 1000), request_url(http, request)) 75 | end 76 | 77 | def request_url(http, request) 78 | URI::DEFAULT_PARSER.unescape("http#{"s" if http.use_ssl?}://#{http.address}:#{http.port}#{request.path}") 79 | end 80 | 81 | def log_request_headers(request) 82 | if configuration.log_headers 83 | request.each_capitalized do |k,v| 84 | log_header(:request, k, v) 85 | end 86 | end 87 | end 88 | 89 | def log_header(type, name, value) 90 | value = "" if name == AUTHORIZATION_HEADER 91 | log("HTTP #{type} header", "#{name}: #{value}") 92 | end 93 | 94 | HTTP_METHODS_WITH_BODY = Set.new(%w(POST PUT GET PATCH)) 95 | 96 | def log_request_body(request) 97 | if configuration.log_request_body 98 | if HTTP_METHODS_WITH_BODY.include?(request.method) 99 | if (body = request.body) && !body.empty? 100 | log("Request body", truncate_body(body)) 101 | end 102 | end 103 | end 104 | end 105 | 106 | def log_response_code(response) 107 | log("Response status", "#{response.class} (#{response.code})") 108 | end 109 | 110 | def log_response_headers(response) 111 | if configuration.log_headers 112 | response.each_capitalized do |k,v| 113 | log_header(:response, k, v) 114 | end 115 | end 116 | end 117 | 118 | def log_response_body(body, binary) 119 | if configuration.log_response_body 120 | if body.is_a?(Net::ReadAdapter) 121 | log("Response body", "") 122 | else 123 | if body && !body.empty? 124 | log( 125 | "Response body", 126 | binary ? "" : truncate_body(body),) 127 | end 128 | end 129 | end 130 | end 131 | 132 | def require_logging?(http, request) 133 | self.logger && !ignored?(http, request) && (http.started? || webmock?(http, request)) 134 | end 135 | 136 | def ignored?(http, request) 137 | url = request_url(http, request) 138 | configuration.ignore.any? do |pattern| 139 | url =~ pattern 140 | end 141 | end 142 | 143 | def webmock?(http, request) 144 | return false unless defined?(::WebMock) 145 | uri = request_uri_as_string(http, request) 146 | method = request.method.downcase.to_sym 147 | signature = WebMock::RequestSignature.new(method, uri) 148 | ::WebMock.registered_request?(signature) 149 | end 150 | 151 | def request_uri_as_string(net_http, request) 152 | protocol = net_http.use_ssl? ? "https" : "http" 153 | 154 | path = request.path 155 | path = URI.parse(request.path).request_uri if request.path =~ /^http/ 156 | 157 | if request["authorization"] =~ /^Basic / 158 | userinfo = WebMock::Utility.decode_userinfo_from_header(request["authorization"]) 159 | userinfo = WebMock::Utility.encode_unsafe_chars_in_userinfo(userinfo) + "@" 160 | else 161 | userinfo = "" 162 | end 163 | 164 | "#{protocol}://#{userinfo}#{net_http.address}:#{net_http.port}#{path}" 165 | end 166 | 167 | def truncate_body(body) 168 | if collapse_body_limit && collapse_body_limit > 0 && body && body.size >= collapse_body_limit 169 | body_piece_size = collapse_body_limit / 2 170 | body[0..body_piece_size] + 171 | "\n\n\n\n" + 172 | body[(body.size - body_piece_size)..body.size] 173 | else 174 | body 175 | end 176 | end 177 | 178 | def log(message, dump) 179 | self.logger.send(configuration.level, format_log_entry(message, dump)) 180 | end 181 | 182 | def format_log_entry(message, dump = nil) 183 | if configuration.colorize 184 | message_color, dump_color = "4;32;1", "0;1" 185 | log_entry = " \e[#{message_color}m#{message}\e[0m " 186 | log_entry << "\e[#{dump_color}m%#{String === dump ? 's' : 'p'}\e[0m" % dump if dump 187 | log_entry 188 | else 189 | "%s %s" % [message, dump] 190 | end 191 | end 192 | 193 | def logger 194 | configuration.logger 195 | end 196 | 197 | def collapse_body_limit 198 | configuration.collapse_body_limit 199 | end 200 | 201 | def configuration 202 | self.class.configuration 203 | end 204 | end 205 | 206 | block = lambda do |a| 207 | alias request_without_net_http_logger request 208 | def request(request, body = nil, &block) 209 | HttpLogger.perform(self, request, body) do 210 | request_without_net_http_logger(request, body, &block) 211 | end 212 | 213 | end 214 | end 215 | 216 | if defined?(::WebMock) 217 | klass = WebMock::HttpLibAdapters::NetHttpAdapter.instance_variable_get("@webMockNetHTTP") 218 | klass.class_eval(&block) 219 | end 220 | 221 | 222 | Net::HTTP.class_eval(&block) 223 | 224 | if defined?(Rails) 225 | if defined?(ActiveSupport) && ActiveSupport.respond_to?(:on_load) 226 | # Rails3 227 | ActiveSupport.on_load(:after_initialize) do 228 | unless HttpLogger.configuration.logger 229 | HttpLogger.configuration.logger = Rails.logger 230 | end 231 | end 232 | end 233 | end 234 | --------------------------------------------------------------------------------