├── .gitignore ├── .rspec ├── CHANGELOG.md ├── Gemfile ├── LICENSE.txt ├── README.md ├── Rakefile ├── lib ├── log_spy.rb └── log_spy │ ├── payload.rb │ ├── spy.rb │ └── version.rb ├── log_spy.gemspec └── spec ├── payload_spec.rb ├── spec_helper.rb └── spy_spec.rb /.gitignore: -------------------------------------------------------------------------------- 1 | *.gem 2 | *.rbc 3 | .bundle 4 | .config 5 | .yardoc 6 | Gemfile.lock 7 | InstalledFiles 8 | _yardoc 9 | coverage 10 | doc/ 11 | lib/bundler/man 12 | pkg 13 | rdoc 14 | spec/reports 15 | test/tmp 16 | test/version_tmp 17 | tmp 18 | *.bundle 19 | *.so 20 | *.o 21 | *.a 22 | mkmf.log 23 | *.swp 24 | -------------------------------------------------------------------------------- /.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | -I ./spec 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | 1.0.0 / 2016-02-09 3 | ================== 4 | 5 | [Changed] 6 | * upgrade `aws-sdk` to 2.x.x and restrict its dependency version 7 | 8 | [Added] 9 | * add `cookies` to payload 10 | 11 | [Fixed] 12 | * `rewind` body before read in payload 13 | 14 | 15 | 0.0.4 / 2014-09-23 16 | ================== 17 | * add `controller_action` to payload, if `env['action_dispatch.request.parameter']` exists 18 | 19 | 0.0.3 / 2014-09-04 20 | ================== 21 | 22 | * accomondate the case when `Rack::Request#body` not readable, fallback to env['RAW_POST_BODY'] if any 23 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Specify your gem's dependencies in log_spy.gemspec 4 | gemspec 5 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014 Yang-Hsing Lin 2 | 3 | MIT License 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | "Software"), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 19 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 20 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 21 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 22 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # LogSpy 2 | 3 | LogSpy is a Rack middleware sending request log to [Amazon SQS](http://aws.amazon.com/sqs/) on each request. 4 | 5 | ## How it works 6 | 7 | After each request, `log_spy` opens a new thread and sends the request log payload as a `json` string onto [AWS SQS](http://aws.amazon.com/sqs/). 8 | 9 | ## Why not use Papertrail or other log collector? 10 | 11 | Logspy does not intend to replace the log collectors like Papertrail or something similar. 12 | The purpose of Logspy is to record each request and its params so that we can easily analyse even replay requests within a certain period. 13 | 14 | ## Installation 15 | 16 | Add this line to your application's Gemfile: 17 | 18 | gem 'log_spy' 19 | 20 | And then execute: 21 | 22 | $ bundle 23 | 24 | Or install it yourself as: 25 | 26 | $ gem install log_spy 27 | 28 | ## Usage 29 | 30 | require and use the middleware: 31 | 32 | - Bare Rack: 33 | 34 | ```ruby 35 | require 'log_spy' 36 | use LogSpy::Spy, 'aws-sqs-url' 37 | ``` 38 | 39 | - Rails: 40 | ```ruby 41 | # config/application.rb 42 | config.middleware.use LogSpy::Spy, 'aws-sqs-url', :region => 'ap-southeast-1', 43 | :access_key_id => 'the-key-id', 44 | :secret_access_key => 'the-secret' 45 | ``` 46 | 47 | ## API Documents: 48 | 49 | ### to `use` the middleware: 50 | - usage: `use LogSpy::Spy, [, ]` 51 | - params: 52 | - `aws-sqs-url`(required): the [Queue URL](http://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/ImportantIdentifiers.html) of SQS, which identifies the queue. 53 | - `options`(optional): if given, `log_spy` would pass it to initialize [`Aws::SQS`](http://docs.aws.amazon.com/sdkforruby/api/Aws/SQS.html) 54 | 55 | ### the payload format sends to AWS SQS: 56 | 57 | ```javascript 58 | { 59 | "path": "/the/request/path", 60 | "status": 200, 61 | "execution_time": 145.3, // in ms 62 | "controller_action": "users#show", // if env['action_dispatch.request.parameters'] exists 63 | "request": { 64 | "content_type": "application/json", 65 | "request_method": "post", 66 | "ip": "123.1.1.1", 67 | "query_string": "query-key=query-val&hello=world", 68 | "body": "body-key=body-val", 69 | "cookies": { 70 | "cookie_key": "cookie_val" 71 | } 72 | }, 73 | 74 | // if got exception 75 | "error": { 76 | "message": "the exception message", 77 | "backtrace": [ "exception back trace" ] 78 | } 79 | } 80 | ``` 81 | 82 | - `error`: `error` would not be included in the payload if no exception was raised 83 | - `request.body`: if the request `Content-Type` is of `multipart`, the body would be an empty string 84 | 85 | ## Testing: 86 | `$ bundle install` 87 | `$ bundle exec rspec spec/` 88 | 89 | 90 | ## Contributing 91 | 92 | 1. Fork it ( https://github.com/[my-github-username]/log_spy/fork ) 93 | 2. Create your feature branch (`git checkout -b my-new-feature`) 94 | 3. Commit your changes (`git commit -am 'Add some feature'`) 95 | 4. Push to the branch (`git push origin my-new-feature`) 96 | 5. Create a new Pull Request 97 | -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | 3 | -------------------------------------------------------------------------------- /lib/log_spy.rb: -------------------------------------------------------------------------------- 1 | require "log_spy/version" 2 | require "log_spy/spy" 3 | 4 | module LogSpy 5 | # Your code goes here... 6 | end 7 | -------------------------------------------------------------------------------- /lib/log_spy/payload.rb: -------------------------------------------------------------------------------- 1 | require 'json' 2 | class LogSpy::Payload 3 | def initialize req, res, begin_at, error = nil 4 | @req = req 5 | @res = res 6 | @error = error 7 | @begin_at = begin_at 8 | end 9 | 10 | def to_json 11 | hash = { 12 | :path => @req.path, 13 | :status => @res.status, 14 | :execution_time => @res.duration, 15 | :begin_at => @begin_at, 16 | :request => { 17 | :content_type => @req.content_type, 18 | :request_method => @req.request_method, 19 | :ip => @req.ip, 20 | :query_string => @req.query_string, 21 | :cookies => @req.cookies, 22 | :body => request_body 23 | } 24 | } 25 | 26 | append_error_if_exists(hash) 27 | append_controller_action_if_exists(hash) 28 | 29 | hash.to_json 30 | end 31 | 32 | def append_error_if_exists hash 33 | if @error 34 | hash[:error] = { :message => @error.message, :backtrace => @error.backtrace } 35 | end 36 | end 37 | private :append_error_if_exists 38 | 39 | def append_controller_action_if_exists hash 40 | if controller_params = @req.env['action_dispatch.request.parameters'] 41 | hash[:controller_action] = "#{controller_params['controller']}##{controller_params['action']}" 42 | end 43 | end 44 | private :append_controller_action_if_exists 45 | 46 | def request_body 47 | return '' if @req.content_type =~ /multipart/ 48 | @req.body.rewind 49 | @req.body.read 50 | rescue IOError 51 | @req.env['RAW_POST_BODY'] 52 | end 53 | private :request_body 54 | end 55 | -------------------------------------------------------------------------------- /lib/log_spy/spy.rb: -------------------------------------------------------------------------------- 1 | require 'aws-sdk' 2 | require 'rack' 3 | require 'log_spy/payload' 4 | require 'json' 5 | require 'ostruct' 6 | 7 | class LogSpy::Spy 8 | attr_reader :sqs_thread 9 | def initialize(app, sqs_url, options = {}) 10 | @app = app 11 | @sqs_url = sqs_url 12 | @options = options 13 | end 14 | 15 | def call env 16 | @env = env 17 | @start_time = Time.now.to_f 18 | 19 | @status, header, body = @app.call(env) 20 | @sqs_thread = send_sqs_async 21 | 22 | [ @status, header, body ] 23 | rescue Exception => err 24 | @sqs_thread = send_sqs_async(err) 25 | raise err 26 | end 27 | 28 | def req 29 | Rack::Request.new @env 30 | end 31 | private :req 32 | 33 | def send_sqs_async(err = nil) 34 | @sqs_thread = Thread.new do 35 | status = err ? 500 : @status 36 | sqs_client = Aws::SQS::Client.new(@options) 37 | duration = ( (Time.now.to_f - @start_time) * 1000 ).round(0) 38 | res = OpenStruct.new({ 39 | :duration => duration, 40 | :status => status 41 | }) 42 | payload = ::LogSpy::Payload.new(req, res, @start_time.to_i, err) 43 | 44 | sqs_client.send_message({ 45 | queue_url: @sqs_url, 46 | message_body: payload.to_json 47 | }) 48 | end 49 | end 50 | private :send_sqs_async 51 | end 52 | -------------------------------------------------------------------------------- /lib/log_spy/version.rb: -------------------------------------------------------------------------------- 1 | module LogSpy 2 | VERSION = "1.0.0" 3 | end 4 | -------------------------------------------------------------------------------- /log_spy.gemspec: -------------------------------------------------------------------------------- 1 | # coding: utf-8 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'log_spy/version' 5 | 6 | Gem::Specification.new do |spec| 7 | spec.name = "log_spy" 8 | spec.version = LogSpy::VERSION 9 | spec.authors = ["Yang-Hsing Lin"] 10 | spec.email = ["yanghsing.lin@gmail.com"] 11 | spec.summary = %q{ send rack application log to Amazon SQS } 12 | spec.description = %q{ LogSpy is a rack middleware sending request log to Amazon SQS } 13 | spec.homepage = "" 14 | spec.license = "MIT" 15 | 16 | spec.files = `git ls-files -z`.split("\x0") 17 | spec.executables = spec.files.grep(%r{^bin/}) { |f| File.basename(f) } 18 | spec.test_files = spec.files.grep(%r{^(test|spec|features)/}) 19 | spec.require_paths = ["lib"] 20 | 21 | spec.add_dependency "aws-sdk", [">= 2.2.14", "< 3.0"] 22 | spec.add_dependency "rack" 23 | spec.add_development_dependency "bundler", "~> 1.6" 24 | spec.add_development_dependency "rake" 25 | spec.add_development_dependency "rspec" 26 | end 27 | -------------------------------------------------------------------------------- /spec/payload_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'log_spy/payload' 3 | 4 | describe LogSpy::Payload do 5 | let(:path) { '/the/request/path' } 6 | let(:request_method) { 'post' } 7 | let(:ip) { '123.1.1.1' } 8 | let(:query_string) { 'query-key=query-val' } 9 | let(:status) { 200 } 10 | let(:duration) { 335 } 11 | let(:body) { double(:body) } 12 | let(:content_type) { 'application/json' } 13 | 14 | before :each do 15 | allow(body).to receive_messages(:rewind => body) 16 | end 17 | 18 | let(:request) do 19 | double(:request, 20 | :path => path, 21 | :content_type => content_type, 22 | :request_method => request_method, 23 | :ip => ip, 24 | :query_string => query_string, 25 | :env => {}, 26 | :cookies => { :key => 'val' }, 27 | :body => body) 28 | end 29 | 30 | let(:response) do 31 | double(:response, 32 | :status => status, 33 | :duration => duration) 34 | end 35 | 36 | let(:error) { Exception.new 'the-message' } 37 | let(:begin_at) { Time.now.to_i } 38 | 39 | describe '::new(request, response, begin_at[, error = nil])' do 40 | it 'takes a request, response and an optional error to init' do 41 | LogSpy::Payload.new(request, response, begin_at) 42 | LogSpy::Payload.new(request, response, begin_at, error) 43 | end 44 | end 45 | 46 | describe '#to_json' do 47 | let(:expected_hash) do 48 | { 49 | :path => path, 50 | :status => status, 51 | :execution_time => duration, 52 | :begin_at => begin_at, 53 | :request => { 54 | :content_type => content_type, 55 | :request_method => request_method, 56 | :ip => ip, 57 | :query_string => query_string, 58 | :cookies => request.cookies 59 | } 60 | } 61 | end 62 | 63 | shared_examples 'ensure_payload_formats' do 64 | it 'returns correct format' do 65 | expect(payload.to_json).to eq(expected_hash.to_json) 66 | end 67 | 68 | it 'returns no body if request content_type is multipart' do 69 | allow(request).to receive_messages(:content_type => 'multipart/form-data') 70 | expected_hash[:request][:content_type] = 'multipart/form-data' 71 | expected_hash[:request][:body] = '' 72 | expect(payload.to_json).to eq(expected_hash.to_json) 73 | end 74 | end 75 | 76 | shared_context "if_body_readable" do 77 | before(:each) do 78 | allow(body).to receive_messages(:rewind => 0) 79 | allow(body).to receive_messages(:read => 'the-raw-body') 80 | expected_hash[:request][:body] = 'the-raw-body' 81 | end 82 | end 83 | 84 | shared_context 'if_body_unreadable' do 85 | before(:each) do 86 | allow(body).to receive(:rewind).and_raise(IOError, 'closed stream') 87 | allow(request).to receive_messages(:env => { 'RAW_POST_BODY' => 'raw-post-body' }) 88 | expected_hash[:request][:body] = 'raw-post-body' 89 | end 90 | 91 | end 92 | 93 | shared_examples "ensure_action_dispatch_controller_params" do 94 | before :each do 95 | controller_params = { 'controller' => 'users', 'action' => 'show' } 96 | env = { 'action_dispatch.request.parameters' => controller_params } 97 | allow(request).to receive_messages(:env => env) 98 | end 99 | 100 | it 'returns hash with `controller_action`' do 101 | expected_hash[:controller_action] = "users#show" 102 | 103 | expect(payload.to_json).to eq(expected_hash.to_json) 104 | end 105 | end 106 | 107 | context "if request ends without error" do 108 | let(:payload) { LogSpy::Payload.new request, response, begin_at } 109 | 110 | context "if env['action_dispatch.request.parameters']" do 111 | include_context "if_body_readable" 112 | include_examples "ensure_action_dispatch_controller_params" 113 | end 114 | 115 | context "if body can be read" do 116 | include_context "if_body_readable" 117 | include_examples 'ensure_payload_formats' 118 | end 119 | 120 | context "if body can not be read" do 121 | include_context "if_body_unreadable" 122 | include_examples 'ensure_payload_formats' 123 | end 124 | 125 | end 126 | 127 | context "if request ends with error" do 128 | let(:payload) { LogSpy::Payload.new request, response, begin_at, error } 129 | before :each do 130 | expected_hash[:error] = { 131 | :message => error.message, 132 | :backtrace => error.backtrace 133 | } 134 | end 135 | 136 | context "if env['action_dispatch.request.parameters']" do 137 | include_context "if_body_readable" 138 | include_examples "ensure_action_dispatch_controller_params" 139 | end 140 | 141 | context "if request body can be read" do 142 | include_context 'if_body_readable' 143 | include_examples 'ensure_payload_formats' 144 | end 145 | 146 | context "if request body can not be read" do 147 | include_context 'if_body_unreadable' 148 | include_examples 'ensure_payload_formats' 149 | end 150 | end 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | lib = File.expand_path('../lib', __FILE__) 2 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 3 | -------------------------------------------------------------------------------- /spec/spy_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require 'log_spy/spy' 3 | 4 | describe LogSpy::Spy do 5 | let(:app) { double(:app) } 6 | let(:sqs_url) { 'the-sqs-url' } 7 | let(:options) { { :reigon => 'ap-southeast-1' } } 8 | 9 | describe '#new(app, sqs_url [, options = {}])' do 10 | it 'takes an app, an sqs_url, and an optional options to init' do 11 | LogSpy::Spy.new(app, sqs_url) 12 | LogSpy::Spy.new(app, sqs_url, options) 13 | end 14 | end 15 | 16 | describe '#call' do 17 | let(:sqs) { double(:sqs, :send_message => true) } 18 | # let(:sqs) { double(:sqs, :queues => double(:[] => queue)) } 19 | # let(:queue) { double(:queue, :send_message => true) } 20 | let(:call_result) { [200, { 'Content-Type' => 'application/json' }, [ 'body' ]] } 21 | let(:env) { {} } 22 | 23 | let(:request) { double(:request) } 24 | let(:payload) { double(:payload, :to_json => {'key' => 'val'}.to_json) } 25 | 26 | let(:middleware) { LogSpy::Spy.new(app, sqs_url, options) } 27 | let(:duration) { 20.12345 } 28 | let(:now) { Time.now } 29 | let(:three_sec_later) { now + duration } 30 | 31 | before :each do 32 | allow(Aws::SQS::Client).to receive_messages(:new => sqs) 33 | allow(app).to receive_messages(:call => call_result) 34 | allow(Rack::Request).to receive_messages(:new => request) 35 | allow(LogSpy::Payload).to receive_messages(:new => payload) 36 | allow(Time).to receive(:now).and_return(now, three_sec_later) 37 | end 38 | 39 | it 'creates a sqs client with options' do 40 | expect(Aws::SQS::Client).to receive(:new).with(options) 41 | 42 | middleware.call env 43 | middleware.sqs_thread.join 44 | end 45 | 46 | it 'creates payload with request, status, request_time' do 47 | expect(Rack::Request).to receive(:new).with(env) 48 | expect(LogSpy::Payload).to receive(:new) do |req, res, begin_at| 49 | expect(req).to be(request) 50 | expect(res.status).to eq(200) 51 | expect(begin_at).to eq(now.to_i) 52 | expect(res.duration).to eq((duration * 1000).round(0)) 53 | end 54 | 55 | middleware.call env 56 | middleware.sqs_thread.join 57 | end 58 | 59 | it 'sends payload json to sqs with queue url' do 60 | expect(sqs).to receive(:send_message).with({ 61 | queue_url: sqs_url, 62 | message_body: payload.to_json 63 | }) 64 | middleware.call env 65 | middleware.sqs_thread.join 66 | end 67 | 68 | 69 | it 'returns original result' do 70 | expect(middleware.call(env)).to eq(call_result) 71 | middleware.sqs_thread.join 72 | end 73 | 74 | context "if `app.call raises`" do 75 | let(:error) { Exception.new } 76 | 77 | before :each do 78 | allow(app).to receive(:call).and_raise(error) 79 | end 80 | 81 | it 'build payload with error' do 82 | expect(LogSpy::Payload).to receive(:new) do |req, res, begin_at, err| 83 | expect(req).to be(request) 84 | expect(res.status).to eq(500) 85 | expect(res.duration).to eq(( duration * 1000 ).round(0)) 86 | expect(begin_at).to eq(now.to_i) 87 | expect(err).to eq(error) 88 | end 89 | 90 | begin 91 | middleware.call(env) 92 | rescue Exception 93 | end 94 | 95 | middleware.sqs_thread.join 96 | end 97 | 98 | it 'forward to error' do 99 | expect { 100 | middleware.call(env) 101 | }.to raise_error(error) 102 | 103 | middleware.sqs_thread.join 104 | end 105 | 106 | end 107 | 108 | end 109 | end 110 | --------------------------------------------------------------------------------