├── spec ├── fixtures │ ├── multiple_files_2.log │ ├── multiple_files_1.log │ ├── rails.db │ ├── decompression.tgz │ ├── decompression.log.bz2 │ ├── decompression.log.gz │ ├── decompression.log.zip │ ├── decompression.tar.gz │ ├── header_and_footer.log │ ├── test_language_combined.log │ ├── test_file_format.log │ ├── test_order.log │ ├── syslog_1x.log │ ├── rails_22.log │ ├── decompression.log │ ├── rails_3_partials.log │ ├── apache_combined.log │ ├── merb_prefixed.log │ ├── apache_common.log │ ├── rails_unordered.log │ ├── rails_22_cached.log │ ├── s3_logs │ │ ├── 2012-10-05-16-26-06-15314AF7F0651839 │ │ └── 2012-10-05-16-18-11-F9AAC5D1A55AEBAD │ ├── oink_22_failure.log │ ├── oink_22.log │ ├── postgresql.log │ ├── rails_1x.log │ └── merb.log ├── unit │ ├── file_format │ │ ├── inheritance_spec.rb │ │ ├── format_autodetection_spec.rb │ │ ├── w3c_format_spec.rb │ │ ├── merb_format_spec.rb │ │ ├── postgresql_format_spec.rb │ │ ├── file_format_api_spec.rb │ │ ├── rack_format_spec.rb │ │ ├── line_definition_spec.rb │ │ ├── delayed_job21_format_spec.rb │ │ ├── common_regular_expressions_spec.rb │ │ ├── delayed_job3_format_spec.rb │ │ ├── delayed_job2_format_spec.rb │ │ ├── oink_format_spec.rb │ │ ├── delayed_job4_format_spec.rb │ │ └── delayed_job_format_spec.rb │ ├── filter │ │ ├── filter_spec.rb │ │ ├── anonymize_filter_spec.rb │ │ ├── timespan_filter_spec.rb │ │ └── field_filter_spec.rb │ ├── controller │ │ ├── log_processor_spec.rb │ │ └── controller_spec.rb │ ├── aggregator │ │ ├── summarizer_spec.rb │ │ └── database_inserter_spec.rb │ ├── tracker │ │ ├── traffic_tracker_spec.rb │ │ ├── duration_tracker_spec.rb │ │ ├── timespan_tracker_spec.rb │ │ ├── frequency_tracker_spec.rb │ │ └── hourly_spread_spec.rb │ ├── mailer_spec.rb │ ├── database │ │ └── connection_spec.rb │ └── request_spec.rb ├── lib │ ├── macros.rb │ ├── testing_format.rb │ ├── helpers.rb │ ├── mocks.rb │ └── matchers.rb ├── database.yml ├── spec_helper.rb └── integration │ ├── munin_plugins_rails_spec.rb │ └── command_line_usage_spec.rb ├── lib ├── request_log_analyzer │ ├── version.rb │ ├── file_format │ │ ├── rack.rb │ │ ├── nginx.rb │ │ ├── rails_development.rb │ │ ├── delayed_job2.rb │ │ ├── delayed_job21.rb │ │ ├── delayed_job.rb │ │ ├── w3c.rb │ │ ├── delayed_job3.rb │ │ ├── delayed_job4.rb │ │ ├── postgresql.rb │ │ ├── amazon_s3.rb │ │ ├── merb.rb │ │ └── mysql.rb │ ├── database │ │ ├── source.rb │ │ ├── warning.rb │ │ ├── request.rb │ │ └── connection.rb │ ├── class_level_inheritable_attributes.rb │ ├── filter.rb │ ├── aggregator │ │ ├── echo.rb │ │ └── database_inserter.rb │ ├── filter │ │ ├── anonymize.rb │ │ ├── field.rb │ │ └── timespan.rb │ ├── aggregator.rb │ ├── tracker │ │ ├── traffic.rb │ │ ├── duration.rb │ │ ├── timespan.rb │ │ ├── hourly_spread.rb │ │ └── frequency.rb │ ├── mailer.rb │ ├── source.rb │ ├── log_processor.rb │ ├── database.rb │ └── output.rb ├── cli │ ├── database_console.rb │ ├── database_console_init.rb │ └── tools.rb └── request_log_analyzer.rb ├── Gemfile ├── Gemfile.activerecord3 ├── Gemfile.activerecord4 ├── .gitignore ├── Rakefile ├── bin └── console ├── .travis.yml ├── LICENSE ├── DESIGN.rdoc ├── README.rdoc ├── CONTRIBUTING.rdoc └── request-log-analyzer.gemspec /spec/fixtures/multiple_files_2.log: -------------------------------------------------------------------------------- 1 | testing is cool 2 | finishing request 1 3 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/version.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer 2 | VERSION = '1.13.4' 3 | end 4 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | platform :rbx do 5 | gem 'rubysl' 6 | end 7 | -------------------------------------------------------------------------------- /spec/fixtures/multiple_files_1.log: -------------------------------------------------------------------------------- 1 | initializing 2 | 3 | processing request 1 4 | 5 | testing is amazing 6 | -------------------------------------------------------------------------------- /spec/fixtures/rails.db: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wvanbergen/request-log-analyzer/HEAD/spec/fixtures/rails.db -------------------------------------------------------------------------------- /spec/fixtures/decompression.tgz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wvanbergen/request-log-analyzer/HEAD/spec/fixtures/decompression.tgz -------------------------------------------------------------------------------- /spec/fixtures/decompression.log.bz2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wvanbergen/request-log-analyzer/HEAD/spec/fixtures/decompression.log.bz2 -------------------------------------------------------------------------------- /spec/fixtures/decompression.log.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wvanbergen/request-log-analyzer/HEAD/spec/fixtures/decompression.log.gz -------------------------------------------------------------------------------- /spec/fixtures/decompression.log.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wvanbergen/request-log-analyzer/HEAD/spec/fixtures/decompression.log.zip -------------------------------------------------------------------------------- /spec/fixtures/decompression.tar.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/wvanbergen/request-log-analyzer/HEAD/spec/fixtures/decompression.tar.gz -------------------------------------------------------------------------------- /spec/fixtures/header_and_footer.log: -------------------------------------------------------------------------------- 1 | start! 2 | this is a header and footer line 3 | blah 4 | 5 | this is a header and footer line 6 | blah 7 | -------------------------------------------------------------------------------- /Gemfile.activerecord3: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | gem 'activerecord', '~> 3.0' 5 | 6 | platform :rbx do 7 | gem 'rubysl' 8 | end 9 | -------------------------------------------------------------------------------- /Gemfile.activerecord4: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | gemspec 3 | 4 | gem 'activerecord', '~> 4.0' 5 | 6 | platform :rbx do 7 | gem 'rubysl' 8 | end 9 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/rack.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | class Rack < Apache 3 | def self.create(*args) 4 | super(:rack, *args) 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/nginx.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | class Nginx < Apache 3 | def self.create(*args) 4 | super(:combined, *args) 5 | end 6 | end 7 | end 8 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .svn/ 2 | .DS_Store 3 | request-log-analyzer-*.gem 4 | requests.db 5 | /pkg 6 | /doc 7 | .yardoc/ 8 | /tmp 9 | /classes 10 | /files 11 | /coverage 12 | Gemfile.lock 13 | /.bundle/ 14 | *.rbc 15 | _site/ 16 | .idea 17 | -------------------------------------------------------------------------------- /spec/fixtures/test_language_combined.log: -------------------------------------------------------------------------------- 1 | initializing 2 | 3 | processing request 1 4 | 5 | testing is amazing 6 | testing is cool 7 | 8 | finishing request 1 9 | 10 | asdfad # garbage 11 | dsaads # garbage 12 | 13 | processing request 1 14 | finishing request 1 -------------------------------------------------------------------------------- /spec/fixtures/test_file_format.log: -------------------------------------------------------------------------------- 1 | processing request 1 2 | testing 3 | testing methods 4 | testing is cool 5 | testing fixtures 6 | testing is amazing 7 | testing can be cumbersome 8 | testing 9 | nonsense 10 | testing is 11 | 12 | more nonsense 13 | finishing request 1 -------------------------------------------------------------------------------- /spec/fixtures/test_order.log: -------------------------------------------------------------------------------- 1 | initializing 2 | 3 | processing request 1 4 | 5 | testing is amazing 6 | testing is cool 7 | 8 | processing request 1 9 | testing is cool 10 | finishing request 1 11 | 12 | asdfad # garbage 13 | dsaads # garbage 14 | 15 | 16 | finishing request 1 -------------------------------------------------------------------------------- /Rakefile: -------------------------------------------------------------------------------- 1 | require "bundler/gem_tasks" 2 | require "rspec/core/rake_task" 3 | 4 | Dir[File.dirname(__FILE__) + "/tasks/*.rake"].each { |file| load(file) } 5 | 6 | RSpec::Core::RakeTask.new(:spec) do |task| 7 | task.pattern = "./spec/**/*_spec.rb" 8 | task.rspec_opts = ['--color'] 9 | end 10 | 11 | task :default => :spec 12 | -------------------------------------------------------------------------------- /spec/unit/file_format/inheritance_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | class DummyInheritedFromRails3 < RequestLogAnalyzer::FileFormat::Rails3 4 | end 5 | 6 | describe DummyInheritedFromRails3 do 7 | 8 | subject { RequestLogAnalyzer::FileFormat.load(DummyInheritedFromRails3) } 9 | 10 | it { should be_well_formed } 11 | it { subject.report_trackers.length.should == 11 } 12 | 13 | end 14 | -------------------------------------------------------------------------------- /bin/console: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # -*- encoding : utf-8 -*- 3 | $LOAD_PATH.unshift(File.expand_path('./../lib', File.dirname(__FILE__))) 4 | #noinspection RubyResolve 5 | require 'request_log_analyzer' 6 | require 'irb' 7 | require 'pp' 8 | 9 | if __FILE__ == $0 10 | IRB.start(__FILE__) 11 | else # check -e option 12 | if /\A-e\z/ =~ $0 13 | IRB.start(__FILE__) 14 | else 15 | IRB.setup(__FILE__) 16 | end 17 | end 18 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/database/source.rb: -------------------------------------------------------------------------------- 1 | class RequestLogAnalyzer::Database::Source < RequestLogAnalyzer::Database::Base 2 | def self.create_table! 3 | unless database.connection.table_exists?(:sources) 4 | database.connection.create_table(:sources) do |t| 5 | t.column :filename, :string 6 | t.column :mtime, :datetime 7 | t.column :filesize, :integer 8 | end 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/unit/filter/filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Filter::Base, 'base filter' do 4 | 5 | before(:each) do 6 | @filter = RequestLogAnalyzer::Filter::Base.new(testing_format) 7 | end 8 | 9 | it 'should return everything' do 10 | @filter.filter(request(ip: '123.123.123.123'))[:ip].should eql('123.123.123.123') 11 | end 12 | 13 | it 'should return nil on nil request' do 14 | @filter.filter(nil).should be_nil 15 | end 16 | 17 | end 18 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/database/warning.rb: -------------------------------------------------------------------------------- 1 | class RequestLogAnalyzer::Database::Warning < RequestLogAnalyzer::Database::Base 2 | def self.create_table! 3 | unless database.connection.table_exists?(:warnings) 4 | database.connection.create_table(:warnings) do |t| 5 | t.column :warning_type, :string, limit: 30, null: false 6 | t.column :message, :string 7 | t.column :source_id, :integer 8 | t.column :lineno, :integer 9 | end 10 | end 11 | end 12 | end 13 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/rails_development.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The RailsDevelopment FileFormat is an extention to the default Rails file format. It includes 3 | # all lines of the normal Rails file format, but parses SQL queries and partial rendering lines 4 | # as well. 5 | class RailsDevelopment < Rails 6 | def self.create 7 | # puts 'DEPRECATED: use --rails-format development instead!' 8 | super('development') 9 | end 10 | end 11 | end 12 | -------------------------------------------------------------------------------- /spec/lib/macros.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::RSpec::Macros 2 | def test_databases 3 | require 'yaml' 4 | hash = YAML.load(File.read("#{File.dirname(__FILE__)}/../database.yml")) 5 | hash.reduce({}) { |res, (name, h)| res[name] = h.map { |(k, v)| "#{k}=#{v}" }.join(';'); res } 6 | end 7 | 8 | # Create or return a new TestingFormat 9 | def testing_format 10 | @testing_format ||= TestingFormat.create 11 | end 12 | 13 | def default_orm_class_names 14 | %w(Warning Request Source) 15 | end 16 | end 17 | -------------------------------------------------------------------------------- /spec/database.yml: -------------------------------------------------------------------------------- 1 | # This file determines what databases are used to test the database 2 | # functionality of request-log-analyzer. By default, only an SQLite3 3 | # memory database is enabled. 4 | # 5 | # The syntax of this file is exactly the same as Rails's database.yml. 6 | 7 | sqlite3: 8 | adapter: "sqlite3" 9 | database: ":memory:" 10 | 11 | mysql: 12 | adapter: "mysql2" 13 | username: "root" 14 | database: "rla_test" 15 | 16 | postgresql: 17 | adapter: "postgresql" 18 | username: "postgres" 19 | database: "rla_test" 20 | -------------------------------------------------------------------------------- /spec/fixtures/syslog_1x.log: -------------------------------------------------------------------------------- 1 | Jul 13 06:25:58 10.1.1.32 app_p [1957]: Processing EmployeeController#index (for 10.1.1.33 at 2008-07-13 06:25:58) [GET] 2 | Jul 13 06:25:58 10.1.1.32 app_p [1957]: Session ID: bd1810833653be11c38ad1e5675635bd 3 | Jul 13 06:25:58 10.1.1.32 app_p [1957]: Parameters: {"format"=>"xml", "action"=>"index} 4 | Jul 13 06:25:58 10.1.1.32 app_p [1957]: Rendering employees 5 | Jul 13 06:25:58 10.1.1.32 app_p [1957]: Completed in 0.21665 (4 reqs/sec) | Rendering: 0.00926 (4%) | DB: 0.00000 (0%) | 200 OK [http://example.com/employee.xml] 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: ruby 2 | before_script: 3 | - mysql -e 'create database rla_test;' 4 | - psql -c 'create database rla_test;' -U postgres 5 | script: bundle exec rake 6 | rvm: 7 | - 1.9.3 8 | - 2.0.0 9 | - 2.1.1 10 | - ruby-head 11 | - jruby-19mode 12 | - jruby-head 13 | - rbx-2 14 | gemfile: 15 | - Gemfile.activerecord3 16 | - Gemfile.activerecord4 17 | matrix: 18 | allow_failures: 19 | - rvm: jruby-head 20 | - rvm: ruby-head 21 | - rvm: rbx-2 22 | notifications: 23 | email: 24 | - info@railsdoctors.com 25 | -------------------------------------------------------------------------------- /spec/unit/controller/log_processor_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | require 'request_log_analyzer/log_processor' 4 | 5 | describe RequestLogAnalyzer::LogProcessor, 'stripping log files' do 6 | 7 | before(:each) do 8 | @log_stripper = RequestLogAnalyzer::LogProcessor.new(testing_format, :strip, {}) 9 | end 10 | 11 | it 'should remove a junk line' do 12 | @log_stripper.strip_line("junk line\n").should be_empty 13 | end 14 | 15 | it 'should keep a teaser line intact' do 16 | @log_stripper.strip_line("processing 1234\n").should be_empty 17 | end 18 | end 19 | -------------------------------------------------------------------------------- /spec/fixtures/rails_22.log: -------------------------------------------------------------------------------- 1 | Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET] 2 | Parameters: {"action"=>"demo", "controller"=>"page"} 3 | Logging in from session data... 4 | Logged in as test@example.com 5 | Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain: 6 | Rendering template within layouts/demo 7 | Rendering page/demo 8 | Rendered shared/_analytics (0.2ms) 9 | Rendered layouts/_actions (0.6ms) 10 | Rendered layouts/_menu (2.2ms) 11 | Rendered layouts/_tabbar (0.5ms) 12 | Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.com/demo] -------------------------------------------------------------------------------- /spec/fixtures/decompression.log: -------------------------------------------------------------------------------- 1 | Processing PageController#demo (for 127.0.0.1 at 2008-12-10 16:28:09) [GET] 2 | Parameters: {"action"=>"demo", "controller"=>"page"} 3 | Logging in from session data... 4 | Logged in as test@example.com 5 | Using locale: en-US, http-accept: ["en-US"], session: , det browser: en-US, det domain: 6 | Rendering template within layouts/demo 7 | Rendering page/demo 8 | Rendered shared/_analytics (0.2ms) 9 | Rendered layouts/_actions (0.6ms) 10 | Rendered layouts/_menu (2.2ms) 11 | Rendered layouts/_tabbar (0.5ms) 12 | Completed in 614ms (View: 120, DB: 31) | 200 OK [http://www.example.coml/demo] -------------------------------------------------------------------------------- /spec/fixtures/rails_3_partials.log: -------------------------------------------------------------------------------- 1 | Started GET "/foobartest" for 127.0.0.1 at 2012-11-21 15:21:13 +0100 2 | Processing by HomeController#foobartest as */* 3 | Rendered home/foobartest_partial.html.slim (10.0ms) 4 | Rendered home/foobartest_partial.html.slim (10.0ms) 5 | Rendered home/foobartest_partial.html.slim (10.0ms) 6 | Completed 200 OK in 58ms (Views: 44.6ms | ActiveRecord: 0.0ms) 7 | 8 | 9 | Started GET "/foobartest" for 127.0.0.1 at 2012-11-21 15:21:31 +0100 10 | Processing by HomeController#foobartest as */* 11 | Rendered home/foobartest_partial.html.slim (100.0ms) 12 | Rendered home/foobartest_partial.html.slim (0.0ms) 13 | Completed 200 OK in 2ms (Views: 0.6ms | ActiveRecord: 0.0ms) 14 | -------------------------------------------------------------------------------- /spec/unit/filter/anonymize_filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Filter::Anonymize, 'anonymize request' do 4 | 5 | before(:each) do 6 | @filter = RequestLogAnalyzer::Filter::Anonymize.new(testing_format) 7 | end 8 | 9 | it 'should anonimize ip' do 10 | @filter.filter(request(ip: '123.123.123.123'))[:ip].should_not eql('123.123.123.123') 11 | end 12 | 13 | it 'should anonimize url' do 14 | @filter.filter(request(url: 'https://test.mysite.com/employees'))[:url].should eql('http://example.com/employees') 15 | end 16 | 17 | it 'should fuzz durations' do 18 | @filter.filter(request(duration: 100))[:duration].should_not eql(100) 19 | end 20 | 21 | end 22 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/database/request.rb: -------------------------------------------------------------------------------- 1 | class RequestLogAnalyzer::Database::Request < RequestLogAnalyzer::Database::Base 2 | # Returns an array of all the Line objects of this request in the correct order. 3 | def lines 4 | @lines ||= begin 5 | lines = [] 6 | self.class.reflections.each { |r, _d| lines += send(r).all } 7 | lines.sort 8 | end 9 | end 10 | 11 | # Creates the table to store requests in. 12 | def self.create_table! 13 | unless database.connection.table_exists?(:requests) 14 | database.connection.create_table(:requests) do |t| 15 | t.column :first_lineno, :integer 16 | t.column :last_lineno, :integer 17 | end 18 | end 19 | end 20 | end 21 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/class_level_inheritable_attributes.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer 2 | module ClassLevelInheritableAttributes 3 | def inheritable_attributes(*args) 4 | @inheritable_attributes ||= [:inheritable_attributes] 5 | @inheritable_attributes += args 6 | args.each do |arg| 7 | class_eval %( 8 | class << self; attr_accessor :#{arg} end 9 | ) 10 | end 11 | @inheritable_attributes 12 | end 13 | 14 | def inherited(subclass) 15 | @inheritable_attributes.each do |inheritable_attribute| 16 | instance_var = "@#{inheritable_attribute}" 17 | subclass.instance_variable_set(instance_var, instance_variable_get(instance_var)) 18 | end 19 | 20 | super 21 | end 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/fixtures/apache_combined.log: -------------------------------------------------------------------------------- 1 | 125.76.230.10 - - [02/Sep/2009:03:33:46 +0200] "GET /cart/install.txt HTTP/1.1" 404 214 "-" "Toata dragostea mea pentru diavola" 2 | 125.76.230.10 - - [02/Sep/2009:03:33:47 +0200] "GET /store/install.txt HTTP/1.1" 404 215 "-" "Toata dragostea mea pentru diavola" 3 | 10.0.1.1 - - [02/Sep/2009:05:08:33 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9" 4 | 10.0.1.1 - - [02/Sep/2009:06:41:51 +0200] "GET / HTTP/1.1" 200 30 "-" "Mozilla/5.0 (Macintosh; U; Intel Mac OS X 10_5_8; en-us) AppleWebKit/531.9 (KHTML, like Gecko) Version/4.0.3 Safari/531.9" 5 | 69.41.0.45 - - [02/Sep/2009:12:02:40 +0200] "GET //phpMyAdmin/ HTTP/1.1" 404 209 "-" "Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)" 6 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/filter.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Filter 2 | # Base filter class used to filter input requests. 3 | # All filters should interit from this base. 4 | class Base 5 | attr_reader :file_format, :options 6 | 7 | # Initializer 8 | # format The file format 9 | # options Are passed to the filters. 10 | def initialize(format, options = {}) 11 | @file_format = format 12 | @options = options 13 | end 14 | 15 | # Return the request if the request should be kept. 16 | # Return nil otherwise. 17 | def filter(request) 18 | request 19 | end 20 | end 21 | end 22 | 23 | require 'request_log_analyzer/filter/field' 24 | require 'request_log_analyzer/filter/timespan' 25 | require 'request_log_analyzer/filter/anonymize' 26 | -------------------------------------------------------------------------------- /spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'bundler/setup' 3 | 4 | require 'rspec' 5 | require 'request_log_analyzer' 6 | require 'stringio' 7 | 8 | module RequestLogAnalyzer::RSpec 9 | end 10 | 11 | # Include all files in the spec_helper directory 12 | Dir[File.dirname(__FILE__) + '/lib/**/*.rb'].each do |file| 13 | require file 14 | end 15 | 16 | Dir.mkdir("#{File.dirname(__FILE__)}/../tmp") unless File.exist?("#{File.dirname(__FILE__)}/../tmp") 17 | 18 | RSpec.configure do |config| 19 | config.include RequestLogAnalyzer::RSpec::Matchers 20 | config.include RequestLogAnalyzer::RSpec::Mocks 21 | config.include RequestLogAnalyzer::RSpec::Helpers 22 | 23 | config.extend RequestLogAnalyzer::RSpec::Macros 24 | 25 | config.expect_with :rspec do |c| 26 | c.syntax = :should 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /lib/cli/database_console.rb: -------------------------------------------------------------------------------- 1 | class DatabaseConsole 2 | IRB = RUBY_PLATFORM =~ /(:?mswin|mingw)/ ? 'irb.bat' : 'irb' 3 | 4 | def initialize(arguments) 5 | @arguments = arguments 6 | end 7 | 8 | def run! 9 | libraries = ['irb/completion', 'rubygems', 'cli/database_console_init'] 10 | libaries_string = libraries.map { |l| "-r #{l}" }.join(' ') 11 | 12 | ENV['RLA_DBCONSOLE_DATABASE'] = @arguments[:database] 13 | if @arguments[:apache_format] 14 | ENV['RLA_DBCONSOLE_FORMAT'] = 'apache' 15 | ENV['RLA_DBCONSOLE_FORMAT_ARGUMENT'] = @arguments[:apache_format] 16 | else 17 | ENV['RLA_DBCONSOLE_FORMAT'] = @arguments[:format] 18 | end 19 | # ENV['RLA_DBCONSOLE_FORMAT_ARGS'] = arguments['database'] 20 | 21 | exec("#{IRB} #{libaries_string} --simple-prompt") 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /spec/fixtures/merb_prefixed.log: -------------------------------------------------------------------------------- 1 | Aug 31 18:35:23 typekit-web001 merb: ~ 2 | Aug 31 18:35:23 typekit-web001 merb: cache: [GET /] miss 3 | Aug 31 18:35:24 typekit-web001 merb: ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009 4 | Aug 31 18:35:24 typekit-web001 merb: ~ Routed to: {"action"=>"index", "controller"=>"home"} 5 | Aug 31 18:35:24 typekit-web001 merb: ~ Params: {"action"=>"index", "controller"=>"home"} 6 | Aug 31 18:35:24 typekit-web001 merb: ~ In repository block default 7 | Aug 31 18:35:24 typekit-web001 merb: ~ (0.000000) SELECT `id`, `created_at`, `updated_at`, `braintree_vault_id`, `cancelled_at` FROM `accounts` WHERE (`id` IN (6214)) ORDER BY `id` 8 | Aug 31 18:35:24 typekit-web001 merb: ~ Redirecting to: /plans (302) 9 | Aug 31 18:35:24 typekit-web001 merb: ~ {:after_filters_time=>0.0, :before_filters_time=>0.0, :dispatch_time=>0.012001, :action_time=>0.012001} 10 | -------------------------------------------------------------------------------- /spec/unit/aggregator/summarizer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Aggregator::Summarizer do 4 | 5 | before(:each) do 6 | @summarizer = RequestLogAnalyzer::Aggregator::Summarizer.new(mock_source, output: mock_output) 7 | @summarizer.prepare 8 | end 9 | 10 | it 'not raise exception when creating a report after aggregating multiple requests' do 11 | @summarizer.aggregate(request(data: 'bluh1')) 12 | @summarizer.aggregate(request(data: 'bluh2')) 13 | 14 | lambda { @summarizer.report(mock_output) }.should_not raise_error 15 | end 16 | 17 | it 'not raise exception when creating a report after aggregating a single request' do 18 | @summarizer.aggregate(request(data: 'bluh1')) 19 | lambda { @summarizer.report(mock_output) }.should_not raise_error 20 | end 21 | 22 | it 'not raise exception when creating a report after aggregating no requests' do 23 | lambda { @summarizer.report(mock_output) }.should_not raise_error 24 | end 25 | 26 | end 27 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/aggregator/echo.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Aggregator 2 | # Echo Aggregator. Writes everything to the screen when it is passed to this aggregator 3 | class Echo < Base 4 | attr_accessor :warnings 5 | 6 | def prepare 7 | @warnings = [] 8 | end 9 | 10 | # Display every parsed line immediately to the terminal 11 | def aggregate(request) 12 | puts "\nRequest: \n" + request.lines.map do |l| 13 | "\t#{l[:lineno]}:#{l[:line_type]}: #{l.reject { |(k, _)| [:lineno, :line_type].include?(k) }.inspect}" 14 | end.join("\n") 15 | end 16 | 17 | # Capture all warnings during parsing 18 | def warning(type, message, lineno) 19 | @warnings << "WARNING #{type.inspect} on line #{lineno}: #{message}" 20 | end 21 | 22 | # Display every warning in the report when finished parsing 23 | def report(output) 24 | output.title('Warnings during parsing') 25 | @warnings.each { |w| output.puts(w) } 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /spec/fixtures/apache_common.log: -------------------------------------------------------------------------------- 1 | 1.56.136.157 - - [08/Sep/2009:07:54:07 -0400] "GET /form_configurations/show.flashxml HTTP/1.1" 304 - 2 | 1.72.56.31 - - [08/Sep/2009:07:54:07 -0400] "GET /users/M17286/projects.xml?per_page=8000 HTTP/1.1" 404 1 3 | 1.82.235.29 - - [08/Sep/2009:07:54:05 -0400] "GET /gallery/fresh?page=23&per_page=16 HTTP/1.1" 200 23414 4 | 1.56.136.157 - - [08/Sep/2009:07:54:08 -0400] "GET /designs/20110457.flashxml HTTP/1.1" 304 - 5 | 1.56.136.157 - - [08/Sep/2009:07:54:08 -0400] "GET /object_libraries/1.flashxml HTTP/1.1" 304 - 6 | 1.249.68.72 - - [08/Sep/2009:07:54:09 -0400] "GET /projects/18182835-my-first-project HTTP/1.1" 200 10435 7 | 1.108.106.20 - - [08/Sep/2009:07:54:09 -0400] "GET /login HTTP/1.1" 200 9522 8 | 1.129.119.13 - - [08/Sep/2009:07:54:09 -0400] "GET /profile/18543424 HTTP/1.0" 200 8223 9 | 1.158.151.25 - - [08/Sep/2009:07:54:09 -0400] "GET /projects/18236114-home/floors/18252888-/designs/19406560-home-1 HTTP/1.1" 200 10434 10 | 1.29.137.1 - - [08/Sep/2009:07:54:10 -0400] "GET /assets/custom/blah/images/print_logo.jpg HTTP/1.1" 200 10604 11 | -------------------------------------------------------------------------------- /spec/unit/tracker/traffic_tracker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Tracker::Traffic do 4 | 5 | describe '#report' do 6 | before(:each) do 7 | @tracker = RequestLogAnalyzer::Tracker::Traffic.new(category: :category, traffic: :traffic) 8 | @tracker.prepare 9 | end 10 | 11 | it 'should generate a report without errors when one category is present' do 12 | @tracker.update(request(category: 'a', traffic: 2)) 13 | @tracker.report(mock_output) 14 | lambda { @tracker.report(mock_output) }.should_not raise_error 15 | end 16 | 17 | it 'should generate a report without errors when no category is present' do 18 | lambda { @tracker.report(mock_output) }.should_not raise_error 19 | end 20 | 21 | it 'should generate a report without errors when multiple categories are present' do 22 | @tracker.update(request(category: 'a', traffic: 2)) 23 | @tracker.update(request(category: 'b', traffic: 2)) 24 | lambda { @tracker.report(mock_output) }.should_not raise_error 25 | end 26 | 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2008-2013 Willem van Bergen & Bart ten Brinke 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /spec/fixtures/rails_unordered.log: -------------------------------------------------------------------------------- 1 | Processing AccountController#dashboard (for 1.1.1.1 at 2008-12-24 07:36:49) [GET] 2 | Parameters: {"action"=>"dashboard", "controller"=>"account", "first_use"=>"true"} 3 | Logging in from session data... 4 | 5 | 6 | Processing ProjectsController#new (for 1.1.1.1 at 2008-12-24 07:36:49) [GET] 7 | Parameters: {"action"=>"new", "controller"=>"projects"} 8 | Rendering template within layouts/default 9 | Rendering account/dashboard 10 | Logging in from session data... 11 | Logging in using cookie... 12 | Using locale: en-US, http-accept: [], session: , det browser: , det domain: , user pref locale: 13 | Rendered shared/_maintenance (0.6ms) 14 | Rendering template within layouts/templates/general_default/index.html.erb 15 | Rendered projects/_recent_designs (4.3ms) 16 | Rendered projects/_project (13.6ms) 17 | Rendered projects/_projects (18.7ms) 18 | Rendered layouts/_menu (1.4ms) 19 | Completed in 36ms (View: 30, DB: 3) | 200 OK [http://www.example.com/projects/new] 20 | Rendered layouts/_actions (0.3ms) 21 | Rendered layouts/_menu (1.6ms) 22 | Rendered layouts/_tabbar (1.9ms) 23 | Rendered layouts/_footer (3.2ms) 24 | Completed in 50ms (View: 41, DB: 4) | 200 OK [http://www.example.com/dashboard?first_use=true] -------------------------------------------------------------------------------- /spec/fixtures/rails_22_cached.log: -------------------------------------------------------------------------------- 1 | Processing CachedController#cached (for 1.1.1.1 at 2008-12-24 07:36:53) [GET] 2 | Parameters: {"action"=>"cached", "controller"=>"cached"} 3 | Logging in from session data... 4 | Logging in using cookie... 5 | Using locale: zh-Hans, http-accept: ["zh-CN", "zh-HK", "zh-TW", "en-US"], session: , det browser: zh-Hans, det domain: , user pref locale: 6 | Referer: http://www.example.com/referer 7 | Cached fragment hit: views/zh-Hans-www-cached-cached-all-CN--- (0.0ms) 8 | Filter chain halted as [#{}, :layout=>nil, :cache_path=>#}>] rendered_or_redirected. 9 | Filter chain halted as [##, :if=>:not_logged_in?, :unless=>nil}, @method=#{}, :layout=>nil, :cache_path=>#}>>] did_not_yield. 10 | Completed in 3ms (View: 0, DB: 0) | 200 OK [http://www.example.com/cached/cached/] 11 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/filter/anonymize.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Filter 2 | # Filter to anonymize parsed values 3 | # Options 4 | # * :mode :reject or :accept. 5 | # * :field Specific field to accept or reject. 6 | # * :value Value that the field should match to be accepted or rejected. 7 | class Anonymize < Base 8 | def generate_random_ip 9 | "#{rand(256)}.#{rand(256)}.#{rand(256)}.#{rand(256)}" 10 | end 11 | 12 | def anonymize_url(value) 13 | value.sub(/^https?\:\/\/[A-Za-z0-9\.-]+\//, 'http://example.com/') 14 | end 15 | 16 | def fuzz(value) 17 | value * ((75 + rand(50)) / 100.0) 18 | end 19 | 20 | def filter(request) 21 | # TODO: request.attributes is bad practice 22 | request.attributes.each do |key, value| 23 | if key == :ip 24 | request.attributes[key] = generate_random_ip 25 | elsif key == :url 26 | request.attributes[key] = anonymize_url(value) 27 | elsif [:duration, :view, :db, :type, :after_filters_time, :before_filters_time, 28 | :action_time].include?(key) 29 | request.attributes[key] = fuzz(value) 30 | end 31 | end 32 | 33 | request 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/unit/mailer_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Mailer, 'mailer' do 4 | 5 | it 'should initialize correctly' do 6 | @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', debug: true) 7 | @mailer.host.should eql('localhost') 8 | @mailer.port.should eql(25) 9 | end 10 | 11 | it 'should allow alternate port settings' do 12 | @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost:2525', debug: true) 13 | @mailer.host.should eql('localhost') 14 | @mailer.port.should eql('2525') 15 | end 16 | 17 | it 'should store printed data' do 18 | @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', debug: true) 19 | 20 | @mailer << 'test1' 21 | @mailer.puts 'test2' 22 | 23 | @mailer.data.should eql(%w(test1 test2)) 24 | end 25 | 26 | it 'should send mail' do 27 | @mailer = RequestLogAnalyzer::Mailer.new('alfa@beta.com', 'localhost', debug: true) 28 | 29 | @mailer << 'test1' 30 | @mailer.puts 'test2' 31 | 32 | mail = @mailer.mail 33 | 34 | mail[0].should include('contact@railsdoctors.com') 35 | mail[0].should include('test1') 36 | mail[0].should include('test2') 37 | 38 | mail[1].should include('contact@railsdoctors.com') 39 | mail[2].should include('alfa@beta.com') 40 | end 41 | 42 | end 43 | -------------------------------------------------------------------------------- /spec/fixtures/s3_logs/2012-10-05-16-26-06-15314AF7F0651839: -------------------------------------------------------------------------------- 1 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:24:46 +0000] 174.44.160.70 - FC6B8CCFF3510D92 REST.GET.OBJECT public/portfolio/29/thumbnail.jpg "GET /lamestuff.com/public/portfolio/29/thumbnail.jpg HTTP/1.1" 200 - 6240 6240 40 39 "http://spike.grobste.in/portfolio" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 2 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:27 +0000] 174.44.160.70 - 6908D7E8249F8AB0 REST.GET.OBJECT public/projects/114/xmas-card-2006-c.jpg "GET /lamestuff.com/public/projects/114/xmas-card-2006-c.jpg HTTP/1.1" 200 - 99874 99874 51 47 "http://spike.grobste.in/portfolio/show?name=xmas-2006" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 3 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:30 +0000] 174.44.160.70 - 41804BB9A626C674 REST.GET.OBJECT public/projects/28/Tim_Horton_Counter__3x12_.jpg "GET /lamestuff.com/public/projects/28/Tim_Horton_Counter__3x12_.jpg HTTP/1.1" 200 - 22256 22256 93 91 "http://spike.grobste.in/project/tim-horton's-at-the-nyse" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 4 | -------------------------------------------------------------------------------- /spec/lib/testing_format.rb: -------------------------------------------------------------------------------- 1 | # Simple log file specification, used to test log parser. 2 | class TestingFormat < RequestLogAnalyzer::FileFormat::Base 3 | format_definition.first do |line| 4 | line.header = true 5 | line.teaser = /processing / 6 | line.regexp = /processing request (\d+)/ 7 | line.captures = [{ name: :request_no, type: :integer }] 8 | end 9 | 10 | format_definition.test do |line| 11 | line.teaser = /testing / 12 | line.regexp = /testing is (\w+)(?: in (\d+\.\d+)ms)?/ 13 | line.captures = [{ name: :test_capture, type: :test_type }, 14 | { name: :duration, type: :duration, unit: :msec }] 15 | end 16 | 17 | format_definition.eval do |line| 18 | line.regexp = /evaluation (\{.*\})/ 19 | line.captures = [{ name: :evaluated, type: :eval, provides: { greating: :string, what: :string } }] 20 | end 21 | 22 | format_definition.last do |line| 23 | line.footer = true 24 | line.teaser = /finishing / 25 | line.regexp = /finishing request (\d+)/ 26 | line.captures = [{ name: :request_no, type: :integer }] 27 | end 28 | 29 | format_definition.combined do |line| 30 | line.header = true 31 | line.footer = true 32 | line.regexp = /this is a header and footer line/ 33 | end 34 | 35 | report do |analyze| 36 | analyze.frequency :test_capture, title: 'What is testing exactly?' 37 | end 38 | 39 | class Request < RequestLogAnalyzer::Request 40 | def convert_test_type(value, _definition) 41 | "Testing is #{value}" 42 | end 43 | end 44 | end 45 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/filter/field.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Filter 2 | # Filter to select or reject a specific field 3 | # Options 4 | # * :mode :reject or :accept. 5 | # * :field Specific field to accept or reject. 6 | # * :value Value that the field should match to be accepted or rejected. 7 | class Field < Base 8 | attr_reader :field, :value, :mode 9 | 10 | def initialize(file_format, options = {}) 11 | super(file_format, options) 12 | setup_filter 13 | end 14 | 15 | # Setup mode, field and value. 16 | def setup_filter 17 | @mode = (@options[:mode] || :accept).to_sym 18 | @field = @options[:field].to_sym 19 | 20 | # Convert the timestamp to the correct formats for quick timestamp comparisons 21 | if @options[:value].is_a?(String) && @options[:value][0, 1] == '/' && @options[:value][-1, 1] == '/' 22 | @value = Regexp.new(@options[:value][1..-2]) 23 | else 24 | @value = @options[:value] # TODO: convert value? 25 | end 26 | end 27 | 28 | # Keep request if @mode == :select and request has the field and value. 29 | # Drop request if @mode == :reject and request has the field and value. 30 | # Returns nil otherwise. 31 | # request Request Object 32 | def filter(request) 33 | found_field = request.every(@field).any? { |value| @value === value.to_s } 34 | return nil if !found_field && @mode == :select 35 | return nil if found_field && @mode == :reject 36 | request 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/delayed_job2.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The DelayedJob2 file format parsed log files that are created by DelayedJob 2.0. 3 | # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log 4 | class DelayedJob2 < Base 5 | extend CommonRegularExpressions 6 | 7 | line_definition :job_lock do |line| 8 | line.header = true 9 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/ 10 | 11 | line.capture(:timestamp).as(:timestamp) 12 | line.capture(:host) 13 | line.capture(:pid).as(:integer) 14 | line.capture(:job) 15 | end 16 | 17 | line_definition :job_completed do |line| 18 | line.footer = true 19 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \* \[JOB\] \S+ host:(#{hostname_or_ip_address}) pid:(\d+) completed after (\d+\.\d+)/ 20 | line.capture(:timestamp).as(:timestamp) 21 | line.capture(:host) 22 | line.capture(:pid).as(:integer) 23 | line.capture(:duration).as(:duration, unit: :sec) 24 | end 25 | 26 | report do |analyze| 27 | analyze.timespan 28 | analyze.hourly_spread 29 | 30 | analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs' 31 | # analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs" 32 | 33 | analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration' 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/unit/controller/controller_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Controller do 4 | 5 | it 'should use a custom output generator correctly' do 6 | 7 | mock_output = double('RequestLogAnalyzer::Output::Base') 8 | mock_output.stub(:io).and_return(mock_io) 9 | mock_output.should_receive(:header) 10 | mock_output.should_receive(:footer) 11 | 12 | controller = RequestLogAnalyzer::Controller.new(mock_source, output: mock_output) 13 | 14 | controller.run! 15 | end 16 | 17 | it 'should call aggregators correctly when run' do 18 | controller = RequestLogAnalyzer::Controller.new(mock_source, output: mock_output) 19 | 20 | mock_aggregator = double('RequestLogAnalyzer::Aggregator::Base') 21 | mock_aggregator.should_receive(:prepare).once.ordered 22 | mock_aggregator.should_receive(:aggregate).with(an_instance_of(testing_format.request_class)).twice.ordered 23 | mock_aggregator.should_receive(:finalize).once.ordered 24 | mock_aggregator.should_receive(:report).once.ordered 25 | 26 | controller.aggregators << mock_aggregator 27 | controller.run! 28 | end 29 | 30 | it 'should call filters when run' do 31 | controller = RequestLogAnalyzer::Controller.new(mock_source, output: mock_output) 32 | 33 | mock_filter = double('RequestLogAnalyzer::Filter::Base') 34 | mock_filter.should_receive(:filter).twice.and_return(nil) 35 | controller.should_receive(:aggregate_request).twice.and_return(nil) 36 | 37 | controller.filters << mock_filter 38 | controller.run! 39 | end 40 | 41 | end 42 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/filter/timespan.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Filter 2 | # Reject all requests not in given timespan 3 | # Options 4 | # * :after Only keep requests after this DateTime. 5 | # * :before Only keep requests before this DateTime. 6 | class Timespan < Base 7 | attr_reader :before, :after 8 | 9 | def initialize(file_format, options = {}) 10 | @after = nil 11 | @before = nil 12 | super(file_format, options) 13 | setup_filter 14 | end 15 | 16 | # Convert the timestamp to the correct formats for quick timestamp comparisons. 17 | # These are stored in the before and after attr_reader fields. 18 | def setup_filter 19 | @after = @options[:after].strftime('%Y%m%d%H%M%S').to_i if options[:after] 20 | @before = @options[:before].strftime('%Y%m%d%H%M%S').to_i if options[:before] 21 | end 22 | 23 | # Returns request if: 24 | # * @after <= request.timestamp <= @before 25 | # * @after <= request.timestamp 26 | # * request.timestamp <= @before 27 | # Returns nil otherwise 28 | # request Request object. 29 | def filter(request) 30 | if @after && @before && request.timestamp && request.timestamp <= @before && @after <= request.timestamp 31 | return request 32 | elsif @after && @before.nil? && request.timestamp && @after <= request.timestamp 33 | return request 34 | elsif @before && @after.nil? && request.timestamp && request.timestamp <= @before 35 | return request 36 | end 37 | 38 | nil 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /spec/fixtures/oink_22_failure.log: -------------------------------------------------------------------------------- 1 | Jun 18 11:25:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:25:37) [GET] 2 | Jun 18 11:25:40 derek rails[67783]: NoMethodError (undefined method `update_domain_account' for nil:NilClass): 3 | 4 | Jun 18 11:27:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:27:37) [GET] 5 | Jun 18 11:27:37 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016 6 | Jun 18 11:27:37 derek rails[67783]: Parameters: {"id"=>"5171"} 7 | Jun 18 11:27:37 derek rails[67783]: Memory usage: 400000 | PID: 67783 8 | Jun 18 11:27:37 derek rails[67783]: Completed in 207ms (View: 0, DB: 258) | 200 OK [http://localhost/file_records/inline/5171] 9 | 10 | Jun 18 11:30:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:27:37) [GET] 11 | Jun 18 11:30:40 derek rails[67783]: NoMethodError (undefined method `update_domain_account' for nil:NilClass): 12 | 13 | Jun 18 11:31:19 derek rails[67783]: Processing InnovationsController#innovation_param (for 127.0.0.1 at 2010-06-18 11:31:19) [GET] 14 | Jun 18 11:31:19 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016 15 | Jun 18 11:31:19 derek rails[67783]: Parameters: {"organization_param"=>"harvard", "innovation_param"=>"potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti"} 16 | Jun 18 11:31:31 derek rails[67783]: Memory usage: 430000 | PID: 67783 17 | Jun 18 11:31:31 derek rails[67783]: Completed in 11922ms (View: 200, DB: 11673) | 200 OK [http://localhost/harvard/potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti] -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/delayed_job21.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The DelayedJob21 file format parsed log files that are created by DelayedJob 2.1 or higher. 3 | # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log 4 | class DelayedJob21 < Base 5 | extend CommonRegularExpressions 6 | 7 | line_definition :job_lock do |line| 8 | line.header = true 9 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] acquired lock on (\S+)/ 10 | 11 | line.capture(:timestamp).as(:timestamp) 12 | line.capture(:host) 13 | line.capture(:pid).as(:integer) 14 | line.capture(:job) 15 | end 16 | 17 | line_definition :job_completed do |line| 18 | line.footer = true 19 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/ 20 | line.capture(:timestamp).as(:timestamp) 21 | line.capture(:host) 22 | line.capture(:pid).as(:integer) 23 | line.capture(:job) 24 | line.capture(:duration).as(:duration, unit: :sec) 25 | end 26 | 27 | report do |analyze| 28 | analyze.timespan 29 | analyze.hourly_spread 30 | 31 | analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs' 32 | # analyze.frequency :job, :if => lambda { |request| request[:attempts] == 1 }, :title => "Failed jobs" 33 | 34 | analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration' 35 | end 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/aggregator.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Aggregator 2 | # The base class of an aggregator. This class provides the interface to which 3 | # every aggregator should comply (by simply subclassing this class). 4 | class Base 5 | attr_reader :options, :source 6 | 7 | # Intializes a new RequestLogAnalyzer::Aggregator::Base instance 8 | # It will include the specific file format module. 9 | def initialize(source, options = {}) 10 | @source = source 11 | @options = options 12 | end 13 | 14 | # The prepare function is called just before parsing starts. This function 15 | # can be used to initialie variables, etc. 16 | def prepare 17 | end 18 | 19 | # The aggregate function is called for every request. 20 | # Implement the aggregating functionality in this method 21 | def aggregate(_request) 22 | end 23 | 24 | # The finalize function is called after all sources are parsed and no more 25 | # requests will be passed to the aggregator 26 | def finalize 27 | end 28 | 29 | # The warning method is called if the parser eits a warning. 30 | def warning(_type, _message, _lineno) 31 | end 32 | 33 | # The report function is called at the end. Implement any result reporting 34 | # in this function. 35 | def report(_output) 36 | end 37 | 38 | # The source_change function gets called when handling a source is started or finished. 39 | def source_change(_change, _filename) 40 | end 41 | end 42 | end 43 | 44 | require 'request_log_analyzer/aggregator/echo' 45 | require 'request_log_analyzer/aggregator/summarizer' 46 | require 'request_log_analyzer/aggregator/database_inserter' 47 | -------------------------------------------------------------------------------- /spec/unit/database/connection_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Database::Connection do 4 | describe '.from_string' do 5 | 6 | it 'should parse a name-value based string' do 7 | string = 'adapter=sqlite3;database=filename.db' 8 | RequestLogAnalyzer::Database::Connection.from_string(string).should == { adapter: 'sqlite3', database: 'filename.db' } 9 | end 10 | 11 | it 'should parse an URI-based string for SQLite3' do 12 | string = 'sqlite3://filename.db' 13 | RequestLogAnalyzer::Database::Connection.from_string(string).should == { adapter: 'sqlite3', database: 'filename.db' } 14 | end 15 | 16 | it 'should parse an URI-based string for MySQL' do 17 | string = 'mysql://localhost.local/database' 18 | RequestLogAnalyzer::Database::Connection.from_string(string).should == 19 | { adapter: 'mysql', database: 'database', host: 'localhost.local' } 20 | end 21 | 22 | it 'should parse an URI-based string for MySQL with only username' do 23 | string = 'mysql://username@localhost.local/database' 24 | RequestLogAnalyzer::Database::Connection.from_string(string).should == 25 | { adapter: 'mysql', database: 'database', host: 'localhost.local', username: 'username' } 26 | end 27 | 28 | it 'should parse an URI-based string for MySQL with username and password' do 29 | string = 'mysql://username:password@localhost.local/database' 30 | RequestLogAnalyzer::Database::Connection.from_string(string).should == 31 | { adapter: 'mysql', database: 'database', host: 'localhost.local', username: 'username', password: 'password' } 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/database/connection.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Database::Connection 2 | def self.from_string(string) 3 | hash = {} 4 | if string =~ /^(?:\w+=(?:[^;])*;)*\w+=(?:[^;])*$/ 5 | string.scan(/(\w+)=([^;]*);?/) { |variable, value| hash[variable.to_sym] = value } 6 | elsif string =~ /^(\w+)\:\/\/(?:(?:([^:]+)(?:\:([^:]+))?\@)?([\w\.-]+)\/)?([\w\:\-\.\/]+)$/ 7 | hash[:adapter], hash[:username], hash[:password], hash[:host], hash[:database] = Regexp.last_match[1], Regexp.last_match[2], Regexp.last_match[3], Regexp.last_match[4], Regexp.last_match[5] 8 | hash.delete_if { |_k, v| v.nil? } 9 | end 10 | hash.empty? ? nil : hash 11 | end 12 | 13 | def connect(connection_identifier) 14 | if connection_identifier.is_a?(Hash) 15 | ActiveRecord::Base.establish_connection(connection_identifier) 16 | elsif connection_identifier == ':memory:' 17 | ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: ':memory:') 18 | elsif connection_hash = RequestLogAnalyzer::Database::Connection.from_string(connection_identifier) 19 | ActiveRecord::Base.establish_connection(connection_hash) 20 | elsif connection_identifier.is_a?(String) # Normal SQLite 3 database file 21 | ActiveRecord::Base.establish_connection(adapter: 'sqlite3', database: connection_identifier) 22 | elsif connection_identifier.nil? 23 | nil 24 | else 25 | fail "Cannot connect with this connection_identifier: #{connection_identifier.inspect}" 26 | end 27 | end 28 | 29 | def disconnect 30 | RequestLogAnalyzer::Database::Base.remove_connection 31 | end 32 | 33 | def connection 34 | RequestLogAnalyzer::Database::Base.connection 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /spec/fixtures/oink_22.log: -------------------------------------------------------------------------------- 1 | Jun 18 11:27:37 derek rails[67783]: Processing FileRecordsController#inline (for 127.0.0.1 at 2010-06-18 11:27:37) [GET] 2 | Jun 18 11:27:37 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016 3 | Jun 18 11:27:37 derek rails[67783]: Parameters: {"id"=>"5171"} 4 | Jun 18 11:27:37 derek rails[67783]: Memory usage: 400000 | PID: 67783 5 | Jun 18 11:27:37 derek rails[67783]: Completed in 207ms (View: 0, DB: 258) | 200 OK [http://localhost/file_records/inline/5171] 6 | 7 | Jun 18 11:28:11 derek rails[700]: Processing InfoController#about (for 127.0.0.1 at 2010-06-18 11:28:11) [GET] 8 | Jun 18 11:28:12 derek rails[700]: Memory usage: 700000 | PID: 67783 9 | Jun 18 11:28:12 derek rails[700]: Completed in 379ms (View: 89, DB: 258) | 200 OK [http://localhost/info/about] 10 | 11 | Jun 18 11:28:11 derek rails[700]: Processing InfoController#board (for 127.0.0.1 at 2010-06-18 11:28:11) [GET] 12 | Jun 18 11:28:12 derek rails[700]: Memory usage: 750000 | PID: 67783 13 | Jun 18 11:28:12 derek rails[700]: Completed in 379ms (View: 89, DB: 258) | 200 OK [http://localhost/info/board] 14 | 15 | Jun 18 11:31:19 derek rails[67783]: Processing InnovationsController#innovation_param (for 127.0.0.1 at 2010-06-18 11:31:19) [GET] 16 | Jun 18 11:31:19 derek rails[67783]: Session ID: 8abbd5bce47cfc8bd438c647f5a9e016 17 | Jun 18 11:31:19 derek rails[67783]: Parameters: {"organization_param"=>"harvard", "innovation_param"=>"potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti"} 18 | Jun 18 11:31:31 derek rails[67783]: Memory usage: 430000 | PID: 67783 19 | Jun 18 11:31:31 derek rails[67783]: Completed in 11922ms (View: 200, DB: 11673) | 200 OK [http://localhost/harvard/potent-ogt-inhibitors-for-the-treatment-of-cancer-and-diabeti] -------------------------------------------------------------------------------- /spec/integration/munin_plugins_rails_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer, 'when harvesting like munin-plugins-rails the YAML output' do 4 | 5 | before(:each) do 6 | cleanup_temp_files! 7 | run("#{log_fixture(:rails_1x)} --dump #{temp_output_file(:yaml)}") 8 | @rla = YAML.load(File.read(temp_output_file(:yaml))) 9 | end 10 | 11 | after(:each) do 12 | cleanup_temp_files! 13 | end 14 | 15 | it 'should contain database times' do 16 | @rla['Database time'].each do |item| 17 | item[1][:min].should_not be_nil 18 | item[1][:max].should_not be_nil 19 | item[1][:hits].should_not be_nil 20 | item[1][:sum].should_not be_nil 21 | end 22 | end 23 | 24 | it 'should contain request times' do 25 | @rla['Request duration'].each do |item| 26 | item[1][:min].should_not be_nil 27 | item[1][:max].should_not be_nil 28 | item[1][:hits].should_not be_nil 29 | item[1][:sum].should_not be_nil 30 | end 31 | end 32 | 33 | it 'should contain failed requests' do 34 | @rla.keys.should include('Failed requests') 35 | end 36 | 37 | it 'should contain Process blockers' do 38 | @rla.keys.should include('Process blockers (> 1 sec duration)') 39 | end 40 | 41 | it 'should contain HTTP Methods' do 42 | @rla['HTTP methods']['GET'].should_not be_nil 43 | end 44 | 45 | it 'should contain HTTP Methods' do 46 | @rla['HTTP methods']['GET'].should_not be_nil 47 | end 48 | 49 | it 'should contain view rendering times' do 50 | @rla['View rendering time'].each do |item| 51 | item[1][:min].should_not be_nil 52 | item[1][:max].should_not be_nil 53 | item[1][:hits].should_not be_nil 54 | item[1][:sum].should_not be_nil 55 | end 56 | end 57 | 58 | end 59 | -------------------------------------------------------------------------------- /lib/cli/database_console_init.rb: -------------------------------------------------------------------------------- 1 | # Setup the include path 2 | $LOAD_PATH.unshift(File.expand_path('..', File.dirname(__FILE__))) 3 | require 'request_log_analyzer' 4 | require 'request_log_analyzer/database' 5 | 6 | $database = RequestLogAnalyzer::Database.new(ENV['RLA_DBCONSOLE_DATABASE']) 7 | $database.load_database_schema! 8 | $database.register_default_orm_classes! 9 | 10 | require 'cli/tools' 11 | 12 | def wordwrap(string, max = 80, indent = '') 13 | strings = [''] 14 | string.split(', ').each do |item| 15 | if strings.last.length == 0 || strings.last.length + item.length <= max 16 | strings.last << item << ', ' 17 | else 18 | strings << (item + ', ') 19 | end 20 | end 21 | strings.map(&:strip).join("\n#{indent}").slice(0..-2) 22 | end 23 | 24 | class Request 25 | def inspect 26 | request_inspect = "Request[id: #{id}]" 27 | request_inspect << " <#{lines.first.source.filename}>" if lines.first.source 28 | 29 | inspected_lines = lines.map do |line| 30 | inspect_line = " - #{line.line_type} (line #{line.lineno})" 31 | if (inspect_attributes = line.attributes.reject { |(k, _v)| [:id, :source_id, :request_id, :lineno].include?(k.to_sym) }).any? 32 | inspect_attributes = inspect_attributes.map { |(k, v)| "#{k} = #{v.inspect}" }.join(', ') 33 | inspect_line << "\n " + wordwrap(inspect_attributes, CommandLine::Tools.terminal_width - 6, ' ') 34 | end 35 | inspect_line 36 | end 37 | 38 | request_inspect << "\n" << inspected_lines.join("\n") << "\n\n" 39 | end 40 | end 41 | 42 | puts 'request-log-analyzer database console' 43 | puts '-------------------------------------' 44 | puts 'The following ActiveRecord classes are available:' 45 | puts $database.orm_classes.map { |k| k.name.split('::').last }.join(', ') 46 | -------------------------------------------------------------------------------- /spec/unit/file_format/format_autodetection_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat do 4 | 5 | describe '.autodetect' do 6 | it 'should autodetect a Merb log' do 7 | file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:merb)) 8 | file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Merb) 9 | end 10 | 11 | it 'should autodetect a MySQL slow query log' do 12 | file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:mysql_slow_query)) 13 | file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Mysql) 14 | end 15 | 16 | it 'should autodetect a Rails 1.x log' do 17 | file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_1x)) 18 | file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Rails) 19 | end 20 | 21 | it 'should autodetect a Rails 2.x log' do 22 | file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:rails_22)) 23 | file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::RailsDevelopment) 24 | end 25 | 26 | it 'should autodetect an Apache access log' do 27 | file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:apache_common)) 28 | file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Apache) 29 | end 30 | 31 | it 'should autodetect a Rack access log' do 32 | file_format = RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:sinatra)) 33 | file_format.should be_instance_of(RequestLogAnalyzer::FileFormat::Rack) 34 | end 35 | 36 | it 'should not find any file format with a bogus file' do 37 | RequestLogAnalyzer::FileFormat.autodetect(log_fixture(:test_order)).should be_nil 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/cli/tools.rb: -------------------------------------------------------------------------------- 1 | module CommandLine 2 | module Tools 3 | extend self 4 | 5 | # Try to determine the terminal with. 6 | # If it is not possible to to so, it returns the default_width. 7 | # default_width Defaults to 81 8 | def terminal_width(default_width = 81, out = STDOUT) 9 | tiocgwinsz = 0x5413 10 | data = [0, 0, 0, 0].pack('SSSS') 11 | if !RUBY_PLATFORM.include?('java') && out.ioctl(tiocgwinsz, data) >= 0 # JRuby crashes on ioctl 12 | _, cols, _, _ = data.unpack('SSSS') 13 | fail unless cols > 0 14 | cols 15 | else 16 | fail 17 | end 18 | rescue 19 | begin 20 | IO.popen('stty -a 2>&1') do |pipe| 21 | column_line = pipe.find { |line| /(\d+) columns/ =~ line } 22 | fail unless column_line 23 | Regexp.last_match[1].to_i 24 | end 25 | rescue 26 | default_width 27 | end 28 | end 29 | 30 | # Copies request-log-analyzer analyzer rake tasks into the /lib/tasks folder of a project, for easy access and 31 | # environment integration. 32 | # install_type Type of project to install into. Defaults to :rails. 33 | # Raises if it cannot find the project folder or if the install_type is now known. 34 | def install_rake_tasks(install_type = :rails) 35 | if install_type.to_sym == :rails 36 | require 'fileutils' 37 | if File.directory?('./lib/tasks/') 38 | task_file = File.expand_path('../../tasks/request_log_analyzer.rake', File.dirname(__FILE__)) 39 | FileUtils.copy(task_file, './lib/tasks/request_log_analyze.rake') 40 | puts 'Installed rake tasks.' 41 | puts 'To use, run: rake rla:report' 42 | else 43 | puts 'Cannot find /lib/tasks folder. Are you in your Rails directory?' 44 | puts 'Installation aborted.' 45 | end 46 | else 47 | fail "Cannot perform this install type! (#{install_type})" 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/delayed_job.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The DelayedJob file format parsed log files that are created by DelayedJob. 3 | # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log 4 | class DelayedJob < Base 5 | line_definition :job_lock do |line| 6 | line.header = true 7 | line.regexp = /\* \[JOB\] acquiring lock on (\S+)/ 8 | 9 | line.capture(:job) 10 | end 11 | 12 | line_definition :job_completed do |line| 13 | line.footer = true 14 | line.regexp = /\* \[JOB\] (\S+) completed after (\d+\.\d+)/ 15 | 16 | line.capture(:completed_job) 17 | line.capture(:duration).as(:duration, unit: :sec) 18 | end 19 | 20 | line_definition :job_failed do |line| 21 | line.footer = true 22 | line.regexp = /\* \[JOB\] (\S+) failed with (\S+)\: .* - (\d+) failed attempts/ 23 | 24 | line.capture(:failed_job) 25 | line.capture(:exception) 26 | line.capture(:attempts).as(:integer) 27 | end 28 | 29 | line_definition :job_lock_failed do |line| 30 | line.footer = true 31 | line.regexp = /\* \[JOB\] failed to acquire exclusive lock for (\S+)/ 32 | 33 | line.capture(:locked_job) 34 | end 35 | 36 | # line_definition :batch_completed do |line| 37 | # line.header = true 38 | # line.footer = true 39 | # line.regexp = /(\d+) jobs processed at (\d+\.\d+) j\/s, (\d+) failed .../ 40 | # 41 | # line.capture(:total_amount).as(:integer) 42 | # line.capture(:mean_duration).as(:duration, :unit => :sec) 43 | # line.capture(:failed_amount).as(:integer) 44 | # end 45 | 46 | report do |analyze| 47 | analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs' 48 | analyze.frequency :job, if: lambda { |request| request[:attempts] == 1 }, title: 'Failed jobs' 49 | 50 | analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration' 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/tracker/traffic.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Tracker 2 | # Analyze the average and total traffic of requests 3 | # 4 | # === Options 5 | # * :category Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER) 6 | # * :traffic The field containing the duration in the request hash. 7 | # * :if Proc that has to return !nil for a request to be passed to the tracker. 8 | # * :line_type The line type that contains the duration field (determined by the category proc). 9 | # * :title Title do be displayed above the report 10 | # * :unless Handle request if this proc is false for the handled request. 11 | class Traffic < NumericValue 12 | # Check if duration and catagory option have been received, 13 | def prepare 14 | options[:value] = options[:traffic] if options[:traffic] 15 | options[:total] = true 16 | super 17 | 18 | @number_of_buckets = options[:number_of_buckets] || 1000 19 | @min_bucket_value = options[:min_bucket_value] ? options[:min_bucket_value].to_f : 1 20 | @max_bucket_value = options[:max_bucket_value] ? options[:max_bucket_value].to_f : 1_000_000_000_000 21 | 22 | # precalculate the bucket size 23 | @bucket_size = (Math.log(@max_bucket_value) - Math.log(@min_bucket_value)) / @number_of_buckets.to_f 24 | end 25 | 26 | # Formats the traffic number using x B/kB/MB/GB etc notation 27 | def display_value(bytes) 28 | return '-' if bytes.nil? 29 | return '0 B' if bytes.zero? 30 | 31 | case [Math.log10(bytes.abs).floor, 0].max 32 | when 0...4 then '%d B' % bytes 33 | when 4...7 then '%d kB' % (bytes / 1000) 34 | when 7...10 then '%d MB' % (bytes / 1_000_000) 35 | when 10...13 then '%d GB' % (bytes / 1_000_000_000) 36 | else '%d TB' % (bytes / 1_000_000_000_000) 37 | end 38 | end 39 | 40 | # Returns the title of this tracker for reports 41 | def title 42 | options[:title] || 'Request traffic' 43 | end 44 | end 45 | end 46 | -------------------------------------------------------------------------------- /DESIGN.rdoc: -------------------------------------------------------------------------------- 1 | === Request-log-analyzer 2 | RLA is set up like a simple pipe and filter system. 3 | 4 | This allows you to easily add extra reports, filters and outputs. 5 | -> Aggregator (database) 6 | Source -> Filter -> Filter -> Aggregator (summary report) -> Output 7 | -> Aggregator (...) 8 | 9 | When the pipeline has been constructed, we Start chunk producer (source) and push requests through pipeline. 10 | 11 | Controller.start 12 | 13 | === Source 14 | RequestLogAnalyzer::Source is an Object that pushes requests into the chain. 15 | At the moment you can only use the log-parser as a source. 16 | It accepts files or stdin and can parse then into request objects using a RequestLogAnalyzer::FileFormat definition. 17 | In the future we want to be able to have a generated request database as source as this will make interactive 18 | down drilling possible. 19 | 20 | === Filter 21 | The filters are all subclasses of the RequestLogAnalyzer::Filter class. 22 | They accept a request object, manipulate or drop it, and then pass the request object on to the next filter 23 | in the chain. 24 | At the moment there are three types of filters available: Anonymize, Field and Timespan. 25 | 26 | === Aggregator 27 | The Aggregators all inherit from the RequestLogAnalyzer::Aggregator class. 28 | All the requests that come out of the Filterchain are fed into all the aggregators in parallel. 29 | These aggregators can do anything what they want with the given request. 30 | For example: the Database aggregator will just store all the requests into a SQLite database while the Summarizer will 31 | generate a wide range of statistical reports from them. 32 | 33 | === Running the pipeline 34 | All Aggregators are asked to report what they have done. For example the database will report: I stuffed x requests 35 | into SQLite database Y. The Summarizer will output its reports. 36 | 37 | Controller.report 38 | 39 | The output is pushed to a RequestLogAnalyzer::Output object, which takes care of the output. 40 | It can generate either ASCII, UTF8 or even HTML output. 41 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/mailer.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer 2 | # Mail report to a specified emailaddress 3 | class Mailer 4 | attr_accessor :data, :to, :host, :port 5 | 6 | # Initialize a mailer 7 | # to to email address to mail to 8 | # host the mailer host (defaults to localhost) 9 | # options Specific style options 10 | # 11 | # Options 12 | # * :debug Do not actually mail 13 | # * :from_alias The from alias 14 | # * :to_alias The to alias 15 | # * :subject The message subject 16 | def initialize(to, host = 'localhost', options = {}) 17 | require 'net/smtp' 18 | @to = to 19 | @host = host 20 | 21 | @port = 25 22 | @options = options 23 | @host, @port = host.split(':') if @host.include?(':') 24 | @data = [] 25 | end 26 | 27 | # Send all data in @data to the email address used during initialization. 28 | # Returns array containg [message_data, from_email_address, to_email_address] of sent email. 29 | def mail 30 | from = @options[:from] || 'contact@railsdoctors.com' 31 | from_alias = @options[:from_alias] || 'Request-log-analyzer reporter' 32 | to_alias = @options[:to_alias] || to 33 | subject = @options[:subject] || "Request log analyzer report - generated on #{Time.now}" 34 | content_type = '' 35 | content_type = 'Content-Type: text/html; charset="ISO-8859-1";' if @data.map { |l| l.include?('html') }.include?(true) 36 | msg = < 38 | To: #{to_alias} <#{@to}> 39 | Subject: #{subject} 40 | #{content_type} 41 | 42 | #{@data.join("\n")} 43 | END_OF_MESSAGE 44 | 45 | unless @options[:debug] 46 | Net::SMTP.start(@host, @port) do |smtp| 47 | smtp.send_message msg, from, to 48 | end 49 | end 50 | 51 | [msg, from, to] 52 | end 53 | 54 | def <<(string) 55 | data << string 56 | end 57 | 58 | def puts(string) 59 | data << string 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /spec/unit/file_format/w3c_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::W3c do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:w3c) } 6 | 7 | it { should be_well_formed } 8 | it do 9 | should have_line_definition(:access).capturing(:timestamp, :remote_ip, :username, :local_ip, :port, 10 | :method, :path, :http_status, :bytes_sent, :bytes_received, :duration, :user_agent, :referer) 11 | end 12 | 13 | it { should satisfy { |ff| ff.report_trackers.length == 10 } } 14 | 15 | let(:sample1) { '2002-05-24 20:18:01 172.224.24.114 - 206.73.118.24 80 GET /Default.htm - 200 7930 248 31 Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server) http://64.224.24.114/' } 16 | let(:irrelevant) { '#Software: Microsoft Internet Information Services 6.0' } 17 | 18 | describe '#parse_line' do 19 | it do 20 | should parse_line(sample1, 'an access line').and_capture( 21 | timestamp: 20_020_524_201_801, 22 | remote_ip: '172.224.24.114', 23 | username: nil, 24 | local_ip: '206.73.118.24', 25 | port: 80, 26 | method: 'GET', 27 | path: '/Default.htm', 28 | http_status: 200, 29 | bytes_sent: 7930, 30 | bytes_received: 248, 31 | duration: 0.031, 32 | user_agent: 'Mozilla/4.0+(compatible;+MSIE+5.01;+Windows+2000+Server)', 33 | referer: 'http://64.224.24.114/') 34 | end 35 | 36 | it { should_not parse_line(irrelevant, 'an irrelevant line') } 37 | it { should_not parse_line('nonsense', 'a nonsense line') } 38 | end 39 | 40 | describe '#parse_io' do 41 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 42 | let(:snippet) { log_snippet(irrelevant, sample1, sample1) } 43 | 44 | it 'should parse a snippet successully without warnings' do 45 | log_parser.should_receive(:handle_request).twice 46 | log_parser.should_not_receive(:warn) 47 | log_parser.parse_io(snippet) 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/w3c.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # FileFormat for W3C access logs. 3 | class W3c < Base 4 | extend CommonRegularExpressions 5 | 6 | line_definition :access do |line| 7 | line.header = true 8 | line.footer = true 9 | line.regexp = /^(#{timestamp('%Y-%m-%d %H:%M:%S')}) (#{ip_address}) (.*) (#{ip_address}) (\d+) (\w+) (\S+) \- (\d+) (\d+) (\d+) (\d+) (.*) (\S+)/ 10 | 11 | line.capture(:timestamp).as(:timestamp) 12 | line.capture(:remote_ip) 13 | line.capture(:username).as(:nillable_string) 14 | line.capture(:local_ip) 15 | line.capture(:port).as(:integer) 16 | line.capture(:method) 17 | line.capture(:path).as(:path) 18 | line.capture(:http_status).as(:integer) 19 | line.capture(:bytes_sent).as(:traffic, unit: :byte) 20 | line.capture(:bytes_received).as(:traffic, unit: :byte) 21 | line.capture(:duration).as(:duration, unit: :msec) 22 | line.capture(:user_agent) 23 | line.capture(:referer) 24 | end 25 | 26 | report do |analyze| 27 | analyze.timespan 28 | analyze.hourly_spread 29 | 30 | analyze.frequency category: :http_method, title: 'HTTP methods' 31 | analyze.frequency category: :http_status, title: 'HTTP statuses' 32 | 33 | analyze.frequency category: :path, title: 'Most popular URIs' 34 | 35 | analyze.frequency category: :user_agent, title: 'User agents' 36 | analyze.frequency category: :referer, title: 'Referers' 37 | 38 | analyze.duration duration: :duration, category: :path, title: 'Request duration' 39 | analyze.traffic traffic: :bytes_sent, category: :path, title: 'Traffic out' 40 | analyze.traffic traffic: :bytes_received, category: :path, title: 'Traffic in' 41 | end 42 | 43 | class Request < RequestLogAnalyzer::Request 44 | # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer 45 | # to speed up parsing. 46 | def convert_timestamp(value, _definition) 47 | "#{value[0, 4]}#{value[5, 2]}#{value[8, 2]}#{value[11, 2]}#{value[14, 2]}#{value[17, 2]}".to_i 48 | end 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/unit/filter/timespan_filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Filter::Timespan, 'both before and after' do 4 | 5 | before(:each) do 6 | @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, after: DateTime.parse('2009-01-01'), before: DateTime.parse('2009-02-02')) 7 | end 8 | 9 | it 'should reject a request before the after date' do 10 | @filter.filter(request(timestamp: 20_081_212_000_000)).should be_nil 11 | end 12 | 13 | it 'should reject a request after the before date' do 14 | @filter.filter(request(timestamp: 20_090_303_000_000)).should be_nil 15 | end 16 | 17 | it 'should accept a request between the after and before dates' do 18 | @filter.filter(request(timestamp: 20_090_102_000_000)).should_not be_nil 19 | end 20 | end 21 | 22 | describe RequestLogAnalyzer::Filter::Timespan, 'only before' do 23 | 24 | before(:each) do 25 | @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, before: DateTime.parse('2009-02-02')) 26 | end 27 | 28 | it 'should accept a request before the after date' do 29 | @filter.filter(request(timestamp: 20_081_212_000_000)).should_not be_nil 30 | end 31 | 32 | it 'should reject a request after the before date' do 33 | @filter.filter(request(timestamp: 20_090_303_000_000)).should be_nil 34 | end 35 | 36 | it 'should accept a request between the after and before dates' do 37 | @filter.filter(request(timestamp: 20_090_102_000_000)).should_not be_nil 38 | end 39 | end 40 | 41 | describe RequestLogAnalyzer::Filter::Timespan, 'only after' do 42 | 43 | before(:each) do 44 | @filter = RequestLogAnalyzer::Filter::Timespan.new(testing_format, after: DateTime.parse('2009-01-01')) 45 | end 46 | 47 | it 'should reject a request before the after date' do 48 | @filter.filter(request(timestamp: 20_081_212_000_000)).should be_nil 49 | end 50 | 51 | it 'should accept a request after the before date' do 52 | @filter.filter(request(timestamp: 20_090_303_000_000)).should_not be_nil 53 | end 54 | 55 | it 'should accept a request between the after and before dates' do 56 | @filter.filter(request(timestamp: 20_090_102_000_000)).should_not be_nil 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /spec/unit/file_format/merb_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::Merb do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:merb) } 6 | 7 | it { should be_well_formed } 8 | it { should have_line_definition(:started).capturing(:timestamp) } 9 | it { should have_line_definition(:params).capturing(:controller, :action, :namespace) } 10 | it { should have_line_definition(:completed).capturing(:dispatch_time, :before_filters_time, :action_time, :after_filters_time) } 11 | it { should satisfy { |ff| ff.report_trackers.length == 4 } } 12 | 13 | describe '#parse_line' do 14 | let(:started_sample) { '~ Started request handling: Fri Aug 29 11:10:23 +0200 2008' } 15 | let(:prefixed_started_sample) { '~ Aug 31 18:35:24 typekit-web001 merb: ~ Started request handling: Mon Aug 31 18:35:25 +0000 2009' } 16 | let(:params_sample) { '~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"delete", "controller"=>"session"}' } 17 | let(:completed_sample) { '~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833}' } 18 | 19 | it { should parse_line(started_sample, 'without prefix').as(:started).and_capture(timestamp: 20_080_829_111_023) } 20 | it { should parse_line(prefixed_started_sample, 'with prefix').as(:started).and_capture(timestamp: 20_090_831_183_525) } 21 | it { should parse_line(params_sample).as(:params).and_capture(controller: 'session', action: 'delete', namespace: nil) } 22 | it do 23 | should parse_line(completed_sample).as(:completed).and_capture(dispatch_time: 0.006117, 24 | before_filters_time: 0.000712, action_time: 0.005833, after_filters_time: 6.1e-05) 25 | end 26 | 27 | it { should_not parse_line('~ nonsense', 'a nonsense line') } 28 | end 29 | 30 | describe '#parse_io' do 31 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 32 | 33 | it 'should parse a log fragment correctly without warnings' do 34 | log_parser.should_receive(:handle_request).exactly(11).times 35 | log_parser.should_not_receive(:warn) 36 | log_parser.parse_file(log_fixture(:merb)) 37 | end 38 | end 39 | end 40 | -------------------------------------------------------------------------------- /spec/lib/helpers.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::RSpec::Helpers 2 | # Create or return a new TestingFormat 3 | def testing_format 4 | @testing_format ||= TestingFormat.create 5 | end 6 | 7 | # Load a log file from the fixture folder 8 | def log_fixture(name, extention = 'log') 9 | File.dirname(__FILE__) + "/../fixtures/#{name}.#{extention}" 10 | end 11 | 12 | # directory of logs 13 | def log_directory_fixture(name) 14 | File.dirname(__FILE__) + "/../fixtures/#{name}" 15 | end 16 | 17 | # Creates a log file given some lines 18 | def log_snippet(*lines) 19 | StringIO.new(lines.join("\n") << "\n") 20 | end 21 | 22 | # Request loopback 23 | def request(fields, format = testing_format) 24 | if fields.is_a?(Array) 25 | format.request(*fields) 26 | else 27 | format.request(fields) 28 | end 29 | end 30 | 31 | # Run a specific command 32 | # Used to call request-log-analyzer through binary 33 | def run(arguments) 34 | binary = "#{File.dirname(__FILE__)}/../../bin/request-log-analyzer" 35 | arguments = arguments.join(' ') if arguments.is_a?(Array) 36 | 37 | output = [] 38 | IO.popen("#{binary} #{arguments} 2>&1") do |pipe| 39 | output = pipe.readlines 40 | end 41 | $?.exitstatus.should == 0 42 | output 43 | end 44 | 45 | # Cleanup all temporary files generated by specs 46 | def cleanup_temp_files! 47 | Dir["#{File.dirname(__FILE__)}/../../tmp/spec.*tmp"].each do |file| 48 | File.unlink(file) 49 | end 50 | end 51 | 52 | # Return a filename that can be used as temporary file in specs 53 | def temp_output_file(file_type) 54 | File.expand_path("#{File.dirname(__FILE__)}/../../tmp/spec.#{file_type}.tmp") 55 | end 56 | 57 | # Check if a given string can be found in the given file 58 | # Returns the line number if found, nil otherwise 59 | def find_string_in_file(string, file, options = {}) 60 | return nil unless File.exist?(file) 61 | 62 | line_counter = 0 63 | 64 | File.open(file) do |io| 65 | io.each do|line| 66 | line_counter += 1 67 | line.chomp! 68 | 69 | p line if options[:debug] 70 | return line_counter if line.include? string 71 | end 72 | end 73 | 74 | nil 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /spec/lib/mocks.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::RSpec::Mocks 2 | def mock_source 3 | source = double('RequestLogAnalyzer::Source::Base') 4 | source.stub(:file_format).and_return(testing_format) 5 | source.stub(:parsed_requests).and_return(2) 6 | source.stub(:skipped_requests).and_return(1) 7 | source.stub(:parse_lines).and_return(10) 8 | 9 | source.stub(:warning=) 10 | source.stub(:progress=) 11 | source.stub(:source_changes=) 12 | 13 | source.stub(:prepare) 14 | source.stub(:finalize) 15 | 16 | source.stub(:each_request). 17 | and_yield(testing_format.request(field: 'value1')). 18 | and_yield(testing_format.request(field: 'value2')) 19 | 20 | source 21 | end 22 | 23 | def mock_io 24 | mio = double('IO') 25 | mio.stub(:print) 26 | mio.stub(:puts) 27 | mio.stub(:write) 28 | mio 29 | end 30 | 31 | def mock_output 32 | output = double('RequestLogAnalyzer::Output::Base') 33 | output.stub(:report_tracker) 34 | output.stub(:header) 35 | output.stub(:footer) 36 | output.stub(:puts) 37 | output.stub(:<<) 38 | output.stub(:colorize).and_return('Fancy text') 39 | output.stub(:link) 40 | output.stub(:title) 41 | output.stub(:line) 42 | output.stub(:with_style) 43 | output.stub(:table).and_yield([]) 44 | output.stub(:io).and_return(mock_io) 45 | output.stub(:options).and_return({}) 46 | output.stub(:slice_results) { |a| a } 47 | output 48 | end 49 | 50 | def mock_database(*stubs) 51 | database = double('RequestLogAnalyzer::Database') 52 | database.stub(:connect) 53 | database.stub(:disconnect) 54 | database.stub(:connection).and_return(mock_connection) 55 | stubs.each { |s| database.stub(s) } 56 | database 57 | end 58 | 59 | def mock_connection 60 | table_creator = double('ActiveRecord table creator') 61 | table_creator.stub(:column) 62 | 63 | connection = double('ActiveRecord::Base.connection') 64 | connection.stub(:add_index) 65 | connection.stub(:remove_index) 66 | connection.stub(:table_exists?).and_return(false) 67 | connection.stub(:create_table).and_yield(table_creator).and_return(true) 68 | connection.stub(:table_creator).and_return(table_creator) 69 | connection 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/delayed_job3.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The DelayedJob3 file format parsed log files that are created by DelayedJob 3.0 or higher. 3 | # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log 4 | class DelayedJob3 < Base 5 | extend CommonRegularExpressions 6 | 7 | line_definition :job_completed do |line| 8 | line.header = true 9 | line.footer = true 10 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (\S+) completed after (\d+\.\d+)/ 11 | line.capture(:timestamp).as(:timestamp) 12 | line.capture(:host) 13 | line.capture(:pid).as(:integer) 14 | line.capture(:job) 15 | line.capture(:duration).as(:duration, unit: :sec) 16 | end 17 | 18 | line_definition :job_failed do |line| 19 | line.header = true 20 | line.footer = true 21 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] (.+) - (\d+) failed attempts/ 22 | line.capture(:timestamp).as(:timestamp) 23 | line.capture(:host) 24 | line.capture(:pid).as(:integer) 25 | line.capture(:job) 26 | line.capture(:attempts).as(:integer) 27 | end 28 | 29 | line_definition :job_deleted do |line| 30 | line.header = true 31 | line.footer = true 32 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] PERMANENTLY removing (\S+) because of (\d+) consecutive failures./ 33 | line.capture(:timestamp).as(:timestamp) 34 | line.capture(:host) 35 | line.capture(:pid).as(:integer) 36 | line.capture(:job) 37 | line.capture(:failures).as(:integer) 38 | end 39 | 40 | report do |analyze| 41 | analyze.timespan 42 | analyze.hourly_spread 43 | 44 | analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs' 45 | analyze.frequency :job, category: :job, line_type: :job_failed, title: 'Failed jobs' 46 | analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs' 47 | analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration' 48 | end 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /lib/request_log_analyzer.rb: -------------------------------------------------------------------------------- 1 | require 'date' 2 | 3 | # RequestLogAnalyzer is the base namespace in which all functionality of RequestLogAnalyzer is implemented. 4 | # This module itself contains some functions to help with class and source file loading. The actual 5 | # application startup code resides in the {RequestLogAnalyzer::Controller} class. 6 | # 7 | # The {RequestLogAnalyzer::VERSION} constant can be used to determine what version of request-log-analyzer 8 | # is running. 9 | module RequestLogAnalyzer 10 | # Convert a string/symbol in camel case ({RequestLogAnalyzer::Controller}) to underscores 11 | # (request_log_analyzer/controller). This function can be used to load the file (using 12 | # require) in which the given constant is defined. 13 | # 14 | # @param [#to_s] str The string-like to convert in the following format: ModuleName::ClassName. 15 | # @return [String] The input string converted to underscore form. 16 | def self.to_underscore(str) 17 | str.to_s.gsub(/::/, '/').gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z\d])([A-Z])/, '\1_\2').tr('-', '_').downcase 18 | end 19 | 20 | # Convert a string/symbol in underscores (request_log_analyzer/controller) to camelcase 21 | # ({RequestLogAnalyzer::Controller}). This can be used to find the class that is defined in a given 22 | # filename. 23 | # 24 | # @param [#to_s] str The string-like to convert in the f`ollowing format: module_name/class_name. 25 | # @return [String] The input string converted to camelcase form. 26 | def self.to_camelcase(str) 27 | str.to_s.gsub(/\/(.?)/) { '::' + Regexp.last_match[1].upcase }.gsub(/(^|_)(.)/) { Regexp.last_match[2].upcase } 28 | end 29 | end 30 | 31 | require 'request_log_analyzer/version' 32 | require 'request_log_analyzer/controller' 33 | require 'request_log_analyzer/aggregator' 34 | require 'request_log_analyzer/class_level_inheritable_attributes' 35 | require 'request_log_analyzer/file_format' 36 | require 'request_log_analyzer/filter' 37 | require 'request_log_analyzer/line_definition' 38 | require 'request_log_analyzer/log_processor' 39 | require 'request_log_analyzer/mailer' 40 | require 'request_log_analyzer/output' 41 | require 'request_log_analyzer/request' 42 | require 'request_log_analyzer/source' 43 | require 'request_log_analyzer/tracker' 44 | -------------------------------------------------------------------------------- /spec/fixtures/s3_logs/2012-10-05-16-18-11-F9AAC5D1A55AEBAD: -------------------------------------------------------------------------------- 1 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:24:46 +0000] 174.44.160.70 - CE74FF983317B326 REST.GET.OBJECT public/portfolio/22/thumbnail.jpg "GET /lamestuff.com/public/portfolio/22/thumbnail.jpg HTTP/1.1" 200 - 9515 9515 42 41 "http://spike.grobste.in/portfolio" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 2 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:20 +0000] 174.44.160.70 - D484E1F8E6DE0AAB REST.GET.OBJECT public/projects/46/resume.png "GET /lamestuff.com/public/projects/46/resume.png HTTP/1.1" 200 - 510856 510856 85 74 "http://spike.grobste.in/portfolio/show?name=lamestuff.com-%282007%29" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 3 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:25 +0000] 174.44.160.70 - E5C48446A01539FC REST.GET.OBJECT public/projects/3/gameboy.jpg "GET /lamestuff.com/public/projects/3/gameboy.jpg HTTP/1.1" 200 - 37458 37458 57 55 "http://spike.grobste.in/project/classic-console-art" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 4 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:27 +0000] 174.44.160.70 - CD6CC6713DE2254A REST.GET.OBJECT public/projects/115/xmas-card-2006-d.jpg "GET /lamestuff.com/public/projects/115/xmas-card-2006-d.jpg HTTP/1.1" 200 - 89892 89892 58 55 "http://spike.grobste.in/portfolio/show?name=xmas-2006" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 5 | 300a66dd319290e36b67767f862a7f39072caa70ba33b8d01c4b79ddd8e6d7ef lamestuff.com [05/Oct/2012:15:25:31 +0000] 174.44.160.70 - B8B9CBDE2EFE51F3 REST.GET.OBJECT public/projects/34/Tim_Horton-Window-Coffee-ba.jpg "GET /lamestuff.com/public/projects/34/Tim_Horton-Window-Coffee-ba.jpg HTTP/1.1" 200 - 27376 27376 41 40 "http://spike.grobste.in/project/tim-horton's-at-the-nyse" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_2) AppleWebKit/536.26.14 (KHTML, like Gecko) Version/6.0.1 Safari/536.26.14" - 6 | -------------------------------------------------------------------------------- /README.rdoc: -------------------------------------------------------------------------------- 1 | = Request-log-analyzer {}[http://travis-ci.org/wvanbergen/request-log-analyzer] 2 | 3 | This is a simple command line tool to analyze request log files in various formats to produce a performance report. Its purpose is to find what actions are best candidates for optimization. 4 | 5 | * Analyzes log files. Currently supports: Amazon S3, Apache, Delayed::Job, Merb, Mysql, PostgreSQL, Rack, Rails and more. 6 | * Combines multiple files and decompresses compressed files, which comes in handy if you are using logrotate. 7 | * Uses several metrics, including cumulative request time, mean request time, process blockers, database and rendering time, HTTP methods and statuses, Rails action cache statistics, etc.) (Sample output: http://github.com/wvanbergen/request-log-analyzer/wiki/sample-output) 8 | * Runs on any MRI 1.9+ compatible Ruby, has a low memory footprint and is reasonably fast, so it is safe to run on a production server. 9 | 10 | See the project wiki at http://github.com/wvanbergen/request-log-analyzer/wiki for documentation and additional information. 11 | 12 | == Installation & basic usage 13 | 14 | Install request-log-analyzer as a Ruby gem (you might need to run this command as root by prepending +sudo+ to it): 15 | 16 | $ gem install request-log-analyzer 17 | 18 | To analyze a Rails log file and produce a performance report, run request-log-analyzer like this: 19 | 20 | $ request-log-analyzer log/production.log 21 | 22 | For more details, other file formats, and available command line options, see the project's wiki at http://github.com/wvanbergen/request-log-analyzer/wiki 23 | 24 | == Additional information 25 | 26 | Request-log-analyzer was designed and built by Willem van Bergen and Bart ten 27 | Brinke. 28 | 29 | Do you have a rails application that is not performing as it should? If you need 30 | an expert to analyze your application, feel free to contact either Willem van 31 | Bergen (willem@railsdoctors.com) or Bart ten Brinke (bart@railsdoctors.com). 32 | 33 | * Project wiki at GitHub: http://github.com/wvanbergen/request-log-analyzer/wiki 34 | * Issue tracker at GitHub: http://github.com/wvanbergen/request-log-analyzer/issues 35 | * The Railsdoctors homepage: http://railsdoctors.com 36 | * This software is MIT licensed. Check out CONTRIBUTING.rdoc if you want to help out on this project. 37 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/delayed_job4.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The DelayedJob4 file format parsed log files that are created by DelayedJob 4.0 or higher. 3 | # By default, the log file can be found in RAILS_ROOT/log/delayed_job.log 4 | class DelayedJob4 < Base 5 | extend CommonRegularExpressions 6 | 7 | line_definition :job_completed do |line| 8 | line.header = true 9 | line.footer = true 10 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] Job (.+) \(id=\d+\) COMPLETED after (\d+\.\d+)/ 11 | line.capture(:timestamp).as(:timestamp) 12 | line.capture(:host) 13 | line.capture(:pid).as(:integer) 14 | line.capture(:job) 15 | line.capture(:duration).as(:duration, unit: :sec) 16 | end 17 | 18 | line_definition :job_failed do |line| 19 | line.header = true 20 | line.footer = true 21 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] Job (.+) FAILED \((\d+) prior attempts\) with (.+)/ 22 | line.capture(:timestamp).as(:timestamp) 23 | line.capture(:host) 24 | line.capture(:pid).as(:integer) 25 | line.capture(:job) 26 | line.capture(:attempts).as(:integer) 27 | line.capture(:error) 28 | end 29 | 30 | line_definition :job_deleted do |line| 31 | line.header = true 32 | line.footer = true 33 | line.regexp = /(#{timestamp('%Y-%m-%dT%H:%M:%S%z')}): \[Worker\(\S+ host:(#{hostname_or_ip_address}) pid:(\d+)\)\] Job (.+) REMOVED permanently because of (\d+) consecutive failures/ 34 | line.capture(:timestamp).as(:timestamp) 35 | line.capture(:host) 36 | line.capture(:pid).as(:integer) 37 | line.capture(:job) 38 | line.capture(:failures).as(:integer) 39 | end 40 | 41 | report do |analyze| 42 | analyze.timespan 43 | analyze.hourly_spread 44 | 45 | analyze.frequency :job, line_type: :job_completed, title: 'Completed jobs' 46 | analyze.frequency :job, category: lambda { |r| "#{r[:job]} #{r[:error]}" }, line_type: :job_failed, title: 'Failed jobs' 47 | analyze.frequency :failures, category: :job, line_type: :job_deleted, title: 'Deleted jobs' 48 | analyze.duration :duration, category: :job, line_type: :job_completed, title: 'Job duration' 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/unit/file_format/postgresql_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::Postgresql do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:Postgresql) } 6 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 7 | 8 | it { should be_well_formed } 9 | 10 | describe '#parse_line' do 11 | it 'should parse a :query line correctly' do 12 | line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"' 13 | subject.should parse_line(line).as(:query).and_capture(timestamp: 20_101_010_135_207, query_fragment: 'SELECT * FROM "delayed_jobs"') 14 | end 15 | 16 | it 'should parse a :query_fragment line correctly' do 17 | line = ' ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",' 18 | subject.should parse_line(line).as(:query_fragment).and_capture(query_fragment: '("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at",') 19 | end 20 | 21 | it 'should parse a :query line correctly' do 22 | line = '2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs"' 23 | subject.should parse_line(line).as(:query).and_capture(query_time: 0.710) 24 | end 25 | end 26 | 27 | # describe '#parse_io' do 28 | # it "should parse a multiline query entry correctly" do 29 | # fixture = <<-EOS 30 | # 2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG: 00000: duration: 0.195 ms statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {} 31 | # ', '2010-10-10 15:00:02.159884', 0, '2010-10-10 16:00:00.000000', 0, NULL, NULL) RETURNING "id" 32 | # 2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION: exec_simple_query, postgres.c:1081 33 | # EOS 34 | # 35 | # log_parser.should_not_receive(:warn) 36 | # log_parser.parse_string(fixture) do |request| 37 | # request[:query].should == 'INSERT INTO delayed_jobs (failed_at, locked_by, created_at, handler, updated_at, priority, run_at, attempts, locked_at, last_error) VALUES(NULL, NULL, :string, E:string, :string, :int, :string, :int, NULL, NULL) RETURNING id' 38 | # end 39 | # end 40 | # end 41 | end 42 | -------------------------------------------------------------------------------- /spec/unit/filter/field_filter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Filter::Field, 'string in accept mode' do 4 | 5 | before(:each) do 6 | @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, field: :test, value: 'test', mode: :select) 7 | end 8 | 9 | it 'should reject a request if the field value does not match' do 10 | @filter.filter(request(test: 'not test')).should be_nil 11 | end 12 | 13 | it 'should reject a request if the field name does not match' do 14 | @filter.filter(request(testing: 'test')).should be_nil 15 | end 16 | 17 | it 'should accept a request if the both name and value match' do 18 | @filter.filter(request(test: 'test')).should_not be_nil 19 | end 20 | 21 | it 'should accept a request if the value is not the first value' do 22 | @filter.filter(request([{ test: 'ignore' }, { test: 'test' }])).should_not be_nil 23 | end 24 | end 25 | 26 | describe RequestLogAnalyzer::Filter::Field, 'string in reject mode' do 27 | 28 | before(:each) do 29 | @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, field: :test, value: 'test', mode: :reject) 30 | end 31 | 32 | it 'should accept a request if the field value does not match' do 33 | @filter.filter(request(test: 'not test')).should_not be_nil 34 | end 35 | 36 | it 'should accept a request if the field name does not match' do 37 | @filter.filter(request(testing: 'test')).should_not be_nil 38 | end 39 | 40 | it 'should reject a request if the both name and value match' do 41 | @filter.filter(request(test: 'test')).should be_nil 42 | end 43 | 44 | it 'should reject a request if the value is not the first value' do 45 | @filter.filter(request([{ test: 'ignore' }, { test: 'test' }])).should be_nil 46 | end 47 | end 48 | 49 | describe RequestLogAnalyzer::Filter::Field, 'regexp in accept mode' do 50 | 51 | before(:each) do 52 | @filter = RequestLogAnalyzer::Filter::Field.new(testing_format, field: :test, value: '/test/', mode: :select) 53 | end 54 | 55 | it 'should reject a request if the field value does not match' do 56 | @filter.filter(request(test: 'a working test')).should_not be_nil 57 | end 58 | 59 | it 'should reject a request if the field name does not match' do 60 | @filter.filter(request(testing: 'test')).should be_nil 61 | end 62 | 63 | it 'should accept a request if the value is not the first value' do 64 | @filter.filter(request([{ test: 'ignore' }, { test: 'testing 123' }])).should_not be_nil 65 | end 66 | end 67 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/tracker/duration.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Tracker 2 | # Analyze the duration of a specific attribute 3 | # 4 | # === Options 5 | # * :category Proc that handles request categorization for given fileformat (REQUEST_CATEGORIZER) 6 | # * :duration The field containing the duration in the request hash. 7 | # * :if Proc that has to return !nil for a request to be passed to the tracker. 8 | # * :line_type The line type that contains the duration field (determined by the category proc). 9 | # * :title Title do be displayed above the report 10 | # * :unless Handle request if this proc is false for the handled request. 11 | # 12 | # The items in the update request hash are set during the creation of the Duration tracker. 13 | # 14 | # Example output: 15 | # Request duration - top 20 by cumulative time | Hits | Sum. | Avg. 16 | # --------------------------------------------------------------------------------- 17 | # EmployeeController#show.html [GET] | 4742 | 4922.56s | 1.04s 18 | # EmployeeController#update.html [POST] | 4647 | 2731.23s | 0.59s 19 | # EmployeeController#index.html [GET] | 5802 | 1477.32s | 0.25s 20 | # ............. 21 | class Duration < NumericValue 22 | # Check if duration and catagory option have been received, 23 | def prepare 24 | options[:value] = options[:duration] if options[:duration] 25 | super 26 | 27 | @number_of_buckets = options[:number_of_buckets] || 1000 28 | @min_bucket_value = options[:min_bucket_value] ? options[:min_bucket_value].to_f : 0.0001 29 | @max_bucket_value = options[:max_bucket_value] ? options[:max_bucket_value].to_f : 1000 30 | 31 | # precalculate the bucket size 32 | @bucket_size = (Math.log(@max_bucket_value) - Math.log(@min_bucket_value)) / @number_of_buckets.to_f 33 | end 34 | 35 | # Display a duration 36 | def display_value(time) 37 | case time 38 | when nil then '-' 39 | when 0...1 then '%0ims' % (time * 1000) 40 | when 1...60 then '%0.02fs' % time 41 | when 60...3600 then '%dm%02ds' % [time / 60, (time % 60).round] 42 | else '%dh%02dm%02ds' % [time / 3600, (time % 3600) / 60, (time % 60).round] 43 | end 44 | end 45 | 46 | # Returns the title of this tracker for reports 47 | def title 48 | options[:title] || 'Request duration' 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /spec/unit/tracker/duration_tracker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Tracker::Duration do 4 | 5 | describe '#report' do 6 | 7 | before(:each) do 8 | @tracker = RequestLogAnalyzer::Tracker::Duration.new(category: :category, value: :duration) 9 | @tracker.prepare 10 | end 11 | 12 | it 'should generate a report without errors when one category is present' do 13 | @tracker.update(request(category: 'a', duration: 0.2)) 14 | lambda { @tracker.report(mock_output) }.should_not raise_error 15 | end 16 | 17 | it 'should generate a report without errors when no category is present' do 18 | lambda { @tracker.report(mock_output) }.should_not raise_error 19 | end 20 | 21 | it 'should generate a report without errors when multiple categories are present' do 22 | @tracker.update(request(category: 'a', duration: 0.2)) 23 | @tracker.update(request(category: 'b', duration: 0.2)) 24 | lambda { @tracker.report(mock_output) }.should_not raise_error 25 | end 26 | 27 | it 'should generate a report with arrays of durations are present' do 28 | @tracker.update(request(category: 'a', duration: [0.1, 0.2])) 29 | @tracker.update(request(category: 'a', duration: [0.2, 0.3])) 30 | lambda { @tracker.report(mock_output) }.should_not raise_error 31 | @tracker.to_yaml_object['a'].should include(min: 0.1, hits: 4, max: 0.3, mean: 0.2, sum: 0.8) 32 | end 33 | 34 | it 'should generate a YAML output' do 35 | @tracker.update(request(category: 'a', duration: 0.2)) 36 | @tracker.update(request(category: 'b', duration: 0.2)) 37 | @tracker.to_yaml_object.keys.should =~ %w(a b) 38 | @tracker.to_yaml_object['a'].should include(min: 0.2, hits: 1, max: 0.2, mean: 0.2, sum: 0.2, sum_of_squares: 0.0) 39 | @tracker.to_yaml_object['b'].should include(min: 0.2, hits: 1, max: 0.2, mean: 0.2, sum: 0.2, sum_of_squares: 0.0) 40 | end 41 | end 42 | 43 | describe '#display_value' do 44 | before(:each) { @tracker = RequestLogAnalyzer::Tracker::Duration.new(category: :category, value: :duration) } 45 | 46 | it 'should only display seconds when time < 60' do 47 | @tracker.display_value(33.12).should == '33.12s' 48 | end 49 | 50 | it 'should display minutes and wholeseconds when time > 60' do 51 | @tracker.display_value(63.12).should == '1m03s' 52 | end 53 | 54 | it 'should display minutes and wholeseconds when time > 60' do 55 | @tracker.display_value(3601.12).should == '1h00m01s' 56 | end 57 | end 58 | end 59 | -------------------------------------------------------------------------------- /CONTRIBUTING.rdoc: -------------------------------------------------------------------------------- 1 | = Contributing to request-log-analyzer 2 | 3 | If you would like to help out, fork the project! If you want me to merge back the changes, please send a pull request. If you have any questions regarding helping out or the codebase, contact us. 4 | 5 | * Look at the issue tracker for things to work on: http://github.com/wvanbergen/request-log-analyzer/issues 6 | * DESIGN.rdoc contains some general information about how the project is set up internally. This may help you get started. 7 | * RDoc documentation can be found at http://rdoc.info/projects/wvanbergen/request-log-analyzer 8 | * See CHANGELOG.rdoc to see what changed over the different releases of request-log-analyzer. 9 | 10 | We really like to receive patches and we will gladly accept them from anybody regardless of race, gender, nationality, sexual orientation, coding ability or species. 11 | 12 | == Getting your changes accepted 13 | 14 | We care about the maintainability and software quality of this project. Because of this, we code review all changes that go in. Please make sure your pull requests conform to the following requirements: 15 | 16 | - Conform to the current coding style. The style is not written down or defined explicitly, but comparing your code with the surrounding lines will give you a ballpark impression. 17 | - Make sure the documentation is up to date for the methods you add or change. We use Yardoc syntax. 18 | - Always add specs for any new functionality. Otherwise, we may unintentionally break it in the future. 19 | - Always make sure that all the specs pass for all the Ruby versions we report. You can do this yourself using your ruby version manager of choice, or rely on Travis CI to do it for you. We will not merge pull requests for which Travis CI is failing. 20 | - Do not change the RequestLogAnalyzer::VERSION constant. 21 | - Add an entry to CHANGELOG.rdoc describing your change in one sentence. 22 | - Make sure you're not including any hard tabs or trailing spaces in your changes. This drives me MAD! 23 | 24 | Some final notes: 25 | 26 | - Inform us that you're working on something. This way we can keep track on what is happening and we can assist if needed. 27 | - Don't hesitate to ask us any questions or advice. 28 | - Don't hesitate to tell us if we are not being helpful. 29 | 30 | == Release process 31 | 32 | - Update RequestLogAnalyzer::VERSION constant. 33 | - Update CHANGELOG.rdoc by moving the items in the unreleased changes sections to a section for the new version. 34 | - Commit the changes. 35 | - Run rake release 36 | -------------------------------------------------------------------------------- /spec/unit/file_format/file_format_api_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat do 4 | 5 | describe '.format_definition' do 6 | 7 | before(:each) do 8 | @first_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base) 9 | @second_file_format = Class.new(RequestLogAnalyzer::FileFormat::Base) 10 | end 11 | 12 | it 'should specify line definitions directly within the file_format' do 13 | @first_file_format.format_definition.direct_test regexp: /test/ 14 | @first_file_format.should have_line_definition(:direct_test) 15 | end 16 | 17 | it 'specify lines with a block for the format definition' do 18 | @first_file_format.format_definition do |format| 19 | format.block_test regexp: /test (\w+)/, captures: [{ name: :tester, type: :string }] 20 | end 21 | 22 | @first_file_format.should have_line_definition(:block_test).capturing(:tester) 23 | end 24 | 25 | it 'should specify a line with a block' do 26 | @first_file_format.format_definition.hash_test do |line| 27 | line.regexp = /test/ 28 | line.captures = [] 29 | end 30 | 31 | @first_file_format.should have_line_definition(:hash_test) 32 | end 33 | 34 | it 'should define lines only for its own language' do 35 | @first_file_format.format_definition.first regexp: /test 123/ 36 | @second_file_format.format_definition.second regexp: /test 456/ 37 | 38 | @first_file_format.should have_line_definition(:first) 39 | @first_file_format.should_not have_line_definition(:second) 40 | @second_file_format.should_not have_line_definition(:first) 41 | @second_file_format.should have_line_definition(:second) 42 | end 43 | end 44 | 45 | describe '.load' do 46 | 47 | it 'should return an instance of a FileFormat class' do 48 | @file_format = RequestLogAnalyzer::FileFormat.load(TestingFormat) 49 | @file_format.should be_kind_of(TestingFormat) 50 | end 51 | 52 | it 'should return itself if it already is a FileFormat::Base instance' do 53 | @file_format = RequestLogAnalyzer::FileFormat.load(testing_format) 54 | @file_format.should be_kind_of(TestingFormat) 55 | end 56 | 57 | it 'should load a predefined file format from the /file_format dir' do 58 | @file_format = RequestLogAnalyzer::FileFormat.load(:rails) 59 | @file_format.should be_kind_of(RequestLogAnalyzer::FileFormat::Rails) 60 | end 61 | 62 | it 'should load a provided format file' do 63 | format_filename = File.expand_path('../../lib/testing_format.rb', File.dirname(__FILE__)) 64 | @file_format = RequestLogAnalyzer::FileFormat.load(format_filename) 65 | @file_format.should be_kind_of(TestingFormat) 66 | end 67 | 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /spec/unit/file_format/rack_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::Rack do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:rack) } 6 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 7 | 8 | it { should be_well_formed } 9 | it do 10 | should have_line_definition(:access).capturing(:remote_host, :user, :remote_logname, 11 | :timestamp, :http_method, :path, :http_version, :http_status, :bytes_sent, :duration) 12 | end 13 | 14 | it { should satisfy { |ff| ff.report_trackers.length == 7 } } 15 | 16 | let(:sample1) { '127.0.0.1 - - [23/Nov/2009 21:47:47] "GET /css/stylesheet.css HTTP/1.1" 200 3782 0.0024' } 17 | let(:sample2) { '127.0.0.1 - - [16/Sep/2009 07:40:08] "GET /favicon.ico HTTP/1.1" 500 63183 0.0453' } 18 | let(:sample3) { '127.0.0.1 - - [01/Oct/2009 07:58:10] "GET / HTTP/1.1" 200 1 0.0045' } 19 | let(:irrelevant) { '== Sinatra/0.9.4 has taken the stage on 4567 for development with backup from Mongrel' } 20 | 21 | describe '#parse_line' do 22 | 23 | it do 24 | should parse_line(sample1, 'a sample access line').and_capture( 25 | remote_host: '127.0.0.1', timestamp: 20_091_123_214_747, user: nil, 26 | http_status: 200, http_method: 'GET', http_version: '1.1', 27 | duration: 0.0024, bytes_sent: 3782, remote_logname: nil, 28 | path: '/css/stylesheet.css') 29 | end 30 | 31 | it do 32 | should parse_line(sample2, 'another sample access line').and_capture( 33 | remote_host: '127.0.0.1', timestamp: 20_090_916_074_008, user: nil, 34 | http_status: 500, http_method: 'GET', http_version: '1.1', 35 | duration: 0.0453, bytes_sent: 63_183, remote_logname: nil, 36 | path: '/favicon.ico') 37 | end 38 | 39 | it do 40 | should parse_line(sample3, 'a third sample access line').and_capture( 41 | remote_host: '127.0.0.1', timestamp: 20_091_001_075_810, user: nil, 42 | http_status: 200, http_method: 'GET', http_version: '1.1', 43 | duration: 0.0045, bytes_sent: 1, remote_logname: nil, 44 | path: '/') 45 | end 46 | 47 | it { should_not parse_line(irrelevant, 'an irrelevant line') } 48 | it { should_not parse_line('nonsense', 'a nonsense line') } 49 | end 50 | 51 | describe '#parse_io' do 52 | let(:snippet) { log_snippet(irrelevant, sample1, sample2, sample3) } 53 | 54 | it 'shouldparse a snippet without warnings' do 55 | log_parser.should_receive(:handle_request).exactly(3).times 56 | log_parser.should_not_receive(:warn) 57 | log_parser.parse_io(snippet) 58 | end 59 | end 60 | end 61 | -------------------------------------------------------------------------------- /spec/fixtures/postgresql.log: -------------------------------------------------------------------------------- 1 | 2010-10-10 13:50:24 GMT [38626]: [1-1] LOG: 00000: database system was shut down at 2010-10-10 13:49:33 GMT 2 | 2010-10-10 13:50:24 GMT [38626]: [2-1] LOCATION: StartupXLOG, xlog.c:4816 3 | 2010-10-10 13:50:24 GMT [38624]: [1-1] LOG: 00000: database system is ready to accept connections 4 | 2010-10-10 13:50:24 GMT [38624]: [2-1] LOCATION: reaper, postmaster.c:2156 5 | 2010-10-10 13:50:24 GMT [38629]: [1-1] LOG: 00000: autovacuum launcher started 6 | 2010-10-10 13:50:24 GMT [38629]: [2-1] LOCATION: AutoVacLauncherMain, autovacuum.c:520 7 | 2010-10-10 13:52:04 GMT [38747]: [1-1] LOG: 00000: connection received: host=127.0.0.1 port=60020 8 | 2010-10-10 13:52:04 GMT [38747]: [2-1] LOCATION: BackendInitialize, postmaster.c:3027 9 | 2010-10-10 13:52:04 GMT [38747]: [3-1] LOG: 00000: connection authorized: user=root database=demonstration 10 | 2010-10-10 13:52:04 GMT [38747]: [4-1] LOCATION: BackendInitialize, postmaster.c:3097 11 | 2010-10-10 13:52:04 GMT [38747]: [5-1] LOG: 00000: duration: 0.329 ms statement: SHOW clients 12 | 2010-10-10 13:52:05 GMT [38747]: [21-1] LOG: 00000: duration: 0.973 ms statement: SELECT tablename 13 | FROM pg_tables 14 | WHERE schemaname IN (E'"$user"',E'public') 15 | 2010-10-10 13:52:07 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs" 16 | 2010-10-10 13:52:08 GMT [38747]: [33-1] LOG: 00000: duration: 0.710 ms statement: SELECT * FROM "delayed_jobs" 17 | 2010-10-10 15:00:02 GMT [38747]: [1667-1] LOG: 00000: duration: 0.024 ms statement: BEGIN 18 | 2010-10-10 15:00:02 GMT [38747]: [1668-1] LOCATION: exec_simple_query, postgres.c:1081 19 | 2010-10-10 15:00:02 GMT [38747]: [1669-1] LOG: 00000: duration: 0.195 ms statement: INSERT INTO "delayed_jobs" ("failed_at", "locked_by", "created_at", "handler", "updated_at", "priority", "run_at", "attempts", "locked_at", "last_error") VALUES(NULL, NULL, '2010-10-10 15:00:02.159884', E'--- !ruby/object:RuntheChooChootrain {} 20 | 21 | ', '2010-10-10 15:00:02.159884', 0, '2010-10-10 16:00:00.000000', 0, NULL, NULL) RETURNING "id" 22 | 2010-10-10 15:00:02 GMT [38747]: [1670-1] LOCATION: exec_simple_query, postgres.c:1081 23 | 2010-10-10 15:00:02 GMT [38747]: [1671-1] LOG: 00000: duration: 0.065 ms statement: COMMIT 24 | 2010-10-10 15:00:02 GMT [38747]: [1672-1] LOCATION: exec_simple_query, postgres.c:1081 25 | 2010-10-10 15:00:02 GMT [38747]: [1673-1] LOG: 00000: duration: 0.020 ms statement: BEGIN 26 | 2010-10-10 15:00:02 GMT [38747]: [1674-1] LOCATION: exec_simple_query, postgres.c:1081 27 | 2010-10-10 15:00:02 GMT [38747]: [1675-1] LOG: 00000: duration: 0.157 ms statement: DELETE FROM "delayed_jobs" WHERE "id" = 123456 28 | 2010-10-10 15:00:02 GMT [38747]: [1676-1] LOCATION: exec_simple_query, postgres.c:1081 29 | 2010-10-10 15:00:02 GMT [38747]: [1677-1] LOG: 00000: duration: 0.050 ms statement: COMMIT -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/postgresql.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # PostgresQL spec 8.3.7 3 | class Postgresql < Base 4 | extend CommonRegularExpressions 5 | 6 | line_definition :query do |line| 7 | line.header = true 8 | line.teaser = /.*LOG\:/ 9 | line.regexp = /(#{timestamp('%Y-%m-%d %k:%M:%S')})\ \S+ \[\d+\]\:\ \[.*\]\ LOG\:\ \ \d+\:\ duration\: (.*)\ ms\ \ statement:\ (.*)/ 10 | 11 | line.capture(:timestamp).as(:timestamp) 12 | line.capture(:query_time).as(:duration, unit: :sec) 13 | line.capture(:query_fragment) 14 | end 15 | 16 | line_definition :location do |line| 17 | line.footer = true 18 | line.teaser = /.*LOCATION:/ 19 | line.regexp = /.*(\ )LOCATION:/ 20 | 21 | line.capture(:query).as(:sql) # Hack to gather up fragments 22 | end 23 | 24 | line_definition :query_fragment do |line| 25 | line.regexp = /^(?!.*LOG)\s*(.*)\s*/ 26 | line.capture(:query_fragment) 27 | end 28 | 29 | report do |analyze| 30 | analyze.timespan 31 | analyze.hourly_spread 32 | analyze.duration :query_time, category: :query, title: 'Query time' 33 | end 34 | 35 | class Request < RequestLogAnalyzer::Request 36 | def convert_sql(value, _definition) 37 | # Recreate the full SQL query by joining all the previous parts and this last line 38 | sql = every(:query_fragment).join("\n") + value 39 | 40 | # Sanitize an SQL query so that it can be used as a category field. 41 | # sql.gsub!(/\/\*.*\*\//, '') # remove comments 42 | sql.gsub!(/\s+/, ' ') # remove excessive whitespace 43 | sql.gsub!(/"([^"]+)"/, '\1') # remove quotes from field names 44 | sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date') # replace dates 45 | sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime') # replace timestamps 46 | sql.gsub!(/'[^']*'/, ':string') # replace strings 47 | sql.gsub!(/\b\d+\b/, ':int') # replace integers 48 | sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list 49 | sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list 50 | 51 | sql.lstrip.rstrip 52 | end 53 | 54 | def host 55 | self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host] 56 | end 57 | 58 | # Convert the timestamp to an integer 59 | def convert_timestamp(value, _definition) 60 | _, y, m, d, h, i, s = value.split(/(\d\d)-(\d\d)-(\d\d)\s+(\d?\d):(\d\d):(\d\d)/) 61 | ('20%s%s%s%s%s%s' % [y, m, d, h.rjust(2, '0'), i, s]).to_i 62 | end 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /spec/unit/file_format/line_definition_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::LineDefinition do 4 | 5 | subject do 6 | RequestLogAnalyzer::LineDefinition.new(:test, 7 | teaser: /Testing /, 8 | regexp: /Testing (\w+), tries\: (\d+)/, 9 | captures: [{ name: :what, type: :string }, { name: :tries, type: :integer }] 10 | ) 11 | end 12 | 13 | describe '#matches' do 14 | 15 | it 'should return false on an unmatching line' do 16 | subject.matches('nonmatching').should == false 17 | end 18 | 19 | it 'should return false when only the teaser matches' do 20 | subject.matches('Testing LineDefinition').should == false 21 | end 22 | 23 | it 'should parse a line and capture the expected values' do 24 | subject.matches('Testing LineDefinition, tries: 123').should == { line_definition: subject, captures: %w(LineDefinition 123) } 25 | end 26 | 27 | it 'should know which names it can capture' do 28 | subject.captures?(:what).should == true 29 | subject.captures?(:tries).should == true 30 | subject.captures?(:bogus).should == false 31 | end 32 | end 33 | 34 | describe '#convert_captured_values' do 35 | let(:request) { double('request', convert_value: 'foo') } 36 | 37 | it 'should call convert_value for every captured value' do 38 | request.should_receive(:convert_value).twice 39 | subject.convert_captured_values(%w(test 123), request) 40 | end 41 | 42 | it 'should set the converted values' do 43 | subject.convert_captured_values(%w(test 123), request).should == { what: 'foo', tries: 'foo' } 44 | end 45 | 46 | context 'when using :provides option' do 47 | 48 | subject do 49 | RequestLogAnalyzer::LineDefinition.new(:test, 50 | regexp: /Hash\: (\{.+\})/, 51 | captures: [{ name: :hash, type: :hash, provides: { bar: :string } }]) 52 | end 53 | 54 | before do 55 | request.stub(:convert_value).with("{:bar=>'baz'}", anything).and_return(bar: 'baz') 56 | request.stub(:convert_value).with('baz', anything).and_return('foo') 57 | end 58 | 59 | it 'should call Request#convert_value for the initial hash and the value in the hash' do 60 | request.should_receive(:convert_value).with("{:bar=>'baz'}", anything).and_return(bar: 'baz') 61 | request.should_receive(:convert_value).with('baz', anything) 62 | subject.convert_captured_values(["{:bar=>'baz'}"], request) 63 | end 64 | 65 | it 'should return the converted hash' do 66 | subject.convert_captured_values(["{:bar=>'baz'}"], request).should include(bar: 'foo') 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/unit/tracker/timespan_tracker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Tracker::Timespan do 4 | 5 | before(:each) do 6 | @tracker = RequestLogAnalyzer::Tracker::Timespan.new 7 | @tracker.prepare 8 | end 9 | 10 | it 'should set the first request timestamp correctly' do 11 | @tracker.update(request(timestamp: 20_090_102_000_000)) 12 | @tracker.update(request(timestamp: 20_090_101_000_000)) 13 | @tracker.update(request(timestamp: 20_090_103_000_000)) 14 | 15 | @tracker.first_timestamp.should == DateTime.parse('Januari 1, 2009 00:00:00') 16 | end 17 | 18 | it 'should set the last request timestamp correctly' do 19 | @tracker.update(request(timestamp: 20_090_102_000_000)) 20 | @tracker.update(request(timestamp: 20_090_101_000_000)) 21 | @tracker.update(request(timestamp: 20_090_103_000_000)) 22 | 23 | @tracker.last_timestamp.should == DateTime.parse('Januari 3, 2009 00:00:00') 24 | end 25 | 26 | it 'should return the correct timespan in days when multiple requests are given' do 27 | @tracker.update(request(timestamp: 20_090_102_000_000)) 28 | @tracker.update(request(timestamp: 20_090_101_000_000)) 29 | @tracker.update(request(timestamp: 20_090_103_000_000)) 30 | 31 | @tracker.timespan.should == 2 32 | end 33 | 34 | it 'should return a timespan of 0 days when only one timestamp is set' do 35 | @tracker.update(request(timestamp: 20_090_103_000_000)) 36 | @tracker.timespan.should == 0 37 | end 38 | 39 | it 'should raise an error when no timestamp is set' do 40 | lambda { @tracker.timespan }.should raise_error 41 | end 42 | end 43 | 44 | describe RequestLogAnalyzer::Tracker::Timespan, 'reporting' do 45 | 46 | before(:each) do 47 | @tracker = RequestLogAnalyzer::Tracker::Timespan.new 48 | @tracker.prepare 49 | end 50 | 51 | it 'should have a title' do 52 | @tracker.title.should_not eql('') 53 | end 54 | 55 | it 'should generate a report without errors when no request was tracked' do 56 | lambda { @tracker.report(mock_output) }.should_not raise_error 57 | end 58 | 59 | it 'should generate a report without errors when multiple requests were tracked' do 60 | @tracker.update(request(category: 'a', timestamp: 20_090_102_000_000)) 61 | @tracker.update(request(category: 'a', timestamp: 20_090_101_000_000)) 62 | @tracker.update(request(category: 'a', timestamp: 20_090_103_000_000)) 63 | lambda { @tracker.report(mock_output) }.should_not raise_error 64 | end 65 | 66 | it 'should generate a YAML output' do 67 | @tracker.update(request(category: 'a', timestamp: 20_090_102_000_000)) 68 | @tracker.update(request(category: 'a', timestamp: 20_090_101_000_000)) 69 | @tracker.update(request(category: 'a', timestamp: 20_090_103_000_000)) 70 | @tracker.to_yaml_object.should == { first: DateTime.parse('20090101000000'), last: DateTime.parse('20090103000000') } 71 | end 72 | 73 | end 74 | -------------------------------------------------------------------------------- /request-log-analyzer.gemspec: -------------------------------------------------------------------------------- 1 | # -*- encoding: utf-8 -*- 2 | lib = File.expand_path('../lib', __FILE__) 3 | $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) 4 | require 'request_log_analyzer/version' 5 | 6 | Gem::Specification.new do |gem| 7 | gem.name = "request-log-analyzer" 8 | gem.rubyforge_project = 'r-l-a' 9 | 10 | gem.version = RequestLogAnalyzer::VERSION 11 | 12 | gem.authors = ['Willem van Bergen', 'Bart ten Brinke'] 13 | gem.email = ['willem@railsdoctors.com', 'bart@railsdoctors.com'] 14 | gem.homepage = 'http://www.request-log-analyzer.com' 15 | gem.license = "MIT" 16 | 17 | gem.summary = "A command line tool to analyze request logs for Apache, Rails, Merb, MySQL and other web application servers" 18 | gem.description = <<-eos 19 | Request log analyzer's purpose is to find out how your web application is being used, how it performs and to 20 | focus your optimization efforts. This tool will parse all requests in the application's log file and aggregate the 21 | information. Once it is finished parsing the log file(s), it will show the requests that take op most server time 22 | using various metrics. It can also insert all parsed request information into a database so you can roll your own 23 | analysis. It supports Rails-, Merb- and Rack-based applications logs, Apache and Amazon S3 access logs and MySQL 24 | slow query logs out of the box, but file formats of other applications can easily be supported by supplying an 25 | easy to write log file format definition. 26 | eos 27 | 28 | gem.rdoc_options << '--title' << gem.name << '--main' << 'README.rdoc' << '--line-numbers' << '--inline-source' 29 | gem.extra_rdoc_files = ['README.rdoc', 'DESIGN.rdoc', 'CONTRIBUTING.rdoc', 'CHANGELOG.rdoc', 'LICENSE'] 30 | 31 | gem.requirements << "To use the database inserter, ActiveRecord and an appropriate database adapter are required." 32 | gem.required_ruby_version = '>= 1.9.3' 33 | 34 | gem.add_development_dependency('rake') 35 | gem.add_development_dependency('rspec', '~> 3') 36 | gem.add_development_dependency('activerecord') 37 | if defined?(JRUBY_VERSION) 38 | gem.add_development_dependency('jdbc-sqlite3') 39 | gem.add_development_dependency('jdbc-mysql') 40 | gem.add_development_dependency('jdbc-postgres') 41 | gem.add_development_dependency('activerecord-jdbcsqlite3-adapter') 42 | gem.add_development_dependency('activerecord-jdbcmysql-adapter') 43 | gem.add_development_dependency('activerecord-jdbcpostgresql-adapter') 44 | else 45 | gem.add_development_dependency('sqlite3') 46 | gem.add_development_dependency('mysql2') 47 | gem.add_development_dependency('pg') 48 | end 49 | 50 | gem.files = `git ls-files`.split($/) 51 | gem.test_files = gem.files.grep(%r{^(test|spec|features)/}) 52 | 53 | gem.executables = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) } 54 | gem.default_executable = 'request-log-analyzer' 55 | gem.bindir = 'bin' 56 | gem.require_paths = ["lib"] 57 | end 58 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/source.rb: -------------------------------------------------------------------------------- 1 | # The RequestLogAnalyzer::Source module contains all functionality that loads requests from a given source 2 | # and feed them to the pipeline for further processing. The requests (see RequestLogAnalyzer::Request) that 3 | # will be parsed from a source, will be piped throug filters (see RequestLogAnalyzer::Filter) and are then 4 | # fed to an aggregator (see RequestLogAnalyzer::Aggregator). The source instance is thus the beginning of 5 | # the RequestLogAnalyzer chain. 6 | # 7 | # - The base class for all sources is RequestLogAnalyzer::Source::Base. All source classes should inherit from this class. 8 | # - Currently, RequestLogAnalyzer::Source::LogParser is the only implemented source. 9 | module RequestLogAnalyzer::Source 10 | # The base Source class. All other sources should inherit from this class. 11 | # 12 | # A source implememtation should at least implement the each_request method, which should yield 13 | # RequestLogAnalyzer::Request instances that will be fed through the pipleine. 14 | class Base 15 | # A hash of options 16 | attr_reader :options 17 | 18 | # The current Request object that is being parsed 19 | attr_reader :current_request 20 | 21 | # The total number of parsed lines 22 | attr_reader :parsed_lines 23 | 24 | # The number of skipped lines because of warnings 25 | attr_reader :skipped_lines 26 | 27 | # The total number of parsed requests. 28 | attr_reader :parsed_requests 29 | 30 | # The total number of skipped requests because of filters. 31 | attr_reader :skipped_requests 32 | 33 | # The FileFormat instance that describes the format of this source. 34 | attr_reader :file_format 35 | 36 | # Initializer, which will register the file format and save any options given as a hash. 37 | # format:: The file format instance 38 | # options:: A hash of options that can be used by a specific Source implementation 39 | def initialize(format, options = {}) 40 | @options = options 41 | @file_format = format 42 | end 43 | 44 | # The prepare method is called before the RequestLogAnalyzer::Source::Base#each_request method is called. 45 | # Use this method to implement any initialization that should occur before this source can produce Request 46 | # instances. 47 | def prepare 48 | end 49 | 50 | # This function is called to actually produce the requests that will be send into the pipeline. 51 | # The implementation should yield instances of RequestLogAnalyzer::Request. 52 | # options:: A Hash of options that can be used in the implementation. 53 | def each_request(_options = {}, &_block) # :yields: request 54 | true 55 | end 56 | 57 | # This function is called after RequestLogAnalyzer::Source::Base#each_request finished. Any code to 58 | # wrap up, free resources, etc. can be put in this method. 59 | def finalize 60 | end 61 | end 62 | end 63 | 64 | require 'request_log_analyzer/source/log_parser' 65 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/tracker/timespan.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Tracker 2 | # Determines the datetime of the first request and the last request 3 | # Also determines the amount of days inbetween these. 4 | # 5 | # Accepts the following options: 6 | # * :field The timestamp field that is looked at. Defaults to :timestamp. 7 | # * :if Proc that has to return !nil for a request to be passed to the tracker. 8 | # * :line_type The line type that contains the duration field (determined by the category proc). 9 | # * :title Title do be displayed above the report. 10 | # * :unless Proc that has to return nil for a request to be passed to the tracker. 11 | # 12 | # Expects the following items in the update request hash 13 | # * :timestamp in YYYYMMDDHHMMSS format. 14 | # 15 | # Example output: 16 | # First request: 2008-07-13 06:25:06 17 | # Last request: 2008-07-20 06:18:06 18 | # Total time analyzed: 7 days 19 | class Timespan < Base 20 | attr_reader :first, :last 21 | 22 | # Check if timestamp field is set in the options. 23 | def prepare 24 | options[:field] ||= :timestamp 25 | @first, @last = 99_999_999_999_999, 0 26 | end 27 | 28 | # Check if the timestamp in the request and store it. 29 | # request The request. 30 | def update(request) 31 | timestamp = request[options[:field]] 32 | @first = timestamp if timestamp < @first 33 | @last = timestamp if timestamp > @last 34 | end 35 | 36 | # First timestamp encountered 37 | def first_timestamp 38 | DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil 39 | end 40 | 41 | # Last timestamp encountered 42 | def last_timestamp 43 | DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil 44 | end 45 | 46 | # Difference between last and first timestamp. 47 | def timespan 48 | last_timestamp - first_timestamp 49 | end 50 | 51 | # Generate an hourly spread report to the given output object. 52 | # Any options for the report should have been set during initialize. 53 | # output The output object 54 | def report(output) 55 | output.title(options[:title]) if options[:title] 56 | 57 | if @last > 0 && @first < 99_999_999_999_999 58 | output.with_style(cell_separator: false) do 59 | output.table({ width: 20 }, {}) do |rows| 60 | rows << ['First request:', first_timestamp.strftime('%Y-%m-%d %H:%M:%I')] 61 | rows << ['Last request:', last_timestamp.strftime('%Y-%m-%d %H:%M:%I')] 62 | rows << ['Total time analyzed:', "#{timespan.ceil} days"] 63 | end 64 | end 65 | end 66 | end 67 | 68 | # Returns the title of this tracker for reports 69 | def title 70 | options[:title] || 'Request timespan' 71 | end 72 | 73 | # A hash that can be exported to YAML with the first and last timestamp encountered. 74 | def to_yaml_object 75 | { first: first_timestamp, last: last_timestamp } 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /spec/unit/file_format/delayed_job21_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::DelayedJob do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:delayed_job21) } 6 | 7 | it { should be_well_formed } 8 | it { should have_line_definition(:job_lock).capturing(:timestamp, :job, :host, :pid) } 9 | it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid) } 10 | it { should satisfy { |ff| ff.report_trackers.length == 4 } } 11 | 12 | describe '#parse_line' do 13 | 14 | let(:job_lock_sample1) { '2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' } 15 | let(:job_lock_sample2) { '2010-05-17T17:37:34+0000: [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' } 16 | let(:job_completed_sample1) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676' } 17 | 18 | it do 19 | should parse_line(job_lock_sample1, 'with a single worker').as(:job_lock).and_capture( 20 | timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888) 21 | end 22 | 23 | it do 24 | should parse_line(job_lock_sample2, 'with multiple workers').as(:job_lock).and_capture( 25 | timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888) 26 | end 27 | 28 | it do 29 | should parse_line(job_completed_sample1).as(:job_completed).and_capture( 30 | timestamp: 20_100_517_173_735, duration: 1.0676, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob') 31 | end 32 | 33 | it { should_not parse_line('nonsense', 'a nonsense line') } 34 | end 35 | 36 | describe '#parse_io' do 37 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 38 | 39 | it 'should parse a batch of completed jobs without warnings' do 40 | fragment = log_snippet(<<-EOLOG) 41 | 2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888 42 | 2010-05-17T17:37:34+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob 43 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676 44 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob 45 | 2010-05-17T17:37:37+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.4407 46 | 2010-05-17T17:37:37+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob 47 | 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 6.9374 48 | 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] 3 jobs processed at 0.3163 j/s, 0 failed ... 49 | 2010-05-19T11:47:26+0000: Exiting... 50 | EOLOG 51 | 52 | log_parser.should_receive(:handle_request).exactly(3).times 53 | log_parser.should_not_receive(:warn) 54 | log_parser.parse_io(fragment) 55 | end 56 | end 57 | end 58 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/amazon_s3.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # FileFormat for Amazon S3 access logs. 3 | # 4 | # Access logs are disabled by default on Amazon S3. To enable logging, see 5 | # http://docs.amazonwebservices.com/AmazonS3/latest/index.html?ServerLogs.html 6 | class AmazonS3 < Base 7 | extend CommonRegularExpressions 8 | 9 | line_definition :access do |line| 10 | line.header = true 11 | line.footer = true 12 | line.regexp = /^([^\ ]+) ([^\ ]+) \[(#{timestamp('%d/%b/%Y:%H:%M:%S %z')})?\] (#{ip_address}) ([^\ ]+) ([^\ ]+) (\w+(?:\.\w+)*) ([^\ ]+) "([^"]+)" (\d+) ([^\ ]+) ([^\ ]+) (\d+) (\d+) ([^\ ]+) "([^"]*)" "([^"]*)"/ 13 | 14 | line.capture(:bucket_owner) 15 | line.capture(:bucket) 16 | line.capture(:timestamp).as(:timestamp) 17 | line.capture(:remote_ip) 18 | line.capture(:requester) 19 | line.capture(:request_id) 20 | line.capture(:operation) 21 | line.capture(:key).as(:nillable_string) 22 | line.capture(:request_uri) 23 | line.capture(:http_status).as(:integer) 24 | line.capture(:error_code).as(:nillable_string) 25 | line.capture(:bytes_sent).as(:traffic, unit: :byte) 26 | line.capture(:object_size).as(:traffic, unit: :byte) 27 | line.capture(:total_time).as(:duration, unit: :msec) 28 | line.capture(:turnaround_time).as(:duration, unit: :msec) 29 | line.capture(:referer).as(:referer) 30 | line.capture(:user_agent).as(:user_agent) 31 | end 32 | 33 | report do |analyze| 34 | analyze.timespan 35 | analyze.hourly_spread 36 | 37 | analyze.frequency category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Most popular items' 38 | analyze.duration duration: :total_time, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Request duration' 39 | analyze.traffic traffic: :bytes_sent, category: lambda { |r| "#{r[:bucket]}/#{r[:key]}" }, title: 'Traffic' 40 | analyze.frequency category: :http_status, title: 'HTTP status codes' 41 | analyze.frequency category: :error_code, title: 'Error codes' 42 | end 43 | 44 | class Request < RequestLogAnalyzer::Request 45 | MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06', 46 | 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' } 47 | 48 | # Do not use DateTime.parse, but parse the timestamp ourselves to return a integer 49 | # to speed up parsing. 50 | def convert_timestamp(value, _definition) 51 | "#{value[7, 4]}#{MONTHS[value[3, 3]]}#{value[0, 2]}#{value[12, 2]}#{value[15, 2]}#{value[18, 2]}".to_i 52 | end 53 | 54 | # Make sure that the string '-' is parsed as a nil value. 55 | def convert_nillable_string(value, _definition) 56 | value == '-' ? nil : value 57 | end 58 | 59 | # Can be implemented in subclasses for improved categorizations 60 | def convert_referer(value, _definition) 61 | value == '-' ? nil : value 62 | end 63 | 64 | # Can be implemented in subclasses for improved categorizations 65 | def convert_user_agent(value, _definition) 66 | value == '-' ? nil : value 67 | end 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /spec/unit/file_format/common_regular_expressions_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::CommonRegularExpressions do 4 | 5 | include RequestLogAnalyzer::FileFormat::CommonRegularExpressions 6 | 7 | describe '.timestamp' do 8 | it 'should parse timestamps with a given format' do 9 | anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should =~ '2009-12-03T00:12:37+0100' 10 | anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03 00:12:37+0100' 11 | anchored(timestamp('%Y-%m-%dT%H:%M:%S%z')).should_not =~ '2009-12-03T00:12:37' 12 | end 13 | end 14 | 15 | describe '.hostname' do 16 | it 'should parse hostnames successfully' do 17 | anchored(hostname).should =~ 'railsdoctors.com' 18 | anchored(hostname).should =~ 'www.rails-doctors.com' 19 | anchored(hostname).should =~ 'hostname.co.uk' 20 | anchored(hostname).should =~ 'localhost' 21 | 22 | anchored(hostname).should_not =~ '192.168.0.1' 23 | anchored(hostname).should_not =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf' 24 | anchored(hostname).should_not =~ 'railsdoctors.' 25 | end 26 | end 27 | 28 | describe '.ip_address' do 29 | it 'should parse IPv4 addresses' do 30 | anchored(ip_address).should =~ '127.0.0.1' 31 | anchored(ip_address).should =~ '255.255.255.255' 32 | 33 | anchored(ip_address).should_not =~ '2552.2552.2552.2552' 34 | anchored(ip_address).should_not =~ '127001' 35 | anchored(ip_address).should_not =~ '' 36 | anchored(ip_address).should_not =~ '-' 37 | anchored(ip_address).should_not =~ 'sub-host.domain.tld' 38 | end 39 | 40 | it 'should pase IPv6 addresses' do 41 | anchored(ip_address).should =~ '::1' 42 | anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf' 43 | anchored(ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:127.0.0.1' 44 | anchored(ip_address).should =~ '::3:200:f8ff:127.0.0.1' 45 | anchored(ip_address).should =~ '0:0:0:0:0:0:0:1' 46 | 47 | anchored(ip_address).should_not =~ 'qqqq:wwww:eeee:3q:200:wf8ff:fe21:67cf' 48 | anchored(ip_address).should_not =~ '3ffe44:1900f:454545:3:200:f8ff:ffff:5432' 49 | end 50 | 51 | it 'should allow blank if true is given as parameter' do 52 | anchored(ip_address(true)).should =~ '' 53 | anchored(ip_address(true)).should_not =~ ' ' 54 | end 55 | 56 | it 'should allow a nil substitute if a string is given as parameter' do 57 | anchored(ip_address('-')).should =~ '-' 58 | anchored(ip_address('-')).should_not =~ ' -' 59 | anchored(ip_address('-')).should_not =~ '--' 60 | anchored(ip_address('-')).should_not =~ '' 61 | end 62 | end 63 | 64 | describe '.hostname_or_ip_address' do 65 | it 'should parse either hostnames or ip addresses' do 66 | anchored(hostname_or_ip_address).should =~ 'railsdoctors.com' 67 | anchored(hostname_or_ip_address).should =~ 'hostname.co.uk' 68 | anchored(hostname_or_ip_address).should =~ 'localhost' 69 | anchored(hostname_or_ip_address).should =~ '192.168.0.1' 70 | anchored(hostname_or_ip_address).should =~ '3ffe:1900:4545:3:200:f8ff:fe21:67cf' 71 | 72 | anchored(hostname_or_ip_address).should_not =~ 'railsdoctors.' 73 | end 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/merb.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | # The Merb file format parses the request header with the timestamp, the params line 3 | # with the most important request information and the durations line which contains 4 | # the different request durations that can be used for analysis. 5 | class Merb < Base 6 | extend CommonRegularExpressions 7 | 8 | # ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008 9 | line_definition :started do |line| 10 | line.header = true 11 | line.teaser = /Started request handling\:/ 12 | line.regexp = /Started request handling\:\ (#{timestamp('%a %b %d %H:%M:%S %z %Y')})/ 13 | line.captures << { name: :timestamp, type: :timestamp } 14 | end 15 | 16 | # ~ Params: {"action"=>"create", "controller"=>"session"} 17 | # ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy"} 18 | line_definition :params do |line| 19 | line.teaser = /Params\:\ / 20 | line.regexp = /Params\:\ (\{.+\})/ 21 | line.captures << { name: :params, type: :eval, provides: { 22 | namespace: :string, controller: :string, action: :string, format: :string, method: :string } } 23 | end 24 | 25 | # ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833} 26 | line_definition :completed do |line| 27 | line.footer = true 28 | # line.teaser = Regexp.new(Regexp.quote('~ {:')) 29 | line.regexp = /(\{.*\:dispatch_time\s*=>\s*\d+\.\d+.*\})/ 30 | line.captures << { name: :times_hash, type: :eval, provides: { 31 | dispatch_time: :duration, after_filters_time: :duration, 32 | before_filters_time: :duration, action_time: :duration } } 33 | end 34 | 35 | REQUEST_CATEGORIZER = proc do |request| 36 | category = "#{request[:controller]}##{request[:action]}" 37 | category = "#{request[:namespace]}::#{category}" if request[:namespace] 38 | category = "#{category}.#{request[:format]}" if request[:format] 39 | category 40 | end 41 | 42 | report do |analyze| 43 | 44 | analyze.timespan 45 | analyze.hourly_spread 46 | 47 | analyze.frequency category: REQUEST_CATEGORIZER, title: 'Top 20 by hits' 48 | analyze.duration :dispatch_time, category: REQUEST_CATEGORIZER, title: 'Request dispatch duration' 49 | 50 | # analyze.duration :action_time, :category => REQUEST_CATEGORIZER, :title => 'Request action duration' 51 | # analyze.duration :after_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request after_filter duration' 52 | # analyze.duration :before_filters_time, :category => REQUEST_CATEGORIZER, :title => 'Request before_filter duration' 53 | end 54 | 55 | class Request < RequestLogAnalyzer::Request 56 | MONTHS = { 'Jan' => '01', 'Feb' => '02', 'Mar' => '03', 'Apr' => '04', 'May' => '05', 'Jun' => '06', 57 | 'Jul' => '07', 'Aug' => '08', 'Sep' => '09', 'Oct' => '10', 'Nov' => '11', 'Dec' => '12' } 58 | 59 | # Speed up timestamp conversion 60 | def convert_timestamp(value, _definition) 61 | "#{value[26, 4]}#{MONTHS[value[4, 3]]}#{value[8, 2]}#{value[11, 2]}#{value[14, 2]}#{value[17, 2]}".to_i 62 | end 63 | end 64 | end 65 | end 66 | -------------------------------------------------------------------------------- /spec/unit/tracker/frequency_tracker_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Tracker::Frequency do 4 | 5 | context 'static category' do 6 | before(:each) do 7 | @tracker = RequestLogAnalyzer::Tracker::Frequency.new(category: :category) 8 | @tracker.prepare 9 | end 10 | 11 | it 'should register a request in the right category' do 12 | @tracker.update(request(category: 'a', blah: 0.2)) 13 | @tracker.categories.should include('a') 14 | end 15 | 16 | it 'should register a request in the right category' do 17 | @tracker.update(request(category: 'a', blah: 0.2)) 18 | @tracker.update(request(category: 'b', blah: 0.2)) 19 | @tracker.update(request(category: 'b', blah: 0.2)) 20 | 21 | @tracker.frequency('a').should == 1 22 | @tracker.frequency('b').should == 2 23 | @tracker.overall_frequency.should == 3 24 | end 25 | 26 | it 'should sort correctly by frequency' do 27 | @tracker.update(request(category: 'a', blah: 0.2)) 28 | @tracker.update(request(category: 'b', blah: 0.2)) 29 | @tracker.update(request(category: 'b', blah: 0.2)) 30 | 31 | @tracker.sorted_by_frequency.should == [['b', 2], ['a', 1]] 32 | end 33 | end 34 | 35 | context 'dynamic category' do 36 | before(:each) do 37 | @categorizer = proc { |request| request[:duration] > 0.2 ? 'slow' : 'fast' } 38 | @tracker = RequestLogAnalyzer::Tracker::Frequency.new(category: @categorizer) 39 | @tracker.prepare 40 | end 41 | 42 | it 'should use the categorizer to determine the right category' do 43 | @tracker.update(request(category: 'a', duration: 0.2)) 44 | @tracker.update(request(category: 'b', duration: 0.3)) 45 | @tracker.update(request(category: 'b', duration: 0.4)) 46 | 47 | @tracker.frequency('fast').should == 1 48 | @tracker.frequency('slow').should == 2 49 | @tracker.frequency('moderate').should == 0 50 | end 51 | end 52 | 53 | describe '#report' do 54 | before(:each) do 55 | @tracker = RequestLogAnalyzer::Tracker::Frequency.new(category: :category) 56 | @tracker.prepare 57 | end 58 | 59 | it 'should generate a report without errors when one category is present' do 60 | @tracker.update(request(category: 'a', blah: 0.2)) 61 | lambda { @tracker.report(mock_output) }.should_not raise_error 62 | end 63 | 64 | it 'should generate a report without errors when no category is present' do 65 | lambda { @tracker.report(mock_output) }.should_not raise_error 66 | end 67 | 68 | it 'should generate a report without errors when multiple categories are present' do 69 | @tracker.update(request(category: 'a', blah: 0.2)) 70 | @tracker.update(request(category: 'b', blah: 0.2)) 71 | lambda { @tracker.report(mock_output) }.should_not raise_error 72 | end 73 | end 74 | 75 | describe '#to_yaml_object' do 76 | before(:each) do 77 | @tracker = RequestLogAnalyzer::Tracker::Frequency.new(category: :category) 78 | @tracker.prepare 79 | end 80 | 81 | it 'should generate a YAML output' do 82 | @tracker.update(request(category: 'a', blah: 0.2)) 83 | @tracker.update(request(category: 'b', blah: 0.2)) 84 | @tracker.to_yaml_object.should == { 'a' => 1, 'b' => 1 } 85 | end 86 | end 87 | end 88 | -------------------------------------------------------------------------------- /spec/unit/file_format/delayed_job3_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::DelayedJob do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:delayed_job3) } 6 | 7 | it { should be_well_formed } 8 | it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid, :job) } 9 | it { should have_line_definition(:job_failed).capturing(:timestamp, :host, :pid, :job, :attempts) } 10 | it { should have_line_definition(:job_deleted).capturing(:timestamp, :host, :pid, :job, :failures) } 11 | it { should satisfy { |ff| ff.report_trackers.length == 6 } } 12 | 13 | describe '#parse_line' do 14 | 15 | let(:job_completed_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob.create completed after 1.0676' } 16 | let(:job_failed_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob.create failed with SocketError: getaddrinfo: Name or service not known - 0 failed attempts' } 17 | let(:job_deleted_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] PERMANENTLY removing S3FileJob.create because of 25 consecutive failures.' } 18 | 19 | it do 20 | should parse_line(job_completed_sample).as(:job_completed).and_capture( 21 | timestamp: 20_100_517_173_735, duration: 1.0676, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create') 22 | end 23 | 24 | it do 25 | should parse_line(job_failed_sample).as(:job_failed).and_capture( 26 | timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create failed with SocketError: getaddrinfo: Name or service not known', attempts: 0) 27 | end 28 | 29 | it do 30 | should parse_line(job_deleted_sample).as(:job_deleted).and_capture( 31 | timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create', failures: 25) 32 | end 33 | 34 | it { should_not parse_line('nonsense', 'a nonsense line') } 35 | end 36 | 37 | describe '#parse_io' do 38 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 39 | 40 | it 'should parse a batch of completed jobs without warnings' do 41 | fragment = log_snippet(<<-EOLOG) 42 | 2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888 43 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.0676 44 | 2010-05-17T17:37:37+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 1.4407 45 | 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob completed after 6.9374 46 | 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] 3 jobs processed at 0.3163 j/s, 0 failed ... 47 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] S3FileJob.create failed with SocketError: getaddrinfo: Name or service not known - 25 failed attempts 48 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] PERMANENTLY removing S3FileJob.create because of 25 consecutive failures. 49 | 2010-05-19T11:47:26+0000: Exiting... 50 | EOLOG 51 | 52 | log_parser.should_receive(:handle_request).exactly(5).times 53 | log_parser.parse_io(fragment) 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /spec/unit/file_format/delayed_job2_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::DelayedJob2 do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:delayed_job2) } 6 | 7 | it { should be_well_formed } 8 | it { should have_line_definition(:job_lock).capturing(:timestamp, :job, :host, :pid) } 9 | it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid) } 10 | it { should satisfy { |ff| ff.report_trackers.length == 4 } } 11 | 12 | describe '#parse_line' do 13 | 14 | let(:job_lock_sample1) { '2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' } 15 | let(:job_lock_sample2) { '2010-05-17T17:37:34+0000: * [Worker(delayed_job.0 host:hostname.co.uk pid:11888)] acquired lock on S3FileJob' } 16 | let(:job_completed_sample) { '2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676' } 17 | let(:starting_sample) { '2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888' } 18 | let(:summary_sample) { '3 jobs processed at 0.3163 j/s, 0 failed ...' } 19 | 20 | it do 21 | should parse_line(job_lock_sample1, 'with a single worker').as(:job_lock).and_capture( 22 | timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888) 23 | end 24 | 25 | it do 26 | should parse_line(job_lock_sample2, 'with multiple workers').as(:job_lock).and_capture( 27 | timestamp: 20_100_517_173_734, job: 'S3FileJob', host: 'hostname.co.uk', pid: 11_888) 28 | end 29 | 30 | it do 31 | should parse_line(job_completed_sample).as(:job_completed).and_capture(timestamp: 20_100_517_173_735, 32 | duration: 1.0676, host: 'hostname.co.uk', pid: 11_888) 33 | end 34 | 35 | it { should_not parse_line(starting_sample, 'a starting line') } 36 | it { should_not parse_line(summary_sample, 'a summary line') } 37 | it { should_not parse_line('nonsense', 'a nonsense line') } 38 | end 39 | 40 | describe '#parse_io' do 41 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 42 | 43 | it 'should parse a batch of completed jobs without warnings' do 44 | fragment = <<-EOLOG 45 | 2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888 46 | 2010-05-17T17:37:34+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob 47 | 2010-05-17T17:37:35+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.0676 48 | 2010-05-17T17:37:35+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob 49 | 2010-05-17T17:37:37+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 1.4407 50 | 2010-05-17T17:37:37+0000: * [Worker(delayed_job host:hostname.co.uk pid:11888)] acquired lock on S3FileJob 51 | 2010-05-17T17:37:44+0000: * [JOB] delayed_job host:hostname.co.uk pid:11888 completed after 6.9374 52 | 2010-05-17T17:37:44+0000: 3 jobs processed at 0.3163 j/s, 0 failed ... 53 | 2010-05-19T11:47:26+0000: Exiting... 54 | EOLOG 55 | 56 | log_parser.should_receive(:handle_request).exactly(3).times 57 | log_parser.should_not_receive(:warn) 58 | log_parser.parse_string(fragment) 59 | end 60 | end 61 | end 62 | -------------------------------------------------------------------------------- /spec/lib/matchers.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::RSpec::Matchers 2 | class HasLineDefinition 3 | def initialize(line_type) 4 | @line_type = line_type.to_sym 5 | @captures = [] 6 | end 7 | 8 | def and_capture(*captures) 9 | @captures += captures 10 | self 11 | end 12 | 13 | alias_method :capturing, :and_capture 14 | 15 | def description 16 | description = "have a #{@line_type.inspect} line definition" 17 | description << " that captures #{@captures.join(', ')}" unless @captures.empty? 18 | description 19 | end 20 | 21 | def matches?(file_format) 22 | file_format = file_format.create if file_format.is_a?(Class) 23 | if ld = file_format.line_definitions[@line_type] 24 | @captures.all? { |c| ld.all_captured_variables.include?(c) } 25 | else 26 | false 27 | end 28 | end 29 | end 30 | 31 | class ParseLine 32 | def initialize(line, line_description = nil) 33 | @line = line 34 | @captures = {} 35 | @line_type = nil 36 | @line_description = line_description 37 | end 38 | 39 | def line_description 40 | @full_line_description ||= if @line_description 41 | if @line_type && @line_description =~ /^(?:with|without|having|using) / 42 | "a #{@line_type.inspect} line #{@line_description}" 43 | else 44 | @line_description 45 | end 46 | elsif @line_type 47 | "a #{@line_type.inspect} line" 48 | else 49 | "line #{@line.inspect}" 50 | end 51 | end 52 | 53 | attr_reader :failure_message 54 | 55 | def as(line_type) 56 | @line_type = line_type 57 | self 58 | end 59 | 60 | def and_capture(captures) 61 | @captures = captures 62 | self 63 | end 64 | 65 | alias_method :capturing, :and_capture 66 | 67 | def fail(message) 68 | @failure_message = message 69 | false 70 | end 71 | 72 | def description 73 | description = "parse #{line_description}" 74 | description << " as line type #{@line_type.inspect}" if @line_type 75 | description << " and capture #{@captures.keys.join(', ')} correctly" unless @captures.empty? 76 | description 77 | end 78 | 79 | def matches?(file_format) 80 | if @line_hash = file_format.parse_line(@line) 81 | if @line_type.nil? || @line_hash[:line_definition].name == @line_type 82 | @request = file_format.request(@line_hash) 83 | @captures.each do |key, value| 84 | return fail("Expected line #{@line.inspect}\n to capture #{key.inspect} as #{value.inspect} but was #{@request[key].inspect}.") if @request[key] != value 85 | end 86 | return true 87 | else 88 | return fail("The line should match the #{@line_type.inspect} line definition, but matched #{@line_hash[:line_definition].name.inspect} instead.") 89 | end 90 | else 91 | return fail('The line did not match any line definition.') 92 | end 93 | end 94 | end 95 | 96 | def have_line_definition(line_type) 97 | HasLineDefinition.new(line_type) 98 | end 99 | 100 | def parse_line(line, line_description = nil) 101 | ParseLine.new(line, line_description) 102 | end 103 | end 104 | -------------------------------------------------------------------------------- /spec/unit/file_format/oink_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::Oink do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:oink) } 6 | 7 | it { should have_line_definition(:memory_usage).capturing(:pid, :memory) } 8 | it { should have_line_definition(:processing).capturing(:pid, :controller, :action, :ip) } 9 | it { should have_line_definition(:instance_type_counter).capturing(:pid, :instance_counts) } 10 | it { should satisfy { |ff| ff.report_trackers.length == 12 } } 11 | 12 | describe '#parse_line' do 13 | let(:memory_usage_sample) { 'Jun 18 11:27:36 derek rails[67783]: Memory usage: 714052 | PID: 67783' } 14 | let(:processing_sample) { 'Aug 14 21:16:30 derek rails[67783]: Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET]' } 15 | let(:instance_type_counter_sample) { 'Dec 13 12:00:44 storenvy rails[26364]: Instantiation Breakdown: Total: 732 | User: 376 | Post: 323 | Comment: 32 | Blog: 1' } 16 | 17 | it 'should parse a :memory_usage line correctly' do 18 | subject.should parse_line(memory_usage_sample).as(:memory_usage).and_capture(pid: 67_783, memory: 714_052) 19 | end 20 | 21 | it 'should parse the PID from a :processing line correctly' do 22 | subject.should parse_line(processing_sample).as(:processing).and_capture(pid: 67_783, controller: 'PeopleController', action: 'index', timestamp: 20_080_814_211_630, method: 'GET', ip: '1.1.1.1') 23 | end 24 | 25 | it 'should parse a :instance_type_counter correctly' do 26 | subject.should parse_line(instance_type_counter_sample).as(:instance_type_counter).and_capture(pid: 26_364, instance_counts: { 'Total' => 732, 'User' => 376, 'Post' => 323, 'Comment' => 32, 'Blog' => 1 }) 27 | end 28 | end 29 | 30 | describe '#parse_io' do 31 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 32 | 33 | context 'Rails 2.2 style log' do 34 | it 'should parse requests' do 35 | log_parser.should_receive(:handle_request).exactly(4).times 36 | log_parser.should_not_receive(:warn) 37 | log_parser.parse_file(log_fixture(:oink_22)) 38 | end 39 | 40 | it 'should not record :memory_diff on first request' do 41 | log_parser.parse_file(log_fixture(:oink_22)) do |request| 42 | request[:memory_diff].should.nil? if log_parser.parsed_requests == 1 43 | end 44 | end 45 | 46 | it 'should record :memory_diff of 2nd tracked PID' do 47 | log_parser.parse_file(log_fixture(:oink_22)) do |request| 48 | request[:memory_diff].should == 50_000 * 1024 if log_parser.parsed_requests == 3 49 | end 50 | end 51 | 52 | it 'should record :memory_diff of 1st tracked PID' do 53 | log_parser.parse_file(log_fixture(:oink_22)) do |request| 54 | request[:memory_diff].should == 30_000 * 1024 if log_parser.parsed_requests == 4 55 | end 56 | end 57 | end 58 | 59 | context 'Rails 2.2 style log w/failure' do 60 | it 'should parse requests' do 61 | log_parser.should_receive(:handle_request).exactly(4).times 62 | log_parser.should_not_receive(:warn) 63 | log_parser.parse_file(log_fixture(:oink_22_failure)) 64 | end 65 | 66 | it 'should ignore memory changes when a failure occurs' do 67 | log_parser.parse_file(log_fixture(:oink_22_failure)) do |request| 68 | request[:memory_diff].should.nil? if log_parser.parsed_requests == 4 69 | end 70 | end 71 | end 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /spec/unit/file_format/delayed_job4_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::DelayedJob do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:delayed_job4) } 6 | 7 | it { should be_well_formed } 8 | it { should have_line_definition(:job_completed).capturing(:timestamp, :duration, :host, :pid, :job) } 9 | it { should have_line_definition(:job_failed).capturing(:timestamp, :host, :pid, :job, :attempts, :error) } 10 | it { should have_line_definition(:job_deleted).capturing(:timestamp, :host, :pid, :job, :failures) } 11 | it { should satisfy { |ff| ff.report_trackers.length == 6 } } 12 | 13 | describe '#parse_line' do 14 | 15 | let(:job_completed_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534785) COMPLETED after 1.0676' } 16 | let(:job_failed_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534785) FAILED (0 prior attempts) with SocketError: getaddrinfo: Name or service not known' } 17 | let(:job_deleted_sample) { '2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534785) REMOVED permanently because of 25 consecutive failures' } 18 | 19 | it do 20 | should parse_line(job_completed_sample).as(:job_completed).and_capture( 21 | timestamp: 20_100_517_173_735, duration: 1.0676, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create') 22 | end 23 | 24 | it do 25 | should parse_line(job_failed_sample).as(:job_failed).and_capture( 26 | timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create (id=534785)', attempts: 0, error: 'SocketError: getaddrinfo: Name or service not known') 27 | end 28 | 29 | it do 30 | should parse_line(job_deleted_sample).as(:job_deleted).and_capture( 31 | timestamp: 20_100_517_173_735, host: 'hostname.co.uk', pid: 11_888, job: 'S3FileJob.create (id=534785)', failures: 25) 32 | end 33 | 34 | it { should_not parse_line('nonsense', 'a nonsense line') } 35 | end 36 | 37 | describe '#parse_io' do 38 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 39 | 40 | it 'should parse a batch of completed jobs without warnings' do 41 | fragment = log_snippet(<<-EOLOG) 42 | 2010-05-17T17:36:44+0000: *** Starting job worker delayed_job host:hostname.co.uk pid:11888 43 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob (id=534785) COMPLETED after 1.0676 44 | 2010-05-17T17:37:37+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob (id=534788) COMPLETED after 1.4407 45 | 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob (id=534789) COMPLETED after 6.9374 46 | 2010-05-17T17:37:44+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] 3 jobs processed at 0.3163 j/s, 0 failed 47 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534786) FAILED (25 prior attempts) with SocketError: getaddrinfo: Name or service not known 48 | 2010-05-17T17:37:35+0000: [Worker(delayed_job host:hostname.co.uk pid:11888)] Job S3FileJob.create (id=534799) REMOVED permanently because of 25 consecutive failures 49 | 2010-05-19T11:47:26+0000: Exiting... 50 | EOLOG 51 | 52 | log_parser.should_receive(:handle_request).exactly(5).times 53 | log_parser.parse_io(fragment) 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /spec/fixtures/rails_1x.log: -------------------------------------------------------------------------------- 1 | Processing DashboardController#index (for 1.1.1.1 at 2008-08-14 21:16:25) [GET] 2 | Session ID: BAh7CToMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUvMjM1MCIKZmxhc2hJ 3 | QzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNoSGFzaHsABjoKQHVz 4 | ZWR7ADoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVjdBI6CUB3aW4wOg1AY291 5 | bnRyeSIHTkw6CkBoYXNoaf3L2Js6DkBvcmlnX3N0ciIKbmwtTkw6DUBpc28z 6 | MDY2MDoNQGNoYXJzZXQiClVURi04Og5AbGFuZ3VhZ2UiB25sOg5AbW9kaWZp 7 | ZXIwOgtAcG9zaXgiCm5sX05MOg1AZ2VuZXJhbCIKbmxfTkw6DUB2YXJpYW50 8 | MDoOQGZhbGxiYWNrMDoMQHNjcmlwdDA6DnBlcnNvbl9pZGkCMgc=--7918aed37151c13360cd370c37b541f136146fbd 9 | Parameters: {"action"=>"index", "controller"=>"dashboard"} 10 | Set language to: nl_NL 11 | Rendering template within layouts/priscilla 12 | Rendering dashboard/index 13 | Completed in 0.22699 (4 reqs/sec) | Rendering: 0.02667 (11%) | DB: 0.03057 (13%) | 200 OK [https://www.example.com/] 14 | 15 | 16 | Processing PeopleController#index (for 1.1.1.1 at 2008-08-14 21:16:30) [GET] 17 | Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo 18 | SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIQL3ByaXNjaWxsYS86DnBlcnNv 19 | bl9pZGkCMgc6DWxhbmd1YWdlbzoTTG9jYWxlOjpPYmplY3QSOg1AY291bnRy 20 | eSIHTkw6CUB3aW4wOg5Ab3JpZ19zdHIiCm5sLU5MOgpAaGFzaGn9y9ibOg5A 21 | bGFuZ3VhZ2UiB25sOg1AY2hhcnNldCIKVVRGLTg6DUBpc28zMDY2MDoOQG1v 22 | ZGlmaWVyMDoLQHBvc2l4IgpubF9OTDoNQHZhcmlhbnQwOg1AZ2VuZXJhbCIK 23 | bmxfTkw6DEBzY3JpcHQwOg5AZmFsbGJhY2sw--48cbe3788ef27f6005f8e999610a42af6e90ffb3 24 | Parameters: {"commit"=>"Zoek", "action"=>"index", "q"=>"gaby", "controller"=>"people"} 25 | Set language to: nl_NL 26 | Redirected to https://www.example.com/people/2545 27 | Completed in 0.04759 (21 reqs/sec) | DB: 0.03719 (78%) | 302 Found [https://www.example.com/people?q=gaby&commit=Zoek] 28 | 29 | 30 | Processing PeopleController#show (for 1.1.1.1 at 2008-08-14 21:16:30) [GET] 31 | Session ID: BAh7CToMcmVmZXJlciIpL3ByaXNjaWxsYS9wZW9wbGU/cT1nYWJ5JmNvbW1p 32 | dD1ab2VrIgpmbGFzaElDOidBY3Rpb25Db250cm9sbGVyOjpGbGFzaDo6Rmxh 33 | c2hIYXNoewAGOgpAdXNlZHsAOg1sYW5ndWFnZW86E0xvY2FsZTo6T2JqZWN0 34 | EjoJQHdpbjA6DUBjb3VudHJ5IgdOTDoKQGhhc2hp/cvYmzoOQG9yaWdfc3Ry 35 | IgpubC1OTDoNQGlzbzMwNjYwOg1AY2hhcnNldCIKVVRGLTg6DkBsYW5ndWFn 36 | ZSIHbmw6DkBtb2RpZmllcjA6C0Bwb3NpeCIKbmxfTkw6DUBnZW5lcmFsIgpu 37 | bF9OTDoNQHZhcmlhbnQwOg5AZmFsbGJhY2swOgxAc2NyaXB0MDoOcGVyc29u 38 | X2lkaQIyBw==--3ad1948559448522a49d289a2a89dc7ccbe8847a 39 | Parameters: {"action"=>"show", "id"=>"2545", "controller"=>"people"} 40 | Set language to: nl_NL 41 | Rendering template within layouts/priscilla 42 | Rendering people/show 43 | person: John Doe, study_year: 2008/2009 44 | Completed in 0.29077 (3 reqs/sec) | Rendering: 0.24187 (83%) | DB: 0.04030 (13%) | 200 OK [https://www.example.com/people/2545] 45 | 46 | 47 | Processing PeopleController#picture (for 1.1.1.1 at 2008-08-14 21:16:35) [GET] 48 | Session ID: BAh7CSIKZmxhc2hJQzonQWN0aW9uQ29udHJvbGxlcjo6Rmxhc2g6OkZsYXNo 49 | SGFzaHsABjoKQHVzZWR7ADoMcmVmZXJlciIbL3ByaXNjaWxsYS9wZW9wbGUv 50 | MjU0NToOcGVyc29uX2lkaQIyBzoNbGFuZ3VhZ2VvOhNMb2NhbGU6Ok9iamVj 51 | dBI6DUBjb3VudHJ5IgdOTDoJQHdpbjA6DkBvcmlnX3N0ciIKbmwtTkw6CkBo 52 | YXNoaf3L2Js6DkBsYW5ndWFnZSIHbmw6DUBjaGFyc2V0IgpVVEYtODoNQGlz 53 | bzMwNjYwOg5AbW9kaWZpZXIwOgtAcG9zaXgiCm5sX05MOg1AdmFyaWFudDA6 54 | DUBnZW5lcmFsIgpubF9OTDoMQHNjcmlwdDA6DkBmYWxsYmFjazA=--797a33f280a482647111397d138d0918f2658167 55 | Parameters: {"action"=>"picture", "id"=>"2545", "controller"=>"people"} 56 | Set language to: nl_NL 57 | Rendering template within layouts/priscilla 58 | Rendering people/picture 59 | Completed in 0.05383 (18 reqs/sec) | Rendering: 0.04622 (85%) | DB: 0.00206 (3%) | 200 OK [https://www.example.com/people/2545/picture] -------------------------------------------------------------------------------- /spec/unit/tracker/hourly_spread_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Tracker::HourlySpread do 4 | 5 | before(:each) do 6 | @tracker = RequestLogAnalyzer::Tracker::HourlySpread.new 7 | @tracker.prepare 8 | end 9 | 10 | it 'should store timestamps correctly' do 11 | @tracker.update(request(timestamp: 20_090_102_000_000)) 12 | @tracker.update(request(timestamp: 20_090_101_000_000)) 13 | @tracker.update(request(timestamp: 20_090_103_000_000)) 14 | 15 | @tracker.hour_frequencies[0].should eql(3) 16 | end 17 | 18 | it 'should count the number of timestamps correctly' do 19 | @tracker.update(request(timestamp: 20_090_102_000_000)) 20 | @tracker.update(request(timestamp: 20_090_101_000_000)) 21 | @tracker.update(request(timestamp: 20_090_103_000_000)) 22 | @tracker.update(request(timestamp: 20_090_103_010_000)) 23 | 24 | @tracker.total_requests.should eql(4) 25 | end 26 | 27 | it 'should set the first request timestamp correctly' do 28 | @tracker.update(request(timestamp: 20_090_102_000_000)) 29 | @tracker.update(request(timestamp: 20_090_101_000_000)) 30 | @tracker.update(request(timestamp: 20_090_103_000_000)) 31 | 32 | @tracker.first_timestamp.should == DateTime.parse('Januari 1, 2009 00:00:00') 33 | end 34 | 35 | it 'should set the last request timestamp correctly' do 36 | @tracker.update(request(timestamp: 20_090_102_000_000)) 37 | @tracker.update(request(timestamp: 20_090_101_000_000)) 38 | @tracker.update(request(timestamp: 20_090_103_000_000)) 39 | 40 | @tracker.last_timestamp.should == DateTime.parse('Januari 3, 2009 00:00:00') 41 | end 42 | 43 | it 'should return the correct timespan in days when multiple requests are given' do 44 | @tracker.update(request(timestamp: 20_090_102_000_000)) 45 | @tracker.update(request(timestamp: 20_090_101_000_000)) 46 | @tracker.update(request(timestamp: 20_090_103_000_000)) 47 | 48 | @tracker.timespan.should == 2 49 | end 50 | 51 | end 52 | 53 | describe RequestLogAnalyzer::Tracker::HourlySpread, 'reporting' do 54 | 55 | before(:each) do 56 | @tracker = RequestLogAnalyzer::Tracker::HourlySpread.new 57 | @tracker.prepare 58 | end 59 | 60 | it 'should generate a report without errors when no request was tracked' do 61 | lambda { @tracker.report(mock_output) }.should_not raise_error 62 | end 63 | 64 | it 'should generate a report without errors when multiple requests were tracked' do 65 | @tracker.update(request(timestamp: 20_090_102_000_000)) 66 | @tracker.update(request(timestamp: 20_090_101_000_000)) 67 | @tracker.update(request(timestamp: 20_090_103_000_000)) 68 | @tracker.update(request(timestamp: 20_090_103_010_000)) 69 | lambda { @tracker.report(mock_output) }.should_not raise_error 70 | end 71 | 72 | it 'should generate a YAML output' do 73 | @tracker.update(request(timestamp: 20_090_102_000_000)) 74 | @tracker.update(request(timestamp: 20_090_101_000_000)) 75 | @tracker.update(request(timestamp: 20_090_103_000_000)) 76 | @tracker.update(request(timestamp: 20_090_103_010_000)) 77 | @tracker.to_yaml_object.should == { '22:00 - 23:00' => 0, '9:00 - 10:00' => 0, '7:00 - 8:00' => 0, '2:00 - 3:00' => 0, '12:00 - 13:00' => 0, '11:00 - 12:00' => 0, '16:00 - 17:00' => 0, '15:00 - 16:00' => 0, '19:00 - 20:00' => 0, '3:00 - 4:00' => 0, '21:00 - 22:00' => 0, '20:00 - 21:00' => 0, '14:00 - 15:00' => 0, '13:00 - 14:00' => 0, '4:00 - 5:00' => 0, '10:00 - 11:00' => 0, '18:00 - 19:00' => 0, '17:00 - 18:00' => 0, '8:00 - 9:00' => 0, '6:00 - 7:00' => 0, '5:00 - 6:00' => 0, '1:00 - 2:00' => 1, '0:00 - 1:00' => 3, '23:00 - 24:00' => 0 } 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /spec/unit/file_format/delayed_job_format_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::FileFormat::DelayedJob do 4 | 5 | subject { RequestLogAnalyzer::FileFormat.load(:delayed_job) } 6 | 7 | it { should be_well_formed } 8 | it { should have_line_definition(:job_lock).capturing(:job) } 9 | it { should have_line_definition(:job_completed).capturing(:completed_job, :duration) } 10 | it { should have_line_definition(:job_lock_failed).capturing(:locked_job) } 11 | it { should have_line_definition(:job_failed).capturing(:failed_job, :attempts, :exception) } 12 | it { should satisfy { |ff| ff.report_trackers.length == 3 } } 13 | 14 | describe '#parse_line' do 15 | let(:job_lock_sample) { '* [JOB] acquiring lock on BackgroundJob::ThumbnailSaver' } 16 | let(:job_completed_sample) { '* [JOB] BackgroundJob::ThumbnailSaver completed after 0.7932' } 17 | let(:job_lock_failed_sample) { '* [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver' } 18 | let(:job_failed_sample) { "* [JOB] BackgroundJob::ThumbnailSaver failed with ActiveRecord::RecordNotFound: Couldn't find Design with ID=20413443 - 1 failed attempts" } 19 | let(:summary_sample) { '1 jobs processed at 1.0834 j/s, 0 failed ...' } 20 | 21 | it { should parse_line(job_lock_sample).as(:job_lock).and_capture(job: 'BackgroundJob::ThumbnailSaver') } 22 | it { should parse_line(job_completed_sample).as(:job_completed).and_capture(duration: 0.7932, completed_job: 'BackgroundJob::ThumbnailSaver') } 23 | it { should parse_line(job_lock_failed_sample).as(:job_lock_failed).and_capture(locked_job: 'BackgroundJob::ThumbnailSaver') } 24 | it { should parse_line(job_failed_sample).as(:job_failed).and_capture(attempts: 1, failed_job: 'BackgroundJob::ThumbnailSaver', exception: 'ActiveRecord::RecordNotFound') } 25 | 26 | it { should_not parse_line(summary_sample, 'a summary line') } 27 | it { should_not parse_line('nonsense', 'a nonsense line') } 28 | end 29 | 30 | describe '#parse_io' do 31 | let(:log_parser) { RequestLogAnalyzer::Source::LogParser.new(subject) } 32 | 33 | it 'should parse a batch of completed jobs without warnings' do 34 | fragment = log_snippet(<<-EOLOG) 35 | * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver 36 | * [JOB] BackgroundJob::ThumbnailSaver completed after 0.9114 37 | * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver 38 | * [JOB] BackgroundJob::ThumbnailSaver completed after 0.9110 39 | 2 jobs processed at 1.0832 j/s, 0 failed ... 40 | EOLOG 41 | 42 | log_parser.should_receive(:handle_request).twice 43 | log_parser.should_not_receive(:warn) 44 | log_parser.parse_io(fragment) 45 | end 46 | 47 | it 'should parse a batch with a failed job without warnings' do 48 | fragment = log_snippet(<<-EOLOG) 49 | * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver 50 | * [JOB] BackgroundJob::ThumbnailSaver completed after 1.0627 51 | * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver 52 | * [JOB] BackgroundJob::ThumbnailSaver failed with ActiveRecord::RecordNotFound: Couldn't find Design with ID=20413443 - 3 failed attempts 53 | Couldn't find Design with ID=20413443 54 | * [JOB] acquiring lock on BackgroundJob::ThumbnailSaver 55 | * [JOB] failed to acquire exclusive lock for BackgroundJob::ThumbnailSaver 56 | 2 jobs processed at 1.4707 j/s, 1 failed ... 57 | EOLOG 58 | 59 | log_parser.should_receive(:handle_request).exactly(3).times 60 | log_parser.should_not_receive(:warn) 61 | log_parser.parse_io(fragment) 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /spec/unit/request_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Request do 4 | 5 | before(:each) do 6 | @request = testing_format.request 7 | end 8 | 9 | it 'should be empty without any captured lines in it' do 10 | @request.should be_empty 11 | end 12 | 13 | context :incomplete do 14 | 15 | before(:each) do 16 | @request << { line_type: :test, lineno: 1, test_capture: 'awesome!' } 17 | end 18 | 19 | it 'should be single if only one line has been added' do 20 | @request.should_not be_empty 21 | end 22 | 23 | it 'should not be a completed request' do 24 | @request.should_not be_completed 25 | end 26 | 27 | it 'should take the line type of the first line as global line_type' do 28 | @request.lines[0][:line_type].should == :test 29 | @request.should =~ :test 30 | end 31 | 32 | it 'should return the first field value' do 33 | @request[:test_capture].should == 'awesome!' 34 | end 35 | 36 | it 'should return nil if no such field is present' do 37 | @request[:nonexisting].should be_nil 38 | end 39 | end 40 | 41 | context :completed do 42 | 43 | before(:each) do 44 | @request << { line_type: :first, lineno: 1, name: 'first line!' } 45 | @request << { line_type: :test, lineno: 4, test_capture: 'testing' } 46 | @request << { line_type: :test, lineno: 7, test_capture: 'testing some more' } 47 | @request << { line_type: :last, lineno: 10, time: 0.03 } 48 | end 49 | 50 | it 'should not be empty when multiple liness are added' do 51 | @request.should_not be_empty 52 | end 53 | 54 | it 'should be a completed request' do 55 | @request.should be_completed 56 | end 57 | 58 | it 'should recognize all line types' do 59 | [:first, :test, :last].each { |type| @request.should =~ type } 60 | end 61 | 62 | it 'should detect the correct field value' do 63 | @request[:name].should == 'first line!' 64 | @request[:time].should == 0.03 65 | end 66 | 67 | it 'should detect the first matching field value' do 68 | @request.first(:test_capture).should == 'testing' 69 | end 70 | 71 | it 'should detect the every matching field value' do 72 | @request.every(:test_capture).should == ['testing', 'testing some more'] 73 | end 74 | 75 | it 'should set the first_lineno for a request to the lowest lineno encountered' do 76 | @request.first_lineno.should eql(1) 77 | end 78 | 79 | it 'should set the first_lineno for a request if a line with a lower lineno is added' do 80 | @request << { line_type: :test, lineno: 0 } 81 | @request.first_lineno.should eql(0) 82 | end 83 | 84 | it 'should set the last_lineno for a request to the highest encountered lineno' do 85 | @request.last_lineno.should eql(10) 86 | end 87 | 88 | it 'should not set the last_lineno for a request if a line with a lower lineno is added' do 89 | @request << { line_type: :test, lineno: 7 } 90 | @request.last_lineno.should eql(10) 91 | end 92 | 93 | it 'should not have a timestamp if no such field is captured' do 94 | @request.timestamp.should be_nil 95 | end 96 | 97 | it 'should set return a timestamp field if such a field is captured' do 98 | @request << { line_type: :first, lineno: 1, name: 'first line!', timestamp: Time.now } 99 | @request.timestamp.should_not be_nil 100 | end 101 | end 102 | 103 | context 'single line' do 104 | # combined is both a header and a footer line 105 | before(:each) { @request << { line_type: :combined, lineno: 1 } } 106 | 107 | it 'should be a completed request if the line is both header and footer' do 108 | @request.should be_completed 109 | end 110 | end 111 | end 112 | -------------------------------------------------------------------------------- /spec/integration/command_line_usage_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer, 'running from command line' do 4 | 5 | before(:each) do 6 | cleanup_temp_files! 7 | end 8 | 9 | after(:each) do 10 | cleanup_temp_files! 11 | end 12 | 13 | it 'should find 4 requests in default mode' do 14 | output = run("#{log_fixture(:rails_1x)}") 15 | output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should == true 16 | end 17 | 18 | it 'should find 2 requests when parsing a compressed file' do 19 | output = run("#{log_fixture(:decompression, :tgz)}") 20 | output.any? { |line| /^Parsed requests\:\s*2\s/ =~ line }.should == true 21 | end 22 | 23 | it 'should skip 1 requests with a --select option' do 24 | output = run("#{log_fixture(:rails_1x)} --select controller PeopleController") 25 | output.any? { |line| /^Skipped requests\:\s*1\s/ =~ line }.should == true 26 | end 27 | 28 | it 'should skip 3 requests with a --reject option' do 29 | output = run("#{log_fixture(:rails_1x)} --reject controller PeopleController") 30 | output.any? { |line| /^Skipped requests\:\s*3\s/ =~ line }.should == true 31 | end 32 | 33 | it 'should not write output with the --silent option' do 34 | output = run("#{log_fixture(:rails_1x)} --silent --file #{temp_output_file(:report)}") 35 | output.should be_empty 36 | File.exist?(temp_output_file(:report)).should == true 37 | end 38 | 39 | it 'should write output to a file with the --file option' do 40 | run("#{log_fixture(:rails_1x)} --file #{temp_output_file(:report)}") 41 | File.exist?(temp_output_file(:report)).should == true 42 | end 43 | 44 | it 'should write only ASCII characters to a file with the --file option' do 45 | run("#{log_fixture(:rails_1x)} --file #{temp_output_file(:report)}") 46 | /^[\x00-\x7F]*$/.match(File.read(temp_output_file(:report))).should_not be_nil 47 | end 48 | 49 | it 'should write HTML if --output HTML is provided' do 50 | output = run("#{log_fixture(:rails_1x)} --output HTML") 51 | output.any? { |line| /]*>/ =~ line }.should == true 52 | end 53 | 54 | it 'should run with the --database option' do 55 | run("#{log_fixture(:rails_1x)} --database #{temp_output_file(:database)}") 56 | File.exist?(temp_output_file(:database)).should == true 57 | end 58 | 59 | it 'should use no colors in the report with the --boring option' do 60 | output = run("#{log_fixture(:rails_1x)} --boring --format rails") 61 | output.any? { |line| /\e/ =~ line }.should == false 62 | end 63 | 64 | it 'should use only ASCII characters in the report with the --boring option' do 65 | output = run("#{log_fixture(:rails_1x)} --boring") 66 | output.all? { |line| /^[\x00-\x7F]*$/ =~ line }.should == true 67 | end 68 | 69 | it 'should parse a Merb file if --format merb is set' do 70 | output = run("#{log_fixture(:merb)} --format merb") 71 | output.any? { |line| /Parsed requests\:\s*11/ =~ line }.should == true 72 | end 73 | 74 | it 'should parse a Apache access log file if --apache-format is set' do 75 | output = run("#{log_fixture(:apache_combined)} --apache-format combined") 76 | output.any? { |line| /Parsed requests\:\s*5/ =~ line }.should == true 77 | end 78 | 79 | it 'should dump the results to a YAML file' do 80 | run("#{log_fixture(:rails_1x)} --yaml #{temp_output_file(:yaml)}") 81 | File.exist?(temp_output_file(:yaml)).should == true 82 | YAML.load(File.read(temp_output_file(:yaml))).length.should be >= 1 83 | end 84 | 85 | it 'should parse 4 requests from the standard input' do 86 | output = run("--format rails - < #{log_fixture(:rails_1x)}") 87 | output.any? { |line| /^Parsed requests\:\s*4\s/ =~ line }.should == true 88 | end 89 | 90 | it 'should accept a directory as a commandline option' do 91 | output = run("#{log_directory_fixture('s3_logs')} --format amazon_s3") 92 | output.any? { |line| /^Parsed requests:\s*8\s/ =~ line }.should == true 93 | end 94 | end 95 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/log_processor.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer 2 | # The Logprocessor class is used to perform simple processing actions over log files. 3 | # It will go over the log file/stream line by line, pass the line to a processor and 4 | # write the result back to the output file or stream. The processor can alter the 5 | # contents of the line, remain it intact or remove it altogether, based on the current 6 | # file format 7 | # 8 | # Currently, one processors is supported: 9 | # * :strip will remove all irrelevent lines (according to the file format) from the 10 | # sources. A compact, information packed log will remain/. 11 | # 12 | class LogProcessor 13 | attr_reader :mode, :options, :sources, :file_format 14 | attr_accessor :output_file 15 | 16 | # Builds a logprocessor instance from the arguments given on the command line 17 | # command The command hat was used to start the log processor. This will set the 18 | # processing mode. Currently, only :strip is supported. 19 | # arguments The parsed command line arguments (a CommandLine::Arguments instance) 20 | def self.build(command, arguments) 21 | options = { 22 | discard_teaser_lines: arguments[:discard_teaser_lines], 23 | keep_junk_lines: arguments[:keep_junk_lines] 24 | } 25 | 26 | log_processor = RequestLogAnalyzer::LogProcessor.new(arguments[:format].to_sym, command, options) 27 | log_processor.output_file = arguments[:output] if arguments[:output] 28 | 29 | arguments.parameters.each do |input| 30 | log_processor.sources << input 31 | end 32 | 33 | log_processor 34 | end 35 | 36 | # Initializes a new LogProcessor instance. 37 | # format The file format to use (e.g. :rails). 38 | # mode The processing mode 39 | # options A hash with options to take into account 40 | def initialize(format, mode, options = {}) 41 | @options = options 42 | @mode = mode 43 | @sources = [] 44 | @file_format = format 45 | $output_file = nil 46 | end 47 | 48 | # Processes input files by opening it and sending the filestream to process_io, 49 | # in which the actual processing is performed. 50 | # file The file to process 51 | def process_file(file) 52 | File.open(file, 'r') { |io| process_io(io) } 53 | end 54 | 55 | # Processes an input stream by iteration over each line and processing it according to 56 | # the current operation mode 57 | # io The IO instance to process. 58 | def process_io(io) 59 | case mode 60 | when :strip then io.each_line { |line| @output << strip_line(line) } 61 | end 62 | end 63 | 64 | # Returns the line itself if the string matches any of the line definitions. If no match is 65 | # found, an empty line is returned, which will strip the line from the output. 66 | # line The line to strip 67 | def strip_line(line) 68 | file_format.line_definitions.any? { |_name, definition| definition =~ line } ? line : '' 69 | end 70 | 71 | # Runs the log processing by setting up the output stream and iterating over all the 72 | # input sources. Input sources can either be filenames (String instances) or IO streams 73 | # (IO instances). The strings "-" and "STDIN" will be substituted for the $stdin variable. 74 | def run! 75 | if @output_file.nil? 76 | @output = $stdout 77 | else 78 | @output = File.new(@output_file, 'a') 79 | end 80 | 81 | @sources.each do |source| 82 | if source.is_a?(String) && File.exist?(source) 83 | process_file(source) 84 | elsif source.is_a?(IO) 85 | process_io(source) 86 | elsif ['-', 'STDIN'].include?(source) 87 | process_io($stdin) 88 | end 89 | end 90 | 91 | ensure 92 | @output.close if @output.is_a?(File) 93 | end 94 | end 95 | end 96 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/aggregator/database_inserter.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Aggregator 2 | # The database aggregator will create an SQLite3 database with all parsed request information. 3 | # 4 | # The prepare method will create a database schema according to the file format definitions. 5 | # It will also create ActiveRecord::Base subclasses to interact with the created tables. 6 | # Then, the aggregate method will be called for every parsed request. The information of 7 | # these requests is inserted into the tables using the ActiveRecord classes. 8 | # 9 | # A requests table will be created, in which a record is inserted for every parsed request. 10 | # For every line type, a separate table will be created with a request_id field to point to 11 | # the request record, and a field for every parsed value. Finally, a warnings table will be 12 | # created to log all parse warnings. 13 | class DatabaseInserter < Base 14 | attr_reader :request_count, :sources, :database 15 | 16 | # Establishes a connection to the database and creates the necessary database schema for the 17 | # current file format 18 | def prepare 19 | require 'request_log_analyzer/database' 20 | 21 | @sources = {} 22 | @database = RequestLogAnalyzer::Database.new(options[:database]) 23 | @database.file_format = source.file_format 24 | 25 | database.drop_database_schema! if options[:reset_database] 26 | database.create_database_schema! 27 | end 28 | 29 | # Aggregates a request into the database 30 | # This will create a record in the requests table and create a record for every line that has been parsed, 31 | # in which the captured values will be stored. 32 | def aggregate(request) 33 | @request_object = RequestLogAnalyzer::Database::Request.new(first_lineno: request.first_lineno, last_lineno: request.last_lineno) 34 | request.lines.each do |line| 35 | class_columns = database.get_class(line[:line_type]).column_names.reject { |column| %w(id source_id request_id).include?(column) } 36 | attributes = Hash[*line.select { |(key, _)| class_columns.include?(key.to_s) }.flatten] 37 | 38 | # Fix encoding patch for 1.9.2 39 | attributes.each do |k, v| 40 | attributes[k] = v.force_encoding('UTF-8') if v.is_a?(String) 41 | end 42 | 43 | @request_object.send("#{line[:line_type]}_lines").build(attributes) 44 | end 45 | @request_object.save! 46 | rescue SQLite3::SQLException => e 47 | raise Interrupt, e.message 48 | end 49 | 50 | # Finalizes the aggregator by closing the connection to the database 51 | def finalize 52 | @request_count = RequestLogAnalyzer::Database::Request.count 53 | database.disconnect 54 | database.remove_orm_classes! 55 | end 56 | 57 | # Records w warining in the warnings table. 58 | def warning(type, message, lineno) 59 | RequestLogAnalyzer::Database::Warning.create!(warning_type: type.to_s, message: message, lineno: lineno) 60 | end 61 | 62 | # Records source changes in the sources table 63 | def source_change(change, filename) 64 | if File.exist?(filename) 65 | case change 66 | when :started 67 | @sources[filename] = RequestLogAnalyzer::Database::Source.create!(filename: filename) 68 | when :finished 69 | @sources[filename].update_attributes!(filesize: File.size(filename), mtime: File.mtime(filename)) 70 | end 71 | end 72 | end 73 | 74 | # Prints a short report of what has been inserted into the database 75 | def report(output) 76 | output.title('Request database created') 77 | 78 | output << "A database file has been created with all parsed request information.\n" 79 | output << "#{@request_count} requests have been added to the database.\n" 80 | output << "\n" 81 | output << "To open a Ruby console to inspect the database, run the following command.\n" 82 | output << output.colorize(" $ request-log-analyzer console -d #{options[:database]}\n", :bold) 83 | output << "\n" 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /spec/unit/aggregator/database_inserter_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | 3 | describe RequestLogAnalyzer::Aggregator::DatabaseInserter do 4 | 5 | before(:all) do 6 | @log_parser = RequestLogAnalyzer::Source::LogParser.new(testing_format) 7 | end 8 | 9 | # The prepare method is called before the parsing starts. It should establish a connection 10 | # to a database that is suitable for inserting requests later on. 11 | describe '#prepare' do 12 | 13 | before(:each) do 14 | @database = mock_database(:create_database_schema!, :drop_database_schema!, :file_format=) 15 | @database_inserter = RequestLogAnalyzer::Aggregator::DatabaseInserter.new(@log_parser) 16 | RequestLogAnalyzer::Database.stub(:new).and_return(@database) 17 | end 18 | 19 | it 'should establish the database connection' do 20 | RequestLogAnalyzer::Database.should_receive(:new).and_return(@database) 21 | @database_inserter.prepare 22 | end 23 | 24 | it 'should set the file_format' do 25 | @database.should_receive(:file_format=).with(testing_format) 26 | @database_inserter.prepare 27 | end 28 | 29 | it 'should create the database schema during preparation' do 30 | @database.should_receive(:create_database_schema!) 31 | @database_inserter.prepare 32 | end 33 | 34 | it 'should not drop the database schema during preparation if not requested' do 35 | @database.should_not_receive(:drop_database_schema!) 36 | @database_inserter.prepare 37 | end 38 | 39 | it 'should drop the database schema during preparation if requested' do 40 | @database_inserter.options[:reset_database] = true 41 | @database.should_receive(:drop_database_schema!) 42 | @database_inserter.prepare 43 | end 44 | end 45 | 46 | test_databases.each do |name, connection| 47 | 48 | context "using a #{name} database" do 49 | 50 | before(:each) do 51 | @database_inserter = RequestLogAnalyzer::Aggregator::DatabaseInserter.new(@log_parser, database: connection, reset_database: true) 52 | @database_inserter.prepare 53 | 54 | @incomplete_request = testing_format.request(line_type: :first, request_no: 564) 55 | @completed_request = testing_format.request({ line_type: :first, request_no: 564 }, 56 | { line_type: :test, test_capture: 'awesome' }, 57 | { line_type: :test, test_capture: 'indeed' }, 58 | { line_type: :eval, evaluated: { greating: 'howdy' }, greating: 'howdy' }, 59 | { line_type: :last, request_no: 564 }) 60 | end 61 | 62 | after(:each) do 63 | @database_inserter.database.send :remove_orm_classes! 64 | end 65 | 66 | it 'should insert a record in the request table' do 67 | lambda do 68 | @database_inserter.aggregate(@incomplete_request) 69 | end.should change(RequestLogAnalyzer::Database::Request, :count).from(0).to(1) 70 | end 71 | 72 | it 'should insert a record in the first_lines table' do 73 | lambda do 74 | @database_inserter.aggregate(@incomplete_request) 75 | end.should change(@database_inserter.database.get_class(:first), :count).from(0).to(1) 76 | end 77 | 78 | it 'should insert records in all relevant line tables' do 79 | @database_inserter.aggregate(@completed_request) 80 | request = RequestLogAnalyzer::Database::Request.first 81 | request.should satisfy { |r| r.test_lines.length == 2 } 82 | request.should satisfy { |r| r.first_lines.length == 1 } 83 | request.should satisfy { |r| r.eval_lines.length == 1 } 84 | request.should satisfy { |r| r.last_lines.length == 1 } 85 | end 86 | 87 | it 'should log a warning in the warnings table' do 88 | RequestLogAnalyzer::Database::Warning.should_receive(:create!).with(hash_including(warning_type: 'test_warning')) 89 | @database_inserter.warning(:test_warning, 'Testing the warning system', 12) 90 | end 91 | end 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/database.rb: -------------------------------------------------------------------------------- 1 | require 'rubygems' 2 | require 'active_record' 3 | 4 | class RequestLogAnalyzer::Database 5 | require 'request_log_analyzer/database/connection' 6 | include RequestLogAnalyzer::Database::Connection 7 | 8 | attr_accessor :file_format 9 | attr_reader :line_classes 10 | 11 | def initialize(connection_identifier = nil) 12 | @line_classes = [] 13 | RequestLogAnalyzer::Database::Base.database = self 14 | connect(connection_identifier) 15 | end 16 | 17 | # Returns the ORM class for the provided line type 18 | def get_class(line_type) 19 | line_type = line_type.name if line_type.respond_to?(:name) 20 | Object.const_get("#{line_type}_line".camelize) 21 | end 22 | 23 | def default_classes 24 | [RequestLogAnalyzer::Database::Request, RequestLogAnalyzer::Database::Source, RequestLogAnalyzer::Database::Warning] 25 | end 26 | 27 | # Loads the ORM classes by inspecting the tables in the current database 28 | def load_database_schema! 29 | connection.tables.map do |table| 30 | case table.to_sym 31 | when :warnings then RequestLogAnalyzer::Database::Warning 32 | when :sources then RequestLogAnalyzer::Database::Source 33 | when :requests then RequestLogAnalyzer::Database::Request 34 | else load_activerecord_class(table) 35 | end 36 | end 37 | end 38 | 39 | # Returns an array of all the ActiveRecord-bases ORM classes for this database 40 | def orm_classes 41 | default_classes + line_classes 42 | end 43 | 44 | # Loads an ActiveRecord-based class that correspond to the given parameter, which can either be 45 | # a table name or a LineDefinition instance. 46 | def load_activerecord_class(linedefinition_or_table) 47 | case linedefinition_or_table 48 | when String, Symbol 49 | klass_name = linedefinition_or_table.to_s.singularize.camelize 50 | klass = RequestLogAnalyzer::Database::Base.subclass_from_table(linedefinition_or_table) 51 | when RequestLogAnalyzer::LineDefinition 52 | klass_name = "#{linedefinition_or_table.name}_line".camelize 53 | klass = RequestLogAnalyzer::Database::Base.subclass_from_line_definition(linedefinition_or_table) 54 | end 55 | 56 | Object.const_set(klass_name, klass) 57 | klass = Object.const_get(klass_name) 58 | @line_classes << klass 59 | klass 60 | end 61 | 62 | def fileformat_classes 63 | fail 'No file_format provided!' unless file_format 64 | line_classes = file_format.line_definitions.map { |(_name, definition)| load_activerecord_class(definition) } 65 | default_classes + line_classes 66 | end 67 | 68 | # Creates the database schema and related ActiveRecord::Base subclasses that correspond to the 69 | # file format definition. These ORM classes will later be used to create records in the database. 70 | def create_database_schema! 71 | fileformat_classes.each { |klass| klass.create_table! } 72 | end 73 | 74 | # Drops the table of all the ORM classes, and unregisters the classes 75 | def drop_database_schema! 76 | file_format ? fileformat_classes.map(&:drop_table!) : orm_classes.map(&:drop_table!) 77 | remove_orm_classes! 78 | end 79 | 80 | # Registers the default ORM classes in the default namespace 81 | def register_default_orm_classes! 82 | Object.const_set('Request', RequestLogAnalyzer::Database::Request) 83 | Object.const_set('Source', RequestLogAnalyzer::Database::Source) 84 | Object.const_set('Warning', RequestLogAnalyzer::Database::Warning) 85 | end 86 | 87 | # Unregisters every ORM class constant 88 | def remove_orm_classes! 89 | orm_classes.each do |klass| 90 | if klass.respond_to?(:name) && !klass.name.blank? 91 | klass_name = klass.name.split('::').last 92 | Object.send(:remove_const, klass_name) if Object.const_defined?(klass_name) 93 | end 94 | end 95 | end 96 | end 97 | 98 | require 'request_log_analyzer/database/base' 99 | require 'request_log_analyzer/database/request' 100 | require 'request_log_analyzer/database/source' 101 | require 'request_log_analyzer/database/warning' 102 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/tracker/hourly_spread.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Tracker 2 | # Determines the average hourly spread of the parsed requests. 3 | # This spread is shown in a graph form. 4 | # 5 | # Accepts the following options: 6 | # * :if Proc that has to return !nil for a request to be passed to the tracker. 7 | # * :line_type The line type that contains the duration field (determined by the category proc). 8 | # * :output Direct output here (defaults to STDOUT) 9 | # * :unless Proc that has to return nil for a request to be passed to the tracker. 10 | # 11 | # Expects the following items in the update request hash 12 | # * :timestamp in YYYYMMDDHHMMSS format. 13 | # 14 | # Example output: 15 | # Requests graph - average per day per hour 16 | # -------------------------------------------------- 17 | # 7:00 - 330 hits : ======= 18 | # 8:00 - 704 hits : ================= 19 | # 9:00 - 830 hits : ==================== 20 | # 10:00 - 822 hits : =================== 21 | # 11:00 - 823 hits : =================== 22 | # 12:00 - 729 hits : ================= 23 | # 13:00 - 614 hits : ============== 24 | # 14:00 - 690 hits : ================ 25 | # 15:00 - 492 hits : =========== 26 | # 16:00 - 355 hits : ======== 27 | # 17:00 - 213 hits : ===== 28 | # 18:00 - 107 hits : == 29 | # ................ 30 | class HourlySpread < Base 31 | attr_reader :hour_frequencies, :first, :last 32 | 33 | # Check if timestamp field is set in the options and prepare the result time graph. 34 | def prepare 35 | options[:field] ||= :timestamp 36 | @hour_frequencies = (0...24).map { 0 } 37 | @first, @last = 99_999_999_999_999, 0 38 | end 39 | 40 | # Check if the timestamp in the request and store it. 41 | # request The request. 42 | def update(request) 43 | timestamp = request.first(options[:field]) 44 | @hour_frequencies[timestamp.to_s[8..9].to_i] += 1 45 | @first = timestamp if timestamp < @first 46 | @last = timestamp if timestamp > @last 47 | end 48 | 49 | # Total amount of requests tracked 50 | def total_requests 51 | @hour_frequencies.reduce(0) { |sum, value| sum + value } 52 | end 53 | 54 | # First timestamp encountered 55 | def first_timestamp 56 | DateTime.parse(@first.to_s, '%Y%m%d%H%M%S') rescue nil 57 | end 58 | 59 | # Last timestamp encountered 60 | def last_timestamp 61 | DateTime.parse(@last.to_s, '%Y%m%d%H%M%S') rescue nil 62 | end 63 | 64 | # Difference between last and first timestamp. 65 | def timespan 66 | last_timestamp - first_timestamp 67 | end 68 | 69 | # Generate an hourly spread report to the given output object. 70 | # Any options for the report should have been set during initialize. 71 | # output The output object 72 | def report(output) 73 | output.title(title) 74 | 75 | if total_requests == 0 76 | output << "None found.\n" 77 | return 78 | end 79 | 80 | days = [1, timespan].max 81 | output.table({}, { align: :right }, { type: :ratio, width: :rest, treshold: 0.15 }) do |rows| 82 | @hour_frequencies.each_with_index do |requests, index| 83 | ratio = requests.to_f / total_requests.to_f 84 | requests_per_day = (requests / days).ceil 85 | rows << ["#{index.to_s.rjust(3)}:00", '%d hits/day' % requests_per_day, ratio] 86 | end 87 | end 88 | end 89 | 90 | # Returns the title of this tracker for reports 91 | def title 92 | options[:title] || 'Request distribution per hour' 93 | end 94 | 95 | # Returns the found frequencies per hour as a hash for YAML exporting 96 | def to_yaml_object 97 | yaml_object = {} 98 | @hour_frequencies.each_with_index do |freq, hour| 99 | yaml_object["#{hour}:00 - #{hour + 1}:00"] = freq 100 | end 101 | yaml_object 102 | end 103 | end 104 | end 105 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/tracker/frequency.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::Tracker 2 | # Catagorize requests by frequency. 3 | # Count and analyze requests for a specific attribute 4 | # 5 | # === Options 6 | # * :category Proc that handles the request categorization. 7 | # * :if Proc that has to return !nil for a request to be passed to the tracker. 8 | # * :line_type The line type that contains the duration field (determined by the category proc). 9 | # * :nils Track undetermined methods. 10 | # * :title Title do be displayed above the report. 11 | # * :unless Proc that has to return nil for a request to be passed to the tracker. 12 | # 13 | # The items in the update request hash are set during the creation of the Duration tracker. 14 | # 15 | # Example output: 16 | # HTTP methods 17 | # ---------------------------------------------------------------------- 18 | # GET | 22248 hits (46.2%) |================= 19 | # PUT | 13685 hits (28.4%) |=========== 20 | # POST | 11662 hits (24.2%) |========= 21 | # DELETE | 512 hits (1.1%) | 22 | class Frequency < Base 23 | attr_reader :categories 24 | 25 | # Check if categories are set up 26 | def prepare 27 | options[:category] = options[:value] if options[:value] && !options[:category] 28 | fail "No categorizer set up for category tracker #{inspect}" unless options[:category] 29 | 30 | @categorizer = create_lambda(options[:category]) unless options[:multiple] 31 | 32 | # Initialize the categories. Use the list of category names to 33 | @categories = {} 34 | options[:all_categories].each { |cat| @categories[cat] = 0 } if options[:all_categories].is_a?(Enumerable) 35 | end 36 | 37 | # Check HTTP method of a request and store that in the categories hash. 38 | # request The request. 39 | def update(request) 40 | if options[:multiple] 41 | cats = request.every(options[:category]) 42 | cats.each do |cat| 43 | if cat || options[:nils] 44 | @categories[cat] ||= 0 45 | @categories[cat] += 1 46 | end 47 | end 48 | 49 | else 50 | cat = @categorizer.call(request) 51 | if cat || options[:nils] 52 | @categories[cat] ||= 0 53 | @categories[cat] += 1 54 | end 55 | end 56 | end 57 | 58 | # Return the amount of times a HTTP method has been encountered 59 | # cat The HTTP method (:get, :put, :post or :delete) 60 | def frequency(cat) 61 | categories[cat] || 0 62 | end 63 | 64 | # Return the overall frequency 65 | def overall_frequency 66 | categories.reduce(0) { |carry, item| carry + item[1] } 67 | end 68 | 69 | # Return the methods sorted by frequency 70 | def sorted_by_frequency 71 | @categories.sort { |a, b| b[1] <=> a[1] } 72 | end 73 | 74 | # Generate a HTTP method frequency report to the given output object. 75 | # Any options for the report should have been set during initialize. 76 | # output The output object 77 | def report(output) 78 | output.title(options[:title]) if options[:title] 79 | 80 | if @categories.empty? 81 | output << "None found.\n" 82 | else 83 | sorted_categories = output.slice_results(sorted_by_frequency) 84 | total_hits = overall_frequency 85 | 86 | output.table({ align: :left }, { align: :right }, { align: :right }, { type: :ratio, width: :rest }) do |rows| 87 | sorted_categories.each do |(cat, count)| 88 | rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)] 89 | end 90 | end 91 | 92 | end 93 | end 94 | 95 | # Returns a hash with the categories of every category that can be exported to YAML 96 | def to_yaml_object 97 | return nil if @categories.empty? 98 | @categories 99 | end 100 | 101 | # Returns the title of this tracker for reports 102 | def title 103 | options[:title] || 'Request frequency' 104 | end 105 | end 106 | end 107 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/file_format/mysql.rb: -------------------------------------------------------------------------------- 1 | module RequestLogAnalyzer::FileFormat 2 | class Mysql < Base 3 | extend CommonRegularExpressions 4 | 5 | line_definition :time do |line| 6 | line.header = :alternative 7 | line.teaser = /\# Time: / 8 | line.regexp = /\# Time: (#{timestamp('%y%m%d %k:%M:%S')})/ 9 | 10 | line.capture(:timestamp).as(:timestamp) 11 | end 12 | 13 | line_definition :user_host do |line| 14 | line.header = :alternative 15 | line.teaser = /\# User\@Host\: / 16 | line.regexp = /\# User\@Host\: ([\w-]+)\[[\w-]+\] \@ (#{hostname(true)}) \[(#{ip_address(true)})\]/ 17 | 18 | line.capture(:user) 19 | line.capture(:host) 20 | line.capture(:ip) 21 | end 22 | 23 | line_definition :query_statistics do |line| 24 | line.header = :alternative 25 | line.teaser = /\# Query_time: / 26 | line.regexp = /\# Query_time: (\d+(?:\.\d+)?)\s+Lock_time: (\d+(?:\.\d+)?)\s+Rows_sent: (\d+)\s+Rows_examined: (\d+)/ 27 | 28 | line.capture(:query_time).as(:duration, unit: :sec) 29 | line.capture(:lock_time).as(:duration, unit: :sec) 30 | line.capture(:rows_sent).as(:integer) 31 | line.capture(:rows_examined).as(:integer) 32 | end 33 | 34 | line_definition :use_database do |line| 35 | line.regexp = /^\s*use (\w+);\s*$/ 36 | line.capture(:database) 37 | end 38 | 39 | line_definition :query_part do |line| 40 | line.regexp = /^\s*(?!(?:use |\# |SET ))(.*[^;\s])\s*$/ 41 | line.capture(:query_fragment) 42 | end 43 | 44 | line_definition :query do |line| 45 | line.footer = true 46 | line.regexp = /^(?!\s*(?:use |\# |SET ))(.*);\s*$/ 47 | line.capture(:query).as(:sql) 48 | end 49 | 50 | PER_USER = :user 51 | PER_QUERY = :query 52 | PER_USER_QUERY = proc { |request| "#{request[:user]}@#{request.host}: #{request[:query]}" } 53 | 54 | report do |analyze| 55 | analyze.timespan line_type: :time 56 | analyze.frequency :user, title: 'Users with most queries' 57 | analyze.duration :query_time, category: PER_USER, title: 'Query time per user' 58 | analyze.duration :query_time, category: PER_USER_QUERY, title: 'Query time' 59 | 60 | analyze.duration :lock_time, category: PER_USER_QUERY, title: 'Lock time', 61 | if: lambda { |request| request[:lock_time] > 0.0 } 62 | 63 | analyze.numeric_value :rows_examined, category: PER_USER_QUERY, title: 'Rows examined' 64 | analyze.numeric_value :rows_sent, category: PER_USER_QUERY, title: 'Rows sent' 65 | end 66 | 67 | class Request < RequestLogAnalyzer::Request 68 | def convert_sql(value, _definition) 69 | # Recreate the full SQL query by joining all the previous parts and this last line 70 | sql = every(:query_fragment).join("\n") + value 71 | 72 | # Sanitize an SQL query so that it can be used as a category field. 73 | # sql.gsub!(/\/\*.*\*\//, '') # remove comments 74 | sql.gsub!(/\s+/, ' ') # remove excessive whitespace 75 | sql.gsub!(/`([^`]+)`/, '\1') # remove quotes from field names 76 | sql.gsub!(/'\d{4}-\d{2}-\d{2}'/, ':date') # replace dates 77 | sql.gsub!(/'\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}'/, ':datetime') # replace timestamps 78 | sql.gsub!(/'[^']*'/, ':string') # replace strings 79 | sql.gsub!(/\b\d+\b/, ':int') # replace integers 80 | sql.gsub!(/(:int,)+:int/, ':ints') # replace multiple ints by a list 81 | sql.gsub!(/(:string,)+:string/, ':strings') # replace multiple strings by a list 82 | 83 | sql.strip 84 | end 85 | 86 | def host 87 | self[:host] == '' || self[:host].nil? ? self[:ip] : self[:host] 88 | end 89 | 90 | # Convert the timestamp to an integer 91 | def convert_timestamp(value, _definition) 92 | _, y, m, d, h, i, s = value.split(/(\d\d)(\d\d)(\d\d)\s+(\d?\d):(\d\d):(\d\d)/) 93 | ('20%s%s%s%s%s%s' % [y, m, d, h.rjust(2, '0'), i, s]).to_i 94 | end 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /spec/fixtures/merb.log: -------------------------------------------------------------------------------- 1 | ~ Loaded DEVELOPMENT Environment... 2 | ~ Connecting to database... 3 | ~ loading gem 'merb_datamapper' ... 4 | ~ loading gem 'gettext' ... 5 | ~ loading gem 'merb_helpers' ... 6 | ~ loading gem 'merb-assets' ... 7 | ~ loading gem 'merb-action-args' ... 8 | ~ loading gem 'merb-mailer' ... 9 | ~ loading gem 'merb_param_protection' ... 10 | ~ loading gem 'merb_has_flash' ... 11 | ~ loading gem 'merb_forgery_protection' ... 12 | ~ loading gem 'dm-validations' ... 13 | ~ loading gem 'dm-timestamps' ... 14 | ~ loading gem 'dm-migrations' ... 15 | ~ loading gem 'dm-aggregates' ... 16 | ~ loading gem 'dm-adjust' ... 17 | ~ loading gem 'dm-serializer' ... 18 | ~ loading gem 'dm-constraints' ... 19 | ~ loading gem 'dm-timeline' ... 20 | ~ loading gem 'dm-searchable' ... 21 | ~ loading gem 'dm-audited' ... 22 | ~ loading gem 'lib/extensions' ... 23 | ~ loading gem 'lib/authenticated_system/authenticated_dependencies' ... 24 | ~ Compiling routes... 25 | ~ Using 'share-nothing' cookie sessions (4kb limit per client) 26 | ~ Using Mongrel adapter 27 | ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008 28 | ~ Params: {"_method"=>"delete", "authenticity_token"=>"[FILTERED]", "action"=>"destroy", "method"=>"delete", "controller"=>"session"} 29 | ~ Cookie deleted: auth_token => nil 30 | ~ Redirecting to: / (302) 31 | ~ {:dispatch_time=>0.243424, :after_filters_time=>6.9e-05, :before_filters_time=>0.213213, :action_time=>0.241652} 32 | ~ 33 | 34 | ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008 35 | ~ Params: {"action"=>"index", "controller"=>"dashboard"} 36 | ~ Redirecting to: /login (302) 37 | ~ {:dispatch_time=>0.002649, :after_filters_time=>7.4e-05, :action_time=>0.001951} 38 | ~ 39 | 40 | ~ Started request handling: Fri Aug 29 11:10:23 +0200 2008 41 | ~ Params: {"action"=>"create", "controller"=>"session"} 42 | ~ {:dispatch_time=>0.006117, :after_filters_time=>6.1e-05, :before_filters_time=>0.000712, :action_time=>0.005833} 43 | ~ 44 | 45 | ~ Started request handling: Fri Aug 29 11:10:27 +0200 2008 46 | ~ Params: {"authenticity_token"=>"[FILTERED]", "action"=>"create", "controller"=>"session", "login"=>"username", "password"=>"[FILTERED]", "remember_me"=>"0"} 47 | ~ Redirecting to: / (302) 48 | ~ {:dispatch_time=>0.006652, :after_filters_time=>0.000143, :before_filters_time=>0.000861, :action_time=>0.006171} 49 | ~ 50 | 51 | ~ Started request handling: Fri Aug 29 11:10:27 +0200 2008 52 | ~ Params: {"action"=>"index", "controller"=>"dashboard"} 53 | ~ Redirecting to: /dashboard (302) 54 | ~ {:dispatch_time=>0.008241, :after_filters_time=>0.000126, :before_filters_time=>0.002632, :action_time=>0.007711} 55 | ~ 56 | 57 | ~ Started request handling: Fri Aug 29 11:10:27 +0200 2008 58 | ~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"} 59 | ~ {:dispatch_time=>0.009458, :after_filters_time=>0.000103, :before_filters_time=>0.00266, :action_time=>0.008742} 60 | ~ 61 | 62 | ~ Started request handling: Fri Aug 29 11:10:29 +0200 2008 63 | ~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"} 64 | ~ {:dispatch_time=>0.102725, :after_filters_time=>0.000115, :before_filters_time=>0.00411, :action_time=>0.101836} 65 | ~ 66 | 67 | ~ Started request handling: Fri Aug 29 11:10:30 +0200 2008 68 | ~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"organisations"} 69 | ~ {:dispatch_time=>0.042575, :after_filters_time=>8.9e-05, :before_filters_time=>0.004267, :action_time=>0.041762} 70 | ~ 71 | 72 | ~ Started request handling: Fri Aug 29 11:10:31 +0200 2008 73 | ~ Params: {"action"=>"index", "namespace"=>"dashboard", "controller"=>"dashboard"} 74 | ~ {:dispatch_time=>0.010311, :after_filters_time=>8.0e-05, :before_filters_time=>0.003195, :action_time=>0.009567} 75 | ~ 76 | 77 | ~ Started request handling: Fri Aug 29 11:10:33 +0200 2008 78 | ~ Params: {"format"=>nil, "action"=>"index", "namespace"=>"dashboard", "controller"=>"employees"} 79 | ~ {:dispatch_time=>0.012913, :after_filters_time=>7.1e-05, :before_filters_time=>0.004422, :action_time=>0.012141} 80 | ~ 81 | 82 | ~ Started request handling: Fri Aug 29 11:10:35 +0200 2008 83 | ~ Params: {"action"=>"new", "namespace"=>"dashboard", "controller"=>"employees"} 84 | ~ {:dispatch_time=>0.013051, :after_filters_time=>7.8e-05, :before_filters_time=>0.003576, :action_time=>0.011773} 85 | -------------------------------------------------------------------------------- /lib/request_log_analyzer/output.rb: -------------------------------------------------------------------------------- 1 | # Module for generating output 2 | module RequestLogAnalyzer::Output 3 | # Loads a Output::Base subclass instance. 4 | def self.load(file_format, *args) 5 | klass = nil 6 | if file_format.is_a?(RequestLogAnalyzer::Output::Base) 7 | # this already is a file format! return itself 8 | return file_format 9 | 10 | elsif file_format.is_a?(Class) && file_format.ancestors.include?(RequestLogAnalyzer::Output::Base) 11 | # a usable class is provided. Use this format class. 12 | klass = file_format 13 | 14 | elsif file_format.is_a?(String) && File.exist?(file_format) 15 | # load a format from a ruby file 16 | require file_format 17 | const = RequestLogAnalyzer.to_camelcase(File.basename(file_format, '.rb')) 18 | if RequestLogAnalyzer::FileFormat.const_defined?(const) 19 | klass = RequestLogAnalyzer::Output.const_get(const) 20 | elsif Object.const_defined?(const) 21 | klass = Object.const_get(const) 22 | else 23 | fail "Cannot load class #{const} from #{file_format}!" 24 | end 25 | 26 | else 27 | # load a provided file format 28 | klass = RequestLogAnalyzer::Output.const_get(RequestLogAnalyzer.to_camelcase(file_format)) 29 | end 30 | 31 | # check the returned klass to see if it can be used 32 | fail "Could not load a file format from #{file_format.inspect}" if klass.nil? 33 | fail 'Invalid FileFormat class' unless klass.is_a?(Class) && klass.ancestors.include?(RequestLogAnalyzer::Output::Base) 34 | 35 | klass.create(*args) # return an instance of the class 36 | end 37 | 38 | # Base Class used for generating output for reports. 39 | # All output should inherit fromt this class. 40 | class Base 41 | attr_accessor :io, :options, :style 42 | 43 | # Initialize a report 44 | # io iO Object (file, STDOUT, etc.) 45 | # options Specific style options 46 | def initialize(io, options = {}) 47 | @io = io 48 | @options = options 49 | @style = options[:style] || { cell_separator: true, table_border: false } 50 | end 51 | 52 | def report_tracker(tracker) 53 | tracker.report(self) 54 | end 55 | 56 | # Apply a style block.. with style :) 57 | def with_style(temp_style = {}) 58 | old_style = @style 59 | @style = @style.merge(temp_style) 60 | yield(self) if block_given? 61 | @style = old_style 62 | end 63 | 64 | # Generate a header for a report 65 | def header 66 | end 67 | 68 | # Generate the footer of a report 69 | def footer 70 | end 71 | 72 | def slice_results(array) 73 | return array if options[:amount] == :all 74 | array.slice(0, options[:amount]) # otherwise 75 | end 76 | 77 | # Generate a report table and push it into the output object. 78 | # Yeilds a rows array into which the rows can be pushed 79 | # *colums Array of Column hashes (see Column options). 80 | # &block: A block yeilding the rows. 81 | # 82 | # === Column options 83 | # Columns is an array of hashes containing the column definitions. 84 | # * :align Alignment :left or :right 85 | # * :treshold Width in characters or :rest 86 | # * :type :ratio or nil 87 | # * :width Width in characters or :rest 88 | # 89 | # === Example 90 | # The output object should support table definitions: 91 | # 92 | # output.table({:align => :left}, {:align => :right }, {:align => :right}, {:type => :ratio, :width => :rest}) do |rows| 93 | # sorted_frequencies.each do |(cat, count)| 94 | # rows << [cat, "#{count} hits", '%0.1f%%' % ((count.to_f / total_hits.to_f) * 100.0), (count.to_f / total_hits.to_f)] 95 | # end 96 | # end 97 | # 98 | def table(*_columns, &_block) 99 | end 100 | 101 | protected 102 | 103 | # Check if a given table defination hash includes a header (title) 104 | # columns The columns hash 105 | def table_has_header?(columns) 106 | columns.any? { |column| !column[:title].nil? } 107 | end 108 | end 109 | end 110 | 111 | require 'request_log_analyzer/output/fixed_width' 112 | require 'request_log_analyzer/output/html' 113 | --------------------------------------------------------------------------------