├── Gemfile ├── Gemfile.lock ├── README.md ├── groktest.rb └── patterns └── pure-ruby ├── asterisk ├── base ├── java └── ruby /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | 3 | gem "jls-grok", :git => "https://github.com/jordansissel/ruby-grok" 4 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GIT 2 | remote: https://github.com/jordansissel/ruby-grok 3 | revision: 05e07494125fce3dbfd27576b3b0e729bf7cbb7f 4 | specs: 5 | jls-grok (0.10.7) 6 | cabin (~> 0.4.0) 7 | 8 | GEM 9 | remote: https://rubygems.org/ 10 | specs: 11 | cabin (0.4.4) 12 | json 13 | json (1.7.5) 14 | 15 | PLATFORMS 16 | ruby 17 | 18 | DEPENDENCIES 19 | jls-grok! 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Grok patterns for Asterisk log 2 | ============================== 3 | 4 | This is a small test for using [grok](http://code.google.com/p/semicomplete/wiki/Grok) 5 | to extract useful data from Asterisk log files. The pattern file can the be used in a 6 | tool like [Logstash](http://logstash.net/). 7 | 8 | Ruby script inspired by http://blog.bealetech.com/content/testing-logstash-grok-filters 9 | 10 | The Asterisk-specifc patterns are located in the file patterns/pure-ruby/asterisk 11 | -------------------------------------------------------------------------------- /groktest.rb: -------------------------------------------------------------------------------- 1 | # This file reads the logfile "fullsmall" and prints something for every line 2 | # where no grok pattern was found. This is for developing new patterns 3 | 4 | require 'rubygems' 5 | require 'bundler/setup' 6 | 7 | require 'grok-pure' 8 | require 'pp' 9 | require 'benchmark' 10 | 11 | grok = Grok.new 12 | 13 | grok.add_patterns_from_file("patterns/pure-ruby/base") 14 | grok.add_patterns_from_file("patterns/pure-ruby/asterisk") 15 | 16 | grok.compile("%{ASTLOG}") 17 | 18 | # Check for content without a ASTCONTENT pattern from the library 19 | # Ignore some of the uknown contents 20 | def unparsed_content?(captures) 21 | content_types = [ 22 | "ASTCONTENT_EXECUTE", 23 | "ASTCONTENT_CHANNELJUMP", 24 | "ASTCONTENT_GOTO", 25 | "ASTCONTENT_SIPCOSMARK", 26 | "ASTCONTENT_PLAYAUDIO", 27 | "ASTCONTENT_DTMF", 28 | "ASTCONTENT_ASTMANAGER", 29 | "ASTCONTENT_CHANNELEVENT1", 30 | "ASTCONTENT_CHANNELEVENT2", 31 | "ASTCONTENT_CHANNELEVENT3", 32 | "ASTCONTENT_REGISTRATION_TIMEOUT", 33 | "ASTCONTENT_CONNECTION_REFUSED", 34 | "ASTCONTENT_SPAWN", 35 | "ASTCONTENT_UNKNOWN_SIPMESSAGE", 36 | "ASTCONTENT_INVALID_EXTENSION", 37 | "ASTCONTENT_QUEUE_EVENT", 38 | "ASTCONTENT_USERINPUT" 39 | ] 40 | ignore = /Asterisk Queue Logger restarted|Remote UNIX connection|timeout set to|ignoring 'image' media offer|Broken pipe|MixMonitor close filestream|Found/ 41 | return false if not captures['ASTCONTENT'].first or ignore.match(captures['ASTCONTENT'].first) 42 | content_types.each { |c| 43 | return false if captures[c] != [nil] and captures[c] != [] 44 | } 45 | return true 46 | end 47 | 48 | def process_line(line, grok) 49 | if line.strip != "" 50 | match = grok.match(line) 51 | if match 52 | if unparsed_content?(match.captures()) 53 | #pp match.captures() 54 | #puts "\n" 55 | puts line 56 | 57 | end 58 | else 59 | puts "Unmatched line: #{line}" 60 | 61 | end 62 | return 1 63 | end 64 | return 0 65 | end 66 | 67 | File.open("fullsmall", "r") do |infile| 68 | lc = 0 69 | time = Benchmark.realtime do 70 | while (line = infile.gets) 71 | lc += process_line(line, grok) 72 | end 73 | end 74 | puts "#{lc} lines processed in #{time*1000} seconds" 75 | end 76 | 77 | 78 | -------------------------------------------------------------------------------- /patterns/pure-ruby/asterisk: -------------------------------------------------------------------------------- 1 | SIP_HEADER [A-Z_]+ 2 | 3 | ASTLEVEL (?:VERBOSE|ERROR|NOTICE|INFO|DEBUG|DTMF|WARNING) 4 | ASTLANGUAGE [a-z]{1,3} 5 | 6 | # Special pattern for "Ext." Logs 7 | ASTEXTNO [A-Za-z0-9_]+ 8 | 9 | # Valid DTMF keys 10 | ASTDTMF [0-9#*] 11 | 12 | # Log source file (Normally a c file) 13 | ASTSRC [-a-z._0-9/]+ 14 | 15 | # Registration info 16 | ASTREGISTRATION %{USERNAME}@%{IPORHOST} 17 | 18 | # Context 19 | ASTCONTEXTEXTENSION [A-Za-z0-9*#~_]+ 20 | ASTCONTEXTNAME [-a-z0-9]+ 21 | ASTCONTEXTPOSITION \d+ 22 | ASTCONTEXT %{ASTCONTEXTEXTENSION}@%{ASTCONTEXTNAME}:%{ASTCONTEXTPOSITION} 23 | 24 | # Application 25 | ASTAPP [A-Z][A-Za-z0-9]+ 26 | ASTAPP_AUDIOFILE [-a-zA-Z0-9_./]+ 27 | 28 | # Interfaces 29 | AST_SIP_INTERFACE SIP/[-_A-Za-z0-9]+ 30 | ASTINTERFACE %{AST_SIP_INTERFACE} 31 | 32 | # Channels 33 | ASTCHANNEL_SIP %{AST_SIP_INTERFACE}-[a-f0-9]+ 34 | ASTCHANNEL_LOCAL Local/[-a-z0-9@;]+ 35 | ASTCHANNEL (:?%{ASTCHANNEL_SIP}|%{ASTCHANNEL_LOCAL}) 36 | 37 | # Different log content types 38 | ASTCONTENT_CHANNELEVENT1 (:?CDR updated on|Timeout on|Stopped music on hold on|Started music on hold, class '%{ASTAPP_AUDIOFILE}', on|(:?Begin|End) MixMonitor Recording) %{ASTCHANNEL:asterisk_channel} 39 | ASTCONTENT_CHANNELEVENT2 (Locally bridging )?%{ASTCHANNEL:asterisk_channel}.*%{ASTCHANNEL:asterisk_channel} 40 | ASTCONTENT_CHANNELEVENT3 %{ASTCHANNEL:asterisk_channel} is ringing 41 | ASTCONTENT_EXECUTE Executing \[%{ASTCONTEXT:asterisk_context}\] %{ASTAPP:asterisk_app}\("%{ASTCHANNEL:asterisk_channel}", "%{GREEDYDATA:asterisk_app_params}"\) in new stack 42 | ASTCONTENT_CHANNELJUMP Channel '%{ASTCHANNEL:asterisk_channel}' jumping out of macro '%{ASTCONTEXTNAME:asterisk_context}' 43 | ASTCONTENT_GOTO Goto \(%{ASTCONTEXTNAME:asterisk_context},%{ASTCONTEXTEXTENSION:asterisk_extension},%{ASTCONTEXTPOSITION:asterisk_context_position}\) 44 | ASTCONTENT_SIPCOSMARK Using SIP RTP CoS mark %{NUMBER} 45 | ASTCONTENT_ASTMANAGER Manager '%{USERNAME:asterisk_manager_name}' logged (:?off|on) from %{IPORHOST:asterisk_manager_host} 46 | ASTCONTENT_CDRUPDATE on %{ASTCHANNEL:asterisk_channel} 47 | ASTCONTENT_PLAYAUDIO <%{ASTCHANNEL:asterisk_channel}> Playing '%{ASTAPP_AUDIOFILE:audiofile}' \(language '%{ASTLANGUAGE}'\) 48 | ASTCONTENT_REGISTRATION_TIMEOUT Registration for '%{ASTREGISTRATION}' timed out, trying again \(Attempt #%{NUMBER:asterisk_registration_attempt}\) 49 | ASTCONTENT_INVALID_EXTENSION Invalid extension '%{ASTCONTEXTEXTENSION:asterisk_extension}' in context '%{ASTCONTEXTNAME:asterisk_context}' on %{ASTCHANNEL} 50 | ASTCONTENT_CONNECTION_REFUSED Connect attempt from '%{IPORHOST:asterisk_connection_refused_host}' unable to authenticate 51 | ASTCONTENT_QUEUE_EVENT Added interface '%{ASTINTERFACE}' to queue '%{NOTSPACE:asterisk_queue}' 52 | ASTCONTENT_UNKNOWN_SIPMESSAGE Unable to parse %{SIP_HEADER:sip_header} message from %{ASTREGISTRATION:asterisk_external} 53 | ASTCONTENT_SPAWN Spawn extension \(%{ASTCONTEXTNAME:asterisk_context}, %{ASTCONTEXTEXTENSION:asterisk_extension}, %{ASTCONTEXTPOSITION:asterisk_context_position}\) exited .* on '%{ASTCHANNEL:asterisk_channel}' 54 | ASTCONTENT_USERINPUT User entered '%{DATA:asterisk_userinput}' 55 | ASTCONTENT_DTMF DTMF (:?begin|end)(:? passthrough| ignored)? '%{ASTDTMF:asterisk_dtmf_number}'(:? received)? on %{ASTCHANNEL}(:?, duration %{NUMBER:asterisk_dtmf_duration} ms)? 56 | ASTCONTENT [-=\s]*(:?%{ASTCONTENT_EXECUTE}|%{ASTCONTENT_CHANNELJUMP}|%{ASTCONTENT_GOTO}|%{ASTCONTENT_PLAYAUDIO}|%{ASTCONTENT_SPAWN}|%{ASTCONTENT_ASTMANAGER}|%{ASTCONTENT_DTMF}|%{ASTCONTENT_REGISTRATION_TIMEOUT}|%{ASTCONTENT_CHANNELEVENT1}|%{ASTCONTENT_CHANNELEVENT2}|%{ASTCONTENT_CHANNELEVENT3}|%{ASTCONTENT_INVALID_EXTENSION}|%{ASTCONTENT_CONNECTION_REFUSED}|%{ASTCONTENT_QUEUE_EVENT}|%{ASTCONTENT_USERINPUT}|%{ASTCONTENT_UNKNOWN_SIPMESSAGE}|%{ASTCONTENT_SIPCOSMARK}|%{GREEDYDATA}) 57 | 58 | # different log types 59 | ASTLOG1 \[%{TIMESTAMP_ISO8601:timestamp}\] %{ASTLEVEL:severity}\[\d+\] %{ASTSRC:asterisk_src_file}:%{ASTCONTENT} 60 | ASTLOG2 \[%{TIMESTAMP_ISO8601:timestamp}\] %{ASTLEVEL:severity}\[\d+\] Ext. %{ASTEXTNO}: %{GREEDYDATA} 61 | ASTTIMEOUT Packet timed out after %{NUMBER}ms with no response 62 | ASTLOG ^(:?%{ASTLOG1}|%{ASTLOG2}|%{ASTTIMEOUT}) -------------------------------------------------------------------------------- /patterns/pure-ruby/base: -------------------------------------------------------------------------------- 1 | USERNAME [a-zA-Z0-9_-]+ 2 | USER %{USERNAME} 3 | INT (?:[+-]?(?:[0-9]+)) 4 | BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) 5 | NUMBER (?:%{BASE10NUM}) 6 | BASE16NUM (?[^\\"]+|\\.)*")|(?:'(?>[^\\']+|\\.)*')|(?:`(?>[^\\`]+|\\.)*`)) 16 | 17 | # Networking 18 | MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) 19 | CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) 20 | WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) 21 | COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) 22 | IP (? 84 | HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT:ZONE} 85 | 86 | # Shortcuts 87 | QS %{QUOTEDSTRING} 88 | 89 | # Log formats 90 | SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: 91 | COMBINEDAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "%{WORD:verb} %{URIPATHPARAM:request} HTTP/%{NUMBER:httpversion}" %{NUMBER:response} (?:%{NUMBER:bytes}|-) "(?:%{URI:referrer}|-)" %{QS:agent} 92 | -------------------------------------------------------------------------------- /patterns/pure-ruby/java: -------------------------------------------------------------------------------- 1 | JAVACLASS (?:[a-z-]+\.)[A-Za-z0-9]+ 2 | JAVAFILE (?:[A-Za-z0-9_.-]}) 3 | JAVASTACKTRACEPART "at %{JAVACLASS:class}\.%{WORD:method}\(%{JAVAFILE:file}:%{NUMBER:line}\) 4 | -------------------------------------------------------------------------------- /patterns/pure-ruby/ruby: -------------------------------------------------------------------------------- 1 | RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO) 2 | RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601} #{POSINT:pid}\] *%{RUBY_LOGLEVEL} -- : %{DATA:message} 3 | 4 | --------------------------------------------------------------------------------