├── .gitignore ├── lib ├── patterns │ ├── mcollective │ ├── redis │ ├── postgresql │ ├── java │ ├── ruby │ ├── mcollective-patterns │ ├── mongodb │ ├── linux-syslog │ ├── junos │ ├── haproxy │ ├── grok-patterns │ ├── firewalls │ └── nagios ├── index.d.ts ├── test.js └── index.js ├── package.json ├── LICENSE └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | *.iml 3 | /.idea 4 | -------------------------------------------------------------------------------- /lib/patterns/mcollective: -------------------------------------------------------------------------------- 1 | MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}: -------------------------------------------------------------------------------- /lib/patterns/redis: -------------------------------------------------------------------------------- 1 | REDISTIMESTAMP %{MONTHDAY} %{MONTH} %{TIME} 2 | REDISLOG \[%{POSINT:pid}\] %{REDISTIMESTAMP:timestamp} \* 3 | -------------------------------------------------------------------------------- /lib/patterns/postgresql: -------------------------------------------------------------------------------- 1 | # Default postgresql pg_log format pattern 2 | POSTGRESQL %{DATESTAMP:timestamp} %{TZ} %{DATA:user_id} %{GREEDYDATA:connection_id} %{POSINT:pid} 3 | -------------------------------------------------------------------------------- /lib/patterns/java: -------------------------------------------------------------------------------- 1 | JAVACLASS (?:[a-zA-Z0-9-]+\.)+[A-Za-z0-9$_]+ 2 | JAVAFILE (?:[A-Za-z0-9_. -]+) 3 | JAVASTACKTRACEPART at %{JAVACLASS:class}\.%{WORD:method}$%{JAVAFILE:file}:%{NUMBER:line}$ -------------------------------------------------------------------------------- /lib/patterns/ruby: -------------------------------------------------------------------------------- 1 | RUBY_LOGLEVEL (?:DEBUG|FATAL|ERROR|WARN|INFO) 2 | RUBY_LOGGER [DFEWI], \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\] *%{RUBY_LOGLEVEL:loglevel} -- +%{DATA:progname}: %{GREEDYDATA:message} -------------------------------------------------------------------------------- /lib/patterns/mcollective-patterns: -------------------------------------------------------------------------------- 1 | # Remember, these can be multi-line events. 2 | MCOLLECTIVE ., \[%{TIMESTAMP_ISO8601:timestamp} #%{POSINT:pid}\]%{SPACE}%{LOGLEVEL:event_level} 3 | 4 | MCOLLECTIVEAUDIT %{TIMESTAMP_ISO8601:timestamp}: -------------------------------------------------------------------------------- /lib/patterns/mongodb: -------------------------------------------------------------------------------- 1 | MONGO_LOG %{SYSLOGTIMESTAMP:timestamp} \[%{WORD:component}\] %{GREEDYDATA:message} 2 | MONGO_QUERY \{ (?<={ ).*(?= } ntoreturn:) \} 3 | MONGO_SLOWQUERY %{WORD} %{MONGO_WORDDASH:database}\.%{MONGO_WORDDASH:collection} %{WORD}: %{MONGO_QUERY:query} %{WORD}:%{NONNEGINT:ntoreturn} %{WORD}:%{NONNEGINT:ntoskip} %{WORD}:%{NONNEGINT:nscanned}.*nreturned:%{NONNEGINT:nreturned}..+ (?[0-9]+)ms 4 | MONGO_WORDDASH \b[\w-]+\b -------------------------------------------------------------------------------- /lib/index.d.ts: -------------------------------------------------------------------------------- 1 | export class GrokPattern { 2 | parse(str: string, callback: { (err: any, result: any): any }): any; 3 | parseSync(str: string): any; 4 | } 5 | 6 | export class GrokCollection { 7 | createPattern(expression: string, id?: string): GrokPattern; 8 | getPattern(id: string): GrokPattern; 9 | load(filePath: string, callback: { (err?: any): any }): any; 10 | loadSync(filePath: string): number; 11 | count(): number; 12 | } 13 | 14 | export function loadDefault(loadModulesOrCallback?: string[] | string | { (err: any, collection: GrokCollection): any }, callback?: { (err: any, collection: GrokCollection): any }): any; 15 | 16 | export function loadDefaultSync(loadModules?: string[] | string): GrokCollection; 17 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "node-grok", 3 | "version": "2.0.3", 4 | "description": "Regular expression template library inspired by logstash grok filter module", 5 | "repository": "https://github.com/Beh01der/node-grok.git", 6 | "main": "./lib/index.js", 7 | "typings": "./lib/index.d.ts", 8 | "scripts": { 9 | "test": "node ./node_modules/mocha/bin/_mocha ./lib/test.js" 10 | }, 11 | "keywords": [ 12 | "regex", 13 | "grok", 14 | "logstash" 15 | ], 16 | "author": "Andrey Chausenko", 17 | "license": "ISC", 18 | "dependencies": { 19 | "async": "1.5.2", 20 | "collections": "3.0.0", 21 | "oniguruma": "6.1.0" 22 | }, 23 | "devDependencies": { 24 | "chai": "3.2.0", 25 | "mocha": "2.2.5" 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2015, Andrey Chausenko 2 | 3 | Permission to use, copy, modify, and/or distribute this software for any 4 | purpose with or without fee is hereby granted, provided that the above 5 | copyright notice and this permission notice appear in all copies. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 8 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 9 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 10 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 11 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 12 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 13 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 14 | 15 | -------------------------------------------------------------------------------- /lib/patterns/linux-syslog: -------------------------------------------------------------------------------- 1 | SYSLOG5424PRINTASCII [!-~]+ 2 | 3 | SYSLOGBASE2 (?:%{SYSLOGTIMESTAMP:timestamp}|%{TIMESTAMP_ISO8601:timestamp8601}) (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: 4 | SYSLOGPAMSESSION %{SYSLOGBASE} (?=%{GREEDYDATA:message})%{WORD:pam_module}$%{DATA:pam_caller}$: session %{WORD:pam_session_state} for user %{USERNAME:username}(?: by %{GREEDYDATA:pam_by})? 5 | 6 | CRON_ACTION [A-Z ]+ 7 | CRONLOG %{SYSLOGBASE} $%{USER:user}$ %{CRON_ACTION:action} $%{DATA:message}$ 8 | 9 | SYSLOGLINE %{SYSLOGBASE2} %{GREEDYDATA:message} 10 | 11 | # IETF 5424 syslog(8) format (see http://www.rfc-editor.org/info/rfc5424) 12 | SYSLOG5424PRI <%{NONNEGINT:syslog5424_pri}> 13 | SYSLOG5424SD \[%{DATA}\]+ 14 | SYSLOG5424BASE %{SYSLOG5424PRI}%{NONNEGINT:syslog5424_ver} +(?:%{TIMESTAMP_ISO8601:syslog5424_ts}|-) +(?:%{HOSTNAME:syslog5424_host}|-) +(-|%{SYSLOG5424PRINTASCII:syslog5424_app}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_proc}) +(-|%{SYSLOG5424PRINTASCII:syslog5424_msgid}) +(?:%{SYSLOG5424SD:syslog5424_sd}|-|) 15 | 16 | SYSLOG5424LINE %{SYSLOG5424BASE} +%{GREEDYDATA:syslog5424_msg} -------------------------------------------------------------------------------- /lib/patterns/junos: -------------------------------------------------------------------------------- 1 | # JUNOS 11.4 RT_FLOW patterns 2 | RT_FLOW_EVENT (RT_FLOW_SESSION_CREATE|RT_FLOW_SESSION_CLOSE|RT_FLOW_SESSION_DENY) 3 | 4 | RT_FLOW1 %{RT_FLOW_EVENT:event}: %{GREEDYDATA:close-reason}: %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{DATA:nat-src-port}->%{IP:nat-dst-ip}/%{DATA:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} \d+$%{DATA:sent}$ \d+$%{DATA:received}$ %{INT:elapsed-time} .* 5 | 6 | RT_FLOW2 %{RT_FLOW_EVENT:event}: session created %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{IP:nat-src-ip}/%{DATA:nat-src-port}->%{IP:nat-dst-ip}/%{DATA:nat-dst-port} %{DATA:src-nat-rule-name} %{DATA:dst-nat-rule-name} %{INT:protocol-id} %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} %{INT:session-id} .* 7 | 8 | RT_FLOW3 %{RT_FLOW_EVENT:event}: session denied %{IP:src-ip}/%{DATA:src-port}->%{IP:dst-ip}/%{DATA:dst-port} %{DATA:service} %{INT:protocol-id}$\d$ %{DATA:policy-name} %{DATA:from-zone} %{DATA:to-zone} .* 9 | -------------------------------------------------------------------------------- /lib/patterns/haproxy: -------------------------------------------------------------------------------- 1 | ## These patterns were tested w/ haproxy-1.4.15 2 | 3 | ## Documentation of the haproxy log formats can be found at the following links: 4 | ## http://code.google.com/p/haproxy-docs/wiki/HTTPLogFormat 5 | ## http://code.google.com/p/haproxy-docs/wiki/TCPLogFormat 6 | 7 | HAPROXYTIME (?!<[0-9])%{HOUR:haproxy_hour}:%{MINUTE:haproxy_minute}(?::%{SECOND:haproxy_second})(?![0-9]) 8 | HAPROXYDATE %{MONTHDAY:haproxy_monthday}/%{MONTH:haproxy_month}/%{YEAR:haproxy_year}:%{HAPROXYTIME:haproxy_time}.%{INT:haproxy_milliseconds} 9 | 10 | # Override these default patterns to parse out what is captured in your haproxy.cfg 11 | HAPROXYCAPTUREDREQUESTHEADERS %{DATA:captured_request_headers} 12 | HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:captured_response_headers} 13 | 14 | # Example: 15 | # These haproxy config lines will add data to the logs that are captured 16 | # by the patterns below. Place them in your custom patterns directory to 17 | # override the defaults. 18 | # 19 | # capture request header Host len 40 20 | # capture request header X-Forwarded-For len 50 21 | # capture request header Accept-Language len 50 22 | # capture request header Referer len 200 23 | # capture request header User-Agent len 200 24 | # 25 | # capture response header Content-Type len 30 26 | # capture response header Content-Encoding len 10 27 | # capture response header Cache-Control len 200 28 | # capture response header Last-Modified len 200 29 | # 30 | # HAPROXYCAPTUREDREQUESTHEADERS %{DATA:request_header_host}\|%{DATA:request_header_x_forwarded_for}\|%{DATA:request_header_accept_language}\|%{DATA:request_header_referer}\|%{DATA:request_header_user_agent} 31 | # HAPROXYCAPTUREDRESPONSEHEADERS %{DATA:response_header_content_type}\|%{DATA:response_header_content_encoding}\|%{DATA:response_header_cache_control}\|%{DATA:response_header_last_modified} 32 | 33 | # parse a haproxy 'httplog' line 34 | HAPROXYHTTP %{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_request}/%{INT:time_queue}/%{INT:time_backend_connect}/%{INT:time_backend_response}/%{NOTSPACE:time_duration} %{INT:http_status_code} %{NOTSPACE:bytes_read} %{DATA:captured_request_cookie} %{DATA:captured_response_cookie} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} (\{%{HAPROXYCAPTUREDREQUESTHEADERS}\})?( )?(\{%{HAPROXYCAPTUREDRESPONSEHEADERS}\})?( )?"(|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))?" 35 | 36 | # parse a haproxy 'tcplog' line 37 | HAPROXYTCP %{SYSLOGTIMESTAMP:syslog_timestamp} %{IPORHOST:syslog_server} %{SYSLOGPROG}: %{IP:client_ip}:%{INT:client_port} \[%{HAPROXYDATE:accept_date}\] %{NOTSPACE:frontend_name} %{NOTSPACE:backend_name}/%{NOTSPACE:server_name} %{INT:time_queue}/%{INT:time_backend_connect}/%{NOTSPACE:time_duration} %{NOTSPACE:bytes_read} %{NOTSPACE:termination_state} %{INT:actconn}/%{INT:feconn}/%{INT:beconn}/%{INT:srvconn}/%{NOTSPACE:retries} %{INT:srv_queue}/%{INT:backend_queue} -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # node-grok 2 | 3 | This library is inspired by logstash grok filter but it's not a port of it. 4 | 5 | This is a templating library that helps reusing existing regular expressions and constructing new, more complex one. The primary goal was to help parsing and transforming plain text logs into JSON objects (one line => one object) based on provided template. 6 | 7 | ## Install 8 | Install locally: `npm install node-grok`. 9 | 10 | ## Quick start 11 | Following simple snippet 12 | ```javascript 13 | var p = '%{IP:client} \\[%{TIMESTAMP_ISO8601:timestamp}\\] "%{WORD:method} %{URIHOST:site}%{URIPATHPARAM:url}" %{INT:code} %{INT:request} %{INT:response} - %{NUMBER:took} \\[%{DATA:cache}\\] "%{DATA:mtag}" "%{DATA:agent}"'; 14 | var str = '203.35.135.165 [2016-03-15T12:42:04+11:00] "GET memz.co/cloud/" 304 962 0 - 0.003 [MISS] "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36"'; 15 | 16 | require('node-grok').loadDefault(function (patterns) { 17 | var pattern = patterns.createPattern(p); 18 | pattern.parse(str, function (err, obj) { 19 | console.log(obj); 20 | }); 21 | }); 22 | ``` 23 | will transform string 24 | ``` 25 | 203.35.135.165 [2016-03-15T12:42:04+11:00] "GET memz.co/cloud/" 304 962 0 - 0.003 [MISS] "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" 26 | ``` 27 | into object 28 | ```json 29 | { 30 | "client": "203.35.135.165", 31 | "timestamp": "2016-03-15T12:42:04+11:00", 32 | "method": "GET", 33 | "site": "memz.co", 34 | "url": "/cloud/", 35 | "code": "304", 36 | "request": "962", 37 | "response": "0", 38 | "took": "0.003", 39 | "cache": "MISS", 40 | "mtag": "-", 41 | "agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36" 42 | } 43 | ``` 44 | 45 | ## Synchronous version of code 46 | ```javascript 47 | var p = '%{IP:client} \\[%{TIMESTAMP_ISO8601:timestamp}\\] "%{WORD:method} %{URIHOST:site}%{URIPATHPARAM:url}" %{INT:code} %{INT:request} %{INT:response} - %{NUMBER:took} \\[%{DATA:cache}\\] "%{DATA:mtag}" "%{DATA:agent}"'; 48 | var str = '203.35.135.165 [2016-03-15T12:42:04+11:00] "GET memz.co/cloud/" 304 962 0 - 0.003 [MISS] "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.116 Safari/537.36"'; 49 | 50 | var patterns = require('node-grok').loadDefaultSync(); 51 | var pattern = patterns.createPattern(p); 52 | console.log(pattern.parseSync(str)); 53 | ``` 54 | 55 | ## API 56 | * **loadDefault([loadModules,] callback)** - creates new pattern collection including all built-in patterns from `./patterns` folder. By providing *loadModules* parameter you can limit number of loaded patterns: `loadDefault(['grok-patterns'] ,...);`. Callback receives *patterns* collection filled in with default templates: `function(err, patterns)`. 57 | 58 | * **loadDefaultSync([loadModules])** - creates new default pattern collection and returns it `GrokCollection`. 59 | 60 | * **new GrokCollection()** - creates a new empty pattern collection. 61 | 62 | * **GrokCollection.createPattern(expression, [id])** - creates new pattern and adds it to the collection. Find out more about pattern syntax [here](http://logstash.net/docs/1.4.2/filters/grok) and about regular expression syntax [here](http://www.geocities.jp/kosako3/oniguruma/doc/RE.txt) 63 | 64 | * **GrokCollection.getPattern(id)** - returns existing pattern `GrokPattern` 65 | 66 | * **GrokCollection.load(filePath, callback)** - asynchronously loads patterns from file. Callback is `function(err)`. 67 | 68 | * **GrokCollection.loadSync(filePath)** - loads patterns from file and returns number of newly loaded patterns `number` 69 | 70 | * **GrokPattern.parse(str, callback)** - parses string using corresponding pattern. Callback function receives optional *error* and resulting object *result*: `function(error, result)` 71 | 72 | * **GrokPattern.parseSync(str)** - parses string using corresponding pattern and returns resulting object `object` 73 | 74 | ## License 75 | **ISC License (ISC)** 76 | 77 | Copyright (c) 2015, Andrey Chausenko 78 | 79 | Permission to use, copy, modify, and/or distribute this software for any 80 | purpose with or without fee is hereby granted, provided that the above 81 | copyright notice and this permission notice appear in all copies. 82 | 83 | THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 84 | WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 85 | MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 86 | ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 87 | WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 88 | ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 89 | OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 90 | -------------------------------------------------------------------------------- /lib/patterns/grok-patterns: -------------------------------------------------------------------------------- 1 | USERNAME [a-zA-Z0-9._-]+ 2 | USER %{USERNAME} 3 | INT (?:[+-]?(?:[0-9]+)) 4 | BASE10NUM (?[+-]?(?:(?:[0-9]+(?:\.[0-9]+)?)|(?:\.[0-9]+))) 5 | NUMBER (?:%{BASE10NUM}) 6 | BASE16NUM (?(?"(?>\\.|[^\\"]+)+"|""|(?>'(?>\\.|[^\\']+)+')|''|(?>`(?>\\.|[^\\`]+)+`)|``)) 17 | UUID [A-Fa-f0-9]{8}-(?:[A-Fa-f0-9]{4}-){3}[A-Fa-f0-9]{12} 18 | 19 | # Networking 20 | MAC (?:%{CISCOMAC}|%{WINDOWSMAC}|%{COMMONMAC}) 21 | CISCOMAC (?:(?:[A-Fa-f0-9]{4}\.){2}[A-Fa-f0-9]{4}) 22 | WINDOWSMAC (?:(?:[A-Fa-f0-9]{2}-){5}[A-Fa-f0-9]{2}) 23 | COMMONMAC (?:(?:[A-Fa-f0-9]{2}:){5}[A-Fa-f0-9]{2}) 24 | IPV6 ((([0-9A-Fa-f]{1,4}:){7}([0-9A-Fa-f]{1,4}|:))|(([0-9A-Fa-f]{1,4}:){6}(:[0-9A-Fa-f]{1,4}|((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){5}(((:[0-9A-Fa-f]{1,4}){1,2})|:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3})|:))|(([0-9A-Fa-f]{1,4}:){4}(((:[0-9A-Fa-f]{1,4}){1,3})|((:[0-9A-Fa-f]{1,4})?:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){3}(((:[0-9A-Fa-f]{1,4}){1,4})|((:[0-9A-Fa-f]{1,4}){0,2}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){2}(((:[0-9A-Fa-f]{1,4}){1,5})|((:[0-9A-Fa-f]{1,4}){0,3}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(([0-9A-Fa-f]{1,4}:){1}(((:[0-9A-Fa-f]{1,4}){1,6})|((:[0-9A-Fa-f]{1,4}){0,4}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:))|(:(((:[0-9A-Fa-f]{1,4}){1,7})|((:[0-9A-Fa-f]{1,4}){0,5}:((25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)(\.(25[0-5]|2[0-4]\d|1\d\d|[1-9]?\d)){3}))|:)))(%.+)? 25 | IPV4 (?/(?>[\w_%!$@:.,-]+|\\.)*)+ 35 | TTY (?:/dev/(pts|tty([pq])?)(\w+)?/?(?:[0-9]+)) 36 | WINPATH (?>[A-Za-z]+:|\\)(?:\\[^\\?*]*)+ 37 | URIPROTO [A-Za-z]+(\+[A-Za-z+]+)? 38 | URIHOST %{IPORHOST}(?::%{POSINT:port})? 39 | 40 | # uripath comes loosely from RFC1738, but mostly from what Firefox 41 | # doesn't turn into %XX 42 | URIPATH (?:/[A-Za-z0-9$.+!*'(){},~:;=@#%_\-]*)+ 43 | #URIPARAM \?(?:[A-Za-z0-9]+(?:=(?:[^&]*))?(?:&(?:[A-Za-z0-9]+(?:=(?:[^&]*))?)?)*)? 44 | URIPARAM \?[A-Za-z0-9$.+!*'|(){},~@#%&/=:;_?\-\[\]]* 45 | URIPATHPARAM %{URIPATH}(?:%{URIPARAM})? 46 | URI %{URIPROTO}://(?:%{USER}(?::[^@]*)?@)?(?:%{URIHOST})?(?:%{URIPATHPARAM})? 47 | 48 | # Months: January, Feb, 3, 03, 12, December 49 | MONTH \b(?:Jan(?:uary)?|Feb(?:ruary)?|Mar(?:ch)?|Apr(?:il)?|May|Jun(?:e)?|Jul(?:y)?|Aug(?:ust)?|Sep(?:tember)?|Oct(?:ober)?|Nov(?:ember)?|Dec(?:ember)?)\b 50 | MONTHNUM (?:0?[1-9]|1[0-2]) 51 | MONTHNUM2 (?:0[1-9]|1[0-2]) 52 | MONTHDAY (?:(?:0[1-9])|(?:[12][0-9])|(?:3[01])|[1-9]) 53 | 54 | # Days: Monday, Tue, Thu, etc... 55 | DAY (?:Mon(?:day)?|Tue(?:sday)?|Wed(?:nesday)?|Thu(?:rsday)?|Fri(?:day)?|Sat(?:urday)?|Sun(?:day)?) 56 | 57 | # Years? 58 | YEAR (?>\d\d){1,2} 59 | HOUR (?:2[0123]|[01]?[0-9]) 60 | MINUTE (?:[0-5][0-9]) 61 | 62 | # '60' is a leap second in most time standards and thus is valid. 63 | SECOND (?:(?:[0-5]?[0-9]|60)(?:[:.,][0-9]+)?) 64 | TIME (?!<[0-9])%{HOUR}:%{MINUTE}(?::%{SECOND})(?![0-9]) 65 | 66 | # datestamp is YYYY/MM/DD-HH:MM:SS.UUUU (or something like it) 67 | DATE_US %{MONTHNUM}[/-]%{MONTHDAY}[/-]%{YEAR} 68 | DATE_EU %{MONTHDAY}[./-]%{MONTHNUM}[./-]%{YEAR} 69 | ISO8601_TIMEZONE (?:Z|[+-]%{HOUR}(?::?%{MINUTE})) 70 | ISO8601_SECOND (?:%{SECOND}|60) 71 | TIMESTAMP_ISO8601 %{YEAR}-%{MONTHNUM}-%{MONTHDAY}[T ]%{HOUR}:?%{MINUTE}(?::?%{SECOND})?%{ISO8601_TIMEZONE}? 72 | DATE %{DATE_US}|%{DATE_EU} 73 | DATESTAMP %{DATE}[- ]%{TIME} 74 | TZ (?:[PMCE][SD]T|UTC) 75 | DATESTAMP_RFC822 %{DAY} %{MONTH} %{MONTHDAY} %{YEAR} %{TIME} %{TZ} 76 | DATESTAMP_RFC2822 %{DAY}, %{MONTHDAY} %{MONTH} %{YEAR} %{TIME} %{ISO8601_TIMEZONE} 77 | DATESTAMP_OTHER %{DAY} %{MONTH} %{MONTHDAY} %{TIME} %{TZ} %{YEAR} 78 | DATESTAMP_EVENTLOG %{YEAR}%{MONTHNUM2}%{MONTHDAY}%{HOUR}%{MINUTE}%{SECOND} 79 | 80 | # Syslog Dates: Month Day HH:MM:SS 81 | SYSLOGTIMESTAMP %{MONTH} +%{MONTHDAY} %{TIME} 82 | PROG (?:[\w._/%-]+) 83 | SYSLOGPROG %{PROG:program}(?:\[%{POSINT:pid}\])? 84 | SYSLOGHOST %{IPORHOST} 85 | SYSLOGFACILITY <%{NONNEGINT:facility}.%{NONNEGINT:priority}> 86 | HTTPDATE %{MONTHDAY}/%{MONTH}/%{YEAR}:%{TIME} %{INT} 87 | 88 | # Shortcuts 89 | QS %{QUOTEDSTRING} 90 | 91 | # Log formats 92 | SYSLOGBASE %{SYSLOGTIMESTAMP:timestamp} (?:%{SYSLOGFACILITY} )?%{SYSLOGHOST:logsource} %{SYSLOGPROG}: 93 | COMMONAPACHELOG %{IPORHOST:clientip} %{USER:ident} %{USER:auth} \[%{HTTPDATE:timestamp}\] "(?:%{WORD:verb} %{NOTSPACE:request}(?: HTTP/%{NUMBER:httpversion})?|%{DATA:rawrequest})" %{NUMBER:response} (?:%{NUMBER:bytes}|-) 94 | COMBINEDAPACHELOG %{COMMONAPACHELOG} %{QS:referrer} %{QS:agent} 95 | 96 | # Log Levels 97 | LOGLEVEL ([Aa]lert|ALERT|[Tt]race|TRACE|[Dd]ebug|DEBUG|[Nn]otice|NOTICE|[Ii]nfo|INFO|[Ww]arn?(?:ing)?|WARN?(?:ING)?|[Ee]rr?(?:or)?|ERR?(?:OR)?|[Cc]rit?(?:ical)?|CRIT?(?:ICAL)?|[Ff]atal|FATAL|[Ss]evere|SEVERE|EMERG(?:ENCY)?|[Ee]merg(?:ency)?) -------------------------------------------------------------------------------- /lib/patterns/firewalls: -------------------------------------------------------------------------------- 1 | # NetScreen firewall logs 2 | NETSCREENSESSIONLOG %{SYSLOGTIMESTAMP:date} %{IPORHOST:device} %{IPORHOST}: NetScreen device_id=%{WORD:device_id}%{DATA}: start_time=%{QUOTEDSTRING:start_time} duration=%{INT:duration} policy_id=%{INT:policy_id} service=%{DATA:service} proto=%{INT:proto} src zone=%{WORD:src_zone} dst zone=%{WORD:dst_zone} action=%{WORD:action} sent=%{INT:sent} rcvd=%{INT:rcvd} src=%{IPORHOST:src_ip} dst=%{IPORHOST:dst_ip} src_port=%{INT:src_port} dst_port=%{INT:dst_port} src-xlated ip=%{IPORHOST:src_xlated_ip} port=%{INT:src_xlated_port} dst-xlated ip=%{IPORHOST:dst_xlated_ip} port=%{INT:dst_xlated_port} session_id=%{INT:session_id} reason=%{GREEDYDATA:reason} 3 | 4 | #== Cisco ASA == 5 | CISCO_TAGGED_SYSLOG ^<%{POSINT:syslog_pri}>%{CISCOTIMESTAMP:timestamp}( %{SYSLOGHOST:sysloghost})?: %%{CISCOTAG:ciscotag}: 6 | CISCOTIMESTAMP %{MONTH} +%{MONTHDAY}(?: %{YEAR})? %{TIME} 7 | CISCOTAG [A-Z0-9]+-%{INT}-(?:[A-Z0-9_]+) 8 | # Common Particles 9 | CISCO_ACTION Built|Teardown|Deny|Denied|denied|requested|permitted|denied by ACL|discarded|est-allowed|Dropping|created|deleted 10 | CISCO_REASON Duplicate TCP SYN|Failed to locate egress interface|Invalid transport field|No matching connection|DNS Response|DNS Query|(?:%{WORD}\s*)* 11 | CISCO_DIRECTION Inbound|inbound|Outbound|outbound 12 | CISCO_INTERVAL first hit|%{INT}-second interval 13 | CISCO_XLATE_TYPE static|dynamic 14 | # ASA-2-106001 15 | CISCOFW106001 %{CISCO_DIRECTION:direction} %{WORD:protocol} connection %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{GREEDYDATA:tcp_flags} on interface %{GREEDYDATA:interface} 16 | # ASA-2-106006, ASA-2-106007, ASA-2-106010 17 | CISCOFW106006_106007_106010 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} (?:from|src) %{IP:src_ip}/%{INT:src_port}($%{DATA:src_fwuser}$)? (?:to|dst) %{IP:dst_ip}/%{INT:dst_port}($%{DATA:dst_fwuser}$)? (?:on interface %{DATA:interface}|due to %{CISCO_REASON:reason}) 18 | # ASA-3-106014 19 | CISCOFW106014 %{CISCO_ACTION:action} %{CISCO_DIRECTION:direction} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}($%{DATA:src_fwuser}$)? dst %{DATA:dst_interface}:%{IP:dst_ip}($%{DATA:dst_fwuser}$)? $type %{INT:icmp_type}, code %{INT:icmp_code}$ 20 | # ASA-6-106015 21 | CISCOFW106015 %{CISCO_ACTION:action} %{WORD:protocol} $%{DATA:policy_id}$ from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} flags %{DATA:tcp_flags} on interface %{GREEDYDATA:interface} 22 | # ASA-1-106021 23 | CISCOFW106021 %{CISCO_ACTION:action} %{WORD:protocol} reverse path check from %{IP:src_ip} to %{IP:dst_ip} on interface %{GREEDYDATA:interface} 24 | # ASA-4-106023 25 | CISCOFW106023 %{CISCO_ACTION:action} %{WORD:protocol} src %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?($%{DATA:src_fwuser}$)? dst %{DATA:dst_interface}:%{IP:dst_ip}(/%{INT:dst_port})?($%{DATA:dst_fwuser}$)?( $type %{INT:icmp_type}, code %{INT:icmp_code}$)? by access-group %{DATA:policy_id} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] 26 | # ASA-5-106100 27 | CISCOFW106100 access-list %{WORD:policy_id} %{CISCO_ACTION:action} %{WORD:protocol} %{DATA:src_interface}/%{IP:src_ip}$%{INT:src_port}$($%{DATA:src_fwuser}$)? -> %{DATA:dst_interface}/%{IP:dst_ip}$%{INT:dst_port}$($%{DATA:src_fwuser}$)? hit-cnt %{INT:hit_count} %{CISCO_INTERVAL:interval} \[%{DATA:hashcode1}, %{DATA:hashcode2}\] 28 | # ASA-6-110002 29 | CISCOFW110002 %{CISCO_REASON:reason} for %{WORD:protocol} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} 30 | # ASA-6-302010 31 | CISCOFW302010 %{INT:connection_count} in use, %{INT:connection_count_max} most used 32 | # ASA-6-302013, ASA-6-302014, ASA-6-302015, ASA-6-302016 33 | CISCOFW302013_302014_302015_302016 %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection %{INT:connection_id} for %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port}( $%{IP:src_mapped_ip}/%{INT:src_mapped_port}$)?($%{DATA:src_fwuser}$)? to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}( $%{IP:dst_mapped_ip}/%{INT:dst_mapped_port}$)?($%{DATA:dst_fwuser}$)?( duration %{TIME:duration} bytes %{INT:bytes})?(?: %{CISCO_REASON:reason})?( $%{DATA:user}$)? 34 | # ASA-6-302020, ASA-6-302021 35 | CISCOFW302020_302021 %{CISCO_ACTION:action}(?: %{CISCO_DIRECTION:direction})? %{WORD:protocol} connection for faddr %{IP:dst_ip}/%{INT:icmp_seq_num}(?:$%{DATA:fwuser}$)? gaddr %{IP:src_xlated_ip}/%{INT:icmp_code_xlated} laddr %{IP:src_ip}/%{INT:icmp_code}( $%{DATA:user}$)? 36 | # ASA-6-305011 37 | CISCOFW305011 %{CISCO_ACTION:action} %{CISCO_XLATE_TYPE:xlate_type} %{WORD:protocol} translation from %{DATA:src_interface}:%{IP:src_ip}(/%{INT:src_port})?($%{DATA:src_fwuser}$)? to %{DATA:src_xlated_interface}:%{IP:src_xlated_ip}/%{DATA:src_xlated_port} 38 | # ASA-3-313001, ASA-3-313004, ASA-3-313008 39 | CISCOFW313001_313004_313008 %{CISCO_ACTION:action} %{WORD:protocol} type=%{INT:icmp_type}, code=%{INT:icmp_code} from %{IP:src_ip} on interface %{DATA:interface}( to %{IP:dst_ip})? 40 | # ASA-4-313005 41 | CISCOFW313005 %{CISCO_REASON:reason} for %{WORD:protocol} error message: %{WORD:err_protocol} src %{DATA:err_src_interface}:%{IP:err_src_ip}($%{DATA:err_src_fwuser}$)? dst %{DATA:err_dst_interface}:%{IP:err_dst_ip}($%{DATA:err_dst_fwuser}$)? $type %{INT:err_icmp_type}, code %{INT:err_icmp_code}$ on %{DATA:interface} interface\. Original IP payload: %{WORD:protocol} src %{IP:orig_src_ip}/%{INT:orig_src_port}($%{DATA:orig_src_fwuser}$)? dst %{IP:orig_dst_ip}/%{INT:orig_dst_port}($%{DATA:orig_dst_fwuser}$)? 42 | # ASA-4-402117 43 | CISCOFW402117 %{WORD:protocol}: Received a non-IPSec packet $protocol= %{WORD:orig_protocol}$ from %{IP:src_ip} to %{IP:dst_ip} 44 | # ASA-4-402119 45 | CISCOFW402119 %{WORD:protocol}: Received an %{WORD:orig_protocol} packet $SPI= %{DATA:spi}, sequence number= %{DATA:seq_num}$ from %{IP:src_ip} $user= %{DATA:user}$ to %{IP:dst_ip} that failed anti-replay checking 46 | # ASA-4-419001 47 | CISCOFW419001 %{CISCO_ACTION:action} %{WORD:protocol} packet from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port}, reason: %{GREEDYDATA:reason} 48 | # ASA-4-419002 49 | CISCOFW419002 %{CISCO_REASON:reason} from %{DATA:src_interface}:%{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} with different initial sequence number 50 | # ASA-4-500004 51 | CISCOFW500004 %{CISCO_REASON:reason} for protocol=%{WORD:protocol}, from %{IP:src_ip}/%{INT:src_port} to %{IP:dst_ip}/%{INT:dst_port} 52 | # ASA-6-602303, ASA-6-602304 53 | CISCOFW602303_602304 %{WORD:protocol}: An %{CISCO_DIRECTION:direction} %{GREEDYDATA:tunnel_type} SA $SPI= %{DATA:spi}$ between %{IP:src_ip} and %{IP:dst_ip} $user= %{DATA:user}$ has been %{CISCO_ACTION:action} 54 | # ASA-7-710001, ASA-7-710002, ASA-7-710003, ASA-7-710005, ASA-7-710006 55 | CISCOFW710001_710002_710003_710005_710006 %{WORD:protocol} (?:request|access) %{CISCO_ACTION:action} from %{IP:src_ip}/%{INT:src_port} to %{DATA:dst_interface}:%{IP:dst_ip}/%{INT:dst_port} 56 | # ASA-6-713172 57 | CISCOFW713172 Group = %{GREEDYDATA:group}, IP = %{IP:src_ip}, Automatic NAT Detection Status:\s+Remote end\s*%{DATA:is_remote_natted}\s*behind a NAT device\s+This\s+end\s*%{DATA:is_local_natted}\s*behind a NAT device 58 | # ASA-4-733100 59 | CISCOFW733100 \[\s*%{DATA:drop_type}\s*\] drop %{DATA:drop_rate_id} exceeded. Current burst rate is %{INT:drop_rate_current_burst} per second, max configured rate is %{INT:drop_rate_max_burst}; Current average rate is %{INT:drop_rate_current_avg} per second, max configured rate is %{INT:drop_rate_max_avg}; Cumulative total count is %{INT:drop_total_count} 60 | #== End Cisco ASA == -------------------------------------------------------------------------------- /lib/patterns/nagios: -------------------------------------------------------------------------------- 1 | ################################################################################## 2 | ################################################################################## 3 | # Chop Nagios log files to smithereens! 4 | # 5 | # A set of GROK filters to process logfiles generated by Nagios. 6 | # While it does not, this set intends to cover all possible Nagios logs. 7 | # 8 | # Some more work needs to be done to cover all External Commands: 9 | # http://old.nagios.org/developerinfo/externalcommands/commandlist.php 10 | # 11 | # If you need some support on these rules please contact: 12 | # Jelle Smet http://smetj.net 13 | # 14 | ################################################################################# 15 | ################################################################################# 16 | 17 | NAGIOSTIME \[%{NUMBER:nagios_epoch}\] 18 | 19 | ############################################### 20 | ######## Begin nagios log types 21 | ############################################### 22 | NAGIOS_TYPE_CURRENT_SERVICE_STATE CURRENT SERVICE STATE 23 | NAGIOS_TYPE_CURRENT_HOST_STATE CURRENT HOST STATE 24 | 25 | NAGIOS_TYPE_SERVICE_NOTIFICATION SERVICE NOTIFICATION 26 | NAGIOS_TYPE_HOST_NOTIFICATION HOST NOTIFICATION 27 | 28 | NAGIOS_TYPE_SERVICE_ALERT SERVICE ALERT 29 | NAGIOS_TYPE_HOST_ALERT HOST ALERT 30 | 31 | NAGIOS_TYPE_SERVICE_FLAPPING_ALERT SERVICE FLAPPING ALERT 32 | NAGIOS_TYPE_HOST_FLAPPING_ALERT HOST FLAPPING ALERT 33 | 34 | NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT SERVICE DOWNTIME ALERT 35 | NAGIOS_TYPE_HOST_DOWNTIME_ALERT HOST DOWNTIME ALERT 36 | 37 | NAGIOS_TYPE_PASSIVE_SERVICE_CHECK PASSIVE SERVICE CHECK 38 | NAGIOS_TYPE_PASSIVE_HOST_CHECK PASSIVE HOST CHECK 39 | 40 | NAGIOS_TYPE_SERVICE_EVENT_HANDLER SERVICE EVENT HANDLER 41 | NAGIOS_TYPE_HOST_EVENT_HANDLER HOST EVENT HANDLER 42 | 43 | NAGIOS_TYPE_EXTERNAL_COMMAND EXTERNAL COMMAND 44 | NAGIOS_TYPE_TIMEPERIOD_TRANSITION TIMEPERIOD TRANSITION 45 | ############################################### 46 | ######## End nagios log types 47 | ############################################### 48 | 49 | ############################################### 50 | ######## Begin external check types 51 | ############################################### 52 | NAGIOS_EC_DISABLE_SVC_CHECK DISABLE_SVC_CHECK 53 | NAGIOS_EC_ENABLE_SVC_CHECK ENABLE_SVC_CHECK 54 | NAGIOS_EC_DISABLE_HOST_CHECK DISABLE_HOST_CHECK 55 | NAGIOS_EC_ENABLE_HOST_CHECK ENABLE_HOST_CHECK 56 | NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT PROCESS_SERVICE_CHECK_RESULT 57 | NAGIOS_EC_PROCESS_HOST_CHECK_RESULT PROCESS_HOST_CHECK_RESULT 58 | NAGIOS_EC_SCHEDULE_SERVICE_DOWNTIME SCHEDULE_SERVICE_DOWNTIME 59 | NAGIOS_EC_SCHEDULE_HOST_DOWNTIME SCHEDULE_HOST_DOWNTIME 60 | ############################################### 61 | ######## End external check types 62 | ############################################### 63 | NAGIOS_WARNING Warning:%{SPACE}%{GREEDYDATA:nagios_message} 64 | 65 | NAGIOS_CURRENT_SERVICE_STATE %{NAGIOS_TYPE_CURRENT_SERVICE_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message} 66 | NAGIOS_CURRENT_HOST_STATE %{NAGIOS_TYPE_CURRENT_HOST_STATE:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statetype};%{DATA:nagios_statecode};%{GREEDYDATA:nagios_message} 67 | 68 | NAGIOS_SERVICE_NOTIFICATION %{NAGIOS_TYPE_SERVICE_NOTIFICATION:nagios_type}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message} 69 | NAGIOS_HOST_NOTIFICATION %{NAGIOS_TYPE_HOST_NOTIFICATION}: %{DATA:nagios_notifyname};%{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_contact};%{GREEDYDATA:nagios_message} 70 | 71 | NAGIOS_SERVICE_ALERT %{NAGIOS_TYPE_SERVICE_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message} 72 | NAGIOS_HOST_ALERT %{NAGIOS_TYPE_HOST_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{NUMBER:nagios_attempt};%{GREEDYDATA:nagios_message} 73 | 74 | NAGIOS_SERVICE_FLAPPING_ALERT %{NAGIOS_TYPE_SERVICE_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_message} 75 | NAGIOS_HOST_FLAPPING_ALERT %{NAGIOS_TYPE_HOST_FLAPPING_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_message} 76 | 77 | NAGIOS_SERVICE_DOWNTIME_ALERT %{NAGIOS_TYPE_SERVICE_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} 78 | NAGIOS_HOST_DOWNTIME_ALERT %{NAGIOS_TYPE_HOST_DOWNTIME_ALERT:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} 79 | 80 | NAGIOS_PASSIVE_SERVICE_CHECK %{NAGIOS_TYPE_PASSIVE_SERVICE_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} 81 | NAGIOS_PASSIVE_HOST_CHECK %{NAGIOS_TYPE_PASSIVE_HOST_CHECK:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_comment} 82 | 83 | NAGIOS_SERVICE_EVENT_HANDLER %{NAGIOS_TYPE_SERVICE_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name} 84 | NAGIOS_HOST_EVENT_HANDLER %{NAGIOS_TYPE_HOST_EVENT_HANDLER:nagios_type}: %{DATA:nagios_hostname};%{DATA:nagios_state};%{DATA:nagios_statelevel};%{DATA:nagios_event_handler_name} 85 | 86 | NAGIOS_TIMEPERIOD_TRANSITION %{NAGIOS_TYPE_TIMEPERIOD_TRANSITION:nagios_type}: %{DATA:nagios_service};%{DATA:nagios_unknown1};%{DATA:nagios_unknown2}; 87 | 88 | #################### 89 | #### External checks 90 | #################### 91 | 92 | #Disable host & service check 93 | NAGIOS_EC_LINE_DISABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service} 94 | NAGIOS_EC_LINE_DISABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_DISABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname} 95 | 96 | #Enable host & service check 97 | NAGIOS_EC_LINE_ENABLE_SVC_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_SVC_CHECK:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service} 98 | NAGIOS_EC_LINE_ENABLE_HOST_CHECK %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_ENABLE_HOST_CHECK:nagios_command};%{DATA:nagios_hostname} 99 | 100 | #Process host & service check 101 | NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_SERVICE_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_service};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result} 102 | NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_PROCESS_HOST_CHECK_RESULT:nagios_command};%{DATA:nagios_hostname};%{DATA:nagios_state};%{GREEDYDATA:nagios_check_result} 103 | 104 | #Schedule host & service downtime 105 | NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME %{NAGIOS_TYPE_EXTERNAL_COMMAND:nagios_type}: %{NAGIOS_EC_SCHEDULE_HOST_DOWNTIME:nagios_command};%{DATA:nagios_hostname};%{NUMBER:nagios_start_time};%{NUMBER:nagios_end_time};%{NUMBER:nagios_fixed};%{NUMBER:nagios_trigger_id};%{NUMBER:nagios_duration};%{DATA:author};%{DATA:comment} 106 | 107 | #End matching line 108 | NAGIOSLOGLINE %{NAGIOSTIME} (?:%{NAGIOS_WARNING}|%{NAGIOS_CURRENT_SERVICE_STATE}|%{NAGIOS_CURRENT_HOST_STATE}|%{NAGIOS_SERVICE_NOTIFICATION}|%{NAGIOS_HOST_NOTIFICATION}|%{NAGIOS_SERVICE_ALERT}|%{NAGIOS_HOST_ALERT}|%{NAGIOS_SERVICE_FLAPPING_ALERT}|%{NAGIOS_HOST_FLAPPING_ALERT}|%{NAGIOS_SERVICE_DOWNTIME_ALERT}|%{NAGIOS_HOST_DOWNTIME_ALERT}|%{NAGIOS_PASSIVE_SERVICE_CHECK}|%{NAGIOS_PASSIVE_HOST_CHECK}|%{NAGIOS_SERVICE_EVENT_HANDLER}|%{NAGIOS_HOST_EVENT_HANDLER}|%{NAGIOS_TIMEPERIOD_TRANSITION}|%{NAGIOS_EC_LINE_DISABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_ENABLE_SVC_CHECK}|%{NAGIOS_EC_LINE_DISABLE_HOST_CHECK|%{NAGIOS_EC_LINE_ENABLE_HOST_CHECK}|%{NAGIOS_EC_LINE_PROCESS_HOST_CHECK_RESULT}|%{NAGIOS_EC_LINE_PROCESS_SERVICE_CHECK_RESULT}|%{NAGIOS_EC_LINE_SCHEDULE_HOST_DOWNTIME}) -------------------------------------------------------------------------------- /lib/test.js: -------------------------------------------------------------------------------- 1 | var grok = require('./index.js'); 2 | var expect = require("chai").expect; 3 | 4 | function testParse(p, str, expected, done) { 5 | grok.loadDefault(function (err, patterns) { 6 | expect(err).to.be.null; 7 | 8 | var pattern = patterns.createPattern(p); 9 | pattern.parse(str, function (err, result) { 10 | expect(err).to.be.null; 11 | expect(result).to.be.eql(expected); 12 | done(); 13 | }); 14 | }); 15 | } 16 | 17 | describe('grok', function() { 18 | describe('loadDefault', function() { 19 | it('is asynchronous', function() { 20 | var isDone = false; 21 | 22 | grok.loadDefault(function(patterns) { 23 | isDone = true; 24 | }); 25 | 26 | expect(isDone, 'was done immediately after return').to.be.false; 27 | }); 28 | }); 29 | 30 | describe('#parseSync()', function () { 31 | it('returns null when a parse fails', function() { 32 | var patterns = grok.loadDefaultSync(); 33 | var pattern = patterns.createPattern('%{WORD:verb} %{WORD:adjective}'); 34 | 35 | var result = pattern.parseSync('test'); 36 | expect(result).to.be.null; 37 | }); 38 | }); 39 | 40 | describe('#parse()', function () { 41 | 42 | it('is asynchronous', function(done) { 43 | grok.loadDefault(function (err, patterns) { 44 | expect(err).to.be.null; 45 | 46 | var pattern = patterns.createPattern('%{WORD:verb}'); 47 | var isDone = false; 48 | 49 | pattern.parse('test', function(err, result) { 50 | isDone = true; 51 | }); 52 | 53 | expect(isDone).to.be.false; 54 | done(); 55 | }); 56 | }); 57 | 58 | it('returns null when a parse fails', function(done) { 59 | grok.loadDefault(function (err, patterns) { 60 | expect(err).to.be.null; 61 | 62 | var pattern = patterns.createPattern('%{WORD:verb} %{WORD:adjective}'); 63 | 64 | pattern.parse('test', function(err, result) { 65 | expect(err).to.not.exist; 66 | expect(result).to.be.null; 67 | done(); 68 | }); 69 | }); 70 | }); 71 | 72 | it('parses to attributes with uppercase in their names', function(done) { 73 | grok.loadDefault(function (err, patterns) { 74 | expect(err).to.be.null; 75 | 76 | var pattern = patterns.createPattern('%{WORD:verb} %{WORD:testVariable}'); 77 | 78 | pattern.parse('test worp', function(err, result) { 79 | expect(err).to.not.exist; 80 | expect(result).to.deep.equal({verb: 'test', testVariable: 'worp'}); 81 | done(); 82 | }); 83 | }); 84 | }); 85 | 86 | it('should parse a simple custom pattern', function (done) { 87 | var p = '(?\\w+)\\s+(?/\\w+)'; 88 | var str = 'DELETE /ping HTTP/1.1'; 89 | var expected = { 90 | verb: 'DELETE', 91 | url: '/ping' 92 | }; 93 | 94 | testParse(p, str, expected, done); 95 | }); 96 | 97 | it('should parse a pattern with some default patterns', function (done) { 98 | var p = '%{WORD:verb} %{URIPATH:url}'; 99 | var str = 'DELETE /ping HTTP/1.1'; 100 | var expected = { 101 | verb: 'DELETE', 102 | url: '/ping' 103 | }; 104 | 105 | testParse(p, str, expected, done); 106 | }); 107 | 108 | it('should parse a pattern with optional parts correctly #1', function (done) { 109 | var p = '(?(%{WORD:verb} %{URIPATH:url}|(?\$ALTERNATIVE\$)))'; 110 | var str = 'DELETE /ping HTTP/1.1'; 111 | var expected = { 112 | all: 'DELETE /ping', 113 | verb: 'DELETE', 114 | url: '/ping' 115 | }; 116 | 117 | testParse(p, str, expected, done); 118 | }); 119 | 120 | it('should parse a pattern with optional parts correctly #2', function (done) { 121 | var p = '(?(%{WORD:verb} %{URIPATH:url}|(?\$ALTERNATIVE\$)))'; 122 | var str = '(ALTERNATIVE)'; 123 | var expected = { 124 | all: '(ALTERNATIVE)', 125 | alternative: '(ALTERNATIVE)' 126 | }; 127 | 128 | testParse(p, str, expected, done); 129 | }); 130 | 131 | 132 | it('should parse parts of the default HAPROXYHTTP pattern', function (done) { 133 | var p = '(|(%{WORD:http_verb} (%{URIPROTO:http_proto}://)?(?:%{USER:http_user}(?::[^@]*)?@)?(?:%{URIHOST:http_host})?(?:%{URIPATHPARAM:http_request})?( HTTP/%{NUMBER:http_version})?))'; 134 | var str = 'GET /ping HTTP/1.1'; 135 | var expected = { 136 | http_verb: 'GET', 137 | http_request: '/ping', 138 | http_version: '1.1' 139 | }; 140 | 141 | testParse(p, str, expected, done); 142 | }); 143 | 144 | it('should parse the full default HAPROXYHTTP pattern', function (done) { 145 | var p = '%{HAPROXYHTTP:haproxy}'; 146 | var str = 'Aug 17 12:06:27 minion haproxy[3274]: 1.2.3.4:50901 [17/Aug/2015:12:06:27.379] http-in backend_gru/minion_8080 1/0/0/142/265 200 259 - - ---- 0/0/0/0/0 0/0 "GET /ping HTTP/1.1"'; 147 | var expected = { 148 | haproxy: str, 149 | syslog_timestamp: 'Aug 17 12:06:27', 150 | syslog_server: 'minion', 151 | pid: '3274', 152 | program: 'haproxy', 153 | client_ip: '1.2.3.4', 154 | client_port: '50901', 155 | accept_date: '17/Aug/2015:12:06:27.379', 156 | haproxy_hour: '12', 157 | haproxy_milliseconds: '379', 158 | haproxy_minute: '06', 159 | haproxy_month: 'Aug', 160 | haproxy_monthday: '17', 161 | haproxy_second: '27', 162 | haproxy_time: '12:06:27', 163 | haproxy_year: '2015', 164 | frontend_name: 'http-in', 165 | backend_name: 'backend_gru', 166 | server_name: 'minion_8080', 167 | time_request: '1', 168 | time_queue: '0', 169 | time_backend_connect: '0', 170 | time_backend_response: '142', 171 | time_duration: '265', 172 | http_status_code: '200', 173 | bytes_read: '259', 174 | captured_request_cookie: '-', 175 | captured_response_cookie: '-', 176 | termination_state: '----', 177 | actconn: '0', 178 | feconn: '0', 179 | beconn: '0', 180 | srvconn: '0', 181 | retries: '0', 182 | srv_queue: '0', 183 | backend_queue: '0', 184 | http_verb: 'GET', 185 | http_request: '/ping', 186 | http_version: '1.1' 187 | }; 188 | 189 | testParse(p, str, expected, done); 190 | }); 191 | 192 | it('should parse the sample pattern of the README.md', function (done) { 193 | var p = '%{IP:client} \\[%{TIMESTAMP_ISO8601:timestamp}\\] "%{WORD:method} %{URIHOST:site}%{URIPATHPARAM:url}" %{INT:code} %{INT:request} %{INT:response} - %{NUMBER:took} \\[%{DATA:cache}\\] "%{DATA:mtag}" "%{DATA:agent}"'; 194 | var str = '65.19.138.33 [2015-05-13T08:04:43+10:00] "GET datasymphony.com.au/ru/feed/" 304 385 0 - 0.140 [HIT] "-" "Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)"'; 195 | var expected = { 196 | client: '65.19.138.33', 197 | timestamp: '2015-05-13T08:04:43+10:00', 198 | method: 'GET', 199 | site: 'datasymphony.com.au', 200 | url: '/ru/feed/', 201 | code: '304', 202 | request: '385', 203 | response: '0', 204 | took: '0.140', 205 | cache: 'HIT', 206 | mtag: '-', 207 | agent: 'Feedly/1.0 (+http://www.feedly.com/fetcher.html; like FeedFetcher-Google)' 208 | }; 209 | 210 | testParse(p, str, expected, done); 211 | }); 212 | 213 | }); 214 | }); 215 | 216 | describe('GrokCollection', function() { 217 | describe('load', function() { 218 | it('is asynchronous', function() { 219 | var coll = new grok.GrokCollection(); 220 | var isDone = false; 221 | 222 | coll.load(require.resolve('./patterns/grok-patterns'), function() { 223 | isDone = true; 224 | }); 225 | 226 | expect(isDone, 'was done immediately after return').to.be.false; 227 | }); 228 | }); 229 | 230 | describe('loadSync', function() { 231 | it('returns number of patterns', function() { 232 | var coll = new grok.GrokCollection(); 233 | var result = coll.loadSync(require.resolve('./patterns/grok-patterns')); 234 | 235 | expect(result, 'should match number of loaded patterns').to.equal(coll.count()); 236 | }); 237 | }); 238 | }); 239 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs'); 2 | var async = require('async'); 3 | var OnigRegExp = require('oniguruma').OnigRegExp; 4 | var Map = require('collections/fast-map'); 5 | 6 | function GrokPattern(expression, id) { 7 | var t = this; 8 | t.id = id; 9 | t.expression = expression; 10 | t.fields = [ null ]; // add a dummy entry at the beginning to swallow the fully captured expression 11 | t.resolved = null; 12 | t.regex = null; 13 | 14 | t.parse = function(str, next) { 15 | if (!t.regexp) { 16 | t.regexp = new OnigRegExp(t.resolved); 17 | } 18 | 19 | t.regexp.search(str, function(err, result) { 20 | if(err || !result) 21 | return next(err, result); 22 | 23 | var r = {}; 24 | 25 | result.forEach(function(item, index) { 26 | var field = t.fields[index]; 27 | if(field && item.match) { 28 | r[field] = item.match; 29 | } 30 | }); 31 | 32 | return next(err, r, result); 33 | }); 34 | }; 35 | 36 | t.parseSync = function(str) { 37 | if (!t.regexp) { 38 | t.regexp = new OnigRegExp(t.resolved); 39 | } 40 | 41 | var result = t.regexp.searchSync(str); 42 | 43 | if(!result) 44 | return null; 45 | 46 | var r = {}; 47 | 48 | result.forEach(function(item, index) { 49 | var field = t.fields[index]; 50 | if(field && item.match) { 51 | r[field] = item.match; 52 | } 53 | }); 54 | 55 | return r; 56 | }; 57 | } 58 | 59 | var subPatternsRegex = /%\{[A-Z0-9_]+(?::[A-Za-z0-9_]+)?\}/g; // %{subPattern} or %{subPattern:fieldName} 60 | var nestedFieldNamesRegex = /($\?<([A-Za-z0-9_]+)>)|\(\?:|\(\?>|\(\?!|\(\?' + subPattern.resolved + ')'); 101 | } else { 102 | expression = expression.replace(matched, subPattern.resolved); 103 | } 104 | }); 105 | 106 | pattern.resolved = expression; 107 | return pattern; 108 | } 109 | 110 | // create mapping table for the fieldNames to capture 111 | function resolveFieldNames (pattern) { 112 | if(!pattern) { return; } 113 | 114 | var nestLevel = 0; 115 | var inRangeDef = 0; 116 | var matched; 117 | while ((matched = nestedFieldNamesRegex.exec(pattern.resolved)) !== null) { 118 | switch(matched[0]) { 119 | case '(': { if(!inRangeDef) { ++nestLevel; pattern.fields.push(null); } break; } 120 | case '\\(': break; // can be ignored 121 | case '\$': break; // can be ignored 122 | case ')': { if(!inRangeDef) { --nestLevel; } break; } 123 | case '[': { ++inRangeDef; break; } 124 | case '\\[': break; // can be ignored 125 | case '\\]': break; // can be ignored 126 | case ']': { --inRangeDef; break; } 127 | case '(?:': // fallthrough // group not captured 128 | case '(?>': // fallthrough // atomic group 129 | case '(?!': // fallthrough // negative look-ahead 130 | case '(? 2) { 150 | var pattern = new GrokPattern(elements[2], elements[1]); 151 | patterns.set(pattern.id, pattern); 152 | i++; 153 | } 154 | }); 155 | } 156 | } 157 | 158 | return i; 159 | } 160 | 161 | t.createPattern = function (expression, id) { 162 | id = id || 'pattern-' + patterns.length; 163 | if (patterns.has(id)) { 164 | console.error('Error: pattern with id %s already exists', id); 165 | } else { 166 | var pattern = new GrokPattern(expression, id); 167 | resolvePattern(pattern); 168 | patterns.set(id, pattern); 169 | return pattern; 170 | } 171 | }; 172 | 173 | t.getPattern = function (id) { 174 | return patterns.get(id); 175 | }; 176 | 177 | t.load = function (filePath, callback) { 178 | fs.readFile(filePath, function(err, file) { 179 | if(err) 180 | return callback(err); 181 | 182 | doLoad(file); 183 | return callback(); 184 | }); 185 | }; 186 | 187 | t.loadSync = function(filePath) { 188 | return doLoad(fs.readFileSync(filePath)); 189 | }; 190 | 191 | t.count = function () { 192 | return patterns.length; 193 | }; 194 | } 195 | 196 | var patternsDir = __dirname + '/patterns/'; 197 | 198 | function doLoadDefaultSync(loadModules) { 199 | var result = new GrokCollection(); 200 | 201 | var files = fs.readdirSync(patternsDir); 202 | if (files && files.length) { 203 | files.filter(function(file) { 204 | return !loadModules || !loadModules.length || loadModules.indexOf(file) !== -1; 205 | }).forEach(function (file) { 206 | result.loadSync(patternsDir + file); 207 | }) 208 | } 209 | 210 | return result; 211 | } 212 | 213 | function doLoadDefault(loadModules, callback) { 214 | return fs.readdir(patternsDir, function(err, files) { 215 | if(err) 216 | return callback(err); 217 | 218 | var result = new GrokCollection(); 219 | 220 | return async.parallel( 221 | files.filter(function(file) { 222 | return !loadModules || !loadModules.length || loadModules.indexOf(file) !== -1; 223 | }).map(function (file) { 224 | return function(callback) { 225 | return result.load(patternsDir + file, callback); 226 | }; 227 | }), 228 | function(err) { 229 | if(err) 230 | return callback(err); 231 | 232 | return callback(null, result); 233 | }); 234 | }); 235 | } 236 | 237 | module.exports = { 238 | loadDefault: function (loadModules, callback) { 239 | if(arguments.length < 2) { 240 | callback = loadModules; 241 | loadModules = null; 242 | } 243 | 244 | doLoadDefault(loadModules, callback); 245 | }, 246 | 247 | loadDefaultSync: doLoadDefaultSync, 248 | 249 | GrokCollection: GrokCollection 250 | }; --------------------------------------------------------------------------------