├── .gitignore ├── LICENSE ├── README.md ├── index.js ├── package-lock.json └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Chris O'Hara 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | **nginx-parser** parse nginx log files in node.js 2 | 3 | ### Installation 4 | 5 | ```bash 6 | $ npm install nginxparser 7 | ``` 8 | 9 | ### Usage 10 | 11 | To read a log file 12 | 13 | ```javascript 14 | var NginxParser = require('nginxparser'); 15 | 16 | var parser = new NginxParser('$remote_addr - $remote_user [$time_local] ' 17 | + '"$request" $status $body_bytes_sent "$http_referer" "$http_user_agent"'); 18 | 19 | parser.read(path, function (row) { 20 | console.log(row); 21 | }, function (err) { 22 | if (err) throw err; 23 | console.log('Done!') 24 | }); 25 | ``` 26 | 27 | To read from stdin, pass `-` as the path. 28 | 29 | To tail a log file (equivalent to `tail -F`) 30 | 31 | ```javascript 32 | parser.read(path, { tail: true }, function (row) { 33 | //... 34 | }); 35 | ``` 36 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var fs = require('fs') 2 | , spawn = require('child_process').spawn; 3 | 4 | /** 5 | * Create a log parser. 6 | * 7 | * @param {String} format 8 | */ 9 | 10 | var Parser = module.exports = function (format) { 11 | this.directives = {}; 12 | 13 | var prefix = format.match(/^[^\$]*/); 14 | if (prefix) { 15 | format = this.escape(prefix[0]) + format.slice(prefix[0].length); 16 | } 17 | 18 | this.parser = format; 19 | 20 | var directive = /\$([a-z_]+)(.)?([^\$]+)?/g 21 | , match, regex, boundary, i = 1; 22 | 23 | while ((match = directive.exec(format))) { 24 | this.directives[match[1]] = i++; 25 | if (match[2]) { 26 | boundary = this.escape(match[2]); 27 | regex = '([^' + boundary + ']*?)' + boundary; 28 | if (match[3]) { 29 | regex += this.escape(match[3]); 30 | } 31 | } else { 32 | regex = '(.+)$'; 33 | } 34 | this.parser = this.parser.replace(match[0], regex); 35 | } 36 | 37 | this.parser = new RegExp(this.parser); 38 | }; 39 | 40 | /** 41 | * Parse a log file. 42 | * 43 | * @param {String} path 44 | * @param {Object} options (optional) 45 | * @param {Function} iterator - called for each line 46 | * @param {Function} callback (optional) - called at the end 47 | */ 48 | 49 | Parser.prototype.read = function (path, options, iterator, callback) { 50 | if (typeof options === 'function') { 51 | callback = iterator; 52 | iterator = options; 53 | } 54 | if (!path || path === '-') { 55 | return this.stdin(iterator, callback); 56 | } else if (options.tail) { 57 | return this.tail(path, iterator, callback); 58 | } 59 | return this.stream(fs.createReadStream(path), iterator, callback); 60 | }; 61 | 62 | /** 63 | * Parse a log file and watch it for changes. 64 | * 65 | * @param {String} path 66 | * @param {Function} iterator - called for each line 67 | * @param {Function} callback (optional) - called at the end 68 | */ 69 | 70 | Parser.prototype.tail = function (path, iterator, callback) { 71 | var stream = spawn('tail', [ '-F', '-c', '+0', path]).stdout; 72 | return this.stream(stream, iterator, callback); 73 | }; 74 | 75 | /** 76 | * Parse a log stream from STDIN. 77 | * 78 | * @param {Function} iterator - called for each line 79 | * @param {Function} callback (optional) - called at the end 80 | */ 81 | 82 | Parser.prototype.stdin = function (iterator, callback) { 83 | return this.stream(process.stdin, iterator, callback); 84 | }; 85 | 86 | /** 87 | * Parse a log stream. 88 | * 89 | * @param {ReadableStream} stream 90 | * @param {Function} iterator - called for each line 91 | * @param {Function} callback (optional) - called at the end 92 | */ 93 | 94 | Parser.prototype.stream = function (stream, iterator, callback) { 95 | var self = this, overflow = new Buffer(0), complete = false; 96 | stream.on('data', function (data) { 97 | var buffer = Buffer.concat([overflow, data]), newline = 0; 98 | for (var i = 0, len = buffer.length; i < len; i++) { 99 | if (buffer[i] === 10) { 100 | self.parseLine(buffer.slice(newline, i), iterator); 101 | newline = i + 1; 102 | } 103 | } 104 | overflow = buffer.slice(newline); 105 | }); 106 | if (callback) { 107 | stream.on('error', function (err) { 108 | if (complete) return; 109 | complete = true; 110 | callback(err); 111 | }); 112 | } 113 | stream.on('end', function () { 114 | if (overflow.length) { 115 | self.parseLine(overflow, iterator); 116 | } 117 | if (complete) return; 118 | complete = true; 119 | if (callback) { 120 | callback(); 121 | } 122 | }); 123 | if (stream.resume) { 124 | process.nextTick(function () { 125 | stream.resume(); 126 | }); 127 | } 128 | return stream; 129 | }; 130 | 131 | /** 132 | * Parse a log line. 133 | * 134 | * @param {Buffer|String} line 135 | * @param {Function} iterator 136 | */ 137 | 138 | Parser.prototype.parseLine = function (line, iterator) { 139 | var match = line.toString().match(this.parser); 140 | if (!match) { 141 | return; 142 | } 143 | 144 | var row = { 145 | msec: null 146 | , time_iso8601: null 147 | , remote_addr: null 148 | , query_string: null 149 | , http_x_forwarded_for: null 150 | , http_user_agent: null 151 | , http_referer: null 152 | , time_local: null 153 | , request: null 154 | , status: null 155 | , request_time: null 156 | , request_length: null 157 | , pipe: null 158 | , connection: null 159 | , bytes_sent: null 160 | , body_bytes_sent: null 161 | 162 | , date: null 163 | , timestamp: null 164 | , ip: null 165 | , ip_str: null 166 | }; 167 | 168 | for (var key in this.directives) { 169 | row[key] = match[this.directives[key]]; 170 | if (row[key] === '-') { 171 | row[key] = null; 172 | } 173 | } 174 | 175 | //Parse the timestamp 176 | if (row.time_iso8601) { 177 | row.date = new Date(row.time_iso8601); 178 | } else if (row.msec) { 179 | row.date = new Date(Number(row.msec.replace('.', ''))); 180 | } 181 | if (row.date) { 182 | row.timestamp = row.date.getTime(); 183 | } 184 | 185 | //Parse the user's IP 186 | if (row.http_x_forwarded_for) { 187 | row.ip_str = row.http_x_forwarded_for; 188 | } else if (row.remote_addr) { 189 | row.ip_str = row.remote_addr; 190 | } 191 | if (row.ip_str) { 192 | var ip = row.ip_str.split('.', 4); 193 | row.ip = Number(ip[0]) * (2 << 23) + 194 | Number(ip[1]) * (2 << 15) + 195 | Number(ip[2]) * (2 << 7) + 196 | Number(ip[3]); 197 | } 198 | 199 | iterator(row); 200 | }; 201 | 202 | /** 203 | * Escape regular expression tokens. 204 | * 205 | * @param {String} str 206 | * @return {String} 207 | */ 208 | 209 | Parser.prototype.escape = function (str) { 210 | return str.replace(new RegExp('[.*+?|()\\[\\]{}]', 'g'), '\\$&'); 211 | }; 212 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nginxparser", 3 | "version": "2.0.0", 4 | "lockfileVersion": 1 5 | } 6 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nginxparser", 3 | "description": "Parse Nginx log files", 4 | "version": "2.1.0", 5 | "homepage": "http://github.com/chriso/nginx-parser", 6 | "author": "Chris O'Hara ", 7 | "bugs": { 8 | "mail": "cohara87@gmail.com", 9 | "url": "http://github.com/chriso/nginx-parser/issues" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "http://github.com/chriso/nginx-parser.git" 14 | }, 15 | "engines": { 16 | "node": ">=0.7.11" 17 | }, 18 | "licenses": [ 19 | { 20 | "type": "MIT", 21 | "url": "http://github.com/chriso/nginx-parser/raw/master/LICENSE" 22 | } 23 | ] 24 | } 25 | --------------------------------------------------------------------------------