├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package.json └── test ├── log └── run.js /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/ 2 | *.lock 3 | *.log 4 | node_modules 5 | test/log.json 6 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: false 3 | node_js: 4 | - "4" 5 | - "5" 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | (The MIT License) 2 | 3 | Copyright (c) 2015 Sofish Lin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining 6 | a copy of this software and associated documentation files (the 7 | 'Software'), to deal in the Software without restriction, including 8 | without limitation the rights to use, copy, modify, merge, publish, 9 | distribute, sublicense, and/or sell copies of the Software, and to 10 | permit persons to whom the Software is furnished to do so, subject to 11 | the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be 14 | included in all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, 17 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 18 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 19 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 20 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 21 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 22 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ![](https://travis-ci.org/sofish/log2json.svg) 2 | 3 | # Log2JSON 4 | 5 | Log2JSON is a small lib that allows you to transform nginx/apache/whatever 6 | logs to JSON. 7 | 8 | Install it will npm / cnpm: 9 | 10 | ```ruby 11 | $ npm install log2json 12 | ``` 13 | 14 | ## Usage 15 | 16 | Run it like: 17 | 18 | ```js 19 | var log2json = require('log2json'); 20 | 21 | log2json(configure, ret => { 22 | if(!ret) return console.log('the file is empty'); 23 | console.log('JSON is generated: %s', ret); 24 | }); 25 | ``` 26 | 27 | Follow the codes below to generate the `configure` object. 28 | 29 | ```js 30 | var path = require('path'); 31 | 32 | 33 | var separator = '•-•'; // separator of the log 34 | var src = path.join(__dirname, './log'); // path of the log 35 | var dist = path.join(__dirname, './log.json'); // the generated JSON file 36 | var removeSrc = true; // remove the log when JSON is 37 | // generated, by default is true 38 | var map = [ // map the fields with keys 39 | 40 | 'fieldName2', // 1. {string} name to the filed 41 | fn2TransformData, // 2. a custom function to transform the field 42 | // should return an object {name, value} 43 | 'fieldName|number', // 3. use built-in directive to 44 | // transform the field 45 | 'fieldName|url', // built-in directives: number, url, array 46 | 'fieldName|array' // - `number` transform string to number 47 | // - `array` transform 'a,b' to ["a","b"] 48 | // - `url` transform querystring to object 49 | // 'a=b&c[]=d&c[]=e' to {a:"b", c: ["d", "e"]} 50 | ]; 51 | 52 | // a transform function should return an object like {name, value} 53 | function fn2TransformData(input) { 54 | var name = 'transformed'; 55 | var value = doSomethingWith(value); 56 | return {name, value}; 57 | } 58 | 59 | // what we need 60 | var configure = {map, separator, src, dist, removeSrc}; 61 | ``` 62 | 63 | ## Test 64 | 65 | simply run `npm test` on your favor terminal app. 66 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const fs = require('fs'); 4 | const ENCODING = 'utf-8'; 5 | const format = { url, array, number }; 6 | const keys = Object.keys(format); 7 | const STOP = 5000; 8 | const _extend = require('util')._extend; 9 | 10 | module.exports = function parser(configure) { 11 | 12 | /* Input Parser: 13 | * @param {object} configure 14 | * { 15 | * src: 'path/to/src/file', // {string} 16 | * dist: 'path/to/dist/file', // {string} 17 | * separator: '•-•', // {string|regexp} beware about to choose a right one 18 | * removeSrc: true // {boolean} [optional] by default is true 19 | 20 | * map: ['filedName', fn, 'name|FORMAT', ...] // {array} 21 | * 22 | * // 1. {string} fieldName, the key/value pairs' key 23 | * // 2. {function} fn, a transform function, accept an argument, the origin value, 24 | * // it returns an object { name, value }, eg. the fn may translating the origin value - 25 | * // `foo=bar&hello=world` to { name: 'query, value: { 'foo: 'bar', hello: 'world' } } 26 | * // 3. {string} separate the filedName and built-in format to transform the origin value 27 | * // eg. 'name|url' will parse the origin value as a queryString 28 | * 29 | * callback: callback(err, result) 30 | * // {string} result 31 | * // 1. if the file is empty, returns an empty string and won't create a new file 32 | * // 2. if a new file is created returns the file path 33 | * } 34 | */ 35 | 36 | var defaults = { 37 | removeSrc: true, 38 | cacheArray: [], 39 | first: 1, 40 | count: 0, 41 | buffer: '', 42 | callback: () => {} 43 | }; 44 | 45 | configure = _extend(defaults, configure); 46 | var stream = fs.createReadStream(configure.src, ENCODING); 47 | 48 | stream.on('data', process.bind(stream, configure)); 49 | stream.on('error', function() { 50 | this.emit('end'); // drop file 51 | }); 52 | stream.on('end', () => { 53 | if(typeof configure.dist === 'string') { 54 | if(configure.first) { 55 | fs.writeFileSync(configure.dist, '[]'); 56 | } else { 57 | append(configure); 58 | fs.appendFileSync(configure.dist, ']'); 59 | }; 60 | } else { 61 | configure.dist(configure.cacheArray.slice()); 62 | } 63 | 64 | // remove empty file 65 | if(configure.first || configure.removeSrc) fs.unlink(configure.src, () => {}); 66 | 67 | configure.cacheArray.length = 0; 68 | configure.callback(null, configure); 69 | }); 70 | 71 | } 72 | 73 | /* Process file with stream 74 | * @param {object} configure 75 | * @param {string} ret, string read as stream in one time 76 | */ 77 | function process(configure, ret) { 78 | ret = configure.buffer + ret; 79 | var pos = ret.lastIndexOf('\n'); 80 | configure.buffer = ret.slice(pos + 1); 81 | ret = ret.slice(0, pos); 82 | ret = ret.split(/\n+/); 83 | ret = ret.map(item => item && mapper(item.split(configure.separator), configure.map)); 84 | 85 | if(!ret.length) return; 86 | configure.count += ret.length; 87 | 88 | if(configure.first && typeof configure.dist === 'string' ) { 89 | configure.first = 0; 90 | fs.writeFileSync(configure.dist, '['); 91 | } 92 | 93 | if(configure.cacheArray.length > STOP) { 94 | // if configure.dist is a function, instead of creating a new file, exec it with the result 95 | // the result should be an array 96 | typeof configure.dist === 'string' ? append(configure) : configure.dist(configure.cacheArray.slice()); 97 | return configure.cacheArray.length = 0; 98 | } 99 | 100 | configure.cacheArray.push.apply(configure.cacheArray, ret); 101 | }; 102 | 103 | /* Append data to file 104 | * @param {object} configure 105 | */ 106 | function append(configure) { 107 | fs.appendFileSync(configure.dist, JSON.stringify(configure.cacheArray).slice(1, -1)); 108 | } 109 | 110 | /* Mapper: mapping fields with a specific map 111 | * @param {array} fields 112 | * @param {array} map 113 | * @param {array} [{}, {}, {}, ...] 114 | */ 115 | function mapper(fields, map) { 116 | var obj = {}; 117 | fields.forEach((cur, i) => { 118 | let kv = pair(map[i], cur, i); 119 | obj[kv.name] = kv.value; 120 | }); 121 | return obj; 122 | } 123 | 124 | /* Generate a proper key / value pair 125 | * @param {string|function} key 126 | */ 127 | function pair(name, value, i) { 128 | if(typeof name === 'function') return name(value); 129 | if(!name) return {name: `notMappedField${i}`, value}; 130 | 131 | var candidateKey = name.split('|'); 132 | var directive = candidateKey[1]; 133 | if(keys.indexOf(directive) !== -1) return {name: candidateKey[0], value: format[directive](value)}; 134 | 135 | return {name, value}; 136 | } 137 | 138 | /* Transform queryString to object 139 | * @param {string} queryString 140 | */ 141 | function url(queryString) { 142 | queryString = decodeURIComponent(queryString); 143 | queryString = queryString.split('&'); 144 | return queryString.reduce((ret, url) => { 145 | url = url.split('='); 146 | var key = url[0]; 147 | var val = url[1]; 148 | 149 | // EXAMPLE: key[]=bar&key[]=world => 150 | // ret = { key: ['bar', 'world'] } 151 | if(key.slice(-2) === '[]') { 152 | key = key.slice(0, -2); 153 | ret[key] ? ret[key].push(val) : (ret[key] = [val]); 154 | 155 | // EXAMPLE: key[foo]=bar&key[hello]=world => 156 | // ret = { key: {foo: 'bar', hello: 'world'} } 157 | } else if(key.match(/\[\w+\]$/)) { 158 | let keys = key.split(/\[|\]/); 159 | let subkey = keys[1]; 160 | key = keys[0]; 161 | 162 | if(!ret[key]) ret[key] = {}; 163 | ret[key][subkey] = val; 164 | } else { 165 | ret[key] = val; 166 | } 167 | return ret; 168 | }, {}); 169 | } 170 | 171 | // Transform a `,` separated string to array 172 | function array(text) { 173 | return text.split(','); 174 | } 175 | 176 | // Transform string to number 177 | function number(str) { 178 | return +str; 179 | } 180 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "log2json", 3 | "version": "1.1.1", 4 | "description": "transform nginx log to JSON", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "node ./test/run.js" 8 | }, 9 | "keywords": [ 10 | "nginx", 11 | "logformat", 12 | "JSON" 13 | ], 14 | "author": "sofish (http://sofi.sh/)", 15 | "license": "MIT", 16 | "directories": { 17 | "test": "test" 18 | }, 19 | "repository": { 20 | "type": "git", 21 | "url": "https://github.com/sofish/log2json.git" 22 | }, 23 | "bugs": { 24 | "url": "https://github.com/sofish/log2json/issues" 25 | }, 26 | "homepage": "https://github.com/sofish/log2json" 27 | } 28 | -------------------------------------------------------------------------------- /test/log: -------------------------------------------------------------------------------- 1 | 2015-11-14T23:59:14+08:00•-•223.104.14.184•-•-•-•http://opensite.ele.me/place/ww24pej69znn•-•CN•-•-•-•35.0000•-•105.0000•-•test=1st•-•Mozilla/5.0 (Linux; Android 4.4.4; m1 note Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 Proxy/cootekservice Proxy/cootekservice•-•a,b,c,d•-•forTestOnly•-•key is missing 2 | 2015-11-14T23:59:16+08:00•-•113.206.187.156•-•-•-•http://opensite.ele.me/place/wm78nvbdy1f4•-•CN•-•Chongqing•-•29.5628•-•106.5528•-•test=1st•-•Mozilla/5.0 (Linux; U; Android 4.4.2; zh-cn; H60-L02 Build/HDH60-L02) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025478 Mobile Safari/533.1 Proxy/cootekservice Proxy/cootekservice 3 | 2015-11-14T23:59:33+08:00•-•117.136.38.174•-•-•-•http://opensite.ele.me/place/wx4eu6hy2zz8•-•CN•-•Beijing•-•39.9289•-•116.3883•-•test=1st•-•Mozilla/5.0 (Linux; Android 4.4.4; Coolpad Y75 Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 4 | 2015-11-14T23:59:36+08:00•-•117.136.41.19•-•-•-•http://opensite.ele.me/place/ws04r95rgjcg•-•CN•-•Guangzhou•-•23.1167•-•113.2500•-•test=1st•-•Mozilla/5.0 (Linux; U; Android 4.4.4; zh-cn; HM NOTE 1LTE Build/KTU84P) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025478 Mobile Safari/533.1 Proxy/cootekservice Proxy/cootekservice 5 | 6 | 2015-11-14T23:59:39+08:00•-•112.17.245.48•-•-•-•http://opensite.ele.me/place/wtjrsrj5wkg2•-•CN•-•Hangzhou•-•30.2936•-•120.1614•-•test=1st•-•Mozilla/5.0 (Linux; Android 5.1; Lovme-T9 Build/LMY47D) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/39.0.0.0 Mobile Safari/537.36 Proxy/cootekservice Proxy/cootekservice 7 | 2015-11-14T23:59:49+08:00•-•61.180.204.27•-•-•-•http://opensite.ele.me/place/yb2jkmeu79fx•-•CN•-•Harbin•-•45.7500•-•126.6500•-•test=1st•-•Mozilla/5.0 (Linux; U; Android 5.1.1; zh-cn; SM-G9250 Build/LMY47X) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025478 Mobile Safari/533.1 Proxy/cootekservice Proxy/cootekservice 8 | 2015-11-15T00:00:01+08:00•-•123.66.221.175•-•-•-•http://opensite.ele.me/place/wx4eubrs7m59•-•CN•-•Beijing•-•39.9289•-•116.3883•-•test=1st•-•Mozilla/5.0 (Linux; U; Android 4.3; zh-cn; GT-N7108 Build/JSS15J) AppleWebKit/533.1 (KHTML, like Gecko)Version/4.0 MQQBrowser/5.4 TBS/025483 Mobile Safari/533.1 Proxy/cootekservice Proxy/cootekservice 9 | 2015-11-15T00:00:04+08:00•-•218.26.55.32•-•-•-•http://opensite.ele.me/place/ww2mtwq4vrbp•-•CN•-•Datong•-•40.0936•-•113.2914•-•test=1st•-•Mozilla/5.0 (iPhone; CPU iPhone OS 7_1 like Mac OS X) AppleWebKit/537.51.2 (KHTML, like Gecko) Mobile/11D167 10 | 2015-11-15T00:00:19+08:00•-•223.104.25.94•-•-•-•http://opensite.ele.me/place/wm783k7tkyfr•-•CN•-•Chongqing•-•29.5628•-•106.5528•-•test=1st•-•Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Mobile/13B143 -------------------------------------------------------------------------------- /test/run.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var log2json = require('../'); 4 | var path = require('path'); 5 | var fs = require('fs'); 6 | var map = ['createdAt', 'origin', 'xForwardFor', 'referrer', 'country', 'city', 7 | 'latitude|number', 'longitude|number', 'query|url', 'userAgent', 'arr|array', transform]; 8 | var separator = '•-•'; 9 | var src = path.join(__dirname, './log'); 10 | var dist = path.join(__dirname, './log.json'); 11 | var removeSrc = false; 12 | var configure = {map, separator, src, dist, removeSrc, callback}; 13 | var configure2 = {map, separator, src, dist: jsonCallback , removeSrc}; 14 | 15 | var the1stobj = { 16 | "createdAt": "2015-11-14T23:59:14+08:00", 17 | "origin": "223.104.14.184", 18 | "xForwardFor": "-", 19 | "referrer": "http://opensite.ele.me/place/ww24pej69znn", 20 | "country": "CN", 21 | "city": "-", 22 | "latitude": 35, 23 | "longitude": 105, 24 | "query": { "test": "1st" }, 25 | "userAgent": "Mozilla/5.0 (Linux; Android 4.4.4; m1 note Build/KTU84P) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/33.0.0.0 Mobile Safari/537.36 Proxy/cootekservice Proxy/cootekservice", 26 | "arr": [ "a", "b", "c", "d" ], 27 | "transformed": "fortestonly", 28 | "notMappedField12": "key is missing" 29 | }; 30 | 31 | log2json(configure); 32 | log2json(configure2); 33 | 34 | function callback(err, ret) { 35 | if(err) throw err; 36 | if(ret) console.log(`✓ JSON is generated: ${shortPath(ret.dist)}`); 37 | 38 | test(ret); // run test 39 | } 40 | 41 | function transform(text) { 42 | var value= ''; 43 | var name = 'transformed'; 44 | if(text) value = text.toLowerCase(); 45 | return {name, value}; 46 | } 47 | 48 | function jsonCallback(ret) { 49 | if(ret.length === 8) return console.log('✓ [2] JSON object is returned instead of creating new file'); 50 | console.log('✘ [2] expected an array'); 51 | throw new Error(); 52 | } 53 | 54 | function test(conf) { 55 | var json = require(conf.dist); 56 | var obj = json[0]; 57 | var keys = Object.keys(obj); 58 | var arr = []; 59 | var fail = 0; 60 | 61 | // parse keys 62 | keys = map.map((item, i) => { 63 | var name = kv(item, obj[item], i); 64 | if(name.directive) arr[i] = name.directive; 65 | return name.name || name; 66 | }); 67 | 68 | // test map 69 | keys.forEach((item, i) => { 70 | if(keys.indexOf(item) !== -1) { 71 | if( 72 | the1stobj[item] === obj[item] || 73 | (obj[item] && typeof obj[item] === 'object' && JSON.stringify(the1stobj[item]) === JSON.stringify(obj[item])) 74 | ) return console.log('✓ mapped as `%s` with right value', item); 75 | fail++; 76 | return console.log(`✘ mapped, but the expected value of \`${item}\` is ${the1stobj[item]}`); 77 | } 78 | fail++; 79 | console.log('✘ item is not mapped %s', item); 80 | }); 81 | 82 | if(obj.notMappedField12 === the1stobj.notMappedField12) { 83 | console.log('✓ when key is missing, should mapped as `notMappedFieldN`, like `%s`', 'notMappedField12'); 84 | } else { 85 | fail++; 86 | console.log('✘ notMappedField12 is not mapped'); 87 | } 88 | 89 | var isSrcExists = true; 90 | try { 91 | isSrcExists = fs.statSync(configure.src).isFile(); 92 | } catch(e) { 93 | isSrcExists = false; 94 | } 95 | 96 | console.log(configure.src); 97 | var short = shortPath(configure.src); 98 | if(configure.removeSrc) { 99 | let icon = isSrcExists ? '✘' : '✓'; 100 | console.log(`${icon} when \`removeSrc = true\` ${short} should be removed`); 101 | } else { 102 | let icon = isSrcExists ? '✓' : '✘'; 103 | console.log(`${icon} when \`removeSrc = false\` ${short} should exists`); 104 | } 105 | 106 | console.log(); 107 | console.log('= SUCCESS: %d, FAIL: %d', Object.keys(the1stobj).length - fail, fail); 108 | console.log('= done!\n' ); 109 | 110 | if(fail > 0) throw new Error(`${fail} testcases failed`); 111 | } 112 | 113 | function kv(name, value, i) { 114 | if(typeof name === 'function') return name(value).name; 115 | if(!name) return `notMappedField${i}`;; 116 | 117 | var candidateKey = name.split('|'); 118 | if(candidateKey[1]) return {name: candidateKey[0], directive: candidateKey[1]}; 119 | 120 | return name; 121 | } 122 | 123 | function shortPath(str) { 124 | if(str.length < 25) return str; 125 | return str.slice(0, 10) + '...' + str.slice(-10); 126 | } 127 | --------------------------------------------------------------------------------