├── .gitignore ├── .jshintrc ├── .tm_properties ├── .travis.yml ├── Makefile ├── README.md ├── lib ├── connect-logger.js ├── elasticsearch-bulk.js ├── elasticsearch-client.js ├── log4js-elasticsearch-layouts.js └── log4js-elasticsearch.js ├── package.json └── test ├── integration └── test-real-eslogger.js ├── test-connect-logger.js ├── test-es-template.js └── test-eslogger.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | .DS_Store 4 | meta.json 5 | .idea 6 | coverage.html 7 | lib-cov 8 | .coverage_data 9 | reports 10 | html-report 11 | build 12 | cobertura-coverage.xml 13 | metadata 14 | *.tgz 15 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "proto": true, 3 | "browser": true, 4 | "curly": true, 5 | "devel": true, 6 | "eqeqeq": true, 7 | "eqnull": true, 8 | "es5": false, 9 | "evil": false, 10 | "immed": false, 11 | "jquery": true, 12 | "latedef": false, 13 | "laxcomma": true, 14 | "newcap": true, 15 | "node": true, 16 | "noempty": true, 17 | "nonew": true, 18 | "predef": 19 | [ 20 | "after", 21 | "afterEach", 22 | "before", 23 | "beforeEach", 24 | "describe", 25 | "it", 26 | "unescape", 27 | "par", 28 | "each", 29 | "setImmediate" 30 | ], 31 | "smarttabs": true, 32 | "trailing": false, 33 | "undef": true, 34 | "strict": false, 35 | "expr": true 36 | } 37 | -------------------------------------------------------------------------------- /.tm_properties: -------------------------------------------------------------------------------- 1 | fontName = "Monaco" 2 | fontSize = 12 3 | 4 | myExtraExcludes = "log,vendor,tmp,node_modules" 5 | excludeInFileChooser = "{$excludeInFileChooser,$myExtraExcludes}" 6 | excludeInFolderSearch = "{$excludeInFolderSearch,$myExtraExcludes}" 7 | excludeInBrowser = "{$excludeInBrowser,log,vendor,tmp,node_modules}" 8 | 9 | showInvisibles = true 10 | softTabs = true 11 | tabSize = 2 12 | wrapColumn = 120 13 | showWrapColumn = true 14 | 15 | projectDirectory = "$CWD" 16 | windowTitle = "$TM_DISPLAYNAME — ${CWD/^.*\///} ($TM_SCM_BRANCH)" 17 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "0.10" 4 | - "0.8" -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | BIN = ./node_modules/.bin 2 | MOCHA_OPTS = --timeout 2000 3 | REPORTER = spec 4 | TEST_FILES = test/*.js 5 | TEST_INTEGRATION_FILES = test/integration/*.js 6 | BIN = ./node_modules/.bin 7 | 8 | lint: 9 | $(BIN)/jshint lib/* test/* --config .jshintrc 10 | 11 | test: lint 12 | $(BIN)/mocha \ 13 | $(MOCHA_OPTS) \ 14 | --reporter $(REPORTER) \ 15 | $(TEST) 16 | 17 | test-integration: lint 18 | $(BIN)/mocha \ 19 | $(MOCHA_OPTS) \ 20 | --reporter $(REPORTER) \ 21 | $(TEST_INTEGRATION_FILES) 22 | 23 | test-ci: 24 | $(MAKE) -k test MOCHA_OPTS="$(MOCHA_OPTS) --watch --growl" REPORTER="min" 25 | 26 | lib-cov: 27 | [ -d "lib-cov" ] && rm -rf lib-cov || true 28 | $(BIN)/istanbul instrument --output lib-cov --no-compact --variable global.__coverage__ lib 29 | 30 | test-cov: lib-cov 31 | @LOG4JS_COV=1 $(MAKE) test "REPORTER=mocha-istanbul" ISTANBUL_REPORTERS=text-summary,html 32 | 33 | clean: 34 | [ -d "lib-cov" ] && rm -rf lib-cov || true 35 | [ -d "reports" ] && rm -rf reports || true 36 | [ -d "build" ] && rm -rf build || true 37 | 38 | .PHONY: test 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | log4js-elasticsearch 2 | ==================== 3 | 4 | log4js-elasticsearch is a log4js log appender to push log messages into [elasticsearch](http://elasticsearch.org). 5 | [Kibana](http://three.kibana.org) is the awesome tool to view the logs. 6 | 7 | The logs produced are compatible with [logstash's elasticsearch_http output](logstash.net/docs/1.1.12/outputs/elasticsearch_http). 8 | 9 | AWS Managed Elasticsearch support 10 | --------------------------------- 11 | Supported here: https://github.com/ironSource/log4js-elasticsearch-aws 12 | 13 | Installation 14 | ------------ 15 | 16 | You can install install log4js-elasticsearch via npm: 17 | 18 | npm install log4js-elasticsearch 19 | 20 | Usage: basic 21 | ------------ 22 | 23 | var log4js = require('log4js'); 24 | var esAppenderConfig = { 25 | url: 'http://user:password@myelasticsearch.com:9200' 26 | }; 27 | var log4jsESAppender = require('log4js-elasticsearch').configure(esAppenderConfig); 28 | log4js.addAppender(log4js, 'tests'); 29 | 30 | The default url of the ES server is http://localhost:9200 31 | 32 | Usage: log4js configuration 33 | --------------------------- 34 | ```javascript 35 | var log4js = require('log4js'); 36 | log4js.configure({ 37 | "appenders": [ 38 | { 39 | "category": "tests", 40 | "type": "logLevelFilter", 41 | "level": "WARN", 42 | "appender": { 43 | "type": "log4js-elasticsearch", 44 | "url": "http://127.0.0.1:9200" 45 | } 46 | }, 47 | { 48 | "category": "tests", 49 | "type": "console" 50 | } 51 | ], 52 | "levels": { 53 | "tests": "DEBUG" 54 | } 55 | }); 56 | 57 | var log = log4js.getLogger('tests'); 58 | 59 | log.error('hello hello'); 60 | 61 | if (setTimeout(function() {}).unref === undefined) { 62 | console.log('force flushing and goodbye for node <= 0.8'); 63 | require('log4js-elasticsearch').flushAll(true); 64 | } 65 | ``` 66 | 67 | Note: clearing the timer used by log4js-elasticsearch 68 | ----------------------------------------------------- 69 | By default the logs are posted every 30 seconds to elasticsearch or when more than 256 log events have been issued. 70 | 71 | Sending the logs by batches on a regular basis is a lot more performant 72 | than one by one. 73 | 74 | However a node process will not exit until it has no more referenced timers. 75 | Since node-0.9 it is possible to have a 'soft' timer that is not referenced. 76 | For node-0.8 and older it is required to close the log4js elasticsearch appenders: 77 | 78 | ```javascript 79 | // manually close the elasticsearch appenders: 80 | require('log4js-elasticsearch').flushAll(true); 81 | ``` 82 | 83 | If the process is a server and it is simply meant to stop when killed then no need to do anything: process event listeners are added and a best attempt is made to send the pending logs before exiting. Tested on node-0.8 and node-0.10. 84 | 85 | Usage: custom 86 | ------------- 87 | 88 | var log4js = require('log4js'); 89 | var uuid = require('node-uuid'); 90 | log4js.configure({ 91 | "appenders": [ { 92 | "type": "log4js-elasticsearch", 93 | "indexName": function(loggingEvent) { 94 | return loggingEvent.categoryName; 95 | }, 96 | "typeName": function(loggingEvent) { 97 | return loggingEvent.level.levelStr; 98 | }, 99 | "url": "http://127.0.0.1:9200", 100 | "logId": function(loggingEvent) { 101 | return uuid.v4(); 102 | }, 103 | "buffersize": 1024, 104 | "timeout": 45000, 105 | "layout": { 106 | "type": "logstash", 107 | "tags": [ "mytag" ], 108 | "sourceHost": function(loggingEvent) { 109 | return "it-depends"; 110 | } 111 | } 112 | } 113 | ], 114 | "levels": { 115 | "tests": "DEBUG" 116 | } 117 | }); 118 | 119 | Usage: connect logger 120 | --------------------- 121 | ```javascript 122 | var log4js = require('log4js'); 123 | var logstashConnectFormatter = require('log4js-elasticsearch').logstashConnectFormatter; 124 | log4js.configure({ 125 | "appenders": [ 126 | { 127 | "type": "log4js-elasticsearch", 128 | "esclient": mockElasticsearchClient, 129 | "buffersize": 1, 130 | "layout": { type: 'logstash' } 131 | } 132 | ] 133 | }); 134 | var logger = log4js.getLogger('express'); 135 | var connectLogger = log4js.connectLogger(logger, { format: logstashConnectFormatter }); 136 | app.use(connectLogger); //where app is the express app. 137 | ``` 138 | 139 | Appender configuration parameters 140 | ================================= 141 | - `url`: the URL of the elasticsearch server. 142 | Basic authentication is supported. 143 | Default: http://localhost:9200 144 | 145 | - `indexName`: the name of the elasticsearch index in which the logs are stored. 146 | Either a static string either a function that is passed the logging event. 147 | Defaults: undefined'; The indexNamePrefix is used by default. 148 | 149 | - `indexNamePrefix`: the prefix of the index name in which the logs are stored. 150 | The name of the actual index is suffixed with the date: `%{+YYYY.MM.dd}` and changes every day, UTC time. 151 | Defaults: 'logstash-'. 152 | 153 | - `typeName`: the name of the elasticsearch object in which the logs are posted. 154 | Either a string or a function that is passed the logging event. 155 | Default: 'nodejs'. 156 | 157 | - `layout`: object descriptor for the layout. 158 | By default the layout is logstash. 159 | 160 | - `buffersize`: number of logs events to buffer before posted in bulks to elasticsearch 161 | Default: 256 162 | 163 | - `timeout`: number of milliseconds to wait until the logs buffer is posted to elasticsearch; regardless of its size. 164 | Default: 30 seconds. 165 | 166 | - `logId`: function that returns the value of the `_id` of the logging event. 167 | Default: undefined to let elasticsearch generate it. 168 | 169 | Additional Built-in layouts 170 | ============================ 171 | 172 | The following layouts are added to the log4js builtin layouts: 173 | - logstash 174 | - simpleJson 175 | 176 | The following parameters are the children of the `layout` parameter in the appender's configuration for those new built-in layouts. 177 | 178 | Default: Logstash layout 179 | ------------------------ 180 | The logstash layout posts logs in the same structure than [logstash's elasticsearch_http output](logstash.net/docs/1.1.12/outputs/elasticsearch_http). 181 | 182 | - `tags`: output as the value of the `@tags` property. 183 | A static array or a function that is passed the logging event. 184 | Default: empty array. 185 | 186 | - `sourceHost`: output as the value of the `@source_host` property. 187 | A static string or a function that is passed the logging event 188 | Default: OS's hostname. 189 | 190 | - `source`: output as the value of the `@source` property. 191 | A string. 192 | Default: 'log4js'. 193 | 194 | - `sourcePath`: output as the value of the `@source_path` property 195 | A string. 196 | Default: working directory of the current process. 197 | 198 | - `logId`: outputs the value of the `_id` field. 199 | A function or undefined to let elasticsearch generates it. 200 | Default: undefined. 201 | 202 | - `template`: the elasticsearch template to define. 203 | Only used if no template with the same name is defined. 204 | Default: from [untergeek's using-templates-to-improve-elasticsearch-caching-with-logstash](http://untergeek.com/2012/09/20/using-templates-to-improve-elasticsearch-caching-with-logstash/). 205 | 206 | simpleJson Layout 207 | ----------------- 208 | A simple message pass through of the loggingEvent. 209 | 210 | License 211 | ======= 212 | MIT 213 | 214 | Copyright 215 | ========= 216 | (c) 2013 Sutoiku, Inc. 217 | -------------------------------------------------------------------------------- /lib/connect-logger.js: -------------------------------------------------------------------------------- 1 | /* 2 | "@fields" => { 3 | "client" => "127.0.0.1", 4 | "duration_usec" => 240, 5 | "status" => 404, 6 | "request" => "/favicon.ico", 7 | "method" => "GET", 8 | "referrer" => "-" 9 | }, 10 | */ 11 | function logstashConnectFormatter(req, res, formattedOutput) { 12 | var fields = { 13 | url: req.originalUrl, 14 | method: req.method, 15 | status: res.__statusCode || res.statusCode, 16 | 'response-time': res.responseTime, 17 | referrer: req.headers.referer || req.headers.referrer || '', 18 | 'http-version': req.httpVersionMajor + '.' + req.httpVersionMinor, 19 | 'remote-addr': req.socket && (req.socket.remoteAddress || (req.socket.socket && req.socket.socket.remoteAddress)), 20 | 'user-agent': req.headers['user-agent'] || '', 21 | 'content-length': (res._headers && res._headers['content-length']) || (res.__headers && res.__headers['Content-Length']) || -1 22 | }; 23 | var message = formattedOutput(':remote-addr - - ":method :url HTTP/:http-version" :status :content-length ":referrer" ":user-agent"', req, res); 24 | var resul = {}; 25 | Object.defineProperty(resul, '@fields', { value: fields, enumerable: true }); 26 | resul.toString = function() { 27 | return message; 28 | }; 29 | return resul; 30 | } 31 | exports.logstashConnectFormatter = logstashConnectFormatter; 32 | -------------------------------------------------------------------------------- /lib/elasticsearch-bulk.js: -------------------------------------------------------------------------------- 1 | var esLogBuffers = []; 2 | 3 | function EsOneAtaTime(layout, esclient, layoutES) { 4 | this.esclient = esclient; 5 | this.layout = layout; 6 | this.layoutES = layoutES; 7 | } 8 | 9 | EsOneAtaTime.prototype.log = function(loggingEvent) { 10 | this.esclient.index(this.layoutES.indexName(loggingEvent) 11 | , this.layoutES.typeName(loggingEvent) 12 | , this.layout(loggingEvent) 13 | , this.layoutES.logId(loggingEvent) 14 | , function() { 15 | //emit an error? 16 | }); 17 | }; 18 | 19 | 20 | function EsLogBuffer(layout, esclient, layoutES, timeout, buffersize) { 21 | this.esclient = esclient; 22 | this.layout = layout; 23 | this.layoutES = layoutES; 24 | if (buffersize > 0) { 25 | this.bulkCmdsLength = buffersize * 2; 26 | } 27 | this.timeout = timeout; 28 | this.bulkCmds = []; 29 | if (this.timeout > 0) { 30 | var self = this; 31 | this.interv = setInterval(function() { 32 | self.flush(); 33 | }, this.timeout); 34 | if (typeof this.interv.unref === 'function') { 35 | this.interv.unref(); 36 | } else { 37 | console.warn('No support for Timer#unref:\n' + 38 | ' the setInterval object will keep running the node\n' + 39 | ' process until it stop for some other reason.\n' + 40 | ' Call require(log4js-elasticsearch).flushAll(true)\n' + 41 | ' Call EsLogBuffer#close to clear the setInterval.'); 42 | } 43 | setupProcessListeners(); 44 | } 45 | } 46 | 47 | 48 | EsLogBuffer.prototype.log = function(loggingEvent) { 49 | this.bulkCmds.push({ 50 | index: { 51 | _id: this.layoutES.logId(loggingEvent) 52 | , _type: this.layoutES.typeName(loggingEvent) 53 | , _index: this.layoutES.indexName(loggingEvent) 54 | } 55 | }); 56 | this.bulkCmds.push(this.layout(loggingEvent)); 57 | if (this.bulkCmds.length >= this.bulkCmdsLength) { 58 | this.flush(); 59 | } 60 | }; 61 | 62 | EsLogBuffer.prototype.close = function() { 63 | if (this.interv) { 64 | clearTimeout(this.interv); 65 | } 66 | }; 67 | 68 | EsLogBuffer.prototype.flush = function(optionalDone) { 69 | if (this.bulkCmds.length === 0) { 70 | if (optionalDone) { 71 | optionalDone(); 72 | } 73 | return; 74 | } 75 | if (!optionalDone) { 76 | optionalDone = function() { 77 | //emit an event to say it is over? 78 | }; 79 | } 80 | var current = this.bulkCmds; 81 | this.bulkCmds = []; 82 | this.esclient.bulk(current, optionalDone); 83 | }; 84 | 85 | /** 86 | * The function that knows how to post to elasticsearch: 87 | * one at a time or in bulk mode regurlarly. 88 | */ 89 | exports.makeEsLog = function(layout, esclient, layoutES, timeout, buffersize) { 90 | if (buffersize === 1) { 91 | return new EsOneAtaTime(layout, esclient, layoutES); 92 | } 93 | if ((!timeout || timeout === 0) && (!buffersize || buffersize <= 0)) { 94 | buffersize = 256; 95 | } 96 | if (!timeout) { 97 | timeout = 30000; //30 seconds 98 | } 99 | 100 | var esBulk = new EsLogBuffer(layout, esclient, layoutES, timeout, buffersize); 101 | esLogBuffers.push(esBulk); 102 | return esBulk; 103 | }; 104 | 105 | function flushAll(doreset, optionalDone) { 106 | var flushthem = esLogBuffers; 107 | if (doreset) { 108 | esLogBuffers = []; 109 | clearProcessListeners(); 110 | } 111 | if (typeof optionalDone === 'function') { 112 | var i = -1; 113 | var flushOneAtATime = function() { 114 | i++; 115 | var curr = flushthem[i]; 116 | if (curr) { 117 | curr.flush(function(err) { 118 | if (doreset) { 119 | curr.close(); 120 | } 121 | flushOneAtATime(); 122 | }); 123 | } else { 124 | optionalDone(); 125 | } 126 | }; 127 | flushOneAtATime(); 128 | } else { 129 | flushthem.forEach(function (esLogBuffer) { 130 | if (doreset) { 131 | esLogBuffer.close(); 132 | } 133 | esLogBuffer.flush(); 134 | }); 135 | } 136 | } 137 | 138 | var processListenersSetup = false; 139 | 140 | var onProcessSigint = function() { 141 | clearProcessListeners(); 142 | flushAll(true, function() { 143 | process.kill(process.pid, 'SIGINT'); 144 | }); 145 | }; 146 | var onProcessExit = function() { 147 | clearProcessListeners(); 148 | flushAll(true, function() { 149 | process.exit(); 150 | }); 151 | }; 152 | 153 | function setupProcessListeners() { 154 | if (!processListenersSetup) { 155 | processListenersSetup = true; 156 | process.addListener('SIGINT', onProcessSigint); 157 | process.addListener('exit', onProcessExit); 158 | } 159 | } 160 | function clearProcessListeners() { 161 | if (processListenersSetup) { 162 | processListenersSetup = false; 163 | process.removeListener('SIGINT', onProcessSigint); 164 | process.removeListener('exit', onProcessExit); 165 | } 166 | } 167 | 168 | exports.flushAll = flushAll; 169 | -------------------------------------------------------------------------------- /lib/elasticsearch-client.js: -------------------------------------------------------------------------------- 1 | var parseUrl = require('url').parse; 2 | var http = require('http'); 3 | var https = require('https'); 4 | var EventEmitter = require('events').EventEmitter; 5 | 6 | function ElasticSearchClient(options) { 7 | this.secure = options.secure || false; 8 | this.host = options.host; 9 | this.port = options.port; 10 | if (options.auth) { 11 | this.auth = options.auth.username; 12 | if (options.auth.password) { 13 | this.auth += ':' + options.auth.password; 14 | } 15 | } 16 | this.timeout = options.timeout || false; 17 | this.httpClient = this.secure ? https : http; 18 | this.agent = options.agent; 19 | } 20 | // inherit nodejs events emitter 21 | ElasticSearchClient.prototype = Object.create(EventEmitter.prototype); 22 | 23 | var numberOfErrorsReported = 3; 24 | // default error listener will report in the console 3 times 25 | var errorDefaultListener = function(err) { 26 | if (numberOfErrorsReported) { 27 | console.log('Error %d occured while posting log to ES:', err); 28 | numberOfErrorsReported--; 29 | } 30 | }; 31 | 32 | ElasticSearchClient.prototype.init = function() { 33 | // node's event emitter will crash if no error listeners were registered 34 | this.on('error', errorDefaultListener); 35 | var self = this; 36 | this.on('newListener', function(eventName, listener) { 37 | if (eventName === 'error' && listener !== errorDefaultListener) { 38 | // Remove the default listener if it had been registered previously. 39 | self.removeListener(eventName, errorDefaultListener); 40 | } 41 | }); 42 | }; 43 | 44 | ElasticSearchClient.prototype.__makeRequestOptions = function(path, method) { 45 | return { 46 | path: path, 47 | method: method, 48 | host: this.host, 49 | port: this.port, 50 | auth: this.auth, 51 | agent: this.agent 52 | }; 53 | }; 54 | 55 | ElasticSearchClient.prototype.index = function(indexName, typeName, data, id, done) { 56 | var path = '/' + indexName + '/' + typeName, method; 57 | if (typeof id === 'function' && done === undefined) { 58 | done = id; 59 | id = undefined; 60 | method = 'POST'; 61 | } else if (id) { 62 | path += "/" + id; 63 | method = 'PUT'; 64 | } else { 65 | method = 'POST'; 66 | } 67 | var reqOptions = this.__makeRequestOptions(path, method); 68 | var request = this.httpClient.request(reqOptions); 69 | this.execRequest(request, data, done); 70 | }; 71 | 72 | /** 73 | * indexName and typeName are optional. 74 | */ 75 | ElasticSearchClient.prototype.bulk = function(indexName, typeName, bulkCmds, done) { 76 | //first arg that is an array is bulkCmds; 77 | if (Array.isArray(indexName)) { 78 | bulkCmds = indexName; 79 | done = typeName; 80 | indexName = undefined; 81 | typeName = undefined; 82 | } else if (Array.isArray(typeName)) { 83 | bulkCmds = typeName; 84 | done = bulkCmds; 85 | typeName = undefined; 86 | } 87 | var path = '/_bulk'; 88 | if (indexName) { 89 | if (typeName) { 90 | path = '/' + indexName + '/' + typeName + path; 91 | } else { 92 | path = '/' + indexName + path; 93 | } 94 | } 95 | var data = ''; //don't even think about buffer, this is what is most efficient in V8 96 | for (var i = 0; i < bulkCmds.length; i++) { 97 | data += JSON.stringify(bulkCmds[i]) + '\n'; 98 | } 99 | var reqOptions = this.__makeRequestOptions(path, 'POST'); 100 | var request = this.httpClient.request(reqOptions); 101 | this.execRequest(request, data, done); 102 | }; 103 | 104 | ElasticSearchClient.prototype.getTemplate = function(templateName, done) { 105 | var path = '/_template/' + templateName; 106 | var reqOptions = this.__makeRequestOptions(path, 'GET'); 107 | var request = this.httpClient.request(reqOptions); 108 | this.execRequest(request, null, done); 109 | }; 110 | 111 | ElasticSearchClient.prototype.defineTemplate = function(templateName, template, done) { 112 | var path = '/_template/' + templateName; 113 | var reqOptions = this.__makeRequestOptions(path, 'PUT'); 114 | var request = this.httpClient.request(reqOptions); 115 | this.execRequest(request, template, done); 116 | }; 117 | 118 | /** 119 | * @param request is either an http.request object or an array [ http.request, postOrPutPayload ] 120 | * @param data is optional. 121 | */ 122 | ElasticSearchClient.prototype.execRequest = function(request, data, done) { 123 | if (done === undefined && typeof data === 'function') { 124 | done = data; 125 | } 126 | var self = this; 127 | if (self.timeout) { 128 | request.setTimeout(self.timeout, function () { 129 | self.emit('error', new Error('timed out after ' + self.timeout + 'ms')); 130 | }); 131 | } 132 | 133 | request.on('error', function (error) { 134 | self.emit("error", error); 135 | }); 136 | 137 | request.on('response', function (response) { 138 | var body = ""; 139 | response.on('data', function (chunk) { 140 | body += chunk; 141 | }); 142 | response.on('end', function () { 143 | if (typeof done === 'function') { 144 | done(undefined, body); 145 | } else { 146 | self.emit("data", body); 147 | self.emit("done", 0); 148 | } 149 | }); 150 | response.on('error', function (error) { 151 | if (typeof self.callback === 'function') { 152 | // console.log('problem', error); 153 | done(error); 154 | } else { 155 | self.emit("error", error); 156 | } 157 | }); 158 | }); 159 | if (data) { 160 | if (typeof data !== 'string') { 161 | data = JSON.stringify(data); 162 | } 163 | request.setHeader('Content-Type', 'application/json'); 164 | request.setHeader('Content-Length', Buffer.byteLength(data, 'utf8')); 165 | request.end(data); 166 | } else { 167 | request.end(''); 168 | } 169 | 170 | }; 171 | 172 | /** 173 | * Parses a URL with optional login / password 174 | * returns the expected json options to configure the connection to ES. 175 | */ 176 | ElasticSearchClient.makeOptions = function(url) { 177 | var urlP = parseUrl(url); 178 | var options = { 179 | host: urlP.hostname 180 | }; 181 | var secure = urlP.protocol === 'https:'; 182 | if (urlP.port !== null && urlP.port !== undefined) { 183 | options.port = urlP.port; 184 | } else if (secure) { 185 | options.port = '443'; 186 | } else { 187 | options.port = '80'; 188 | } 189 | options.secure = secure; 190 | if (urlP.auth) { 191 | var toks = urlP.auth.split(':'); 192 | if (toks.length === 2) { 193 | options.auth = { username: toks[0], password: toks[1] }; 194 | } else { 195 | options.auth = { username: toks[0], password: '' }; 196 | } 197 | } 198 | return options; 199 | }; 200 | 201 | module.exports = ElasticSearchClient; -------------------------------------------------------------------------------- /lib/log4js-elasticsearch-layouts.js: -------------------------------------------------------------------------------- 1 | /** 2 | some idea; maybe not so good: 3 | support the pattern layout but make it as a json serialization ? 4 | */ 5 | var layouts = require('log4js').layouts; 6 | module.exports = layouts; 7 | 8 | var messagePassThroughLayout = layouts.messagePassThroughLayout; 9 | 10 | /* 11 | log4js logging event: 12 | startTime: date, 13 | categoryName: string, 14 | level: { level: int, levelStr: levelStr }, 15 | data: [ args of logger ], 16 | logger: ... circular ... 17 | */ 18 | 19 | 20 | /** 21 | * Outputs a JSON object 22 | */ 23 | function simpleJsonLayout(loggingEvent) { 24 | var data = __formatData(loggingEvent); 25 | var message = data[0], errorMsg = data[1], stack = data[2]; 26 | var base = { 27 | startTime: loggingEvent.startTime, 28 | category: loggingEvent.categoryName, 29 | level: loggingEvent.level.level, 30 | levelStr: loggingEvent.level.levelStr, 31 | message: message 32 | }; 33 | if (errorMsg !== undefined) { 34 | base.error = errorMsg; 35 | base.stack = stack; 36 | } 37 | return base; 38 | } 39 | 40 | /** 41 | * @param the logging event 42 | * @return The JSON 43 | */ 44 | function logstashLayout(loggingEvent) { 45 | var data, fields, message, errorMsg, stack; 46 | if (loggingEvent.data[0] && loggingEvent.data[0]['@fields']) { 47 | fields = loggingEvent.data[0]['@fields']; 48 | message = loggingEvent.data[0]['@message'] || loggingEvent.data.toString(); 49 | } else { 50 | data = __formatData(loggingEvent); 51 | message = data[0], errorMsg = data[1], stack = data[2]; 52 | fields = {}; 53 | } 54 | var eslogger = loggingEvent.logger; 55 | var base = { 56 | '@timestamp': loggingEvent.startTime, 57 | '@message': message 58 | }; 59 | fields.level = loggingEvent.level.level; 60 | fields.levelStr = loggingEvent.level.levelStr; 61 | fields.category = loggingEvent.categoryName; 62 | if (errorMsg) { 63 | fields.error = errorMsg; 64 | fields.stack = stack; 65 | } 66 | base['@fields'] = fields; 67 | return base; 68 | } 69 | 70 | /** 71 | * Extracts the message, error-message and stack track. 72 | */ 73 | function __formatData(loggingEvent) { 74 | var data = loggingEvent.data; 75 | var message, errorMsg, stack; 76 | if (data[data.length -1] instanceof Error) { 77 | var error = data[data.length - 1]; 78 | errorMsg = error.message; 79 | if (typeof error.stack === 'string') { 80 | stack = error.stack.split('\n'); 81 | } else { 82 | stack = error.stack; 83 | } 84 | data = data.splice(0, data.length -1); 85 | message = messagePassThroughLayout({data: data}); 86 | } else { 87 | message = messagePassThroughLayout(loggingEvent); 88 | } 89 | return [ message, errorMsg, stack ]; 90 | } 91 | 92 | layouts.logstashLayout = logstashLayout; 93 | layouts.simpleJsonLayout = simpleJsonLayout; 94 | 95 | 96 | var defaultHostname = require('os').hostname(); 97 | function logstashLayoutMaker(layoutConfig) { 98 | var typeName = layoutConfig.typeName; 99 | var source = layoutConfig.source ? layoutConfig.source : 'log4js'; 100 | 101 | var sourceHost; 102 | if (typeof layoutConfig.sourceHost === 'function') { 103 | sourceHost = layoutConfig.sourceHost; 104 | } else { 105 | sourceHost = function() { 106 | return layoutConfig.sourceHost || defaultHostname; 107 | }; 108 | } 109 | var tags; 110 | if (typeof layoutConfig.tags === 'function') { 111 | tags = layoutConfig.tags; 112 | } else { 113 | tags = function() { 114 | return layoutConfig.tags || []; 115 | }; 116 | } 117 | var sourcePath = layoutConfig.sourcePath ? layoutConfig.sourcePath : process.cwd(); 118 | return function(loggingEvent) { 119 | var layoutOutput = logstashLayout(loggingEvent); 120 | layoutOutput['@type'] = typeName(loggingEvent); 121 | layoutOutput['@source'] = source; 122 | layoutOutput['@source_host'] = sourceHost(loggingEvent); 123 | layoutOutput['@source_path'] = sourcePath; 124 | layoutOutput['@tags'] = tags(loggingEvent); 125 | return layoutOutput; 126 | }; 127 | } 128 | 129 | // add the new layouts: 130 | var oriLayoutMaker = layouts.layout; 131 | if (oriLayoutMaker.name !== 'layoutEs') { 132 | // really sure we don't double monky patch or yagni ? 133 | layouts.layout = function layoutEs(name, config) { 134 | if (name === 'logstash') { 135 | return logstashLayoutMaker(config); 136 | } else if (name === 'simpleJson') { 137 | return layouts.simpleJson; 138 | } else { 139 | return oriLayoutMaker(name, config); 140 | } 141 | }; 142 | } 143 | 144 | function __getOpt(path, options, defaultV) { 145 | if (options === undefined) { 146 | return defaultV; 147 | } 148 | var curr = options; 149 | for(var i = 0; i < path.length; i++) { 150 | var p = path[i]; 151 | if (p) { 152 | curr = curr[p]; 153 | if (curr === undefined) { 154 | return defaultV; 155 | } 156 | } 157 | } 158 | if (curr === '__delete__') { 159 | return undefined; 160 | } else { 161 | return curr; 162 | } 163 | } 164 | 165 | layouts.esTemplateMakers = {}; 166 | 167 | // http://untergeek.com/2012/09/20/using-templates-to-improve-elasticsearch-caching-with-logstash/ 168 | layouts.esTemplateMakers.logstash = function(templateName, templateConfig) { 169 | var nbOfShards = __getOpt(['settings', 'number_of_shards'], templateConfig, 170 | parseInt(process.env.ES_LOG4JS_SHARDS_NUMBER, 10) || 4); 171 | var nbOfReplicas = __getOpt(['settings', 'number_of_replicas'], templateConfig, 172 | parseInt(process.env.ES_LOG4JS_REPLICAS_NUMBER, 10) || 1); 173 | var totalShardsPerNode = __getOpt(['settings', 'index.routing.allocation.total_shards_per_node'], templateConfig, 2); 174 | var cacheFieldType = __getOpt(['settings', 'index.cache.field.type'], templateConfig, "soft"); 175 | var refreshInterval = __getOpt(['settings', 'index.refresh_interval'], templateConfig, "25s"); 176 | var defaultField = __getOpt(['settings', 'index.default_field'], templateConfig, "@message"); 177 | var enableAll = __getOpt(['mappings', '_default_', '_all', 'enabled'], templateConfig, false); 178 | return { 179 | "template" : templateName || "logstash-*", 180 | "settings" : { 181 | "number_of_shards" : nbOfShards, 182 | "number_of_replicas" : nbOfReplicas, 183 | "index.cache.field.type" : cacheFieldType, 184 | "index.refresh_interval" : refreshInterval, 185 | "index.store.compress.stored" : true, 186 | "index.query.default_field" : defaultField, 187 | "index.routing.allocation.total_shards_per_node" : totalShardsPerNode 188 | }, 189 | "mappings" : { 190 | "_default_" : { 191 | "_all" : {"enabled" : enableAll}, 192 | "properties" : { 193 | "@fields" : { "type" : "object", "dynamic": true, "path": "full" }, 194 | "@message": { "type": "string", "index": "analyzed" }, 195 | "@source": { "type": "string", "index": "not_analyzed" }, 196 | "@source_host": { "type": "string", "index": "not_analyzed" }, 197 | "@source_path": { "type": "string", "index": "not_analyzed" }, 198 | "@tags": { "type": "string", "index": "not_analyzed" }, 199 | "@timestamp": { "type": "date", "index": "not_analyzed" }, 200 | "@type": { "type": "string", "index": "not_analyzed" } 201 | } 202 | } 203 | } 204 | }; 205 | }; 206 | 207 | layouts.esTemplateMakers.simpleJson = function(templateName, templateConfig) { 208 | return { 209 | "template" : templateName || "log4js*", 210 | "settings" : { 211 | "number_of_shards" : parseInt(process.env.ES_LOG4JS_SHARDS_NUMBER, 10) || 4, 212 | "number_of_replicas" : parseInt(process.env.ES_LOG4JS_REPLICAS_NUMBER, 10) || 1, 213 | "index.cache.field.type" : "soft", 214 | "index.refresh_interval" : "5s", 215 | "index.store.compress.stored" : true, 216 | "index.query.default_field" : "message", 217 | "index.routing.allocation.total_shards_per_node" : 2 218 | }, 219 | "mappings" : { 220 | "_default_" : { 221 | "_all" : {"enabled" : false}, 222 | "properties" : { 223 | "category": { "type": "string", "index": "not_analyzed" }, 224 | "level": { "type": "integer" }, 225 | "levelStr": { "type": "string", "index": "not_analyzed" }, 226 | "startTime": { "type": "date" }, 227 | "message": { "type": "string", "index": "analyzed" }, 228 | "error": { "type": "string", "index": "analyzed" }, 229 | "stack": { "type": "object", "dynamic": true } 230 | } 231 | } 232 | } 233 | }; 234 | }; 235 | -------------------------------------------------------------------------------- /lib/log4js-elasticsearch.js: -------------------------------------------------------------------------------- 1 | var ElasticsearchClient = require('./elasticsearch-client'); 2 | var layouts = require('./log4js-elasticsearch-layouts'); 3 | var logstashConnectFormatter = require('./connect-logger').logstashConnectFormatter; 4 | var bulk = require('./elasticsearch-bulk'); 5 | 6 | function createAppender(layout, config, options, done) { 7 | var layoutES = makeESHelper(layout, config); 8 | var esclient = initESClient(config, options, layoutES.template, done); 9 | var eslog = bulk.makeEsLog(layout, esclient, layoutES 10 | , config.timeout, config.buffersize); 11 | return function(loggingEvent) { 12 | eslog.log(loggingEvent); 13 | }; 14 | } 15 | 16 | function configure(config, options, done) { 17 | var layout; 18 | config = loadAppenderConfig(config); 19 | layout = layouts.layout(config.layout.type, config.layout); 20 | if (typeof layout !== 'function') { 21 | console.error('Unable to find a layout named ' + config.layout.type); 22 | } 23 | return createAppender(layout, config, options, done); 24 | } 25 | 26 | function loadAppenderConfig(config) { 27 | if (!config) { 28 | config = {}; 29 | } 30 | if (typeof config.typeName !== 'function') { 31 | var value = config.typeName || 'nodejs'; 32 | config.typeName = function(loggingEvent) { 33 | return value; 34 | }; 35 | } 36 | if (!config.layout) { 37 | config.layout = { type: 'logstash' }; 38 | } 39 | 40 | //we need to pass the typeName to the layout config. 41 | //it is used both by the logstash layout and by the ES client. 42 | config.layout.typeName = config.typeName; 43 | 44 | return config; 45 | } 46 | 47 | function initESClient(config, options, template, done) { 48 | var esOptions; 49 | if (config.url) { 50 | esOptions = ElasticsearchClient.makeOptions(config.url); 51 | } else if (config.esOptions) { 52 | esOptions = config.esOptions; 53 | } else { 54 | esOptions = ElasticsearchClient.makeOptions('http://localhost:9200'); 55 | } 56 | var esclient = config.esclient || new ElasticsearchClient(esOptions); 57 | if (esclient.init) { 58 | esclient.init(); 59 | } 60 | if (template) { 61 | var templateName = template.template; 62 | esclient.getTemplate(templateName, function(err, res) { 63 | if (res === '{}' || config.forceDefineTemplate) { 64 | esclient.defineTemplate(templateName, template, function() { 65 | //let it be or plug an event emitter in there 66 | if (typeof done === 'function') { 67 | done(); 68 | } 69 | }); 70 | } else if (typeof done === 'function') { 71 | done(); 72 | } 73 | }); 74 | } else if (typeof done === 'function') { 75 | done(); 76 | } 77 | return esclient; 78 | } 79 | 80 | function makeESHelper(layout, config) { 81 | var logId = config.logId || function() {}; 82 | var typeName; 83 | if (typeof config.typeName === 'function') { 84 | typeName = config.typeName; 85 | } else { 86 | typeName = function(loggingEvent) { 87 | return config.typeName || 'nodejs'; 88 | }; 89 | } 90 | 91 | var indexName; 92 | var templateName; 93 | var template; 94 | if (typeof config.indexName === 'function') { 95 | indexName = config.indexName; 96 | } else if (typeof config.indexName === 'string') { 97 | indexName = function() { 98 | return config.indexName; 99 | }; 100 | } else { 101 | var prefix = config.indexNamePrefix || 'logstash-'; 102 | templateName = prefix + '*'; 103 | indexName = function() { 104 | function pad(n){ 105 | return n<10 ? '0'+n : n; 106 | } 107 | var date = new Date(); 108 | var vDay = pad(date.getUTCDate()); 109 | var vMonth = pad(date.getUTCMonth()+1); 110 | var vYearLong = pad(date.getUTCFullYear()); 111 | //'logstash-%{+YYYY.MM.dd}'; 112 | return prefix + vYearLong + '.' + vMonth + '.' + vDay; 113 | }; 114 | if (config.layout && config.layout.template !== false) { 115 | if (config.layout.type === 'logstash') { 116 | template = layouts.esTemplateMakers.logstash(templateName, config.layout.template); 117 | } else if (config.layout.type === 'simpleJson') { 118 | template = layouts.esTemplateMakers.simpleJson(templateName, config.layout.template); 119 | } 120 | } 121 | } 122 | return { 123 | indexName: indexName, 124 | typeName: typeName, 125 | logId: logId, 126 | template: template 127 | }; 128 | } 129 | 130 | exports.appender = createAppender; 131 | exports.configure = configure; 132 | exports.flushAll = bulk.flushAll; 133 | exports.logstashConnectFormatter = logstashConnectFormatter; 134 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "log4js-elasticsearch", 3 | "version": "0.0.8", 4 | "description": "log4js appender for node that targets elasticsearch.\nCompatible with logstash's elasticsearch_http output; Viewable with Kibana.", 5 | "main": "lib/log4js-elasticsearch.js", 6 | "scripts": { 7 | "test": "make test" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git://github.com/hmalphettes/log4js-elasticsearch.git" 12 | }, 13 | "keywords": [ 14 | "log4js", 15 | "log4js-node", 16 | "elasticsearch", 17 | "kibana" 18 | ], 19 | "author": { 20 | "name": "Hugues Malphettes", 21 | "email": "hmalphettes@gmail.com" 22 | }, 23 | "license": "MIT", 24 | "dependencies": {}, 25 | "devDependencies": { 26 | "log4js": "*", 27 | "chai": "*", 28 | "mocha": "*", 29 | "istanbul": "*", 30 | "mocha-istanbul": "*", 31 | "sandboxed-module": "*", 32 | "jshint": "*" 33 | }, 34 | "optionalDependencies": {} 35 | } 36 | -------------------------------------------------------------------------------- /test/integration/test-real-eslogger.js: -------------------------------------------------------------------------------- 1 | // run this for some real life logging 2 | var expect = require('chai').expect; 3 | var sandbox = require('sandboxed-module'); 4 | var libpath = process.env.COVERAGE ? '../../lib-cov' : '../../lib'; 5 | var log4jsElasticSearch = require(libpath + '/log4js-elasticsearch'); 6 | 7 | // a single shard for testing is enough. 8 | if (!process.env.ES_DEFAULT_SHARDS_NUMBER) { 9 | process.env.ES_DEFAULT_SHARDS_NUMBER = 1; 10 | } 11 | 12 | describe('When configuring a logger posting events to elasticsearch', function() { 13 | var log4js = require('log4js'); 14 | before(function(done) { 15 | var config = { 16 | typeName: 'log4js', 17 | buffersize: 1, 18 | url: process.env.ES_URL, 19 | layout: { 20 | type: 'logstash' 21 | } 22 | }; 23 | log4js.clearAppenders(); 24 | log4js.addAppender(log4jsElasticSearch.configure(config, null, done), 'unittest'); 25 | }); 26 | 27 | describe("When logging", function() { 28 | it('Must send events to elasticsearch', function(done) { 29 | var log = log4js.getLogger('unittest'); 30 | var nolog = log4js.getLogger('notunittest'); 31 | nolog.error('nono'); 32 | log.error('aha'); 33 | log.info('huhu', 'hehe'); 34 | log.warn('ohoho', new Error('pants on fire')); 35 | log.error('ohoho %s', 'a param', new Error('pants on fire')); 36 | setTimeout(done, 700); 37 | }); 38 | }); 39 | }); 40 | 41 | describe('When configuring an es logger', function() { 42 | var log4js = sandbox.require('log4js', { 43 | requires: { 44 | 'log4js-elasticsearch': log4jsElasticSearch 45 | } 46 | }); 47 | before(function(done) { 48 | var config = { 49 | "appenders": [ 50 | { 51 | "type": "log4js-elasticsearch", 52 | "url": process.env.ES_URL || 'http://127.0.0.1:9200', 53 | "forceDefineTemplate": true, 54 | "layout": { 55 | "type": "logstash", 56 | "tags": [ "goodie" ], 57 | "sourceHost": "aspecialhost" 58 | } 59 | } 60 | ] 61 | }; 62 | log4js.clearAppenders(); 63 | log4js.configure(config); 64 | setTimeout(done, 1000); 65 | }); 66 | it('Must log where expected', function(done) { 67 | log4js.getLogger('tests').warn('and one for ES and the console'); 68 | log4js.getLogger('tests').debug('and one for the console alone'); 69 | log4jsElasticSearch.flushAll(true); 70 | setTimeout(done, 700); 71 | }); 72 | }); 73 | 74 | describe.skip('When configuring a filtered es logger', function() { 75 | // if someone knows how to setup the sandbox to make it load right 76 | // it would be nicer! 77 | var log4js = sandbox.require('log4js', { 78 | requires: { 79 | 'log4js-elasticsearch': log4jsElasticSearch 80 | } 81 | }); 82 | before(function(done) { 83 | var config = { 84 | "appenders": [ 85 | { 86 | "category": "tests2", 87 | "type": "logLevelFilter", 88 | "level": "WARN", 89 | "appender": { 90 | "type": "log4js-elasticsearch", 91 | "url": process.env.ES_URL, 92 | "layout": { 93 | "type": "logstash" 94 | } 95 | } 96 | }, 97 | { 98 | "category": "tests2", 99 | "type": "console", 100 | "layout": { 101 | "type": "messagePassThrough" 102 | } 103 | } 104 | ], 105 | "levels": { 106 | "tests": "DEBUG" 107 | } 108 | }; 109 | log4js.clearAppenders(); 110 | log4js.configure(config); 111 | setTimeout(done, 1000); 112 | }); 113 | it('Must log where expected', function(done) { 114 | log4js.getLogger('tests2').warn('and one for ES and the console'); 115 | log4js.getLogger('tests2').debug('and one for the console alone'); 116 | log4jsElasticSearch.flushAll(true); 117 | setTimeout(done, 700); 118 | }); 119 | }); 120 | -------------------------------------------------------------------------------- /test/test-connect-logger.js: -------------------------------------------------------------------------------- 1 | var sandbox = require('sandboxed-module'); 2 | var libpath = process.env.COVERAGE ? '../lib-cov' : '../lib'; 3 | var log4jsElasticSearch = require(libpath + '/log4js-elasticsearch'); 4 | var logstashConnectFormatter = log4jsElasticSearch.logstashConnectFormatter; 5 | var expect = require('chai').expect; 6 | var levels = require('log4js').levels; 7 | 8 | function MockLogger() { 9 | 10 | var that = this; 11 | this.messages = []; 12 | 13 | this.log = function(level, message, exception) { 14 | that.messages.push({ level: level, message: message }); 15 | }; 16 | 17 | this.isLevelEnabled = function(level) { 18 | return level.isGreaterThanOrEqualTo(that.level); 19 | }; 20 | 21 | this.level = levels.TRACE; 22 | } 23 | 24 | function MockRequest(remoteAddr, method, originalUrl) { 25 | 26 | this.socket = { remoteAddress: remoteAddr }; 27 | this.originalUrl = originalUrl; 28 | this.method = method; 29 | this.httpVersionMajor = '5'; 30 | this.httpVersionMinor = '0'; 31 | this.headers = {}; 32 | 33 | } 34 | 35 | function MockResponse(statusCode) { 36 | 37 | this.statusCode = statusCode; 38 | this.end = function(chunk, encoding) {}; 39 | 40 | } 41 | 42 | describe.skip('When using a connect logger', function() { 43 | var log4js = require('log4js'); 44 | var ml, cl; 45 | before(function() { 46 | ml = new MockLogger(); 47 | cl = log4js.connectLogger(ml, { format: logstashConnectFormatter }); 48 | expect(cl).to.be.a['function']; 49 | }); 50 | it('Must format a logging event for logstash from the execution of a connect request', function(done) { 51 | var req = new MockRequest('my.remote.addr', 'GET', 'http://url'); 52 | var res = new MockResponse(200); 53 | cl(req, res, function() { 54 | res.end('chunk', 'encoding'); 55 | done(); 56 | }); 57 | }); 58 | }); 59 | describe('When using a connect logger on the es appender', function() { 60 | var log4js = sandbox.require('log4js', { 61 | requires: { 'log4js-elasticsearch': log4jsElasticSearch } 62 | }); 63 | var ml, cl; 64 | var mockElasticsearchClient = { 65 | index: function(indexName, typeName, logObj, id, cb) { 66 | // console.log('logObj', logObj); 67 | expect(logObj['@message']).to.equal('my.remote.addr - - "GET http://url HTTP/5.0" 200 - "" ""'); 68 | var fields = logObj['@fields']; 69 | expect(fields.url).to.equal('http://url'); 70 | expect(fields.method).to.equal('GET'); 71 | expect(fields['response-time']).to.be.a.number; 72 | expect(fields['remote-addr']).to.equal('my.remote.addr'); 73 | expect(fields['content-length']).to.be.a.number; 74 | expect(fields['http-version']).to.equal('5.0'); 75 | cb(); 76 | }, defineTemplate: function(templateName, template, cb) { 77 | cb(null, 'ok'); 78 | }, getTemplate: function(templateName, cb) { 79 | cb(null, '{}'); 80 | } 81 | }; 82 | before(function() { 83 | log4js.configure({ 84 | "appenders": [ 85 | { 86 | "type": "log4js-elasticsearch", 87 | "esclient": mockElasticsearchClient, 88 | "buffersize": 1, 89 | "layout": { type: 'logstash' } 90 | } 91 | ] 92 | }); 93 | ml = log4js.getLogger('unittest'); 94 | cl = log4js.connectLogger(ml, { format: logstashConnectFormatter }); 95 | expect(cl).to.be.a['function']; 96 | }); 97 | it('Must format a logging event for logstash from the execution of a connect request', function(done) { 98 | var req = new MockRequest('my.remote.addr', 'GET', 'http://url'); 99 | var res = new MockResponse(200); 100 | cl(req, res, function() { 101 | res.end('chunk', 'encoding'); 102 | done(); 103 | }); 104 | }); 105 | }); 106 | 107 | 108 | 109 | 110 | -------------------------------------------------------------------------------- /test/test-es-template.js: -------------------------------------------------------------------------------- 1 | var expect = require('chai').expect; 2 | var sandbox = require('sandboxed-module'); 3 | var libpath = process.env.COVERAGE ? '../lib-cov' : '../lib'; 4 | var lsLayouts = require(libpath + '/log4js-elasticsearch-layouts').esTemplateMakers.logstash; 5 | 6 | describe('When passing options to the es-template', function() { 7 | it('Must return the template when no options are passed', function() { 8 | var r = lsLayouts('test'); 9 | expect(r.template).to.equal('test'); 10 | }); 11 | it('Must keep the default value when no options are passed', function() { 12 | var r = lsLayouts('test'); 13 | expect(r.settings['index.query.default_field']).to.equal('@message'); 14 | }); 15 | it('Must override the number of shards via the options', function() { 16 | var r = lsLayouts('test', {settings: { number_of_shards: 1 }}); 17 | expect(r.settings.number_of_shards).to.equal(1); 18 | }); 19 | it('Must override the total_shards_per_node via the options', function() { 20 | var r = lsLayouts('test', {settings: { 'index.routing.allocation.total_shards_per_node': 10 }}); 21 | expect(r.settings['index.routing.allocation.total_shards_per_node']).to.equal(10); 22 | }); 23 | it('Must delete the index.cache.field.type via the options', function() { 24 | var r = lsLayouts('test', {settings: { 'index.cache.field.type': '__delete__' }}); 25 | expect(r.settings['index.cache.field.type']).to.not.exist; 26 | }); 27 | it('Must enable the _all field via the options', function() { 28 | var r = lsLayouts('test', {mappings: { _default_: { _all: {enabled: true} } }}); 29 | expect(r.mappings._default_._all.enabled).to.equal(true); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /test/test-eslogger.js: -------------------------------------------------------------------------------- 1 | var expect = require('chai').expect; 2 | var sandbox = require('sandboxed-module'); 3 | var libpath = process.env.COVERAGE ? '../lib-cov' : '../lib'; 4 | var log4jsElasticSearch = require(libpath + '/log4js-elasticsearch'); 5 | 6 | describe('When configuring a logger posting events to elasticsearch', function() { 7 | var log4js = require('log4js'); 8 | var mockElasticsearchClient = { 9 | index: function(indexName, typeName, logObj, newLogId, cb) { 10 | expect(indexName).to.match(/^logstash-/); 11 | expect(typeName).to.equal('nodejs'); 12 | expect(newLogId).to.not.exist; 13 | expect(logObj['@fields'].category).to.equal('unittest'); 14 | expect(logObj['@source']).to.equal('log4js'); 15 | expect(logObj['@source_host']).to.equal(require('os').hostname()); 16 | expect(logObj['@source_path']).to.equal(process.cwd()); 17 | expect(logObj['@tags'].length).to.equal(0); 18 | if (currentMsg) { 19 | expect(logObj['@message']).to.equal(currentMsg); 20 | currentMsg = null; 21 | } else { 22 | expect(logObj['@message']).to.exist; 23 | } 24 | if (currentErrorMsg) { 25 | expect(currentErrorMsg).to.equal(logObj['@fields'].error); 26 | expect(logObj['@fields'].stack).to.be['instanceof'](Array); 27 | currentErrorMsg = null; 28 | } 29 | if (currentLevelStr) { 30 | expect(logObj['@fields'].levelStr).to.equal(currentLevelStr); 31 | currentLevelStr = null; 32 | } 33 | if (currentCallback) { 34 | var callit = currentCallback; 35 | currentCallback = null; 36 | callit(); 37 | } 38 | cb(); 39 | }, defineTemplate: function(templateName, template, done) { 40 | expect(templateName).to.equal('logstash-*'); 41 | defineTemplateWasCalled = true; 42 | done(); 43 | }, getTemplate: function(templateName, cb) { 44 | cb(null, '{}'); 45 | } 46 | }; 47 | var currentMsg; 48 | var currentCallback; 49 | var currentErrorMsg; 50 | var currentLevelStr; 51 | var defineTemplateWasCalled = false; 52 | before(function(done) { 53 | var config = { esclient: mockElasticsearchClient, buffersize: 1 }; 54 | log4js.clearAppenders(); 55 | log4js.addAppender(log4jsElasticSearch.configure(config, null, done), 'unittest'); 56 | }); 57 | it("Must have created the template", function() { 58 | expect(defineTemplateWasCalled).to.equal(true); 59 | }); 60 | 61 | describe("When logging", function() { 62 | it('Must send events to elasticsearch', function(done) { 63 | var log = log4js.getLogger('unittest'); 64 | var nolog = log4js.getLogger('notunittest'); 65 | currentErrorMsg = 'I should not be called at all'; 66 | nolog.error('nono'); 67 | currentErrorMsg = null; 68 | 69 | currentLevelStr = 'ERROR'; 70 | currentMsg = 'aha'; 71 | log.error('aha'); 72 | expect(currentMsg).to.be['null']; 73 | 74 | currentLevelStr = 'INFO'; 75 | currentMsg = 'huhu \'hehe\''; 76 | log.info('huhu', 'hehe'); 77 | expect(currentMsg).to.be['null']; 78 | 79 | currentLevelStr = 'WARN'; 80 | currentMsg = 'ohoho'; 81 | currentErrorMsg = 'pants on fire'; 82 | log.warn('ohoho', new Error('pants on fire')); 83 | 84 | currentCallback = done; 85 | currentMsg = 'ohoho a param'; 86 | currentErrorMsg = 'pants on fire'; 87 | log.error('ohoho %s', 'a param', new Error('pants on fire')); 88 | }); 89 | }); 90 | }); 91 | 92 | describe('When configuring an elasticsearch appender', function() { 93 | var log4js = sandbox.require('log4js', { 94 | requires: { 95 | 'log4js-elasticsearch': log4jsElasticSearch 96 | } 97 | }); 98 | 99 | var currentMsg; 100 | var defineTemplateWasCalled = false; 101 | var mockElasticsearchClient = { 102 | index: function(indexName, typeName, logObj, id, cb) { 103 | expect(logObj['@message']).to.equal(currentMsg); 104 | currentMsg = null; 105 | cb(); 106 | }, defineTemplate: function(templateName, template, cb) { 107 | defineTemplateWasCalled = true; 108 | cb(null, 'ok'); 109 | }, getTemplate: function(templateName, cb) { 110 | cb(null, '{}'); 111 | } 112 | }; 113 | before(function() { 114 | log4js.configure({ 115 | "appenders": [ 116 | { 117 | "type": "log4js-elasticsearch", 118 | "esclient": mockElasticsearchClient, 119 | "buffersize": 1, 120 | "layout": { type: 'logstash' } 121 | } 122 | ] 123 | }); 124 | expect(defineTemplateWasCalled).to.be['true']; 125 | }); 126 | it('Must have configured the appender', function() { 127 | currentMsg = 'hello'; 128 | log4js.getLogger('unittest').info('hello'); 129 | expect(currentMsg).to.be['null']; 130 | }); 131 | }); 132 | 133 | describe('When configuring an elasticsearch logstash appender layout', function() { 134 | var log4js = sandbox.require('log4js', { 135 | requires: { 136 | 'log4js-elasticsearch': log4jsElasticSearch 137 | } 138 | }); 139 | 140 | var currentMsg; 141 | var defineTemplateWasCalled = false; 142 | var expectedNumberOfShardsInTemplate = -1; 143 | var mockElasticsearchClient = { 144 | index: function(indexName, typeName, logObj, id, cb) { 145 | expect(logObj['@message']).to.equal(currentMsg); 146 | expect(logObj['@tags'][0]).to.equal('goodie'); 147 | expect(logObj['@source_host']).to.equal('aspecialhost'); 148 | expect(typeName).to.equal('customType'); 149 | currentMsg = null; 150 | cb(); 151 | }, defineTemplate: function(templateName, template, cb) { 152 | defineTemplateWasCalled = true; 153 | if (expectedNumberOfShardsInTemplate !== -1) { 154 | expect(template.settings.number_of_shards).to.equal(expectedNumberOfShardsInTemplate); 155 | expectedNumberOfShardsInTemplate = -1; 156 | } 157 | cb(null, 'something'); 158 | }, getTemplate: function(templateName, cb) { 159 | cb(null, '{}'); 160 | } 161 | }; 162 | it('Must have configured the appender with static params', function() { 163 | expectedNumberOfShardsInTemplate = 1; 164 | log4js.configure({ 165 | "appenders": [ 166 | { 167 | "type": "log4js-elasticsearch", 168 | "esclient": mockElasticsearchClient, 169 | "typeName": "customType", 170 | "buffersize": 1, 171 | "layout": { 172 | "type": "logstash", 173 | "tags": [ "goodie" ], 174 | "sourceHost": "aspecialhost", 175 | "template": {"settings": { "number_of_shards": 1 }} 176 | } 177 | } 178 | ] 179 | }); 180 | expect(defineTemplateWasCalled).to.be['true']; 181 | defineTemplateWasCalled = undefined; 182 | 183 | currentMsg = 'hello'; 184 | log4js.getLogger('unittest').info('hello'); 185 | expect(currentMsg).to.be['null']; 186 | }); 187 | 188 | it('Must have configured the appender with dynamic params', function() { 189 | expectedNumberOfShardsInTemplate = 4; 190 | log4js.configure({ 191 | "appenders": [ 192 | { 193 | "type": "log4js-elasticsearch", 194 | "esclient": mockElasticsearchClient, 195 | "buffersize": 1, 196 | "typeName": function(loggingEvent) { 197 | return 'customType'; 198 | }, 199 | "layout": { 200 | "type": "logstash", 201 | "tags": function(loggingEvent) { 202 | return [ 'goodie' ]; 203 | }, 204 | "sourceHost": function(loggingEvent) { 205 | return "aspecialhost"; 206 | } 207 | } 208 | } 209 | ] 210 | }); 211 | expect(defineTemplateWasCalled).to.be['true']; 212 | defineTemplateWasCalled = undefined; 213 | 214 | currentMsg = 'hello'; 215 | log4js.getLogger('unittest').info('hello'); 216 | expect(currentMsg).to.be['null']; 217 | }); 218 | }); 219 | 220 | describe('When sending the logs in bulk', function() { 221 | var log4js = sandbox.require('log4js', { 222 | requires: { 223 | 'log4js-elasticsearch': log4jsElasticSearch 224 | } 225 | }); 226 | 227 | var currentMsg; 228 | var bulkWasCalled = 0; 229 | var expectedBulkcmdsSize; 230 | var mockElasticsearchClient = { 231 | bulk: function(bulkCmds, cb) { 232 | if (expectedBulkcmdsSize) { 233 | expect(bulkCmds.length).to.equal(expectedBulkcmdsSize); 234 | expectedBulkcmdsSize = null; 235 | } 236 | bulkWasCalled++; 237 | cb(); 238 | }, getTemplate: function(templateName, cb) { 239 | cb(null, 'notempty'); 240 | } 241 | }; 242 | function reset(done) { 243 | log4jsElasticSearch.flushAll(true); 244 | bulkWasCalled = 0; 245 | done(); 246 | } 247 | afterEach(reset); 248 | beforeEach(reset); 249 | it('Must send logs in bulks when the bufferSize is reached', function(done) { 250 | log4js.configure({ 251 | "appenders": [ 252 | { 253 | "type": "log4js-elasticsearch", 254 | "esclient": mockElasticsearchClient, 255 | "buffersize": 2 256 | } 257 | ] 258 | }); 259 | expectedBulkcmdsSize = 4; 260 | log4js.getLogger('unittest').info('hello'); 261 | expect(bulkWasCalled).to.equal(0); 262 | log4js.getLogger('unittest').info('goodbye'); 263 | expect(bulkWasCalled).to.equal(1); 264 | done(); 265 | }); 266 | it('Must send logs in bulks after a timeout', function(done) { 267 | log4js.configure({ 268 | "appenders": [ 269 | { 270 | "type": "log4js-elasticsearch", 271 | "esclient": mockElasticsearchClient, 272 | "buffersize": 2, 273 | "timeout": 20 274 | } 275 | ] 276 | }); 277 | expectedBulkcmdsSize = 2; 278 | log4js.getLogger('unittest').info('hello'); 279 | expect(bulkWasCalled).to.equal(0); 280 | setTimeout(function() { 281 | expect(bulkWasCalled).to.equal(1); 282 | done(); 283 | }, 80); 284 | }); 285 | }); 286 | 287 | --------------------------------------------------------------------------------