├── .npmignore ├── .gitignore ├── .editorconfig ├── test ├── request_logentry_1.json └── test.js ├── .eslintrc ├── .travis.yml ├── transformer.js ├── index-template-mapping.json ├── LICENSE ├── package.json ├── index.d.ts ├── index.js ├── bulk_writer.js ├── CHANGELOG.md └── README.md /.npmignore: -------------------------------------------------------------------------------- 1 | .* 2 | coverage 3 | test 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | npm-debug.log 3 | 4 | coverage/ 5 | 6 | .vscode 7 | .project 8 | .settings 9 | 10 | *.log 11 | *~ 12 | .idea/ 13 | .nyc_output 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | indent_size = 2 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | 11 | [*.md] 12 | trim_trailing_whitespace = false 13 | -------------------------------------------------------------------------------- /test/request_logentry_1.json: -------------------------------------------------------------------------------- 1 | { 2 | "message": "logmessage", 3 | "level": "info", 4 | "meta": { 5 | "method": "GET", 6 | "url": "/sitemap.xml", 7 | "headers": { 8 | "host": "www.example.com", 9 | "user-agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 10 | "accept": "*/*", 11 | "accept-encoding": "gzip,deflate", 12 | "from": "googlebot(at)googlebot.com", 13 | "if-modified-since": "Tue, 30 Sep 2015 11:34:56 GMT", 14 | "x-forwarded-for": "66.249.78.19" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "eslint-config-airbnb-base" 4 | ], 5 | "env": { 6 | "node": true, 7 | "mocha": true 8 | }, 9 | "rules": { 10 | "arrow-body-style": 0, 11 | "consistent-this": [ 12 | "error", 13 | "thiz" 14 | ], 15 | "comma-dangle": 0, 16 | "global-require": 0, 17 | "no-unused-vars": [ 18 | "error", 19 | { 20 | "vars": "all", 21 | "args": "none" 22 | } 23 | ], 24 | "prefer-template": 0, 25 | "prefer-const": [ 26 | 2, 27 | { 28 | "destructuring": "any", 29 | "ignoreReadBeforeAssign": true 30 | } 31 | ], 32 | "space-before-function-paren": 0, 33 | "prefer-object-spread": 0, 34 | "strict": 0 35 | }, 36 | "plugins": [ 37 | "json" 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | sudo: required 3 | dist: xenial 4 | node_js: 5 | - "14" 6 | jdk: 7 | - oraclejdk11 8 | env: 9 | global: 10 | - COVERALLS=1 11 | - WAIT_FOR_ES=1 12 | matrix: 13 | - ES_VERSION=7.11.2 14 | install: 15 | - mkdir /tmp/elasticsearch 16 | - wget -O - https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz | tar xz --directory=/tmp/elasticsearch --strip-components=1 17 | - /tmp/elasticsearch/bin/elasticsearch --daemonize -E path.data=/tmp 18 | - until curl --silent -XGET --fail http://localhost:9200; do printf '.'; sleep 1; done 19 | - npm ci 20 | git: 21 | depth: 10 22 | before_install: 23 | - echo $JAVA_HOME 24 | - java -version 25 | before_script: 26 | - sleep 100 27 | after_success: 28 | - "npm run coveralls" 29 | - "cat ./coverage/lcov.info | ./node_modules/coveralls/bin/coveralls.js" 30 | -------------------------------------------------------------------------------- /transformer.js: -------------------------------------------------------------------------------- 1 | /** 2 | Transformer function to transform log data as provided by winston into 3 | a message structure which is more appropriate for indexing in ES. 4 | 5 | @param {Object} logData 6 | @param {Object} logData.message - the log message 7 | @param {Object} logData.level - the log level 8 | @param {Object} logData.meta - the log meta data (JSON object) 9 | @returns {Object} transformed message 10 | */ 11 | const transformer = function transformer(logData) { 12 | const transformed = {}; 13 | transformed['@timestamp'] = logData.timestamp ? logData.timestamp : new Date().toISOString(); 14 | transformed.message = logData.message; 15 | transformed.severity = logData.level; 16 | transformed.fields = logData.meta; 17 | 18 | if (logData.meta['transaction.id']) transformed.transaction = { id: logData.meta['transaction.id'] }; 19 | if (logData.meta['trace.id']) transformed.trace = { id: logData.meta['trace.id'] }; 20 | if (logData.meta['span.id']) transformed.span = { id: logData.meta['span.id'] }; 21 | 22 | return transformed; 23 | }; 24 | 25 | module.exports = transformer; 26 | -------------------------------------------------------------------------------- /index-template-mapping.json: -------------------------------------------------------------------------------- 1 | { 2 | "priority": 200, 3 | "template": { 4 | "settings": { 5 | "index": { 6 | "mapping": { 7 | "total_fields": { 8 | "limit": "3000" 9 | } 10 | }, 11 | "refresh_interval": "5s", 12 | "number_of_shards": "1", 13 | "number_of_replicas": "0" 14 | } 15 | }, 16 | "mappings": { 17 | "_source": { 18 | "enabled": true 19 | }, 20 | "properties": { 21 | "severity": { 22 | "index": true, 23 | "type": "keyword" 24 | }, 25 | "source": { 26 | "index": true, 27 | "type": "keyword" 28 | }, 29 | "@timestamp": { 30 | "type": "date" 31 | }, 32 | "@version": { 33 | "type": "keyword" 34 | }, 35 | "fields": { 36 | "dynamic": true, 37 | "type": "object" 38 | }, 39 | "message": { 40 | "index": true, 41 | "type": "text" 42 | } 43 | } 44 | } 45 | }, 46 | "index_patterns": [ 47 | "logs-app-default*" 48 | ], 49 | "data_stream": {}, 50 | "composed_of": [] 51 | } 52 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 - 2023 Thomas Hoppe. 4 | Copyright (c) 2013 Jacques-Olivier D. Bernier. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in 14 | all copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 22 | THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "winston-elasticsearch", 3 | "version": "0.19.0", 4 | "description": "An Elasticsearch transport for winston", 5 | "main": "index", 6 | "authors": [ 7 | { 8 | "name": "Jacques-Olivier D. Bernier", 9 | "url": "https://github.com/jackdbernier" 10 | }, 11 | { 12 | "name": "Thomas Hoppe", 13 | "url": "https://github.com/vanthome", 14 | "email": "thomas.hoppe@n-fuse.co" 15 | } 16 | ], 17 | "contributors": [ 18 | { 19 | "name": "Andy Potanin", 20 | "url": "https://github.com/andypotanin" 21 | } 22 | ], 23 | "repository": { 24 | "type": "git", 25 | "url": "http://github.com/vanthome/winston-elasticsearch.git" 26 | }, 27 | "license": "MIT", 28 | "keywords": [ 29 | "logging", 30 | "winston", 31 | "elasticsearch", 32 | "transport", 33 | "logstash" 34 | ], 35 | "optionalDependencies": { 36 | "elastic-apm-node": "^3.20.0" 37 | }, 38 | "dependencies": { 39 | "@elastic/elasticsearch": "^8.13.1", 40 | "dayjs": "^1.11.11", 41 | "debug": "^4.3.4", 42 | "lodash.defaults": "^4.2.0", 43 | "lodash.omit": "^4.5.0", 44 | "promise": "^8.3.0", 45 | "retry": "^0.13.1", 46 | "winston": "^3.13.0", 47 | "winston-transport": "^4.7.0" 48 | }, 49 | "devDependencies": { 50 | "babel-eslint": "^10.1.0", 51 | "coveralls": "^3.1.1", 52 | "eslint": "^9.2.0", 53 | "eslint-config-airbnb-base": "^15.0.0", 54 | "eslint-plugin-import": "^2.29.1", 55 | "eslint-plugin-json": "^3.1.0", 56 | "mocha": "^10.4.0", 57 | "nyc": "^15.1.0", 58 | "should": "^13.2.3" 59 | }, 60 | "scripts": { 61 | "test": "nyc mocha", 62 | "lint": "eslint *.json *.js", 63 | "mocha": "mocha --full-trace./test/* -- --trace-warnings", 64 | "coveralls": "nyc report --reporter=text-lcov | coveralls" 65 | }, 66 | "engines": { 67 | "node": ">= 8.0.0" 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /index.d.ts: -------------------------------------------------------------------------------- 1 | import { Client, ClientOptions, estypes } from '@elastic/elasticsearch'; 2 | import TransportStream = require('winston-transport'); 3 | 4 | export interface LogData { 5 | message: any; 6 | level: string; 7 | meta: { [key: string]: any }; 8 | timestamp?: string; 9 | } 10 | 11 | export interface Transformer { 12 | (logData: LogData): any; 13 | } 14 | 15 | export interface ElasticsearchTransportOptions extends TransportStream.TransportStreamOptions { 16 | dataStream?: boolean; 17 | apm?: any; // typeof Agent; 18 | timestamp?: () => string; 19 | level?: string; 20 | index?: string; 21 | indexPrefix?: string | Function; 22 | indexSuffixPattern?: string; 23 | transformer?: Transformer; 24 | useTransformer?: boolean; 25 | indexTemplate?: { [key: string]: any }; 26 | ensureIndexTemplate?: boolean; 27 | flushInterval?: number; 28 | waitForActiveShards?: number | 'all'; 29 | handleExceptions?: boolean; 30 | pipeline?: string; 31 | client?: Client; 32 | clientOpts?: ClientOptions; 33 | buffering?: boolean; 34 | bufferLimit?: number; 35 | healthCheckTimeout?: string; 36 | healthCheckWaitForStatus?: string; 37 | healthCheckWaitForNodes?: string; 38 | source?: string; 39 | retryLimit?: number; 40 | } 41 | 42 | export class ElasticsearchTransport extends TransportStream { 43 | constructor(opts?: ElasticsearchTransportOptions); 44 | flush(): Promise; 45 | 46 | query(options: any, callback?: () => void): Promise>; 47 | query(q: string): Promise>; 48 | getIndexName(opts: ElasticsearchTransportOptions): string; 49 | } 50 | 51 | interface TransformedData { 52 | '@timestamp': string 53 | message: string 54 | severity: string 55 | fields: string 56 | transaction?: { id: string } 57 | trace?: { id: string } 58 | span?: { id: string } 59 | } 60 | 61 | export function ElasticsearchTransformer(logData: LogData): TransformedData; 62 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const winston = require('winston'); 4 | const Transport = require('winston-transport'); 5 | const dayjs = require('dayjs'); 6 | const defaults = require('lodash.defaults'); 7 | const omit = require('lodash.omit'); 8 | const { Client } = require('@elastic/elasticsearch'); 9 | const defaultTransformer = require('./transformer'); 10 | const BulkWriter = require('./bulk_writer'); 11 | const mappingTemplate = require('./index-template-mapping.json'); 12 | 13 | class ElasticsearchTransport extends Transport { 14 | constructor(opts) { 15 | super(opts); 16 | this.name = 'elasticsearch'; 17 | this.handleExceptions = opts.handleExceptions || false; 18 | this.handleRejections = opts.handleRejections || false; 19 | this.exitOnError = false; 20 | this.source = null; 21 | 22 | this.on('pipe', (source) => { 23 | this.source = source; 24 | }); 25 | 26 | this.on('error', (err) => { 27 | this.source.pipe(this); // re-pipes readable 28 | }); 29 | 30 | this.opts = opts || {}; 31 | 32 | // Set defaults 33 | defaults(opts, { 34 | level: 'info', 35 | index: opts.dataStream ? 'logs-app-default' : null, 36 | indexPrefix: 'logs', 37 | indexSuffixPattern: 'YYYY.MM.DD', 38 | transformer: defaultTransformer, 39 | useTransformer: true, 40 | ensureIndexTemplate: true, 41 | flushInterval: 2000, 42 | waitForActiveShards: 1, 43 | handleExceptions: false, 44 | exitOnError: false, 45 | pipeline: null, 46 | bufferLimit: null, 47 | buffering: true, 48 | healthCheckTimeout: '30s', 49 | healthCheckWaitForStatus: 'yellow', 50 | healthCheckWaitForNodes: '>=1', 51 | dataStream: false, 52 | internalLogger: console.error, 53 | }); 54 | 55 | // Use given client or create one 56 | if (opts.client) { 57 | this.client = opts.client; 58 | } else { 59 | defaults(opts, { 60 | clientOpts: { 61 | log: [ 62 | { 63 | type: 'console', 64 | level: 'error', 65 | } 66 | ] 67 | } 68 | }); 69 | 70 | // Create a new ES client 71 | // http://localhost:9200 is the default of the client already 72 | const copts = { ...this.opts.clientOpts }; 73 | this.client = new Client(copts); 74 | } 75 | 76 | const bulkWriterOpts = { 77 | index: opts.index, 78 | interval: opts.flushInterval, 79 | waitForActiveShards: opts.waitForActiveShards, 80 | pipeline: opts.pipeline, 81 | ensureIndexTemplate: opts.ensureIndexTemplate, 82 | indexTemplate: opts.indexTemplate || mappingTemplate, 83 | indexPrefix: opts.indexPrefix, 84 | buffering: opts.buffering, 85 | bufferLimit: opts.buffering ? opts.bufferLimit : 0, 86 | healthCheckTimeout: opts.healthCheckTimeout, 87 | healthCheckWaitForStatus: opts.healthCheckWaitForStatus, 88 | healthCheckWaitForNodes: opts.healthCheckWaitForNodes, 89 | dataStream: opts.dataStream, 90 | retryLimit: opts.retryLimit, 91 | internalLogger: opts.internalLogger, 92 | }; 93 | 94 | this.bulkWriter = new BulkWriter(this, this.client, bulkWriterOpts); 95 | this.bulkWriter.start(); 96 | } 97 | 98 | async flush() { 99 | await this.bulkWriter.flush(); 100 | } 101 | 102 | // end() will be called from here: https://github.com/winstonjs/winston/blob/master/lib/winston/logger.js#L328 103 | end(chunk, encoding, callback) { 104 | this.bulkWriter.schedule = () => { }; 105 | this.bulkWriter.flush().then(() => { 106 | setImmediate(() => { 107 | super.end(chunk, encoding, callback); // this emits finish event from stream 108 | }); 109 | }); 110 | } 111 | 112 | async log(info, callback) { 113 | const { level, message, timestamp } = info; 114 | const meta = Object.assign({}, omit(info, ['level', 'message'])); 115 | setImmediate(() => { 116 | this.emit('logged', level); 117 | }); 118 | 119 | const logData = { 120 | message, 121 | level, 122 | timestamp, 123 | meta, 124 | }; 125 | 126 | const entry = this.opts.useTransformer 127 | ? await this.opts.transformer(logData) 128 | : info; 129 | 130 | let index = this.opts.dataStream 131 | ? this.opts.index 132 | : this.getIndexName(this.opts); 133 | 134 | if (this.opts.source) { 135 | entry.source = this.opts.source; 136 | } 137 | 138 | if (entry.indexInterfix !== undefined) { 139 | index = this.opts.dataStream 140 | ? this.getDataStreamName(this.opts, entry.indexInterfix) 141 | : this.getIndexName(this.opts, entry.indexInterfix); 142 | delete entry.indexInterfix; 143 | } 144 | 145 | if (this.opts.apm) { 146 | const apm = this.opts.apm.currentTraceIds; 147 | if (apm['transaction.id']) entry.transaction = { id: apm['transaction.id'], ...entry.transaction }; 148 | if (apm['trace.id']) entry.trace = { id: apm['trace.id'], ...entry.trace }; 149 | if (apm['span.id']) entry.span = { id: apm['span.id'], ...entry.span }; 150 | } 151 | 152 | this.bulkWriter.append(index, entry); 153 | 154 | callback(); 155 | } 156 | 157 | getIndexName(opts, indexInterfix) { 158 | this.test = 'test'; 159 | let indexName = opts.index; 160 | if (indexName === null) { 161 | // eslint-disable-next-line prefer-destructuring 162 | let indexPrefix = opts.indexPrefix; 163 | if (typeof indexPrefix === 'function') { 164 | // eslint-disable-next-line prefer-destructuring 165 | indexPrefix = opts.indexPrefix(); 166 | } 167 | const now = dayjs(); 168 | const dateString = now.format(opts.indexSuffixPattern); 169 | indexName = indexPrefix 170 | + (indexInterfix !== undefined ? '-' + indexInterfix : '') 171 | + '-' 172 | + dateString; 173 | } 174 | return indexName; 175 | } 176 | } 177 | 178 | winston.transports.Elasticsearch = ElasticsearchTransport; 179 | 180 | module.exports = { 181 | ElasticsearchTransport, 182 | ElasticsearchTransformer: defaultTransformer 183 | }; 184 | -------------------------------------------------------------------------------- /bulk_writer.js: -------------------------------------------------------------------------------- 1 | /* eslint no-underscore-dangle: ['error', { 'allow': ['_index', '_type'] }] */ 2 | 3 | const Promise = require('promise'); 4 | const debug = require('debug')('winston:elasticsearch'); 5 | const retry = require('retry'); 6 | 7 | const BulkWriter = function BulkWriter(transport, client, options) { 8 | this.transport = transport; 9 | this.client = client; 10 | this.options = options; 11 | this.interval = options.interval || 5000; 12 | this.healthCheckTimeout = options.healthCheckTimeout || '30s'; 13 | this.healthCheckWaitForStatus = options.healthCheckWaitForStatus || 'yellow'; 14 | this.healthCheckWaitForNodes = options.healthCheckWaitForNodes || '>=1'; 15 | this.waitForActiveShards = options.waitForActiveShards || '1'; 16 | this.pipeline = options.pipeline; 17 | this.retryLimit = options.retryLimit || 400; 18 | 19 | this.bulk = []; // bulk to be flushed 20 | this.running = false; 21 | this.timer = false; 22 | debug('created', this); 23 | }; 24 | 25 | BulkWriter.prototype.start = function start() { 26 | this.checkEsConnection(this.retryLimit); 27 | debug('started'); 28 | }; 29 | 30 | BulkWriter.prototype.stop = function stop() { 31 | this.running = false; 32 | if (!this.timer) { 33 | return; 34 | } 35 | clearTimeout(this.timer); 36 | this.timer = null; 37 | debug('stopped'); 38 | }; 39 | 40 | BulkWriter.prototype.schedule = function schedule() { 41 | const thiz = this; 42 | this.timer = setTimeout(() => { 43 | thiz.tick(); 44 | }, this.interval); 45 | }; 46 | 47 | BulkWriter.prototype.tick = function tick() { 48 | debug('tick'); 49 | const thiz = this; 50 | if (!this.running) { 51 | return; 52 | } 53 | this.flush() 54 | .then(() => { 55 | // Emulate finally with last .then() 56 | }) 57 | .then(() => { 58 | // finally() 59 | thiz.schedule(); 60 | }); 61 | }; 62 | 63 | BulkWriter.prototype.flush = function flush() { 64 | // write bulk to elasticsearch 65 | if (this.bulk.length === 0) { 66 | debug('nothing to flush'); 67 | return new Promise((resolve) => { 68 | // pause the buffering process when there's no more bulk to flush 69 | // thus allowing the process to be terminated 70 | this.running = false; 71 | return resolve(); 72 | }); 73 | } 74 | const bulk = this.bulk.concat(); 75 | this.bulk = []; 76 | const body = []; 77 | // eslint-disable-next-line object-curly-newline 78 | bulk.forEach(({ index, doc, attempts }) => { 79 | body.push( 80 | { 81 | [this.options.dataStream ? 'create' : 'index']: { 82 | _index: index, 83 | pipeline: this.pipeline 84 | }, 85 | attempts 86 | }, 87 | doc 88 | ); 89 | }); 90 | debug('bulk writer is going to write', body); 91 | return this.write(body); 92 | }; 93 | 94 | BulkWriter.prototype.append = function append(index, doc) { 95 | if (this.options.buffering === true) { 96 | if ( 97 | typeof this.options.bufferLimit === 'number' 98 | && this.bulk.length >= this.options.bufferLimit 99 | ) { 100 | debug('message discarded because buffer limit exceeded'); 101 | // @todo: i guess we can use callback to notify caller 102 | return; 103 | } 104 | this.bulk.unshift({ 105 | index, 106 | doc, 107 | attempts: 0 108 | }); 109 | // resume the buffering process 110 | if (!this.running) { 111 | this.running = true; 112 | this.tick(); 113 | } 114 | } else { 115 | this.write([ 116 | { [this.options.dataStream ? 'create' : 'index']: { _index: index, pipeline: this.pipeline } }, 117 | doc 118 | ]); 119 | } 120 | }; 121 | 122 | BulkWriter.prototype.write = function write(body) { 123 | const thiz = this; 124 | const operation = [thiz.options.dataStream ? 'create' : 'index']; 125 | debug('writing to ES'); 126 | return this.client 127 | .bulk({ 128 | body, 129 | wait_for_active_shards: this.waitForActiveShards, 130 | timeout: this.interval + 'ms', 131 | }) 132 | .then((res) => { 133 | if (res && res.errors && res.items) { 134 | const err = new Error('Elasticsearch error'); 135 | res.items.forEach((item, itemIndex) => { 136 | const bodyData = body[itemIndex * 2 + 1]; 137 | const opKey = Object.keys(item)[0]; 138 | if (item[opKey] && item[opKey].error) { 139 | debug('elasticsearch indexing error', item[opKey].error); 140 | thiz.options.internalLogger('elasticsearch indexing error', item[opKey].error, bodyData); 141 | err.indexError = item[opKey].error; 142 | err.causedBy = bodyData; 143 | } 144 | }); 145 | throw err; 146 | } 147 | }) 148 | .catch((e) => { 149 | // rollback this.bulk array 150 | const newBody = []; 151 | body.forEach((chunk, index, chunks) => { 152 | const { attempts, created } = chunk; 153 | if (!created && attempts < thiz.retryLimit) { 154 | newBody.push({ 155 | index: chunk[operation]._index, 156 | doc: chunks[index + 1], 157 | attempts: attempts + 1, 158 | }); 159 | } else { 160 | debug('retry attempts exceeded'); 161 | } 162 | }); 163 | 164 | const lenSum = thiz.bulk.length + newBody.length; 165 | if (thiz.options.bufferLimit && lenSum >= thiz.options.bufferLimit) { 166 | thiz.bulk = newBody.concat( 167 | thiz.bulk.slice(0, thiz.options.bufferLimit - newBody.length) 168 | ); 169 | } else { 170 | thiz.bulk = newBody.concat(thiz.bulk); 171 | } 172 | debug('error occurred during writing', e); 173 | this.stop(); 174 | this.checkEsConnection(thiz.retryLimit) 175 | .catch((err) => thiz.transport.emit('error', err)); 176 | thiz.transport.emit('warning', e); 177 | 178 | thiz.bulk.forEach((bulk) => { 179 | if (bulk.attempts === thiz.retryLimit) { 180 | this.transport.emit('error', e); 181 | } 182 | }); 183 | }); 184 | }; 185 | 186 | BulkWriter.prototype.checkEsConnection = function checkEsConnection(retryLimit) { 187 | const thiz = this; 188 | thiz.esConnection = false; 189 | 190 | const operation = retry.operation({ 191 | forever: false, 192 | retries: retryLimit, 193 | factor: 1, 194 | minTimeout: 1000, 195 | maxTimeout: 10 * 1000, 196 | randomize: false 197 | }); 198 | return new Promise((fulfill, reject) => { 199 | operation.attempt((currentAttempt) => { 200 | debug('checking for ES connection'); 201 | thiz.client.cluster.health({ 202 | timeout: thiz.healthCheckTimeout, 203 | wait_for_nodes: thiz.healthCheckWaitForNodes, 204 | wait_for_status: thiz.healthCheckWaitForStatus 205 | }) 206 | .then( 207 | (res) => { 208 | thiz.esConnection = true; 209 | const start = () => { 210 | if (thiz.options.buffering === true) { 211 | debug('starting bulk writer'); 212 | thiz.running = true; 213 | thiz.tick(); 214 | } 215 | }; 216 | // Ensure mapping template is existing if desired 217 | if (thiz.options.ensureIndexTemplate) { 218 | thiz.ensureIndexTemplate((res1) => { 219 | fulfill(res1); 220 | start(); 221 | }, reject); 222 | } else { 223 | fulfill(true); 224 | start(); 225 | } 226 | }, 227 | (err) => { 228 | debug('re-checking for connection to ES'); 229 | if (operation.retry(err)) { 230 | return; 231 | } 232 | thiz.esConnection = false; 233 | debug('cannot connect to ES'); 234 | reject(new Error('Cannot connect to ES')); 235 | } 236 | ); 237 | }); 238 | }); 239 | }; 240 | 241 | BulkWriter.prototype.ensureIndexTemplate = function ensureIndexTemplate( 242 | fulfill, 243 | reject 244 | ) { 245 | const thiz = this; 246 | 247 | const indexPrefix = typeof thiz.options.indexPrefix === 'function' 248 | ? thiz.options.indexPrefix() 249 | : thiz.options.indexPrefix; 250 | 251 | const { indexTemplate } = thiz.options; 252 | 253 | let templateName = indexPrefix; 254 | if (thiz.options.dataStream) { 255 | if (!thiz.options.index) { 256 | // hm, has this to be a console error or better a throw? is it needed at all? 257 | thiz.options.internalLogger('Error while deriving templateName with options', thiz.options); 258 | } else { 259 | templateName = thiz.options.index; 260 | } 261 | } 262 | 263 | const tmplCheckMessage = { 264 | name: 'template_' + templateName 265 | }; 266 | debug('Checking tpl name', tmplCheckMessage); 267 | thiz.client.indices.existsIndexTemplate(tmplCheckMessage).then( 268 | (res) => { 269 | if (res.statusCode && res.statusCode === 404) { 270 | const tmplMessage = { 271 | name: 'template_' + templateName, 272 | create: true, 273 | body: indexTemplate 274 | }; 275 | thiz.client.indices.putIndexTemplate(tmplMessage).then( 276 | (res1) => { 277 | debug('Index template created successfully'); 278 | fulfill(res1.body); 279 | }, 280 | (err1) => { 281 | debug('Failed to create index template'); 282 | thiz.transport.emit('warning', err1); 283 | reject(err1); 284 | } 285 | ); 286 | } else { 287 | fulfill(res.body); 288 | } 289 | }, 290 | (res) => { 291 | debug('Failed to check for index template'); 292 | thiz.transport.emit('warning', res); 293 | reject(res); 294 | } 295 | ); 296 | }; 297 | 298 | module.exports = BulkWriter; 299 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const should = require('should'); 3 | const winston = require('winston'); 4 | const http = require('http'); 5 | const { Client } = require('@elastic/elasticsearch'); 6 | 7 | require('../index'); 8 | const defaultTransformer = require('../transformer'); 9 | 10 | const logMessage = JSON.parse( 11 | fs.readFileSync('./test/request_logentry_1.json', 'utf8') 12 | ); 13 | 14 | /* 15 | * Note: To run the tests, a running elasticsearch instance is required. 16 | */ 17 | 18 | // A null logger to prevent ES client spamming the console for deliberately failed tests 19 | function NullLogger(config) { 20 | this.error = (msg) => {}; 21 | this.warning = (msg) => {}; 22 | this.info = (msg) => {}; 23 | this.debug = (msg) => {}; 24 | this.trace = (msg) => {}; 25 | this.close = (msg) => {}; 26 | } 27 | 28 | const clientOpts = { 29 | log: NullLogger, 30 | node: 'http://localhost:9200' 31 | }; 32 | 33 | function createLogger(buffering) { 34 | const logger = winston.createLogger({ 35 | transports: [ 36 | new winston.transports.Elasticsearch({ 37 | flushInterval: 1, 38 | buffering, 39 | // index: 'logs-myapp-mything', 40 | dataStream: false, 41 | clientOpts, 42 | source: 'test-source', 43 | }) 44 | ] 45 | }); 46 | // logger.on('error', (error) => { 47 | // console.error('Error caught', error); 48 | // process.exit(1); 49 | // }); 50 | return logger; 51 | } 52 | 53 | before(() => { 54 | return new Promise((resolve) => { 55 | // get ES version being used 56 | http.get(clientOpts.node, (res) => { 57 | res.setEncoding('utf8'); 58 | let body = ''; 59 | res.on('data', (data) => { 60 | body += data; 61 | }); 62 | res.on('error', () => { resolve(); }); 63 | res.on('end', () => { 64 | body = JSON.parse(body); 65 | resolve(); 66 | }); 67 | }); 68 | }); 69 | }); 70 | 71 | describe('the default transformer', () => { 72 | it('should transform log data from winston into a logstash like structure', (done) => { 73 | const transformed = defaultTransformer({ 74 | message: 'some message', 75 | level: 'error', 76 | meta: { 77 | someField: true 78 | } 79 | }); 80 | should.exist(transformed['@timestamp']); 81 | transformed.severity.should.equal('error'); 82 | transformed.fields.someField.should.be.true(); 83 | done(); 84 | }); 85 | }); 86 | 87 | describe('a buffering logger', () => { 88 | it('can be instantiated', function(done) { 89 | this.timeout(8000); 90 | try { 91 | const logger = createLogger(true); 92 | logger.end(); 93 | } catch (err) { 94 | should.not.exist(err); 95 | } 96 | 97 | // Wait for index template to settle 98 | setTimeout(() => { 99 | done(); 100 | }, 4000); 101 | }); 102 | 103 | it('can end logging without calling `logger.end`', function() { 104 | this.timeout(800000); 105 | createLogger(true); 106 | }); 107 | 108 | it('should log simple message to Elasticsearch', function(done) { 109 | this.timeout(8000); 110 | const logger = createLogger(true); 111 | 112 | logger.log(logMessage.level, `${logMessage.message}1`); 113 | logger.on('finish', () => { 114 | done(); 115 | }); 116 | logger.on('error', (err) => { 117 | should.not.exist(err); 118 | }); 119 | logger.end(); 120 | }); 121 | 122 | it('should log with or without metadata', function(done) { 123 | this.timeout(8000); 124 | const logger = createLogger(true); 125 | 126 | logger.info('test test'); 127 | logger.info('test test', 'hello world'); 128 | logger.info({ message: 'test test', foo: 'bar' }); 129 | logger.log(logMessage.level, `${logMessage.message}2`, logMessage.meta); 130 | logger.on('finish', () => { 131 | done(); 132 | }); 133 | logger.on('error', (err) => { 134 | should.not.exist(err); 135 | }); 136 | logger.end(); 137 | }); 138 | 139 | it('should update buffer properly in case of an error from elasticsearch.', function(done) { 140 | this.timeout(80000); 141 | const logger = createLogger(true); 142 | const transport = logger.transports[0]; 143 | transport.bulkWriter.bulk.should.have.lengthOf(0); 144 | 145 | logger.on('error', (err) => { 146 | should.exist(err); 147 | transport.bulkWriter.bulk.should.have.lengthOf(1); 148 | // manually clear the buffer of stop transport from attempting to flush logs 149 | transport.bulkWriter.bulk = []; 150 | done(); 151 | }); 152 | // mock client.bulk to throw an error 153 | logger.info('test'); 154 | transport.client.bulk = () => { 155 | return Promise.reject(new Error('Test Error')); 156 | }; 157 | logger.info('test'); 158 | logger.end(); 159 | }); 160 | 161 | /* 162 | describe('the logged message', () => { 163 | it('should be found in the index', function(done) { 164 | const elasticsearch = require('@elastic/elasticsearch'); 165 | const client = new elasticsearch.Client({ 166 | host: 'localhost:9200', 167 | log: 'error' 168 | }); 169 | client.search(`message:${logMessage.message}`).then( 170 | (res) => { 171 | res.hits.total.should.be.above(0); 172 | done(); 173 | }, 174 | (err) => { 175 | should.not.exist(err); 176 | } 177 | ).catch((e) => { 178 | // prevent '[DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated' 179 | }); 180 | }); 181 | }); 182 | */ 183 | }); 184 | 185 | describe('a non buffering logger', () => { 186 | it('can be instantiated', function(done) { 187 | this.timeout(8000); 188 | try { 189 | const logger = createLogger(false); 190 | logger.end(); 191 | done(); 192 | } catch (err) { 193 | // console.log('1111111111111'); 194 | should.not.exist(err); 195 | } 196 | }); 197 | 198 | it('should log simple message to Elasticsearch', function(done) { 199 | this.timeout(8000); 200 | const logger = createLogger(false); 201 | 202 | logger.log(logMessage.level, `${logMessage.message}1`); 203 | logger.on('finish', () => { 204 | done(); 205 | }); 206 | logger.on('error', (err) => { 207 | // eslint-disable-next-line no-console 208 | console.error('no', err); 209 | should.not.exist(err); 210 | }); 211 | logger.end(); 212 | }); 213 | }); 214 | 215 | function createLoggerWithDataStream(opts) { 216 | const logger = winston.createLogger({ 217 | transports: [ 218 | new winston.transports.Elasticsearch({ 219 | flushInterval: 1, 220 | buffering: false, 221 | dataStream: true, 222 | clientOpts, 223 | source: 'test-source', 224 | ...opts, 225 | }) 226 | ] 227 | }); 228 | logger.on('error', (error) => { 229 | console.error('Error caught', error.meta.body.error); 230 | process.exit(1); 231 | }); 232 | return logger; 233 | } 234 | 235 | function sleep(ms) { 236 | return new Promise((resolve) => setTimeout(resolve, ms)); 237 | } 238 | 239 | describe('an Elasticsearch datastream', () => { 240 | it('should create a datastream called "logs-app-default" with default settings', function(done) { 241 | this.timeout(16000); 242 | const logger = createLoggerWithDataStream(); 243 | 244 | logger.log(logMessage.level, `${logMessage.message}S1-${new Date()}`); 245 | logger.on('finish', () => { 246 | sleep(5000).then(() => { 247 | new Client(clientOpts).indices.getDataStream({ name: 'logs-app-default' }).then(() => { 248 | done(); 249 | }).catch((e) => { 250 | done(e); 251 | }); 252 | }); 253 | }); 254 | logger.on('error', (err) => { 255 | should.not.exist(err); 256 | }); 257 | logger.end(); 258 | }); 259 | 260 | it('should create a datastream called "logs-myapp-mything" when using customization', function(done) { 261 | this.timeout(16000); 262 | const logger = createLoggerWithDataStream({ 263 | index: 'logs-myapp-mything', 264 | transformer: (event) => ({ 265 | ...defaultTransformer(event) 266 | }) 267 | }); 268 | 269 | logger.log(logMessage.level, `${logMessage.message}S2-${new Date()}`); 270 | logger.on('finish', () => { 271 | sleep(5000).then(() => { 272 | new Client(clientOpts).indices.getDataStream({ name: 'logs-myapp-mything' }).then(() => { 273 | done(); 274 | }).catch((e) => { 275 | done(e); 276 | }); 277 | }); 278 | }); 279 | logger.on('error', (err) => { 280 | should.not.exist(err); 281 | }); 282 | logger.end(); 283 | }); 284 | }); 285 | 286 | // describe('a defective log transport', () => { 287 | // it('emits an error', function(done) { 288 | // this.timeout(500000); 289 | // const transport = new (winston.transports.Elasticsearch)({ 290 | // clientOpts: { 291 | // node: 'http://does-not-exist.test:9200', 292 | // log: NullLogger 293 | // } 294 | // }); 295 | 296 | // transport.on('error', (err) => { 297 | // should.exist(err); 298 | // done(); 299 | // }); 300 | 301 | // const defectiveLogger = winston.createLogger({ 302 | // transports: [ 303 | // transport 304 | // ] 305 | // }); 306 | 307 | // defectiveLogger.info('test'); 308 | // }); 309 | // }); 310 | 311 | // Manual test which allows to test re-connection of the ES client for unavailable ES instance. 312 | // Must be combined with --no-timeouts option for mocha 313 | /* 314 | describe('ES Re-Connection Test', () => { 315 | it('test', function(done) { 316 | this.timeout(400000); 317 | setInterval(() => { 318 | // eslint-disable-next-line no-console 319 | console.log('LOGGING...'); 320 | const logger = createLogger(false); 321 | logger.log(logMessage.level, logMessage.message, logMessage.meta, 322 | (err) => { 323 | should.not.exist(err); 324 | }); 325 | }, 3000); 326 | }); 327 | }); 328 | */ 329 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 0.19.0 / 2024-05-11 2 | =================== 3 | 4 | - Upgrade dependencies 5 | 6 | 0.18.0 / 2024-03-04 7 | =================== 8 | 9 | - Upgrade dependencies 10 | 11 | 0.17.3 / 2023-09-01 12 | =================== 13 | 14 | - Upgrade dependencies 15 | 16 | 0.17.2 / 2023-02-16 17 | =================== 18 | 19 | - Fix typings 20 | 21 | 0.17.1 / 2022-05-15 22 | =================== 23 | 24 | - Fix typings 25 | 26 | 0.17.0 / 2022-05-15 27 | =================== 28 | 29 | - Upgrade dependencies, major new ES client version 30 | 31 | 0.16.1 / 2022-01-24 32 | =================== 33 | 34 | - Upgrade dependencies 35 | - Expose option `internalLogger` for last resort logger 36 | 37 | 0.16.0 / 2021-12-08 38 | =================== 39 | 40 | - Add more debug output 41 | - Upgrade dependencies 42 | - Fix duplicate entries when buffering is used 43 | 44 | 0.15.9 / 2021-09-14 45 | =================== 46 | 47 | - Fix last resort error logging 48 | 49 | 0.15.8 / 2021-07-25 50 | =================== 51 | 52 | - Fix last resort error logging 53 | 54 | 0.15.7 / 2021-06-25 55 | =================== 56 | 57 | - Upgrade optional APM dependency for making it work with node 16. 58 | 59 | 0.15.6 / 2021-06-02 60 | =================== 61 | 62 | - Upgrade ES client. 63 | 64 | 0.15.5 / 2021-05-25 65 | =================== 66 | 67 | - Expose transformer function so that it can be adapted. 68 | 69 | 0.15.4 / 2021-04-09 70 | =================== 71 | 72 | - Upgrade ES client 73 | - Add console.log output in case of bulk writer errors 74 | 75 | 0.15.3 / 2021-03-20 76 | =================== 77 | 78 | - Fix indexPrefix can be of type string | Function 79 | - Fix travis build process (ES JS Client (7.11.2 instead of 11), nodejs 8 deprecated on yargs) 80 | - Fix templateName is undefined or null if options.index is undefined or null 81 | - Correct npm link in README.md (browsenpm.org isn't connected to npm anymore) 82 | - Prevents resending successfully sent items. 83 | - Add `retryLimit` option 84 | - Fix index name not being passed to bulk writer 85 | 86 | 0.15.1 / 2021-03-02 87 | =================== 88 | 89 | - Fix index name not being passed to bulk writer 90 | 91 | 0.15.0 / 2021-02-28 92 | =================== 93 | 94 | - Introduce `source` parameter 95 | - Fix some datastream issues 96 | - Remove _type and don't write it 97 | 98 | 0.14.0 / 2021-02-24 99 | =================== 100 | 101 | - Rename config Property `ensureMappingTemplate` to `ensureIndexTemplate` and `mappingTemplate` accordingly 102 | - Make Index Templates not appear as Legacy ones in Kibana, use `putIndexTemplate` instead of `putTemplate` 103 | 104 | 0.13.0 / 2021-02-12 105 | =================== 106 | 107 | - ES 11 JS Client 108 | - Support for data streams 109 | - Drop support for ES <= 6 110 | 111 | 0.12.3 / 2020-12-04 112 | =================== 113 | 114 | - Increase retries from 5 to 400 (almost 7 minutes) as some massive uServices systems take a while until they spun up 115 | 116 | 0.12.2 / 2020-11-26 117 | =================== 118 | 119 | - Inject indexing error in case writing to ES fails 120 | 121 | 0.12.1 / 2020-11-13 122 | =================== 123 | 124 | - Use latest ES client 125 | - Docs fixes 126 | 127 | 0.12.0 / 2020-11-02 128 | =================== 129 | 130 | - Make retry mechanism give up after 5 retries 131 | - Make emit a warning only when the max retry has been reached 132 | - Make APM param optional 133 | - Move APM dependency into normal dependencies 134 | 135 | 0.11.0 / 2020-10-22 136 | =================== 137 | 138 | - Improve typings 139 | - Added flush() method to ES transport 140 | 141 | 0.10.0 / 2020-08-18 142 | =================== 143 | 144 | - Emit transport internal errors no longer as `error` but as `warning` because even when listened to with `.on('error', ...)` it lead to an `UnhandledPromiseRejectionWarning`. 145 | 146 | 0.9.0 / 2020-05-16 147 | ================== 148 | 149 | - Upgrade ES Client 150 | - New approach to error handling where, in case of an emitted error the source stream is automatically re-attached to the transport stream 151 | - This commit adds a retry counter - after the number of retries has been exceeded then the document is discarded. 152 | - Exposes healthcheck options 153 | - ES version specific mapping templates 154 | 155 | 0.8.8 / 2020-04-07 156 | ================== 157 | 158 | - Fix typings 159 | 160 | 0.8.7 / 2020-04-02 161 | ================== 162 | 163 | - Fix typings 164 | 165 | 0.8.6 / 2020-04-01 166 | ================== 167 | 168 | - Remove default export from typings 169 | 170 | 0.8.5 / 2020-03-09 171 | ================== 172 | 173 | - Upgrade dependencies 174 | - Correct order for entries with same timestamp 175 | - Fix edge case with `this.client.bulk` 176 | 177 | 0.8.4 / 2020-01-20 178 | ================== 179 | 180 | - Upgrade typings 181 | - Make compatible with ES client 7.6 182 | 183 | 0.8.3 / 2019-01-20 184 | ================== 185 | 186 | - Upgrade deps 187 | - Make compatible with ES client 7.5 188 | - Add `-*` to given index pattern prefix for creating index pattern 189 | 190 | 0.8.2 / 2019-11-01 191 | ================== 192 | 193 | - Upgrade deps 194 | - Use `existsTemplate` instead of `getTemplate` 195 | - Make buffering in case of outages more stable 196 | - Don't provide `type` anymore with ES client's bulk operation 197 | - Default to `all` for `waitForActiveShards` 198 | - Emit more `error` events in error cases 199 | - Adapt default index template to conform with newer ES version 200 | 201 | 0.8.1 / 2019-10-01 202 | ================== 203 | 204 | - Update typings 205 | 206 | 0.8.0 / 2019-09-01 207 | ================== 208 | 209 | - Switch to new official ES client 210 | - Emit `error` events when an error happens 211 | 212 | 0.7.13 / 2019-08-28 213 | ================== 214 | 215 | - Upgrade to ES client 216 | - Remove unneeded dependency 217 | - Reduce package size by swapping moment for days.js and other measured 218 | 219 | 0.7.12 / 2019-05-15 220 | ================== 221 | 222 | - Upgrade to ES client v16 223 | - Allow to pass the timestamp through log() so that it's not generated by the transformer 224 | - Fix 2 severe issues described here: https://github.com/vanthome/winston-elasticsearch/pull/87 225 | 226 | 0.7.11 / 2019-04-23 227 | ================== 228 | 229 | - Prevent .git from being published to npm 230 | 231 | 0.7.10 / 2019-04-17 232 | ================== 233 | 234 | - Properly implement non buffering case --> immediately write any message to ES 235 | - Upgrade deps and several other cleanups 236 | - Upgrade of typings 237 | - Correct documentation that we use `_doc` as default type` 238 | 239 | 0.7.9 / 2019-02-12 240 | ================== 241 | 242 | - Upgrade deps, also Winston 3.2.1 243 | - Incorporate fix to prevent flush called twice after each interval instead of once 244 | - Better ignore extraneous resources pushed to npm 245 | 246 | 0.7.8 / 2019-01-27 247 | ================== 248 | 249 | - Upgrade deps, also Winston 3.2.0 250 | - Update the documentation 251 | 252 | 0.7.7 / 2018-12-28 253 | ================== 254 | 255 | - Add feature to limit message buffering 256 | 257 | 0.7.6 / 2018-12-28 258 | ================== 259 | 260 | - Upgrade deps 261 | - Allow the process to exit when the logger is ended. 262 | 263 | 0.7.5 / 2018-09-30 264 | ================== 265 | 266 | - Upgrade to Winston 3.1.0 267 | - Documentation fixes 268 | 269 | 0.7.4 / 2018-07-23 270 | ================== 271 | 272 | - Make sure no messages are lost in case of an ES level fault 273 | - Add a Typescript declaration file 274 | - Fix bug in ES fault handler 275 | 276 | 0.7.3 / 2018-06-30 277 | ================== 278 | 279 | - Possibility to have an `indexInterfix` 280 | 281 | 0.7.2 / 2018-06-30 282 | ================== 283 | 284 | - Remove _all field from default mapping as it is deprecated in ES 285 | - Support `logger.info({ message: 'Test', foo: 'bar' });` signature as well (make sure that field `foo` ends up in `fields` in the index) 286 | 287 | 0.7.1 / 2018-06-18 288 | ================== 289 | 290 | - Correct usage docs in README.md 291 | - Clone opts of ES client https://github.com/elastic/elasticsearch-js/issues/33 292 | 293 | 0.7.0 / 2018-06-17 294 | ================== 295 | 296 | - Upgrade to winston 3.0 and elasticsearch 15.0 297 | - Actually use message type _doc as default 298 | - Mapping template changed --> if you have a custom mapping please check! 299 | 300 | 0.6.0 / 2018-04-28 301 | ================== 302 | 303 | - Remove a lot of unsupported settings in the default mapping. **CHECK YOR MAPPING**, if you are using a custom one 304 | - Remove default mapping due to the fact that they are deprecated in ES v6.0.0 and greater 305 | - See https://www.elastic.co/guide/en/elasticsearch/reference/current/removal-of-types.html 306 | 307 | 0.5.9 / 2018-04-16 308 | ================== 309 | 310 | - Fix bug that lead to mapping template not being used 311 | - Upgrade deps 312 | 313 | 0.5.8 / 2018-03-28 314 | ================== 315 | 316 | Better error handling. 317 | 318 | 0.5.7 / 2018-02-14 319 | ================== 320 | 321 | - In order to prevent `UnhandledPromiseRejectionWarning` and tackle node.js deprecation DEP0018, catching and logging to console is now the default way of handling internal errors 322 | - Enable `sniffOnConnectionFault` on ES client by default 323 | - Change default mapping: `template` --> `index_patterns` 324 | - Migrate default mapping according to https://www.elastic.co/blog/strings-are-dead-long-live-strings 325 | - Moved retry logic into bulkwriter to handle intermittent connectivity interruptions gracefully AND be able to resume operation 326 | - Connection testing is now running infinitely which means the logger never gives up 327 | - Messages logged during a connection outage are buffered 328 | 329 | 0.5.6 / 2017-12-24 330 | ================== 331 | 332 | - Rename debug key from `bulk writer` to `winston:elasticsearch` 333 | - use `finally()` instead of `then()` to schedule bulk writes even in case of exceptions 334 | 335 | 0.5.5 / 2017-12-15 336 | ================== 337 | 338 | - Fix issue with loading built-in mapping 339 | - Upgrade to Elasticsearch client 14 (Elasticsearch 6) 340 | - Ignore 404 errors for silent creation of new indexes 341 | 342 | 0.5.3 / 2017-10-02 343 | ================== 344 | 345 | - Upgrade to Winston 2.4.0 346 | 347 | 0.5.2 / 2017-09-28 348 | ================== 349 | 350 | - Add pipeline option for elasticsearch 351 | 352 | 0.5.1 / 2017-09-24 353 | ================== 354 | 355 | - Upgrade all deps 356 | - Fix linting issues 357 | - Fix loading of template file previously done with require() 358 | 359 | 0.5.0 / 2016-12-01 360 | ================== 361 | 362 | - Release for Elasticsearch 5 363 | - Remove `consistency` option 364 | - Introduce `waitForActiveShards` option 365 | 366 | 0.4.2 / 2016-11-12 367 | ================== 368 | 369 | - Allow `consistency` to be disabled using `false` as value 370 | - Upgrade deps 371 | 372 | 0.4.1 / 2016-10-26 373 | ================== 374 | 375 | - Add timestamp automatically to log messages 376 | 377 | 0.4.0 / 2016-05-31 378 | ================== 379 | 380 | - Minimum node.js version 6 381 | - Version upgrades 382 | - Migrate to eslint from jshint and jscs 383 | 384 | 0.3.1 / 2016-04-22 385 | ================== 386 | 387 | - Fix for dependencies - move winstom from devDependencies to dependencies in package.json 388 | 389 | 0.3.0 / 2016-03-30 390 | ================== 391 | 392 | - Test with ES 2.3.1 393 | - Add time driven bulk write functionality. 394 | - Remove retry functionality for writes as now covered by bulk writer. 395 | - Tests for ES unavailable case. 396 | 397 | 0.2.6 / 2016-01-19 398 | ================== 399 | 400 | - Fix for Windows platform -- make default mapping file readable. 401 | 402 | 0.2.5 / 2015-12-01 403 | ================== 404 | 405 | - ES 2.1 support (driver update). 406 | 407 | 0.2.4 / 2015-11-08 408 | ================== 409 | 410 | - Support ES 2.0, really. 411 | - Support for single index usage, without data suffix via `index` option. 412 | - Fix bug when ES client is provided. 413 | 414 | 0.2.3 / 2015-11-01 415 | ================== 416 | 417 | - Support ES 2.0. 418 | 419 | 0.2.2 / 2015-09-23 420 | ================== 421 | 422 | - Add jshint jscs. 423 | - Minor cleanups. 424 | 425 | 0.2.1 / 2015-09-10 426 | ================== 427 | 428 | - Add transport name. 429 | - Call super constructor for inheritance. 430 | 431 | 0.2.0 / 2015-09-03 432 | ================== 433 | 434 | - 90% rewrite. 435 | - Use current dependencies. 436 | - Removed feature to generate process stats automatically. 437 | - Removed functionality to generate other implicit fields like @tags. 438 | - Add transformer functionality to modify structure of messages before logging. 439 | - Introduce connection state checking and basic retry mechanism for main ES operations. 440 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # winston-elasticsearch 2 | 3 | [![Version npm][version]](https://www.npmjs.com/package/winston-elasticsearch) 4 | [![Build Status][build]](https://travis-ci.org/vanthome/winston-elasticsearch) 5 | [![Dependencies][dependencies]](https://david-dm.org/vanthome/winston-elasticsearch) 6 | [![Coverage Status][cover]](https://coveralls.io/r/vanthome/winston-elasticsearch?branch=master) 7 | 8 | [version]: http://img.shields.io/npm/v/winston-elasticsearch.svg?style=flat-square 9 | [build]: http://img.shields.io/travis/vanthome/winston-elasticsearch/master.svg?style=flat-square 10 | [dependencies]: https://img.shields.io/librariesio/release/npm/winston-elasticsearch.svg?style=flat-square 11 | [cover]: http://img.shields.io/coveralls/vanthome/winston-elasticsearch/master.svg?style=flat-square 12 | 13 | An [elasticsearch](https://www.elastic.co/products/elasticsearch) 14 | transport for the [winston](https://github.com/winstonjs/winston) logging toolkit. 15 | 16 | ## Features 17 | 18 | - [logstash](https://www.elastic.co/products/logstash) compatible message structure. 19 | - Thus consumable with [kibana](https://www.elastic.co/products/kibana). 20 | - Date pattern based index names. 21 | - Custom transformer function to transform logged data into a different message structure. 22 | - Buffering of messages in case of unavailability of ES. The limit is the memory as all unwritten messages are kept in memory. 23 | 24 | ### Compatibility 25 | 26 | For **Winston 3.7**, **Elasticsearch 8.0** and later, use the >= `0.17.0`. 27 | For **Winston 3.4**, **Elasticsearch 7.8** and later, use the >= `0.16.0`. 28 | For **Winston 3.x**, **Elasticsearch 7.0** and later, use the >= `0.7.0`. 29 | For **Elasticsearch 6.0** and later, use the `0.6.0`. 30 | For **Elasticsearch 5.0** and later, use the `0.5.9`. 31 | For earlier versions, use the `0.4.x` series. 32 | 33 | ### Unsupported / Todo 34 | 35 | - Querying. 36 | 37 | ## Installation 38 | 39 | ```sh 40 | npm install --save winston winston-elasticsearch 41 | ``` 42 | 43 | ## Usage 44 | 45 | ```js 46 | const winston = require('winston'); 47 | const { ElasticsearchTransport } = require('winston-elasticsearch'); 48 | 49 | const esTransportOpts = { 50 | level: 'info' 51 | }; 52 | const esTransport = new ElasticsearchTransport(esTransportOpts); 53 | const logger = winston.createLogger({ 54 | transports: [ 55 | esTransport 56 | ] 57 | }); 58 | // Compulsory error handling 59 | logger.on('error', (error) => { 60 | console.error('Error in logger caught', error); 61 | }); 62 | esTransport.on('error', (error) => { 63 | console.error('Error in logger caught', error); 64 | }); 65 | ``` 66 | 67 | The [winston API for logging](https://github.com/winstonjs/winston#streams-objectmode-and-info-objects) 68 | can be used with one restriction: Only one JS object can only be logged and indexed as such. 69 | If multiple objects are provided as arguments, the contents are stringified. 70 | 71 | ## Options 72 | 73 | - `level` [`info`] Messages logged with a severity greater or equal to the given one are logged to ES; others are discarded. 74 | - `index` [none | when `dataStream` is `true`, `logs-app-default`] The index to be used. This option is mutually exclusive with `indexPrefix`. 75 | - `indexPrefix` [`logs`] The prefix to use to generate the index name according to the pattern `-`. Can be string or function, returning the string to use. 76 | - `indexSuffixPattern` [`YYYY.MM.DD`] a Day.js compatible date/ time pattern. 77 | - `transformer` [see below] A transformer function to transform logged data into a different message structure. 78 | - `useTransformer` [`true`] If set to `true`, the given `transformer` will be used (or the default). Set to `false` if you want to apply custom transformers during Winston's `createLogger`. 79 | - `ensureIndexTemplate` [`true`] If set to `true`, the given `indexTemplate` is checked/ uploaded to ES when the module is sending the first log message to make sure the log messages are mapped in a sensible manner. 80 | - `indexTemplate` [see file `index-template-mapping.json`] the mapping template to be ensured as parsed JSON. 81 | `ensureIndexTemplate` is `true` and `indexTemplate` is `undefined` 82 | - `flushInterval` [`2000`] Time span between bulk writes in ms. 83 | - `retryLimit` [`400`] Number of retries to connect to ES before giving up. 84 | - `healthCheckTimeout` [`30s`] Timeout for one health check (health checks will be retried forever). 85 | - `healthCheckWaitForStatus` [`yellow`] Status to wait for when check upon health. See [its API docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) for supported options. 86 | - `healthCheckWaitForNodes` [`>=1`] Nodes to wait for when check upon health. See [its API docs](https://www.elastic.co/guide/en/elasticsearch/reference/current/cluster-health.html) for supported options. 87 | - `client` An [elasticsearch client](https://www.npmjs.com/package/@elastic/elasticsearch) instance. If given, the `clientOpts` are ignored. 88 | - `clientOpts` An object passed to the ES client. See [its docs](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/client-configuration.html) for supported options. 89 | - `waitForActiveShards` [`1`] Sets the number of shard copies that must be active before proceeding with the bulk operation. 90 | - `pipeline` [none] Sets the pipeline id to pre-process incoming documents with. See [the bulk API docs](https://www.elastic.co/guide/en/elasticsearch/client/javascript-api/current/api-reference.html#api-bulk). 91 | - `buffering` [`true`] Boolean flag to enable or disable messages buffering. The `bufferLimit` option is ignored if set to `false`. 92 | - `bufferLimit` [`null`] Limit for the number of log messages in the buffer. 93 | - `apm` [`null`] Inject [apm client](https://www.npmjs.com/package/elastic-apm-node) to link elastic logs with elastic apm traces. 94 | - `dataStream` [`false`] Use Elasticsearch [datastreams](https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html). 95 | - `source` [none] the source of the log message. This can be useful for microservices to understand from which service a log message origins. 96 | - `internalLogger` [`console.error`] A logger of last resort to log internal errors. 97 | 98 | ### Logging of ES Client 99 | 100 | The default client and options will log through `console`. 101 | 102 | ### Interdependencies of Options 103 | 104 | When changing the `indexPrefix` and/or the `transformer`, 105 | make sure to provide a matching `indexTemplate`. 106 | 107 | ## Transformer 108 | 109 | The transformer function allows mutation of log data as provided 110 | by winston into a shape more appropriate for indexing in Elasticsearch. 111 | 112 | The default transformer generates a `@timestamp` and rolls any `meta` 113 | objects into an object called `fields`. 114 | 115 | Params: 116 | 117 | - `logdata` An object with the data to log. Properties are: 118 | - `timestamp` [`new Date().toISOString()`] The timestamp of the log entry 119 | - `level` The log level of the entry 120 | - `message` The message for the log entry 121 | - `meta` The meta data for the log entry 122 | 123 | Returns: Object with the following properties 124 | 125 | - `@timestamp` The timestamp of the log entry 126 | - `severity` The log level of the entry 127 | - `message` The message for the log entry 128 | - `fields` The meta data for the log entry 129 | 130 | The default transformer function's transformation is shown below. 131 | 132 | Input A: 133 | 134 | ```js 135 | { 136 | "message": "Some message", 137 | "level": "info", 138 | "meta": { 139 | "method": "GET", 140 | "url": "/sitemap.xml", 141 | ... 142 | } 143 | } 144 | ``` 145 | 146 | Output A: 147 | 148 | ```js 149 | { 150 | "@timestamp": "2019-09-30T05:09:08.282Z", 151 | "message": "Some message", 152 | "severity": "info", 153 | "fields": { 154 | "method": "GET", 155 | "url": "/sitemap.xml", 156 | ... 157 | } 158 | } 159 | ``` 160 | 161 | The default transformer can be imported and extended 162 | ### Example 163 | ```js 164 | const { ElasticsearchTransformer } = require('winston-elasticsearch'); 165 | const esTransportOpts = { 166 | transformer: (logData) => { 167 | const transformed = ElasticsearchTransformer(logData); 168 | transformed.fields.customField = 'customValue' 169 | return transformed; 170 | }}; 171 | const esTransport = new ElasticsearchTransport(esTransportOpts); 172 | 173 | ``` 174 | 175 | Note that in current logstash versions, the only "standard fields" are 176 | `@timestamp` and `@version`, anything else is just free. 177 | 178 | A custom transformer function can be provided in the options initiation. 179 | 180 | ## Events 181 | 182 | - `error`: in case of any error. 183 | 184 | ## Example 185 | 186 | An example assuming default settings. 187 | 188 | ### Log Action 189 | 190 | ```js 191 | logger.info('Some message', {}); 192 | ``` 193 | 194 | Only JSON objects are logged from the `meta` field. Any non-object is ignored. 195 | 196 | ### Generated Message 197 | 198 | The log message generated by this module has the following structure: 199 | 200 | ```js 201 | { 202 | "@timestamp": "2019-09-30T05:09:08.282Z", 203 | "message": "Some log message", 204 | "severity": "info", 205 | "fields": { 206 | "method": "GET", 207 | "url": "/sitemap.xml", 208 | "headers": { 209 | "host": "www.example.com", 210 | "user-agent": "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", 211 | "accept": "*/*", 212 | "accept-encoding": "gzip,deflate", 213 | "from": "googlebot(at)googlebot.com", 214 | "if-modified-since": "Tue, 30 Sep 2019 11:34:56 GMT", 215 | "x-forwarded-for": "66.249.78.19" 216 | } 217 | } 218 | } 219 | ``` 220 | 221 | ### Target Index 222 | 223 | This message would be POSTed to the following endpoint: 224 | 225 | http://localhost:9200/logs-2019.09.30/log/ 226 | 227 | So the default mapping uses an index pattern `logs-*`. 228 | 229 | ## Logs correlation with Elastic APM 230 | 231 | ### Instrument your code 232 | 233 | - Install the official nodejs client for [elastic-apm](https://www.npmjs.com/package/elastic-apm-node) 234 | 235 | ```sh 236 | yarn add elastic-apm-node 237 | - or - 238 | npm install elastic-apm-node 239 | ``` 240 | 241 | Then, before any other require in your code, do: 242 | 243 | ```js 244 | const apm = require("elastic-apm-node").start({ 245 | serverUrl: "" 246 | }) 247 | 248 | // Set up the logger 249 | var winston = require('winston'); 250 | var Elasticsearch = require('winston-elasticsearch'); 251 | 252 | var esTransportOpts = { 253 | apm, 254 | level: 'info', 255 | clientOpts: { node: "" } 256 | }; 257 | var logger = winston.createLogger({ 258 | transports: [ 259 | new Elasticsearch(esTransportOpts) 260 | ] 261 | }); 262 | ``` 263 | 264 | ### Inject apm traces into logs 265 | 266 | ```js 267 | logger.info('Some log message'); 268 | ``` 269 | 270 | Will produce: 271 | 272 | ```js 273 | { 274 | "@timestamp": "2021-03-13T20:35:28.129Z", 275 | "message": "Some log message", 276 | "severity": "info", 277 | "fields": {}, 278 | "transaction": { 279 | "id": "1f6c801ffc3ae6c6" 280 | }, 281 | "trace": { 282 | "id": "1f6c801ffc3ae6c6" 283 | } 284 | } 285 | ``` 286 | 287 | ### Notice 288 | 289 | Some "custom" logs may not have the apm trace. 290 | 291 | If that is the case, you can retrieve traces using `apm.currentTraceIds` like so: 292 | 293 | ```js 294 | logger.info("Some log message", { ...apm.currentTracesIds }) 295 | ``` 296 | 297 | The transformer function (see above) will place the apm trace in the root object 298 | so that kibana can link Logs to APMs. 299 | 300 | **Custom traces WILL TAKE PRECEDENCE** 301 | 302 | If you are using a custom transformer, you should add the following code into it: 303 | 304 | ```js 305 | if (logData.meta['transaction.id']) transformed.transaction = { id: logData.meta['transaction.id'] }; 306 | if (logData.meta['trace.id']) transformed.trace = { id: logData.meta['trace.id'] }; 307 | if (logData.meta['span.id']) transformed.span = { id: logData.meta['span.id'] }; 308 | ``` 309 | 310 | This scenario may happen on a server (e.g. restify) where you want to log the query 311 | after it was sent to the client (e.g. using `server.on('after', (req, res, route, error) => log.debug("after", { route, error }))`). 312 | In that case you will not get the traces into the response because traces would 313 | have stopped (as the server sent the response to the client). 314 | 315 | In that scenario, you could do something like so: 316 | 317 | ```js 318 | server.use((req, res, next) => { 319 | req.apm = apm.currentTracesIds 320 | next() 321 | }) 322 | server.on("after", (req, res, route, error) => log.debug("after", { route, error, ...req.apm })) 323 | ``` 324 | 325 | ## Manual Flushing 326 | 327 | Flushing can be manually triggered like this: 328 | 329 | ```js 330 | const esTransport = new ElasticsearchTransport(esTransportOpts); 331 | esTransport.flush(); 332 | ``` 333 | 334 | ## Datastreams 335 | 336 | Elasticsearch 7.9 and higher supports [Datastreams](https://www.elastic.co/guide/en/elasticsearch/reference/master/data-streams.html). 337 | 338 | When `dataStream: true` is set, bulk indexing happens with `create` instead of `index`, and also the default naming convention is `logs-*-*`, which will match the built-in [Index template](https://www.elastic.co/guide/en/elasticsearch/reference/master/index-templates.html) and [ILM](https://www.elastic.co/guide/en/elasticsearch/reference/master/index-lifecycle-management.html) policy, 339 | automatically creating a datastream. 340 | 341 | By default, the datastream will be named `logs-app-default`, but alternatively, you can set the `index` option to anything that matches `logs-*-*` to make use of the built-in template and ILM policy. 342 | 343 | If `dataStream: true` is enabled, AND ( you are using Elasticsearch < 7.9 OR (you have set a custom `index` that does not match `logs-*-*` AND you have not created a custom matching template in Elasticsearch)), a normal index will be created. 344 | --------------------------------------------------------------------------------