├── .gitignore ├── .travis.yml ├── example └── winston │ ├── data │ └── fluentd.d │ │ └── fluent.conf │ ├── docker-compose.yml │ └── index.js ├── lib ├── event-time.js ├── index.js ├── logger-error.js ├── winston.js ├── index.d.ts ├── testHelper.js └── sender.js ├── package.json ├── test ├── test.event-time.js ├── test.winston.js └── test.sender.js ├── .eslintrc.js ├── CHANGELOG.md ├── LICENSE.txt └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | /package-lock.json 3 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | node_js: 4 | - "6" 5 | - "8" 6 | - "10" 7 | -------------------------------------------------------------------------------- /example/winston/data/fluentd.d/fluent.conf: -------------------------------------------------------------------------------- 1 | 2 | @type forward 3 | bind 0.0.0.0 4 | port 24224 5 | 6 | 7 | 8 | @type stdout 9 | -------------------------------------------------------------------------------- /example/winston/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: "3.5" 2 | services: 3 | fluentd: 4 | image: fluent/fluentd:v1.2.3 5 | entrypoint: ["fluentd", "-c", "/etc/fluent/fluent.conf", "-vv"] 6 | ports: 7 | - "24224:24224" 8 | - "24224:24224/udp" 9 | volumes: 10 | - "./data/fluentd.d/fluent.conf:/etc/fluent/fluent.conf" -------------------------------------------------------------------------------- /example/winston/index.js: -------------------------------------------------------------------------------- 1 | const winston = require('winston'); 2 | const fluentNodeLogger = require('../../lib'); 3 | 4 | const logger = winston.createLogger({ 5 | transports: [ 6 | new (fluentNodeLogger.support.winstonTransport())( 7 | '___specialcustomtesttag', 8 | { 9 | host: 'localhost', 10 | port: 24224, 11 | timeout: 3.0, 12 | requireAckResponse: true, 13 | } 14 | ), 15 | ], 16 | }); 17 | 18 | (function repeatLog() { 19 | setTimeout(() => { 20 | logger.info('it works'); 21 | repeatLog(); 22 | }, 1000) 23 | })(); 24 | -------------------------------------------------------------------------------- /lib/event-time.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = class EventTime { 4 | constructor(epoch, nano) { 5 | this.epoch = epoch; 6 | this.nano = nano; 7 | } 8 | 9 | static pack(eventTime) { 10 | const b = Buffer.allocUnsafe(8); 11 | b.writeUInt32BE(eventTime.epoch, 0); 12 | b.writeUInt32BE(eventTime.nano, 4); 13 | return b; 14 | } 15 | 16 | static unpack(buffer) { 17 | const e = buffer.readUInt32BE(0); 18 | const n = buffer.readUInt32BE(4); 19 | return new EventTime(e, n); 20 | } 21 | 22 | static now() { 23 | const now = Date.now(); 24 | return EventTime.fromTimestamp(now); 25 | } 26 | 27 | static fromDate(date) { 28 | const t = date.getTime(); 29 | return EventTime.fromTimestamp(t); 30 | } 31 | 32 | static fromTimestamp(t) { 33 | const epoch = Math.floor(t / 1000); 34 | const nano = t % 1000 * 1000000; 35 | return new EventTime(epoch, nano); 36 | } 37 | }; 38 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fluent-logger", 3 | "version": "3.4.1", 4 | "main": "./lib/index.js", 5 | "scripts": { 6 | "test": "mocha -t 10000 --recursive", 7 | "lint": "eslint ." 8 | }, 9 | "author": { 10 | "name": "Yohei Sasaki", 11 | "email": "yssk22@gmail.com", 12 | "url": "http://github.com/yssk22" 13 | }, 14 | "contributors": [ 15 | { 16 | "name": "Eduardo Silva", 17 | "email": "eduardo@treasure-data.com", 18 | "url": "http://edsiper.linuxchile.cl" 19 | }, 20 | { 21 | "name": "okkez", 22 | "email": "okkez000@gmail.com", 23 | "url": "http://okkez.net" 24 | } 25 | ], 26 | "repository": { 27 | "type": "git", 28 | "url": "https://github.com/fluent/fluent-logger-node" 29 | }, 30 | "engines": { 31 | "node": ">=6" 32 | }, 33 | "dependencies": { 34 | "msgpack-lite": "*" 35 | }, 36 | "devDependencies": { 37 | "async": "*", 38 | "chai": "*", 39 | "eslint": "^5.1.0", 40 | "eslint-plugin-node": "*", 41 | "mocha": "*", 42 | "selfsigned": "*", 43 | "winston": "*" 44 | }, 45 | "license": "Apache-2.0", 46 | "keywords": [ 47 | "logger", 48 | "fluent", 49 | "fluentd" 50 | ] 51 | } 52 | -------------------------------------------------------------------------------- /test/test.event-time.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* globals describe, it */ 3 | /* eslint node/no-unpublished-require: ["error", {"allowModules": ["chai"]}] */ 4 | const expect = require('chai').expect; 5 | const EventTime = require('../lib/event-time'); 6 | const msgpack = require('msgpack-lite'); 7 | 8 | const codec = msgpack.createCodec(); 9 | codec.addExtPacker(0x00, EventTime, EventTime.pack); 10 | codec.addExtUnpacker(0x00, EventTime.unpack); 11 | 12 | describe('EventTime', () => { 13 | it('should equal to decoded value', (done) => { 14 | const eventTime = EventTime.now(); 15 | const encoded = msgpack.encode(eventTime, { codec: codec }); 16 | const decoded = msgpack.decode(encoded, { codec: codec }); 17 | expect(JSON.stringify(decoded)).to.equal(JSON.stringify(eventTime)); 18 | done(); 19 | }); 20 | it('should equal fromDate and fromTimestamp', (done) => { 21 | const now = new Date(1489543720999); // 2017-03-15T02:08:40.999Z 22 | const timestamp = now.getTime(); 23 | const eventTime = JSON.stringify(new EventTime(1489543720, 999000000)); 24 | const eventTime1 = JSON.stringify(EventTime.fromDate(now)); 25 | const eventTime2 = JSON.stringify(EventTime.fromTimestamp(timestamp)); 26 | expect(eventTime1).to.equal(eventTime); 27 | expect(eventTime2).to.equal(eventTime); 28 | done(); 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "env": { 3 | "es6": true, 4 | "node": true 5 | }, 6 | "plugins": ["node"], 7 | "extends": ["eslint:recommended", "plugin:node/recommended"], 8 | "parserOptions": { 9 | "sourceType": "module" 10 | }, 11 | "rules": { 12 | "indent": [ 13 | "error", 14 | 2, { 15 | "MemberExpression": 1 16 | } 17 | ], 18 | "linebreak-style": [ 19 | "error", 20 | "unix" 21 | ], 22 | "quotes": [ 23 | "error", 24 | "single" 25 | ], 26 | "semi": [ 27 | "error", 28 | "always" 29 | ], 30 | "brace-style": [ 31 | "error" 32 | ], 33 | "block-spacing": [ 34 | "error" 35 | ], 36 | "space-before-blocks": [ 37 | "error" 38 | ], 39 | "keyword-spacing": [ 40 | "error" 41 | ], 42 | "space-in-parens": [ 43 | "error" 44 | ], 45 | "space-infix-ops": [ 46 | "error" 47 | ], 48 | "space-unary-ops": [ 49 | "error" 50 | ], 51 | "prefer-arrow-callback": [ 52 | "warn" 53 | ], 54 | "no-unused-vars": [ 55 | "error", { 56 | "varsIgnorePattern": "^_", 57 | "args": "none" 58 | } 59 | ], 60 | "no-var": [ 61 | "error" 62 | ], 63 | "node/exports-style": [ 64 | "error", 65 | "module.exports" 66 | ] 67 | } 68 | }; 69 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const FluentSender = require('./sender'); 4 | const EventTime = require('./event-time'); 5 | let sender = new FluentSender('debug'); 6 | 7 | module.exports = { 8 | configure: function(tag, options) { 9 | sender.end(); 10 | sender = new FluentSender(tag, options); 11 | sender._setupErrorHandler(); 12 | // Optimization -- see note at end 13 | module.exports.emit = sender.emit.bind(sender); 14 | }, 15 | 16 | createFluentSender: function(tag, options) { 17 | const _sender = new FluentSender(tag, options); 18 | _sender._setupErrorHandler(); 19 | return _sender; 20 | }, 21 | 22 | support: { 23 | winstonTransport: function() { 24 | const transport = require('../lib/winston'); 25 | return transport; 26 | } 27 | }, 28 | 29 | EventTime: EventTime 30 | }; 31 | 32 | // delegate logger interfaces to default sender object 33 | const methods = ['end', 'addListener', 'on', 'once', 'removeListener', 'removeAllListeners', 'setMaxListeners', 'getMaxListeners']; 34 | methods.forEach((attr, i) => { 35 | module.exports[attr] = function() { 36 | if (sender) { 37 | return sender[attr].apply(sender, arguments); 38 | } 39 | return undefined; 40 | }; 41 | }); 42 | 43 | // Export emit() directly so that calls to it can be inlined properly. 44 | module.exports.emit = sender.emit.bind(sender); 45 | -------------------------------------------------------------------------------- /lib/logger-error.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | module.exports = {}; 4 | 5 | class BaseError extends Error { 6 | constructor(message, options) { 7 | super(); 8 | Error.captureStackTrace(this, this.constructor); 9 | this.name = this.constructor.name; 10 | this.message = message; 11 | this.options = options; 12 | } 13 | } 14 | 15 | class ConfigError extends BaseError { 16 | constructor(message, options) { 17 | super(message, options); 18 | } 19 | } 20 | 21 | class MissingTagError extends BaseError { 22 | constructor(message, options) { 23 | super(message, options); 24 | } 25 | } 26 | 27 | class ResponseError extends BaseError { 28 | constructor(message, options) { 29 | super(message, options); 30 | } 31 | } 32 | 33 | class ResponseTimeoutError extends BaseError { 34 | constructor(message, options) { 35 | super(message, options); 36 | } 37 | } 38 | 39 | class DataTypeError extends BaseError { 40 | constructor(message, options) { 41 | super(message, options); 42 | } 43 | } 44 | 45 | class HandshakeError extends BaseError { 46 | constructor(message, options) { 47 | super(message, options); 48 | } 49 | } 50 | 51 | module.exports = { 52 | ConfigError: ConfigError, 53 | MissingTag: MissingTagError, 54 | ResponseError: ResponseError, 55 | DataTypeError: DataTypeError, 56 | ResponseTimeout: ResponseTimeoutError, 57 | HandshakeError: HandshakeError 58 | }; 59 | -------------------------------------------------------------------------------- /test/test.winston.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* globals describe, it */ 3 | /* eslint node/no-unpublished-require: ["error", {"allowModules": ["async", "chai", "winston"]}] */ 4 | const expect = require('chai').expect; 5 | const winstonSupport = require('../lib/winston'); 6 | const winston = require('winston'); 7 | const runServer = require('../lib/testHelper').runServer; 8 | 9 | describe('winston', () => { 10 | describe('name', () => { 11 | it('should be "fluent"', (done) => { 12 | expect((new winstonSupport()).name).to.be.equal('fluent'); 13 | done(); 14 | }); 15 | }); 16 | 17 | describe('transport', () => { 18 | it('should send log records', (done) => { 19 | runServer({}, {}, (server, finish) => { 20 | const logger = winston.createLogger({ 21 | format: winston.format.combine( 22 | winston.format.splat(), 23 | winston.format.simple() 24 | ), 25 | transports: [ 26 | new winstonSupport({tag: 'debug', port: server.port}) 27 | ] 28 | }); 29 | 30 | logger.info('foo %s', 'bar', {x: 1}); 31 | setTimeout(() => { 32 | finish((data) => { 33 | expect(data[0].tag).to.be.equal('debug'); 34 | expect(data[0].data).exist; 35 | expect(data[0].time).exist; 36 | expect(data[0].data.message).to.be.equal('foo bar'); 37 | expect(data[0].data.level).to.be.equal('info'); 38 | expect(data[0].data.x).to.be.equal(1); 39 | done(); 40 | }); 41 | }, 1000); 42 | }); 43 | }); 44 | }); 45 | }); 46 | -------------------------------------------------------------------------------- /lib/winston.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /** 3 | * winston transport support 4 | */ 5 | 6 | const FluentSender = require('./sender'); 7 | /* eslint-disable-next-line node/no-extraneous-require */ 8 | const Transport = require('winston-transport'); 9 | const DEFAULT_TAG = 'winston'; 10 | 11 | module.exports = class FluentTransport extends Transport { 12 | constructor(_tag = DEFAULT_TAG, _options = {}) { 13 | // following check is to maintain compatibility with code that 14 | // uses the :tag parameter as the options object, 15 | // TODO: remove this check on major release 16 | const tagIsString = (typeof _tag === 'string'); 17 | const tagIsObject = (typeof _tag === 'object'); 18 | const tag = ( 19 | (tagIsString) 20 | ? _tag 21 | : ( 22 | tagIsObject 23 | ? (_tag.tag || DEFAULT_TAG) 24 | : null 25 | ) 26 | ); 27 | const options = ( 28 | (tagIsObject) 29 | ? _tag 30 | : (_options || {}) 31 | ); 32 | super(options); 33 | this.name = 'fluent'; 34 | this.sender = new FluentSender(tag, options); 35 | this.sender._setupErrorHandler(); 36 | } 37 | 38 | log(info, callback) { 39 | setImmediate(() => { 40 | this.sender.emit(info, (error) => { 41 | if (error) { 42 | this.emit('error', info); 43 | callback && callback(error, false); 44 | } else { 45 | this.emit('logged', info); 46 | callback && callback(null, true); 47 | } 48 | }); 49 | }); 50 | } 51 | 52 | _final(callback) { 53 | if (!this.sender) return process.nextTick(callback); 54 | 55 | this.sender.end(null, null, () => { 56 | this.sender = null; 57 | callback(); 58 | }); 59 | } 60 | }; 61 | -------------------------------------------------------------------------------- /lib/index.d.ts: -------------------------------------------------------------------------------- 1 | // Type definitions for fluent-logger-node 2 | // Project: https://github.com/fluent/fluent-logger-node 3 | // Definitions by: navono 4 | // Definitions: https://github.com/fluent/fluent-logger-node 5 | 6 | /// 7 | 8 | import { Writable } from 'stream'; 9 | 10 | declare namespace fluentLogger { 11 | interface Options { 12 | eventMode?: string; 13 | host?: string; 14 | port?: number; 15 | path?: string; 16 | timeout?: number; 17 | tls?: any; 18 | tlsOptions?: any; 19 | reconnectInterval?: number; 20 | requireAckResponse?: boolean; 21 | ackResponseTimeout?: number; 22 | milliseconds?: number; 23 | flushInterval?: number; 24 | sendQueueSizeLimit?: number; 25 | security?: Security; 26 | internalLogger?: Logger; 27 | } 28 | 29 | interface Security { 30 | clientHostname: string; 31 | sharedKey: string; 32 | username?: string; 33 | password?: string; 34 | } 35 | 36 | interface StreamOptions { 37 | label?: string; 38 | encoding?: string; 39 | } 40 | 41 | interface Logger { 42 | info: LogFunction; 43 | error: LogFunction; 44 | [other: string]: any; 45 | } 46 | 47 | interface LogFunction { 48 | (message: any, data?: any, ...extra: any[]): any 49 | } 50 | 51 | type Timestamp = number | Date; 52 | type Callback = (err?: Error) => void; 53 | 54 | class FluentSender { 55 | constructor(tagPrefix: string, options: Options); 56 | 57 | emit(data: T, callback?: Callback): void; 58 | emit(data: T, timestamp: Timestamp, callback?: Callback): void; 59 | emit(label: string, data: T, callback?: Callback): void; 60 | emit(label: string, data: T, timestamp: Timestamp, callback?: Callback): void; 61 | end(label: string, data: T, callback: Callback): void; 62 | toStream(label: string): Writable; 63 | toStream(opt: StreamOptions): Writable; 64 | } 65 | 66 | class InnerEventTime { 67 | epoch: number; 68 | nano: string; 69 | 70 | constructor(epoch: number, nano: number); 71 | 72 | static pack(eventTime: InnerEventTime): Buffer; 73 | static unpack(buffer: Buffer): InnerEventTime; 74 | static now(): InnerEventTime; 75 | static fromDate(date: Date): InnerEventTime; 76 | static fromTimestamp(t: number): InnerEventTime; 77 | } 78 | 79 | let support: { 80 | winstonTransport: any 81 | }; 82 | 83 | let EventTime: InnerEventTime; 84 | 85 | function configure(tag: string, options: Options): void; 86 | function createFluentSender(tag: string, options: Options): FluentSender; 87 | 88 | function emit(data: T, callback?: Callback): void; 89 | function emit(data: T, timestamp: Timestamp, callback?: Callback): void; 90 | function emit(label: string, data: T, callback?: Callback): void; 91 | function emit(label: string, data: T, timestamp: Timestamp, callback?: Callback): void; 92 | function end(label: string, data: T, callback: Callback): void; 93 | 94 | function on(event: string | symbol, listener: (...args: any[]) => void): void; 95 | function once(event: string | symbol, listener: (...args: any[]) => void): void; 96 | function removeListener(event: string | symbol, listener: (...args: any[]) => void): void; 97 | function removeAllListeners(event?: string | symbol): void; 98 | function setMaxListeners(n: number): void; 99 | function getMaxListeners(): number; 100 | } 101 | 102 | export = fluentLogger; 103 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # v3.x 2 | 3 | ## v3.4.1 - 2019-12-14 4 | 5 | ### Fixes 6 | 7 | * Fix type definition for winston support #163 8 | 9 | ## v3.4.0 - 2019-11-20 10 | 11 | ### Improvements 12 | 13 | * Gracefully free resources on `.end()` #144 14 | * Update type definitions for TypeScript #145, #147 15 | * Add new option messageQueueSizeLimit #152 16 | 17 | ### Fixes 18 | 19 | * Fix packets on multiple tags get corrput and multiple calls of callbacks on error #155 20 | 21 | ## v3.3.1 - 2019-02-19 22 | 23 | ### Fixes 24 | 25 | * Set up default error handler for winston #136 26 | * Flush sendQueue after reconnect #136 27 | 28 | ## v3.3.0 - 2019-01-31 29 | 30 | ### Improvements 31 | 32 | * Improve performance #131 33 | 34 | ## v3.2.3 - 2019-01-10 35 | 36 | ### Improvements 37 | 38 | * Update type definition according to documentation for security #127 39 | 40 | ### Fixes 41 | 42 | * Fix user based authentication #128 #129 43 | 44 | ## v3.2.2 - 2018-12-13 45 | 46 | ### Improvements 47 | 48 | * Improve TypeScript definitions #118 #125 49 | 50 | ## v3.2.1 - 2018-10-19 51 | 52 | ### Fixes 53 | 54 | * Update TypeScript declaration file #114 55 | 56 | ## v3.2.0 - 2018-09-10 57 | 58 | ### Improvements 59 | 60 | * Allow using a flag to disable automatic reconnect #111 61 | 62 | ## v3.1.0 - 2018-09-10 63 | 64 | ### Improvements 65 | 66 | * Add sendQueueSizeLimit option #102 67 | * Add TypeScrip declaration file #110 68 | 69 | ### Fixes 70 | 71 | * Fix winston support #108 72 | 73 | ## v3.0.0 - 2018-07-24 74 | 75 | ### Improvements 76 | 77 | * Drop Node.js 4 support 78 | * Support winston 3 79 | 80 | # v2.x 81 | 82 | ## v2.8.1 - 2018-09-10 83 | 84 | ### Fixes 85 | 86 | * Fix supported winston version 87 | 88 | ## v2.8.0 - 2018-07-19 89 | 90 | ### Fixes 91 | 92 | * Reset send queue size #99 93 | 94 | ## v2.7.0 - 2018-05-11 95 | 96 | ### Improvements 97 | 98 | * Support TLS #92 99 | 100 | ## v2.6.2 - 2018-02-27 101 | 102 | ### Improvements 103 | 104 | * Introduce ESLint #88 105 | 106 | ### Fixes 107 | 108 | * Avoid writing to closed socket #90 109 | 110 | ## v2.6.1 - 2017-11-16 111 | 112 | ### Improvements 113 | 114 | * Support log level configuration #85 115 | 116 | ## v2.6.0 - 2017-10-23 117 | 118 | ### Improvements 119 | 120 | * Replace built-in log4js appender to log4js-fluent-appender #82 121 | 122 | ## v2.5.0 - 2017-10-11 123 | 124 | ### Improvements 125 | 126 | * Support Fluentd v1 protocol handshake #80 127 | 128 | ## v2.4.4 - 2017-10-10 129 | 130 | ### Fixes 131 | 132 | * Invoke callback function asynchronously in winston transport #81 133 | 134 | ## v2.4.3 - 2017-09-28 135 | 136 | ### Fixes 137 | 138 | * Fix bugs in v2.4.2 #77 139 | 140 | ## v2.4.2 - 2017-09-26 141 | 142 | This release has bugs. 143 | 144 | ### Improvements 145 | 146 | * Use arrow functions #76 147 | 148 | ## v2.4.1 - 2017-07-26 149 | 150 | ### Fixes 151 | 152 | * Clear setTimeout when ack is received #68, #69 153 | * Mark log4js 2.0 and later as unsupported #71 154 | * Flush queue step by step #72 155 | 156 | ## v2.4.0 - 2017-05-30 157 | 158 | ### Improvements 159 | 160 | * Add internal logger #63 161 | 162 | ### Fixes 163 | 164 | * Update supported engines #64 165 | 166 | ## v2.3.0 - 2017-03-16 167 | 168 | ### Improvements 169 | 170 | * Support EventTime #61 171 | * Support connect event #62 172 | 173 | ## v2.2.0 - 2016-10-18 174 | 175 | ### Fixes 176 | 177 | * Wrap dataString to record object #54 178 | 179 | ## v2.1.0 - 2016-09-28 180 | 181 | ### Improvements 182 | 183 | * Support stream #53 184 | 185 | ## v2.0.1 - 2016-07-27 186 | 187 | ### Fixes 188 | 189 | * Fix CI settings 190 | 191 | ## v2.0.0 - 2016-07-27 192 | 193 | ### Improvements 194 | 195 | * Support requireAckResponse 196 | * Support setting tag_prefix to `null` 197 | * Improve error handling 198 | * Add winston transport 199 | 200 | # v1.x 201 | 202 | ## v1.2.1 - 2016-07-15 203 | ## v1.2.0 - 2016-07-15 204 | ## v1.1.1 - 2016-05-09 205 | ## v1.1.0 - 2016-02-01 206 | ## v1.0.0 - 2016-01-12 207 | 208 | # v0.x 209 | 210 | Ancient releases. 211 | 212 | ## v0.5.0 - 2015-12-14 213 | ## v0.4.2 - 2015-11-09 214 | ## v0.4.1 - 2015-10-30 215 | ## v0.4.0 - 2015-10-30 216 | ## v0.3.0 - 2015-09-25 217 | ## v0.2.5 - 2013-11-24 218 | ## v0.2.4 - 2013-06-02 219 | ## v0.2.3 - 2013-06-02 220 | ## v0.2.2 - 2013-06-02 221 | ## v0.2.1 - 2013-01-10 222 | ## v0.2.0 - 2013-01-07 223 | ## v0.1.0 - 2012-04-20 224 | 225 | ## v0.0.2 - 2012-02-17 226 | 227 | Initial release 228 | -------------------------------------------------------------------------------- /lib/testHelper.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const net = require('net'); 3 | const tls = require('tls'); 4 | const msgpack = require('msgpack-lite'); 5 | const crypto = require('crypto'); 6 | const zlib = require('zlib'); 7 | 8 | class MockFluentdServer { 9 | constructor(options, tlsOptions) { 10 | this._port = null; 11 | this._options = options; 12 | this._tlsOptions = tlsOptions || {}; 13 | this._received = []; 14 | this._clients = {}; 15 | this._state = null; 16 | this._nonce = null; 17 | this._userAuthSalt = null; 18 | let server = (socket) => { 19 | const clientKey = socket.remoteAddress + ':' + socket.remotePort; 20 | this._clients[clientKey] = socket; 21 | socket.on('end', () => { 22 | delete this._clients[clientKey]; 23 | }); 24 | const stream = msgpack.createDecodeStream(); 25 | socket.pipe(stream).on('data', (m) => { 26 | if (this._state === 'pingpong') { 27 | const authResult = this._checkPing(m); 28 | socket.write(msgpack.encode(this._generatePong(authResult, this._nonce, this._options.security.sharedKey))); 29 | if (authResult.succeeded) { 30 | this._state = 'established'; 31 | } else { 32 | socket.end(); 33 | } 34 | } else if (this._state === 'established') { 35 | let entries = m[1]; 36 | let options = null; 37 | if (entries instanceof Buffer) { 38 | options = m[2]; 39 | if (options.compressed === 'gzip') { 40 | entries = zlib.gunzipSync(entries); 41 | } 42 | let s = msgpack.createDecodeStream(); 43 | s.on('data', (data) => { 44 | let _time = data[0]; 45 | let record = data[1]; 46 | this._received.push({ 47 | tag: m[0], 48 | data: record, 49 | options: options 50 | }); 51 | }); 52 | s.write(entries); 53 | } else { 54 | this._received.push({ 55 | tag: m[0], 56 | time: m[1], 57 | data: m[2], 58 | options: m[3] 59 | }); 60 | options = m[3]; 61 | } 62 | if (this._options.requireAckResponse && options && options.chunk) { 63 | const response = { 64 | ack: options.chunk 65 | }; 66 | socket.write(msgpack.encode(response)); 67 | } 68 | } 69 | }); 70 | }; 71 | let connectionEventType = 'connection'; 72 | if (this._tlsOptions.tls) { 73 | connectionEventType = 'secureConnection'; 74 | this._server = tls.createServer(this._tlsOptions, server); 75 | } else { 76 | this._server = net.createServer(server); 77 | } 78 | this._server.on(connectionEventType, (socket) => { 79 | if (this._options.security && this._options.security.sharedKey && this._options.security.serverHostname) { 80 | this._state = 'helo'; 81 | this._nonce = crypto.randomBytes(16); 82 | this._userAuthSalt = crypto.randomBytes(16); 83 | } else { 84 | this._state = 'established'; 85 | } 86 | if (this._state === 'helo') { 87 | socket.write(msgpack.encode(this._generateHelo(this._nonce, this._userAuthSalt))); 88 | this._state = 'pingpong'; 89 | } 90 | }); 91 | } 92 | 93 | get port() { 94 | return this._port; 95 | } 96 | 97 | get messages() { 98 | return this._received; 99 | } 100 | 101 | _generateHelo(nonce, userAuthSalt) { 102 | // ['HELO', options(hash)] 103 | let options = { 104 | 'nonce': nonce, 105 | 'auth': this._options.security ? userAuthSalt : '', 106 | 'keepalive': false 107 | }; 108 | return ['HELO', options]; 109 | } 110 | 111 | _checkPing(m) { 112 | // this._options.checkPing() should return { succeeded: true, reason: 'why', sharedKeySalt: 'salt' } 113 | if (this._options.checkPing) { 114 | return this._options.checkPing(m); 115 | } else { 116 | // ['PING', self_hostname, shared_key_salt, sha512_hex(shared_key_salt + self_hostname + nonce + shared_key), username || '', sha512_hex(auth_salt + username + password) || ''] 117 | if (m.length !== 6) { 118 | return { succeeded: false, reason: 'Invalid ping message size' }; 119 | } 120 | if (m[0] !== 'PING') { 121 | return { succeeded: false, reason: 'Invalid ping message' }; 122 | } 123 | const _ping = m[0]; 124 | const hostname = m[1]; 125 | const sharedKeySalt = m[2]; 126 | const sharedKeyHexDigest = m[3]; 127 | const _username = m[4]; 128 | const passwordDigest = m[5]; 129 | const serverSideDigest = crypto.createHash('sha512') 130 | .update(sharedKeySalt) 131 | .update(hostname) 132 | .update(this._nonce) 133 | .update(this._options.security.sharedKey) 134 | .digest('hex'); 135 | if (sharedKeyHexDigest !== serverSideDigest) { 136 | return { succeeded: false, reason: 'shared key mismatch' }; 137 | } 138 | if (this._options.security.username && this._options.security.password) { 139 | const serverSidePasswordDigest = crypto.createHash('sha512') 140 | .update(this._userAuthSalt) 141 | .update(this._options.security.username) 142 | .update(this._options.security.password) 143 | .digest('hex'); 144 | if (passwordDigest !== serverSidePasswordDigest) { 145 | return { succeeded: false, reason: 'username/password mismatch' }; 146 | } 147 | } 148 | return { succeeded: true, sharedKeySalt: sharedKeySalt }; 149 | } 150 | } 151 | 152 | _generatePong(authResult, nonce, sharedKey) { 153 | // this._options.generatePong() should return PONG message 154 | // [ 155 | // 'PONG', 156 | // bool(authentication result), 157 | // 'reason if authentication failed', 158 | // serverHostname, 159 | // sha512_hex(salt + serverHostname + nonce + sharedkey) 160 | // ] 161 | if (authResult.succeeded) { 162 | const sharedKeyDigestHex = crypto.createHash('sha512') 163 | .update(authResult.sharedKeySalt) 164 | .update(this._options.security.serverHostname) 165 | .update(nonce) 166 | .update(sharedKey) 167 | .digest('hex'); 168 | return ['PONG', true, '', this._options.security.serverHostname, sharedKeyDigestHex]; 169 | } else { 170 | return ['PONG', false, authResult.reason, '', '']; 171 | } 172 | } 173 | 174 | listen(callback) { 175 | let options = { 176 | port: this._options.port 177 | }; 178 | this._server.listen(options, () => { 179 | this._port = this._server.address().port; 180 | callback(); 181 | }); 182 | } 183 | 184 | close(callback) { 185 | this._server.close(() => { 186 | callback(); 187 | }); 188 | for (const i in this._clients) { 189 | this._clients[i].end(); 190 | // this._clients[i].destroy(); 191 | } 192 | } 193 | } 194 | 195 | module.exports = { 196 | runServer: function(options, tlsOptions, callback) { 197 | const server = new MockFluentdServer(options, tlsOptions); 198 | server.listen(() => { 199 | callback(server, (_callback) => { 200 | // wait 100 ms to receive all messages and then close 201 | setTimeout(() => { 202 | const messages = server.messages; 203 | server.close(() => { 204 | _callback && _callback(messages); 205 | }); 206 | }, 100); 207 | }); 208 | }); 209 | } 210 | }; 211 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright 2012-2013 Yohei Sasaki 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | 204 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # fluent-logger for Node.js 2 | 3 | fluent-logger implementation for Node.js inspired by [fluent-logger-python]. 4 | 5 | [![NPM](https://nodei.co/npm/fluent-logger.png?downloads=true&downloadRank=true)](https://nodei.co/npm/fluent-logger/) 6 | 7 | [![Build Status](https://secure.travis-ci.org/fluent/fluent-logger-node.png?branch=master,develop)](http://travis-ci.org/fluent/fluent-logger-node) 8 | 9 | ## Install 10 | 11 | $ npm install fluent-logger 12 | 13 | ## Prerequistes 14 | 15 | Fluent daemon should listen on TCP port. 16 | 17 | Simple configuration is following: 18 | 19 | ```aconf 20 | 21 | @type forward 22 | port 24224 23 | 24 | 25 | 26 | @type stdout 27 | 28 | ``` 29 | 30 | ## Usage 31 | 32 | ### Send an event record to Fluentd 33 | 34 | Singleton style 35 | 36 | ```js 37 | var logger = require('fluent-logger') 38 | // The 2nd argument can be omitted. Here is a default value for options. 39 | logger.configure('tag_prefix', { 40 | host: 'localhost', 41 | port: 24224, 42 | timeout: 3.0, 43 | reconnectInterval: 600000 // 10 minutes 44 | }); 45 | 46 | // send an event record with 'tag.label' 47 | logger.emit('label', {record: 'this is a log'}); 48 | ``` 49 | 50 | Instance style 51 | 52 | ```js 53 | var logger = require('fluent-logger').createFluentSender('tag_prefix', { 54 | host: 'localhost', 55 | port: 24224, 56 | timeout: 3.0, 57 | reconnectInterval: 600000 // 10 minutes 58 | }); 59 | ``` 60 | 61 | The emit method has following signature 62 | 63 | ```js 64 | .emit([label string], , [timestamp number/date], [callback function]) 65 | ``` 66 | 67 | Where only the `record` argument is required. If the label is set it will be 68 | appended to the configured tag. 69 | 70 | ### Disable automatic reconnect 71 | Both Singleton and Instance style can disable automatic reconnect allowing the user to handle reconnect himself 72 | ```js 73 | logger.configure('tag_prefix', { 74 | host: 'localhost', 75 | port: 24224, 76 | timeout: 3.0, 77 | enableReconnect: false // defaults to true 78 | }); 79 | ``` 80 | 81 | ### Shared key authentication 82 | 83 | Logger configuration: 84 | 85 | ```js 86 | var logger = require('fluent-logger').createFluentSender('dummy', { 87 | host: 'localhost', 88 | port: 24224, 89 | timeout: 3.0, 90 | reconnectInterval: 600000, // 10 minutes 91 | security: { 92 | clientHostname: "client.localdomain", 93 | sharedKey: "secure_communication_is_awesome" 94 | } 95 | }); 96 | logger.emit('debug', { message: 'This is a message' }); 97 | ``` 98 | 99 | Server configuration: 100 | 101 | ```aconf 102 | 103 | @type forward 104 | port 24224 105 | 106 | self_hostname input.testing.local 107 | shared_key secure_communication_is_awesome 108 | 109 | 110 | 111 | 112 | @type stdout 113 | 114 | ``` 115 | 116 | See also [Fluentd](https://github.com/fluent/fluentd) examples. 117 | 118 | ### TLS/SSL encryption 119 | 120 | Logger configuration: 121 | 122 | ```js 123 | var logger = require('fluent-logger').createFluentSender('dummy', { 124 | host: 'localhost', 125 | port: 24224, 126 | timeout: 3.0, 127 | reconnectInterval: 600000, // 10 minutes 128 | security: { 129 | clientHostname: "client.localdomain", 130 | sharedKey: "secure_communication_is_awesome" 131 | }, 132 | tls: true, 133 | tlsOptions: { 134 | ca: fs.readFileSync('/path/to/ca_cert.pem') 135 | } 136 | }); 137 | logger.emit('debug', { message: 'This is a message' }); 138 | ``` 139 | 140 | Server configuration: 141 | 142 | ```aconf 143 | 144 | @type forward 145 | port 24224 146 | 147 | ca_cert_path /path/to/ca_cert.pem 148 | ca_private_key_path /path/to/ca_key.pem 149 | ca_private_key_passphrase very_secret_passphrase 150 | 151 | 152 | self_hostname input.testing.local 153 | shared_key secure_communication_is_awesome 154 | 155 | 156 | 157 | 158 | @type stdout 159 | 160 | ``` 161 | 162 | FYI: You can generate certificates using fluent-ca-generate command since Fluentd 1.1.0. 163 | 164 | See also [How to enable TLS/SSL encryption](https://docs.fluentd.org/input/forward#how-to-enable-tls-encryption). 165 | 166 | ### Mutual TLS Authentication 167 | 168 | Logger configuration: 169 | 170 | ```js 171 | var logger = require('fluent-logger').createFluentSender('dummy', { 172 | host: 'localhost', 173 | port: 24224, 174 | timeout: 3.0, 175 | reconnectInterval: 600000, // 10 minutes 176 | security: { 177 | clientHostname: "client.localdomain", 178 | sharedKey: "secure_communication_is_awesome" 179 | }, 180 | tls: true, 181 | tlsOptions: { 182 | ca: fs.readFileSync('/path/to/ca_cert.pem'), 183 | cert: fs.readFileSync('/path/to/client-cert.pem'), 184 | key: fs.readFileSync('/path/to/client-key.pem'), 185 | passphrase: 'very-secret' 186 | } 187 | }); 188 | logger.emit('debug', { message: 'This is a message' }); 189 | ``` 190 | 191 | Server configuration: 192 | 193 | ```aconf 194 | 195 | @type forward 196 | port 24224 197 | 198 | ca_path /path/to/ca-cert.pem 199 | cert_path /path/to/server-cert.pem 200 | private_key_path /path/to/server-key.pem 201 | private_key_passphrase very_secret_passphrase 202 | client_cert_auth true 203 | 204 | 205 | self_hostname input.testing.local 206 | shared_key secure_communication_is_awesome 207 | 208 | 209 | 210 | 211 | @type stdout 212 | 213 | ``` 214 | 215 | ### EventTime support 216 | 217 | We can also specify [EventTime](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#eventtime-ext-format) as timestamp. 218 | 219 | ```js 220 | var FluentLogger = require('fluent-logger'); 221 | var EventTime = FluentLogger.EventTime; 222 | var logger = FluentLogger.createFluentSender('tag_prefix', { 223 | var eventTime = new EventTime(1489547207, 745003500); // 2017-03-15 12:06:47 +0900 224 | logger.emit('tag', { message: 'This is a message' }, eventTime); 225 | ``` 226 | 227 | ### Events 228 | 229 | * `connect` : Handle [net.Socket Event: connect](https://nodejs.org/api/net.html#net_event_connect) 230 | * `error` : Handle [net.Socket Event: error](https://nodejs.org/api/net.html#net_event_error_1) 231 | 232 | ```js 233 | var logger = require('fluent-logger').createFluentSender('tag_prefix', { 234 | host: 'localhost', 235 | port: 24224, 236 | timeout: 3.0, 237 | reconnectInterval: 600000 // 10 minutes 238 | }); 239 | logger.on('error', (error) => { 240 | console.log(error); 241 | }); 242 | logger.on('connect', () => { 243 | console.log('connected!'); 244 | }); 245 | ``` 246 | 247 | ## Logging Library Support 248 | 249 | ### log4js 250 | 251 | Use [log4js-fluent-appender](https://www.npmjs.com/package/log4js-fluent-appender). 252 | 253 | ### winston 254 | 255 | Before using [winston](https://github.com/winstonjs/winston) support, you should install it IN YOUR APPLICATION. 256 | 257 | ```js 258 | var winston = require('winston'); 259 | var config = { 260 | host: 'localhost', 261 | port: 24224, 262 | timeout: 3.0, 263 | requireAckResponse: true // Add this option to wait response from Fluentd certainly 264 | }; 265 | var fluentTransport = require('fluent-logger').support.winstonTransport(); 266 | var fluent = new fluentTransport('mytag', config); 267 | var logger = winston.createLogger({ 268 | transports: [fluent, new (winston.transports.Console)()] 269 | }); 270 | 271 | logger.on('flush', () => { 272 | console.log("flush"); 273 | }) 274 | 275 | logger.on('finish', () => { 276 | console.log("finish"); 277 | fluent.sender.end("end", {}, () => {}) 278 | }); 279 | 280 | logger.log('info', 'this log record is sent to fluent daemon'); 281 | logger.info('this log record is sent to fluent daemon'); 282 | logger.info('end of log message'); 283 | logger.end(); 284 | ``` 285 | 286 | **NOTE** If you use `winston@2`, you can use `fluent-logger@2.7.0` or earlier. If you use `winston@3`, you can use `fluent-logger@2.8` or later. 287 | 288 | ### stream 289 | 290 | Several libraries use stream as output. 291 | 292 | ```js 293 | 'use strict'; 294 | const Console = require('console').Console; 295 | var sender = require('fluent-logger').createFluentSender('tag_prefix', { 296 | host: 'localhost', 297 | port: 24224, 298 | timeout: 3.0, 299 | reconnectInterval: 600000 // 10 minutes 300 | }); 301 | var logger = new Console(sender.toStream('stdout'), sender.toStream('stderr')); 302 | logger.log('this log record is sent to fluent daemon'); 303 | setTimeout(()=> sender.end(), 5000); 304 | ``` 305 | 306 | 307 | ## Options 308 | 309 | **tag_prefix** 310 | 311 | The tag prefix string. 312 | You can specify `null` when you use `FluentSender` directly. 313 | In this case, you must specify `label` when you call `emit`. 314 | 315 | **host** 316 | 317 | The hostname. Default value = 'localhost'. 318 | 319 | See [socket.connect][1] 320 | 321 | **port** 322 | 323 | The port to listen to. Default value = 24224. 324 | 325 | See [socket.connect][1] 326 | 327 | **path** 328 | 329 | The path to your Unix Domain Socket. 330 | If you set `path` then fluent-logger ignores `host` and `port`. 331 | 332 | See [socket.connect][1] 333 | 334 | **timeout** 335 | 336 | Set the socket to timetout after `timeout` milliseconds of inactivity 337 | on the socket. 338 | 339 | See [socket.setTimeout][2] 340 | 341 | **reconnectInterval** 342 | 343 | Set the reconnect interval in milliseconds. 344 | If error occurs then reconnect after this interval. 345 | 346 | [1]: https://nodejs.org/api/net.html#net_socket_connect_path_connectlistener 347 | [2]: https://nodejs.org/api/net.html#net_socket_settimeout_timeout_callback 348 | 349 | **requireAckResponse** 350 | 351 | Change the protocol to at-least-once. The logger waits the ack from destination. 352 | 353 | **ackResponseTimeout** 354 | 355 | This option is used when requireAckResponse is true. The default is 190. This default value is based on popular `tcp_syn_retries`. 356 | 357 | **eventMode** 358 | 359 | Set [Event Modes](https://github.com/fluent/fluentd/wiki/Forward-Protocol-Specification-v1#event-modes). This logger supports `Message`, `PackedForward` and `CompressedPackedForward`. 360 | Default is `Message`. 361 | 362 | NOTE: We will change default to `PackedForward` and drop `Message` in next major release. 363 | 364 | **flushInterval** 365 | 366 | Set flush interval in milliseconds. This option has no effect in `Message` mode. 367 | The logger stores emitted events in buffer and flush events for each interval. 368 | Default `100`. 369 | 370 | **messageQueueSizeLimit** 371 | 372 | Maximum number of messages that can be in queue at the same time. If a new message is received and it overflows the queue then the oldest message will be removed before adding the new item. This option has effect only in `Message` mode. No limit by default. 373 | 374 | **security.clientHostname** 375 | 376 | Set hostname of this logger. Use this value for hostname based authentication. 377 | 378 | **security.sharedKey** 379 | 380 | Shared key between client and server. 381 | 382 | **security.username** 383 | 384 | Set username for user based authentication. Default values is empty string. 385 | 386 | **security.password** 387 | 388 | Set password for user based authentication. Default values is empty string. 389 | 390 | **sendQueueSizeLimit** 391 | 392 | Queue size limit in bytes. This option has no effect in `Message` mode. Default is `8 MiB`. 393 | 394 | **tls** 395 | 396 | Enable TLS for socket. 397 | 398 | **tlsOptions** 399 | 400 | Options to pass to tls.connect when tls is true. 401 | 402 | For more details, see following documents 403 | * [tls.connect()](https://nodejs.org/api/tls.html#tls_tls_connect_options_callback) 404 | * [tls.createSecureContext()](https://nodejs.org/api/tls.html#tls_tls_createsecurecontext_options) 405 | 406 | **internalLogger** 407 | 408 | Set internal logger object for FluentLogger. Use `console` by default. 409 | This logger requires `info` and `error` method. 410 | 411 | ## Examples 412 | ### Winston Integration 413 | An example of integrating with Winston can be found at [`./example/winston`](./example/winston). 414 | 415 | You will need Docker Compose to run it. After navigating to `./example/winston`, run `docker-compose up` and then `node index.js`. You should see the Docker logs having an `"it works"` message being output to FluentD. 416 | 417 | ## License 418 | 419 | Apache License, Version 2.0. 420 | 421 | [fluent-logger-python]: https://github.com/fluent/fluent-logger-python 422 | 423 | 424 | ## About NodeJS versions 425 | 426 | This package is compatible with NodeJS versions >= 6. 427 | -------------------------------------------------------------------------------- /lib/sender.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | const EventEmitter = require('events').EventEmitter; 3 | const msgpack = require('msgpack-lite'); 4 | const net = require('net'); 5 | const stream = require('stream'); 6 | const crypto = require('crypto'); 7 | const tls = require('tls'); 8 | const zlib = require('zlib'); 9 | const FluentLoggerError = require('./logger-error'); 10 | const EventTime = require('./event-time'); 11 | 12 | const codec = msgpack.createCodec(); 13 | codec.addExtPacker(0x00, EventTime, EventTime.pack); 14 | codec.addExtUnpacker(0x00, EventTime.unpack); 15 | 16 | class FluentSender { 17 | constructor(tag_prefix, options) { 18 | options = options || {}; 19 | this._eventMode = options.eventMode || 'Message'; // Message, PackedForward, CompressedPackedForward 20 | if (!/^Message|PackedForward|CompressedPackedForward$/.test(this._eventMode)) { 21 | throw new FluentLoggerError.ConfigError('Unknown event mode: ' + this._eventMode); 22 | } 23 | this.tag_prefix = tag_prefix; 24 | this.host = options.host || 'localhost'; 25 | this.port = options.port || 24224; 26 | this.path = options.path; 27 | this.timeout = options.timeout || 3.0; 28 | this.tls = !!options.tls; 29 | this.tlsOptions = options.tlsOptions || {}; 30 | this.enableReconnect = typeof options.enableReconnect === 'boolean' ? options.enableReconnect : true; 31 | this.reconnectInterval = options.reconnectInterval || 600000; // Default is 10 minutes 32 | this.requireAckResponse = options.requireAckResponse; 33 | this.ackResponseTimeout = options.ackResponseTimeout || 190000; // Default is 190 seconds 34 | this.internalLogger = options.internalLogger || console; 35 | this._timeResolution = options.milliseconds ? 1 : 1000; 36 | this._socket = null; 37 | if (this._eventMode === 'Message') { 38 | this._sendQueue = []; // queue for items waiting for being sent. 39 | this._flushInterval = 0; 40 | this._messageQueueSizeLimit = options.messageQueueSizeLimit || 0; 41 | } else { 42 | this._sendQueue = new Map(); 43 | this._flushInterval = options.flushInterval || 100; 44 | this._sendQueueSizeLimit = options.sendQueueSizeLimit || 8 * 1024 * 1024; // 8MiB 45 | this._sendQueueSize = 0; 46 | this._flushSendQueueTimeoutId = null; 47 | this._compressed = (this._eventMode === 'CompressedPackedForward'); 48 | } 49 | this._eventEmitter = new EventEmitter(); 50 | // options.security = { clientHostname: "client.localdomain", sharedKey: "very-secret-shared-key" } 51 | this.security = options.security || { 52 | clientHostname: null, 53 | sharedKey: null, 54 | username: '', 55 | password: '' 56 | }; 57 | this.sharedKeySalt = crypto.randomBytes(16).toString('hex'); 58 | // helo, pingpong, established 59 | this._status = null; 60 | this._connecting = false; 61 | } 62 | 63 | emit(/*[label] , [timestamp], [callback] */a0, a1, a2, a3) { 64 | let label, data, timestamp, callback; 65 | let timestampOrCallback, cbArg; 66 | // Label must be string always 67 | // Data can be almost anything 68 | // Date can be either timestamp number or Date object 69 | // Last argument is an optional callback 70 | if (typeof a0 === 'string') { 71 | label = a0; 72 | data = a1; 73 | timestampOrCallback = a2; 74 | cbArg = a3; 75 | } else { 76 | data = a0; 77 | timestampOrCallback = a1; 78 | cbArg = a2; 79 | } 80 | 81 | if (typeof timestampOrCallback === 'function') { 82 | callback = timestampOrCallback; 83 | } else if (timestampOrCallback) { 84 | timestamp = timestampOrCallback; 85 | if (typeof cbArg === 'function') { 86 | callback = cbArg; 87 | } 88 | } 89 | 90 | const tag = this._makeTag(label); 91 | let error; 92 | let options; 93 | if (tag === null) { 94 | options = { 95 | tag_prefix: this.tag_prefix, 96 | label: label 97 | }; 98 | error = new FluentLoggerError.MissingTag('tag is missing', options); 99 | this._handleEvent('error', error, callback); 100 | return; 101 | } 102 | if (typeof data !== 'object') { 103 | options = { 104 | tag_prefix: this.tag_prefix, 105 | label: label, 106 | record: data 107 | }; 108 | error = new FluentLoggerError.DataTypeError('data must be an object', options); 109 | this._handleEvent('error', error, callback); 110 | return; 111 | } 112 | 113 | this._push(tag, timestamp, data, callback); 114 | this._connect(() => { 115 | this._flushSendQueue(); 116 | }); 117 | } 118 | 119 | end(label, data, callback) { 120 | if ((label != null && data != null)) { 121 | this.emit(label, data, (err) => { 122 | this._close(); 123 | if (err) { 124 | this._handleEvent('error', err, callback); 125 | } else { 126 | callback && callback(); 127 | } 128 | }); 129 | } else { 130 | process.nextTick(() => { 131 | this._close(); 132 | callback && callback(); 133 | }); 134 | } 135 | } 136 | 137 | _close() { 138 | if (this._socket) { 139 | this._socket.end(); 140 | this._socket = null; 141 | this._status = null; 142 | } 143 | } 144 | 145 | _makeTag(label) { 146 | let tag = null; 147 | if (this.tag_prefix && label) { 148 | tag = `${this.tag_prefix}.${label}`; 149 | } else if (this.tag_prefix) { 150 | tag = this.tag_prefix; 151 | } else if (label) { 152 | tag = label; 153 | } 154 | return tag; 155 | } 156 | 157 | _makePacketItem(tag, time, data) { 158 | if (!time || (typeof time !== 'number' && !(time instanceof EventTime))) { 159 | time = Math.floor((time ? time.getTime() : Date.now()) / this._timeResolution); 160 | } 161 | 162 | const packet = [tag, time, data]; 163 | const options = {}; 164 | if (this.requireAckResponse) { 165 | options.chunk = crypto.randomBytes(16).toString('base64'); 166 | packet.push(options); 167 | } 168 | return { 169 | packet: msgpack.encode(packet, { codec: codec }), 170 | tag: tag, 171 | time: time, 172 | data: data, 173 | options: options 174 | }; 175 | } 176 | 177 | _makeEventEntry(time, data) { 178 | if (!time || (typeof time !== 'number' && !(time instanceof EventTime))) { 179 | time = Math.floor((time ? time.getTime() : Date.now()) / this._timeResolution); 180 | } 181 | 182 | return msgpack.encode([time, data], { codec: codec }); 183 | } 184 | 185 | _push(tag, time, data, callback) { 186 | if (this._eventMode === 'Message') { 187 | // Message mode 188 | const item = this._makePacketItem(tag, time, data); 189 | item.callback = callback; 190 | if (this._messageQueueSizeLimit && this._sendQueue.length === this._messageQueueSizeLimit) { 191 | this._sendQueue.shift(); 192 | } 193 | this._sendQueue.push(item); 194 | } else { 195 | // PackedForward mode 196 | const eventEntry = this._makeEventEntry(time, data); 197 | this._sendQueueSize += eventEntry.length; 198 | if (this._sendQueue.has(tag)) { 199 | const eventEntryData = this._sendQueue.get(tag); 200 | eventEntryData.eventEntries.push(eventEntry); 201 | eventEntryData.size += eventEntry.length; 202 | if (callback) eventEntryData.callbacks.push(callback); 203 | } else { 204 | const callbacks = callback ? [callback] : []; 205 | this._sendQueue.set(tag, { 206 | eventEntries: [eventEntry], 207 | size: eventEntry.length, 208 | callbacks: callbacks 209 | }); 210 | } 211 | } 212 | } 213 | 214 | _connect(callback) { 215 | if (this._connecting) { 216 | return; 217 | } 218 | 219 | if (this._socket === null) { 220 | this._connecting = true; 221 | this._doConnect(() => { 222 | this._connecting = false; 223 | callback(); 224 | }); 225 | } else if (!this._socket.writable) { 226 | this._disconnect(); 227 | this._connect(callback); 228 | } else { 229 | this._connecting = false; 230 | process.nextTick(callback); 231 | } 232 | } 233 | 234 | _doConnect(callback) { 235 | const addHandlers = () => { 236 | const errorHandler = (err) => { 237 | if (this._socket) { 238 | this._disconnect(); 239 | this._handleEvent('error', err); 240 | } 241 | }; 242 | this._socket.on('error', errorHandler); 243 | this._socket.on('connect', () => { 244 | this._handleEvent('connect'); 245 | }); 246 | if (this.tls) { 247 | this._socket.on('tlsClientError', errorHandler); 248 | this._socket.on('secureConnect', () => { 249 | this._handleEvent('connect'); 250 | }); 251 | } 252 | }; 253 | if (!this.tls) { 254 | this._socket = new net.Socket(); 255 | this._socket.setTimeout(this.timeout); 256 | addHandlers(); 257 | } 258 | if (this.path) { 259 | if (this.tls) { 260 | this._socket = tls.connect(Object.assign({}, this.tlsOptions, { path: this.path }), () => { 261 | callback(); 262 | }); 263 | addHandlers(); 264 | } else { 265 | this._socket.connect(this.path, () => { 266 | callback(); 267 | }); 268 | } 269 | } else { 270 | const postConnect = () => { 271 | if (this.security.clientHostname && this.security.sharedKey !== null) { 272 | this._handshake(callback); 273 | } else { 274 | this._status = 'established'; 275 | callback(); 276 | } 277 | }; 278 | if (this.tls) { 279 | this._socket = tls.connect(Object.assign({}, this.tlsOptions, { host: this.host, port: this.port }), () => { 280 | postConnect(); 281 | }); 282 | addHandlers(); 283 | } else { 284 | this._socket.connect(this.port, this.host, () => { 285 | postConnect(); 286 | }); 287 | } 288 | } 289 | } 290 | 291 | _disconnect() { 292 | this._socket && this._socket.destroy(); 293 | this._socket = null; 294 | this._status = null; 295 | this._connecting = false; 296 | } 297 | 298 | _handshake(callback) { 299 | if (this._status === 'established') { 300 | return; 301 | } 302 | this._status = 'helo'; 303 | this._socket.once('data', (data) => { 304 | this._socket.pause(); 305 | const heloStatus = this._checkHelo(data); 306 | if (!heloStatus.succeeded) { 307 | this.internalLogger.error('Received invalid HELO message from ' + this._socket.remoteAddress); 308 | this._disconnect(); 309 | return; 310 | } 311 | this._status = 'pingpong'; 312 | this._socket.write(this._generatePing(), () => { 313 | this._socket.resume(); 314 | this._socket.once('data', (data) => { 315 | const pongStatus = this._checkPong(data); 316 | if (!pongStatus.succeeded) { 317 | this.internalLogger.error(pongStatus.message); 318 | const error = new FluentLoggerError.HandshakeError(pongStatus.message); 319 | this._handleEvent('error', error); 320 | this._disconnect(); 321 | return; 322 | } 323 | this._status = 'established'; 324 | this.internalLogger.info('Established'); 325 | callback(); 326 | }); 327 | }); 328 | }); 329 | } 330 | 331 | _flushSendQueue() { 332 | if (this._flushingSendQueue) 333 | return; 334 | 335 | this._flushingSendQueue = true; 336 | this._waitToWrite(); 337 | } 338 | 339 | _waitToWrite() { 340 | if (!this._socket) { 341 | this._flushingSendQueue = false; 342 | return; 343 | } 344 | 345 | if (this._socket.writable) { 346 | if (this._eventMode === 'Message') { 347 | this._doFlushSendQueue(); 348 | } else { 349 | if (this._sendQueueSize >= this._sendQueueSizeLimit) { 350 | this._flushSendQueueTimeoutId && clearTimeout(this._flushSendQueueTimeoutId); 351 | this._doFlushSendQueue(); 352 | } else { 353 | this._flushSendQueueTimeoutId && clearTimeout(this._flushSendQueueTimeoutId); 354 | this._flushSendQueueTimeoutId = setTimeout(() => { 355 | if (!this._socket) { 356 | this._flushingSendQueue = false; 357 | return; 358 | } 359 | this._doFlushSendQueue(); 360 | }, this._flushInterval); 361 | } 362 | } 363 | } else { 364 | process.nextTick(() => { 365 | this._waitToWrite(); 366 | }); 367 | } 368 | } 369 | 370 | _doFlushSendQueue(timeoutId) { 371 | if (this._eventMode === 'Message') { 372 | const item = this._sendQueue.shift(); 373 | if (item === undefined) { 374 | this._flushingSendQueue = false; 375 | // nothing written; 376 | return; 377 | } 378 | this._doWrite(item.packet, item.options, timeoutId, [item.callback]); 379 | } else { 380 | if (this._sendQueue.size === 0) { 381 | this._flushingSendQueue = false; 382 | return; 383 | } 384 | const first = this._sendQueue.entries().next().value; 385 | const tag = first[0]; 386 | const eventEntryData = first[1]; 387 | let entries = Buffer.concat(eventEntryData.eventEntries, eventEntryData.size); 388 | let size = entries.length; 389 | this._sendQueue.delete(tag); 390 | if (this._compressed) { 391 | entries = zlib.gzipSync(entries); 392 | size = entries.length; 393 | } 394 | const options = { 395 | chunk: crypto.randomBytes(16).toString('base64'), 396 | size: size, 397 | compressed: this._compressed ? 'gzip' : 'text', 398 | eventEntryDataSize: eventEntryData.size 399 | }; 400 | const packet = msgpack.encode([tag, entries, options], { codec: codec }); 401 | this._doWrite(packet, options, timeoutId, eventEntryData.callbacks); 402 | } 403 | } 404 | 405 | _doWrite(packet, options, timeoutId, callbacks) { 406 | const sendPacketSize = (options && options.eventEntryDataSize) || this._sendQueueSize; 407 | this._socket.write(packet, () => { 408 | if (this.requireAckResponse) { 409 | this._socket.once('data', (data) => { 410 | timeoutId && clearTimeout(timeoutId); 411 | const response = msgpack.decode(data, { codec: codec }); 412 | if (response.ack !== options.chunk) { 413 | const error = new FluentLoggerError.ResponseError( 414 | 'ack in response and chunk id in sent data are different', 415 | { ack: response.ack, chunk: options.chunk } 416 | ); 417 | callbacks.forEach((callback) => { 418 | this._handleEvent('error', error, callback); 419 | }); 420 | } else { // no error on ack 421 | callbacks.forEach((callback) => { 422 | callback && callback(); 423 | }); 424 | } 425 | this._sendQueueSize -= sendPacketSize; 426 | process.nextTick(() => { 427 | this._waitToWrite(); 428 | }); 429 | }); 430 | timeoutId = setTimeout(() => { 431 | const error = new FluentLoggerError.ResponseTimeout('ack response timeout'); 432 | callbacks.forEach((callback) => { 433 | this._handleEvent('error', error, callback); 434 | }); 435 | }, this.ackResponseTimeout); 436 | } else { 437 | this._sendQueueSize -= sendPacketSize; 438 | callbacks.forEach((callback) => { 439 | callback && callback(); 440 | }); 441 | process.nextTick(() => { 442 | this._waitToWrite(); 443 | }); 444 | } 445 | }); 446 | } 447 | 448 | _handleEvent(signal, data, callback) { 449 | callback && callback(data); 450 | if (this._eventEmitter.listenerCount(signal) > 0) { 451 | this._eventEmitter.emit(signal, data); 452 | } 453 | } 454 | 455 | _setupErrorHandler(callback) { 456 | if (!this.reconnectInterval || !this.enableReconnect) { 457 | return; 458 | } 459 | this.on('error', (error) => { 460 | this._flushingSendQueue = false; 461 | this._status = null; 462 | this.internalLogger.error('Fluentd error', error); 463 | this.internalLogger.info('Fluentd will reconnect after ' + this.reconnectInterval / 1000 + ' seconds'); 464 | const timeoutId = setTimeout(() => { 465 | this.internalLogger.info('Fluentd is reconnecting...'); 466 | this._connect(() => { 467 | this._flushSendQueue(); 468 | this.internalLogger.info('Fluentd reconnection finished!!'); 469 | }); 470 | }, this.reconnectInterval); 471 | callback && callback(timeoutId); 472 | }); 473 | } 474 | 475 | _checkHelo(data) { 476 | // ['HELO', options(hash)] 477 | this.internalLogger.info('Checking HELO...'); 478 | const message = msgpack.decode(data); 479 | if (message.length !== 2) { 480 | return { succeeded: false, message: 'Invalid format for HELO message' }; 481 | } 482 | if (message[0] !== 'HELO') { 483 | return { succeeded: false, message: 'Invalid format for HELO message' }; 484 | } 485 | const options = message[1] || {}; 486 | this.sharedKeyNonce = options['nonce'] || ''; 487 | this.authentication = options['auth'] || ''; 488 | return { succeeded: true }; 489 | } 490 | 491 | _generatePing() { 492 | // [ 493 | // 'PING', 494 | // client_hostname, 495 | // shared_key_salt, 496 | // sha512_hex(sharedkey_salt + client_hostname + nonce + shared_key), 497 | // username || '', 498 | // sha512_hex(auth_salt + username + password) || '' 499 | // ] 500 | const sharedKeyHexdigest = crypto.createHash('sha512') 501 | .update(this.sharedKeySalt) 502 | .update(this.security.clientHostname) 503 | .update(this.sharedKeyNonce) 504 | .update(this.security.sharedKey) 505 | .digest('hex'); 506 | const ping = ['PING', this.security.clientHostname, this.sharedKeySalt, sharedKeyHexdigest]; 507 | if (Buffer.isBuffer(this.authentication) && this.authentication.length !== 0) { 508 | const passwordHexDigest = crypto.createHash('sha512') 509 | .update(this.authentication) 510 | .update(this.security.username || '') 511 | .update(this.security.password || '') 512 | .digest('hex'); 513 | ping.push(this.security.username, passwordHexDigest); 514 | } else { 515 | ping.push('', ''); 516 | } 517 | return msgpack.encode(ping); 518 | } 519 | 520 | _checkPong(data) { 521 | // [ 522 | // 'PONG', 523 | // bool(authentication result), 524 | // 'reason if authentication failed', 525 | // server_hostname, 526 | // sha512_hex(salt + server_hostname + nonce + sharedkey) 527 | // ] 528 | this.internalLogger.info('Checking PONG...'); 529 | const message = msgpack.decode(data); 530 | if (message.length !== 5) { 531 | return false; 532 | } 533 | if (message[0] !== 'PONG') { 534 | return { succeeded: false, message: 'Invalid format for PONG message' }; 535 | } 536 | const _pong = message[0]; 537 | const authResult = message[1]; 538 | const reason = message[2]; 539 | const hostname = message[3]; 540 | const sharedKeyHexdigest = message[4]; 541 | if (!authResult) { 542 | return { succeeded: false, message: 'Authentication failed: ' + reason }; 543 | } 544 | if (hostname === this.security.clientHostname) { 545 | return { succeeded: false, message: 'Same hostname between input and output: invalid configuration' }; 546 | } 547 | const clientsideHexdigest = crypto.createHash('sha512') 548 | .update(this.sharedKeySalt) 549 | .update(hostname) 550 | .update(this.sharedKeyNonce) 551 | .update(this.security.sharedKey) 552 | .digest('hex'); 553 | if (sharedKeyHexdigest !== clientsideHexdigest) { 554 | return { succeeded: false, message: 'Sharedkey mismatch' }; 555 | } 556 | return { succeeded: true }; 557 | } 558 | 559 | toStream(options) { 560 | if (typeof options === 'string') { 561 | options = {label: options}; 562 | } else { 563 | options = options || {}; 564 | } 565 | const label = options.label; 566 | if (!label) { 567 | throw new Error('label is needed'); 568 | } 569 | const defaultEncoding = options.encoding || 'UTF-8'; 570 | const writable = new stream.Writable(); 571 | let dataString = ''; 572 | writable._write = (chunk, encoding, callback) => { 573 | const dataArray = chunk.toString(defaultEncoding).split(/\n/); 574 | const next = () => { 575 | if (dataArray.length) { 576 | dataString += dataArray.shift(); 577 | } 578 | if (!dataArray.length) { 579 | process.nextTick(callback); 580 | return; 581 | } 582 | this.emit(label, { message: dataString }, (err) => { 583 | if (err) { 584 | this._handleEvent('error', err, callback); 585 | return; 586 | } 587 | dataString = ''; 588 | next(); 589 | }); 590 | }; 591 | next(); 592 | }; 593 | return writable; 594 | } 595 | } 596 | 597 | ['addListener', 'on', 'once', 'removeListener', 'removeAllListeners', 'setMaxListeners', 'getMaxListeners'].forEach((attr, i) => { 598 | FluentSender.prototype[attr] = function() { 599 | return this._eventEmitter[attr].apply(this._eventEmitter, Array.from(arguments)); 600 | }; 601 | }); 602 | 603 | module.exports = FluentSender; 604 | -------------------------------------------------------------------------------- /test/test.sender.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | /* globals describe, it */ 3 | /* eslint brace-style: ["error", "1tbs", {"allowSingleLine": true}] */ 4 | /* eslint no-unused-vars: ["error", {"args": "none"}] */ 5 | /* eslint node/no-unpublished-require: ["error", {"allowModules": ["async", "chai"]}] */ 6 | const expect = require('chai').expect; 7 | const FluentSender = require('../lib/sender'); 8 | const EventTime = require('../lib/event-time'); 9 | const runServer = require('../lib/testHelper').runServer; 10 | const stream = require('stream'); 11 | const async = require('async'); 12 | const EventEmitter = require('events').EventEmitter; 13 | const msgpack = require('msgpack-lite'); 14 | 15 | const codec = msgpack.createCodec(); 16 | codec.addExtPacker(0x00, EventTime, EventTime.pack); 17 | codec.addExtUnpacker(0x00, EventTime.unpack); 18 | 19 | let doTest = (tls) => { 20 | let serverOptions = {}; 21 | let clientOptions = {}; 22 | if (tls) { 23 | /* eslint-disable-next-line node/no-unpublished-require */ 24 | const selfsigned = require('selfsigned'); 25 | const attrs = [{ name: 'commonName', value: 'foo.com' }]; 26 | const pems = selfsigned.generate(attrs, { days: 365 }); 27 | serverOptions = { tls: true, key: pems.private, cert: pems.cert, ca: pems.cert }; 28 | clientOptions = { tls: true, tlsOptions: { rejectUnauthorized: false } }; 29 | } 30 | it('should throw error', (done) => { 31 | try { 32 | new FluentSender('debug', Object.assign({}, clientOptions, { eventMode: 'Unknown' })); 33 | } catch (e) { 34 | expect(e.message).to.be.equal('Unknown event mode: Unknown'); 35 | done(); 36 | } 37 | }); 38 | 39 | it('should send records', (done) => { 40 | runServer({}, serverOptions, (server, finish) => { 41 | const s1 = new FluentSender('debug', Object.assign({}, clientOptions, { port: server.port })); 42 | const emits = []; 43 | function emit(k) { 44 | emits.push((done) => { s1.emit('record', k, done); }); 45 | } 46 | for (let i = 0; i < 10; i++) { 47 | emit({ number: i }); 48 | } 49 | emits.push(() => { 50 | finish((data) => { 51 | expect(data.length).to.be.equal(10); 52 | for (let i = 0; i < 10; i++) { 53 | expect(data[i].tag).to.be.equal('debug.record'); 54 | expect(data[i].data.number).to.be.equal(i); 55 | } 56 | done(); 57 | }); 58 | }); 59 | async.series(emits); 60 | }); 61 | }); 62 | 63 | it('should emit connect event', (done) => { 64 | runServer({}, serverOptions, (server, finish) => { 65 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 66 | let called = false; 67 | s.on('connect', () => { 68 | called = true; 69 | }); 70 | s.emit({message: '1st message'}, () => { 71 | finish((data) => { 72 | expect(called).to.equal(true); 73 | done(); 74 | }); 75 | }); 76 | }); 77 | }); 78 | 79 | it('should raise error when connection fails', (done) => { 80 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 81 | host: 'localhost', 82 | port: 65535 83 | })); 84 | s.on('error', (err) => { 85 | expect(err.code).to.be.equal('ECONNREFUSED'); 86 | done(); 87 | }); 88 | s.emit('test connection error', { message: 'foobar' }); 89 | }); 90 | 91 | it('should log error when connection fails w/ internal logger', (done) => { 92 | const logger = { 93 | buffer: { 94 | info: [], 95 | error: [] 96 | }, 97 | info: function(message) { 98 | this.buffer.info.push(message); 99 | }, 100 | error: function(message) { 101 | this.buffer.error.push(message); 102 | } 103 | }; 104 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 105 | host: 'localhost', 106 | port: 65535, 107 | internalLogger: logger 108 | })); 109 | s._setupErrorHandler((timeoutId) => { 110 | expect(logger.buffer.info).to.have.lengthOf(1); 111 | expect(logger.buffer.info[0]).to.be.equal('Fluentd will reconnect after 600 seconds'); 112 | expect(logger.buffer.error).to.have.lengthOf(1); 113 | expect(logger.buffer.error[0]).to.be.equal('Fluentd error'); 114 | clearTimeout(timeoutId); 115 | done(); 116 | }); 117 | s.emit('test connection error', { message: 'foobar' }); 118 | }); 119 | 120 | 121 | it('should assure the sequence.', (done) => { 122 | runServer({}, serverOptions, (server, finish) => { 123 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 124 | s.emit('1st record', { message: '1st data' }); 125 | s.emit('2nd record', { message: '2nd data' }); 126 | s.end('last record', { message: 'last data' }, () => { 127 | finish((data) => { 128 | expect(data[0].tag).to.be.equal('debug.1st record'); 129 | expect(data[0].data.message).to.be.equal('1st data'); 130 | expect(data[1].tag).to.be.equal('debug.2nd record'); 131 | expect(data[1].data.message).to.be.equal('2nd data'); 132 | expect(data[2].tag).to.be.equal('debug.last record'); 133 | expect(data[2].data.message).to.be.equal('last data'); 134 | done(); 135 | }); 136 | }); 137 | }); 138 | }); 139 | 140 | it('should allow to emit with a custom timestamp', (done) => { 141 | runServer({}, serverOptions, (server, finish) => { 142 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 143 | const timestamp = new Date(2222, 12, 4); 144 | const timestamp_seconds_since_epoch = Math.floor(timestamp.getTime() / 1000); 145 | 146 | s.emit('1st record', { message: '1st data' }, timestamp, () => { 147 | finish((data) => { 148 | expect(data[0].time).to.be.equal(timestamp_seconds_since_epoch); 149 | done(); 150 | }); 151 | }); 152 | }); 153 | }); 154 | 155 | it('should allow to emit with a custom numeric timestamp', (done) => { 156 | runServer({}, serverOptions, (server, finish) => { 157 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 158 | const timestamp = Math.floor(new Date().getTime() / 1000); 159 | 160 | s.emit('1st record', { message: '1st data' }, timestamp, () => { 161 | finish((data) => { 162 | expect(data[0].time).to.be.equal(timestamp); 163 | done(); 164 | }); 165 | }); 166 | }); 167 | }); 168 | 169 | it('should allow to emit with a EventTime', (done) => { 170 | runServer({}, serverOptions, (server, finish) => { 171 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 172 | const eventTime = EventTime.now(); 173 | 174 | s.emit('1st record', { message: '1st data' }, eventTime, () => { 175 | finish((data) => { 176 | const decoded = EventTime.unpack(data[0].time.buffer); 177 | expect(JSON.stringify(decoded)).to.equal(JSON.stringify(eventTime)); 178 | done(); 179 | }); 180 | }); 181 | }); 182 | }); 183 | 184 | it('should resume the connection automatically and flush the queue', (done) => { 185 | const s = new FluentSender('debug', clientOptions); 186 | s.emit('1st record', { message: '1st data' }); 187 | s.on('error', (err) => { 188 | expect(err.code).to.be.equal('ECONNREFUSED'); 189 | runServer({}, serverOptions, (server, finish) => { 190 | s.port = server.port; 191 | s.emit('2nd record', { message: '2nd data' }); 192 | s.end('last record', { message: 'last data' }, () => { 193 | finish((data) => { 194 | expect(data[0].tag).to.be.equal('debug.1st record'); 195 | expect(data[0].data.message).to.be.equal('1st data'); 196 | expect(data[1].tag).to.be.equal('debug.2nd record'); 197 | expect(data[1].data.message).to.be.equal('2nd data'); 198 | expect(data[2].tag).to.be.equal('debug.last record'); 199 | expect(data[2].data.message).to.be.equal('last data'); 200 | done(); 201 | }); 202 | }); 203 | }); 204 | }); 205 | }); 206 | 207 | it('should reconnect when fluentd close the client socket suddenly', (done) => { 208 | runServer({}, serverOptions, (server, finish) => { 209 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 210 | s.emit('foo', 'bar', () => { 211 | // connected 212 | server.close(() => { 213 | // waiting for the server closing all client socket. 214 | (function waitForUnwritable() { 215 | if (!(s._socket && s._socket.writable)) { 216 | runServer({}, serverOptions, (_server2, finish) => { 217 | s.port = _server2.port; // in actuall case, s.port does not need to be updated. 218 | s.emit('bar', { message: 'hoge' }, () => { 219 | finish((data) => { 220 | expect(data[0].tag).to.be.equal('debug.bar'); 221 | expect(data[0].data.message).to.be.equal('hoge'); 222 | done(); 223 | }); 224 | }); 225 | }); 226 | } else { 227 | setTimeout(() => { 228 | waitForUnwritable(); 229 | }, 100); 230 | } 231 | })(); 232 | }); 233 | }); 234 | }); 235 | }); 236 | 237 | it('should send records with requireAckResponse', (done) => { 238 | runServer({requireAckResponse: true}, serverOptions, (server, finish) => { 239 | const s1 = new FluentSender('debug', Object.assign({}, clientOptions, { 240 | port: server.port, 241 | requireAckResponse: true 242 | })); 243 | const emits = []; 244 | function emit(k) { 245 | emits.push((done) => { s1.emit('record', k, done); }); 246 | } 247 | for (let i = 0; i < 10; i++) { 248 | emit({ number: i }); 249 | } 250 | emits.push(() => { 251 | finish((data) => { 252 | expect(data.length).to.be.equal(10); 253 | for (let i = 0; i < 10; i++) { 254 | expect(data[i].tag).to.be.equal('debug.record'); 255 | expect(data[i].data.number).to.be.equal(i); 256 | expect(data[i].options.chunk).to.be.equal(server.messages[i].options.chunk); 257 | } 258 | done(); 259 | }); 260 | }); 261 | async.series(emits); 262 | }); 263 | }); 264 | 265 | it('should send records ackResponseTimeout', (done) => { 266 | runServer({requireAckResponse: false }, serverOptions, (server, finish) => { 267 | const s1 = new FluentSender('debug', Object.assign({}, clientOptions, { 268 | port: server.port, 269 | requireAckResponse: false, 270 | ackResponseTimeout: 1000 271 | })); 272 | s1.on('response-timeout', (error) => { 273 | expect(error).to.be.equal('ack response timeout'); 274 | }); 275 | s1.emit('record', { number: 1 }); 276 | finish((data) => { 277 | expect(data.length).to.be.equal(1); 278 | done(); 279 | }); 280 | }); 281 | }); 282 | 283 | it('should set error handler', (done) => { 284 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 285 | reconnectInterval: 100 286 | })); 287 | expect(s._eventEmitter.listeners('error').length).to.be.equal(0); 288 | s._setupErrorHandler(); 289 | expect(s._eventEmitter.listeners('error').length).to.be.equal(1); 290 | done(); 291 | }); 292 | 293 | it('should flush queue on reconnect', (done) => { 294 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 295 | port: 43210, 296 | reconnectInterval: 20, 297 | internalLogger: { 298 | info: () => {}, 299 | error: () => {} 300 | } 301 | })); 302 | s._setupErrorHandler(); 303 | s.emit('record', { number: 1}); 304 | s.emit('record', { number: 2}); 305 | setTimeout(() => { 306 | runServer({requireAckResponse: true, port: 43210}, serverOptions, (server, finish) => { 307 | setTimeout(() => { 308 | finish((data) => { 309 | expect(data.length).to.be.equal(2); 310 | expect(data[0].data.number).to.be.equal(1); 311 | expect(data[1].data.number).to.be.equal(2); 312 | done(); 313 | }); 314 | }, 20); 315 | }); 316 | }, 100); 317 | }); 318 | 319 | it('should send messages with different tags correctly in PackedForward', (done) => { 320 | runServer({}, serverOptions, (server, finish) => { 321 | const s1 = new FluentSender('debug', Object.assign({}, clientOptions, { 322 | port: server.port, 323 | eventMode: 'PackedForward' 324 | })); 325 | const emits = []; 326 | const total = 4; 327 | function emit(messageData) { 328 | emits.push((asyncDone) => { 329 | if (messageData.number === total) { // end 330 | s1.emit(`multi-${messageData.number}`, { text: messageData.text}, asyncDone); // wait for send 331 | } else { 332 | s1.emit(`multi-${messageData.number}`, { text: messageData.text}); 333 | asyncDone(); // run immediately do not wait for ack 334 | } 335 | }); 336 | } 337 | for (let i = 0; i <= total; i++) { 338 | emit({ number: i, text: `This is text No ${i}` }); 339 | } 340 | emits.push(() => { 341 | finish((data) => { 342 | expect(data.length).to.be.equal(5); 343 | data.forEach((element, index) => { 344 | expect(element.tag).to.be.equal(`debug.multi-${index}`); 345 | expect(element.data.text).to.be.equal(`This is text No ${index}`); 346 | }); 347 | done(); 348 | }); 349 | }); 350 | async.series(emits); 351 | }); 352 | }); 353 | 354 | [ 355 | { 356 | name: 'tag and record', 357 | args: ['foo', { bar: 1 }], 358 | expect: { 359 | tag: 'debug.foo', 360 | data: { bar: 1 } 361 | } 362 | }, 363 | 364 | { 365 | name: 'tag, record and time', 366 | args: ['foo', { bar: 1 }, 12345], 367 | expect: { 368 | tag: 'debug.foo', 369 | data: { bar: 1 }, 370 | time: 12345 371 | } 372 | }, 373 | 374 | { 375 | name: 'tag, record and callback', 376 | args: ['foo', { bar: 1 }, function cb() { cb.called = true; }], 377 | expect: { 378 | tag: 'debug.foo', 379 | data: { bar: 1 } 380 | } 381 | }, 382 | 383 | { 384 | name: 'tag, record, time and callback', 385 | args: ['foo', { bar: 1 }, 12345, function cb() { cb.called = true; }], 386 | expect: { 387 | tag: 'debug.foo', 388 | data: { bar: 1 }, 389 | time: 12345 390 | } 391 | }, 392 | 393 | { 394 | name: 'record', 395 | args: [{ bar: 1 }], 396 | expect: { 397 | tag: 'debug', 398 | data: { bar: 1 } 399 | } 400 | }, 401 | 402 | { 403 | name: 'record and time', 404 | args: [{ bar: 1 }, 12345], 405 | expect: { 406 | tag: 'debug', 407 | data: { bar: 1 }, 408 | time: 12345 409 | } 410 | }, 411 | 412 | { 413 | name: 'record and callback', 414 | args: [{ bar: 1 }, function cb() { cb.called = true; }], 415 | expect: { 416 | tag: 'debug', 417 | data: { bar: 1 } 418 | } 419 | }, 420 | 421 | { 422 | name: 'record, time and callback', 423 | args: [{ bar: 1 }, 12345, function cb() { cb.called = true; }], 424 | expect: { 425 | tag: 'debug', 426 | data: { bar: 1 }, 427 | time: 12345 428 | } 429 | }, 430 | 431 | { 432 | name: 'record and date object', 433 | args: [{ bar: 1 }, new Date(1384434467952)], 434 | expect: { 435 | tag: 'debug', 436 | data: { bar: 1 }, 437 | time: 1384434467 438 | } 439 | } 440 | ].forEach((testCase) => { 441 | it('should send records with ' + testCase.name + ' arguments', (done) => { 442 | runServer({}, serverOptions, (server, finish) => { 443 | const s1 = new FluentSender('debug', Object.assign({}, clientOptions, { port: server.port })); 444 | s1.emit.apply(s1, testCase.args); 445 | 446 | finish((data) => { 447 | expect(data[0].tag).to.be.equal(testCase.expect.tag); 448 | expect(data[0].data).to.be.deep.equal(testCase.expect.data); 449 | if (testCase.expect.time) { 450 | expect(data[0].time).to.be.deep.equal(testCase.expect.time); 451 | } 452 | 453 | testCase.args.forEach((arg) => { 454 | if (typeof arg === 'function') { 455 | expect(arg.called, 'callback must be called').to.be.true; 456 | } 457 | }); 458 | 459 | done(); 460 | }); 461 | 462 | }); 463 | }); 464 | }); 465 | 466 | [ 467 | { 468 | name: 'tag and record', 469 | args: ['foo', { bar: 1 }], 470 | expect: { 471 | tag: 'foo', 472 | data: { bar: 1 } 473 | } 474 | }, 475 | 476 | { 477 | name: 'tag, record and time', 478 | args: ['foo', { bar: 1 }, 12345], 479 | expect: { 480 | tag: 'foo', 481 | data: { bar: 1 }, 482 | time: 12345 483 | } 484 | }, 485 | 486 | { 487 | name: 'tag, record and callback', 488 | args: ['foo', { bar: 1 }, function cb() { cb.called = true; }], 489 | expect: { 490 | tag: 'foo', 491 | data: { bar: 1 } 492 | } 493 | }, 494 | 495 | { 496 | name: 'tag, record, time and callback', 497 | args: ['foo', { bar: 1 }, 12345, function cb() { cb.called = true; }], 498 | expect: { 499 | tag: 'foo', 500 | data: { bar: 1 }, 501 | time: 12345 502 | } 503 | } 504 | ].forEach((testCase) => { 505 | it('should send records with ' + testCase.name + ' arguments without a default tag', (done) => { 506 | runServer({}, serverOptions, (server, finish) => { 507 | const s1 = new FluentSender(null, Object.assign({}, clientOptions, { port: server.port })); 508 | s1.emit.apply(s1, testCase.args); 509 | 510 | finish((data) => { 511 | expect(data[0].tag).to.be.equal(testCase.expect.tag); 512 | expect(data[0].data).to.be.deep.equal(testCase.expect.data); 513 | if (testCase.expect.time) { 514 | expect(data[0].time).to.be.deep.equal(testCase.expect.time); 515 | } 516 | 517 | testCase.args.forEach((arg) => { 518 | if (typeof arg === 'function') { 519 | expect(arg.called, 'callback must be called').to.be.true; 520 | } 521 | }); 522 | 523 | done(); 524 | }); 525 | 526 | }); 527 | }); 528 | }); 529 | 530 | [ 531 | { 532 | name: 'record', 533 | args: [{ bar: 1 }] 534 | }, 535 | 536 | { 537 | name: 'record and time', 538 | args: [{ bar: 1 }, 12345] 539 | }, 540 | 541 | { 542 | name: 'record and callback', 543 | args: [{ bar: 1 }, function cb() { cb.called = true; }] 544 | }, 545 | 546 | { 547 | name: 'record, time and callback', 548 | args: [{ bar: 1 }, 12345, function cb() { cb.called = true; }] 549 | }, 550 | 551 | { 552 | name: 'record and date object', 553 | args: [{ bar: 1 }, new Date(1384434467952)] 554 | } 555 | ].forEach((testCase) => { 556 | it('should not send records with ' + testCase.name + ' arguments without a default tag', (done) => { 557 | runServer({}, serverOptions, (server, finish) => { 558 | const s1 = new FluentSender(null, Object.assign({}, clientOptions, { port: server.port })); 559 | s1.on('error', (error) => { 560 | expect(error.name).to.be.equal('MissingTagError'); 561 | }); 562 | s1.emit.apply(s1, testCase.args); 563 | 564 | finish((data) => { 565 | expect(data.length).to.be.equal(0); 566 | testCase.args.forEach((arg) => { 567 | if (typeof arg === 'function') { 568 | expect(arg.called, 'callback must be called').to.be.true; 569 | } 570 | }); 571 | 572 | done(); 573 | }); 574 | 575 | }); 576 | }); 577 | }); 578 | 579 | it('should not send records is not object', (done) => { 580 | runServer({}, serverOptions, (server, finish) => { 581 | const s1 = new FluentSender(null, Object.assign({}, clientOptions, { port: server.port })); 582 | s1.on('error', (error) => { 583 | expect(error.name).to.be.equal('DataTypeError'); 584 | }); 585 | s1.emit('label', 'string'); 586 | finish((data) => { 587 | expect(data.length).to.be.equal(0); 588 | }); 589 | done(); 590 | }); 591 | }); 592 | 593 | it('should set max listeners', (done) => { 594 | const s = new FluentSender('debug', clientOptions); 595 | if (EventEmitter.prototype.getMaxListeners) { 596 | expect(s.getMaxListeners()).to.be.equal(10); 597 | } 598 | s.setMaxListeners(100); 599 | if (EventEmitter.prototype.getMaxListeners) { 600 | expect(s.getMaxListeners()).to.be.equal(100); 601 | } else { 602 | expect(s._eventEmitter._maxListeners).to.be.equal(100); 603 | } 604 | done(); 605 | }); 606 | 607 | it('should set sendQueueSizeLimit', (done) => { 608 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 609 | sendQueueSizeLimit: 1000, 610 | eventMode: 'PackedForward' 611 | })); 612 | 613 | expect(s._sendQueueSizeLimit).to.be.equal(1000); 614 | done(); 615 | }); 616 | 617 | // Internal behavior test. 618 | it('should not flush queue if existing connection is unavailable.', (done) => { 619 | runServer({}, serverOptions, (server, finish) => { 620 | const s = new FluentSender('debug', Object.assign({}, clientOptions, {port: server.port})); 621 | s.emit('1st record', { message: '1st data' }, () => { 622 | s._disconnect(); 623 | s.emit('2nd record', { message: '2nd data' }, () => { 624 | finish((data) => { 625 | expect(data[0].tag).to.be.equal('debug.1st record'); 626 | expect(data[0].data.message).to.be.equal('1st data'); 627 | expect(data[1].tag).to.be.equal('debug.2nd record'); 628 | expect(data[1].data.message).to.be.equal('2nd data'); 629 | done(); 630 | }); 631 | }); 632 | }); 633 | }); 634 | }); 635 | 636 | it('should write stream.', (done) => { 637 | runServer({}, serverOptions, (server, finish) => { 638 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { port: server.port })); 639 | const ss = s.toStream('record'); 640 | const pt = new stream.PassThrough(); 641 | pt.pipe(ss); 642 | pt.push('data1\n'); 643 | pt.push('data2\ndata'); 644 | pt.push('3\ndata4\n'); 645 | pt.end(); 646 | ss.on('finish', () => { 647 | s.end(null, null, () => { 648 | finish((data) => { 649 | expect(data[0].data.message).to.be.equal('data1'); 650 | expect(data[1].data.message).to.be.equal('data2'); 651 | expect(data[2].data.message).to.be.equal('data3'); 652 | expect(data[3].data.message).to.be.equal('data4'); 653 | done(); 654 | }); 655 | }); 656 | }); 657 | }); 658 | }); 659 | 660 | it('should flush queue on reconnect for stream', (done) => { 661 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 662 | port: 43210, 663 | reconnectInterval: 20, 664 | internalLogger: { 665 | info: () => {}, 666 | error: () => {} 667 | } 668 | })); 669 | s._setupErrorHandler(); 670 | const ss = s.toStream('record'); 671 | const pt = new stream.PassThrough(); 672 | pt.pipe(ss); 673 | pt.push('data1\n'); 674 | pt.push('data2\ndata'); 675 | pt.push('3\ndata4\n'); 676 | pt.end(); 677 | setTimeout(() => { 678 | runServer({port: 43210}, serverOptions, (server, finish) => { 679 | setTimeout(() => { 680 | finish((data) => { 681 | expect(data[0].data.message).to.be.equal('data1'); 682 | expect(data[1].data.message).to.be.equal('data2'); 683 | expect(data[2].data.message).to.be.equal('data3'); 684 | expect(data[3].data.message).to.be.equal('data4'); 685 | done(); 686 | }); 687 | }, 20); 688 | }); 689 | }, 100); 690 | }); 691 | 692 | it('should process messages step by step on requireAckResponse=true', (done) => { 693 | runServer({ requireAckResponse: true }, serverOptions, (server, finish) => { 694 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 695 | port: server.port, 696 | timeout: 3.0, 697 | reconnectInterval: 600000, 698 | requireAckResponse: true 699 | })); 700 | const errors = []; 701 | s.on('error', (err) => { 702 | errors.push(count + ': ' + err); 703 | }); 704 | const maxCount = 20; 705 | let count = 0; 706 | const sendMessage = function() { 707 | const time = Math.round(Date.now() / 1000); 708 | const data = { 709 | count: count 710 | }; 711 | s.emit('test', data, time); 712 | count++; 713 | if (count > maxCount) { 714 | clearInterval(timer); 715 | finish(); 716 | expect(errors.join('\n')).to.be.equal(''); 717 | done(); 718 | } 719 | }; 720 | let timer = setInterval(sendMessage, 10); 721 | }); 722 | }); 723 | 724 | it('should process entries when using PackedForward Mode', (done) => { 725 | runServer({}, serverOptions, (server, finish) => { 726 | const loggerOptions = { 727 | port: server.port, 728 | eventMode: 'PackedForward', 729 | internalLogger: { 730 | info: () => {}, 731 | error: () => {} 732 | } 733 | }; 734 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 735 | s.emit('test', { message: 'This is test 0' }); 736 | s.end('test', { message: 'This is test 1' }); 737 | setTimeout(() => { 738 | finish((data) => { 739 | expect(data.length).to.be.equal(2); 740 | expect(data[0].tag).to.be.equal('debug.test'); 741 | expect(data[0].data.message).to.be.equal('This is test 0'); 742 | expect(data[1].tag).to.be.equal('debug.test'); 743 | expect(data[1].data.message).to.be.equal('This is test 1'); 744 | expect(s._sendQueueSize).to.be.equal(0); 745 | done(); 746 | }); 747 | }, 200); 748 | }); 749 | }); 750 | 751 | it('should compress entries when using CompressedPackedForward Mode', (done) => { 752 | runServer({}, serverOptions, (server, finish) => { 753 | const loggerOptions = { 754 | port: server.port, 755 | eventMode: 'CompressedPackedForward', 756 | internalLogger: { 757 | info: () => {}, 758 | error: () => {} 759 | } 760 | }; 761 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 762 | s.emit('test', { message: 'This is test 0' }); 763 | s.emit('test', { message: 'This is test 1' }); 764 | setTimeout(() => { 765 | finish((data) => { 766 | expect(data.length).to.be.equal(2); 767 | expect(data[0].tag).to.be.equal('debug.test'); 768 | expect(data[0].data.message).to.be.equal('This is test 0'); 769 | expect(data[1].tag).to.be.equal('debug.test'); 770 | expect(data[1].data.message).to.be.equal('This is test 1'); 771 | expect(s._sendQueueSize).to.be.equal(0); 772 | done(); 773 | }); 774 | }, 200); 775 | }); 776 | }); 777 | 778 | it('should process handshake sahred key', (done) => { 779 | const sharedKey = 'sharedkey'; 780 | const options = { 781 | security: { 782 | serverHostname: 'server.example.com', 783 | sharedKey: sharedKey 784 | } 785 | }; 786 | runServer(options, serverOptions, (server, finish) => { 787 | const loggerOptions = { 788 | port: server.port, 789 | security: { 790 | clientHostname: 'client.example.com', 791 | sharedKey: sharedKey 792 | }, 793 | internalLogger: { 794 | info: () => {}, 795 | error: () => {} 796 | } 797 | }; 798 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 799 | s.emit('test', { message: 'This is test 0' }); 800 | s.emit('test', { message: 'This is test 1' }); 801 | finish((data) => { 802 | expect(data.length).to.be.equal(2); 803 | expect(data[0].tag).to.be.equal('debug.test'); 804 | expect(data[0].data.message).to.be.equal('This is test 0'); 805 | expect(data[1].tag).to.be.equal('debug.test'); 806 | expect(data[1].data.message).to.be.equal('This is test 1'); 807 | done(); 808 | }); 809 | }); 810 | }); 811 | 812 | it('should process handshake sahred key mismatch', (done) => { 813 | const sharedKey = 'sharedkey'; 814 | const options = { 815 | security: { 816 | serverHostname: 'server.example.com', 817 | sharedKey: sharedKey 818 | } 819 | }; 820 | runServer(options, serverOptions, (server, finish) => { 821 | const loggerOptions = { 822 | port: server.port, 823 | security: { 824 | clientHostname: 'client.example.com', 825 | sharedKey: 'wrongSharedKey' 826 | }, 827 | internalLogger: { 828 | info: () => {}, 829 | error: () => {} 830 | } 831 | }; 832 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 833 | s.on('error', (error) => { 834 | expect(error.message).to.be.equal('Authentication failed: shared key mismatch'); 835 | }); 836 | s.emit('test', { message: 'This is test 0' }); 837 | finish((data) => { 838 | expect(data.length).to.be.equal(0); 839 | done(); 840 | }); 841 | }); 842 | }); 843 | 844 | it('should process handshake user based authentication', (done) => { 845 | const sharedKey = 'sharedkey'; 846 | const options = { 847 | security: { 848 | serverHostname: 'server.example.com', 849 | sharedKey: sharedKey, 850 | username: 'fluentd', 851 | password: 'password' 852 | } 853 | }; 854 | runServer(options, serverOptions, (server, finish) => { 855 | const loggerOptions = { 856 | port: server.port, 857 | security: { 858 | clientHostname: 'client.example.com', 859 | sharedKey: sharedKey, 860 | username: 'fluentd', 861 | password: 'password' 862 | }, 863 | internalLogger: { 864 | info: () => {}, 865 | error: () => {} 866 | } 867 | }; 868 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 869 | s.emit('test', { message: 'This is test 0' }); 870 | s.emit('test', { message: 'This is test 1' }); 871 | finish((data) => { 872 | expect(data.length).to.be.equal(2); 873 | expect(data[0].tag).to.be.equal('debug.test'); 874 | expect(data[0].data.message).to.be.equal('This is test 0'); 875 | expect(data[1].tag).to.be.equal('debug.test'); 876 | expect(data[1].data.message).to.be.equal('This is test 1'); 877 | done(); 878 | }); 879 | }); 880 | }); 881 | 882 | it('should process handshake user based authentication failed', (done) => { 883 | const sharedKey = 'sharedkey'; 884 | const options = { 885 | security: { 886 | serverHostname: 'server.example.com', 887 | sharedKey: sharedKey, 888 | username: 'fluentd', 889 | password: 'password' 890 | } 891 | }; 892 | runServer(options, serverOptions, (server, finish) => { 893 | const loggerOptions = { 894 | port: server.port, 895 | security: { 896 | clientHostname: 'client.example.com', 897 | sharedKey: sharedKey, 898 | username: 'fluentd', 899 | password: 'wrongPassword' 900 | }, 901 | internalLogger: { 902 | info: () => {}, 903 | error: () => {} 904 | } 905 | }; 906 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 907 | s.on('error', (error) => { 908 | expect(error.message).to.be.equal('Authentication failed: username/password mismatch'); 909 | }); 910 | s.emit('test', { message: 'This is test 0' }); 911 | finish((data) => { 912 | expect(data.length).to.be.equal(0); 913 | done(); 914 | }); 915 | }); 916 | }); 917 | 918 | it('should process handshake failed', (done) => { 919 | const sharedKey = 'sharedkey'; 920 | const options = { 921 | security: { 922 | serverHostname: 'server.example.com', 923 | sharedKey: sharedKey 924 | }, 925 | checkPing: (data) => { return { succeeded: false, reason: 'reason', sharedKeySalt: null }; } 926 | }; 927 | runServer(options, serverOptions, (server, finish) => { 928 | const loggerOptions = { 929 | port: server.port, 930 | security: { 931 | clientHostname: 'client.example.com', 932 | sharedKey: sharedKey 933 | }, 934 | internalLogger: { 935 | info: () => {}, 936 | error: () => {} 937 | } 938 | }; 939 | const s = new FluentSender('debug', Object.assign({}, clientOptions, loggerOptions)); 940 | s.on('error', (err) => { 941 | expect(err.message).to.be.equal('Authentication failed: reason'); 942 | }); 943 | s.emit('test', { message: 'This is test 0' }); 944 | finish((data) => { 945 | expect(data.length).to.be.equal(0); 946 | done(); 947 | }); 948 | }); 949 | }); 950 | 951 | it('should limit messages stored in queue if server is not available', (done) => { 952 | runServer({}, serverOptions, (server, finish) => { 953 | finish(() => { 954 | const s = new FluentSender('debug', Object.assign({}, clientOptions, { 955 | port: server.port, 956 | messageQueueSizeLimit: 3 957 | })); 958 | s.emit('message1', {}); 959 | s.emit('message2', {}); 960 | s.emit('message3', {}); 961 | s.emit('message4', {}); 962 | expect(s._sendQueue.length).to.be.equal(3); 963 | expect(s._sendQueue[0].tag).to.be.equal('debug.message2'); 964 | expect(s._sendQueue[1].tag).to.be.equal('debug.message3'); 965 | expect(s._sendQueue[2].tag).to.be.equal('debug.message4'); 966 | done(); 967 | }); 968 | }); 969 | }); 970 | }; 971 | 972 | describe('FluentSender', () => { 973 | doTest(); 974 | }); 975 | 976 | describe('FluentSenderWithTLS', () => { 977 | doTest(true); 978 | }); 979 | --------------------------------------------------------------------------------