├── .gitignore ├── CONTRIBUTION.md ├── LICENSE ├── README.md ├── docs.mli ├── index.js ├── package.json └── test ├── index.js ├── is-disable.js ├── kafka.js └── lib ├── kafka-message.js ├── kafka-rest-proxy-server.js └── kafka-server.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled source # 2 | ################### 3 | *.com 4 | *.class 5 | *.dll 6 | *.exe 7 | *.a 8 | *.o 9 | *.so 10 | *.node 11 | bin/* 12 | 13 | # Node Waf Byproducts # 14 | ####################### 15 | .lock-wscript 16 | build/ 17 | autom4te.cache/ 18 | 19 | # Node Modules # 20 | ################ 21 | # Better to let npm install these from the package.json defintion 22 | # rather than maintain this manually 23 | node_modules/ 24 | 25 | # Packages # 26 | ############ 27 | # it's better to unpack these files and commit the raw source 28 | # git has its own built in compression methods 29 | *.7z 30 | *.dmg 31 | *.gz 32 | *.iso 33 | *.jar 34 | *.rar 35 | *.tar 36 | *.zip 37 | 38 | # Logs and databases # 39 | ###################### 40 | *.log 41 | dump.rdb 42 | *.js.tap 43 | *.coffee.tap 44 | 45 | # OS generated files # 46 | ###################### 47 | .DS_Store? 48 | .DS_Store 49 | ehthumbs.db 50 | Icon? 51 | Thumbs.db 52 | coverage 53 | 54 | # Text Editor Byproducts # 55 | ########################## 56 | *.swp 57 | *.swo 58 | .idea/ 59 | 60 | *.pyc 61 | -------------------------------------------------------------------------------- /CONTRIBUTION.md: -------------------------------------------------------------------------------- 1 | # Want to contribute? 2 | 3 | Great! That's why this is an open source project. We use this project in our infrastructure at Uber, and we hope that it's useful to others as well. 4 | 5 | Before you get started, here are some suggestions: 6 | 7 | - Check open issues for what you want. 8 | - If there is an open issue, comment on it. Otherwise open an issue describing your bug or feature with use cases. 9 | - Before undertaking a major change, please discuss this on the issue. We'd hate to see you spend a lot of time working on something that conflicts with other goals or requirements that might not be obvious. 10 | - Write code to fix the problem, then open a pull request with tests and documentation. 11 | - The pull requests gets reviewed and then merged assuming there are no problems. 12 | - A new release version gets cut. 13 | 14 | ## Licencing 15 | 16 | - Every file must have a licence block at the top. This is enforced using `uber-licence` 17 | - If you contribute to a file in this project and are not an Uber employee, then you should 18 | add your name to the copyright section of the licence file. 19 | - Work that you contribute must be your own. 20 | 21 | ## Releases 22 | 23 | Declaring formal releases requires peer review. 24 | 25 | - A reviewer of a pull request should recommend a new version number (patch, minor or major). 26 | - Once your change is merged feel free to bump the version as recommended by the reviewer. 27 | - A new version number should not be cut without peer review unless done by the project maintainer. 28 | 29 | ### Cutting a new version 30 | 31 | - Get your branch merged on master 32 | - Run `npm version major` or `npm version minor` or `npm version patch` 33 | - `git push origin master --tags` 34 | - If you are a project owner, then `npm publish` 35 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2014 Uber Technologies, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # kafka-logger 2 | 3 | A [kafka](http://kafka.apache.org/) transport for [winston](https://github.com/winstonjs/winston) 4 | 5 | ## Usage 6 | 7 | ```js 8 | var winston = require('winston'); 9 | 10 | winston.transports.Kafka = require('kafka-logger'); 11 | 12 | winston.add(winston.transports.Kafka, options); 13 | ``` 14 | 15 | The Kafka transport currently uses [node-kafka-rest-client](). The Kafka transport takes the following options: 16 | 17 | * `topic` - The Kafka topic to publish to. 18 | * `proxyHost` - The Kafka REST Proxy host to publish to. 19 | * `proxyPort` - The Kafka REST Proxy port to publish to. 20 | * `properties` - Top-level properties that should be added to the JSON object published to the kafka topic; useful if multiple processes use the same topic 21 | * `dateFormats` - An object of date formats to use; keys are the names of the keys the format should be added to, values are the names of the formats (useful for cross-language usage of the logs to reduce transforms on the consumers). These formats are: `epoch` (time in sec since Jan 1, 1970), `jsepoch` (time in ms since Jan 1, 1970), `pyepoch` (time in sec since Jan 1, 1970, but floating point with ms resolution), `iso` (ISO datestring format) 22 | 23 | ## Install 24 | 25 | ```sh 26 | npm install winston kafka-logger 27 | ``` 28 | 29 | ## Testing 30 | 31 | ```sh 32 | npm test 33 | ``` 34 | 35 | There is a `kafka.js` that will talk to kafka if it is running and just 36 | gets skipped if its not running. 37 | 38 | To run kafka run zookeeper & kafka with `npm run start-zk` and 39 | `npm run start-kafka` 40 | 41 | ## MIT Licenced 42 | -------------------------------------------------------------------------------- /docs.mli: -------------------------------------------------------------------------------- 1 | type WinstonTransport := { 2 | log: (level: String, msg: String, meta: Object, callback: Callback<>) 3 | } 4 | 5 | type KafkaClient := { 6 | produce: (topic: String, msg: Object, callback: Callback<>) => void 7 | } 8 | 9 | type Prober := { 10 | probe: (thunk: (Callback<>) => void) => void 11 | } 12 | 13 | winston-kafka := ({ 14 | topic: String, 15 | leafHost: String, 16 | leafPort: Number, 17 | properties?: Object, 18 | dateFormats?: Object, 19 | peerId?: Number, 20 | workerId?: Number, 21 | kafkaProber?: Prober, 22 | failureHandler?: (Error) => void, 23 | kafkaClient?: KafkaClient 24 | }) => WinstonTransport 25 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | /* jshint forin: false */ 22 | var util = require('util'); 23 | var Transport = require('winston-uber').Transport; 24 | var KafkaRestClient = require('kafka-rest-client'); 25 | var hostName = require('os').hostname(); 26 | var extend = require('xtend'); 27 | 28 | function KafkaLogger(options) { 29 | if (!(this instanceof KafkaLogger)) { 30 | return new KafkaLogger(options); 31 | } 32 | 33 | var self = this; 34 | 35 | function onConnect(err) { 36 | if (!err) { 37 | if (self.logger) { 38 | self.logger.info('KafkaClient connected to kafka'); 39 | } 40 | self.connected = true; 41 | if (!this.kafkaRestClient || (this.kafkaRestClient && this.kafkaRestClientConnected)) { 42 | self._flush(); 43 | } 44 | } else { 45 | if (self.logger) { 46 | self.logger.warn('KafkaClient could not connect to kafka'); 47 | } 48 | // connection failed, purge queue. 49 | self.initQueue.length = 0; 50 | } 51 | } 52 | 53 | 54 | function onKafkaRestClientConnect(err) { 55 | if (!err) { 56 | if (self.logger) { 57 | self.logger.info('KafkaRestClient connected to kafka'); 58 | } 59 | self.kafkaRestClientConnected = true; 60 | if (!this.kafkaClient || (this.kafkaClient && this.connected)) { 61 | self._flush(); 62 | } 63 | } else { 64 | if (self.logger) { 65 | self.logger.warn('KafkaRestClient could not connect to kafka'); 66 | } 67 | } 68 | } 69 | 70 | options = options || {}; 71 | 72 | Transport.call(this, options); 73 | this.topic = options.topic || 'unknown'; 74 | this.leafHost = options.leafHost; 75 | this.leafPort = options.leafPort; 76 | this.proxyHost = options.proxyHost || 'localhost'; 77 | if ('proxyPort' in options && options.proxyPort) { 78 | this.proxyPort = options.proxyPort; 79 | } 80 | this.logger = options.logger; 81 | this.properties = options.properties || {}; 82 | this.dateFormats = options.dateFormats || { isodate: 'iso' }; 83 | this.peerId = options.hasOwnProperty('peerId') ? options.peerId : -1; 84 | this.workerId = options.hasOwnProperty('workerId') ? options.workerId : -1; 85 | this.logTemplate = { 86 | host: hostName, 87 | level: "LOGLEVELHERE", 88 | msg: "MESSAGEHERE" 89 | }; 90 | for (var dateFormat in this.dateFormats) { 91 | this.logTemplate[dateFormat] = this.dateFormats[dateFormat]; 92 | } 93 | for (var property in this.properties) { 94 | this.logTemplate[property] = this.properties[property]; 95 | } 96 | 97 | this.kafkaProber = options.kafkaProber || null; 98 | this.failureHandler = options.failureHandler || null; 99 | this.kafkaClient = null; 100 | this.isDisabled = options.isDisabled || null; 101 | 102 | this.connected = true; 103 | this.kafkaRestClientConnected = false; 104 | this.initQueue = []; 105 | this.initTime = Date.now(); 106 | this.statsd = options.statsd || null; 107 | if (!this.kafkaRestClient) { 108 | if (this.proxyPort) { 109 | var kafkaRestClientOptions = { 110 | proxyHost: this.proxyHost, 111 | proxyPort: this.proxyPort, 112 | statsd: this.statsd 113 | }; 114 | if ('maxRetries' in options) { 115 | kafkaRestClientOptions['maxRetries'] = options.maxRetries; 116 | } 117 | if ('statsd' in options) { 118 | kafkaRestClientOptions['statsd'] = options.statsd; 119 | } 120 | if ('batching' in options) { 121 | kafkaRestClientOptions['batching'] = options.batching; 122 | } 123 | if (options.batchingWhitelist) { 124 | kafkaRestClientOptions['batchingWhitelist'] = options.batchingWhitelist; 125 | } 126 | this.kafkaRestClient = new KafkaRestClient(kafkaRestClientOptions); 127 | this.kafkaRestClient.connect(onKafkaRestClientConnect); 128 | } 129 | } else { 130 | this.kafkaRestClientConnected = true; 131 | } 132 | 133 | if (this.leafHost || this.leafPort) { 134 | throw new Error('[kafka-logger] kafka7 no longer supported'); 135 | } 136 | } 137 | 138 | util.inherits(KafkaLogger, Transport); 139 | 140 | KafkaLogger.prototype.name = 'KafkaLogger'; 141 | 142 | KafkaLogger.prototype.destroy = function destroy() { 143 | if (this.kafkaClient) { 144 | var producer = this.kafkaClient.get_producer(this.topic); 145 | 146 | if (producer && producer.connection && 147 | producer.connection.connection && 148 | producer.connection.connection._connection 149 | ) { 150 | producer.connection.connection._connection.destroy(); 151 | } 152 | } 153 | 154 | if (this.kafkaRestClient) { 155 | this.kafkaRestClient.close(); 156 | } 157 | }; 158 | 159 | KafkaLogger.prototype._flush = function _flush() { 160 | while (this.initQueue.length > 0) { 161 | var tuple = this.initQueue.shift(); 162 | produceMessage(this, tuple[0], tuple[1]); 163 | } 164 | }; 165 | 166 | KafkaLogger.prototype.log = function(level, msg, meta, callback) { 167 | var logMessage = extend(this.logTemplate); 168 | meta = meta || {}; 169 | 170 | var d = new Date(); 171 | var timestamp = d.getTime(); 172 | for (var dateFormat in this.dateFormats) { 173 | switch(this.dateFormats[dateFormat]) { 174 | case 'epoch': 175 | logMessage[dateFormat] = Math.floor(timestamp / 1000); 176 | break; 177 | case 'jsepoch': 178 | logMessage[dateFormat] = timestamp; 179 | break; 180 | case 'pyepoch': 181 | logMessage[dateFormat] = timestamp / 1000; 182 | break; 183 | case 'iso': 184 | /* falls through */ 185 | default: 186 | logMessage[dateFormat] = d.toISOString(); 187 | break; 188 | } 189 | } 190 | if (!logMessage.ts) { 191 | logMessage.ts = timestamp; 192 | } 193 | logMessage.level = level; 194 | logMessage.msg = msg; 195 | logMessage.fields = meta; 196 | 197 | if (((this.kafkaClient && !this.connected) || (this.kafkaRestClient && !this.kafkaRestClientConnected)) && Date.now() < this.initTime + 5000) { 198 | return this.initQueue.push([logMessage, callback]); 199 | } else if (this.connected && this.initQueue.length) { 200 | this._flush(); 201 | } 202 | 203 | produceMessage(this, logMessage, callback) 204 | }; 205 | 206 | function produceMessage(self, logMessage, callback) { 207 | if (self.isDisabled && self.isDisabled()) { 208 | if (callback) { 209 | process.nextTick(callback); 210 | } 211 | return; 212 | } 213 | 214 | var failureHandler = self.failureHandler 215 | 216 | if (self.kafkaClient) { 217 | if (self.kafkaProber) { 218 | var thunk = self.kafkaClient.produce.bind(self.kafkaClient, 219 | self.topic, logMessage); 220 | self.kafkaProber.probe(thunk, onFailure); 221 | } else { 222 | self.kafkaClient.produce(self.topic, logMessage, callback); 223 | } 224 | } 225 | 226 | if (self.kafkaRestClientConnected) { 227 | self.kafkaRestClient.produce(self.topic, JSON.stringify(logMessage), logMessage.ts); 228 | if (!self.kafkaClient && callback) { 229 | callback(); 230 | } 231 | } 232 | 233 | function onFailure(err) { 234 | if (failureHandler) { 235 | failureHandler(err, logMessage); 236 | } 237 | 238 | if (callback && typeof callback === 'function') { 239 | callback(err); 240 | } 241 | } 242 | } 243 | 244 | module.exports = KafkaLogger; 245 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kafka-logger", 3 | "version": "7.1.0", 4 | "description": "A kafka logger for winston", 5 | "main": "./index.js", 6 | "directories": { 7 | "test": "test" 8 | }, 9 | "scripts": { 10 | "cover": "istanbul cover --report none --print detail _mocha -- test --reporter tap", 11 | "view-cover": "istanbul report html && open ./coverage/index.html", 12 | "check-cover": "istanbul check-coverage", 13 | "test": "node ./test" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "git@github.com:uber/kafka-logger" 18 | }, 19 | "keywords": [ 20 | "winston", 21 | "kafka", 22 | "logger" 23 | ], 24 | "author": "David Ellis ", 25 | "collaborators": [ 26 | { 27 | "name": "David Ellis" 28 | }, 29 | { 30 | "name": "Raynos" 31 | } 32 | ], 33 | "license": "MIT", 34 | "bugs": { 35 | "url": "https://github.com/uber/kafka-logger/issues" 36 | }, 37 | "dependencies": { 38 | "kafka-rest-client": "^1.11.7", 39 | "winston-uber": "^1.0.0", 40 | "xtend": "^4.0.0" 41 | }, 42 | "devDependencies": { 43 | "istanbul": "~0.1.46", 44 | "tape": "^3.0.3", 45 | "uber-licence": "^1.5.1", 46 | "uber-nodesol": "^0.4.0", 47 | "uuid": "~1.4.1" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | require('./kafka.js'); 22 | require('./is-disable.js'); 23 | -------------------------------------------------------------------------------- /test/is-disable.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | var test = require('tape'); 22 | 23 | var KafkaLogger = require('../index.js'); 24 | var KafkaRestProxyServer = require('./lib/kafka-rest-proxy-server'); 25 | 26 | test('KafkaLogger writes to a real kafka server', function (assert) { 27 | var server = new KafkaRestProxyServer(4444, function (msg) { 28 | server.emit('msg', msg); 29 | }); 30 | 31 | server.start(); 32 | 33 | var isDisabledFlag = false; 34 | var logger = new KafkaLogger({ 35 | topic: 'testTopic0', 36 | proxyHost: 'localhost', 37 | proxyPort: 4444, 38 | blacklistMigrator: true, 39 | blacklistMigratorUrl: 'localhost:2222', 40 | isDisabled: function () { 41 | return isDisabledFlag; 42 | } 43 | }); 44 | 45 | server.once('msg', function (msg) { 46 | assert.ok(msg); 47 | 48 | isDisabledFlag = true; 49 | logger.log('error', 'some message', {}, function () { 50 | isDisabledFlag = false; 51 | 52 | server.removeListener('msg', failure); 53 | logger.log('error', 'some message'); 54 | server.once('msg', function (msg) { 55 | assert.ok(msg); 56 | 57 | logger.destroy(); 58 | server.stop(); 59 | assert.end(); 60 | }); 61 | }); 62 | 63 | server.on('msg', failure); 64 | 65 | function failure(msg) { 66 | assert.ok(false, 'wrote a message'); 67 | } 68 | }); 69 | 70 | logger.log('error', 'some message'); 71 | }); 72 | -------------------------------------------------------------------------------- /test/kafka.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | var test = require('tape'); 22 | 23 | var KafkaLogger = require('../index.js'); 24 | var KafkaServer = require('./lib/kafka-server.js'); 25 | 26 | var KafkaRestProxyServer = require('./lib/kafka-rest-proxy-server'); 27 | test('KafkaLogger double writes to a real kafka server', function (assert) { 28 | var restServer = new KafkaRestProxyServer(4444, function (msg) { 29 | assert.equal(msg.host ,require('os').hostname()); 30 | assert.equal(msg.level,'error'); 31 | assert.equal(msg.msg , 'some message'); 32 | }); 33 | 34 | restServer.start(); 35 | 36 | var logger = new KafkaLogger({ 37 | topic: 'testTopic0', 38 | proxyHost: 'localhost', 39 | proxyPort: 4444, 40 | blacklistMigrator: true, 41 | blacklistMigratorUrl: 'localhost:2222' 42 | }); 43 | 44 | logger.log('error', 'some message'); 45 | 46 | setTimeout(function stopRestServer() { 47 | restServer.stop(); 48 | logger.destroy(); 49 | assert.end(); 50 | }, 1000); 51 | 52 | }); 53 | -------------------------------------------------------------------------------- /test/lib/kafka-message.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | /*jshint maxparams: 8 */ 22 | 'use strict'; 23 | 24 | /* 25 | 0 1 2 3 26 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 27 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 28 | | REQUEST_LENGTH | 29 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 30 | | REQUEST_TYPE | TOPIC_LENGTH | 31 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 32 | / / 33 | / TOPIC / 34 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 35 | | PARTITION | 36 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 37 | 38 | 0 1 2 3 39 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 40 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 41 | / REQUEST HEADER (above) / 42 | / / 43 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 44 | | MESSAGES_LENGTH | 45 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 46 | / / 47 | / MESSAGES / 48 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 49 | 50 | 0 1 2 3 51 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 52 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 53 | | LENGTH | 54 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 55 | | MAGIC | COMPRESSION | CHECKSUM | 56 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 57 | | CHECKSUM (cont.) | PAYLOAD / 58 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ / 59 | / PAYLOAD (cont.) / 60 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 61 | */ 62 | 63 | module.exports = parseMessage; 64 | 65 | function parseMessage(msg) { 66 | var records = []; 67 | 68 | var recordOffset = 0; 69 | 70 | while (recordOffset < msg.length) { 71 | var length = msg.readUInt32BE(recordOffset); 72 | 73 | var type = msg.readUInt16BE(recordOffset + 4); 74 | var topicLength = 75 | msg.readUInt16BE(recordOffset + 6); 76 | var topic = msg.slice( 77 | recordOffset + 8, recordOffset + 8 + topicLength); 78 | var partition = 79 | msg.readUInt32BE(recordOffset + 8 + topicLength); 80 | 81 | var remainder = msg .slice( 82 | recordOffset + 12 + topicLength, 83 | recordOffset + 4 + length); 84 | 85 | var messagesLength = remainder.readUInt32BE(0); 86 | var messages = remainder.slice(4, 4 + messagesLength); 87 | var messageStructs = []; 88 | 89 | var messageOffset = 0; 90 | 91 | while (messageOffset < messagesLength) { 92 | var mLength = messages.readUInt32BE(messageOffset); 93 | var mMagic = messages 94 | .readUInt8(messageOffset + 4); 95 | var mCompression = messages 96 | .readUInt8(messageOffset + 5); 97 | var mChecksum = messages 98 | .readUInt32BE(messageOffset + 6); 99 | 100 | var mPayload = messages 101 | .slice(10, mLength + 4); 102 | 103 | messageStructs.push(new KafkaMessage( 104 | mLength, 105 | mMagic, 106 | mCompression, 107 | mChecksum, 108 | JSON.parse(String(mPayload)) 109 | )); 110 | messageOffset = messageOffset + (mLength + 4); 111 | } 112 | 113 | records.push(new KafkaRequest( 114 | length, 115 | type, 116 | topicLength, 117 | String(topic), 118 | partition, 119 | messagesLength, 120 | messageStructs 121 | )); 122 | recordOffset = recordOffset + length + 4; 123 | } 124 | 125 | return records; 126 | } 127 | 128 | // @constructor 129 | /* eslint-disable max-params */ 130 | function KafkaMessage( 131 | length, magic, compression, checksum, payload 132 | ) { 133 | this.length = length; 134 | this.magic = magic; 135 | this.compression = compression; 136 | this.checksum = checksum; 137 | this.payload = payload; 138 | } 139 | /* eslint-enable max-params */ 140 | 141 | // @constructor 142 | /* eslint-disable max-params */ 143 | function KafkaRequest( 144 | length, type, topicLength, topic, partition, mLength, msgs 145 | ) { 146 | this.length = length; 147 | this.type = type; 148 | this.topicLength = topicLength; 149 | this.topic = topic; 150 | this.partition = partition; 151 | this.messagesLength = mLength; 152 | this.messages = msgs; 153 | } 154 | /* eslint-enable max-params */ 155 | -------------------------------------------------------------------------------- /test/lib/kafka-rest-proxy-server.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | var util = require('util'); 23 | var http = require('http'); 24 | 25 | function KafkaRestProxyServer(port, assertion) { 26 | var self = this; 27 | self.listenPort = port; 28 | self.sockets = {}; 29 | self.nextSocketId = 0; 30 | self.assertion = assertion; 31 | http.Server.call(this, this.handle); 32 | } 33 | 34 | util.inherits(KafkaRestProxyServer, http.Server); 35 | 36 | KafkaRestProxyServer.prototype.handle = function handle(req, res) { 37 | var self = this; 38 | self.started = false; 39 | var messages = { 40 | 'localhost:4444': ['testTopic0', 'testTopic1', 'testTopic2', 'testTopic3'], 41 | 'localhost:5555': ['testTopic4', 'testTopic5', 'testTopic6', 'testTopic7'] 42 | }; 43 | if (req.method === 'GET') { 44 | res.end(JSON.stringify(messages)); 45 | } else if (req.method === 'POST') { 46 | 47 | var body = ''; 48 | req.on('data', function (data) { 49 | data = data.slice(8, data.length); 50 | body += data; 51 | }); 52 | req.on('end', function () { 53 | self.assertion(JSON.parse(body)) 54 | }); 55 | if (req.headers.timestamp) { 56 | res.end('{ version : 1, Status : SENT, message : {}}'); 57 | } else { 58 | res.end('Not found timestamp field in request header!'); 59 | } 60 | } 61 | }; 62 | 63 | KafkaRestProxyServer.prototype.start = function start() { 64 | var self = this; 65 | this.listen(self.listenPort, function started() { 66 | self.started = true; 67 | // console.log('Listening for HTTP requests on port %d.', 68 | // self.listenPort); 69 | }); 70 | this.on('connection', function connect(socket) { 71 | // Add a newly connected socket 72 | var socketId = self.nextSocketId++; 73 | self.sockets[socketId] = socket; 74 | // console.log('socket', socketId, 'opened'); 75 | 76 | // Remove the socket when it closes 77 | socket.on('close', function close() { 78 | // console.log('socket', socketId, 'closed'); 79 | delete self.sockets[socketId]; 80 | }); 81 | }); 82 | }; 83 | 84 | KafkaRestProxyServer.prototype.stop = function stop() { 85 | var self = this; 86 | self.started = false; 87 | self.close(function close() { 88 | // console.log('Stopped listening.'); 89 | }); 90 | // Destroy all open sockets 91 | for (var socketId in self.sockets) { 92 | if (self.sockets.hasOwnProperty(socketId)) { 93 | // console.log('socket', socketId, 'destroyed'); 94 | self.sockets[socketId].destroy(); 95 | } 96 | } 97 | }; 98 | 99 | module.exports = KafkaRestProxyServer; 100 | -------------------------------------------------------------------------------- /test/lib/kafka-server.js: -------------------------------------------------------------------------------- 1 | // Copyright (c) 2015 Uber Technologies, Inc. 2 | // 3 | // Permission is hereby granted, free of charge, to any person obtaining a copy 4 | // of this software and associated documentation files (the "Software"), to deal 5 | // in the Software without restriction, including without limitation the rights 6 | // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | // copies of the Software, and to permit persons to whom the Software is 8 | // furnished to do so, subject to the following conditions: 9 | // 10 | // The above copyright notice and this permission notice shall be included in 11 | // all copies or substantial portions of the Software. 12 | // 13 | // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | // THE SOFTWARE. 20 | 21 | 'use strict'; 22 | 23 | var parseMessage = require('./kafka-message.js'); 24 | 25 | module.exports = KafkaServer; 26 | 27 | function KafkaServer(listener) { 28 | var net = require('net'); 29 | 30 | var server = net.createServer(); 31 | server.on('connection', onConnection); 32 | var PORT = 10000 + Math.floor(Math.random() * 20000); 33 | 34 | server.listen(PORT); 35 | server.port = PORT; 36 | 37 | return server; 38 | 39 | function onConnection(socket) { 40 | socket.on('data', onMessage); 41 | 42 | function onMessage(buf) { 43 | try { 44 | var messages = parseMessage(buf); 45 | messages.forEach(function eachMessage(msg) { 46 | listener(null, msg); 47 | }); 48 | } catch (e) { 49 | listener(e); 50 | } 51 | } 52 | } 53 | } 54 | --------------------------------------------------------------------------------