├── .gitignore ├── AUTHORS ├── LICENSE ├── README.md ├── cwlogger.js ├── index.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | test.sh 3 | node_modules 4 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Damian Beresford 2 | Matteo Collina 3 | Tim Koopmans -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) nearForm Ltd. and Contributors 4 | All rights reserved. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | 24 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Stream interface to CloudWatch Logs 2 | 3 | Nice streaming interface to [CloudWatch Logs](http://docs.aws.amazon.com/AmazonCloudWatch/latest/DeveloperGuide/WhatIsCloudWatchLogs.html). Can be used from code or as a standalone Agent. 4 | 5 | ## Install 6 | 7 | ``` 8 | npm install cloudwatchlogs-stream --save 9 | ``` 10 | 11 | Or to install globally as an Agent: 12 | 13 | ``` 14 | npm install -g cloudwatchlogs-stream --save 15 | cloudwatchlogs -h 16 | ``` 17 | 18 | ## Usage 19 | 20 | See `cloudwatchlogs --help`. 21 | 22 | Example code: 23 | 24 | ``` 25 | var opts = { 26 | 'accessKeyId': 'ACCESS_KEY', 27 | 'secretAccessKey': 'SECRET_KEY', 28 | 'region': 'REGION', 29 | 'logGroupName': 'GROUP_NAME', 30 | 'logStreamName': 'STREAM_NAME', 31 | 'bulkIndex': 10, 32 | 'timeout': 20 // seconds 33 | }; 34 | var stream = new CloudWatchLogsStream(opts); 35 | something.pipe(stream) 36 | ``` 37 | 38 | Example command line: 39 | 40 | ``` 41 | tail -f my.log | cloudwatchlogs -a ACCESSKEY -s SECRET_KEY -r REGION -g GROUP_NAME -t STREAM_NAME -b 20 -o 20 42 | 43 | ``` 44 | 45 | -------------------------------------------------------------------------------- /cwlogger.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | 3 | // Global AWS config: http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-configuring.html 4 | // CloudWatchLogs API: http://docs.amazonaws.cn/en_us/AWSJavaScriptSDK/latest/AWS/CloudWatchLogs.html 5 | 6 | module.exports = function cwlogger (opts) { 7 | var cloudWatchLogs = new AWS.CloudWatchLogs(opts); 8 | 9 | function capitalize (str) { 10 | return str.charAt(0).toUpperCase() + str.slice(1); 11 | }; 12 | 13 | function createLogResourseFunction (name) { 14 | return function (params, cb) { 15 | cloudWatchLogs['create' + capitalize(name)](params, function (err, data) { 16 | if (err && err.code === 'ResourceAlreadyExistsException') { 17 | return cb(); 18 | } 19 | return cb(err, data); 20 | }); 21 | }; 22 | }; 23 | 24 | var _createLogGroup = createLogResourseFunction('logGroup'); 25 | var _createLogStream = createLogResourseFunction('logStream'); 26 | 27 | function createLogGroup (logGroupName, cb) { 28 | var params = { 29 | logGroupNamePrefix: logGroupName 30 | }; 31 | 32 | cloudWatchLogs.describeLogGroups(params, function (err, data) { 33 | if (err) return cb(err); 34 | if (data.logGroups && data.logGroups[0]) { 35 | return cb(); 36 | } 37 | params = { 38 | logGroupName: logGroupName 39 | }; 40 | _createLogGroup(params, cb); 41 | }); 42 | }; 43 | 44 | function createLogStream (logGroupName, logStreamName, cb) { 45 | var params = { 46 | logGroupName: logGroupName, 47 | logStreamNamePrefix: logStreamName 48 | }; 49 | cloudWatchLogs.describeLogStreams(params, function (err, data) { 50 | if (err) return cb(err); 51 | 52 | if (data.logStreams && data.logStreams[0]) { 53 | return cb(null, data.logStreams[0].uploadSequenceToken); 54 | } 55 | 56 | params = { 57 | logGroupName: logGroupName, 58 | logStreamName: logStreamName 59 | }; 60 | _createLogStream(params, function (err, data) { 61 | return cb(err, null); // sequence token null for new stream 62 | }); 63 | }); 64 | }; 65 | 66 | var re = /expected sequenceToken is: (\w+)/; 67 | function putLogEvents (params, cb) { 68 | cloudWatchLogs.putLogEvents(params, function (err, data) { 69 | if (err && err.code === 'InvalidSequenceTokenException') { 70 | var match = re.exec(err.message); 71 | if (!match) return cb(err); 72 | 73 | // retry once more.. 74 | var sequenceToken = match[1]; 75 | params.sequenceToken = sequenceToken; 76 | return cloudWatchLogs.putLogEvents(params, cb); 77 | } 78 | return cb(err, data); 79 | }); 80 | }; 81 | 82 | return { 83 | createLogGroup: createLogGroup, 84 | createLogStream: createLogStream, 85 | putLogEvents: putLogEvents 86 | }; 87 | }; 88 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var stream = require('stream'); 4 | var util = require('util'); 5 | var minimist = require('minimist'); 6 | var debug = require('debug')('cloudwatchlogs'); 7 | var cwlogger = require('./cwlogger.js'); 8 | var Deque = require('collections/deque'); 9 | 10 | function CloudWatchLogsStream (opts) { 11 | debug('opts', opts); 12 | stream.Writable.call(this); 13 | this.logGroupName = opts.logGroupName; 14 | this.logStreamName = opts.logStreamName; 15 | this.bulkIndex = opts.bulkIndex || null; 16 | this.sequenceToken = null; 17 | this.cwlogger = cwlogger(opts); 18 | this.firstMsg = null; 19 | this.queue = new Deque(); 20 | this.timeout = opts.timeout ? Number(opts.timeout) * 1000 : null; 21 | this.timer = null; 22 | this.startTimer(); 23 | 24 | var self = this; 25 | self.cwlogger.createLogGroup(self.logGroupName, function (err) { 26 | if (err) return self.emit('error', err); 27 | self.cwlogger.createLogStream(self.logGroupName, self.logStreamName, function (err, sequenceToken) { 28 | if (err) return self.emit('error', err); 29 | self.sequenceToken = sequenceToken; 30 | self._write = write; 31 | if (self.firstMsg) { 32 | self._write(self.firstMsg.chunk, self.firstMsg.encoding, self.firstMsg.done); 33 | } 34 | }); 35 | }); 36 | }; 37 | util.inherits(CloudWatchLogsStream, stream.Writable); 38 | 39 | function write (chunk, encoding, done) { 40 | var self = this; 41 | 42 | self.queue.push({ 43 | message: chunk.toString(), 44 | timestamp: new Date().getTime() 45 | }); 46 | 47 | // if we're not doing any batching, send now 48 | if (!self.bulkIndex && !self.timeout) return self.sendEvents(); 49 | 50 | // if we're bulk batching and we've hit the batch limit, send now 51 | if (self.bulkIndex && self.queue.length > self.bulkIndex) return self.sendEvents(done); 52 | 53 | done(); 54 | } 55 | 56 | CloudWatchLogsStream.prototype.sendEvents = function (cb) { 57 | var self = this; 58 | var events = self.queue.toArray(); 59 | self.clearTimer(); 60 | 61 | if (events.length === 0) { 62 | self.startTimer(); 63 | return setImmediate(cb); 64 | } 65 | 66 | var params = { 67 | logEvents: events, 68 | logGroupName: self.logGroupName, 69 | logStreamName: self.logStreamName, 70 | sequenceToken: self.sequenceToken 71 | }; 72 | 73 | this.cwlogger.putLogEvents(params, function (err, data) { 74 | if (err) { 75 | self.startTimer(); 76 | return self.emit('error', err); 77 | } 78 | 79 | self.queue.clear(); 80 | self.sequenceToken = data.nextSequenceToken; 81 | self.startTimer(); 82 | if (cb) cb(); 83 | }); 84 | }; 85 | 86 | CloudWatchLogsStream.prototype.startTimer = function () { 87 | if (!this.timeout) return; 88 | 89 | this.timer = setTimeout( 90 | (function (self) { 91 | return function () { 92 | self.sendEvents(); 93 | }; 94 | })(this), 95 | this.timeout 96 | ); 97 | }; 98 | 99 | CloudWatchLogsStream.prototype.clearTimer = function () { 100 | if (this.timer) clearTimeout(this.timer); 101 | }; 102 | 103 | CloudWatchLogsStream.prototype._write = function (chunk, encoding, done) { 104 | this.firstMsg = { 105 | chunk: chunk, 106 | encoding: encoding, 107 | done: done 108 | }; 109 | }; 110 | 111 | CloudWatchLogsStream.prototype.destroy = function (err) { 112 | this.clearTimer(); 113 | if (err) this.emit('error', err); 114 | this.emit('close'); 115 | }; 116 | 117 | function main () { 118 | var argv = minimist(process.argv.slice(2), { 119 | alias: { 120 | 'accessKeyId': 'a', 121 | 'secretAccessKey': 's', 122 | 'region': 'r', 123 | 'logGroupName': 'g', 124 | 'logStreamName': 't', 125 | 'bulkIndex': 'b', 126 | 'timeout': 'o' 127 | } 128 | }); 129 | 130 | if (!(argv.accesskey || argv.secretkey || argv.groupname || argv.streamname || argv.region)) { 131 | console.log('Usage: cloudwatchlogs [-a ACCESS_KEY] [-s SECRET_KEY]\n' + 132 | ' [-r REGION] [-g GROUP_NAME] [-t STREAM_NAME]\n' + 133 | ' [-b BULK_INDEX] [-o TIMEOUT]'); 134 | process.exit(1); 135 | } 136 | 137 | var str = new CloudWatchLogsStream(argv); 138 | process.stdin.pipe(str); 139 | } 140 | 141 | if (require.main === module) { 142 | main(); 143 | }; 144 | 145 | module.exports = CloudWatchLogsStream; 146 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudwatchlogs-stream", 3 | "version": "1.2.0", 4 | "repository": { 5 | "type": "git", 6 | "url": "https://github.com/nearform/cloudwatchlogs-stream" 7 | }, 8 | "description": "Stream interface to AWS CloudWatch Logs", 9 | "main": "index.js", 10 | "scripts": { 11 | "lint": "semistandard", 12 | "test": "echo \"Error: no test specified\" && exit 1" 13 | }, 14 | "pre-commit": [ 15 | "lint" 16 | ], 17 | "bin": { 18 | "cloudwatchlogs": "./index.js" 19 | }, 20 | "keywords": [ 21 | "aws", 22 | "cloudwatch", 23 | "amazon", 24 | "logging" 25 | ], 26 | "author": "Damian Beresford (npmjs.com/~damianberesford)", 27 | "license": "MIT", 28 | "dependencies": { 29 | "aws-sdk": "^2.1.21", 30 | "debug": "^2.1.3", 31 | "minimist": "^1.1.1", 32 | "collections": "^1.2.2" 33 | }, 34 | "devDependencies": { 35 | "pre-commit": "^1.0.6", 36 | "semistandard": "^4.1.3" 37 | } 38 | } 39 | --------------------------------------------------------------------------------