├── .gitignore ├── package.json ├── RELEASES.md ├── index.js ├── example.js ├── README.md ├── metrics.js └── logs.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | ._* 3 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudwatch-buddy", 3 | "version": "0.0.14", 4 | "description": "AWS CloudWatch metrics and logging for node apps", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/matthewdfuller/cloudwatch-buddy.git" 12 | }, 13 | "keywords": [ 14 | "aws", 15 | "cloudwatch", 16 | "logs", 17 | "logging", 18 | "metrics", 19 | "cloudwatchmetrics", 20 | "cloudwatchlogs", 21 | "amazon" 22 | ], 23 | "author": "matthewdfuller", 24 | "license": "MIT", 25 | "bugs": { 26 | "url": "https://github.com/matthewdfuller/cloudwatch-buddy/issues" 27 | }, 28 | "homepage": "https://github.com/matthewdfuller/cloudwatch-buddy", 29 | "dependencies": { 30 | "async": "^0.9.0", 31 | "aws-sdk": "^2.1.24" 32 | } 33 | } -------------------------------------------------------------------------------- /RELEASES.md: -------------------------------------------------------------------------------- 1 | ### 0.0.14 2 | * Fixed bug where no space was added between instance ID and the message for string-format logs 3 | * Fixed bug causing successive log streams to be skipped if the first failed 4 | 5 | ### 0.0.13 6 | * Fixed bug where unit was overwritten when multiple stats submitted in one group 7 | 8 | ### 0.0.12 9 | * Fixed bug causing minimum metric value to be set to zero when only one sample 10 | 11 | ### 0.0.11 12 | * If sample count is 0, change to 1 but keep values as 0 so the charts maintain a connected line 13 | 14 | ### 0.0.10 15 | * Fixed bug affecting metrics with dimensions submitted when sample count was 0 16 | 17 | ### 0.0.9 18 | * Adding ability to subfolder logs in S3 19 | 20 | ### 0.0.8 21 | * Removing spacing in JSON stringify so logs in S3 are stored as one line 22 | 23 | ### 0.0.7 24 | * Fixing a bug - accidentally published version where logs default timeout was 5 seconds 25 | 26 | ### 0.0.6 27 | * Added the ability to copy logs to S3 using s3Bucket and s3Prefix options DO NOT USE - contains bug (see above) 28 | 29 | ### 0.0.5 30 | * Added debug option to log to console all events 31 | 32 | ### 0.0.4 33 | * Added multi-dimension support 34 | 35 | ### 0.0.3 36 | * Fixing bug where instance id was hard-coded as a string rather than the actual instance ID 37 | 38 | ### 0.0.2 39 | * Fixing bug where metrics were not sent due to a return statement before the cloudwatch call 40 | 41 | ### 0.0.1 42 | * First release, beta -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var AWS = require('aws-sdk'); 2 | 3 | var metricsHelper = require(__dirname + '/metrics.js'); 4 | var logsHelper = require(__dirname + '/logs.js'); 5 | 6 | var CloudWatchBuddy = function(aws){ 7 | // Ensure a region is passed 8 | if (!aws || !aws.region) { 9 | throw new Error('Valid AWS config with a region is required'); 10 | } 11 | 12 | AWS.config.update(aws); 13 | 14 | return { 15 | metrics: function(options) { 16 | if (!options || !options.namespace) { 17 | throw new Error('Valid metrics config with namespace is required'); 18 | } 19 | 20 | if (options.namespace.indexOf('AWS/') === 0) { 21 | throw new Error('Namespace cannot begin with "AWS/"'); 22 | } 23 | 24 | var cloudwatch = new AWS.CloudWatch(aws); 25 | return new metricsHelper(cloudwatch, options); 26 | }, 27 | logs: function(options) { 28 | if (!options || !options.logGroup || typeof options.logGroup !== 'string') { 29 | throw new Error('Valid logs config with log group is required'); 30 | } 31 | 32 | if (options.logGroup.length > 512) { 33 | throw new Error('Log group name cannot be more than the AWS limit of 512 characters'); 34 | } 35 | 36 | var cloudwatchlogs = new AWS.CloudWatchLogs(aws); 37 | var svc = new AWS.MetadataService(aws); 38 | var s3 = new AWS.S3(aws); 39 | 40 | return new logsHelper(cloudwatchlogs, svc, s3, options); 41 | } 42 | } 43 | } 44 | 45 | module.exports = CloudWatchBuddy; -------------------------------------------------------------------------------- /example.js: -------------------------------------------------------------------------------- 1 | var CloudWatchBuddy = require(__dirname + '/index.js'); 2 | 3 | var awsOptions = { 4 | accessKeyId: '', 5 | secretAccessKey: '', 6 | sessionToken: '', 7 | region: 'us-east-1' 8 | }; 9 | 10 | var cwbMetricsOptions = { 11 | namespace: 'test-data', 12 | timeout: 60 13 | }; 14 | 15 | var cwbLogsOptions = { 16 | logGroup: 'test-data', 17 | timeout: 60, 18 | addInstanceId: false, 19 | addTimestamp: true, 20 | logFormat: 'json' //|| 'json' 21 | }; 22 | 23 | var cwbMetrics = new CloudWatchBuddy(awsOptions).metrics(cwbMetricsOptions); 24 | var cwbLogs = new CloudWatchBuddy(awsOptions).logs(cwbLogsOptions); 25 | 26 | cwbMetrics.increment('pageviews'); 27 | cwbMetrics.increment('pageviews'); 28 | cwbMetrics.increment('pageviews'); 29 | cwbMetrics.increment('pageviews'); 30 | cwbMetrics.increment('pageviews'); 31 | 32 | cwbMetrics.stat('loadtime', 10, 'Milliseconds'); 33 | cwbMetrics.stat('loadtime', 15, 'Milliseconds'); 34 | cwbMetrics.stat('loadtime', 7, 'Milliseconds'); 35 | cwbMetrics.stat('loadtime', 100, 'Milliseconds'); 36 | 37 | cwbMetrics.stat('pagesize', 10, 'Megabytes'); 38 | 39 | cwbMetrics.stat('serverload', 10, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 40 | cwbMetrics.stat('serverload', 10, 'Percent', {serverName:'web02.example.com',region:'us-east-1'}); 41 | 42 | cwbMetrics.stat('serverload', 20, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 43 | cwbMetrics.stat('serverload', 21, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 44 | cwbMetrics.stat('serverload', 24, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 45 | cwbMetrics.stat('serverload', 26, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 46 | cwbMetrics.stat('serverload', 21, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 47 | cwbMetrics.stat('serverload', 29, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 48 | cwbMetrics.stat('serverload', 24, 'Percent', {serverName:'web01.example.com',region:'us-east-1'}); 49 | cwbMetrics.stat('serverload', 30, 'Percent', {serverName:'web03.example.com',region:'us-east-1'}); 50 | cwbMetrics.stat('serverload', 30, 'Percent', {serverName:'web03.example.com',region:'us-east-1',serverId:'xyz'}); 51 | 52 | //cwbLogs.log('errors', 'Dual message'); 53 | //cwbLogs.log('signups', 'Some user'); 54 | 55 | var i=0; 56 | setInterval(function(){ 57 | i++; 58 | cwbMetrics.stat('pagesize', 10, 'Megabytes'); 59 | //cwbLogs.log('errors', 'Dual message: ' + i); 60 | //cwbLogs.log('signups', 'Some user: ' + i); 61 | },500); -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | cloudwatch-buddy 2 | =============== 3 | 4 | WARNING: This is currently in "beta" and has not been extensively tested. Please wait for a 1.0.0 release before using in production. Until then, feel free to create issues or pull requests. 5 | 6 | ## Description 7 | 8 | cloudwatch-buddy is a node module which easily allows for the sending of CloudWatch metrics, statistics, and logs to AWS CloudWatch. Using this module, you can easily replace StatsD with AWS CloudWatch. It gracefully hadles single increments (such as pageviews), as well as more complex measurements (page load times or size) with sum, minimums, and maximums with custom dimensions. Additionally, it can stream logs to AWS, dealing with the timestamps and formatting issues. It also manages periodic sending of logs and the AWS "next token" pattern, as well as uploading copies of the log files to an S3 bucket of your choosing. 9 | 10 | ## Features 11 | 12 | * Simple "increment" measurements 13 | * Statistics (sum, minimum, maximum) across a timeframe 14 | * Custom dimensions 15 | * Periodic uploading 16 | * Log ordering 17 | * Retrying of failed data submissions 18 | * Sending 0 as value so application has data 19 | * S3 bucket uploads 20 | 21 | ## Usage 22 | 23 | ``` 24 | npm install cloudwatch-buddy 25 | ``` 26 | 27 | ``` 28 | var CloudWatchBuddy = require('cloudwatch-buddy'); 29 | 30 | var awsOptions = { 31 | accessKeyId: 'access', // Optional. It is suggested to use IAM roles (see permissions below) 32 | secretAccessKey: 'secret', // Optional 33 | sessionToken: 'session', // Optional 34 | region: 'us-east-1' // Required. You must enter a valid AWS region 35 | }; 36 | 37 | var metricsOptions = { // You can use either or both metric and log collection. 38 | namespace: 'test-data', 39 | timeout: 60 // See below for a complete list of options 40 | }; 41 | 42 | var logsOptions = { 43 | logGroup: 'my-application', 44 | timeout: 60, 45 | maxSize: 10000, 46 | addInstanceId: true, 47 | addTimestamp: true, 48 | logFormat: 'string', 49 | debug: true, 50 | s3Bucket: 'mybucket.example.com', 51 | s3Prefix: 'app/test', 52 | s3Subfolders: true 53 | }; 54 | 55 | var cwbMetrics = new CloudWatchBuddy(awsOptions).metrics(metricsOptions); 56 | var cwbLogs = new CloudWatchBuddy(awsOptions).logs(logsOptions); 57 | 58 | // Submit a simple Count metric for page views 59 | cwbMetrics.increment('pageviews'); 60 | 61 | // Submit a metric with a unit, keeping track of sum, minimum, maximum 62 | cwbMetrics.stat('loadtime', 10, 'Milliseconds'); 63 | 64 | // Submit a metric with a unit, with custom dimensions 65 | cwbMetrics.stat('serverload', 10, 'Percent', { 66 | serverName:'web01.example.com',region:'us-east-1' 67 | }); 68 | 69 | // Send a log 70 | cwbLogs.log('errors', 'Test message'); 71 | cwbLogs.log('signups', 'New user'); 72 | ``` 73 | 74 | ## Options 75 | 76 | ### AWS Options 77 | 78 | The AWS options object must be a valid config object supported by AWS (see: http://docs.aws.amazon.com/AWSJavaScriptSDK/latest/AWS/Config.html). You *must* provide a region. 79 | 80 | ### Metrics 81 | 82 | The following metrics options are supported: 83 | 84 | `namespace` (Required) - The AWS name space that all metrics should be submitted under. Note: to use multiple namespaces, you must instantiate separate metrics objects. 85 | 86 | `timeout` - The interval, in seconds, to submit your metrics. The minimum is 60 seconds (AWS CloudWatch charts only show minute-level metrics) and the maximum is 30 minutes (1800 s). The default is 120 seconds. 87 | 88 | `debug` - Whether messages should be printed to the console; helps with debugging 89 | 90 | ### Logs 91 | 92 | The following logs options are supported: 93 | 94 | `logGroup` (Required) - The log group name in AWS. This must already exist (the module will create streams for you, but not log groups). 95 | 96 | `timeout` - The interval, in seconds, to submit your logs. The minimum is 60 seconds and the maximum is 30 minutes (1800 s). The default is 120 seconds. 97 | 98 | `maxSize` - The maximum size, in bytes, of a log stream group between uploads. The minimum is 5000 and the maximum is 1048576 (~1MB). The default is 200000 (200KB). Note that this is a *rough estimate*. 99 | 100 | `logFormat` - Either "string" or "json". For string, the logs are written as timestamp - instance id - log message. For JSON, it is saved as an object with these same keys. The default is "string". 101 | 102 | `addTimestamp` - Whether the timestamp should be included in the log message. All logs have timestamps sent to AWS (used for ordering), but this option allows the timestamp to be included in the actual log message contents as well. The default is false. 103 | 104 | `addInstanceId` - Whether the EC2 instance ID should be included in the log message. The default is false. The module will attempt to load the instance ID from the AWS metadata service. If it cannot be determined, the instance ID will become "unknown". 105 | 106 | `debug` - Whether messages should be printed to the console; helps with debugging 107 | 108 | `s3Bucket` - If included, copies of the logs will be uploaded to this bucket (also requires s3Prefix to be set). 109 | 110 | `s3Prefix` - If included, copies of the logs will be uploaded to this prefix within the bucket (also requires s3Bucket to be set). 111 | 112 | `s3Subfolders` - Whether logs should be put in subfolders by year / month / day. (If true, path would be bucket/2015/04/15/22-45-153.log vs bucket/2015-04-15-22-45-153.log). 113 | 114 | ## Permissions 115 | 116 | You can either hard code AWS credentials into the application (please don't do this unless you're testing locally) or use IAM roles. When using IAM roles, your instance must have the following permissions in order to allow cloudwatch-buddy to create log streams, put log events, and put metrics: 117 | 118 | ``` 119 | { 120 | "Effect" : "Allow", 121 | "Action" : [ 122 | "cloudwatch:PutMetricData", 123 | "logs:CreateLogStream", 124 | "logs:DescribeLogGroups", 125 | "logs:DescribeLogStreams", 126 | "logs:PutLogEvents" 127 | ], 128 | "Resource" : "*" 129 | } 130 | ``` 131 | 132 | ## Known Limitations 133 | 134 | * Because of the 40KB limit imposed by API calls to CloudWatch, the number of stats that can be recorded is limited to approximately 100. This means that you can have 100 separate metric names ("page load time," "page size," etc.). You can update them as often as needed, either as increments or with dimensions, but you cannot create too many. Note: this is probably a good limit, as AWS imposes limits on the number of distinct metric names you can have as well. 135 | 136 | * The log group must exist on AWS before the module can write the logs. Future versions may allow for the log group to be created automatically, but it would require more permissions than are preferred. At this point, simply create the log group for your application before running this module. -------------------------------------------------------------------------------- /metrics.js: -------------------------------------------------------------------------------- 1 | var validMetricValues = [ 2 | 'Seconds', 3 | 'Microseconds', 4 | 'Milliseconds', 5 | 'Bytes', 6 | 'Kilobytes', 7 | 'Megabytes', 8 | 'Gigabytes', 9 | 'Terabytes', 10 | 'Bits', 11 | 'Kilobits', 12 | 'Megabits', 13 | 'Gigabits', 14 | 'Terabits', 15 | 'Percent', 16 | 'Count', 17 | 'Bytes/Second', 18 | 'Kilobytes/Second', 19 | 'Megabytes/Second', 20 | 'Gigabytes/Second', 21 | 'Terabytes/Second', 22 | 'Bits/Second', 23 | 'Kilobits/Second', 24 | 'Megabits/Second', 25 | 'Gigabits/Second', 26 | 'Terabits/Second', 27 | 'Count/Second', 28 | 'None' 29 | ]; 30 | 31 | var CloudWatchBuddyMetrics = function(cloudwatch, options){ 32 | 33 | var api = {}; 34 | 35 | var _increments = {}; 36 | var _stats = {}; 37 | var _statsWithDimensions = []; 38 | 39 | var _uploadInterval; 40 | 41 | var _namespace = options.namespace; 42 | var _timeout = (options.timeout && typeof options.timeout === 'number' && options.timeout >= 60 && options.timeout <= 1800) ? options.timeout : 120; 43 | var _debug = (options.debug && typeof options.debug === 'boolean') ? options.debug : false; 44 | //var _maxSize = (typeof options.maxSize === 'number' && options.maxSize < 40000) ? options.maxSize : 40000; // Max upload size if 40KB // TODO: add this option 45 | 46 | var putMetricData = function() { 47 | clearInterval(_uploadInterval); 48 | 49 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Put metrics called'); } 50 | 51 | var params = { 52 | MetricData:[], 53 | Namespace: _namespace 54 | }; 55 | 56 | // Add the increments 57 | 58 | if (Object.keys(_increments).length > 0) { 59 | for (key in _increments) { 60 | var val = _increments[key]; 61 | 62 | params.MetricData.push({ 63 | MetricName: key, 64 | Timestamp: new Date, 65 | Unit: 'Count', 66 | Value: val, 67 | }); 68 | _increments[key] = 0; // reset for next time 69 | } 70 | } 71 | 72 | // Add the stats 73 | if (Object.keys(_stats).length > 0) { 74 | for (key in _stats) { 75 | if (_stats[key].SampleCount === 0) { 76 | // This allows CWB to always send something, even if it is a 0 value 77 | _stats[key].SampleCount = 1; 78 | } 79 | var obj = _stats[key]; 80 | 81 | var pushObj = { 82 | MetricName: key, 83 | Timestamp: new Date, 84 | Unit: obj.Unit, 85 | StatisticValues: { 86 | Maximum: obj.Maximum, 87 | Minimum: obj.Minimum, 88 | SampleCount: obj.SampleCount, 89 | Sum: obj.Sum 90 | } 91 | }; 92 | 93 | params.MetricData.push(pushObj); 94 | 95 | // Reset the key 96 | _stats[key] = { 97 | Maximum: 0, 98 | Minimum: 0, 99 | SampleCount: 0, 100 | Sum: 0, 101 | Unit: obj.Unit 102 | }; 103 | } 104 | } 105 | 106 | // Add stats with dimensions 107 | if (_statsWithDimensions.length > 0) { 108 | for (index in _statsWithDimensions) { 109 | if (_statsWithDimensions[index].SampleCount === 0) { 110 | // This allows CWB to always send something, even if it is a 0 value 111 | _statsWithDimensions[index].SampleCount = 1; 112 | } 113 | var obj = _statsWithDimensions[index]; 114 | 115 | var pushObj = { 116 | MetricName: obj.MetricName, 117 | Timestamp: new Date, 118 | Unit: obj.Unit, 119 | StatisticValues: { 120 | Maximum: obj.Maximum, 121 | Minimum: obj.Minimum, 122 | SampleCount: obj.SampleCount, 123 | Sum: obj.Sum 124 | }, 125 | Dimensions: obj.Dimensions 126 | }; 127 | 128 | params.MetricData.push(pushObj); 129 | 130 | _statsWithDimensions[index] = { 131 | MetricName: obj.MetricName, 132 | Unit: obj.Unit, 133 | Maximum: 0, 134 | Minimum: 0, 135 | SampleCount: 0, 136 | Sum: 0, 137 | Dimensions: obj.Dimensions 138 | }; 139 | } 140 | } 141 | 142 | if (params.MetricData.length > 0) { 143 | // TODO: Check size before sending and split if needed 144 | cloudwatch.putMetricData(params, function(err, data){ 145 | if (err && _debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : ERROR : Put metrics error : ' + err); } 146 | if (!err && _debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Put metrics success'); } 147 | // TODO: if err, see if retryable 148 | setUploadInterval(); 149 | }); 150 | } else { 151 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : No metrics to put'); } 152 | setUploadInterval(); 153 | } 154 | } 155 | 156 | var setUploadInterval = function() { 157 | _uploadInterval = setInterval(function(){ 158 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Timer expired, calling put metrics'); } 159 | putMetricData(); 160 | }, _timeout * 1000); 161 | } 162 | 163 | setUploadInterval(); // Call it first to start 164 | 165 | // Public functions 166 | 167 | api.increment = function(key){ 168 | if (!key || typeof key !== 'string' || key.length < 1 || key.length > 100) { return; } 169 | 170 | if (_increments[key] === undefined) { 171 | _increments[key] = 0; 172 | } 173 | 174 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Incrementing metric : ' + key); } 175 | _increments[key]++; 176 | }; 177 | 178 | api.stat = function(key, value, unit, dimensions) { 179 | if (!unit || validMetricValues.indexOf(unit) === -1) { 180 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : ERROR : Error adding stat : Invalid unit : ' + unit); } 181 | return; 182 | } // Only accept valid AWS metrics 183 | 184 | if (_stats[key] === undefined && !dimensions) { 185 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Adding stats key : ' + key); } 186 | _stats[key] = { 187 | Unit: unit, 188 | Maximum: value, 189 | Minimum: value, 190 | SampleCount: 1, 191 | Sum: value 192 | }; 193 | } else if (dimensions && typeof dimensions === 'object' && Object.keys(dimensions).length > 0) { 194 | var convertedDimensions = []; 195 | 196 | for (name in dimensions) { 197 | if (typeof name !== 'string' || typeof dimensions[name] === 'object') { 198 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : ERROR : Invalid dimensions : ' + JSON.stringify(dimensions[name])); } 199 | continue; 200 | } // Only allow string - string / integer mappings 201 | 202 | convertedDimensions.push({ 203 | Name: name, 204 | Value: dimensions[name] 205 | }); 206 | } 207 | 208 | // Handle different dimensions sets 209 | var dimensionsFound = false; 210 | if (_statsWithDimensions.length > 0) { 211 | for (index in _statsWithDimensions) { 212 | // Loop through each dimensions stat 213 | if (JSON.stringify(_statsWithDimensions[index].Dimensions) === JSON.stringify(convertedDimensions)) { 214 | // Edit the existing dimensions set 215 | _statsWithDimensions[index].Maximum = value > _statsWithDimensions[index].Maximum ? value : _statsWithDimensions[index].Maximum; 216 | _statsWithDimensions[index].Minimum = value < _statsWithDimensions[index].Minimum ? value : _statsWithDimensions[index].Minimum; 217 | _statsWithDimensions[index].SampleCount++; 218 | _statsWithDimensions[index].Sum += value; 219 | dimensionsFound = true; 220 | break; // Don't continue the for loop once detected 221 | } 222 | } 223 | } 224 | 225 | if (!dimensionsFound) { 226 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Adding stat with dimensions : ' + key); } 227 | 228 | _statsWithDimensions.push({ 229 | MetricName: key, 230 | Unit: unit, 231 | Maximum: value, 232 | Minimum: value, 233 | SampleCount: 1, 234 | Sum: value, 235 | Dimensions: convertedDimensions 236 | }); 237 | } 238 | } else { 239 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyMetrics : INFO : Updating existing stats : ' + key); } 240 | 241 | _stats[key].Maximum = value > _stats[key].Maximum ? value : _stats[key].Maximum; 242 | _stats[key].Minimum = (value < _stats[key].Minimum || _stats[key].Minimum === 0) ? value : _stats[key].Minimum; 243 | _stats[key].SampleCount++; 244 | _stats[key].Sum += value; 245 | } 246 | }; 247 | 248 | return api; 249 | } 250 | 251 | module.exports = CloudWatchBuddyMetrics; -------------------------------------------------------------------------------- /logs.js: -------------------------------------------------------------------------------- 1 | var async = require('async'); 2 | 3 | var CloudWatchBuddyLogs = function(cloudwatchlogs, svc, s3, options){ 4 | 5 | var api = {}; 6 | 7 | var _logs = {}; 8 | var _logsToSend; 9 | 10 | var _existingLogStreams = {}; // Holds known streams and their sequence tokens 11 | var _queuedSize = {}; // bytes to upload for each stream 12 | 13 | var _uploadInterval; 14 | 15 | var _logGroup = options.logGroup; 16 | var _timeout = (options.timeout && typeof options.timeout === 'number' && options.timeout >= 60 && options.timeout <= 1800) ? options.timeout : 120; 17 | var _maxSize = (options.maxSize && typeof options.maxSize === 'number' && options.maxSize < 1048576 && options.maxSize > 5000) ? options.maxSize : 200000; // Default upload size of 200KB, AWS max of 1,048,576 bytes 18 | var _logFormat = (options.logFormat && typeof options.logFormat === 'string' && (options.logFormat === 'string' || options.logFormat === 'json')) ? options.logFormat : 'string'; 19 | var _addTimestamp = (options.addTimestamp && typeof options.addTimestamp === 'boolean') ? options.addTimestamp : false; 20 | var _addInstanceId = (options.addInstanceId && typeof options.addInstanceId === 'boolean') ? options.addInstanceId : false; 21 | var _debug = (options.debug && typeof options.debug === 'boolean') ? options.debug : false; 22 | var _s3Subfolders = (options.s3Subfolders && typeof options.s3Subfolders === 'boolean') ? options.s3Subfolders : false; 23 | var _s3Bucket = (options.s3Bucket && typeof options.s3Bucket === 'string') ? options.s3Bucket : false; 24 | var _s3Prefix = (options.s3Prefix && typeof options.s3Prefix === 'string') ? options.s3Prefix : false; 25 | 26 | var _instanceId = 'unknown'; 27 | 28 | // If _addInstanceId is set, then request the instance ID from AWS 29 | if (_addInstanceId) { 30 | svc.request('/latest/meta-data/instance-id', function(err, data){ 31 | if (err && _debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : ERROR : Error retrieving instance ID from AWS : ' + err); } 32 | if (!err && data) { 33 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Instance ID retrieved from AWS : ' + data); } 34 | _instanceId = data; // set the instance ID in the background during initiation 35 | } 36 | }); 37 | } 38 | 39 | var putLogData = function() { 40 | clearInterval(_uploadInterval); 41 | 42 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Put logs called'); } 43 | // Copy the log object so logs saved during the upload process aren't lost 44 | _logsToSend = JSON.parse(JSON.stringify(_logs)); // copy; don't reference 45 | _logs = {}; // Reset the logs instantly so none are lost 46 | 47 | // Reset the queued sizes 48 | for (key in _logsToSend) { 49 | _queuedSize[key] = 0; 50 | } 51 | 52 | async.eachSeries(Object.keys(_logsToSend), function(stream, callback){ 53 | 54 | if (!_logsToSend[stream].length) { 55 | return callback(); // go to the next one 56 | } 57 | 58 | // If S3 option is selected, upload to S3 too 59 | if (_s3Bucket && _s3Prefix) { 60 | uploadLogsToS3(stream, _logsToSend[stream]); // Do this synchronously 61 | } 62 | 63 | checkIfLogStreamExistsAndCreateItIfItDoesNot(stream, function(err, data){ 64 | if (err) { 65 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : ERROR : Error checking if log stream exists : ' + err); } 66 | callback(); // Don't callback an error - one log stream failure shouldn't affect the other streams 67 | } else { 68 | // Stream now exists 69 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Log stream exists : ' + stream); } 70 | var params = { 71 | logEvents: _logsToSend[stream], 72 | logGroupName: _logGroup, 73 | logStreamName: stream, 74 | sequenceToken: _existingLogStreams[stream] 75 | }; 76 | cloudwatchlogs.putLogEvents(params, function(err, data){ 77 | if (err) { 78 | if (err.code === 'InvalidSequenceTokenException') { 79 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : InvalidSequenceTokenException from AWS, retrying : ' + err); } 80 | params.sequenceToken = err.message.substring(err.message.indexOf(':') + 2); 81 | cloudwatchlogs.putLogEvents(params, function(err, data){ 82 | if (err) { 83 | // Still having issues 84 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : ERROR : Error putting logs : ' + err); } 85 | callback(); // Don't callback an error for a single stream 86 | } else { 87 | _existingLogStreams[stream] = data.nextSequenceToken; // Set this for next time 88 | callback(null, data); 89 | } 90 | }); 91 | } else { 92 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : ERROR : Error putting logs : ' + err); } 93 | callback(); // An error for one stream shouldn't prevent others from being uploaded 94 | } 95 | } else { 96 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Successfully put logs for stream : ' + stream); } 97 | _existingLogStreams[stream] = data.nextSequenceToken; // Set this for next time 98 | callback(); 99 | } 100 | }); 101 | } 102 | }); 103 | }, function(err){ 104 | if (err) { 105 | 106 | } 107 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Finished putting logs. Resetting timer'); } 108 | setUploadInterval(); // Reset timer for next loop 109 | }); 110 | }; 111 | 112 | var uploadLogsToS3 = function(stream, logData) { 113 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Adding logs to S3 : ' + stream); } 114 | 115 | logFile = ''; 116 | for (log in logData) { 117 | logFile += logData[log].message + '\n'; 118 | } 119 | 120 | var timestamp = new Date(); 121 | 122 | if (_s3Subfolders) { 123 | var key = _s3Prefix + '/' + stream + '/' + timestamp.getFullYear() + '/' + ('0' + (timestamp.getMonth()+1)).slice(-2) + '/' + ('0' + (timestamp.getDate())).slice(-2) + '/' + ('0' + (timestamp.getHours())).slice(-2) + '-' + ('0' + (timestamp.getMinutes())).slice(-2) + '-' + ('0' + (timestamp.getSeconds())).slice(-2) + '-' + timestamp.getMilliseconds() + '.log'; 124 | } else { 125 | var key = _s3Prefix + '/' + stream + '/' + timestamp.getFullYear() + '-' + ('0' + (timestamp.getMonth()+1)).slice(-2) + '-' + ('0' + (timestamp.getDate())).slice(-2) + '-' + ('0' + (timestamp.getHours())).slice(-2) + '-' + ('0' + (timestamp.getMinutes())).slice(-2) + '-' + ('0' + (timestamp.getSeconds())).slice(-2) + '-' + timestamp.getMilliseconds() + '.log'; 126 | } 127 | 128 | var params = { 129 | Bucket: _s3Bucket, 130 | Key: key, 131 | Body: logFile 132 | }; 133 | 134 | s3.putObject(params, function(err, data){ 135 | if (err && _debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : ERROR : S3 Error uploading logs : ' + stream + ' : ' + err); } 136 | else if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Logs uploaded to S3 successfully : ' + stream); } 137 | }); 138 | }; 139 | 140 | var checkIfLogStreamExistsAndCreateItIfItDoesNot = function(stream, callback) { 141 | if (_existingLogStreams[stream]) { 142 | callback(); 143 | } else { 144 | var params = { 145 | logGroupName: _logGroup, 146 | logStreamNamePrefix: stream, 147 | limit: 1 148 | }; 149 | cloudwatchlogs.describeLogStreams(params, function(err, data){ 150 | if (!err && data.logStreams.length > 0) { 151 | // The stream already exists, so add it to our known array and continue 152 | // TODO: make sure the whole name matches, not just the prefix 153 | _existingLogStreams[stream] = null; // Will eventually hold the sequence token 154 | callback(); 155 | } else { 156 | // Create the stream 157 | var params = { 158 | logGroupName: _logGroup, 159 | logStreamName: stream 160 | }; 161 | cloudwatchlogs.createLogStream(params, callback); 162 | } 163 | }); 164 | } 165 | }; 166 | 167 | 168 | var setUploadInterval = function() { 169 | _uploadInterval = setInterval(function(){ 170 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Timer expired, calling put logs'); } 171 | putLogData(); 172 | }, _timeout * 1000); 173 | }; 174 | 175 | setUploadInterval(); // Call it first to start 176 | 177 | // Public functions 178 | 179 | api.log = function(stream, msg) { 180 | if (_logs[stream] === undefined) { 181 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Adding new local log stream : ' + stream); } 182 | _logs[stream] = []; 183 | } 184 | 185 | if (_logFormat === 'string') { 186 | if (typeof msg === 'object') { 187 | msg = JSON.stringify(msg); 188 | } 189 | 190 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Adding log string to local stream : ' + stream); } 191 | 192 | _logs[stream].push({ 193 | timestamp: new Date().getTime(), 194 | message: (_addTimestamp ? new Date + ' ' : '') + (_addInstanceId ? _instanceId + ' ' : '') + msg 195 | }); 196 | } else { 197 | var logObj = {}; 198 | if (_addTimestamp) { logObj['timestamp'] = new Date; } 199 | if (_addInstanceId) { logObj['instance_id'] = _instanceId; } 200 | logObj['message'] = msg; 201 | 202 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Adding log JSON to local stream : ' + stream); } 203 | 204 | _logs[stream].push({ 205 | timestamp: new Date().getTime(), 206 | message: JSON.stringify(logObj) // AWS only accepts a string 207 | }); 208 | } 209 | 210 | _queuedSize[stream] += (26 + JSON.stringify(_logs[stream]).length * 2); //~2 bytes per character plus 26 bytes of overhead per log 211 | 212 | if (((_queuedSize[stream]) >= (_maxSize - 1000)) || _logs[stream].length > 9000) { // Leave some room (AWS max is 10,000 logs) 213 | if (_debug) { console.log (new Date() + ' : CloudWatchBuddyLogs : INFO : Size of log queue for stream ' + stream + ' ' + _queuedSize[stream] + ' bytes is greater than max size of ' + _maxSize + ' bytes'); } 214 | putLogData(); 215 | } 216 | }; 217 | 218 | return api; 219 | } 220 | 221 | module.exports = CloudWatchBuddyLogs; --------------------------------------------------------------------------------