├── .gitignore ├── README.md ├── package.json ├── functions ├── test-api │ └── handler.js ├── usage-metrics │ ├── lib.js │ └── handler.js └── custom-metrics │ ├── handler.js │ └── lib.js ├── serverless.yml ├── lib ├── cloudwatch.js └── parse.js ├── .vscode └── launch.json ├── LICENSE └── events └── log-message.json /.gitignore: -------------------------------------------------------------------------------- 1 | # package directories 2 | node_modules 3 | jspm_packages 4 | 5 | # Serverless directories 6 | .serverless -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # lambda-logging-metrics-demo 2 | How to apply Datadog's approach for sending custom metrics asynchronously. 3 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-logging-metrics-demo", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "devDependencies": { 12 | "aws-sdk": "^2.104.0", 13 | "serverless": "^1.20.2", 14 | "serverless-pseudo-parameters": "^1.1.5" 15 | }, 16 | "dependencies": { 17 | "bluebird": "^3.5.0", 18 | "co": "^4.6.0", 19 | "lodash": "^4.17.4" 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /functions/test-api/handler.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const namespace = "theburningmonk.com"; 4 | 5 | function sendMetric(name, value, unit) { 6 | console.log(`MONITORING|${value}|${unit}|${name}|${namespace}`); 7 | } 8 | 9 | module.exports.handler = (event, context, callback) => { 10 | let pageViews = Math.ceil(Math.random() * 42); 11 | sendMetric("page_view", pageViews, "count"); 12 | 13 | let latency = Math.random() * 42; 14 | sendMetric("latency", latency, "milliseconds"); 15 | 16 | let response = { 17 | statusCode: 200, 18 | body: JSON.stringify({ 19 | message: "dracarys" 20 | }) 21 | }; 22 | callback(null, response); 23 | }; -------------------------------------------------------------------------------- /functions/usage-metrics/lib.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const co = require('co'); 5 | const Promise = require('bluebird'); 6 | const parse = require('../../lib/parse'); 7 | const cloudwatch = require('../../lib/cloudwatch'); 8 | const namespace = "theburningmonk.com"; 9 | 10 | let processAll = co.wrap(function* (logGroup, logStream, logEvents) { 11 | let funcName = parse.functionName(logGroup); 12 | let lambdaVer = parse.lambdaVersion(logStream); 13 | let metricDatum = 14 | _.flatMap(logEvents, e => parse.usageMetrics(funcName, lambdaVer, e.message)); 15 | 16 | yield cloudwatch.publish(metricDatum, namespace); 17 | }); 18 | 19 | module.exports = processAll; -------------------------------------------------------------------------------- /functions/custom-metrics/handler.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const co = require('co'); 4 | const Promise = require('bluebird'); 5 | const processAll = require('./lib'); 6 | const zlib = Promise.promisifyAll(require('zlib')); 7 | 8 | module.exports.handler = co.wrap(function* (event, context, callback) { 9 | console.log(JSON.stringify(event)); 10 | 11 | let payload = new Buffer(event.awslogs.data, 'base64'); 12 | let json = (yield zlib.gunzipAsync(payload)).toString('utf8'); 13 | console.log(json); 14 | 15 | let logEvent = JSON.parse(json); 16 | 17 | yield processAll(logEvent.logGroup, logEvent.logStream, logEvent.logEvents); 18 | callback(null, `Successfully processed ${logEvent.logEvents.length} log events.`); 19 | }); -------------------------------------------------------------------------------- /functions/usage-metrics/handler.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const co = require('co'); 4 | const Promise = require('bluebird'); 5 | const processAll = require('./lib'); 6 | const zlib = Promise.promisifyAll(require('zlib')); 7 | 8 | module.exports.handler = co.wrap(function* (event, context, callback) { 9 | console.log(JSON.stringify(event)); 10 | 11 | let payload = new Buffer(event.awslogs.data, 'base64'); 12 | let json = (yield zlib.gunzipAsync(payload)).toString('utf8'); 13 | console.log(json); 14 | 15 | let logEvent = JSON.parse(json); 16 | 17 | yield processAll(logEvent.logGroup, logEvent.logStream, logEvent.logEvents); 18 | callback(null, `Successfully processed ${logEvent.logEvents.length} log events.`); 19 | }); -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: logging-metrics-demo 2 | 3 | plugins: 4 | - serverless-pseudo-parameters 5 | 6 | provider: 7 | name: aws 8 | runtime: nodejs6.10 9 | 10 | iamRoleStatements: 11 | - Effect: "Allow" 12 | Action: 13 | - "cloudwatch:PutMetricData" 14 | Resource: "*" 15 | 16 | functions: 17 | custom-metrics: 18 | handler: functions/custom-metrics/handler.handler 19 | description: Sends custom CloudWatch metrics asynchronously 20 | 21 | usage-metrics: 22 | handler: functions/usage-metrics/handler.handler 23 | description: Tracks Memory Usage and Billed Duration as metrics 24 | 25 | test-api: 26 | handler: functions/test-api/handler.handler 27 | events: 28 | - http: 29 | path: / 30 | method: get -------------------------------------------------------------------------------- /functions/custom-metrics/lib.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const co = require('co'); 5 | const Promise = require('bluebird'); 6 | const parse = require('../../lib/parse'); 7 | const cloudwatch = require('../../lib/cloudwatch'); 8 | 9 | let processAll = co.wrap(function* (logGroup, logStream, logEvents) { 10 | let funcName = parse.functionName(logGroup); 11 | let lambdaVer = parse.lambdaVersion(logStream); 12 | let metricDatum = 13 | logEvents 14 | .map(e => parse.customMetric(funcName, lambdaVer, e.message)) 15 | .filter(m => m != null && m != undefined); 16 | 17 | let metricDatumByNamespace = _.groupBy(metricDatum, m => m.Namespace); 18 | let namespaces = _.keys(metricDatumByNamespace); 19 | for (let namespace of namespaces) { 20 | let datum = metricDatumByNamespace[namespace]; 21 | yield cloudwatch.publish(datum, namespace); 22 | } 23 | }); 24 | 25 | module.exports = processAll; -------------------------------------------------------------------------------- /lib/cloudwatch.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const co = require('co'); 5 | const Promise = require('bluebird'); 6 | const AWS = require('aws-sdk'); 7 | const cloudWatch = Promise.promisifyAll(new AWS.CloudWatch()); 8 | 9 | let publish = co.wrap(function* (metricDatum, namespace) { 10 | let metricData = metricDatum.map(m => { 11 | return { 12 | MetricName : m.MetricName, 13 | Dimensions : m.Dimensions, 14 | Timestamp : m.Timestamp, 15 | Unit : m.Unit, 16 | Value : m.Value 17 | }; 18 | }); 19 | 20 | // cloudwatch only allows 20 metrics per request 21 | let chunks = _.chunk(metricData, 20); 22 | 23 | for (let chunk of chunks) { 24 | let req = { 25 | MetricData: chunk, 26 | Namespace: namespace 27 | }; 28 | 29 | yield cloudWatch.putMetricDataAsync(req); 30 | 31 | console.log(`sent [${chunk.length}] metrics`); 32 | } 33 | }); 34 | 35 | module.exports = { 36 | publish 37 | }; -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible Node.js debug attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "custom-metrics", 11 | "program": "${workspaceRoot}/node_modules/.bin/sls", 12 | "args": [ 13 | "invoke", 14 | "local", 15 | "-f", 16 | "custom-metrics", 17 | "-p", 18 | "events/log-message.json" 19 | ] 20 | }, 21 | { 22 | "type": "node", 23 | "request": "launch", 24 | "name": "usage-metrics", 25 | "program": "${workspaceRoot}/node_modules/.bin/sls", 26 | "args": [ 27 | "invoke", 28 | "local", 29 | "-f", 30 | "usage-metrics", 31 | "-p", 32 | "events/log-message.json" 33 | ] 34 | } 35 | ] 36 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Yan Cui 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /events/log-message.json: -------------------------------------------------------------------------------- 1 | { 2 | "awslogs": { 3 | "data": "H4sIAAAAAAAAAL2Xb2/iRhDGvwpCfRnDzszuzC7vcgqNIl2SKtC+uEsUGbOkVvmTYpM0Pe67dyA5HUFxzoTmhGTE7nr9PDOe3w5fmpNYFOlN7D/cxmaneXTYP7w+7fZ6h8fd5kFzdj+Ncx22IRiwxNZZ0OHx7OZ4Plvc6kw7vS/a43QyGKZtHb7JpzfJJJbzPCuSYZzM9HKXlLEok/Q2f7y1V85jOtF70YC0jW+jtD//8vGw3+31r8CzjzhwwxhSK2kW0nSYjjiyc5zZTHSLYjEosnl+W+az6a/5uIzzotn53Py4FvG4+XWllGxRlLPJt/Hm1VpR9y5Oy9UmX5r5UIUROXIS0Iugs8EZA+hCYLEMzrAna8AgEFhksM4BAnq9qrgy14CW6URjo0tJAgcG7/HgW6CffCfGJyh9Ax3EjoWWLvl0WZIPkkqGiU8zTgCiJD46SQYDCQSCaRR3WZ6en530zy9Ozo6XFpbZbDEtl7e69fVdHu+X5Z9xsJhP1f1kNv2rlc0ml9Pm14P9rNHPt0bUcmI9BudJILiwnOTjcV7EbDYdFstxWsZp9rCD3ZVjI04keDQOSa/Bi9Mvqx+xOmnAg/GBTLVd2rTbPTtqXMS/F7rwZNhp1PK4vzqoqe6i+9v5RX9ngeXRYp6uiqvTwJYPjUlxWX7Q0Mdh4/sMGKMTDc2YVtb8odHL/42rUbSN0w86mv7TeJr5vYj6ZDTr8ZfcB9A3TawNTGpAjAcSCoaZUNQVGWDPAMFCEGuwKjfEyJvue/3DLfMhGwEb2jCf2oCJC8YMRzSEQZDGH0qTtcEnIFUIRueDIwWivp9C1nqNviUAYhMCWTAq1gp4YTBV6SImeL12qKVLVrVTQ/nz2uE3YmFHa1XEe09rtuUNilEurAuH98HCjnarKLhtdxsLdTzur87WVPcCFmol4Xvxm5b1748Fh9aRYfSoDLSBnLeekMUjgDh9owysehLxJgBKVW6cHnOvYyHNFLN+tGE+ZDxS8xYzDX02Sl09LKjgIMyaH7SalyAmWM2T12mnEGcd8cysQXUIhqsF8+u141q6ZFU7NZQ/qx18KxV2cyY/35lrsagS4GDJO9Efe2FhN7++pt9tLNQxub+6UFPdC1iolYVNLBC9Pxa8B20GnKWgUFC/iA7YIgXH5Ow6LBBAOwU9GyxXnVCs0XodCwNtirKBbDJxyC4BfQjGUeoNUD0sqGDx3mk/Y0xQbFtYVb8YPcWCQW9JWBsdJb2ACFR1Cyr4B8XDLf9YPDWUPz9S3Vu5sJu1V3LxbtZ8y+mfGNJHKBYIzX7twm5+q46kbb/bXKhjcn91Ve3CtroXuFArC8+4gP87F66+/gczcsNIvBAAAA==" 4 | } 5 | } -------------------------------------------------------------------------------- /lib/parse.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | // logGroup looks like this: 4 | // "logGroup": "/aws/lambda/service-env-funcName" 5 | let functionName = function (logGroup) { 6 | return logGroup.split('/').reverse()[0]; 7 | }; 8 | 9 | // logStream looks like this: 10 | // "logStream": "2016/08/17/[76]afe5c000d5344c33b5d88be7a4c55816" 11 | let lambdaVersion = function (logStream) { 12 | let start = logStream.indexOf('['); 13 | let end = logStream.indexOf(']'); 14 | return logStream.substring(start+1, end); 15 | }; 16 | 17 | let isDate = function (str) { 18 | return !isNaN(Date.parse(str)); 19 | } 20 | 21 | // this won't work for some units like Bits/Second, Count/Second, etc. 22 | // but hey, this is a demo ;-) 23 | let toCamelCase = function(str) { 24 | return str.substr( 0, 1 ).toUpperCase() + str.substr( 1 ); 25 | } 26 | 27 | // a typical log message looks like this: 28 | // "2017-04-26T10:41:09.023Z\tdb95c6da-2a6c-11e7-9550-c91b65931beb\tmy log message\n" 29 | // the metrics message we're looking for looks like this: 30 | // "2017-04-26T10:41:09.023Z\tdb95c6da-2a6c-11e7-9550-c91b65931beb\tMONITORING|metric_value|metric_unit|metric_name|namespace\n" 31 | let customMetric = function (functionName, version, message) { 32 | let parts = message.split('\t', 3); 33 | 34 | if (parts.length === 3 && isDate(parts[0]) && parts[2].startsWith("MONITORING")) { 35 | let timestamp = parts[0]; 36 | let requestId = parts[1]; 37 | let logMessage = parts[2]; 38 | 39 | let metricData = logMessage.split('|'); 40 | return { 41 | Value : parseFloat(metricData[1]), 42 | Unit : toCamelCase(metricData[2].trim()), 43 | MetricName : metricData[3].trim(), 44 | Dimensions : [ 45 | { Name: "FunctionName", Value: functionName }, 46 | { Name: "FunctionVersion", Value: version } 47 | ], 48 | Timestamp : new Date(timestamp), 49 | Namespace : metricData[4].trim() 50 | }; 51 | } 52 | 53 | return null; 54 | }; 55 | 56 | let parseFloatWith = (regex, input) => { 57 | let res = regex.exec(input); 58 | return parseFloat(res[1]); 59 | } 60 | 61 | let mkMetric = (value, unit, name, dimensions) => { 62 | return { 63 | Value : value, 64 | Unit : toCamelCase(unit), 65 | MetricName : name, 66 | Dimensions : dimensions, 67 | Timestamp : new Date() 68 | }; 69 | } 70 | 71 | // a typical report message looks like this: 72 | // "REPORT RequestId: 3897a7c2-8ac6-11e7-8e57-bb793172ae75\tDuration: 2.89 ms\tBilled Duration: 100 ms \tMemory Size: 1024 MB\tMax Memory Used: 20 MB\t\n" 73 | let usageMetrics = function (functionName, version, message) { 74 | if (message.startsWith("REPORT RequestId:")) { 75 | let parts = message.split("\t", 5); 76 | 77 | let billedDuration = parseFloatWith(/Billed Duration: (.*) ms/i, parts[2]); 78 | let memorySize = parseFloatWith(/Memory Size: (.*) MB/i, parts[3]); 79 | let memoryUsed = parseFloatWith(/Max Memory Used: (.*) MB/i, parts[4]); 80 | let dimensions = [ 81 | { Name: "FunctionName", Value: functionName }, 82 | { Name: "FunctionVersion", Value: version } 83 | ]; 84 | 85 | return [ 86 | mkMetric(billedDuration, "milliseconds", "BilledDuration", dimensions), 87 | mkMetric(memorySize, "megabytes", "MemorySize", dimensions), 88 | mkMetric(memoryUsed, "megabytes", "MemoryUsed", dimensions), 89 | ]; 90 | } 91 | 92 | return []; 93 | } 94 | 95 | module.exports = { 96 | functionName, 97 | lambdaVersion, 98 | customMetric, 99 | usageMetrics 100 | }; --------------------------------------------------------------------------------