├── .circleci └── config.yml ├── .gitignore ├── LICENSE ├── Makefile ├── README.md ├── index.js ├── package-lock.json ├── package.json └── test ├── test-datadog.js └── test-logger.js /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | jobs: 4 | test: 5 | working_directory: ~/src 6 | docker: 7 | - image: circleci/node:8 8 | steps: 9 | - checkout 10 | - run: npm install 11 | - run: | 12 | echo '{"host": "logs.papertrailapp.com", "port": 1234, "appname": "app_name", "program": "default", "datadog": "api key", "waitForFlush": false}' > env.json 13 | - run: npm test 14 | 15 | workflows: 16 | version: 2 17 | test: 18 | jobs: 19 | - test 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | # Custom 40 | code.zip 41 | env.json 42 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2016 Apiary Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | APP ?= app_name 2 | PROGRAM ?= default 3 | LOG_GROUP ?= log_group_name 4 | HOST ?= logs.papertrailapp.com 5 | PORT ?= 1234 6 | DATADOG ?= 7 | WAIT_FOR_FLUSH ?= false 8 | 9 | ALNUM_LOG_GROUP = $(shell echo $(LOG_GROUP) | sed 's/[^[:alnum:]]/_/g') 10 | ACCOUNT_ID = $(shell aws sts get-caller-identity --output text --query Account) 11 | 12 | all: lambda log 13 | 14 | deps: 15 | rm -rf node_modules 16 | npm install 17 | 18 | env: 19 | rm -f env.json 20 | echo "{\"host\": \"$(HOST)\", \"port\": $(PORT), \"appname\": \"$(APP)\", \"program\": \"$(PROGRAM)\", \"datadog\": \"$(DATADOG)\", \"waitForFlush\": $(WAIT_FOR_FLUSH)}" > env.json 21 | 22 | create-zip: 23 | rm -f code.zip 24 | zip code.zip -r index.js env.json node_modules 25 | 26 | lambda: deps env create-zip 27 | aws lambda create-function --publish \ 28 | --function-name $(APP)-$(PROGRAM)-to-papertrail \ 29 | --runtime nodejs8.10 \ 30 | --handler index.handler \ 31 | --zip-file fileb://code.zip \ 32 | --role arn:aws:iam::$(ACCOUNT_ID):role/lambda_basic_execution 33 | 34 | deploy: deps env create-zip 35 | aws lambda update-function-code --publish \ 36 | --function-name $(APP)-$(PROGRAM)-to-papertrail \ 37 | --zip-file fileb://code.zip 38 | 39 | log: 40 | aws lambda add-permission \ 41 | --function-name $(APP)-$(PROGRAM)-to-papertrail \ 42 | --statement-id $(ALNUM_LOG_GROUP)__$(APP)-$(PROGRAM)-to-papertrail \ 43 | --principal logs.$(AWS_DEFAULT_REGION).amazonaws.com \ 44 | --action lambda:InvokeFunction \ 45 | --source-arn arn:aws:logs:$(AWS_DEFAULT_REGION):$(ACCOUNT_ID):log-group:$(LOG_GROUP):* \ 46 | --source-account $(ACCOUNT_ID) 47 | 48 | aws logs put-subscription-filter \ 49 | --log-group-name $(LOG_GROUP) \ 50 | --destination-arn arn:aws:lambda:$(AWS_DEFAULT_REGION):$(ACCOUNT_ID):function:$(APP)-$(PROGRAM)-to-papertrail \ 51 | --filter-name LambdaStream_$(APP)-$(PROGRAM)-to-papertrail \ 52 | --filter-pattern "" 53 | 54 | clean: 55 | rm -f code.zip env.json 56 | 57 | test: env 58 | docker pull lambci/lambda 59 | # docker run --name lambda --rm -v $(pwd):/var/task lambci/lambda index.handler '{}' 60 | 61 | destroy: 62 | aws lambda delete-function \ 63 | --function-name $(APP)-$(PROGRAM)-to-papertrail 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # cloudwatch-to-papertrail 2 | Lambda to send logs from Cloudwatch to Papertrail 3 | 4 | ## Deprecation Notice 5 | 6 | This package is no longer maintained by Apiary, use at your own risk. We 7 | welcome you to fork and maintain this package yourself. 8 | 9 | ## Usage 10 | 11 | First, ensure an IAM role exists called `lambda_basic_execution`, 12 | with the `AWSLambdaBasicExecutionRole` policy. 13 | 14 | Then create lambda function and streams logs from the specified log group to this function: 15 | 16 | ```bash 17 | $ export AWS_DEFAULT_REGION=us-east-1 18 | $ export HOST=logs.papertrailapp.com PORT=1234 19 | $ export DATADOG=1234567890abcdef1234567890abcdef12345678 20 | $ APP=helium PROGRAM=lambda LOG_GROUP=/aws/lambda/helium_transform make 21 | ``` 22 | 23 | `DATADOG` setting is optional. If you only want logging to papertrail, leave `DATADOG` blank. 24 | 25 | > NOTE: Datadog API key, not APP key 26 | 27 | > NOTE: Metric logs should be in the following format. `2016-08-05T15:20:11.819Z - info: parse: message metric#time=39 metric#tag#valid=1`. The metrics sent will be `helium.lambda.parse.time` with tag `valid:1` 28 | 29 | To update existing lambda function: 30 | 31 | ```bash 32 | $ export HOST=logs.papertrailapp.com PORT=1234 33 | $ export DATADOG=1234567890abcdef1234567890abcdef12345678 34 | $ APP=helium PROGRAM=lambda make deploy 35 | ``` 36 | 37 | To stream another log group to already existing lambda: 38 | 39 | ```bash 40 | $ export AWS_DEFAULT_REGION=us-east-1 41 | $ APP=helium PROGRAM=lambda LOG_GROUP=/aws/lambda/helium_compose make log 42 | ``` 43 | 44 | By default, lambda doesn't wait for the event loop to empty before shutting down the function. 45 | This means logs may not be sent immediately to papertrail, but instead wait for future 46 | invocations. It's also possible some logs may not get sent to papertrail at all. See 47 | [callbackWaitsForEmptyEventLoop](http://docs.aws.amazon.com/lambda/latest/dg/nodejs-prog-model-context.html) 48 | 49 | To override this behavior, set `WAIT_FOR_FLUSH=true`, i.e. 50 | ```bash 51 | $ APP=helium PROGRAM=lambda WAIT_FOR_FLUSH=true make deploy 52 | ``` 53 | 54 | Logs will be sent immediately to papertrail, at the expense of longer lambda execution times. 55 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var zlib = require('zlib'); 2 | var winston = require('winston'); 3 | var papertrailTransport = require('winston-papertrail').Papertrail; 4 | var dogapi = require('dogapi'); 5 | var config = require('./env.json'); 6 | 7 | const ignoredPatterns = [ 8 | /^.*API Key authorized because method '.+' does not require API Key. Request will not contribute to throttle or quota limits$/, 9 | /^.*Usage Plan check succeeded for API Key and API Stage .*$/, 10 | /^.*Verifying Usage Plan for request: [\d\w-]+. API Key: API Stage: .*/, 11 | ]; 12 | 13 | function addLambdaMetrics(data, match) { 14 | var now = dogapi.now(); 15 | 16 | data.push({ 17 | metric: 'aws.lambda.billed', 18 | points: [ 19 | [now, match[1]] 20 | ] 21 | }); 22 | 23 | data.push({ 24 | metric: 'aws.lambda.maxmemory', 25 | points: [ 26 | [now, match[2]] 27 | ] 28 | }); 29 | }; 30 | 31 | function addAppMetrics(data, match) { 32 | var now = parseInt((new Date(match[1])).getTime()/1000); 33 | 34 | var tags = []; 35 | var points = []; 36 | 37 | match[3].split(' ').forEach(function (metric) { 38 | var keyValue = metric.split('='); 39 | 40 | if (keyValue[0].indexOf('metric#') == -1) { 41 | return; 42 | } 43 | 44 | if (keyValue[0].indexOf('metric#tag#') != -1) { 45 | return tags.push(keyValue[0].replace('metric#tag#', '') + ':' + keyValue[1]); 46 | } 47 | 48 | points.push({ 49 | metric: [config.appname, config.program, match[2], keyValue[0].replace('metric#', '')].join('.'), 50 | points: [ 51 | [now, parseInt(keyValue[1])] 52 | ] 53 | }); 54 | }); 55 | 56 | points.forEach(function (item) { 57 | item.tags = tags; 58 | data.push(item); 59 | }); 60 | }; 61 | 62 | dogapi.initialize({ 63 | api_key: config.datadog 64 | }); 65 | 66 | function handleEvent(event, logger, cb) { 67 | var payload = new Buffer(event.awslogs.data, 'base64'); 68 | 69 | zlib.gunzip(payload, function (err, result) { 70 | if (err) { 71 | return cb(err); 72 | } 73 | 74 | var data = JSON.parse(result.toString('utf8')); 75 | 76 | var metricRegex = /^(\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\ -\ info:\ ([a-z-]+):.*?(metric#.*)+$/; 77 | var reportRegex = /^REPORT\ RequestId.*Billed\ Duration:\ ([0-9]+)\ ms.*Used:\ ([0-9]+)\ MB$/; 78 | 79 | var metricPoints = []; 80 | var reportPoints = []; 81 | 82 | data.logEvents.forEach(function (line) { 83 | if (ignoredPatterns.some(rule => rule.test(line.message))) { 84 | return; 85 | } 86 | 87 | logger.info(line.message); 88 | 89 | if (config.datadog !== '') { 90 | var metricMatch = line.message.trim().match(metricRegex); 91 | 92 | if (metricMatch != null) { 93 | return addAppMetrics(metricPoints, metricMatch); 94 | } 95 | 96 | var reportMatch = line.message.trim().match(reportRegex); 97 | 98 | if (reportMatch != null) { 99 | return addLambdaMetrics(reportPoints, reportMatch); 100 | } 101 | } 102 | }); 103 | 104 | if (config.datadog === '') { 105 | logger.close(); 106 | return cb(); 107 | } 108 | 109 | dogapi.metric.send_all(metricPoints, function () { 110 | dogapi.metric.send_all(reportPoints, function () { 111 | logger.close(); 112 | cb(); 113 | }); 114 | }); 115 | }); 116 | }; 117 | 118 | function handler(event, context, cb) { 119 | context.callbackWaitsForEmptyEventLoop = config.waitForFlush; 120 | 121 | const logger = new winston.Logger({ 122 | transports: [] 123 | }); 124 | 125 | logger.add(papertrailTransport, { 126 | host: config.host, 127 | port: config.port, 128 | program: config.program, 129 | hostname: config.appname, 130 | flushOnClose: true, 131 | logFormat: function (level, message) { 132 | return message; 133 | } 134 | }); 135 | 136 | handleEvent(event, logger, cb); 137 | } 138 | 139 | module.exports = { 140 | dogapi, 141 | handleEvent, 142 | handler, 143 | }; 144 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudwatch-to-papertrail", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "assertion-error": { 8 | "version": "1.1.0", 9 | "resolved": "https://registry.npmjs.org/assertion-error/-/assertion-error-1.1.0.tgz", 10 | "integrity": "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw==", 11 | "dev": true 12 | }, 13 | "async": { 14 | "version": "1.0.0", 15 | "resolved": "https://registry.npmjs.org/async/-/async-1.0.0.tgz", 16 | "integrity": "sha1-+PwEyjoTeErenhZBr5hXjPvWR6k=" 17 | }, 18 | "balanced-match": { 19 | "version": "1.0.0", 20 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 21 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 22 | "dev": true 23 | }, 24 | "bignumber.js": { 25 | "version": "1.1.1", 26 | "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-1.1.1.tgz", 27 | "integrity": "sha1-GkFdmsAUwTJWrx/u2dGj5XF6jPc=" 28 | }, 29 | "brace-expansion": { 30 | "version": "1.1.11", 31 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 32 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 33 | "dev": true, 34 | "requires": { 35 | "balanced-match": "^1.0.0", 36 | "concat-map": "0.0.1" 37 | } 38 | }, 39 | "browser-stdout": { 40 | "version": "1.3.1", 41 | "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", 42 | "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", 43 | "dev": true 44 | }, 45 | "chai": { 46 | "version": "4.2.0", 47 | "resolved": "https://registry.npmjs.org/chai/-/chai-4.2.0.tgz", 48 | "integrity": "sha512-XQU3bhBukrOsQCuwZndwGcCVQHyZi53fQ6Ys1Fym7E4olpIqqZZhhoFJoaKVvV17lWQoXYwgWN2nF5crA8J2jw==", 49 | "dev": true, 50 | "requires": { 51 | "assertion-error": "^1.1.0", 52 | "check-error": "^1.0.2", 53 | "deep-eql": "^3.0.1", 54 | "get-func-name": "^2.0.0", 55 | "pathval": "^1.1.0", 56 | "type-detect": "^4.0.5" 57 | } 58 | }, 59 | "check-error": { 60 | "version": "1.0.2", 61 | "resolved": "https://registry.npmjs.org/check-error/-/check-error-1.0.2.tgz", 62 | "integrity": "sha1-V00xLt2Iu13YkS6Sht1sCu1KrII=", 63 | "dev": true 64 | }, 65 | "colors": { 66 | "version": "1.0.3", 67 | "resolved": "https://registry.npmjs.org/colors/-/colors-1.0.3.tgz", 68 | "integrity": "sha1-BDP0TYCWgP3rYO0mDxsMJi6CpAs=" 69 | }, 70 | "commander": { 71 | "version": "2.15.1", 72 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", 73 | "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", 74 | "dev": true 75 | }, 76 | "concat-map": { 77 | "version": "0.0.1", 78 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 79 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 80 | "dev": true 81 | }, 82 | "core-util-is": { 83 | "version": "1.0.2", 84 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 85 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=", 86 | "dev": true 87 | }, 88 | "cycle": { 89 | "version": "1.0.3", 90 | "resolved": "https://registry.npmjs.org/cycle/-/cycle-1.0.3.tgz", 91 | "integrity": "sha1-IegLK+hYD5i0aPN5QwZisEbDStI=" 92 | }, 93 | "debug": { 94 | "version": "3.1.0", 95 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", 96 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", 97 | "dev": true, 98 | "requires": { 99 | "ms": "2.0.0" 100 | } 101 | }, 102 | "deep-eql": { 103 | "version": "3.0.1", 104 | "resolved": "https://registry.npmjs.org/deep-eql/-/deep-eql-3.0.1.tgz", 105 | "integrity": "sha512-+QeIQyN5ZuO+3Uk5DYh6/1eKO0m0YmJFGNmFHGACpf1ClL1nmlV/p4gNgbl2pJGxgXb4faqo6UE+M5ACEMyVcw==", 106 | "dev": true, 107 | "requires": { 108 | "type-detect": "^4.0.0" 109 | } 110 | }, 111 | "deep-extend": { 112 | "version": "0.6.0", 113 | "resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz", 114 | "integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==" 115 | }, 116 | "diff": { 117 | "version": "3.5.0", 118 | "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", 119 | "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", 120 | "dev": true 121 | }, 122 | "dogapi": { 123 | "version": "2.8.3", 124 | "resolved": "https://registry.npmjs.org/dogapi/-/dogapi-2.8.3.tgz", 125 | "integrity": "sha512-CzJIllAcv17H8hylyuljldJv+2EGal5+Fa+gsXFrXDo1zl8cq9jk7FDCXDqH4C/tPNAusS9ckV8wuBFtCiuTUg==", 126 | "requires": { 127 | "extend": "^3.0.0", 128 | "json-bigint": "^0.1.4", 129 | "lodash": "^4.17.10", 130 | "minimist": "^1.1.1", 131 | "rc": "^1.2.8" 132 | } 133 | }, 134 | "escape-string-regexp": { 135 | "version": "1.0.5", 136 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 137 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", 138 | "dev": true 139 | }, 140 | "extend": { 141 | "version": "3.0.2", 142 | "resolved": "https://registry.npmjs.org/extend/-/extend-3.0.2.tgz", 143 | "integrity": "sha512-fjquC59cD7CyW6urNXK0FBufkZcoiGG80wTuPujX590cB5Ttln20E2UB4S/WARVqhXffZl2LNgS+gQdPIIim/g==" 144 | }, 145 | "eyes": { 146 | "version": "0.1.8", 147 | "resolved": "https://registry.npmjs.org/eyes/-/eyes-0.1.8.tgz", 148 | "integrity": "sha1-Ys8SAjTGg3hdkCNIqADvPgzCC8A=" 149 | }, 150 | "fs.realpath": { 151 | "version": "1.0.0", 152 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 153 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 154 | "dev": true 155 | }, 156 | "get-func-name": { 157 | "version": "2.0.0", 158 | "resolved": "https://registry.npmjs.org/get-func-name/-/get-func-name-2.0.0.tgz", 159 | "integrity": "sha1-6td0q+5y4gQJQzoGY2YCPdaIekE=", 160 | "dev": true 161 | }, 162 | "glob": { 163 | "version": "7.1.2", 164 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 165 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 166 | "dev": true, 167 | "requires": { 168 | "fs.realpath": "^1.0.0", 169 | "inflight": "^1.0.4", 170 | "inherits": "2", 171 | "minimatch": "^3.0.4", 172 | "once": "^1.3.0", 173 | "path-is-absolute": "^1.0.0" 174 | } 175 | }, 176 | "glossy": { 177 | "version": "0.1.7", 178 | "resolved": "https://registry.npmjs.org/glossy/-/glossy-0.1.7.tgz", 179 | "integrity": "sha1-dptZhKlvYGarnqdYIkgl7mwhDws=" 180 | }, 181 | "growl": { 182 | "version": "1.10.5", 183 | "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", 184 | "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", 185 | "dev": true 186 | }, 187 | "has-flag": { 188 | "version": "3.0.0", 189 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", 190 | "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", 191 | "dev": true 192 | }, 193 | "he": { 194 | "version": "1.1.1", 195 | "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", 196 | "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", 197 | "dev": true 198 | }, 199 | "inflight": { 200 | "version": "1.0.6", 201 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 202 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 203 | "dev": true, 204 | "requires": { 205 | "once": "^1.3.0", 206 | "wrappy": "1" 207 | } 208 | }, 209 | "inherits": { 210 | "version": "2.0.3", 211 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz", 212 | "integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4=", 213 | "dev": true 214 | }, 215 | "ini": { 216 | "version": "1.3.5", 217 | "resolved": "https://registry.npmjs.org/ini/-/ini-1.3.5.tgz", 218 | "integrity": "sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==" 219 | }, 220 | "isarray": { 221 | "version": "1.0.0", 222 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 223 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=", 224 | "dev": true 225 | }, 226 | "isstream": { 227 | "version": "0.1.2", 228 | "resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz", 229 | "integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo=" 230 | }, 231 | "json-bigint": { 232 | "version": "0.1.4", 233 | "resolved": "https://registry.npmjs.org/json-bigint/-/json-bigint-0.1.4.tgz", 234 | "integrity": "sha1-tdQLipAJ6S8Vf3wHnbCXABgw4B4=", 235 | "requires": { 236 | "bignumber.js": "~1.1.1" 237 | } 238 | }, 239 | "lodash": { 240 | "version": "4.17.11", 241 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.11.tgz", 242 | "integrity": "sha512-cQKh8igo5QUhZ7lg38DYWAxMvjSAKG0A8wGSVimP07SIUEK2UO+arSRKbRZWtelMtN5V0Hkwh5ryOto/SshYIg==" 243 | }, 244 | "minimatch": { 245 | "version": "3.0.4", 246 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 247 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 248 | "dev": true, 249 | "requires": { 250 | "brace-expansion": "^1.1.7" 251 | } 252 | }, 253 | "minimist": { 254 | "version": "1.2.0", 255 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.0.tgz", 256 | "integrity": "sha1-o1AIsg9BOD7sH7kU9M1d95omQoQ=" 257 | }, 258 | "mkdirp": { 259 | "version": "0.5.1", 260 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 261 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 262 | "dev": true, 263 | "requires": { 264 | "minimist": "0.0.8" 265 | }, 266 | "dependencies": { 267 | "minimist": { 268 | "version": "0.0.8", 269 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 270 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", 271 | "dev": true 272 | } 273 | } 274 | }, 275 | "mocha": { 276 | "version": "5.2.0", 277 | "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", 278 | "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", 279 | "dev": true, 280 | "requires": { 281 | "browser-stdout": "1.3.1", 282 | "commander": "2.15.1", 283 | "debug": "3.1.0", 284 | "diff": "3.5.0", 285 | "escape-string-regexp": "1.0.5", 286 | "glob": "7.1.2", 287 | "growl": "1.10.5", 288 | "he": "1.1.1", 289 | "minimatch": "3.0.4", 290 | "mkdirp": "0.5.1", 291 | "supports-color": "5.4.0" 292 | } 293 | }, 294 | "ms": { 295 | "version": "2.0.0", 296 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 297 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", 298 | "dev": true 299 | }, 300 | "once": { 301 | "version": "1.4.0", 302 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 303 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 304 | "dev": true, 305 | "requires": { 306 | "wrappy": "1" 307 | } 308 | }, 309 | "path-is-absolute": { 310 | "version": "1.0.1", 311 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 312 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 313 | "dev": true 314 | }, 315 | "pathval": { 316 | "version": "1.1.0", 317 | "resolved": "https://registry.npmjs.org/pathval/-/pathval-1.1.0.tgz", 318 | "integrity": "sha1-uULm1L3mUwBe9rcTYd74cn0GReA=", 319 | "dev": true 320 | }, 321 | "process-nextick-args": { 322 | "version": "2.0.0", 323 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.0.tgz", 324 | "integrity": "sha512-MtEC1TqN0EU5nephaJ4rAtThHtC86dNN9qCuEhtshvpVBkAW5ZO7BASN9REnF9eoXGcRub+pFuKEpOHE+HbEMw==", 325 | "dev": true 326 | }, 327 | "rc": { 328 | "version": "1.2.8", 329 | "resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz", 330 | "integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==", 331 | "requires": { 332 | "deep-extend": "^0.6.0", 333 | "ini": "~1.3.0", 334 | "minimist": "^1.2.0", 335 | "strip-json-comments": "~2.0.1" 336 | } 337 | }, 338 | "readable-stream": { 339 | "version": "2.3.6", 340 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.6.tgz", 341 | "integrity": "sha512-tQtKA9WIAhBF3+VLAseyMqZeBjW0AHJoxOtYqSUZNJxauErmLbVm2FW1y+J/YA9dUrAC39ITejlZWhVIwawkKw==", 342 | "dev": true, 343 | "requires": { 344 | "core-util-is": "~1.0.0", 345 | "inherits": "~2.0.3", 346 | "isarray": "~1.0.0", 347 | "process-nextick-args": "~2.0.0", 348 | "safe-buffer": "~5.1.1", 349 | "string_decoder": "~1.1.1", 350 | "util-deprecate": "~1.0.1" 351 | } 352 | }, 353 | "safe-buffer": { 354 | "version": "5.1.2", 355 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 356 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==", 357 | "dev": true 358 | }, 359 | "stack-trace": { 360 | "version": "0.0.10", 361 | "resolved": "https://registry.npmjs.org/stack-trace/-/stack-trace-0.0.10.tgz", 362 | "integrity": "sha1-VHxws0fo0ytOEI6hoqFZ5f3eGcA=" 363 | }, 364 | "string_decoder": { 365 | "version": "1.1.1", 366 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 367 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 368 | "dev": true, 369 | "requires": { 370 | "safe-buffer": "~5.1.0" 371 | } 372 | }, 373 | "strip-json-comments": { 374 | "version": "2.0.1", 375 | "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", 376 | "integrity": "sha1-PFMZQukIwml8DsNEhYwobHygpgo=" 377 | }, 378 | "supports-color": { 379 | "version": "5.4.0", 380 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", 381 | "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", 382 | "dev": true, 383 | "requires": { 384 | "has-flag": "^3.0.0" 385 | } 386 | }, 387 | "triple-beam": { 388 | "version": "1.3.0", 389 | "resolved": "https://registry.npmjs.org/triple-beam/-/triple-beam-1.3.0.tgz", 390 | "integrity": "sha512-XrHUvV5HpdLmIj4uVMxHggLbFSZYIn7HEWsqePZcI50pco+MPqJ50wMGY794X7AOOhxOBAjbkqfAbEe/QMp2Lw==", 391 | "dev": true 392 | }, 393 | "type-detect": { 394 | "version": "4.0.8", 395 | "resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz", 396 | "integrity": "sha512-0fr/mIH1dlO+x7TlcMy+bIDqKPsw/70tVyeHW787goQjhmqaZe10uwLujubK9q9Lg6Fiho1KUKDYz0Z7k7g5/g==", 397 | "dev": true 398 | }, 399 | "util-deprecate": { 400 | "version": "1.0.2", 401 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 402 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=", 403 | "dev": true 404 | }, 405 | "winston": { 406 | "version": "2.4.4", 407 | "resolved": "https://registry.npmjs.org/winston/-/winston-2.4.4.tgz", 408 | "integrity": "sha512-NBo2Pepn4hK4V01UfcWcDlmiVTs7VTB1h7bgnB0rgP146bYhMxX0ypCz3lBOfNxCO4Zuek7yeT+y/zM1OfMw4Q==", 409 | "requires": { 410 | "async": "~1.0.0", 411 | "colors": "1.0.x", 412 | "cycle": "1.0.x", 413 | "eyes": "0.1.x", 414 | "isstream": "0.1.x", 415 | "stack-trace": "0.0.x" 416 | } 417 | }, 418 | "winston-papertrail": { 419 | "version": "github:hyrwork/winston-papertrail#ee23fad5d8f242daef4a9e0f8b8430264db1e98a", 420 | "from": "github:hyrwork/winston-papertrail", 421 | "requires": { 422 | "glossy": "^0.1.7" 423 | } 424 | }, 425 | "winston-transport": { 426 | "version": "4.3.0", 427 | "resolved": "https://registry.npmjs.org/winston-transport/-/winston-transport-4.3.0.tgz", 428 | "integrity": "sha512-B2wPuwUi3vhzn/51Uukcao4dIduEiPOcOt9HJ3QeaXgkJ5Z7UwpBzxS4ZGNHtrxrUvTwemsQiSys0ihOf8Mp1A==", 429 | "dev": true, 430 | "requires": { 431 | "readable-stream": "^2.3.6", 432 | "triple-beam": "^1.2.0" 433 | } 434 | }, 435 | "wrappy": { 436 | "version": "1.0.2", 437 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 438 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 439 | "dev": true 440 | } 441 | } 442 | } 443 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "cloudwatch-to-papertrail", 3 | "version": "1.0.0", 4 | "description": "Lambda to send cloudwatch logs to papertrail", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha" 8 | }, 9 | "dependencies": { 10 | "dogapi": "", 11 | "winston": "^2.4.4", 12 | "winston-papertrail": "github:hyrwork/winston-papertrail" 13 | }, 14 | "repository": { 15 | "type": "git", 16 | "url": "git+https://github.com/apiaryio/cloudwatch-to-papertrail.git" 17 | }, 18 | "author": "", 19 | "license": "MIT", 20 | "bugs": { 21 | "url": "https://github.com/apiaryio/cloudwatch-to-papertrail/issues" 22 | }, 23 | "homepage": "https://github.com/apiaryio/cloudwatch-to-papertrail#readme", 24 | "devDependencies": { 25 | "chai": "^4.2.0", 26 | "mocha": "^5.2.0", 27 | "winston-transport": "^4.3.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/test-datadog.js: -------------------------------------------------------------------------------- 1 | const { promisify } = require('util'); 2 | const { gzipSync } = require('zlib'); 3 | const { expect } = require('chai'); 4 | const index = require('../index'); 5 | 6 | const handler = promisify(index.handler); 7 | 8 | describe('data dog', function () { 9 | let metrics; 10 | let reportPoints; 11 | 12 | before(function () { 13 | metrics = []; 14 | reportPoints = []; 15 | 16 | index.dogapi.metric.send_all = function (_metrics, callback) { 17 | if (metrics.length > 0) { 18 | reportPoints = _metrics; 19 | } else { 20 | metrics = _metrics; 21 | } 22 | callback(); 23 | } 24 | }); 25 | 26 | afterEach(function() { 27 | metrics = []; 28 | reportPoints = []; 29 | }); 30 | 31 | it('sends metrics to datadog', async function () { 32 | const event = { 33 | awslogs: { 34 | data: gzipSync(JSON.stringify({ 35 | logEvents: [ 36 | { 37 | message: '2018-06-29T12:30:00.000Z - info: helium: metric#valid=1' 38 | } 39 | ] 40 | })).toString('base64') 41 | } 42 | } 43 | 44 | await handler(event, {}); 45 | 46 | expect(metrics).to.deep.equal([ 47 | { 48 | metric: 'app_name.default.helium.valid', 49 | points: [[1530275400, 1]], 50 | tags: [] 51 | } 52 | ]); 53 | }); 54 | 55 | it('sends metrics to datadog with dashes in name', async function () { 56 | const event = { 57 | awslogs: { 58 | data: gzipSync(JSON.stringify({ 59 | logEvents: [ 60 | { 61 | message: '2018-09-17T10:31:08.433Z - info: content-encoding: metric#unencoded_size=49227 metric#encoded_size=3544 metric#tag#encoding=gzip' 62 | } 63 | ] 64 | })).toString('base64') 65 | } 66 | } 67 | 68 | await handler(event, {}); 69 | 70 | expect(metrics).to.deep.equal([ 71 | { 72 | metric: 'app_name.default.content-encoding.unencoded_size', 73 | points: [[1537180268, 49227]], 74 | tags: ['encoding:gzip'] 75 | }, 76 | { 77 | metric: 'app_name.default.content-encoding.encoded_size', 78 | points: [[1537180268, 3544]], 79 | tags: ['encoding:gzip'] 80 | } 81 | ]); 82 | }); 83 | }); 84 | -------------------------------------------------------------------------------- /test/test-logger.js: -------------------------------------------------------------------------------- 1 | const { promisify } = require('util'); 2 | const { gzipSync } = require('zlib'); 3 | const { expect } = require('chai'); 4 | const winston = require('winston'); 5 | const Transport = require('winston-transport'); 6 | const index = require('../index'); 7 | 8 | const handleEvent = promisify(index.handleEvent); 9 | 10 | class TestTransport extends Transport { 11 | constructor() { 12 | super(); 13 | this.name = 'test'; 14 | this.events = []; 15 | } 16 | 17 | log(level, message) { 18 | this.events.push({ level, message }) 19 | } 20 | } 21 | 22 | describe('#handler', function () { 23 | let logger; 24 | let transport; 25 | 26 | beforeEach(function () { 27 | logger = new winston.Logger({ 28 | transports: [] 29 | }); 30 | logger.add(TestTransport); 31 | transport = logger.transports.test; 32 | }); 33 | 34 | afterEach(function() { 35 | logger.clear(); 36 | transport = null; 37 | logger = null; 38 | }); 39 | 40 | it('logs given awslogs to winston', async function () { 41 | const event = { 42 | awslogs: { 43 | data: 'H4sIAAAAAAAAAHWPwQqCQBCGX0Xm7EFtK+smZBEUgXoLCdMhFtKV3akI8d0bLYmibvPPN3wz00CJxmQnTO41whwWQRIctmEcB6sQbFC3CjW3XW8kxpOpP+OC22d1Wml1qZkQGtoMsScxaczKN3plG8zlaHIta5KqWsozoTYw3/djzwhpLwivWFGHGpAFe7DL68JlBUk+l7KSN7tCOEJ4M3/qOI49vMHj+zCKdlFqLaU2ZHV2a4Ct/an0/ivdX8oYc1UVX860fQDQiMdxRQEAAA==' 44 | } 45 | } 46 | 47 | await handleEvent(event, logger); 48 | 49 | expect(transport.events).to.deep.equal([ 50 | { 51 | level: 'info', 52 | message: '[ERROR] First test message', 53 | }, 54 | { 55 | level: 'info', 56 | message: '[ERROR] Second test message', 57 | } 58 | ]); 59 | }); 60 | 61 | it('ignores CloudWatch messages for request not contributing to throttle / quota limits', async function () { 62 | const event = { 63 | awslogs: { 64 | data: gzipSync(JSON.stringify({ 65 | logEvents: [ 66 | { 67 | message: "(some-uuid-is-here) API Key authorized because method 'POST /validate' does not require API Key. Request will not contribute to throttle or quota limits", 68 | }, 69 | ], 70 | })).toString('base64'), 71 | }, 72 | }; 73 | 74 | await handleEvent(event, logger); 75 | 76 | expect(transport.events.length).to.equal(0); 77 | }); 78 | 79 | it('ignores CloudWatch messages for checking usage plans', async function () { 80 | const event = { 81 | awslogs: { 82 | data: gzipSync(JSON.stringify({ 83 | logEvents: [ 84 | { 85 | message: "(some-uuid-is-here) Usage Plan check succeeded for API Key and API Stage stageid/production ", 86 | }, 87 | ], 88 | })).toString('base64'), 89 | }, 90 | }; 91 | 92 | await handleEvent(event, logger); 93 | 94 | expect(transport.events.length).to.equal(0); 95 | }); 96 | 97 | it('ignores CloudWatch messages for verifying usage plans', async function () { 98 | const event = { 99 | awslogs: { 100 | data: gzipSync(JSON.stringify({ 101 | logEvents: [ 102 | { 103 | message: "(some-uuid-is-here) Verifying Usage Plan for request: request-uuid. API Key: API Stage: stageid/production ", 104 | }, 105 | ], 106 | })).toString('base64'), 107 | }, 108 | }; 109 | 110 | await handleEvent(event, logger); 111 | 112 | expect(transport.events.length).to.equal(0); 113 | }); 114 | }); 115 | --------------------------------------------------------------------------------