├── .eslintignore ├── test ├── assets │ ├── message.txt │ ├── csv.source.txt │ ├── tabs.source.txt │ ├── table.jsonstring.txt │ ├── message.txt.gz │ ├── table.json │ ├── log.string.txt │ ├── log.string.config.txt │ ├── log.string.base64.txt │ ├── cloudfront.source.json │ ├── cloudtrail.source.json │ ├── log.json │ ├── config.source.json │ ├── cloudtrail.format.json │ ├── config.format.json │ ├── cloudwatch.parse.json │ ├── cloudwatch.source.txt │ ├── cloudfront.format.json │ ├── cloudwatch.data.json │ ├── elbv1.parse.json │ ├── elbv2.source.txt │ ├── cloudwatch.format.json │ ├── elbv1.format.json │ ├── elbv2.parse.json │ ├── elbv2.format.json │ ├── s3access.source.txt │ ├── s3access.parse.json │ └── s3access.format.json ├── decodeBase64.spec.js ├── formatCloudfront.spec.js ├── parseJson.spec.js ├── decompressGzip.spec.js ├── outputJsonLines.spec.js ├── parseCsv.spec.js ├── parseTabs.spec.js ├── formatELBv2.spec.js ├── formatS3Access.spec.js ├── formatConfig.spec.js ├── formatCloudtrail.spec.js ├── formatCloudwatchLogs.spec.js ├── formatELBv1.spec.js ├── convertString.spec.js ├── parseSpaces.spec.js ├── getS3Object.spec.js ├── shipTcp.spec.js ├── shipHttp.spec.js ├── shipElasticsearch.spec.js └── index.spec.js ├── .gitignore ├── .eslintrc.js ├── handlers ├── decodeBase64.js ├── parseJson.js ├── outputJsonLines.js ├── decompressGzip.js ├── parseCsv.js ├── parseSpaces.js ├── parseTabs.js ├── getS3Object.js ├── shipTcp.js ├── shipHttp.js ├── formatCloudtrail.js ├── formatConfig.js ├── convertString.js ├── formatCloudfront.js ├── formatELBv1.js ├── formatELBv2.js ├── formatS3Access.js ├── shipElasticsearch.js └── formatCloudwatchLogs.js ├── .travis.yml ├── example ├── package.json └── index.js ├── CHANGELOG.md ├── LICENSE-MIT ├── package.json ├── index.js └── README.md /.eslintignore: -------------------------------------------------------------------------------- 1 | /coverage 2 | -------------------------------------------------------------------------------- /test/assets/message.txt: -------------------------------------------------------------------------------- 1 | hi, hello! 2 | 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /coverage 2 | /node_modules 3 | /.nyc_output 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "extends": "google" 3 | }; -------------------------------------------------------------------------------- /test/assets/csv.source.txt: -------------------------------------------------------------------------------- 1 | col1,col2,col3 2 | 1,2,3 3 | 4,5,6 4 | -------------------------------------------------------------------------------- /test/assets/tabs.source.txt: -------------------------------------------------------------------------------- 1 | col1 col2 col3 2 | 1 2 3 3 | 4 5 6 4 | -------------------------------------------------------------------------------- /test/assets/table.jsonstring.txt: -------------------------------------------------------------------------------- 1 | ["col1","col2","col3"] 2 | ["1","2","3"] 3 | ["4","5","6"] 4 | -------------------------------------------------------------------------------- /test/assets/message.txt.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/arithmetric/lambda-stash/HEAD/test/assets/message.txt.gz -------------------------------------------------------------------------------- /test/assets/table.json: -------------------------------------------------------------------------------- 1 | [ 2 | ["col1", "col2", "col3"], 3 | ["1", "2", "3"], 4 | ["4", "5", "6"] 5 | ] 6 | -------------------------------------------------------------------------------- /test/assets/log.string.txt: -------------------------------------------------------------------------------- 1 | date="2016-03-23T12:33:00Z" level="40" title="user authentication" 2 | date_time="2016-03-23T12:33:00Z" applevel="40" title="user authentication" 3 | -------------------------------------------------------------------------------- /handlers/decodeBase64.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('decodeBase64'); 3 | config.data = new Buffer(config.data, 'base64'); 4 | return Promise.resolve(config); 5 | }; 6 | -------------------------------------------------------------------------------- /test/assets/log.string.config.txt: -------------------------------------------------------------------------------- 1 | prefix date="2016-03-23T12:33:00Z" level="40" title="user authentication" suffix 2 | prefix date_time="2016-03-23T12:33:00Z" applevel="40" title="user authentication" suffix 3 | -------------------------------------------------------------------------------- /test/assets/log.string.base64.txt: -------------------------------------------------------------------------------- 1 | ZGF0ZT0iMjAxNi0wMy0yM1QxMjozMzowMFoiIGxldmVsPSI0MCIgdGl0bGU9InVzZXIgYXV0aGVudGljYXRpb24iCmRhdGVfdGltZT0iMjAxNi0wMy0yM1QxMjozMzowMFoiIGFwcGxldmVsPSI0MCIgdGl0bGU9InVzZXIgYXV0aGVudGljYXRpb24iCg== 2 | -------------------------------------------------------------------------------- /test/assets/cloudfront.source.json: -------------------------------------------------------------------------------- 1 | [ 2 | ["#Fields: date time col1 col2"], 3 | ["2016-03-23", "03:23:16", "val1", "val2", "extra1"], 4 | ["2016-06-16", "14:01:02", "val3", "val4", "extra2"], 5 | ["#ignore this line"], 6 | [] 7 | ] 8 | -------------------------------------------------------------------------------- /test/assets/cloudtrail.source.json: -------------------------------------------------------------------------------- 1 | { 2 | "Records": [ 3 | { 4 | "eventTime": "2016-06-16T14:19:13Z", 5 | "key1": "value1" 6 | }, 7 | { 8 | "eventTime": "2016-06-16T14:19:57Z", 9 | "key1": "value2" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /test/assets/log.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "date": "2016-03-23T12:33:00Z", 4 | "level": 40, 5 | "title": "user authentication" 6 | }, 7 | { 8 | "date-time": "2016-03-23T12:33:00Z", 9 | "app.level": 40, 10 | "(title)": "user authentication" 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - '8.10' 4 | - 10 5 | - 12 6 | matrix: 7 | allow_failures: 8 | - node_js: '8.10' 9 | script: 10 | - npm run lint 11 | - npm run check-coverage 12 | after_script: 13 | - cat ./coverage/lcov.info | ./node_modules/.bin/coveralls 14 | -------------------------------------------------------------------------------- /example/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-stash-example", 3 | "version": "1.0.0", 4 | "description": "Example deployment package for lambda-stash to AWS Lambda.", 5 | "main": "index.js", 6 | "dependencies": { 7 | "lambda-stash": "*" 8 | }, 9 | "private": true 10 | } 11 | -------------------------------------------------------------------------------- /handlers/parseJson.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('parseJson'); 3 | try { 4 | config.data = JSON.parse(config.data); 5 | } catch (err) { 6 | return Promise.reject('Unable to parse JSON: ' + err); 7 | } 8 | return Promise.resolve(config); 9 | }; 10 | -------------------------------------------------------------------------------- /test/assets/config.source.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurationItems": [ 3 | { 4 | "resourceCreationTime": "2016-06-16T14:19:13Z", 5 | "key1": "value1" 6 | }, 7 | { 8 | "resourceCreationTime": "2016-06-16T14:19:57Z", 9 | "key1": "value2" 10 | } 11 | ] 12 | } 13 | -------------------------------------------------------------------------------- /test/assets/cloudtrail.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "eventTime": "2016-06-16T14:19:13Z", 4 | "key1": "value1", 5 | "date": "2016-06-16T14:19:13Z" 6 | }, 7 | { 8 | "eventTime": "2016-06-16T14:19:57Z", 9 | "key1": "value2", 10 | "date": "2016-06-16T14:19:57Z" 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /test/assets/config.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "resourceCreationTime": "2016-06-16T14:19:13Z", 4 | "key1": "value1", 5 | "date": "2016-06-16T14:19:13Z" 6 | }, 7 | { 8 | "resourceCreationTime": "2016-06-16T14:19:57Z", 9 | "key1": "value2", 10 | "date": "2016-06-16T14:19:57Z" 11 | } 12 | ] 13 | -------------------------------------------------------------------------------- /test/assets/cloudwatch.parse.json: -------------------------------------------------------------------------------- 1 | {"messageType":"DATA_MESSAGE","owner":"1234567890","logGroup":"/aws/lambda/LambdaStashTest","logStream":"2016/06/18/[$LATEST]1234567890abcdef","subscriptionFilters":["LambdaStashTest-Filter"],"logEvents":[{"id":"1234567890","timestamp":1466263061409,"message":"START RequestId: abcdef-1234-5678-9012-abcdef Version: $LATEST\n"}]} 2 | -------------------------------------------------------------------------------- /handlers/outputJsonLines.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('outputJsonLines'); 3 | try { 4 | config.data = config.data.reduce(function(str, item) { 5 | return str + JSON.stringify(item) + '\n'; 6 | }, ''); 7 | } catch (err) { 8 | return Promise.reject(err); 9 | } 10 | return Promise.resolve(config); 11 | }; 12 | -------------------------------------------------------------------------------- /test/assets/cloudwatch.source.txt: -------------------------------------------------------------------------------- 1 | H4sICDeMZVcAA2N3Lmpzb24AXY/NasMwEITvfQojeoyR5Lhq7Jugbiikl0j0kpoix9tU4L9KckMJefeu45ZCTguzs7PfnEgL3psD6O8BSE4epJZvz4VScl2QBemPHTiUebJM78T9KmMoNv1h7fpxQJ2ao6eNaava0M1lqGD8hwYfZqMKDkyLzoRxQZmgfEV3txupC6XL/1RT7Wt4xxM/Vn7v7BBs3z3aJoDzJN+Rq+x4XpHy8qP4gi5MthOx9TVssFgwmBZpeSpEIpZM8JRli7/ieKC03OpoC58jWp/qPJpx4ikonpLijPEkntXoBZmQLo9+a7x25Fyeb34AE0XspUoBAAA= 2 | -------------------------------------------------------------------------------- /handlers/decompressGzip.js: -------------------------------------------------------------------------------- 1 | var zlib = require('zlib'); 2 | 3 | exports.process = function(config) { 4 | console.log('decompressGzip'); 5 | return new Promise(function(resolve, reject) { 6 | zlib.gunzip(config.data, function(err, result) { 7 | if (err) { 8 | return reject(err); 9 | } 10 | config.data = result.toString(); 11 | return resolve(config); 12 | }); 13 | }); 14 | }; 15 | -------------------------------------------------------------------------------- /test/assets/cloudfront.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "date": "2016-03-23T03:23:16", 4 | "time": "03:23:16", 5 | "col1": "val1", 6 | "col2": "val2", 7 | "4": "extra1", 8 | "originalDate": "2016-03-23" 9 | }, 10 | { 11 | "date": "2016-06-16T14:01:02", 12 | "time": "14:01:02", 13 | "col1": "val3", 14 | "col2": "val4", 15 | "4": "extra2", 16 | "originalDate": "2016-06-16" 17 | } 18 | ] 19 | -------------------------------------------------------------------------------- /handlers/parseCsv.js: -------------------------------------------------------------------------------- 1 | var parse = require('csv-parse'); 2 | 3 | exports.process = function(config) { 4 | console.log('parseCsv'); 5 | return new Promise(function(resolve, reject) { 6 | parse(config.data, { 7 | relax_column_count: true, // eslint-disable-line camelcase 8 | trim: true 9 | }, function(err, data) { 10 | if (err) { 11 | return reject(err); 12 | } 13 | config.data = data; 14 | resolve(config); 15 | }); 16 | }); 17 | }; 18 | -------------------------------------------------------------------------------- /handlers/parseSpaces.js: -------------------------------------------------------------------------------- 1 | var parse = require('csv-parse'); 2 | 3 | exports.process = function(config) { 4 | console.log('parseSpaces'); 5 | return new Promise(function(resolve, reject) { 6 | parse(config.data, { 7 | delimiter: ' ', 8 | relax_column_count: true, // eslint-disable-line camelcase 9 | trim: true 10 | }, function(err, data) { 11 | if (err) { 12 | return reject(err); 13 | } 14 | config.data = data; 15 | resolve(config); 16 | }); 17 | }); 18 | }; 19 | -------------------------------------------------------------------------------- /handlers/parseTabs.js: -------------------------------------------------------------------------------- 1 | var parse = require('csv-parse'); 2 | 3 | exports.process = function(config) { 4 | console.log('parseTabs'); 5 | return new Promise(function(resolve, reject) { 6 | parse(config.data, { 7 | delimiter: '\t', 8 | relax_column_count: true, // eslint-disable-line camelcase 9 | trim: true 10 | }, function(err, data) { 11 | if (err) { 12 | return reject(err); 13 | } 14 | config.data = data; 15 | resolve(config); 16 | }); 17 | }); 18 | }; 19 | -------------------------------------------------------------------------------- /handlers/getS3Object.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('getS3Object'); 3 | return new Promise(function(resolve, reject) { 4 | var AWS = require('aws-sdk'); 5 | var s3 = new AWS.S3(); 6 | s3.getObject({ 7 | Bucket: config.S3.srcBucket, 8 | Key: config.S3.srcKey 9 | }, function(err, result) { 10 | if (err) { 11 | return reject(err); 12 | } else if (!result || !result.hasOwnProperty('Body')) { 13 | return reject('Unexpected data received from s3.getObject().'); 14 | } 15 | config.data = result.Body; 16 | resolve(config); 17 | }); 18 | }); 19 | }; 20 | -------------------------------------------------------------------------------- /handlers/shipTcp.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('shipTcp'); 3 | var finished = false; 4 | return new Promise(function(resolve, reject) { 5 | var net = require('net'); 6 | var client = net.connect(config.tcp.port, config.tcp.host, function() { 7 | var keyData = config.tcp.keyData || 'data'; 8 | client.write(config[keyData], function() { 9 | finished = true; 10 | client.end(); 11 | resolve(config); 12 | }); 13 | }); 14 | client.on('close', function() { 15 | if (!finished) { 16 | reject('Socket closed unexpectedly.'); 17 | } 18 | }); 19 | }); 20 | }; 21 | -------------------------------------------------------------------------------- /handlers/shipHttp.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('shipHttp'); 3 | return new Promise(function(resolve, reject) { 4 | var http = require('http'); 5 | var url = require('url'); 6 | var options = url.parse(config.http.url); 7 | options.method = 'POST'; 8 | var req = http.request(options, function(res) { 9 | console.log('Received HTTP response status code: ' + res.statusCode); 10 | resolve(config); 11 | }); 12 | req.on('error', function(err) { 13 | reject('An error occurred when making an HTTP request: ' + err); 14 | }); 15 | var keyData = config.http.keyData || 'data'; 16 | req.write(config[keyData]); 17 | req.end(); 18 | }); 19 | }; 20 | -------------------------------------------------------------------------------- /handlers/formatCloudtrail.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('formatCloudtrail'); 3 | if (!config.data || 4 | !config.data.hasOwnProperty('Records') || 5 | (!config.data.Records.length && config.data.Records.length !== 0)) { 6 | return Promise.reject('Received unexpected CloudTrail JSON format:' + 7 | JSON.stringify(config.data)); 8 | } 9 | 10 | var items = []; 11 | var num = config.data.Records.length; 12 | var i; 13 | var item; 14 | for (i = 0; i < num; i++) { 15 | item = config.data.Records[i]; 16 | if (config.dateField && config.dateField !== 'eventTime') { 17 | item[config.dateField] = item.eventTime; 18 | } 19 | items.push(item); 20 | } 21 | config.data = items; 22 | return Promise.resolve(config); 23 | }; 24 | -------------------------------------------------------------------------------- /test/assets/cloudwatch.data.json: -------------------------------------------------------------------------------- 1 | { 2 | "logEvents": [ 3 | {"id":"1234567890","timestamp":1466263061409,"message":"START RequestId: abcdef-1234-5678-9012-abcdef Version: $LATEST\n"}, 4 | {"id":"1234567891","timestamp":1466263061419,"message":"2016-06-18T15:17:41.419Z\tabcdef-1234-5678-9012-abcdef\tExample log message 1\n"}, 5 | {"id":"1234567892","timestamp":1466263061429,"message":"2016-06-18T15:17:41.429Z\tabcdef-1234-5678-9012-abcdef\tExample log message 2\nLine 2\n"}, 6 | {"id":"1234567893","timestamp":1466263061439,"message":"END RequestId: abcdef-1234-5678-9012-abcdef\n"}, 7 | {"id":"1234567894","timestamp":1466263061449,"message":"REPORT RequestId: abcdef-1234-5678-9012-abcdef\tDuration: 432.10 ms\tBilled Duration: 500 ms\tMemory Size: 256 MB\tMax Memory Used: 123 MB\n"} 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /handlers/formatConfig.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'); 2 | 3 | exports.process = function(config) { 4 | console.log('formatConfig'); 5 | if (!config.data || 6 | !config.data.hasOwnProperty('configurationItems') || 7 | _.isNil(config.data.configurationItems.length)) { 8 | return Promise.reject('Received unexpected AWS Config JSON format:' + 9 | JSON.stringify(config.data)); 10 | } 11 | 12 | var items = []; 13 | var num = config.data.configurationItems.length; 14 | var i; 15 | var item; 16 | for (i = 0; i < num; i++) { 17 | item = config.data.configurationItems[i]; 18 | if (config.dateField && config.dateField !== 'resourceCreationTime') { 19 | item[config.dateField] = item.resourceCreationTime; 20 | } 21 | items.push(item); 22 | } 23 | config.data = items; 24 | return Promise.resolve(config); 25 | }; 26 | -------------------------------------------------------------------------------- /handlers/convertString.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'); 2 | 3 | exports.process = function(config) { 4 | console.log('convertString'); 5 | if (!_.isArray(config.data)) { 6 | return Promise.reject('Non-array data passed to convertString.'); 7 | } 8 | 9 | config.data = _.map(config.data, function(datum) { 10 | var parts = _.map(datum, function(value, key) { 11 | key = String(key).replace('-', '_').replace(/\W/g, ''); 12 | value = String(value).replace(/\n/g, ' '); 13 | return key + '="' + value + '"'; 14 | }); 15 | 16 | if (config.string && config.string.prefix) { 17 | parts.unshift(config.string.prefix); 18 | } 19 | 20 | if (config.string && config.string.suffix) { 21 | parts.push(config.string.suffix); 22 | } 23 | 24 | return parts.join(' ') + '\n'; 25 | }).join(''); 26 | 27 | return Promise.resolve(config); 28 | }; 29 | -------------------------------------------------------------------------------- /test/assets/elbv1.parse.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "2018-05-09T12:20:13.464900Z", 4 | "elb-test", 5 | "12.23.34.45:5678", 6 | "10.0.10.1:8080", 7 | "0.000039", 8 | "0.000905", 9 | "0.000022", 10 | "301", 11 | "301", 12 | "0", 13 | "178", 14 | "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1", 15 | "Amazon Route 53 Health Check Service; ref:1234567890;", 16 | "-", 17 | "-" 18 | ], 19 | [ 20 | "2018-05-09T12:20:14.464900Z", 21 | "elb-test", 22 | "12.23.34.45:5678", 23 | "10.0.10.1:8080", 24 | "0.000039", 25 | "0.000905", 26 | "0.000022", 27 | "301", 28 | "301", 29 | "0", 30 | "178", 31 | "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1", 32 | "Amazon Route 53 Health Check Service; ref:1234567890;", 33 | "-", 34 | "-", 35 | "new field" 36 | ] 37 | ] 38 | -------------------------------------------------------------------------------- /test/assets/elbv2.source.txt: -------------------------------------------------------------------------------- 1 | https 2016-08-10T23:39:43.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012 0 2 | https 2016-08-10T23:39:44.065466Z app/my-loadbalancer/50dc6c495c0c9188 192.168.131.39:2817 10.0.0.1:80 0.086 0.048 0.037 200 200 0 57 "GET https://www.example.com:443/ HTTP/1.1" "curl/7.46.0" ECDHE-RSA-AES128-GCM-SHA256 TLSv1.2 arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067 "Root=1-58337281-1d84f3d73c47ec4e58577259" www.example.com arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012 0 "new field" 3 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | # Change Log for lambda-stash 2 | 3 | ## 4.0.0 [2020/07/01] 4 | 5 | - Fixing parsing of Cloudwatch Logs streamed to Lambda. 6 | - Updating dependencies to resolve security warnings. 7 | - Dropping support for Node.js v6. 8 | 9 | ## 3.0.0 [2019/08/06] 10 | 11 | - Adding support for the S3 access log format. 12 | - Dropping support for Node.js v4. 13 | 14 | ## 2.0.0 [2018/05/23] 15 | 16 | - Adding support for ELB Classic Load Balancer (elbv1) and Application Load 17 | Balancer (elbv2) log formats. 18 | - Adding Elasticsearch requestTimeout configuration. 19 | - Fixing http-aws-es dependency. 20 | 21 | ## 1.1.0 [2017/01/20] 22 | 23 | - Allowing custom handlers to be provided through the configuration script. 24 | 25 | ## 1.0.2 [2016/06/21] 26 | 27 | - Improving test coverage. 28 | 29 | ## 1.0.1 [2016/06/19] 30 | 31 | - Fixing parsing of CloudWatch Log messages. 32 | - Fixing name of formatCloudtrail.spec.js test file. 33 | 34 | ## 1.0.0 [2016/06/18] 35 | 36 | - Initial release of lambda-stash. 37 | -------------------------------------------------------------------------------- /test/decodeBase64.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require('../handlers/decodeBase64'); 7 | 8 | describe('handler/decodeBase64.js', function() { 9 | describe('#process()', function() { 10 | var dataBase64; 11 | var dataPlain; 12 | 13 | before(function() { 14 | dataBase64 = fs.readFileSync("test/assets/log.string.base64.txt") 15 | .toString(); 16 | dataPlain = fs.readFileSync("test/assets/log.string.txt").toString(); 17 | }); 18 | 19 | it('should decode Base64 data', 20 | function(done) { 21 | handler.process({data: dataBase64, setting: true}) 22 | .then(function(result) { 23 | assert.ok(result.hasOwnProperty('setting'), 24 | 'process returns config object'); 25 | assert.strictEqual(result.data.toString(), dataPlain, 26 | 'Base64 data decoded successfully'); 27 | done(); 28 | }); 29 | }); 30 | }); 31 | }); 32 | -------------------------------------------------------------------------------- /example/index.js: -------------------------------------------------------------------------------- 1 | var shipper = require('lambda-stash'); 2 | 3 | exports.handler = function(event, context, callback) { 4 | var config = { 5 | elasticsearch: { 6 | host: 'https://search-abcdefghi.us-west-2.es.amazonaws.com', 7 | index: 'logs', 8 | region: 'us-west-2', 9 | useAWS: true 10 | }, 11 | mappings: [ 12 | { 13 | bucket: 'my-cloudtrail-logs', 14 | processors: [ 15 | 'decompressGzip', 16 | 'parseJson', 17 | 'formatCloudtrail', 18 | 'shipElasticsearch' 19 | ], 20 | elasticsearch: { 21 | type: 'cloudtrail' 22 | }, 23 | dateField: 'date' 24 | }, 25 | { 26 | bucket: 'my-cloudfront-logs', 27 | processors: [ 28 | 'decompressGzip', 29 | 'parseTabs', 30 | 'formatCloudfront', 31 | 'shipElasticsearch' 32 | ], 33 | elasticsearch: { 34 | type: 'cloudfront' 35 | }, 36 | dateField: 'date' 37 | } 38 | ] 39 | }; 40 | shipper.handler(config, event, context, callback); 41 | }; 42 | -------------------------------------------------------------------------------- /handlers/formatCloudfront.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('formatCloudfront'); 3 | var output = []; 4 | var fields = []; 5 | var numRows = config.data.length; 6 | var numCols; 7 | var i; 8 | var j; 9 | var row; 10 | var item; 11 | var label; 12 | 13 | for (i = 0; i < numRows; i++) { 14 | row = config.data[i]; 15 | numCols = row.length; 16 | if (numCols === 1) { 17 | row = row[0]; 18 | var pos = row.indexOf('#Fields: '); 19 | if (pos !== -1) { 20 | row = row.substr(pos + 9); 21 | fields = row.split(" "); 22 | } 23 | } else if (numCols) { 24 | item = {}; 25 | for (j = 0; j < numCols; j++) { 26 | label = (j < fields.length) ? fields[j] : String(j); 27 | item[label] = row[j]; 28 | } 29 | if (config.dateField) { 30 | if (config.dateField === 'date') { 31 | item.originalDate = item.date; 32 | } 33 | item[config.dateField] = item.date + 'T' + item.time; 34 | } 35 | output.push(item); 36 | } 37 | } 38 | config.data = output; 39 | return Promise.resolve(config); 40 | }; 41 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2018 Joe Turgeon (@arithmetric) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /test/formatCloudfront.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatCloudfront"); 7 | 8 | describe('handler/formatCloudfront.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/cloudfront.source.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/cloudfront.format.json")); 18 | }); 19 | 20 | it('should format parsed Cloudfront data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'Cloudfront data formatted successfully'); 33 | done(); 34 | }); 35 | }); 36 | }); 37 | }); 38 | -------------------------------------------------------------------------------- /test/parseJson.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/parseJson"); 7 | 8 | describe('handler/parseJson.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = fs.readFileSync("test/assets/table.json"); 15 | dataJson = JSON.parse(dataSource); 16 | }); 17 | 18 | it('should parse JSON data', 19 | function(done) { 20 | var config = {data: dataSource, setting: true}; 21 | handler.process(config) 22 | .then(function(result) { 23 | assert.ok(result.hasOwnProperty('setting'), 24 | 'process returns config object'); 25 | assert.deepStrictEqual(result.data, dataJson, 26 | 'JSON data parsed successfully'); 27 | done(); 28 | }); 29 | }); 30 | 31 | it('should fail if malformed JSON data is provided', 32 | function(done) { 33 | handler.process({data: dataJson}) 34 | .catch(function(err) { 35 | assert.ok(err, 'failure reported for malformed JSON data'); 36 | done(); 37 | }); 38 | }); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /test/assets/cloudwatch.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id":"1234567890", 4 | "timestamp":1466263061409, 5 | "logType": "start", 6 | "requestId": "abcdef-1234-5678-9012-abcdef", 7 | "lambdaVersion": "$LATEST", 8 | "date": "2016-06-18T15:17:41.409Z" 9 | }, 10 | { 11 | "id":"1234567891", 12 | "timestamp":1466263061419, 13 | "date": "2016-06-18T15:17:41.419Z", 14 | "requestId": "abcdef-1234-5678-9012-abcdef", 15 | "logType": "message", 16 | "message":"Example log message 1\n" 17 | }, 18 | { 19 | "id":"1234567892", 20 | "timestamp":1466263061429, 21 | "date": "2016-06-18T15:17:41.429Z", 22 | "requestId": "abcdef-1234-5678-9012-abcdef", 23 | "logType": "message", 24 | "message":"Example log message 2\nLine 2\n" 25 | }, 26 | { 27 | "id":"1234567893", 28 | "timestamp":1466263061439, 29 | "date": "2016-06-18T15:17:41.439Z", 30 | "requestId": "abcdef-1234-5678-9012-abcdef", 31 | "logType": "end" 32 | }, 33 | { 34 | "id":"1234567894", 35 | "timestamp":1466263061449, 36 | "date": "2016-06-18T15:17:41.449Z", 37 | "requestId": "abcdef-1234-5678-9012-abcdef", 38 | "logType": "report", 39 | "duration": "432.10", 40 | "durationBilled": "500", 41 | "memConfigured": "256", 42 | "memUsed": "123" 43 | } 44 | ] 45 | -------------------------------------------------------------------------------- /test/decompressGzip.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/decompressGzip"); 7 | 8 | describe('handler/decompressGzip.js', function() { 9 | describe('#process()', function() { 10 | var dataGzip; 11 | var dataPlain; 12 | 13 | before(function() { 14 | dataGzip = fs.readFileSync("test/assets/message.txt.gz"); 15 | dataPlain = fs.readFileSync("test/assets/message.txt").toString(); 16 | }); 17 | 18 | it('should decompress gzip data', 19 | function(done) { 20 | handler.process({data: dataGzip, setting: true}) 21 | .then(function(result) { 22 | assert.ok(result.hasOwnProperty('setting'), 23 | 'process returns config object'); 24 | assert.strictEqual(result.data, dataPlain, 25 | 'gzipped data decompressed successfully'); 26 | done(); 27 | }); 28 | }); 29 | 30 | it('should fail if malformed gzip data is provided', 31 | function(done) { 32 | handler.process({data: dataPlain}) 33 | .catch(function(err) { 34 | assert.ok(err, 'failure reported for malformed gzip data'); 35 | done(); 36 | }); 37 | }); 38 | }); 39 | }); 40 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "lambda-stash", 3 | "version": "4.0.0", 4 | "description": "AWS Lambda script for shipping data from S3 or other cloud data sources to data stores, like Elasticsearch", 5 | "main": "index.js", 6 | "dependencies": { 7 | "aws-sdk": "^2.290.0", 8 | "csv-parse": "^4.10.1", 9 | "elasticsearch": "^15.5.0", 10 | "http-aws-es": "^6.0.0", 11 | "lodash": "^4.17.5" 12 | }, 13 | "devDependencies": { 14 | "babel-eslint": "^10.0.2", 15 | "coveralls": "^3.0.5", 16 | "eslint": "^6.1.0", 17 | "eslint-config-google": "^0.6.0", 18 | "mocha": "^6.2.0", 19 | "nyc": "^14.1.1" 20 | }, 21 | "engines": { 22 | "node": ">=8.0" 23 | }, 24 | "scripts": { 25 | "check-coverage": "nyc --check-coverage --statements=100 --reporter=lcov --reporter=text-summary npm run test", 26 | "lint": "eslint .", 27 | "test": "mocha --check-leaks --timeout 5000" 28 | }, 29 | "author": { 30 | "name": "Joe Turgeon", 31 | "email": "arithmetric@gmail.com" 32 | }, 33 | "repository": { 34 | "type": "git", 35 | "url": "https://github.com/arithmetric/lambda-stash.git" 36 | }, 37 | "keywords": [ 38 | "aws", 39 | "lambda", 40 | "log", 41 | "logs", 42 | "logger", 43 | "logstash", 44 | "shipper" 45 | ], 46 | "license": "MIT" 47 | } 48 | -------------------------------------------------------------------------------- /test/outputJsonLines.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/outputJsonLines"); 7 | 8 | describe('handler/outputJsonLines.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync("test/assets/table.json")); 15 | dataJson = fs.readFileSync("test/assets/table.jsonstring.txt").toString(); 16 | }); 17 | 18 | it('should convert arrays to lines of JSON', 19 | function(done) { 20 | var config = {data: dataSource, setting: true}; 21 | handler.process(config) 22 | .then(function(result) { 23 | assert.ok(result.hasOwnProperty('setting'), 24 | 'process returns config object'); 25 | assert.strictEqual(result.data, dataJson, 26 | 'JSON data parsed successfully'); 27 | done(); 28 | }); 29 | }); 30 | 31 | it('should fail if a non-array value is provided', 32 | function(done) { 33 | handler.process({data: dataJson}) 34 | .catch(function(err) { 35 | assert.ok(err, 'failure reported for malformed JSON data'); 36 | done(); 37 | }); 38 | }); 39 | }); 40 | }); 41 | -------------------------------------------------------------------------------- /test/parseCsv.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/parseCsv"); 7 | 8 | describe('handler/parseCsv.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = fs.readFileSync("test/assets/csv.source.txt"); 15 | dataJson = JSON.parse(fs.readFileSync("test/assets/table.json")); 16 | }); 17 | 18 | it('should parse CSV data', 19 | function(done) { 20 | var config = {data: dataSource, setting: true}; 21 | handler.process(config) 22 | .then(function(result) { 23 | assert.ok(result.hasOwnProperty('setting'), 24 | 'process returns config object'); 25 | assert.deepStrictEqual(result.data, dataJson, 26 | 'CSV data parsed successfully'); 27 | done(); 28 | }); 29 | }); 30 | 31 | it('should fail if malformed CSV data is provided', 32 | function(done) { 33 | // An unclosed quote should throw an error when processed by csv-parse. 34 | const badData = '"test test\t'; 35 | handler.process({data: badData}) 36 | .catch(function(err) { 37 | assert.ok(err, 'failure reported for malformed CSV data'); 38 | done(); 39 | }); 40 | }); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /test/assets/elbv1.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "timestamp": "2018-05-09T12:20:13.464900Z", 4 | "elb": "elb-test", 5 | "client": "12.23.34.45:5678", 6 | "backend": "10.0.10.1:8080", 7 | "request_processing_time": "0.000039", 8 | "backend_processing_time": "0.000905", 9 | "response_processing_time": "0.000022", 10 | "elb_status_code": "301", 11 | "backend_status_code": "301", 12 | "received_bytes": "0", 13 | "sent_bytes": "178", 14 | "request": "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1", 15 | "user_agent": "Amazon Route 53 Health Check Service; ref:1234567890;", 16 | "ssl_cipher": "-", 17 | "ssl_protocol": "-", 18 | "date": "2018-05-09T12:20:13.464900Z" 19 | }, 20 | { 21 | "timestamp": "2018-05-09T12:20:14.464900Z", 22 | "elb": "elb-test", 23 | "client": "12.23.34.45:5678", 24 | "backend": "10.0.10.1:8080", 25 | "request_processing_time": "0.000039", 26 | "backend_processing_time": "0.000905", 27 | "response_processing_time": "0.000022", 28 | "elb_status_code": "301", 29 | "backend_status_code": "301", 30 | "received_bytes": "0", 31 | "sent_bytes": "178", 32 | "request": "GET http://elb-test-1234567890.us-east-1.elb.amazonaws.com:80/ HTTP/1.1", 33 | "user_agent": "Amazon Route 53 Health Check Service; ref:1234567890;", 34 | "ssl_cipher": "-", 35 | "ssl_protocol": "-", 36 | "date": "2018-05-09T12:20:14.464900Z" 37 | } 38 | ] 39 | -------------------------------------------------------------------------------- /test/assets/elbv2.parse.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "https", 4 | "2016-08-10T23:39:43.065466Z", 5 | "app/my-loadbalancer/50dc6c495c0c9188", 6 | "192.168.131.39:2817", 7 | "10.0.0.1:80", 8 | "0.086", 9 | "0.048", 10 | "0.037", 11 | "200", 12 | "200", 13 | "0", 14 | "57", 15 | "GET https://www.example.com:443/ HTTP/1.1", 16 | "curl/7.46.0", 17 | "ECDHE-RSA-AES128-GCM-SHA256", 18 | "TLSv1.2", 19 | "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", 20 | "Root=1-58337281-1d84f3d73c47ec4e58577259", 21 | "www.example.com", 22 | "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012", 23 | "0" 24 | ], 25 | [ 26 | "https", 27 | "2016-08-10T23:39:44.065466Z", 28 | "app/my-loadbalancer/50dc6c495c0c9188", 29 | "192.168.131.39:2817", 30 | "10.0.0.1:80", 31 | "0.086", 32 | "0.048", 33 | "0.037", 34 | "200", 35 | "200", 36 | "0", 37 | "57", 38 | "GET https://www.example.com:443/ HTTP/1.1", 39 | "curl/7.46.0", 40 | "ECDHE-RSA-AES128-GCM-SHA256", 41 | "TLSv1.2", 42 | "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", 43 | "Root=1-58337281-1d84f3d73c47ec4e58577259", 44 | "www.example.com", 45 | "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012", 46 | "0", 47 | "new field" 48 | ] 49 | ] 50 | -------------------------------------------------------------------------------- /test/parseTabs.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/parseTabs"); 7 | 8 | describe('handler/parseTabs.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = fs.readFileSync("test/assets/tabs.source.txt"); 15 | dataJson = JSON.parse(fs.readFileSync("test/assets/table.json")); 16 | }); 17 | 18 | it('should parse tab separated data', 19 | function(done) { 20 | var config = {data: dataSource, setting: true}; 21 | handler.process(config) 22 | .then(function(result) { 23 | assert.ok(result.hasOwnProperty('setting'), 24 | 'process returns config object'); 25 | assert.deepStrictEqual(result.data, dataJson, 26 | 'tab separated data parsed successfully'); 27 | done(); 28 | }); 29 | }); 30 | 31 | it('should fail if malformed tab separated data is provided', 32 | function(done) { 33 | // An unclosed quote should throw an error when processed by csv-parse. 34 | const badData = '"test test\t'; 35 | handler.process({data: badData}) 36 | .catch(function(err) { 37 | assert.ok(err, 'failure reported for malformed tab separated data'); 38 | done(); 39 | }); 40 | }); 41 | }); 42 | }); 43 | -------------------------------------------------------------------------------- /handlers/formatELBv1.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('formatELB for Classic Load Balancers'); 3 | if (!config.data || 4 | (!config.data.length && config.data.length !== 0)) { 5 | return Promise.reject('Received unexpected ELB log format:' + 6 | JSON.stringify(config.data)); 7 | } 8 | 9 | var output = []; 10 | var fields = [ 11 | 'timestamp', 12 | 'elb', 13 | 'client', 14 | 'backend', 15 | 'request_processing_time', 16 | 'backend_processing_time', 17 | 'response_processing_time', 18 | 'elb_status_code', 19 | 'backend_status_code', 20 | 'received_bytes', 21 | 'sent_bytes', 22 | 'request', 23 | 'user_agent', 24 | 'ssl_cipher', 25 | 'ssl_protocol' 26 | ]; 27 | var numRows = config.data.length; 28 | var numCols; 29 | var i; 30 | var j; 31 | var row; 32 | var item; 33 | 34 | for (i = 0; i < numRows; i++) { 35 | row = config.data[i]; 36 | if (row.length !== 15) { 37 | console.log('Expected 15 columns in row. Found ' + row.length + ': ' + 38 | config.data[i]); 39 | } 40 | 41 | item = {}; 42 | numCols = Math.min(row.length, fields.length); 43 | for (j = 0; j < numCols; j++) { 44 | item[fields[j]] = row[j]; 45 | } 46 | if (config.dateField && config.dateField !== 'timestamp') { 47 | item[config.dateField] = item.timestamp; 48 | } 49 | output.push(item); 50 | } 51 | config.data = output; 52 | return Promise.resolve(config); 53 | }; 54 | -------------------------------------------------------------------------------- /test/formatELBv2.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatELBv2"); 7 | 8 | describe('handler/formatELBv2.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/elbv2.parse.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/elbv2.format.json")); 18 | }); 19 | 20 | it('should format parsed ELB data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'ELB data formatted successfully'); 33 | done(); 34 | }); 35 | }); 36 | 37 | it('should fail if malformed ELB data is provided', 38 | function(done) { 39 | var config = { 40 | data: {malformed: 'data'}, 41 | setting: true 42 | }; 43 | handler.process(config) 44 | .catch(function(err) { 45 | assert.ok(err, 'error is thrown'); 46 | done(); 47 | }); 48 | }); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /test/formatS3Access.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatS3Access"); 7 | 8 | describe('handler/formatS3Access.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/s3access.parse.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/s3access.format.json")); 18 | }); 19 | 20 | it('should format parsed S3 Access data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'S3 Access data formatted successfully'); 33 | done(); 34 | }); 35 | }); 36 | 37 | it('should fail if malformed S3 Access data is provided', 38 | function(done) { 39 | var config = { 40 | data: {malformed: 'data'}, 41 | setting: true 42 | }; 43 | handler.process(config) 44 | .catch(function(err) { 45 | assert.ok(err, 'error is thrown'); 46 | done(); 47 | }); 48 | }); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /test/formatConfig.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatConfig"); 7 | 8 | describe('handler/formatConfig.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/config.source.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/config.format.json")); 18 | }); 19 | 20 | it('should format parsed Config data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'Config data formatted successfully'); 33 | done(); 34 | }); 35 | }); 36 | 37 | it('should fail if malformed Config data is provided', 38 | function(done) { 39 | var config = { 40 | data: dataJson, 41 | dateField: 'date', 42 | setting: true 43 | }; 44 | handler.process(config) 45 | .catch(function(err) { 46 | assert.ok(err, 'error is thrown'); 47 | done(); 48 | }); 49 | }); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /test/formatCloudtrail.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatCloudtrail"); 7 | 8 | describe('handler/formatCloudtrail.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/cloudtrail.source.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/cloudtrail.format.json")); 18 | }); 19 | 20 | it('should format parsed Cloudtrail data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'Cloudtrail data formatted successfully'); 33 | done(); 34 | }); 35 | }); 36 | 37 | it('should fail if malformed Cloudtrail data is provided', 38 | function(done) { 39 | var config = { 40 | data: dataJson, 41 | dateField: 'date', 42 | setting: true 43 | }; 44 | handler.process(config) 45 | .catch(function(err) { 46 | assert.ok(err, 'error is thrown'); 47 | done(); 48 | }); 49 | }); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /test/formatCloudwatchLogs.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatCloudwatchLogs"); 7 | 8 | describe('handler/formatCloudwatchLogs.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/cloudwatch.data.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/cloudwatch.format.json")); 18 | }); 19 | 20 | it('should format parsed CloudWatch Logs data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'CloudWatch Logs data formatted successfully'); 33 | done(); 34 | }); 35 | }); 36 | 37 | it('should fail if malformed Cloudtrail data is provided', 38 | function(done) { 39 | var config = { 40 | data: dataJson, 41 | dateField: 'date', 42 | setting: true 43 | }; 44 | handler.process(config) 45 | .catch(function(err) { 46 | assert.ok(err, 'error is thrown'); 47 | done(); 48 | }); 49 | }); 50 | }); 51 | }); 52 | -------------------------------------------------------------------------------- /test/formatELBv1.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/formatELBv1"); 7 | 8 | describe('handler/formatELBv1.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync( 15 | "test/assets/elbv1.parse.json")); 16 | dataJson = JSON.parse(fs.readFileSync( 17 | "test/assets/elbv1.format.json")); 18 | }); 19 | 20 | it('should format parsed ELB data', 21 | function(done) { 22 | var config = { 23 | data: dataSource, 24 | dateField: 'date', 25 | setting: true 26 | }; 27 | handler.process(config) 28 | .then(function(result) { 29 | assert.ok(result.hasOwnProperty('setting'), 30 | 'process returns config object'); 31 | assert.deepStrictEqual(result.data, dataJson, 32 | 'ELB data formatted successfully'); 33 | done(); 34 | }) 35 | .catch(function(err) { 36 | console.log(err); 37 | }); 38 | }); 39 | 40 | it('should fail if malformed ELB data is provided', 41 | function(done) { 42 | var config = { 43 | data: {malformed: 'data'}, 44 | setting: true 45 | }; 46 | handler.process(config) 47 | .catch(function(err) { 48 | assert.ok(err, 'error is thrown'); 49 | done(); 50 | }); 51 | }); 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /handlers/formatELBv2.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('formatELBv2 for Application Load Balancers'); 3 | if (!config.data || 4 | (!config.data.length && config.data.length !== 0)) { 5 | return Promise.reject('Received unexpected ELB log format:' + 6 | JSON.stringify(config.data)); 7 | } 8 | 9 | var output = []; 10 | var fields = [ 11 | 'type', 12 | 'timestamp', 13 | 'elb', 14 | 'client', 15 | 'target', 16 | 'request_processing_time', 17 | 'target_processing_time', 18 | 'response_processing_time', 19 | 'elb_status_code', 20 | 'target_status_code', 21 | 'received_bytes', 22 | 'sent_bytes', 23 | 'request', 24 | 'user_agent', 25 | 'ssl_cipher', 26 | 'ssl_protocol', 27 | 'target_group_arn', 28 | 'trace_id', 29 | 'domain_name', 30 | 'chosen_cert_arn', 31 | 'matched_rule_priority' 32 | ]; 33 | var numRows = config.data.length; 34 | var numCols; 35 | var i; 36 | var j; 37 | var row; 38 | var item; 39 | 40 | for (i = 0; i < numRows; i++) { 41 | row = config.data[i]; 42 | if (row.length !== 21) { 43 | console.log('Expected 21 columns in row. Found ' + row.length + ': ' + 44 | config.data[i]); 45 | } 46 | 47 | item = {}; 48 | numCols = Math.min(row.length, fields.length); 49 | for (j = 0; j < numCols; j++) { 50 | item[fields[j]] = row[j]; 51 | } 52 | if (config.dateField && config.dateField !== 'timestamp') { 53 | item[config.dateField] = item.timestamp; 54 | } 55 | output.push(item); 56 | } 57 | config.data = output; 58 | return Promise.resolve(config); 59 | }; 60 | -------------------------------------------------------------------------------- /handlers/formatS3Access.js: -------------------------------------------------------------------------------- 1 | exports.process = function(config) { 2 | console.log('formatS3Access for S3 hosted application logs'); 3 | if (!config.data || 4 | (!config.data.length && config.data.length !== 0)) { 5 | return Promise.reject('Received unexpected S3 log format:' + 6 | JSON.stringify(config.data)); 7 | } 8 | 9 | var output = []; 10 | var fields = [ 11 | 'bucket_owner', 12 | 'bucket', 13 | 'time', 14 | 'zone', 15 | 'remote_ip', 16 | 'requester', 17 | 'request_id', 18 | 'operation', 19 | 'key', 20 | 'request_uri', 21 | 'http_status', 22 | 'error_code', 23 | 'bytes_sent', 24 | 'object_size', 25 | 'total_time', 26 | 'turn_around_time', 27 | 'referrer', 28 | 'user_agent', 29 | 'version_id', 30 | 'host_id', 31 | 'signature_version', 32 | 'cipher_suite', 33 | 'authentication_type', 34 | 'host_header', 35 | 'tls_version' 36 | ]; 37 | var numRows = config.data.length; 38 | var numCols; 39 | var i; 40 | var j; 41 | var row; 42 | var item; 43 | 44 | for (i = 0; i < numRows; i++) { 45 | row = config.data[i]; 46 | if (row.length !== 25) { 47 | console.log('Expected 25 columns in row. Found ' + row.length + ': ' + 48 | config.data[i]); 49 | } 50 | 51 | item = {}; 52 | numCols = Math.min(row.length, fields.length); 53 | for (j = 0; j < numCols; j++) { 54 | item[fields[j]] = row[j]; 55 | } 56 | item.time = item.time.replace(/:/, ' ').replace(/\[/, ''); 57 | item.zone = item.zone.replace(/\]/, ''); 58 | item[config.dateField] = new Date(item.time + item.zone).toISOString(); 59 | output.push(item); 60 | } 61 | config.data = output; 62 | return Promise.resolve(config); 63 | }; 64 | -------------------------------------------------------------------------------- /test/convertString.spec.js: -------------------------------------------------------------------------------- 1 | /* global before, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/convertString"); 7 | 8 | describe('handler/convertString.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataString; 12 | 13 | before(function() { 14 | dataSource = JSON.parse(fs.readFileSync("test/assets/log.json")); 15 | dataString = fs.readFileSync("test/assets/log.string.txt").toString(); 16 | }); 17 | 18 | it('should convert an array to lines of text', 19 | function(done) { 20 | var config = {data: dataSource, setting: true}; 21 | handler.process(config) 22 | .then(function(result) { 23 | assert.ok(result.hasOwnProperty('setting'), 24 | 'process returns config object'); 25 | assert.equal(result.data, dataString, 26 | 'converted to string successfully'); 27 | done(); 28 | }); 29 | }); 30 | 31 | it('should support a configurable prefix and suffix', 32 | function(done) { 33 | var config = { 34 | data: dataSource, 35 | setting: true, 36 | string: { 37 | prefix: 'prefix', 38 | suffix: 'suffix' 39 | } 40 | }; 41 | dataString = fs.readFileSync("test/assets/log.string.config.txt") 42 | .toString(); 43 | handler.process(config) 44 | .then(function(result) { 45 | assert.ok(result.hasOwnProperty('setting'), 46 | 'process returns config object'); 47 | assert.equal(result.data, dataString, 48 | 'converted to string with prefix and suffix successfully'); 49 | done(); 50 | }); 51 | }); 52 | 53 | it('should fail if a non-array value is provided', 54 | function(done) { 55 | handler.process({data: dataString}) 56 | .catch(function(err) { 57 | assert.ok(err, 'failure reported for malformed JSON data'); 58 | done(); 59 | }); 60 | }); 61 | }); 62 | }); 63 | -------------------------------------------------------------------------------- /test/parseSpaces.spec.js: -------------------------------------------------------------------------------- 1 | /* global describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var handler = require("../handlers/parseSpaces"); 7 | 8 | describe('handler/parseSpaces.js', function() { 9 | describe('#process()', function() { 10 | var dataSource; 11 | var dataJson; 12 | 13 | it('should parse space separated ELBv2 data', 14 | function(done) { 15 | dataSource = fs.readFileSync("test/assets/elbv2.source.txt"); 16 | dataJson = JSON.parse(fs.readFileSync("test/assets/elbv2.parse.json")); 17 | var config = {data: dataSource, setting: true}; 18 | handler.process(config) 19 | .then(function(result) { 20 | assert.ok(result.hasOwnProperty('setting'), 21 | 'process returns config object'); 22 | assert.deepStrictEqual(result.data, dataJson, 23 | 'space separated data parsed successfully'); 24 | done(); 25 | }); 26 | }); 27 | 28 | it('should parse space separated S3 log data', 29 | function(done) { 30 | dataSource = fs.readFileSync("test/assets/s3access.source.txt"); 31 | dataJson = 32 | JSON.parse(fs.readFileSync("test/assets/s3access.parse.json")); 33 | var config = {data: dataSource, setting: true}; 34 | handler.process(config) 35 | .then(function(result) { 36 | assert.ok(result.hasOwnProperty('setting'), 37 | 'process returns config object'); 38 | assert.deepStrictEqual(result.data, dataJson, 39 | 'space separated data parsed successfully'); 40 | done(); 41 | }); 42 | }); 43 | 44 | it('should fail if malformed space separated data is provided', 45 | function(done) { 46 | // An unclosed quote should throw an error when processed by csv-parse. 47 | const badData = '"test test'; 48 | handler.process({data: badData}) 49 | .catch(function(err) { 50 | assert.ok(err, 51 | 'failure reported for malformed space separated data'); 52 | done(); 53 | }); 54 | }); 55 | }); 56 | }); 57 | -------------------------------------------------------------------------------- /handlers/shipElasticsearch.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'); 2 | 3 | exports.process = function(config) { 4 | console.log('shipElasticsearch'); 5 | 6 | var esConfig = { 7 | host: config.elasticsearch.host 8 | }; 9 | 10 | if (config.elasticsearch.requestTimeout) { 11 | esConfig.requestTimeout = config.elasticsearch.requestTimeout; 12 | } 13 | 14 | if (config.elasticsearch.useAWS) { 15 | esConfig.connectionClass = require('http-aws-es'); 16 | esConfig.amazonES = { 17 | region: config.elasticsearch.region 18 | }; 19 | if (config.elasticsearch.accessKey && 20 | config.elasticsearch.secretKey) { 21 | esConfig.amazonES.accessKey = config.elasticsearch.accessKey; 22 | esConfig.amazonES.secretKey = config.elasticsearch.secretKey; 23 | } else { 24 | var AWS = require('aws-sdk'); 25 | esConfig.amazonES.credentials = new AWS.EnvironmentCredentials('AWS'); 26 | } 27 | } 28 | 29 | var es = require('elasticsearch').Client(esConfig); // eslint-disable-line new-cap 30 | var docs = []; 31 | var maxChunkSize = config.elasticsearch.maxChunkSize || 1000; 32 | var promises = []; 33 | 34 | var ship = function(docs) { 35 | console.log('Preparing to ship ' + (docs.length / 2) + 36 | ' records to Elasticsearch.'); 37 | return new Promise(function(resolve, reject) { 38 | es.bulk({body: docs}, function(err, result) { 39 | if (err) { 40 | return reject(err); 41 | } else if (result.errors) { 42 | return reject(result); 43 | } 44 | resolve(); 45 | }); 46 | }); 47 | }; 48 | 49 | _.forEach(config.data, function(datum) { 50 | docs.push({ 51 | index: { 52 | _index: config.elasticsearch.index, 53 | _type: config.elasticsearch.type 54 | } 55 | }); 56 | docs.push(datum); 57 | 58 | if ((docs.length / 2) >= maxChunkSize) { 59 | var chunk = docs; 60 | promises.push(ship(chunk)); 61 | docs = []; 62 | } 63 | }); 64 | if (docs.length) { 65 | promises.push(ship(docs)); 66 | } 67 | 68 | return Promise.all(promises) 69 | .then(function() { 70 | return config; 71 | }); 72 | }; 73 | -------------------------------------------------------------------------------- /test/assets/elbv2.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "type": "https", 4 | "timestamp": "2016-08-10T23:39:43.065466Z", 5 | "elb": "app/my-loadbalancer/50dc6c495c0c9188", 6 | "client": "192.168.131.39:2817", 7 | "target": "10.0.0.1:80", 8 | "request_processing_time": "0.086", 9 | "target_processing_time": "0.048", 10 | "response_processing_time": "0.037", 11 | "elb_status_code": "200", 12 | "target_status_code": "200", 13 | "received_bytes": "0", 14 | "sent_bytes": "57", 15 | "request": "GET https://www.example.com:443/ HTTP/1.1", 16 | "user_agent": "curl/7.46.0", 17 | "ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", 18 | "ssl_protocol": "TLSv1.2", 19 | "target_group_arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", 20 | "trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259", 21 | "domain_name": "www.example.com", 22 | "chosen_cert_arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012", 23 | "matched_rule_priority": "0", 24 | "date": "2016-08-10T23:39:43.065466Z" 25 | }, 26 | { 27 | "type": "https", 28 | "timestamp": "2016-08-10T23:39:44.065466Z", 29 | "elb": "app/my-loadbalancer/50dc6c495c0c9188", 30 | "client": "192.168.131.39:2817", 31 | "target": "10.0.0.1:80", 32 | "request_processing_time": "0.086", 33 | "target_processing_time": "0.048", 34 | "response_processing_time": "0.037", 35 | "elb_status_code": "200", 36 | "target_status_code": "200", 37 | "received_bytes": "0", 38 | "sent_bytes": "57", 39 | "request": "GET https://www.example.com:443/ HTTP/1.1", 40 | "user_agent": "curl/7.46.0", 41 | "ssl_cipher": "ECDHE-RSA-AES128-GCM-SHA256", 42 | "ssl_protocol": "TLSv1.2", 43 | "target_group_arn": "arn:aws:elasticloadbalancing:us-east-2:123456789012:targetgroup/my-targets/73e2d6bc24d8a067", 44 | "trace_id": "Root=1-58337281-1d84f3d73c47ec4e58577259", 45 | "domain_name": "www.example.com", 46 | "chosen_cert_arn": "arn:aws:acm:us-east-2:123456789012:certificate/12345678-1234-1234-1234-123456789012", 47 | "matched_rule_priority": "0", 48 | "date": "2016-08-10T23:39:44.065466Z" 49 | } 50 | ] 51 | -------------------------------------------------------------------------------- /test/assets/s3access.source.txt: -------------------------------------------------------------------------------- 1 | 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 3E57427F3EXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 7 - "-" "S3Console/0.4" - s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 2 | 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 891CE47D2EXAMPLE REST.GET.LOGGING_STATUS - "GET /awsexamplebucket?logging HTTP/1.1" 200 - 242 - 11 - "-" "S3Console/0.4" - 9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 3 | 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:00:38 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be A1206F460EXAMPLE REST.GET.BUCKETPOLICY - "GET /awsexamplebucket?policy HTTP/1.1" 404 NoSuchBucketPolicy 297 - 38 - "-" "S3Console/0.4" - BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 4 | 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:00 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be 7B4A0FABBEXAMPLE REST.GET.VERSIONING - "GET /awsexamplebucket?versioning HTTP/1.1" 200 - 113 - 33 - "-" "S3Console/0.4" - Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234= SigV2 ECDHE-RSA-AES128-GCM-SHA256 AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 5 | 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be awsexamplebucket [06/Feb/2019:00:01:57 +0000] 192.0.2.3 79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be DD6CC733AEXAMPLE REST.PUT.OBJECT s3-dg.pdf "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1" 200 - - 4406583 41754 28 "-" "S3Console/0.4" - 10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234= SigV4 ECDHE-RSA-AES128-SHA AuthHeader awsexamplebucket.s3.amazonaws.com TLSV1.1 "extra field" 6 | -------------------------------------------------------------------------------- /handlers/formatCloudwatchLogs.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'); 2 | 3 | exports.process = function(config) { 4 | console.log('formatCloudwatchLogs'); 5 | if (!config.data || 6 | !config.data.hasOwnProperty('logEvents') || 7 | _.isNil(config.data.logEvents.length)) { 8 | return Promise.reject('Received unexpected AWS Cloudwatch Logs format:' + 9 | JSON.stringify(config.data)); 10 | } 11 | 12 | var items = []; 13 | var num = config.data.logEvents.length; 14 | var i; 15 | var item; 16 | var parts; 17 | for (i = 0; i < num; i++) { 18 | item = config.data.logEvents[i]; 19 | 20 | parts = item.message.match(/^(\w+)/); 21 | if (parts && parts[1] === 'START') { 22 | item.logType = 'start'; 23 | parts = item.message.match(/^START RequestId: ([a-z0-9-]+) Version: (\S+)/); // eslint-disable-line max-len 24 | if (parts && parts.length >= 3) { 25 | item.requestId = parts[1]; 26 | item.lambdaVersion = parts[2]; 27 | delete item.message; 28 | } 29 | } else if (parts && parts[1] === 'REPORT') { 30 | item.logType = 'report'; 31 | parts = item.message.match(/^REPORT RequestId: ([a-z0-9-]+)\tDuration: ([0-9.]+) ms\tBilled Duration: ([0-9.]+) ms\tMemory Size: ([0-9.]+) MB\tMax Memory Used: ([0-9.]+)/); // eslint-disable-line max-len 32 | if (parts && parts.length >= 6) { 33 | item.requestId = parts[1]; 34 | item.duration = parts[2]; 35 | item.durationBilled = parts[3]; 36 | item.memConfigured = parts[4]; 37 | item.memUsed = parts[5]; 38 | delete item.message; 39 | } 40 | } else if (parts && parts[1] === 'END') { 41 | item.logType = 'end'; 42 | parts = item.message.match(/^END RequestId: ([a-z0-9-]+)/); 43 | if (parts && parts[1]) { 44 | item.requestId = parts[1]; 45 | delete item.message; 46 | } 47 | } else { 48 | item.logType = 'message'; 49 | parts = item.message.match(/^(.*)\t(.*)\t((.|\n)*)/m); 50 | if (parts && parts.length >= 5) { 51 | item.requestId = parts[2]; 52 | item.message = parts[3]; 53 | } 54 | } 55 | 56 | if (config.dateField && config.dateField !== 'timestamp') { 57 | item[config.dateField] = new Date(item.timestamp).toISOString(); 58 | } 59 | items.push(item); 60 | } 61 | config.data = items; 62 | return Promise.resolve(config); 63 | }; 64 | -------------------------------------------------------------------------------- /test/getS3Object.spec.js: -------------------------------------------------------------------------------- 1 | /* global describe, it */ 2 | 3 | var assert = require("assert"); 4 | 5 | var handler = require("../handlers/getS3Object"); 6 | 7 | describe('handler/getS3Object.js', function() { 8 | describe('#process()', function() { 9 | it('should return data for an S3 key', 10 | function(done) { 11 | var AWS = require('aws-sdk'); 12 | AWS.S3 = function() { 13 | return { 14 | getObject: function(params, callback) { 15 | if (params.Bucket === 'source-bucket' && 16 | params.Key === 'source/key') { 17 | callback(null, {Body: 'successful-response'}); 18 | } 19 | } 20 | }; 21 | }; 22 | var config = { 23 | S3: { 24 | srcBucket: 'source-bucket', 25 | srcKey: 'source/key' 26 | } 27 | }; 28 | handler.process(config) 29 | .then(function(result) { 30 | assert.strictEqual(result.data, 'successful-response', 31 | 'S3.getObject() returned data successfully'); 32 | done(); 33 | }); 34 | }); 35 | 36 | it('should fail if S3.getObject() returns an error', 37 | function(done) { 38 | var AWS = require('aws-sdk'); 39 | AWS.S3 = function() { 40 | return { 41 | getObject: function(params, callback) { 42 | callback(new Error('test error')); 43 | } 44 | }; 45 | }; 46 | var config = { 47 | S3: { 48 | srcBucket: 'source-bucket', 49 | srcKey: 'source/key' 50 | } 51 | }; 52 | handler.process(config) 53 | .catch(function(err) { 54 | assert.ok(err, 'error was thrown and caught'); 55 | done(); 56 | }); 57 | }); 58 | 59 | it('should fail if S3.getObject() returns unexpected data', 60 | function(done) { 61 | var AWS = require('aws-sdk'); 62 | AWS.S3 = function() { 63 | return { 64 | getObject: function(params, callback) { 65 | callback(null, 'malformed data'); 66 | } 67 | }; 68 | }; 69 | var config = { 70 | S3: { 71 | srcBucket: 'source-bucket', 72 | srcKey: 'source/key' 73 | } 74 | }; 75 | handler.process(config) 76 | .catch(function(err) { 77 | assert.ok(err, 'error was thrown and caught'); 78 | done(); 79 | }); 80 | }); 81 | }); 82 | }); 83 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'); 2 | 3 | exports.handler = function(config, event, context, callback) { 4 | var taskNames = []; 5 | var eventType = ''; 6 | 7 | if (event.hasOwnProperty('Records') && event.Records.length && 8 | event.Records[0].eventSource === 'aws:s3') { 9 | config.S3 = { 10 | srcBucket: event.Records[0].s3.bucket.name, 11 | srcKey: event.Records[0].s3.object.key 12 | }; 13 | eventType = 'S3'; 14 | taskNames.push('getS3Object'); 15 | console.log('Handling event for s3://' + config.S3.srcBucket + '/' + 16 | config.S3.srcKey); 17 | } else if (event.hasOwnProperty('awslogs') && 18 | event.awslogs.hasOwnProperty('data')) { 19 | config.data = event.awslogs.data; 20 | eventType = 'CloudWatch'; 21 | taskNames.push('decodeBase64'); 22 | taskNames.push('decompressGzip'); 23 | taskNames.push('parseJson'); 24 | console.log('Handling event for CloudWatch logs'); 25 | } 26 | 27 | var currentMapping; 28 | if (config.mappings) { 29 | _.some(config.mappings, function(item) { 30 | if (item.type === eventType || 31 | (config.S3 && item.bucket === config.S3.srcBucket)) { 32 | currentMapping = item; 33 | console.log('Selected mapping for S3 event:', item); 34 | if (item.hasOwnProperty('processors')) { 35 | taskNames = taskNames.concat(item.processors); 36 | } 37 | config = _.merge({}, config, item); 38 | return true; 39 | } 40 | }); 41 | delete config.mappings; 42 | } 43 | 44 | if (!currentMapping) { 45 | console.log('Event did not match any mappings.'); 46 | return callback(null, 'Event did not match any mappings.'); 47 | } 48 | 49 | console.log('Running ' + taskNames.length + ' handlers with config:', config); 50 | var tasks = []; 51 | var processor; 52 | _.some(taskNames, function(taskName) { 53 | if (_.isFunction(taskName)) { 54 | tasks.push(taskName); 55 | return false; 56 | } 57 | 58 | try { 59 | processor = require('./handlers/' + taskName); 60 | } catch (err) { 61 | context.fail(err); 62 | return true; 63 | } 64 | if (processor.hasOwnProperty('process')) { 65 | tasks.push(processor.process); 66 | } 67 | }); 68 | 69 | console.log('Starting to run processor tasks...'); 70 | 71 | Promise.series(tasks, config) 72 | .then(function(/* config */) { 73 | console.log('Successfully shipped data!'); 74 | callback(null, 'Successfully shipped data!'); 75 | }) 76 | .catch(function(err) { 77 | console.log('Error occurred while preparing to ship data:', err); 78 | context.fail('Error occurred while preparing to ship data'); 79 | }); 80 | }; 81 | 82 | Promise.series = function(promises, initValue) { 83 | return promises.reduce(function(chain, promise) { 84 | return chain.then(promise); 85 | }, Promise.resolve(initValue)); 86 | }; 87 | -------------------------------------------------------------------------------- /test/shipTcp.spec.js: -------------------------------------------------------------------------------- 1 | /* global beforeEach, describe, it */ 2 | 3 | var assert = require("assert"); 4 | 5 | var handler; 6 | 7 | describe('handler/shipTcp.js', function() { 8 | describe('#process()', function() { 9 | beforeEach(function() { 10 | // Clear the module cache to reset overridden functions. 11 | delete require.cache[require.resolve('../handlers/shipTcp')]; 12 | handler = require('../handlers/shipTcp'); 13 | }); 14 | 15 | it('should ship data using the net module', 16 | function(done) { 17 | var net = require('net'); 18 | var tcpReqSent = false; 19 | var tcpReqEnded = false; 20 | net.connect = function(port, host, _callback) { 21 | var closeCallback; 22 | assert.strictEqual(port, '7654', 'TCP port provided'); 23 | assert.strictEqual(host, 'mock', 'TCP host provided'); 24 | process.nextTick(_callback); 25 | return { 26 | end: function() { 27 | tcpReqEnded = true; 28 | closeCallback(); 29 | }, 30 | on: function(event, _callback) { 31 | closeCallback = _callback; 32 | }, 33 | write: function(data, _callback) { 34 | assert.strictEqual(data, 'test data 123456', 35 | 'data written in request'); 36 | tcpReqSent = true; 37 | _callback(); 38 | } 39 | }; 40 | }; 41 | var config = { 42 | tcp: { 43 | host: 'mock', 44 | port: '7654' 45 | }, 46 | data: 'test data 123456', 47 | test: 'test' 48 | }; 49 | handler.process(config) 50 | .then(function(result) { 51 | assert.ok(tcpReqSent, 'TCP request sent'); 52 | assert.ok(tcpReqEnded, 'TCP request closed'); 53 | assert.strictEqual(result.test, 'test', 'config data returned'); 54 | done(); 55 | }); 56 | }); 57 | 58 | it('should fail if the TCP request throws an error', 59 | function(done) { 60 | var net = require('net'); 61 | net.connect = function(port, host, _callback) { 62 | process.nextTick(_callback); 63 | return { 64 | end: function() { 65 | }, 66 | on: function(event, _callback) { 67 | assert.strictEqual(event, 'close', 'TCP close event handled'); 68 | _callback(); 69 | }, 70 | write: function(data, _callback) { 71 | } 72 | }; 73 | }; 74 | var config = { 75 | tcp: { 76 | host: 'mock', 77 | port: '7654' 78 | }, 79 | data: 'test data 123456', 80 | test: 'test' 81 | }; 82 | handler.process(config) 83 | .catch(function(err) { 84 | assert.ok(err, 'error is thrown'); 85 | done(); 86 | }); 87 | }); 88 | }); 89 | }); 90 | -------------------------------------------------------------------------------- /test/shipHttp.spec.js: -------------------------------------------------------------------------------- 1 | /* global beforeEach, describe, it */ 2 | 3 | var assert = require("assert"); 4 | 5 | var handler; 6 | 7 | describe('handler/shipHttp.js', function() { 8 | describe('#process()', function() { 9 | beforeEach(function() { 10 | // Clear the module cache to reset overridden functions. 11 | delete require.cache[require.resolve('../handlers/shipHttp')]; 12 | handler = require('../handlers/shipHttp'); 13 | }); 14 | 15 | it('should ship data using the http module', 16 | function(done) { 17 | var http = require('http'); 18 | var httpReqSent = false; 19 | var httpReqEnded = false; 20 | http.request = function(options, _callback) { 21 | var callback = _callback; 22 | assert.strictEqual(options.method, 'POST', 'HTTP method provided'); 23 | assert.strictEqual(options.protocol, 'http:', 24 | 'HTTP protocol provided'); 25 | assert.strictEqual(options.hostname, 'mock', 'HTTP host provided'); 26 | assert.strictEqual(options.pathname, '/logs', 'HTTP path provided'); 27 | assert.strictEqual(options.query, 'app=abc123', 28 | 'HTTP query provided'); 29 | return { 30 | end: function() { 31 | httpReqEnded = true; 32 | callback({statusCode: 201}); 33 | }, 34 | on: function(event, _callback) { 35 | return; 36 | }, 37 | write: function(data) { 38 | assert.strictEqual(data, 'test data 123456', 39 | 'data written in request'); 40 | httpReqSent = true; 41 | } 42 | }; 43 | }; 44 | var config = { 45 | http: { 46 | url: 'http://mock/logs?app=abc123' 47 | }, 48 | data: 'test data 123456', 49 | test: 'test' 50 | }; 51 | handler.process(config) 52 | .then(function(result) { 53 | assert.ok(httpReqSent, 'HTTP request sent'); 54 | assert.ok(httpReqEnded, 'HTTP request closed'); 55 | assert.strictEqual(result.test, 'test', 'config data returned'); 56 | done(); 57 | }); 58 | }); 59 | 60 | it('should fail if the HTTP request throws an error', 61 | function(done) { 62 | var http = require('http'); 63 | http.request = function(options, _callback) { 64 | return { 65 | end: function() { 66 | }, 67 | on: function(event, callback) { 68 | callback(new Error('test error')); 69 | }, 70 | write: function(data) { 71 | } 72 | }; 73 | }; 74 | var config = { 75 | http: { 76 | url: 'http://mock/logs?app=abc123' 77 | }, 78 | data: 'test data 123456', 79 | test: 'test' 80 | }; 81 | handler.process(config) 82 | .catch(function(err) { 83 | assert.ok(err, 'error is thrown'); 84 | done(); 85 | }); 86 | }); 87 | }); 88 | }); 89 | -------------------------------------------------------------------------------- /test/assets/s3access.parse.json: -------------------------------------------------------------------------------- 1 | [ 2 | [ 3 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 4 | "awsexamplebucket", 5 | "[06/Feb/2019:00:00:38", 6 | "+0000]", 7 | "192.0.2.3", 8 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 9 | "3E57427F3EXAMPLE", 10 | "REST.GET.VERSIONING", 11 | "-", 12 | "GET /awsexamplebucket?versioning HTTP/1.1", 13 | "200", 14 | "-", 15 | "113", 16 | "-", 17 | "7", 18 | "-", 19 | "-", 20 | "S3Console/0.4", 21 | "-", 22 | "s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=", 23 | "SigV2", 24 | "ECDHE-RSA-AES128-GCM-SHA256", 25 | "AuthHeader", 26 | "awsexamplebucket.s3.amazonaws.com", 27 | "TLSV1.1" 28 | ], 29 | [ 30 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 31 | "awsexamplebucket", 32 | "[06/Feb/2019:00:00:38", 33 | "+0000]", 34 | "192.0.2.3", 35 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 36 | "891CE47D2EXAMPLE", 37 | "REST.GET.LOGGING_STATUS", 38 | "-", 39 | "GET /awsexamplebucket?logging HTTP/1.1", 40 | "200", 41 | "-", 42 | "242", 43 | "-", 44 | "11", 45 | "-", 46 | "-", 47 | "S3Console/0.4", 48 | "-", 49 | "9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=", 50 | "SigV2", 51 | "ECDHE-RSA-AES128-GCM-SHA256", 52 | "AuthHeader", 53 | "awsexamplebucket.s3.amazonaws.com", 54 | "TLSV1.1" 55 | ], 56 | [ 57 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 58 | "awsexamplebucket", 59 | "[06/Feb/2019:00:00:38", 60 | "+0000]", 61 | "192.0.2.3", 62 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 63 | "A1206F460EXAMPLE", 64 | "REST.GET.BUCKETPOLICY", 65 | "-", 66 | "GET /awsexamplebucket?policy HTTP/1.1", 67 | "404", 68 | "NoSuchBucketPolicy", 69 | "297", 70 | "-", 71 | "38", 72 | "-", 73 | "-", 74 | "S3Console/0.4", 75 | "-", 76 | "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=", 77 | "SigV2", 78 | "ECDHE-RSA-AES128-GCM-SHA256", 79 | "AuthHeader", 80 | "awsexamplebucket.s3.amazonaws.com", 81 | "TLSV1.1" 82 | ], 83 | [ 84 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 85 | "awsexamplebucket", 86 | "[06/Feb/2019:00:01:00", 87 | "+0000]", 88 | "192.0.2.3", 89 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 90 | "7B4A0FABBEXAMPLE", 91 | "REST.GET.VERSIONING", 92 | "-", 93 | "GET /awsexamplebucket?versioning HTTP/1.1", 94 | "200", 95 | "-", 96 | "113", 97 | "-", 98 | "33", 99 | "-", 100 | "-", 101 | "S3Console/0.4", 102 | "-", 103 | "Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=", 104 | "SigV2", 105 | "ECDHE-RSA-AES128-GCM-SHA256", 106 | "AuthHeader", 107 | "awsexamplebucket.s3.amazonaws.com", 108 | "TLSV1.1" 109 | ], 110 | [ 111 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 112 | "awsexamplebucket", 113 | "[06/Feb/2019:00:01:57", 114 | "+0000]", 115 | "192.0.2.3", 116 | "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 117 | "DD6CC733AEXAMPLE", 118 | "REST.PUT.OBJECT", 119 | "s3-dg.pdf", 120 | "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1", 121 | "200", 122 | "-", 123 | "-", 124 | "4406583", 125 | "41754", 126 | "28", 127 | "-", 128 | "S3Console/0.4", 129 | "-", 130 | "10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234=", 131 | "SigV4", 132 | "ECDHE-RSA-AES128-SHA", 133 | "AuthHeader", 134 | "awsexamplebucket.s3.amazonaws.com", 135 | "TLSV1.1", 136 | "extra field" 137 | ] 138 | ] 139 | -------------------------------------------------------------------------------- /test/shipElasticsearch.spec.js: -------------------------------------------------------------------------------- 1 | /* global beforeEach, describe, it */ 2 | 3 | var assert = require("assert"); 4 | 5 | var handler; 6 | 7 | describe('handler/shipElasticsearch.js', function() { 8 | describe('#process()', function() { 9 | beforeEach(function() { 10 | // Clear the module cache to reset overridden functions. 11 | delete require.cache[require.resolve('../handlers/shipElasticsearch')]; 12 | handler = require('../handlers/shipElasticsearch'); 13 | }); 14 | 15 | it('should split data by the maxChunkSize', 16 | function(done) { 17 | var passVal1 = false; 18 | var passVal2 = false; 19 | var es = require('elasticsearch'); 20 | es.Client = function(config) { 21 | assert.strictEqual(config.host, 'http://mock', 'host param provided'); 22 | return { 23 | bulk: function(params, callback) { 24 | if (params.body[1].value === '1') { 25 | passVal1 = true; 26 | } else if (params.body[1].value === '2') { 27 | passVal2 = true; 28 | } 29 | callback(null, {errors: false}); 30 | } 31 | }; 32 | }; 33 | var config = { 34 | elasticsearch: { 35 | host: 'http://mock', 36 | index: 'index', 37 | type: 'type', 38 | maxChunkSize: 1 39 | }, 40 | data: [ 41 | { 42 | value: '1' 43 | }, 44 | { 45 | value: '2' 46 | } 47 | ], 48 | test: 'test' 49 | }; 50 | handler.process(config) 51 | .then(function(result) { 52 | assert.ok(passVal1, 'chunk 1 shipped'); 53 | assert.ok(passVal2, 'chunk 2 shipped'); 54 | assert.strictEqual(result.test, 'test', 'config data returned'); 55 | done(); 56 | }); 57 | }); 58 | 59 | it('should accept configuration for AWS Elasticsearch', 60 | function(done) { 61 | var config = { 62 | elasticsearch: { 63 | index: 'index', 64 | type: 'type', 65 | useAWS: true, 66 | region: 'us-east-1', 67 | accessKey: 'access_key', 68 | secretKey: 'shh_secret', 69 | requestTimeout: 90000 70 | }, 71 | data: [ 72 | { 73 | value: '1' 74 | }, 75 | { 76 | value: '2' 77 | } 78 | ], 79 | test: 'test' 80 | }; 81 | var es = require('elasticsearch'); 82 | es.Client = function(config) { 83 | assert.strictEqual(config.amazonES.region, 'us-east-1', 84 | 'region param provided'); 85 | assert.strictEqual(config.amazonES.accessKey, 'access_key', 86 | 'key param provided'); 87 | assert.strictEqual(config.amazonES.secretKey, 'shh_secret', 88 | 'secret param provided'); 89 | assert.strictEqual(config.requestTimeout, 90000, 90 | 'request timeout param provided'); 91 | done(); 92 | }; 93 | handler.process(config); 94 | }); 95 | 96 | it('should allow using AWS environment credentials', 97 | function(done) { 98 | var config = { 99 | elasticsearch: { 100 | index: 'index', 101 | type: 'type', 102 | useAWS: true, 103 | region: 'us-east-1' 104 | }, 105 | data: [ 106 | { 107 | value: '1' 108 | } 109 | ], 110 | test: 'test' 111 | }; 112 | var es = require('elasticsearch'); 113 | es.Client = function(config) { 114 | assert.strictEqual(config.amazonES.credentials.envPrefix, 'AWS', 115 | 'environment credentials provided'); 116 | assert.strictEqual(config.amazonES.region, 'us-east-1', 117 | 'region param provided'); 118 | done(); 119 | }; 120 | handler.process(config); 121 | }); 122 | 123 | it('should fail if Elasticsearch bulk results have an error', 124 | function(done) { 125 | var es = require('elasticsearch'); 126 | es.Client = function() { 127 | return { 128 | bulk: function(params, callback) { 129 | callback(null, {errors: true}); 130 | } 131 | }; 132 | }; 133 | var config = { 134 | elasticsearch: { 135 | index: 'index', 136 | type: 'type' 137 | }, 138 | data: [ 139 | { 140 | value: '1' 141 | } 142 | ] 143 | }; 144 | handler.process(config) 145 | .catch(function(err) { 146 | assert.ok(err, 'error is thrown'); 147 | done(); 148 | }); 149 | }); 150 | 151 | it('should fail if Elasticsearch bulk post throws an error', 152 | function(done) { 153 | var es = require('elasticsearch'); 154 | es.Client = function() { 155 | return { 156 | bulk: function(params, callback) { 157 | callback(new Error('test error')); 158 | } 159 | }; 160 | }; 161 | var config = { 162 | elasticsearch: { 163 | index: 'index', 164 | type: 'type' 165 | }, 166 | data: [ 167 | { 168 | value: '1' 169 | } 170 | ] 171 | }; 172 | handler.process(config) 173 | .catch(function(err) { 174 | assert.ok(err, 'error is thrown'); 175 | done(); 176 | }); 177 | }); 178 | }); 179 | }); 180 | -------------------------------------------------------------------------------- /test/assets/s3access.format.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 4 | "bucket": "awsexamplebucket", 5 | "time": "06/Feb/2019 00:00:38", 6 | "zone": "+0000", 7 | "remote_ip": "192.0.2.3", 8 | "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 9 | "request_id": "3E57427F3EXAMPLE", 10 | "operation": "REST.GET.VERSIONING", 11 | "key": "-", 12 | "request_uri": "GET /awsexamplebucket?versioning HTTP/1.1", 13 | "http_status": "200", 14 | "error_code": "-", 15 | "bytes_sent": "113", 16 | "object_size": "-", 17 | "total_time": "7", 18 | "turn_around_time": "-", 19 | "referrer": "-", 20 | "user_agent": "S3Console/0.4", 21 | "version_id": "-", 22 | "host_id": "s9lzHYrFp76ZVxRcpX9+5cjAnEH2ROuNkd2BHfIa6UkFVdtjf5mKR3/eTPFvsiP/XV/VLi31234=", 23 | "signature_version": "SigV2", 24 | "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", 25 | "authentication_type": "AuthHeader", 26 | "host_header": "awsexamplebucket.s3.amazonaws.com", 27 | "tls_version": "TLSV1.1", 28 | "date": "2019-02-06T00:00:38.000Z" 29 | }, 30 | { 31 | "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 32 | "bucket": "awsexamplebucket", 33 | "time": "06/Feb/2019 00:00:38", 34 | "zone": "+0000", 35 | "remote_ip": "192.0.2.3", 36 | "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 37 | "request_id": "891CE47D2EXAMPLE", 38 | "operation": "REST.GET.LOGGING_STATUS", 39 | "key": "-", 40 | "request_uri": "GET /awsexamplebucket?logging HTTP/1.1", 41 | "http_status": "200", 42 | "error_code": "-", 43 | "bytes_sent": "242", 44 | "object_size": "-", 45 | "total_time": "11", 46 | "turn_around_time": "-", 47 | "referrer": "-", 48 | "user_agent": "S3Console/0.4", 49 | "version_id": "-", 50 | "host_id": "9vKBE6vMhrNiWHZmb2L0mXOcqPGzQOI5XLnCtZNPxev+Hf+7tpT6sxDwDty4LHBUOZJG96N1234=", 51 | "signature_version": "SigV2", 52 | "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", 53 | "authentication_type": "AuthHeader", 54 | "host_header": "awsexamplebucket.s3.amazonaws.com", 55 | "tls_version": "TLSV1.1", 56 | "date": "2019-02-06T00:00:38.000Z" 57 | }, 58 | { 59 | "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 60 | "bucket": "awsexamplebucket", 61 | "time": "06/Feb/2019 00:00:38", 62 | "zone": "+0000", 63 | "remote_ip": "192.0.2.3", 64 | "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 65 | "request_id": "A1206F460EXAMPLE", 66 | "operation": "REST.GET.BUCKETPOLICY", 67 | "key": "-", 68 | "request_uri": "GET /awsexamplebucket?policy HTTP/1.1", 69 | "http_status": "404", 70 | "error_code": "NoSuchBucketPolicy", 71 | "bytes_sent": "297", 72 | "object_size": "-", 73 | "total_time": "38", 74 | "turn_around_time": "-", 75 | "referrer": "-", 76 | "user_agent": "S3Console/0.4", 77 | "version_id": "-", 78 | "host_id": "BNaBsXZQQDbssi6xMBdBU2sLt+Yf5kZDmeBUP35sFoKa3sLLeMC78iwEIWxs99CRUrbS4n11234=", 79 | "signature_version": "SigV2", 80 | "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", 81 | "authentication_type": "AuthHeader", 82 | "host_header": "awsexamplebucket.s3.amazonaws.com", 83 | "tls_version": "TLSV1.1", 84 | "date": "2019-02-06T00:00:38.000Z" 85 | }, 86 | { 87 | "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 88 | "bucket": "awsexamplebucket", 89 | "time": "06/Feb/2019 00:01:00", 90 | "zone": "+0000", 91 | "remote_ip": "192.0.2.3", 92 | "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 93 | "request_id": "7B4A0FABBEXAMPLE", 94 | "operation": "REST.GET.VERSIONING", 95 | "key": "-", 96 | "request_uri": "GET /awsexamplebucket?versioning HTTP/1.1", 97 | "http_status": "200", 98 | "error_code": "-", 99 | "bytes_sent": "113", 100 | "object_size": "-", 101 | "total_time": "33", 102 | "turn_around_time": "-", 103 | "referrer": "-", 104 | "user_agent": "S3Console/0.4", 105 | "version_id": "-", 106 | "host_id": "Ke1bUcazaN1jWuUlPJaxF64cQVpUEhoZKEG/hmy/gijN/I1DeWqDfFvnpybfEseEME/u7ME1234=", 107 | "signature_version": "SigV2", 108 | "cipher_suite": "ECDHE-RSA-AES128-GCM-SHA256", 109 | "authentication_type": "AuthHeader", 110 | "host_header": "awsexamplebucket.s3.amazonaws.com", 111 | "tls_version": "TLSV1.1", 112 | "date": "2019-02-06T00:01:00.000Z" 113 | }, 114 | { 115 | "bucket_owner": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 116 | "bucket": "awsexamplebucket", 117 | "time": "06/Feb/2019 00:01:57", 118 | "zone": "+0000", 119 | "remote_ip": "192.0.2.3", 120 | "requester": "79a59df900b949e55d96a1e698fbacedfd6e09d98eacf8f8d5218e7cd47ef2be", 121 | "request_id": "DD6CC733AEXAMPLE", 122 | "operation": "REST.PUT.OBJECT", 123 | "key": "s3-dg.pdf", 124 | "request_uri": "PUT /awsexamplebucket/s3-dg.pdf HTTP/1.1", 125 | "http_status": "200", 126 | "error_code": "-", 127 | "bytes_sent": "-", 128 | "object_size": "4406583", 129 | "total_time": "41754", 130 | "turn_around_time": "28", 131 | "referrer": "-", 132 | "user_agent": "S3Console/0.4", 133 | "version_id": "-", 134 | "host_id": "10S62Zv81kBW7BB6SX4XJ48o6kpcl6LPwEoizZQQxJd5qDSCTLX0TgS37kYUBKQW3+bPdrg1234=", 135 | "signature_version": "SigV4", 136 | "cipher_suite": "ECDHE-RSA-AES128-SHA", 137 | "authentication_type": "AuthHeader", 138 | "host_header": "awsexamplebucket.s3.amazonaws.com", 139 | "tls_version": "TLSV1.1", 140 | "date": "2019-02-06T00:01:57.000Z" 141 | } 142 | ] 143 | -------------------------------------------------------------------------------- /test/index.spec.js: -------------------------------------------------------------------------------- 1 | /* global beforeEach, describe, it */ 2 | 3 | var assert = require("assert"); 4 | var fs = require("fs"); 5 | 6 | var index; 7 | 8 | describe('index.js', function() { 9 | describe('#handler()', function() { 10 | beforeEach(function() { 11 | // Clear the module cache to reset overridden functions. 12 | delete require.cache; 13 | index = require("../index"); 14 | }); 15 | 16 | it('selects mapping based on S3 bucket and completes with mock', 17 | function(done) { 18 | var AWS = require('aws-sdk'); 19 | AWS.S3 = function() { 20 | return { 21 | getObject: function(params, callback) { 22 | assert.strictEqual(params.Bucket, 'source-bucket', 23 | 'S3.getObject invoked with correct S3 bucket'); 24 | callback(null, {Body: '{"status": "successful-response"}'}); 25 | } 26 | }; 27 | }; 28 | var config = { 29 | mappings: [ 30 | { 31 | bucket: 'non-match-bucket', 32 | processors: [ 33 | 'parseJson' 34 | ] 35 | }, 36 | { 37 | bucket: 'source-bucket', 38 | processors: [ 39 | 'parseJson' 40 | ] 41 | } 42 | ] 43 | }; 44 | var event = { 45 | Records: [ 46 | { 47 | eventSource: 'aws:s3', 48 | s3: { 49 | bucket: { 50 | name: 'source-bucket' 51 | }, 52 | object: { 53 | key: 'source/key' 54 | } 55 | } 56 | } 57 | ] 58 | }; 59 | var context; 60 | var callback = function(err /* , config */) { 61 | assert.ok(!err, 'no error is given'); 62 | done(); 63 | }; 64 | index.handler(config, event, context, callback); 65 | }); 66 | 67 | it('handles a CloudWatch Logs event and completes parsing', 68 | function(done) { 69 | var config = { 70 | mappings: [ 71 | { 72 | type: 'CloudWatch', 73 | processors: [ 74 | 'shipElasticsearch' 75 | ] 76 | } 77 | ] 78 | }; 79 | var dataSource = fs.readFileSync("test/assets/cloudwatch.source.txt"); 80 | var dataJson = JSON.parse(fs.readFileSync( 81 | "test/assets/cloudwatch.parse.json")); 82 | var event = { 83 | awslogs: { 84 | data: dataSource.toString() 85 | } 86 | }; 87 | var context; 88 | var callback = function() {}; 89 | var shipElasticsearch = require('../handlers/shipElasticsearch'); 90 | shipElasticsearch.process = function(config) { 91 | assert.deepStrictEqual(config.data, dataJson, 92 | 'CloudWatch Log is parsed'); 93 | done(); 94 | }; 95 | index.handler(config, event, context, callback); 96 | }); 97 | 98 | it('handles a custom handler provided as a function', 99 | function(done) { 100 | var AWS = require('aws-sdk'); 101 | AWS.S3 = function() { 102 | return { 103 | getObject: function(params, callback) { 104 | assert.strictEqual(params.Bucket, 'source-bucket', 105 | 'S3.getObject invoked with correct S3 bucket'); 106 | callback(null, {Body: '{"status": "successful-response"}'}); 107 | } 108 | }; 109 | }; 110 | var ranCustomHandler = false; 111 | var customHandler = function(config) { 112 | console.log('customHandler'); 113 | ranCustomHandler = true; 114 | return Promise.resolve(config); 115 | }; 116 | var config = { 117 | mappings: [ 118 | { 119 | bucket: 'source-bucket', 120 | processors: [ 121 | customHandler 122 | ] 123 | } 124 | ] 125 | }; 126 | var event = { 127 | Records: [ 128 | { 129 | eventSource: 'aws:s3', 130 | s3: { 131 | bucket: { 132 | name: 'source-bucket' 133 | }, 134 | object: { 135 | key: 'source/key' 136 | } 137 | } 138 | } 139 | ] 140 | }; 141 | var context = {}; 142 | var callback = function(err /* , result */) { 143 | assert.ok(!err, 'no error is given'); 144 | assert.ok(ranCustomHandler, 'custom handler was run'); 145 | done(); 146 | }; 147 | index.handler(config, event, context, callback); 148 | }); 149 | 150 | it('throws error if a handler throws an error', 151 | function(done) { 152 | var AWS = require('aws-sdk'); 153 | AWS.S3 = function() { 154 | return { 155 | getObject: function(/* params, callback */) { 156 | throw new Error('test error'); 157 | } 158 | }; 159 | }; 160 | var config = { 161 | mappings: [ 162 | { 163 | bucket: 'source-bucket' 164 | } 165 | ] 166 | }; 167 | var event = { 168 | Records: [ 169 | { 170 | eventSource: 'aws:s3', 171 | s3: { 172 | bucket: { 173 | name: 'source-bucket' 174 | }, 175 | object: { 176 | key: 'source/key' 177 | } 178 | } 179 | } 180 | ] 181 | }; 182 | var context = { 183 | fail: function(err) { 184 | assert.ok(err, 'error is thrown'); 185 | done(); 186 | } 187 | }; 188 | var callback = function() {}; 189 | index.handler(config, event, context, callback); 190 | }); 191 | 192 | it('throws error for an invalid processor', 193 | function(done) { 194 | var AWS = require('aws-sdk'); 195 | AWS.S3 = function() { 196 | return { 197 | getObject: function(params, callback) { 198 | callback(null, {Body: 'successful-response'}); 199 | } 200 | }; 201 | }; 202 | var config = { 203 | mappings: [ 204 | { 205 | bucket: 'source-bucket', 206 | processors: [ 207 | 'invalid-processor' 208 | ] 209 | } 210 | ] 211 | }; 212 | var event = { 213 | Records: [ 214 | { 215 | eventSource: 'aws:s3', 216 | s3: { 217 | bucket: { 218 | name: 'source-bucket' 219 | }, 220 | object: { 221 | key: 'source/key' 222 | } 223 | } 224 | } 225 | ] 226 | }; 227 | var context = { 228 | fail: function(err) { 229 | assert.ok(err, 'error is thrown'); 230 | done(); 231 | } 232 | }; 233 | var callback = function() {}; 234 | index.handler(config, event, context, callback); 235 | }); 236 | 237 | it('ends without error if no mapping is found', 238 | function(done) { 239 | var config = {}; 240 | var event = { 241 | Records: [ 242 | { 243 | eventSource: 'aws:s3', 244 | s3: { 245 | bucket: { 246 | name: 'source-bucket' 247 | }, 248 | object: { 249 | key: 'source/key' 250 | } 251 | } 252 | } 253 | ] 254 | }; 255 | var context; 256 | var callback = function(err, result) { 257 | assert.ok(!err, 'no error is given'); 258 | assert.strictEqual(result, 'Event did not match any mappings.', 259 | 'invokes callback with status'); 260 | done(); 261 | }; 262 | index.handler(config, event, context, callback); 263 | }); 264 | }); 265 | }); 266 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Lambda Stash 2 | 3 | [![npm version](https://badge.fury.io/js/lambda-stash.svg)](https://www.npmjs.com/package/lambda-stash) 4 | [![Travis CI test status](https://travis-ci.org/arithmetric/lambda-stash.svg?branch=master)](https://travis-ci.org/arithmetric/lambda-stash) 5 | [![Coverage Status](https://coveralls.io/repos/github/arithmetric/lambda-stash/badge.svg?branch=master)](https://coveralls.io/github/arithmetric/lambda-stash?branch=master) 6 | 7 | `lambda-stash` is an AWS Lambda script for shipping data from S3 or other cloud 8 | data sources to data stores, like Elasticsearch. 9 | 10 | ## Features 11 | 12 | - Supported input formats: AWS Cloudfront access logs, AWS Cloudtrail API logs, 13 | AWS CloudWatch logs, AWS Config change logs, AWS Elastic Load Balancer access 14 | logs, AWS S3 access logs, and other formats by implementing a custom handler. 15 | 16 | - Supported output formats: JSON, plain text key-value pairs. 17 | 18 | - Supported shipping methods: Elasticsearch (includes signing for AWS domains), 19 | HTTP/S POST, TCP socket. 20 | 21 | ## Set Up 22 | 23 | 1. Set up a Lambda deployment package that includes `lambda-stash` and a script 24 | with your configuration. See the example included (under `example/`) and the 25 | configuration documentation below to get started. 26 | 27 | 2. Use the AWS Management Console to create a Lambda function using the Node.js 28 | 12.x runtime. Upload your package, configure it with event sources as desired 29 | (S3 buckets or CloudWatch logs). Be sure the Lambda function has an IAM role 30 | with any necessary permissions, like getting data from an S3 bucket or accessing 31 | an AWS Elasticsearch domain. 32 | 33 | 3. Check CloudWatch Logs, where a Log Group should be created once the Lambda 34 | script runs the first time. Review the logs to make sure the script finishes 35 | successfully within the time allowed. You can set up a CloudWatch Alarm to be 36 | notified if an error occurs and to track for how long the function is running. 37 | The elasticsearch client timeout default is 30 seconds. You can set 38 | config.elasticsearch.requestTimeout to milliseconds or 'Infinity'. 39 | 40 | ## Configuration 41 | 42 | `lambda-stash` is intended to be implemented by a script that provides the 43 | configuration, such as what processors to use for specific events. 44 | 45 | See the included example in `example/` and the handlers documentation below for 46 | details on how to implement the handlers. 47 | 48 | ## Handlers 49 | 50 | ### convertString 51 | 52 | Converts an array of objects to key-value strings with the format: 53 | `prefix key1="value1" key2="value2" ... suffix` 54 | 55 | **Inputs**: 56 | 57 | - `config.data` should be an array of data objects. 58 | - `config.string.prefix` (optional) is a string to add at the beginning of each 59 | output string. 60 | - `config.string.suffix` (optional) is a string to add at the end of each output 61 | string. 62 | 63 | **Outputs**: 64 | 65 | - `config.data` is transformed to a string with the result of the handler. 66 | 67 | ### decodeBase64 68 | 69 | Decodes a Base64 encoded string. 70 | 71 | **Inputs**: 72 | 73 | - `config.data` should be a string in Base64 format. 74 | 75 | **Outputs**: 76 | 77 | - `config.data` is transformed to a buffer with the decoded data. 78 | 79 | ### decompressGzip 80 | 81 | Decompresses gzipped data. 82 | 83 | **Inputs**: 84 | 85 | - `config.data` should be a buffer in gzip format. 86 | 87 | **Outputs**: 88 | 89 | - `config.data` is transformed to a string with the decompressed data. 90 | 91 | ### formatCloudfront 92 | 93 | Processes parsed data from Cloudfront access logs and normalizes it in key-value 94 | objects. 95 | 96 | Raw Cloudfront access log files in S3 should be processed with `decompressGzip` 97 | and `parseTabs` before using this format handler. 98 | 99 | **Inputs**: 100 | 101 | - `config.data` should be an array of log records, each of which is an array of 102 | field values. 103 | 104 | - `config.dateField` (optional) is the key name in the output that should 105 | contain the log's date time value (combined from the date and time fields). 106 | 107 | **Outputs**: 108 | 109 | - `config.data` is transformed to an array of objects with log data. 110 | 111 | ### formatCloudtrail 112 | 113 | Processes parsed data from Cloudtrail logs and normalizes it in key-value 114 | objects. 115 | 116 | Raw Cloudtrail access log files in S3 should be processed with `decompressGzip` 117 | and `parseJson` before using this format handler. 118 | 119 | **Inputs**: 120 | 121 | - `config.data` should have a `Records` property that is an array of objects 122 | for each log item. 123 | 124 | - `config.dateField` (optional) is the key name in the output that should 125 | contain the log's date time value. 126 | 127 | **Outputs**: 128 | 129 | - `config.data` is transformed to an array of objects with log data. 130 | 131 | ### formatCloudwatchLogs 132 | 133 | Processes parsed data from CloudWatch logs and normalizes it in key-value 134 | objects. Includes handling for Lambda function start, end, and report logs. 135 | 136 | This handler is meant to be used with data provided through a CloudWatch Logs 137 | subscription event, which is automatically processed with `decodeBase64`, 138 | `decompressGzip`, and `parseJson`. 139 | 140 | **Inputs**: 141 | 142 | - `config.data` should have a `logEvents` property that is an array of objects 143 | for each log item. 144 | 145 | - `config.dateField` (optional) is the key name in the output that should 146 | contain the log's date time value. 147 | 148 | **Outputs**: 149 | 150 | - `config.data` is transformed to an array of objects with log data. 151 | 152 | ### formatConfig 153 | 154 | Processes parsed data from AWS Config logs and normalizes it in key-value 155 | objects. 156 | 157 | Raw Config log files in S3 should be processed with `decompressGzip` 158 | and `parseJson` before using this format handler. 159 | 160 | **Inputs**: 161 | 162 | - `config.data` should have a `configurationItems` property that is an array of 163 | objects for each log item. 164 | 165 | - `config.dateField` (optional) is the key name in the output that should 166 | contain the log's date time value. 167 | 168 | **Outputs**: 169 | 170 | - `config.data` is transformed to an array of objects with log data. 171 | 172 | ### formatELBv1 173 | 174 | Processes parsed data from AWS Elastic Load Balancer (ELB) Classic Load Balancer 175 | (ELB version 1) logs and normalizes it in key-value objects. 176 | 177 | Raw ELB log files in S3 should be processed with `parseSpaces` before using this 178 | format handler. 179 | 180 | **Inputs**: 181 | 182 | - `config.data` should be an array (one item per log record) of arrays (one item 183 | per record field). 184 | 185 | - `config.dateField` (optional) is the key name in the output that should 186 | contain the log's date time value. 187 | 188 | **Outputs**: 189 | 190 | - `config.data` is transformed to an array of objects with log data. 191 | 192 | ### formatELBv2 193 | 194 | Processes parsed data from AWS Elastic Load Balancer (ELB) Application Load 195 | Balancer (ELB version 2) logs and normalizes it in key-value objects. 196 | 197 | Raw ELB log files in S3 should be processed with `parseSpaces` before using this 198 | format handler. 199 | 200 | **Inputs**: 201 | 202 | - `config.data` should be an array (one item per log record) of arrays (one item 203 | per record field). 204 | 205 | - `config.dateField` (optional) is the key name in the output that should 206 | contain the log's date time value. 207 | 208 | **Outputs**: 209 | 210 | - `config.data` is transformed to an array of objects with log data. 211 | 212 | ### formatS3Access 213 | 214 | Processes parsed data from AWS Simple Storage Service (S3) access logs and 215 | normalizes it in key-value objects. 216 | 217 | Raw S3 log files should be processed with `parseSpaces` before using this 218 | format handler. 219 | 220 | **Inputs**: 221 | 222 | - `config.data` should be an array (one item per log record) of arrays (one item 223 | per record field). 224 | 225 | - `config.dateField` (optional) is the key name in the output that should 226 | contain the log's date time value. 227 | 228 | **Outputs**: 229 | 230 | - `config.data` is transformed to an array of objects with log data. 231 | 232 | ### getS3Object 233 | 234 | Fetches an object from S3. 235 | 236 | **Inputs**: 237 | 238 | - `config.S3.srcBucket` is the S3 bucket name for the object to fetch. 239 | 240 | - `config.S3.srcKey` is the S3 key name for the object to fetch. 241 | 242 | **Outputs**: 243 | 244 | - `config.data` is set to the buffer with the S3 object data. 245 | 246 | ### outputJsonLines 247 | 248 | Transforms output data into a series of JSON strings delimited by a newline. 249 | This output format is compatible with the 250 | [Loggly HTTP/S bulk endpoint](https://www.loggly.com/docs/http-bulk-endpoint/). 251 | 252 | **Inputs**: 253 | 254 | - `config.data` is an array of objects with log data. 255 | 256 | **Outputs**: 257 | 258 | - `config.data` is transformed into JSON strings delimited by newlines. 259 | 260 | ### parseCsv 261 | 262 | Parses CSV data into an array of records, each of which is an array of column 263 | values. 264 | 265 | **Inputs**: 266 | 267 | - `config.data` is a string with CSV data. 268 | 269 | **Outputs**: 270 | 271 | - `config.data` is transformed into an array of records, each of which is an 272 | array of strings with column values. 273 | 274 | ### parseJson 275 | 276 | Parses a JSON string into its equivalent JavaScript object. 277 | 278 | **Inputs**: 279 | 280 | - `config.data` is a JSON string. 281 | 282 | **Outputs**: 283 | 284 | - `config.data` is the equivalent JavaScript object. 285 | 286 | ### parseTabs 287 | 288 | Parses tab separated value data into an array of records, each of which is an 289 | array of column values. 290 | 291 | **Inputs**: 292 | 293 | - `config.data` is a string with tab separated data. 294 | 295 | **Outputs**: 296 | 297 | - `config.data` is transformed into an array of records, each of which is an 298 | array of strings with column values. 299 | 300 | ### shipElasticsearch 301 | 302 | Ships the data to an Elasticsearch index. The Elasticsearch `_index` and `_type` 303 | values are configurable, and support for the AWS Elasticsearch request signing 304 | process is provided. 305 | 306 | **Inputs**: 307 | 308 | - `config.data` is an array of objects with log data. 309 | 310 | - `config.elasticsearch.host` is an HTTP/S URL to the Elasticsearch domain to 311 | which to send data. 312 | 313 | - `config.elasticsearch.index` is a string with the `_index` value to add to log 314 | objects. 315 | 316 | - `config.elasticsearch.type` is a string with the `_type` value to add to log 317 | objects. 318 | 319 | - `config.elasticsearch.useAWS` should be set to `true` to connect with an AWS 320 | Elasticsearch domain and automatically sign requests for AWS. 321 | 322 | - `config.elasticsearch.region` is the AWS region where the AWS Elasticsearch 323 | domain is hosted. 324 | 325 | - `config.elasticsearch.accessKey` (optional) is the AWS access key to use for 326 | signing the request. If an access key and secret key are not provided, the 327 | environment credentials are used instead (i.e. the IAM role under which the 328 | Lambda function is running). 329 | 330 | - `config.elasticsearch.secretKey` (optional) is the AWS secret key to use for 331 | signing the request. 332 | 333 | - `config.elasticsearch.maxChunkSize` (optional) is the maximum number of log 334 | items to ship in one request. By default, set to 1000. 335 | 336 | - `config.elasticsearch.requestTimeout` (optional) is the Elasticsearch client 337 | timeout in milliseconds or can be set to `Infinity` for no timeout. The 338 | Elasticsearch client's default is 30000 (30 seconds). 339 | 340 | ### shipHttp 341 | 342 | Ships the data to an HTTP/S endpoint. 343 | 344 | **Inputs**: 345 | 346 | - `config.data` is the string data to ship. An alternate key can be configured. 347 | 348 | - `config.http.url` is an HTTP/S URL to which to send data. 349 | 350 | - `config.http.keyData` (optional) is the key name in the `config` object of the 351 | data to ship. By default, set to `data`. 352 | 353 | ### shipTcp 354 | 355 | Ships the data to a TCP service. 356 | 357 | **Inputs**: 358 | 359 | - `config.data` is the string data to ship. An alternate key can be configured. 360 | 361 | - `config.tcp.host` is a hostname to which to open a TCP socket. 362 | 363 | - `config.tcp.port` is a port on which to open the socket. 364 | 365 | - `config.tcp.keyData` (optional) is the key name in the `config` object of the 366 | data to ship. By default, set to `data`. 367 | 368 | ## Recipes 369 | 370 | ### Ship Cloudtrail logs to AWS Elasticsearch 371 | 372 | The following script can be used to ship Cloudtrail logs from S3 to an AWS 373 | Elasticsearch instance. 374 | 375 | ``` 376 | var shipper = require('lambda-stash'); 377 | var config = { 378 | elasticsearch: { 379 | host: 'https://search-test-es-abcdef...', 380 | region: 'us-east-1', 381 | useAWS: true 382 | }, 383 | mappings: [ 384 | { 385 | bucket: 'my-cloudtrail-logs', 386 | processors: [ 387 | 'decompressGzip', 388 | 'parseJson', 389 | 'formatCloudtrail', 390 | 'shipElasticsearch' 391 | ], 392 | elasticsearch: { 393 | index: 'logs', 394 | type: 'cloudtrail' 395 | }, 396 | dateField: 'date' 397 | } 398 | ] 399 | }; 400 | shipper.handler(config, event, context, callback); 401 | ``` 402 | --------------------------------------------------------------------------------