├── renovate.json ├── README.md ├── .envrc.skeleton ├── tsconfig.json ├── package.json ├── serverless.yml ├── src └── webhook.ts ├── .gitignore ├── resources.yml └── yarn.lock /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "config:base" 4 | ] 5 | } 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # serverless-kinesis-firehose 2 | 3 | JSON collector endpoint powered by Serverless Framework, Amazon Kinesis Firehose, Amazon Athena 4 | -------------------------------------------------------------------------------- /.envrc.skeleton: -------------------------------------------------------------------------------- 1 | # Amazon Web Service 2 | export AWS_ACCESS_KEY_ID={aws_access_key_id} 3 | export AWS_SECRET_ACCESS_KEY={aws_secret_access_key} 4 | export AWS_DEFAULT_REGION=us-west-2 5 | export AWS_DEFAULT_OUTPUT=json 6 | 7 | # Serverless Framework 8 | export STAGE=dev 9 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "es5", 4 | "moduleResolution": "node", 5 | "lib": ["es2015"], 6 | "rootDir": "./", 7 | "strict": true, 8 | "preserveConstEnums": true, 9 | "strictNullChecks": true, 10 | "sourceMap": true 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "serverless-kinesis-firehose", 3 | "version": "0.0.1", 4 | "main": "index.js", 5 | "repository": "ssh://git@github.com/otofu-square/serverless-kinesis-firehose", 6 | "license": "MIT", 7 | "devDependencies": { 8 | "@types/aws-lambda": "0.0.30", 9 | "@types/node": "9.3.0", 10 | "@types/ramda": "0.25.16", 11 | "aws-sdk": "2.188.0", 12 | "serverless-plugin-typescript": "1.1.5" 13 | }, 14 | "dependencies": { 15 | "ramda": "^0.25.0" 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: serverless-kinesis-firehose 2 | 3 | provider: 4 | name: aws 5 | runtime: nodejs6.10 6 | region: us-west-2 7 | iamRoleStatements: 8 | - Effect: Allow 9 | Action: 10 | - firehose:DeleteDeliveryStream 11 | - firehose:PutRecord 12 | - firehose:PutRecordBatch 13 | - firehose:UpdateDestination 14 | Resource: '*' 15 | 16 | package: 17 | exclude: 18 | - README.md 19 | - .envrc 20 | - .envrc.skeleton 21 | - package.json 22 | - node_modules/** 23 | - yarn.lock 24 | - tsconfig.json 25 | - .gitignore 26 | 27 | functions: 28 | webhook: 29 | handler: src/webhook.execute 30 | environment: 31 | STAGE: ${env:STAGE} 32 | events: 33 | - http: 34 | path: /webhook 35 | method: post 36 | 37 | resources: ${file(resources.yml)} 38 | 39 | plugins: 40 | - serverless-plugin-typescript 41 | -------------------------------------------------------------------------------- /src/webhook.ts: -------------------------------------------------------------------------------- 1 | import { APIGatewayEvent, Callback, Context } from 'aws-lambda'; 2 | import { Firehose } from 'aws-sdk'; 3 | 4 | const DeliveryStreamName = `${process.env.STAGE}-serverless-kinesis-firehose`; 5 | const firehose = new Firehose(); 6 | 7 | export const execute = ( 8 | event: APIGatewayEvent, 9 | context: Context, 10 | callback: Callback, 11 | ) => { 12 | const json: any = JSON.parse(event.body as string); 13 | const params = { 14 | DeliveryStreamName, 15 | Record: { 16 | Data: JSON.stringify(json.events[0]) + '\n', 17 | }, 18 | }; 19 | firehose.putRecord(params, (err, data) => { 20 | if (err) { 21 | console.log(JSON.stringify(err)); 22 | callback(undefined, { statusCode: 200, body: JSON.stringify(err) }); 23 | } else { 24 | console.log(JSON.stringify(data)); 25 | callback(undefined, { statusCode: 200, body: JSON.stringify(data) }); 26 | } 27 | }); 28 | }; 29 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (http://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # Typescript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # Serverless directories 61 | .serverless 62 | -------------------------------------------------------------------------------- /resources.yml: -------------------------------------------------------------------------------- 1 | --- 2 | Resources: 3 | FirehoseToS3Role: 4 | Type: AWS::IAM::Role 5 | Properties: 6 | RoleName: FirehoseToS3Role 7 | AssumeRolePolicyDocument: 8 | Statement: 9 | - Effect: Allow 10 | Principal: 11 | Service: 12 | - firehose.amazonaws.com 13 | Action: 14 | - sts:AssumeRole 15 | Policies: 16 | - PolicyName: FirehoseToS3Policy 17 | PolicyDocument: 18 | Statement: 19 | - Effect: Allow 20 | Action: 21 | - s3:AbortMultipartUpload 22 | - s3:GetBucketLocation 23 | - s3:GetObject 24 | - s3:ListBucket 25 | - s3:ListBucketMultipartUploads 26 | - s3:PutObject 27 | Resource: '*' 28 | ServerlessKinesisFirehoseBucket: 29 | Type: AWS::S3::Bucket 30 | DeletionPolicy: Retain 31 | Properties: 32 | BucketName: ${env:STAGE}-serverless-kinesis-firehose-bucket 33 | ServerlessKinesisFirehose: 34 | Type: AWS::KinesisFirehose::DeliveryStream 35 | Properties: 36 | DeliveryStreamName: ${env:STAGE}-serverless-kinesis-firehose 37 | S3DestinationConfiguration: 38 | BucketARN: 39 | Fn::Join: 40 | - '' 41 | - - 'arn:aws:s3:::' 42 | - Ref: ServerlessKinesisFirehoseBucket 43 | BufferingHints: 44 | IntervalInSeconds: "60" 45 | SizeInMBs: "1" 46 | CompressionFormat: "UNCOMPRESSED" 47 | Prefix: "raw/" 48 | RoleARN: { Fn::GetAtt: [ FirehoseToS3Role, Arn ] } 49 | -------------------------------------------------------------------------------- /yarn.lock: -------------------------------------------------------------------------------- 1 | # THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. 2 | # yarn lockfile v1 3 | 4 | 5 | "@types/aws-lambda@0.0.30": 6 | version "0.0.30" 7 | resolved "https://registry.yarnpkg.com/@types/aws-lambda/-/aws-lambda-0.0.30.tgz#3944949f39453fd690bebf0b41a81ad031bb4af2" 8 | 9 | "@types/node@9.3.0": 10 | version "9.3.0" 11 | resolved "https://registry.yarnpkg.com/@types/node/-/node-9.3.0.tgz#3a129cda7c4e5df2409702626892cb4b96546dd5" 12 | 13 | "@types/ramda@0.25.16": 14 | version "0.25.16" 15 | resolved "https://registry.yarnpkg.com/@types/ramda/-/ramda-0.25.16.tgz#1d4eeb78d3247d1ceef971b458dd8469646cd1b4" 16 | 17 | array-union@^1.0.1: 18 | version "1.0.2" 19 | resolved "https://registry.yarnpkg.com/array-union/-/array-union-1.0.2.tgz#9a34410e4f4e3da23dea375be5be70f24778ec39" 20 | dependencies: 21 | array-uniq "^1.0.1" 22 | 23 | array-uniq@^1.0.1: 24 | version "1.0.3" 25 | resolved "https://registry.yarnpkg.com/array-uniq/-/array-uniq-1.0.3.tgz#af6ac877a25cc7f74e058894753858dfdb24fdb6" 26 | 27 | arrify@^1.0.1: 28 | version "1.0.1" 29 | resolved "https://registry.yarnpkg.com/arrify/-/arrify-1.0.1.tgz#898508da2226f380df904728456849c1501a4b0d" 30 | 31 | aws-sdk@2.188.0: 32 | version "2.188.0" 33 | resolved "https://registry.yarnpkg.com/aws-sdk/-/aws-sdk-2.188.0.tgz#9062abc7dba6393459fa2f3423cf5d294f004611" 34 | dependencies: 35 | buffer "4.9.1" 36 | events "^1.1.1" 37 | jmespath "0.15.0" 38 | querystring "0.2.0" 39 | sax "1.2.1" 40 | url "0.10.3" 41 | uuid "3.1.0" 42 | xml2js "0.4.17" 43 | xmlbuilder "4.2.1" 44 | 45 | balanced-match@^1.0.0: 46 | version "1.0.0" 47 | resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" 48 | 49 | base64-js@^1.0.2: 50 | version "1.2.1" 51 | resolved "https://registry.yarnpkg.com/base64-js/-/base64-js-1.2.1.tgz#a91947da1f4a516ea38e5b4ec0ec3773675e0886" 52 | 53 | brace-expansion@^1.1.7: 54 | version "1.1.8" 55 | resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.8.tgz#c07b211c7c952ec1f8efd51a77ef0d1d3990a292" 56 | dependencies: 57 | balanced-match "^1.0.0" 58 | concat-map "0.0.1" 59 | 60 | buffer@4.9.1: 61 | version "4.9.1" 62 | resolved "https://registry.yarnpkg.com/buffer/-/buffer-4.9.1.tgz#6d1bb601b07a4efced97094132093027c95bc298" 63 | dependencies: 64 | base64-js "^1.0.2" 65 | ieee754 "^1.1.4" 66 | isarray "^1.0.0" 67 | 68 | concat-map@0.0.1: 69 | version "0.0.1" 70 | resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" 71 | 72 | dir-glob@^2.0.0: 73 | version "2.0.0" 74 | resolved "https://registry.yarnpkg.com/dir-glob/-/dir-glob-2.0.0.tgz#0b205d2b6aef98238ca286598a8204d29d0a0034" 75 | dependencies: 76 | arrify "^1.0.1" 77 | path-type "^3.0.0" 78 | 79 | events@^1.1.1: 80 | version "1.1.1" 81 | resolved "https://registry.yarnpkg.com/events/-/events-1.1.1.tgz#9ebdb7635ad099c70dcc4c2a1f5004288e8bd924" 82 | 83 | fs-extra@^5.0.0: 84 | version "5.0.0" 85 | resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-5.0.0.tgz#414d0110cdd06705734d055652c5411260c31abd" 86 | dependencies: 87 | graceful-fs "^4.1.2" 88 | jsonfile "^4.0.0" 89 | universalify "^0.1.0" 90 | 91 | fs.realpath@^1.0.0: 92 | version "1.0.0" 93 | resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f" 94 | 95 | glob@^7.1.2: 96 | version "7.1.2" 97 | resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.2.tgz#c19c9df9a028702d678612384a6552404c636d15" 98 | dependencies: 99 | fs.realpath "^1.0.0" 100 | inflight "^1.0.4" 101 | inherits "2" 102 | minimatch "^3.0.4" 103 | once "^1.3.0" 104 | path-is-absolute "^1.0.0" 105 | 106 | globby@^7.0.0: 107 | version "7.1.1" 108 | resolved "https://registry.yarnpkg.com/globby/-/globby-7.1.1.tgz#fb2ccff9401f8600945dfada97440cca972b8680" 109 | dependencies: 110 | array-union "^1.0.1" 111 | dir-glob "^2.0.0" 112 | glob "^7.1.2" 113 | ignore "^3.3.5" 114 | pify "^3.0.0" 115 | slash "^1.0.0" 116 | 117 | graceful-fs@^4.1.2, graceful-fs@^4.1.6: 118 | version "4.1.11" 119 | resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.1.11.tgz#0e8bdfe4d1ddb8854d64e04ea7c00e2a026e5658" 120 | 121 | ieee754@^1.1.4: 122 | version "1.1.8" 123 | resolved "https://registry.yarnpkg.com/ieee754/-/ieee754-1.1.8.tgz#be33d40ac10ef1926701f6f08a2d86fbfd1ad3e4" 124 | 125 | ignore@^3.3.5: 126 | version "3.3.7" 127 | resolved "https://registry.yarnpkg.com/ignore/-/ignore-3.3.7.tgz#612289bfb3c220e186a58118618d5be8c1bab021" 128 | 129 | inflight@^1.0.4: 130 | version "1.0.6" 131 | resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9" 132 | dependencies: 133 | once "^1.3.0" 134 | wrappy "1" 135 | 136 | inherits@2: 137 | version "2.0.3" 138 | resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.3.tgz#633c2c83e3da42a502f52466022480f4208261de" 139 | 140 | isarray@^1.0.0: 141 | version "1.0.0" 142 | resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11" 143 | 144 | jmespath@0.15.0: 145 | version "0.15.0" 146 | resolved "https://registry.yarnpkg.com/jmespath/-/jmespath-0.15.0.tgz#a3f222a9aae9f966f5d27c796510e28091764217" 147 | 148 | jsonfile@^4.0.0: 149 | version "4.0.0" 150 | resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb" 151 | optionalDependencies: 152 | graceful-fs "^4.1.6" 153 | 154 | lodash@^4.0.0, lodash@^4.17.4: 155 | version "4.17.4" 156 | resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.4.tgz#78203a4d1c328ae1d86dca6460e369b57f4055ae" 157 | 158 | minimatch@^3.0.4: 159 | version "3.0.4" 160 | resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083" 161 | dependencies: 162 | brace-expansion "^1.1.7" 163 | 164 | once@^1.3.0: 165 | version "1.4.0" 166 | resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1" 167 | dependencies: 168 | wrappy "1" 169 | 170 | path-is-absolute@^1.0.0: 171 | version "1.0.1" 172 | resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f" 173 | 174 | path-type@^3.0.0: 175 | version "3.0.0" 176 | resolved "https://registry.yarnpkg.com/path-type/-/path-type-3.0.0.tgz#cef31dc8e0a1a3bb0d105c0cd97cf3bf47f4e36f" 177 | dependencies: 178 | pify "^3.0.0" 179 | 180 | pify@^3.0.0: 181 | version "3.0.0" 182 | resolved "https://registry.yarnpkg.com/pify/-/pify-3.0.0.tgz#e5a4acd2c101fdf3d9a4d07f0dbc4db49dd28176" 183 | 184 | punycode@1.3.2: 185 | version "1.3.2" 186 | resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" 187 | 188 | querystring@0.2.0: 189 | version "0.2.0" 190 | resolved "https://registry.yarnpkg.com/querystring/-/querystring-0.2.0.tgz#b209849203bb25df820da756e747005878521620" 191 | 192 | ramda@^0.25.0: 193 | version "0.25.0" 194 | resolved "https://registry.yarnpkg.com/ramda/-/ramda-0.25.0.tgz#8fdf68231cffa90bc2f9460390a0cb74a29b29a9" 195 | 196 | sax@1.2.1: 197 | version "1.2.1" 198 | resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.1.tgz#7b8e656190b228e81a66aea748480d828cd2d37a" 199 | 200 | sax@>=0.6.0: 201 | version "1.2.4" 202 | resolved "https://registry.yarnpkg.com/sax/-/sax-1.2.4.tgz#2816234e2378bddc4e5354fab5caa895df7100d9" 203 | 204 | serverless-plugin-typescript@1.1.5: 205 | version "1.1.5" 206 | resolved "https://registry.yarnpkg.com/serverless-plugin-typescript/-/serverless-plugin-typescript-1.1.5.tgz#a62ed1f04ea76875c72700b36b8bf95002a8d6a0" 207 | dependencies: 208 | fs-extra "^5.0.0" 209 | globby "^7.0.0" 210 | lodash "^4.17.4" 211 | typescript "^2.2.2" 212 | 213 | slash@^1.0.0: 214 | version "1.0.0" 215 | resolved "https://registry.yarnpkg.com/slash/-/slash-1.0.0.tgz#c41f2f6c39fc16d1cd17ad4b5d896114ae470d55" 216 | 217 | typescript@^2.2.2: 218 | version "2.5.2" 219 | resolved "https://registry.yarnpkg.com/typescript/-/typescript-2.5.2.tgz#038a95f7d9bbb420b1bf35ba31d4c5c1dd3ffe34" 220 | 221 | universalify@^0.1.0: 222 | version "0.1.1" 223 | resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.1.tgz#fa71badd4437af4c148841e3b3b165f9e9e590b7" 224 | 225 | url@0.10.3: 226 | version "0.10.3" 227 | resolved "https://registry.yarnpkg.com/url/-/url-0.10.3.tgz#021e4d9c7705f21bbf37d03ceb58767402774c64" 228 | dependencies: 229 | punycode "1.3.2" 230 | querystring "0.2.0" 231 | 232 | uuid@3.1.0: 233 | version "3.1.0" 234 | resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.1.0.tgz#3dd3d3e790abc24d7b0d3a034ffababe28ebbc04" 235 | 236 | wrappy@1: 237 | version "1.0.2" 238 | resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f" 239 | 240 | xml2js@0.4.17: 241 | version "0.4.17" 242 | resolved "https://registry.yarnpkg.com/xml2js/-/xml2js-0.4.17.tgz#17be93eaae3f3b779359c795b419705a8817e868" 243 | dependencies: 244 | sax ">=0.6.0" 245 | xmlbuilder "^4.1.0" 246 | 247 | xmlbuilder@4.2.1, xmlbuilder@^4.1.0: 248 | version "4.2.1" 249 | resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-4.2.1.tgz#aa58a3041a066f90eaa16c2f5389ff19f3f461a5" 250 | dependencies: 251 | lodash "^4.0.0" 252 | --------------------------------------------------------------------------------