├── .gitignore ├── LICENSE.txt ├── README.md ├── install.js ├── jest.config.js ├── package-lock.json ├── package.json ├── prompts.js ├── serverless.yml ├── src ├── database │ ├── Database.ts │ └── Email.ts ├── event-uploaders │ ├── AmplitudeUploader.ts │ ├── MixpanelUploader.ts │ ├── SegmentUploader.ts │ └── UploaderFunctions.ts ├── handlers │ ├── Transform.ts │ └── Upload.ts ├── model │ ├── AmplitudeEvent.ts │ ├── BranchEvent.ts │ ├── MixpanelEvent.ts │ ├── Models.ts │ └── SegmentEvent.ts ├── templates │ ├── amplitude │ │ ├── AMPLITUDE.mst │ │ └── partials │ │ │ └── campaign.partial │ ├── email │ │ ├── JOBREPORT.mjml │ │ ├── JOBREPORT_html.mst │ │ └── JOBREPORT_plain.mst │ ├── mixpanel │ │ ├── MIXPANEL.mst │ │ └── partials │ │ │ └── lastAttributedTouchData.partial │ └── segment │ │ ├── SEGMENT.mst │ │ └── partials │ │ ├── advertisingIds.partial │ │ ├── campaign.partial │ │ ├── device.partial │ │ ├── lastAttributedTouchData.partial │ │ └── properties.partial ├── transformers │ ├── AmplitudeTransformer.ts │ ├── MixpanelTransformer.ts │ ├── SegmentTransformer.ts │ └── Transformer.ts └── utils │ ├── ArrayUtils.ts │ ├── Compression.ts │ ├── Config.ts │ ├── Secrets.ts │ ├── StringUtils.ts │ └── s3.ts ├── tests ├── BranchEvent.spec.ts ├── Database.spec.ts ├── Functions.spec.ts ├── Models.spec.ts ├── TestData.ts ├── TestUtils.ts └── Transformers.spec.ts ├── tsconfig.json └── webpack.config.js /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | .serverless/cloudformation-template-create-stack.json 63 | .serverless/cloudformation-template-update-stack.json 64 | .serverless/data-export-service.zip 65 | .serverless/serverless-state.json 66 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2019 Branch Metrics, Inc. 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Overview 3 | 4 | This software is able to transform and upload raw event data to multiple destinations. Supported destinations are: 5 | - Segment 6 | - Mixpanel 7 | - Amplitude 8 | 9 | Please open a PR should you wish to include a new export destination. 10 | 11 | # Features 12 | - Tested to support files up to 1GB in size 13 | - Simple interactive CLI setup 14 | - Low cost - runs on Lambda, DynamoDB and S3 15 | - Automatic retry 16 | - Error log storage 17 | - Upload file exclusion support 18 | - Securely stores all API keys and secrets 19 | 20 | # Setup 21 | 22 | Create a bucket on S3 to store your event files or get the name of the bucket you would like to use with this upload service. e.g. `my-exported-event-data-bucket` 23 | 24 | Install Homebrew 25 | 26 | Run `brew install node` 27 | 28 | Install serverless `npm -g install serverless` 29 | Install typescript `npm install -g typescript` 30 | 31 | Then run `npm install` from the root folder to install dependencies 32 | 33 | Test by running `serverless --help` 34 | 35 | Note: You may need to install Docker for deployment to work if you have not installed it already. The simplest is to install Docker Desktop: https://docs.docker.com/install/ 36 | 37 | Run `npm run install` to configure your serverless environment (note you may need to add your AWS Access Key ID and Secret to the package.json setup script) 38 | 39 | If you want to avoid the prompts you can create a `.env` file in the root of your project and add the following: 40 | 41 | ``` 42 | { 43 | "appName": // your app name - this needs to be unique, 44 | "stage": // either dev, stg or prd, 45 | "region": // AWS region e.g. us-east-1, 46 | "awsAccessKeyId": // AWS key id used to create buckets and upload files, 47 | "awsSecretKey": // AWS secret key, 48 | "segmentKey": // If using Segment enter your Segment.com write key, 49 | "segmentExcludedTopics": // Events to exclude from the upload see below, 50 | "amplitudeKey": // If using Amplitude, your Amplitude API key, 51 | "amplitudeExcludedTopics": // Events to exclude from the upload see below, 52 | "mixpanelAPIKey": // Your Mixpanel API key, 53 | "mixpanelToken": // Your Mixpanel token, 54 | "mixpanelExcludedTopics": // Event to exclude from the upload see below, 55 | "includeOrganic": //"true" or "false" (as String values). Organic events will be ignored from uploading 56 | "downloadsBucket": "my-exported-event-data-bucket" 57 | } 58 | ``` 59 | 60 | Possible event types that can be excluded from being uploaded: 61 | ``` 62 | Click 63 | View 64 | Commerce 65 | Content 66 | Install 67 | Open 68 | PageView 69 | Reinstall 70 | SMSSent 71 | UserLifecycleEvent 72 | WebSessionStart 73 | WebToAppAutoRedirect 74 | ``` 75 | 76 | # AWS Setup 77 | 78 | If you don't already have an AWS account visit: https://aws.amazon.com to create one. 79 | 80 | Next create an IAM User with programmatic access and the following policy permissions: 81 | 82 | - SecretsManagerReadWrite 83 | - AWSLambdaFullAccess 84 | - AmazonDynamoDBFullAccess 85 | - AmazonAPIGatewayAdministrator 86 | - AWSCodeDeployRoleForLambda 87 | - AWSDeepRacerCloudFormationAccessPolicy 88 | - IAMFullAccess 89 | 90 | Save the AWS key and secret for later use 91 | 92 | Install the AWS CLI: https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html 93 | 94 | *Note:* if you already have the AWS CLI installed make sure the credentials in the `~/.aws/credentials` file match the use you just created above. 95 | 96 | # Deployment & Updating 97 | 98 | Run `npm run update` 99 | 100 | All setup at this point should be complete. 101 | -------------------------------------------------------------------------------- /install.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk') 2 | const fs = require('fs') 3 | const path = require('path') 4 | const prompt = require('prompt') 5 | const prompts = require('./prompts') 6 | 7 | const templatesBucketSuffix = 'data-export-transform-templates' 8 | 9 | module.exports.install = async function () { 10 | //add keys 11 | const result = await initialiseConfig() 12 | console.log(`Configured keys: ${JSON.stringify(Object.keys(result))}`) 13 | 14 | let awsConfig = { 15 | accessKeyId: result["awsAccessKeyId"], 16 | secretAccessKey: result["awsSecretKey"], 17 | region: result["region"] 18 | } 19 | 20 | //create AWS Secrets Manager entries for the keys 21 | const secretKeys = [ 22 | "awsAccessKeyId", 23 | "awsSecretKey", 24 | "amplitudeKey", 25 | "segmentKey", 26 | "mixpanelToken", 27 | "mixpanelAPIKey" 28 | ] 29 | let secrets = {} 30 | Object.keys(result) 31 | .filter(k => secretKeys.find(p => p === k) && result[k].length > 0) 32 | .forEach(k => secrets[k] = result[k]) 33 | 34 | try { 35 | const uploadResults = await this.uploadSecrets(secrets, result) 36 | console.log(`Secrets uploaded to Secrets Manager`) 37 | } catch (error) { 38 | console.log(JSON.stringify(error)) 39 | } 40 | 41 | const services = this.servicesFromSecrets(secrets) 42 | const bucket = `${result.appName}-${result.stage}-${templatesBucketSuffix}` 43 | let userConfig = { templatesBucket: bucket, services } 44 | Object.keys(result) 45 | .filter(k => !secretKeys.find(p => p === k) && result[k].length > 0) 46 | .forEach(k => userConfig[k] = result[k]) 47 | await this.saveConfig(userConfig) 48 | 49 | const s3 = new AWS.S3(awsConfig) 50 | await this.createBucket(s3, bucket) 51 | await this.copyTemplates(s3, bucket) 52 | } 53 | 54 | const initialiseConfig = async () => { 55 | const envPath = path.join(__dirname, '.env') 56 | console.log(`envPath: ${envPath}`) 57 | try { 58 | const result = await this.readFile(envPath) 59 | 60 | return JSON.parse(result.toString()) 61 | } catch (error) { 62 | console.log(`.env not found or invalid: ${error}`) 63 | console.log('Loading config from prompts:') 64 | } 65 | prompt.start() 66 | const result = await new Promise((resolve, reject) => { 67 | prompt.get(prompts.prompts(), function (err, result) { 68 | if (!!err) { 69 | reject(err) 70 | return 71 | } 72 | resolve(result) 73 | }) 74 | }) 75 | return result 76 | } 77 | 78 | module.exports.createBucket = async function (s3, bucket) { 79 | const params = { 80 | Bucket: bucket 81 | } 82 | const { Buckets } = await s3.listBuckets().promise() 83 | if (!!Buckets.find(b => b.Name === bucket)) { 84 | console.log(`Bucket already exists, skipping...`) 85 | return 86 | } 87 | console.log(`Creating bucket: ${bucket}`) 88 | await s3.createBucket(params).promise() 89 | } 90 | 91 | module.exports.copyTemplates = async function (s3, bucket) { 92 | const templatesFolder = path.join(__dirname, 'src/templates') 93 | const files = await this.readDir(templatesFolder) 94 | if (!files || files.length === 0) { 95 | console.log(`${templatesFolder} is empty or does not exist.`) 96 | return 97 | } 98 | await Promise.all(files.map(fileName => { 99 | const filePath = path.join(templatesFolder, fileName) 100 | if (fs.lstatSync(filePath).isDirectory()) { 101 | const { name } = path.parse(fileName) 102 | return this.uploadDirectory(s3, filePath, `${bucket}/${name}`) 103 | } 104 | return this.uploadFile(s3, filePath, bucket) 105 | })) 106 | } 107 | 108 | module.exports.uploadFile = async function (s3, filePath, bucket) { 109 | const { name, ext } = path.parse(filePath) 110 | const fileContent = await this.readFile(filePath) 111 | console.log(`Uploading file: ${filePath} to: ${bucket}`) 112 | await s3.putObject({ 113 | Bucket: bucket, 114 | Key: `${name}${ext}`, 115 | Body: fileContent 116 | }).promise() 117 | console.log(`Successfully uploaded ${name}`) 118 | } 119 | 120 | module.exports.uploadDirectory = async function (s3, folder, bucket) { 121 | console.log(`Uploading: ${folder} to: ${bucket}`) 122 | const files = await this.readDir(folder) 123 | if (!files || files.length === 0) { 124 | return 125 | } 126 | return Promise.all(files.map(fileName => { 127 | const filePath = path.join(folder, fileName) 128 | //recurse if it is a directory 129 | if (fs.lstatSync(filePath).isDirectory()) { 130 | console.log(`${filePath} is a sub-folder, syncing...`) 131 | const { name } = path.parse(filePath) 132 | return this.uploadDirectory(s3, filePath, `${bucket}/${name}`) 133 | } 134 | return this.uploadFile(s3, filePath, bucket) 135 | })) 136 | } 137 | 138 | module.exports.readFile = async function (path) { 139 | return new Promise((resolve, reject) => { 140 | fs.readFile(path, (err, data) => { 141 | if (!!err) { 142 | reject(err) 143 | return 144 | } 145 | resolve(data) 146 | }) 147 | }) 148 | } 149 | 150 | module.exports.readDir = async function (path) { 151 | return new Promise((resolve, reject) => { 152 | fs.readdir(path, async (err, files) => { 153 | if (!!err) { 154 | reject(err) 155 | return 156 | } 157 | resolve(files) 158 | }) 159 | }) 160 | } 161 | 162 | module.exports.uploadSecrets = async function (secrets, config) { 163 | console.log(`Uploading secrets: ${JSON.stringify(secrets)}`) 164 | const secretsManager = new AWS.SecretsManager({ 165 | accessKeyId: config["awsAccessKeyId"], 166 | secretAccessKey: config["awsSecretKey"], 167 | region: config["region"] 168 | }) 169 | const { appName, stage, region } = config 170 | return Promise.all(Object.keys(secrets).map(async key => { 171 | const secretName = `${appName}-${stage}-${region}-${key}` 172 | const value = secrets[key] 173 | console.log(`Uploading secret: ${key}`) 174 | try { 175 | const result = await secretsManager.putSecretValue({ 176 | SecretId: secretName, 177 | SecretString: value 178 | }).promise() 179 | return result 180 | } catch (error) { 181 | if (error.code !== "ResourceNotFoundException") { 182 | console.log(`Error updating secret: ${JSON.stringify(error)}`) 183 | } 184 | } 185 | return secretsManager.createSecret({ 186 | Name: secretName, 187 | SecretString: value 188 | }).promise() 189 | })) 190 | } 191 | 192 | module.exports.saveConfig = async function (config) { 193 | const filePath = path.join(__dirname, 'config.json') 194 | console.log(`Saving ${filePath}: ${JSON.stringify(config)}`) 195 | return new Promise((resolve, reject) => { 196 | fs.writeFile(filePath, JSON.stringify(config), function (err) { 197 | if (!!err) { 198 | reject(err) 199 | return 200 | } 201 | resolve() 202 | }) 203 | }) 204 | } 205 | 206 | module.exports.servicesFromSecrets = function (secrets) { 207 | return Object.keys(secrets).filter(k => { 208 | return k === 'amplitudeKey' || k === 'segmentKey' || k === 'mixpanelToken' 209 | }).map(k => { 210 | if (k === 'amplitudeKey') return "Amplitude" 211 | if (k === 'segmentKey') return "Segment" 212 | if (k === 'mixpanelToken') return "Mixpanel" 213 | }).join(",") 214 | } -------------------------------------------------------------------------------- /jest.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | preset: 'ts-jest', 3 | testEnvironment: 'node' 4 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aws-nodejs-typescript", 3 | "version": "1.0.0", 4 | "description": "Serverless webpack example using Typescript", 5 | "main": "handler.js", 6 | "scripts": { 7 | "test": "jest --config ./jest.config.js", 8 | "start": "./node_modules/.bin/sls offline start -r ap-southeast-1 --noTimeout", 9 | "deploy": "sls deploy --verbose ", 10 | "update": "npm run install && npm test && npm run deploy", 11 | "install": "node -e 'require(\"./install\").install()'", 12 | "debug": "export SLS_DEBUG=* && node --inspect-brk=9229 ./node_modules/.bin/serverless offline start -r ap-southeast-1 --noTimeout dev", 13 | "setup": "./node_modules/.bin/sls config credentials --provider aws --key KEY --secret SECRET", 14 | "install:lambda": "docker pull lambci/lambda", 15 | "install:dynamodb": "./node_modules/.bin/sls dynamodb install", 16 | "start:dynamodb": "./node_modules/.bin/sls dynamodb start -p 8000 --migrate true" 17 | }, 18 | "dependencies": { 19 | "analytics-node": "^3.4.0-beta.1", 20 | "axios": "^0.21.1", 21 | "dotenv": "^8.2.0", 22 | "mixpanel": "^0.10.2", 23 | "mustache": "^3.1.0", 24 | "papaparse": "^5.2.0", 25 | "scramjet": "^4.27.1", 26 | "serverless-dynamodb-client": "0.0.2", 27 | "source-map-support": "^0.5.13", 28 | "uniqid": "^5.2.0" 29 | }, 30 | "devDependencies": { 31 | "@types/aws-lambda": "^8.10.31", 32 | "@types/axios": "^0.14.0", 33 | "@types/jest": "^24.0.18", 34 | "@types/mixpanel": "^2.14.1", 35 | "@types/mustache": "^0.8.32", 36 | "@types/node": "^10.14.18", 37 | "@types/node-fetch": "^2.5.1", 38 | "@types/papaparse": "^5.0.1", 39 | "jest": "^24.9.0", 40 | "prompt": "^1.0.0", 41 | "serverless": "^1.52.2", 42 | "serverless-dynamodb-local": "^0.2.38", 43 | "serverless-offline": "^5.11.0", 44 | "serverless-webpack": "^5.3.1", 45 | "terser-webpack-plugin": "^2.1.3", 46 | "ts-jest": "^24.1.0", 47 | "ts-loader": "^5.4.5", 48 | "typescript": "^3.6.3", 49 | "webpack": "^4.40.2" 50 | }, 51 | "author": "Michael Gaylord https://", 52 | "license": "MIT" 53 | } 54 | -------------------------------------------------------------------------------- /prompts.js: -------------------------------------------------------------------------------- 1 | const topics = [ 2 | "Click", 3 | "View", 4 | "Commerce", 5 | "Content", 6 | "Install", 7 | "Open", 8 | "PageView", 9 | "Reinstall", 10 | "SMSSent", 11 | "UserLifecycleEvent", 12 | "WebSessionStart", 13 | "WebToAppAutoRedirect" 14 | ] 15 | 16 | module.exports.prompts = function () { 17 | return { 18 | properties: { 19 | appName: { 20 | description: 'Enter your AWS app name', 21 | pattern: /^[a-zA-Z\-]+$/, 22 | message: 'App name must be only letters and dashes', 23 | required: true 24 | }, 25 | stage: { 26 | description: 'Enter a stage e.g. dev, stg, prd (default is dev)', 27 | }, 28 | region: { 29 | description: 'Enter region (default: us-east-1)', 30 | default: 'us-east-1' 31 | }, 32 | downloadsBucket: { 33 | description: 'Provide your S3 bucket name where your Branch export files are located', 34 | pattern: /^[a-zA-Z0-9\+\/\=]+$/, 35 | message: 'Invalid bucket name', 36 | required: true 37 | }, 38 | awsAccessKeyId: { 39 | description: 'Provide your AWS Access Key ID', 40 | pattern: /^[A-Z0-9]+$/, 41 | message: 'Invalid AWS Access Key ID', 42 | required: true 43 | }, 44 | awsSecretKey: { 45 | description: 'Provide your AWS Secret Key', 46 | pattern: /^[a-zA-Z0-9\+\/\=]+$/, 47 | message: 'Invalid AWS Secret Key', 48 | required: true 49 | }, 50 | includeOrganic: { 51 | description: `Include organic events in your uploads. Type 'true' or 'false'` 52 | }, 53 | segmentKey: { 54 | description: 'Provide your Segment.io write key: (Enter nothing to disable Segment upload)', 55 | pattern: /^[a-zA-Z0-9]+$/, 56 | message: 'Invalid Segment write key', 57 | }, 58 | segmentExcludedTopics: { 59 | description: `Comma separated list of events to exclude sending to Segment (Valid options: ${topics.join(', ')})`, 60 | }, 61 | amplitudeKey: { 62 | description: 'Provide your Amplitude API key: (Enter nothing to disable Amplitude upload)', 63 | pattern: /^[a-zA-Z0-9]+$/, 64 | message: 'Invalid Amplitude API key', 65 | }, 66 | amplitudeExcludedTopics: { 67 | description: `Comma separated list of events to exclude sending to Amplitude (Valid options: ${topics.join(', ')})`, 68 | }, 69 | mixpanelToken: { 70 | description: 'Provide your Mixpanel Project Token: (Enter nothing to disable Mixpanel upload)', 71 | pattern: /^[a-zA-Z0-9]+$/, 72 | message: 'Invalid Mixpanel Token', 73 | }, 74 | mixpanelAPIKey: { 75 | description: 'Provide your Mixpanel API key:', 76 | pattern: /^[a-zA-Z0-9]+$/, 77 | message: 'Invalid Mixpanel API key', 78 | }, 79 | mixpanelExcludedTopics: { 80 | description: `Comma separated list of events to exclude sending to Mixpanel (Valid options: ${topics.join(', ')})`, 81 | } 82 | } 83 | } 84 | } -------------------------------------------------------------------------------- /serverless.yml: -------------------------------------------------------------------------------- 1 | service: 2 | name: event-uploader 3 | 4 | # Add the serverless-webpack plugin 5 | plugins: 6 | - serverless-webpack 7 | - serverless-dynamodb-local 8 | - serverless-offline 9 | 10 | # serverless offline config 11 | custom: 12 | config: ${file(config.json)} 13 | app-name: ${self:custom.config.appName} 14 | serverless-offline: 15 | port: 4000 16 | babelOptions: 17 | presets: ["es2017"] 18 | dynamodb: 19 | stages: 20 | - ${self:custom.config.stage} 21 | start: 22 | migrate: true 23 | s3Buckets: 24 | BUCKET_NAME: ${self:custom.config.downloadsBucket} 25 | 26 | provider: 27 | name: aws 28 | runtime: nodejs12.x 29 | region: ${self:custom.config.region} 30 | environment: 31 | REGION: ${self:custom.config.region} 32 | STAGE: ${self:custom.config.stage} 33 | APP_NAME: ${self:custom.config.appName} 34 | FILES_TABLE: ${self:custom.app-name}-${self:service}-${self:custom.config.stage}-files 35 | BATCH_UPLOAD_TABLE: ${self:custom.app-name}-${self:service}-${self:custom.config.stage}-upload 36 | SEGMENT_EXCLUDED_TOPICS: ${self:custom.config.segmentExcludedTopics} 37 | AMPLITUDE_EXCLUDED_TOPICS: ${self:custom.config.amplitudeExcludedTopics} 38 | MIXPANEL_EXCLUDED_TOPICS: ${self:custom.config.mixpanelExcludedTopics} 39 | EXPORT_SERVICES: ${self:custom.config.services} 40 | TEMPLATES_BUCKET: ${self:custom.config.templatesBucket} 41 | INCLUDE_ORGANIC: ${self:custom.config.includeOrganic} 42 | iamRoleStatements: 43 | - Effect: Allow 44 | Action: 45 | - dynamodb:Query 46 | - dynamodb:Scan 47 | - dynamodb:GetItem 48 | - dynamodb:PutItem 49 | - dynamodb:UpdateItem 50 | - dynamodb:DeleteItem 51 | - lambda:InvokeFunction 52 | - ses:SendEmail 53 | - secretsmanager:GetSecretValue 54 | - s3:* 55 | Resource: 56 | - "arn:aws:dynamodb:${self:custom.config.region}:*:table/${self:provider.environment.FILES_TABLE}" 57 | - "arn:aws:dynamodb:${self:custom.config.region}:*:table/${self:provider.environment.FILES_TABLE}/index/*" 58 | - "arn:aws:dynamodb:${self:custom.config.region}:*:table/${self:provider.environment.BATCH_UPLOAD_TABLE}" 59 | - "arn:aws:dynamodb:${self:custom.config.region}:*:table/${self:provider.environment.BATCH_UPLOAD_TABLE}/index/*" 60 | - "arn:aws:lambda:${self:custom.config.region}:*:function:*" 61 | - "arn:aws:ses:${self:custom.config.region}:*" 62 | - "arn:aws:ses:${self:custom.config.region}:*:identity/*" 63 | - "arn:aws:secretsmanager:${self:custom.config.region}:*" 64 | - "arn:aws:s3:::${self:provider.environment.TEMPLATES_BUCKET}/*" 65 | - "arn:aws:s3:::${self:custom.s3Buckets.BUCKET_NAME}/*" 66 | - "arn:aws:s3:::${self:custom.s3Buckets.BUCKET_NAME}" 67 | - "arn:aws:s3:::${self:provider.environment.TEMPLATES_BUCKET}" 68 | - "arn:aws:logs:${self:custom.config.region}:*:log-group:*" 69 | - "arn:aws:logs:${self:custom.config.region}:*:log-group:*:*:*" 70 | 71 | functions: 72 | transform: 73 | handler: src/handlers/Transform.run 74 | timeout: 900 75 | environment: 76 | FUNCTION_PREFIX: ${self:service}-${opt:stage, self:provider.stage} 77 | events: 78 | - s3: 79 | existing: true 80 | bucket: ${self:custom.s3Buckets.BUCKET_NAME} 81 | event: s3:ObjectCreated:* 82 | 83 | 84 | upload: 85 | handler: src/handlers/Upload.run 86 | timeout: 30 87 | environment: 88 | FUNCTION_PREFIX: ${self:service}-${opt:stage, self:provider.stage} 89 | events: 90 | - http: 91 | method: post 92 | path: upload 93 | 94 | resources: 95 | Resources: 96 | FilesTable: 97 | Type: 'AWS::DynamoDB::Table' 98 | DeletionPolicy: Retain 99 | Properties: 100 | AttributeDefinitions: 101 | - AttributeName: downloadPath 102 | AttributeType: S 103 | KeySchema: 104 | - AttributeName: downloadPath 105 | KeyType: HASH 106 | ProvisionedThroughput: 107 | ReadCapacityUnits: 1 108 | WriteCapacityUnits: 1 109 | TableName: ${self:provider.environment.FILES_TABLE} 110 | BatchUploadTable: 111 | Type: 'AWS::DynamoDB::Table' 112 | DeletionPolicy: Retain 113 | Properties: 114 | AttributeDefinitions: 115 | - AttributeName: identifier 116 | AttributeType: S 117 | - AttributeName: status 118 | AttributeType: S 119 | KeySchema: 120 | - AttributeName: identifier 121 | KeyType: HASH 122 | GlobalSecondaryIndexes: 123 | - IndexName: UploadedIndex 124 | KeySchema: 125 | - AttributeName: status 126 | KeyType: HASH 127 | Projection: 128 | ProjectionType: KEYS_ONLY 129 | BillingMode: PAY_PER_REQUEST 130 | TableName: ${self:provider.environment.BATCH_UPLOAD_TABLE} 131 | TransformLambdaPermissionS3: 132 | Type: 'AWS::Lambda::Permission' 133 | Properties: 134 | FunctionName: 135 | 'Fn::GetAtt': 136 | - TransformLambdaFunction 137 | - Arn 138 | Principal: 's3.amazonaws.com' 139 | Action: 'lambda:InvokeFunction' 140 | SourceAccount: 141 | Ref: AWS::AccountId 142 | SourceArn: 'arn:aws:s3:::${self:custom.s3Buckets.BUCKET_NAME}' -------------------------------------------------------------------------------- /src/database/Database.ts: -------------------------------------------------------------------------------- 1 | import { DynamoDB } from 'aws-sdk' 2 | import { 3 | BatchUploadDatabaseItem, 4 | BatchUpload, 5 | UploadResultStatus, 6 | DownloadDatabaseItem, 7 | reducedStatus, 8 | File 9 | } from '../model/Models' 10 | import { DocumentClient } from 'aws-sdk/clients/dynamodb' 11 | import * as AWS from 'aws-sdk' 12 | // @ts-ignore 13 | import dotenv from 'dotenv' 14 | import { batchUploadTableName, filesTable } from '../utils/Config' 15 | import { compressString, decompressString } from '../utils/Compression' 16 | 17 | export class Database { 18 | dynamoDb: DocumentClient 19 | batchUploadTable = batchUploadTableName 20 | 21 | constructor() { 22 | AWS.config.update({ region: process.env.REGION }) 23 | dotenv.config() 24 | // if we're running offline we need to specify the endpoint as localhost 25 | const endpoint = process.env.OFFLINE ? { endpoint: 'http://localhost:8000' } : {} 26 | this.dynamoDb = new DynamoDB.DocumentClient({ 27 | region: process.env.REGION, 28 | ...endpoint 29 | }) 30 | } 31 | 32 | saveFiles(files: File[]): Promise { 33 | console.info(`Saving ${files.length} to database`) 34 | return Promise.all( 35 | files.map(file => { 36 | return this.saveFile(file) 37 | }) 38 | ) 39 | } 40 | 41 | async saveFile(file: File): Promise { 42 | const { dynamoDb } = this 43 | const result = await dynamoDb 44 | .put({ 45 | TableName: filesTable, 46 | Item: fileToItem(file) 47 | }) 48 | .promise() 49 | console.info(`DB Save result: ${JSON.stringify(result)} for file: ${JSON.stringify(file)}`) 50 | } 51 | 52 | async updateFileMetrics(filename: string, batchCount: number, eventCount: number): Promise { 53 | const files = await this.listFilesByFilename(filename) 54 | const updated = files.map(file => { 55 | return appendMetrics(file, batchCount, eventCount) 56 | }) 57 | return this.saveFiles(updated) 58 | } 59 | 60 | async saveBatchUpload(upload: BatchUpload): Promise { 61 | const { batchUploadTable, dynamoDb } = this 62 | const item = await batchUploadToItem(upload) 63 | const result = await dynamoDb 64 | .put({ 65 | TableName: batchUploadTable, 66 | Item: item 67 | }) 68 | .promise() 69 | console.info(`DB Save result: ${JSON.stringify(result)}`) 70 | } 71 | 72 | async listBatchUploadsByStatus(status: UploadResultStatus): Promise { 73 | return this.listBatchUploads('status = :s', status) 74 | } 75 | 76 | async listDownloads(): Promise { 77 | const { dynamoDb } = this 78 | const data = await dynamoDb 79 | .scan({ 80 | TableName: filesTable, 81 | FilterExpression: 'downloaded = :l', 82 | ExpressionAttributeValues: { 83 | ':l': '0' 84 | } 85 | }) 86 | .promise() 87 | if (!data.Items) { 88 | return [] 89 | } 90 | return data.Items.map( 91 | (item): File => { 92 | return itemToFile(item) 93 | } 94 | ) 95 | } 96 | 97 | async uploadStatus(filename: string): Promise<{file: File, status: UploadResultStatus}> { 98 | const uploads = await this.listBatchUploads('filename = :s', filename) 99 | const status = reducedStatus(uploads.map(f => f.status)) 100 | const files = await this.listFilesByFilename(filename) 101 | console.debug(`Files listed: ${JSON.stringify(files)}`) 102 | if (files.length === 0) { 103 | throw new Error(`Download with filename: ${filename} does not exist.`) 104 | } 105 | return {file: files[0], status} 106 | } 107 | 108 | async listFilesByDownloadPath(path: string): Promise { 109 | return this.listFilesByFilterExpression('downloadPath = :l', path) 110 | } 111 | 112 | private async listFilesByFilename(filename: string): Promise { 113 | return this.listFilesByFilterExpression('contains(downloadPath, :l)', filename) 114 | } 115 | 116 | private async listFilesByFilterExpression(expression: string, param: string): Promise { 117 | const { dynamoDb } = this 118 | const data = await dynamoDb.scan( 119 | { 120 | TableName: filesTable, 121 | FilterExpression: expression, 122 | ExpressionAttributeValues: { 123 | ':l': param 124 | } 125 | } 126 | ).promise() 127 | if (!data.Items) { 128 | return [] 129 | } 130 | return data.Items.map(item => { 131 | return itemToFile(item) 132 | }) 133 | } 134 | 135 | private async listBatchUploads(expression: string, value: string | number): Promise { 136 | const { batchUploadTable, dynamoDb } = this 137 | const data = await dynamoDb 138 | .scan({ 139 | TableName: batchUploadTable, 140 | FilterExpression: expression, 141 | ExpressionAttributeValues: { 142 | ':s': `${value}` 143 | } 144 | }) 145 | .promise() 146 | if (!data.Items) { 147 | return [] 148 | } 149 | return Promise.all( 150 | data.Items.map(item => { 151 | return itemToBatchUpload(item) 152 | }) 153 | ) 154 | } 155 | } 156 | 157 | export function itemToFile(item: any): File { 158 | return { 159 | downloadPath: item.downloadPath as string, 160 | batchCount: Number.isInteger(item.batchCount) ? parseInt(item.batchCount) : undefined, 161 | eventCount: Number.isInteger(item.eventCount) ? parseInt(item.eventCount) : undefined 162 | } 163 | } 164 | 165 | export function fileToItem(file: File): DownloadDatabaseItem { 166 | return { 167 | downloadPath: `${file.downloadPath}`, 168 | batchCount: `${file.batchCount}`, 169 | eventCount: `${file.eventCount}` 170 | } 171 | } 172 | 173 | export async function batchUploadToItem(upload: BatchUpload): Promise { 174 | const { filename, sequence, status, events, errors } = upload 175 | const identifier = `${filename}->${sequence}` 176 | const compressedEvents = await compressString(!!events ? JSON.stringify(events) : undefined) 177 | const compressedErrors = await compressString(!!errors ? JSON.stringify(errors) : undefined) 178 | return { 179 | status: `${status}`, 180 | compressedEvents, 181 | compressedErrors, 182 | identifier 183 | } 184 | } 185 | 186 | export async function itemToBatchUpload(item: any): Promise { 187 | const { identifier, compressedEvents, compressedErrors, status } = item 188 | const components = identifier.split('->') 189 | const filename = components[0] 190 | const sequence = parseInt(components[1], 10) 191 | const eventsString = await decompressString(compressedEvents) 192 | const errorsString = await decompressString(compressedErrors) 193 | const events = !!eventsString ? JSON.parse(eventsString) : [] 194 | const errors = !!errorsString ? JSON.parse(errorsString) : [] 195 | return { 196 | status: parseInt(status), 197 | filename, 198 | sequence, 199 | events, 200 | errors 201 | } 202 | } 203 | 204 | export function appendMetrics(file: File, batchCount: number, eventCount: number): File { 205 | return {...file, batchCount, eventCount} 206 | } 207 | -------------------------------------------------------------------------------- /src/database/Email.ts: -------------------------------------------------------------------------------- 1 | import * as AWS from 'aws-sdk' 2 | // @ts-ignore 3 | import dotenv from 'dotenv' 4 | import { SendEmailRequest } from 'aws-sdk/clients/ses' 5 | 6 | export function sendEmail(request: SendEmailRequest): Promise { 7 | AWS.config.update({ region: process.env.REGION }) 8 | dotenv.config() 9 | const ses = new AWS.SES() 10 | return new Promise((resolve, reject) => { 11 | ses.sendEmail(request, function (err, data) { 12 | // If something goes wrong, print an error message. 13 | if (!!err) { 14 | reject(err) 15 | return 16 | } 17 | resolve(data.MessageId) 18 | }) 19 | }) 20 | } -------------------------------------------------------------------------------- /src/event-uploaders/AmplitudeUploader.ts: -------------------------------------------------------------------------------- 1 | import BranchEvent from '../model/BranchEvent' 2 | import { FailedEvent, ExportService } from '../model/Models' 3 | import { getFile, loadTemplates } from '../utils/s3' 4 | import { templatesBucket } from '../utils/Config' 5 | import { hasData } from '../utils/StringUtils' 6 | import axios from 'axios' 7 | import { AmplitudeTransformer } from '../transformers/AmplitudeTransformer' 8 | import AmplitudeEvent from '../model/AmplitudeEvent' 9 | import { getSecret, Secret } from '../utils/Secrets' 10 | import { shouldUpload } from './UploaderFunctions' 11 | 12 | export async function uploadToAmplitude(events: BranchEvent[], filename: string): Promise { 13 | try { 14 | const excludedConfig = process.env.AMPLITUDE_EXCLUDED_TOPICS 15 | if (!shouldUpload(filename, excludedConfig)) { 16 | const message = `File - ${filename} marked as excluded, skipping...` 17 | console.info(message) 18 | return 19 | } 20 | console.debug(`Uploading file: ${filename}`) 21 | return upload(events) 22 | } catch (error) { 23 | console.error(`Error uploading to Amplitude: ${error}`) 24 | return events.map(event => { 25 | return { 26 | event, 27 | service: ExportService.Amplitude, 28 | reason: JSON.stringify(error) 29 | } 30 | }) 31 | } 32 | 33 | async function upload(events: BranchEvent[]): Promise { 34 | const template = await getFile(templatesBucket, 'amplitude/AMPLITUDE.mst') 35 | const partials = await loadTemplates(templatesBucket, 'amplitude/partials') 36 | const transformer = new AmplitudeTransformer(template, partials) 37 | var errors = new Array() 38 | var transformedEvents = new Array() 39 | for (let i = 0; i < events.length; i++) { 40 | const event = events[i] 41 | try { 42 | const amplitudeEvent = transformer.transform(event) 43 | if (!amplitudeEvent) { 44 | throw new Error(`Transform failed for event`) 45 | } 46 | if (hasData(amplitudeEvent.device_id, amplitudeEvent.user_id)) { 47 | transformedEvents.push(amplitudeEvent) 48 | } else { 49 | const errorMessage = `Skipped event due to missing deviceId or userId` 50 | // console.debug(errorMessage) 51 | errors.push({ event, service: ExportService.Amplitude, reason: errorMessage}) 52 | } 53 | } catch (error) { 54 | console.error(`Error transforming event: ${JSON.stringify(error)}`) 55 | console.debug(`Event: ${JSON.stringify(event)}`) 56 | errors.push({ event, service: ExportService.Amplitude, reason: JSON.stringify(error) }) 57 | } 58 | } 59 | if (!transformedEvents || transformedEvents.length === 0) { 60 | throw new Error('No events available to upload to Amplitude') 61 | } 62 | await sendData(transformedEvents) 63 | console.info(`Events uploaded successfully to Amplitude: ${transformedEvents.length}`) 64 | return errors 65 | } 66 | 67 | async function sendData(events: AmplitudeEvent[]): Promise { 68 | const api = axios.create({ 69 | baseURL: "https://api.amplitude.com", 70 | headers: { 71 | 'Content-Type': 'application/json', 72 | 'Accept': '*/*' 73 | } 74 | }) 75 | try { 76 | const body = { 77 | "api_key": await getSecret(Secret.AmplitudeApiKey), 78 | "events": events 79 | } 80 | console.debug(`Uploading ${events.length} events to Amplitude...`) 81 | const response = await api.post('/batch', body) 82 | console.debug(`Amplitude upload completed with response: ${response.status}`) 83 | return response 84 | } catch (error) { 85 | console.warn(`Error uploading events to Amplitude: ${error.message}`) 86 | throw error 87 | } 88 | } 89 | } -------------------------------------------------------------------------------- /src/event-uploaders/MixpanelUploader.ts: -------------------------------------------------------------------------------- 1 | import BranchEvent from '../model/BranchEvent' 2 | import { FailedEvent, ExportService } from '../model/Models' 3 | import { getFile, loadTemplates } from '../utils/s3' 4 | import { templatesBucket } from '../utils/Config' 5 | import { shouldUpload } from './UploaderFunctions' 6 | import MixpanelEvent from '../model/MixpanelEvent' 7 | import { MixpanelTransformer } from '../transformers/MixpanelTransformer' 8 | import * as Mixpanel from 'mixpanel' 9 | import { Secret, getSecret } from '../utils/Secrets' 10 | 11 | export async function uploadToMixpanel(events: BranchEvent[], filename: string): Promise { 12 | try { 13 | const excludedConfig = process.env.MIXPANEL_EXCLUDED_TOPICS 14 | if (!shouldUpload(filename, excludedConfig)) { 15 | const message = `File - ${filename} marked as excluded, skipping...` 16 | console.info(message) 17 | return 18 | } 19 | return upload(events) 20 | } catch (error) { 21 | console.error(`Error uploading to Mixpanel: ${error}`) 22 | return events.map(event => { 23 | return { 24 | event, 25 | service: ExportService.Mixpanel, 26 | reason: JSON.stringify(error) 27 | } 28 | }) 29 | } 30 | } 31 | 32 | async function upload(events: BranchEvent[]): Promise { 33 | const template = await getFile(templatesBucket, 'mixpanel/MIXPANEL.mst') 34 | const partials = await loadTemplates(templatesBucket, 'mixpanel/partials') 35 | const transformer = new MixpanelTransformer(template, partials) 36 | var errors = new Array() 37 | var transformedEvents = new Array() 38 | for (let i = 0; i < events.length; i++) { 39 | const event = events[i] 40 | try { 41 | const mixpanelEvent = transformer.transform(event) 42 | if (!mixpanelEvent) { 43 | throw new Error(`Transform failed for event`) 44 | } 45 | transformedEvents.push(mixpanelEvent) 46 | } catch (error) { 47 | console.error(`Error transforming event: ${JSON.stringify(error)}`) 48 | errors.push({ 49 | event, 50 | service: ExportService.Mixpanel, 51 | reason: JSON.stringify(error) 52 | }) 53 | } 54 | } 55 | if (transformedEvents.length === 0) { 56 | throw new Error('No events available to upload to Mixpanel') 57 | } 58 | const token = await getSecret(Secret.MixpanelToken) 59 | const apiKey = await getSecret(Secret.MixpanelAPIKey) 60 | await sendData(transformedEvents, token, apiKey) 61 | console.info(`Events uploaded successfully to Mixpanel: ${transformedEvents.length}`) 62 | return errors 63 | } 64 | 65 | async function sendData(events: MixpanelEvent[], token: string, apiKey: string): Promise { 66 | const mixpanel = Mixpanel.init(token, { 67 | key: apiKey, 68 | protocol: 'https' 69 | }) 70 | return new Promise((resolve, reject) => { 71 | mixpanel.import_batch(events, errors => { 72 | if (!!errors) { 73 | reject(errors) 74 | return 75 | } 76 | resolve() 77 | }) 78 | }) 79 | } 80 | -------------------------------------------------------------------------------- /src/event-uploaders/SegmentUploader.ts: -------------------------------------------------------------------------------- 1 | //@ts-ignore 2 | import Analytics from 'analytics-node' 3 | import BranchEvent from '../model/BranchEvent' 4 | import { FailedEvent, ExportService } from '../model/Models' 5 | import { getFile, loadTemplates } from '../utils/s3' 6 | import { SegmentTransformer } from '../transformers/SegmentTransformer' 7 | import { templatesBucket } from '../utils/Config' 8 | import { getSecret, Secret } from '../utils/Secrets' 9 | import { shouldUpload } from './UploaderFunctions' 10 | 11 | export async function uploadToSegment(events: BranchEvent[], filename: string): Promise { 12 | try { 13 | const excludedConfig = process.env.SEGMENT_EXCLUDED_TOPICS 14 | if (!shouldUpload(filename, excludedConfig)) { 15 | const message = `File - ${filename} marked as excluded, skipping...` 16 | console.info(message) 17 | return 18 | } 19 | return upload(events) 20 | } catch (error) { 21 | console.error(`Uploading to Segment failed due to: ${error}`) 22 | return events.map(event => { 23 | return { 24 | event, 25 | service: ExportService.Segment, 26 | reason: JSON.stringify(error) 27 | } 28 | }) 29 | } 30 | 31 | async function upload(events: BranchEvent[]): Promise { 32 | const template = await getFile(templatesBucket, 'segment/SEGMENT.mst') 33 | const partials = await loadTemplates(templatesBucket, 'segment/partials') 34 | const transformer = new SegmentTransformer(template, partials) 35 | let errors = new Array() 36 | const segmentWriteKey = await getSecret(Secret.SegmentWriteKey) 37 | const analytics = new Analytics(segmentWriteKey) 38 | for (let i = 0; i < events.length; i++) { 39 | const event = events[i] 40 | // More in the docs here: https://segment.com/docs/spec/track/ 41 | try { 42 | const segmentEvent = transformer.transform(event) 43 | if (!segmentEvent) { 44 | throw new Error(`Transform failed for event`) 45 | } 46 | analytics.track(segmentEvent) 47 | } catch (error) { 48 | errors.push({ event, service: ExportService.Segment, reason: JSON.stringify(error) }) 49 | } 50 | } 51 | await completed(analytics) 52 | return errors 53 | } 54 | } 55 | 56 | const completed = (analytics: Analytics): Promise => { 57 | return new Promise((resolve, reject) => { 58 | console.debug('Flushing Segment data...') 59 | analytics.flush((err, batch) => { 60 | if (!!err) { 61 | reject(err) 62 | return 63 | } 64 | console.debug('Segment data flushed successfully.') 65 | resolve(batch) 66 | }) 67 | }) 68 | } -------------------------------------------------------------------------------- /src/event-uploaders/UploaderFunctions.ts: -------------------------------------------------------------------------------- 1 | import { excludedTopics } from "../utils/Config" 2 | 3 | export function shouldUpload(filename: string, config: string): Boolean { 4 | const topics = excludedTopics(config).filter(topic => filename.indexOf(topic) >= 0) 5 | return topics.length === 0 6 | } -------------------------------------------------------------------------------- /src/handlers/Transform.ts: -------------------------------------------------------------------------------- 1 | import { Context, Callback, S3CreateEvent } from 'aws-lambda' 2 | import { StringStream } from 'scramjet' 3 | import { Response } from '../model/Models' 4 | import { getStream } from '../utils/s3' 5 | import { lambda } from '../utils/Config' 6 | import BranchEvent from '../model/BranchEvent' 7 | import { parse } from 'papaparse' 8 | import { Database } from '../database/Database' 9 | 10 | export const run = async (event: S3CreateEvent, _context: Context, _callback: Callback): Promise => { 11 | console.info(`New file arrived: ${JSON.stringify(event.Records[0])}`) 12 | const bucket = event.Records[0].s3.bucket.name 13 | const filename = decodeURIComponent(event.Records[0].s3.object.key.replace(/\+/g, ' ')) 14 | try { 15 | const file = { 16 | downloadPath: filename 17 | } 18 | await new Database().saveFile(file) 19 | const stream = getStream(bucket, filename) 20 | console.debug(`Uploading results for: ${bucket}/${filename}`) 21 | const uploadResults = await transformAndUpload(stream, filename) 22 | console.debug(`Upload completed`) 23 | const result: Response = { 24 | statusCode: 200, 25 | body: `Upload results: ${JSON.stringify(uploadResults)}`, 26 | isBase64Encoded: false 27 | } 28 | return result 29 | } catch (error) { 30 | console.error('Unable to download ' + bucket + '/' + filename + ' and upload events' + ' due to an error: ' + error) 31 | const failed: Response = { 32 | statusCode: 400, 33 | body: error.message || 'Unknown error occurred', 34 | isBase64Encoded: false 35 | } 36 | return failed 37 | } 38 | } 39 | 40 | export async function transformAndUpload( 41 | stream: NodeJS.ReadableStream, 42 | filename: string 43 | ): Promise<{batchCount: number, eventCount: number}> { 44 | let counter = 0 45 | let sequence = 0 46 | var header: string 47 | await StringStream.from(stream, { maxParallel: 10 }) 48 | .lines('\n') 49 | .batch(500) 50 | .map(async function(chunks: Array) { 51 | var input = '' 52 | if (!header) { 53 | header = chunks[0] 54 | input = chunks.join('\n') 55 | } else { 56 | input = header + '\n' + chunks.join('\n') 57 | } 58 | const events = parseEvent(input) 59 | counter = counter + events.length 60 | await uploadEvents(events, filename, sequence) 61 | sequence++ 62 | // const results = await upload(events, filename) 63 | return [] 64 | }) 65 | .run() 66 | .catch(e => { 67 | console.error(`Error uploading events: ${e.stack} counter: ${counter}`) 68 | throw e 69 | }) 70 | console.debug(`Total lines processed: ${counter} - Total sequences: ${sequence}`) 71 | const database = new Database() 72 | await database.updateFileMetrics(filename, sequence, counter) 73 | return { batchCount: sequence, eventCount: counter} 74 | } 75 | 76 | export async function uploadEvents(events: BranchEvent[], filename: string, sequence: number) { 77 | const functionName = `${process.env.FUNCTION_PREFIX}-upload` 78 | try { 79 | await lambda 80 | .invoke({ 81 | LogType: 'None', 82 | FunctionName: functionName, 83 | Payload: JSON.stringify({ events, filename, sequence }) // pass params 84 | }) 85 | .promise() 86 | return { success: true } 87 | } catch (error) { 88 | console.error(`Error executing lambda due to: ${error}`) 89 | return { success: false, error } 90 | } 91 | } 92 | 93 | export function parseEvent(input: string): BranchEvent[] { 94 | const events: BranchEvent[] = parse(input, { 95 | delimiter: ',', 96 | header: true, 97 | skipEmptyLines: true 98 | }).data 99 | return events 100 | } -------------------------------------------------------------------------------- /src/handlers/Upload.ts: -------------------------------------------------------------------------------- 1 | import { Context, Callback, APIGatewayProxyHandler, APIGatewayEvent } from 'aws-lambda' 2 | import { Response, ExportService, UploadResultStatus, FailedEvent } from '../model/Models' 3 | import BranchEvent from '../model/BranchEvent' 4 | import { configuredServices, includeOrganic } from '../utils/Config' 5 | import { uploadToSegment } from '../event-uploaders/SegmentUploader' 6 | import { uploadToAmplitude } from '../event-uploaders/AmplitudeUploader' 7 | import { uploadToMixpanel } from '../event-uploaders/MixpanelUploader' 8 | import { Database } from '../database/Database' 9 | import { isOrganic } from '../model/BranchEvent' 10 | 11 | const database = new Database() 12 | 13 | export const run: APIGatewayProxyHandler = async (event: APIGatewayEvent, _context: Context, _callback: Callback) => { 14 | // @ts-ignore 15 | const { events, filename, sequence } = event 16 | try { 17 | if (!events || events.length === 0) { 18 | throw new Error('Events is empty or undefined, unable to upload') 19 | } 20 | console.debug(`Uploading ${events.length} events for: ${filename}`) 21 | await database.saveBatchUpload({ 22 | status: UploadResultStatus.NotUploaded, 23 | events, 24 | filename, 25 | sequence, 26 | }) 27 | const errors = await upload(events, filename) 28 | console.debug(`Upload completed.`) 29 | const status = statusFromErrors(errors, events.length) 30 | await database.saveBatchUpload({ 31 | filename, 32 | sequence, 33 | status, 34 | errors 35 | }) 36 | const jobStatus = await database.uploadStatus(filename) 37 | console.debug(`Job status: ${JSON.stringify(jobStatus)}`) 38 | const message = `Upload results: ${status} batch ${sequence} of ${jobStatus.file.batchCount}` 39 | console.info(message) 40 | const result: Response = { 41 | statusCode: 200, 42 | body: message, 43 | isBase64Encoded: false 44 | } 45 | return result 46 | } catch (error) { 47 | console.error('Unable to upload events for ' + filename + ' due to an error: ' + error) 48 | const failed: Response = { 49 | statusCode: 400, 50 | body: error.message || 'Unknown error occurred', 51 | isBase64Encoded: false 52 | } 53 | return failed 54 | } 55 | } 56 | 57 | export async function upload(events: BranchEvent[], filename: string): Promise { 58 | const services = configuredServices() 59 | if (events.length === 0) { 60 | console.warn(`Events empty, nothing to upload.`) 61 | return [] 62 | } 63 | // filter out organic events if necessary 64 | const filteredEvents = events.filter(event => { 65 | if (includeOrganic()) { 66 | return true 67 | } 68 | return !isOrganic(event) 69 | }) 70 | console.info(`Excluding organic events: ${!includeOrganic()}`) 71 | console.info(`Total events to upload: ${filteredEvents.length}`) 72 | if (filteredEvents.length === 0) { 73 | console.info(`Total events to upload empty, skipping...`) 74 | return [] 75 | } 76 | console.info(`Uploading ${filteredEvents.length} events to: ${services.join(', ')}`) 77 | const errors = await Promise.all( 78 | services.map(async service => { 79 | console.debug(`Uploading to ${service}...`) 80 | switch (service) { 81 | case ExportService.Segment: 82 | return await uploadToSegment(filteredEvents, filename) 83 | case ExportService.Amplitude: 84 | return await uploadToAmplitude(filteredEvents, filename) 85 | case ExportService.Mixpanel: 86 | return await uploadToMixpanel(filteredEvents, filename) 87 | } 88 | }) 89 | ) 90 | return [].concat(...errors) 91 | } 92 | 93 | function statusFromErrors(errors: FailedEvent[], eventCount: number): UploadResultStatus { 94 | const { length } = errors 95 | if (length === 0) { 96 | return UploadResultStatus.Successful 97 | } 98 | if (length === eventCount) { 99 | return UploadResultStatus.Failed 100 | } 101 | return UploadResultStatus.ContainsErrors 102 | } 103 | -------------------------------------------------------------------------------- /src/model/AmplitudeEvent.ts: -------------------------------------------------------------------------------- 1 | export default interface AmplitudeEvent { 2 | device_id?: string, 3 | user_id?: string 4 | } -------------------------------------------------------------------------------- /src/model/BranchEvent.ts: -------------------------------------------------------------------------------- 1 | import { ExportService } from './Models' 2 | import uniqid from 'uniqid' 3 | 4 | export default interface BranchEvent { 5 | exportService: ExportService 6 | id: number, 7 | name: string, 8 | timestamp: number, 9 | timestamp_iso: string, 10 | origin: string, 11 | last_attributed_touch_type: string, 12 | last_attributed_touch_timestamp: string, 13 | last_attributed_touch_timestamp_iso: string, 14 | last_attributed_touch_data_tilde_id: string, 15 | last_attributed_touch_data_tilde_campaign: string, 16 | last_attributed_touch_data_tilde_campaign_id: string, 17 | last_attributed_touch_data_tilde_channel: string, 18 | last_attributed_touch_data_tilde_feature: string, 19 | last_attributed_touch_data_tilde_stage: string, 20 | last_attributed_touch_data_tilde_tags: string, 21 | last_attributed_touch_data_tilde_advertising_partner_name: string, 22 | last_attributed_touch_data_tilde_secondary_publisher: string, 23 | last_attributed_touch_data_tilde_creative_name: string, 24 | last_attributed_touch_data_tilde_creative_id: string, 25 | last_attributed_touch_data_tilde_ad_set_name: string, 26 | last_attributed_touch_data_tilde_ad_set_id: string, 27 | last_attributed_touch_data_tilde_ad_name: string, 28 | last_attributed_touch_data_tilde_ad_id: string, 29 | last_attributed_touch_data_tilde_branch_ad_format: string, 30 | last_attributed_touch_data_tilde_technology_partner: string, 31 | last_attributed_touch_data_tilde_banner_dimensions: string, 32 | last_attributed_touch_data_tilde_placement: string, 33 | last_attributed_touch_data_tilde_keyword_id: string, 34 | last_attributed_touch_data_tilde_agency: string, 35 | last_attributed_touch_data_tilde_optimization_model: string, 36 | last_attributed_touch_data_tilde_secondary_ad_format: string, 37 | last_attributed_touch_data_tilde_journey_name: string, 38 | last_attributed_touch_data_tilde_journey_id: string, 39 | last_attributed_touch_data_tilde_view_name: string, 40 | last_attributed_touch_data_tilde_view_id: string, 41 | last_attributed_touch_data_plus_current_feature: string, 42 | last_attributed_touch_data_plus_via_features: string, 43 | last_attributed_touch_data_dollar_3p: string, 44 | last_attributed_touch_data_plus_web_format: string, 45 | last_attributed_touch_data_custom_fields: any, 46 | days_from_last_attributed_touch_to_event: string, 47 | hours_from_last_attributed_touch_to_event: string, 48 | minutes_from_last_attributed_touch_to_event: string, 49 | seconds_from_last_attributed_touch_to_event: string, 50 | last_cta_view_timestamp: string, 51 | last_cta_view_timestamp_iso: string, 52 | last_cta_view_data_tilde_id: string, 53 | last_cta_view_data_tilde_campaign: string, 54 | last_cta_view_data_tilde_campaign_id: string, 55 | last_cta_view_data_tilde_channel: string, 56 | last_cta_view_data_tilde_feature: string, 57 | last_cta_view_data_tilde_stage: string, 58 | last_cta_view_data_tilde_tags: string, 59 | last_cta_view_data_tilde_advertising_partner_name: string, 60 | last_cta_view_data_tilde_secondary_publisher: string, 61 | last_cta_view_data_tilde_creative_name: string, 62 | last_cta_view_data_tilde_creative_id: string, 63 | last_cta_view_data_tilde_ad_set_name: string, 64 | last_cta_view_data_tilde_ad_set_id: string, 65 | last_cta_view_data_tilde_ad_name: string, 66 | last_cta_view_data_tilde_ad_id: string, 67 | last_cta_view_data_tilde_branch_ad_format: string, 68 | last_cta_view_data_tilde_technology_partner: string, 69 | last_cta_view_data_tilde_banner_dimensions: string, 70 | last_cta_view_data_tilde_placement: string, 71 | last_cta_view_data_tilde_keyword_id: string, 72 | last_cta_view_data_tilde_agency: string, 73 | last_cta_view_data_tilde_optimization_model: string, 74 | last_cta_view_data_tilde_secondary_ad_format: string, 75 | last_cta_view_data_plus_via_features: string, 76 | last_cta_view_data_dollar_3p: string, 77 | last_cta_view_data_plus_web_format: string, 78 | last_cta_view_data_custom_fields: string, 79 | deep_linked: string, 80 | first_event_for_user: string, 81 | user_data_os: string, 82 | user_data_os_version: string, 83 | user_data_model: string, 84 | user_data_browser: string, 85 | user_data_geo_country_code: string, 86 | user_data_app_version: string, 87 | user_data_sdk_version: string, 88 | user_data_geo_dma_code: number, 89 | user_data_environment: string, 90 | user_data_platform: string, 91 | user_data_aaid: string, 92 | user_data_idfa: string, 93 | user_data_idfv: string, 94 | user_data_android_id: string, 95 | user_data_limit_ad_tracking: string, 96 | user_data_user_agent: string, 97 | user_data_ip: string, 98 | user_data_developer_identity: string, 99 | user_data_language: string, 100 | user_data_brand: string, 101 | di_match_click_token: number, 102 | event_data_revenue_in_usd: string, 103 | event_data_exchange_rate: string, 104 | event_data_transaction_id: string, 105 | event_data_revenue: string, 106 | event_data_currency: string, 107 | event_data_shipping: string, 108 | event_data_tax: string, 109 | event_data_coupon: string, 110 | event_data_affiliation: string, 111 | event_data_search_query: string, 112 | event_data_description: string, 113 | custom_data: string, 114 | last_attributed_touch_data_tilde_keyword: string, 115 | user_data_cross_platform_id: string, 116 | user_data_past_cross_platform_ids: string, 117 | user_data_prob_cross_platform_ids: string, 118 | store_install_begin_timestamp: string, 119 | referrer_click_timestamp: string, 120 | user_data_os_version_android: string, 121 | user_data_geo_city_code: number, 122 | user_data_geo_city_en: string, 123 | user_data_http_referrer: string, 124 | event_timestamp: number, 125 | customer_event_alias: string, 126 | last_attributed_touch_data_tilde_customer_campaign: string, 127 | last_attributed_touch_data_tilde_campaign_type: string, 128 | last_cta_view_data_tilde_campaign_type: string, 129 | last_attributed_touch_data_tilde_agency_id: string, 130 | last_attributed_touch_data_plus_touch_id: string, 131 | last_cta_view_data_plus_touch_id: string, 132 | user_data_installer_package_name: string, 133 | user_data_device_locale: string, 134 | user_data_carrier_name: string, 135 | user_data_screen_width?: number, 136 | user_data_screen_height?: number, 137 | user_data_build: string, 138 | user_data_internet_connection_type: string, 139 | user_data_cpu_type: string, 140 | hash_version: string, 141 | 142 | // Functions - each of these need to be defined and enabled below 143 | timestampMillisFunction?: Function, 144 | joinedTagsFunction?: Function, 145 | lowerCasedFunction?: Function, 146 | touchDataFunction?: Function, 147 | flattenedTouchDataFunction?: Function, 148 | deviceIdFunction?: Function, 149 | userIdFunction?: Function, 150 | joinedFeaturesFunction?: Function, 151 | randomIdFunction?:Function, 152 | hexIdFunction?:Function 153 | } 154 | 155 | const TimestampMillis = function (): number { 156 | if (!this.timestamp) { 157 | return (new Date()).getTime() 158 | } 159 | return Math.ceil(this.timestamp / 1000) 160 | } 161 | 162 | const JoinedTags = function (): string { 163 | const tagString: string = this.last_attributed_touch_data_tilde_tags 164 | if (tagString.length === 0) { 165 | return '' 166 | } 167 | const tags = JSON.parse(tagString) 168 | return tags.join(',') 169 | } 170 | 171 | const JoinedFeatures = function (): string { 172 | const featuresString: string = this.last_attributed_touch_data_plus_via_features 173 | if (featuresString.length === 0) { 174 | return '' 175 | } 176 | const features = JSON.parse(featuresString) 177 | return features.join(',') 178 | } 179 | 180 | const LowerCased = function (): Function { 181 | return (text: string, render: Function) => { 182 | return render(text).toLowerCase() 183 | } 184 | } 185 | 186 | const TouchData = function (): string { 187 | return AllTouchData(this) 188 | } 189 | 190 | const FlattenedTouchData = function(): string { 191 | const touchData = AllTouchData(this).replace(/^\{|}$/g, '') 192 | if (touchData.length >= 0) { 193 | return "," + touchData 194 | } 195 | return "" 196 | } 197 | 198 | function AllTouchData(event: BranchEvent): string { 199 | const exclusions = event.exportService === ExportService.Mixpanel ? 200 | ["last_attributed_touch_data_custom_fields", 201 | "last_attributed_touch_data_plus_via_features", 202 | "last_attributed_touch_data_tilde_tags"] : [] 203 | 204 | var lastAttributedTouchData = {} 205 | for (const key of Object.keys(event)) { 206 | const value = event[key] 207 | if (key !== 'last_attributed_touch_data_custom_fields' && 208 | key.startsWith('last_attributed_touch_data') && 209 | exclusions.indexOf(key) < 0 && value.length > 0) { 210 | lastAttributedTouchData[key] = value 211 | } 212 | } 213 | 214 | try { 215 | if (!!event.last_attributed_touch_data_custom_fields && event.last_attributed_touch_data_custom_fields.length > 0) { 216 | const deserializedCustomFields = JSON.parse(event.last_attributed_touch_data_custom_fields) 217 | lastAttributedTouchData = {...deserializedCustomFields, ...lastAttributedTouchData} 218 | } 219 | } catch (error) { 220 | console.warn(`Errors deserializing custom fields: ${event.last_attributed_touch_data_custom_fields}\nerror: ${error}`) 221 | } 222 | return JSON.stringify(lastAttributedTouchData) 223 | } 224 | 225 | const AnyDeviceId = function (): string | undefined { 226 | if (typeof this === 'string') { //hack for now, need to understand why the device id is being called twice here 227 | return this 228 | } 229 | const device = this.user_data_aaid || this.user_data_android_id || this.user_data_idfa || this.user_data_idfv 230 | return device 231 | } 232 | 233 | const UserId = function(): string | undefined { 234 | if (typeof this === 'string') { 235 | return this 236 | } 237 | try { 238 | if (!this.custom_data || this.custom_data.length === 0) { 239 | return this.user_data_developer_identity 240 | } 241 | const parsed = JSON.parse(this.custom_data) 242 | switch (this.exportService) { 243 | case ExportService.Mixpanel: 244 | return parsed.$mixpanel_distinct_id || this.user_data_developer_identity 245 | case ExportService.Amplitude: 246 | return parsed.$amplitude_user_id || this.user_data_developer_identity 247 | } 248 | } catch (error) { 249 | console.debug(`Error parsing custom_data on event: ${this.custom_data} error: ${error}`) 250 | } 251 | return this.user_data_developer_identity 252 | } 253 | 254 | const RandomId = function(): string { 255 | return uniqid() 256 | } 257 | 258 | const HexId = function(): string { 259 | return Number(this.id).toString(16) 260 | } 261 | 262 | export function isOrganic(event: BranchEvent): Boolean { 263 | return !event.last_attributed_touch_type 264 | || event.last_attributed_touch_type.length === 0 265 | } 266 | 267 | export function enableFunctions(event: BranchEvent, service: ExportService) { 268 | event.exportService = service 269 | event.timestampMillisFunction = TimestampMillis 270 | event.joinedTagsFunction = JoinedTags 271 | event.lowerCasedFunction = LowerCased 272 | event.deviceIdFunction = AnyDeviceId 273 | event.joinedFeaturesFunction = JoinedFeatures 274 | event.touchDataFunction = TouchData 275 | event.flattenedTouchDataFunction = FlattenedTouchData 276 | event.userIdFunction = UserId 277 | event.randomIdFunction = RandomId 278 | event.hexIdFunction = HexId 279 | } -------------------------------------------------------------------------------- /src/model/MixpanelEvent.ts: -------------------------------------------------------------------------------- 1 | export default interface MixpanelEvent { 2 | event: string, 3 | properties: { 4 | time: Date, 5 | distinct_id: string 6 | } 7 | } -------------------------------------------------------------------------------- /src/model/Models.ts: -------------------------------------------------------------------------------- 1 | import BranchEvent from './BranchEvent' 2 | 3 | export interface File { 4 | downloadPath: string 5 | batchCount?: number 6 | eventCount?: number 7 | } 8 | 9 | export interface DownloadDatabaseItem { 10 | downloadPath: string 11 | batchCount?: string 12 | eventCount?: string 13 | } 14 | 15 | export enum Destinations { 16 | Segment, 17 | Amplitude, 18 | mParticle 19 | } 20 | 21 | export interface FailedEvent { 22 | service: ExportService 23 | event: BranchEvent 24 | reason: string 25 | } 26 | 27 | export interface BatchUpload { 28 | filename: string 29 | sequence: number 30 | events?: BranchEvent[] 31 | status: UploadResultStatus 32 | errors?: FailedEvent[] 33 | } 34 | 35 | export interface BatchUploadDatabaseItem { 36 | identifier: string 37 | compressedEvents?: Buffer 38 | compressedErrors?: Buffer 39 | status: string 40 | } 41 | 42 | export interface UploadResult { 43 | totalBatches: number 44 | totalEvents: number 45 | file: string 46 | dateOfFile: string 47 | status: UploadResultStatus 48 | } 49 | 50 | export enum ExportService { 51 | None = 'None', 52 | Segment = 'Segment', 53 | Amplitude = 'Amplitude', 54 | Mixpanel = 'Mixpanel' 55 | } 56 | 57 | export enum EventTopic { 58 | Click = 'eo_click', 59 | View = 'eo_branch_cta_view', 60 | Commerce = 'eo_commerce_event', 61 | Content = 'eo_content_event', 62 | Install = 'eo_install', 63 | Open = 'eo_open', 64 | PageView = 'eo_pageview', 65 | Reinstall = 'eo_reinstall', 66 | SMSSent = 'eo_sms_sent', 67 | UserLifecycleEvent = 'eo_user_lifecycle_event', 68 | WebSessionStart = 'eo_web_session_start', 69 | WebToAppAutoRedirect = 'eo_web_to_app_auto_redirect' 70 | } 71 | 72 | export interface Response { 73 | statusCode: number 74 | body: string 75 | isBase64Encoded: boolean 76 | } 77 | 78 | export enum UploadResultStatus { 79 | NotUploaded = 0, 80 | Successful = 1, 81 | ContainsErrors = 2, 82 | Failed = 3 83 | } 84 | export function reducedStatus(statuses: UploadResultStatus[]): UploadResultStatus { 85 | return statuses.reduce((previousStatus: UploadResultStatus, status, _currentIndex, _array) => { 86 | if (previousStatus === UploadResultStatus.NotUploaded) { 87 | return UploadResultStatus.NotUploaded 88 | } 89 | if (previousStatus === UploadResultStatus.Failed && status !== UploadResultStatus.NotUploaded) { 90 | return UploadResultStatus.Failed 91 | } 92 | if (previousStatus === UploadResultStatus.ContainsErrors && 93 | status !== UploadResultStatus.Failed && 94 | status !== UploadResultStatus.NotUploaded) { 95 | return UploadResultStatus.ContainsErrors 96 | } 97 | return status 98 | }, UploadResultStatus.Successful) 99 | } 100 | -------------------------------------------------------------------------------- /src/model/SegmentEvent.ts: -------------------------------------------------------------------------------- 1 | export default interface SegmentEvent { 2 | properties: any , 3 | campaign?: { 4 | content?: string, 5 | medium?: string, 6 | name?: string, 7 | source?: string 8 | }, 9 | device?: { 10 | advertisingId?: string, 11 | id?: string 12 | }, 13 | idfa?: string, 14 | idfv?: string, 15 | aaid?: string, 16 | android_id?: string, 17 | ip?: string, 18 | user_agent?: string, 19 | os?: string, 20 | } -------------------------------------------------------------------------------- /src/templates/amplitude/AMPLITUDE.mst: -------------------------------------------------------------------------------- 1 | { 2 | {{#userIdFunction}}"user_id": "{{userIdFunction}}",{{/userIdFunction}} 3 | {{#deviceIdFunction}}"device_id": "{{deviceIdFunction}}",{{/deviceIdFunction}} 4 | "event_type": "{{name}}", 5 | "time": {{timestamp}}, 6 | "event_properties": {{touchDataFunction}}, 7 | "user_properties": { 8 | {{> campaign}} 9 | }, 10 | "groups": {}, 11 | "app_version": "{{user_data_app_version}}", 12 | "platform": "{{user_data_os}}", 13 | "os_name": "{{user_data_os}}", 14 | "os_version": "{{user_data_os_version}}", 15 | "device_brand": "{{user_data_brand}}", 16 | "device_model": "{{user_data_model}}", 17 | "country": "{{user_data_geo_country_code}}", 18 | "language": "{{user_data_language}}", 19 | "ip": "{{user_data_ip}}", 20 | "idfa": "{{user_data_idfa}}", 21 | "idfv": "{{user_data_idfv}}", 22 | "adid": "{{user_data_aaid}}", 23 | "android_id": "{{user_data_android_id}}", 24 | "insert_id": "{{id}}" 25 | } -------------------------------------------------------------------------------- /src/templates/amplitude/partials/campaign.partial: -------------------------------------------------------------------------------- 1 | "name": "{{last_attributed_touch_data_tilde_campaign}}", 2 | "source": "{{last_attributed_touch_data_tilde_channel}}", 3 | "medium": "{{last_attributed_touch_data_tilde_feature}}", 4 | "content": "{{last_attributed_touch_data_tilde_tags)}}", 5 | "tags": "{{joinedTagsFunction}}" -------------------------------------------------------------------------------- /src/templates/email/JOBREPORT.mjml: -------------------------------------------------------------------------------- 1 | 2 | 3 | Data Export Report 4 | 5 | 6 | 7 | 8 | 9 | .body-section { -webkit-box-shadow: 1px 4px 11px 0px rgba(0, 0, 0, 0.15); -moz-box-shadow: 1px 4px 11px 0px rgba(0, 0, 0, 0.15); box-shadow: 1px 4px 11px 0px rgba(0, 0, 0, 0.15); } 10 | 11 | 12 | .text-link { color: #5e6ebf } 13 | 14 | 15 | .footer-link { color: #888888 } 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | Data Export Upload to {{service}} 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | Files Processed: 33 | 34 | 35 | {{files_count}} 36 | 37 | 38 | 39 | 40 | Events Uploaded: 41 | 42 | 43 | {{events_count}} 44 | 45 | 46 | 47 | 48 | Failed Events: 49 | 50 | 51 | {{failed_events_count}} 52 | 53 | 54 | 55 | 56 | 57 | {{messages}} 58 | 59 | 60 | 61 | 62 | {{errors}} 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /src/templates/email/JOBREPORT_html.mst: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Data Export Report 6 | 7 | 8 | 9 | 10 | 11 | 56 | 57 | 67 | 68 | 76 | 81 | 93 | 103 | 104 | 105 | 106 |
107 | 114 |
115 | 116 | 117 | 118 | 156 | 157 | 158 |
119 | 128 |
129 | 130 | 131 | 140 | 141 | 142 | 145 | 146 |
132 | 133 | 134 | 135 | 136 | 137 | 138 |
139 |
143 |
Data Export Upload to {{service}}
144 |
147 |
148 | 155 |
159 |
160 | 171 |
172 | 173 | 174 | 175 | 391 | 392 | 393 |
176 | 190 |
191 | 192 | 193 | 194 | 268 | 269 | 270 |
195 | 204 |
205 | 206 | 207 | 210 | 211 | 212 | 215 | 216 |
208 |
Files Processed:
209 |
213 |
{{files_count}}
214 |
217 |
218 | 225 |
226 | 227 | 228 | 231 | 232 | 233 | 236 | 237 |
229 |
Events Uploaded:
230 |
234 |
{{events_count}}
235 |
238 |
239 | 246 |
247 | 248 | 249 | 252 | 253 | 254 | 257 | 258 |
250 |
Failed Events:
251 |
255 |
{{failed_events_count}}
256 |
259 |
260 | 267 |
271 |
272 | 291 |
292 | 293 | 294 | 295 | 322 | 323 | 324 |
296 | 305 |
306 | 307 | 308 | 311 | 312 |
309 |
{{messages}}
310 |
313 |
314 | 321 |
325 |
326 | 345 |
346 | 347 | 348 | 349 | 376 | 377 | 378 |
350 | 359 |
360 | 361 | 362 | 365 | 366 |
363 |
{{errors}}
364 |
367 |
368 | 375 |
379 |
380 | 390 |
394 |
395 | 400 |
401 | 402 | 403 | -------------------------------------------------------------------------------- /src/templates/email/JOBREPORT_plain.mst: -------------------------------------------------------------------------------- 1 | Events uploaded for export date: {{date}} 2 | 3 | File processed: {{filename}} 4 | Events processed: {{totalEvents}} 5 | Batches processed: {{totalBatches}} 6 | -------------------------------------------------------------------------------- /src/templates/mixpanel/MIXPANEL.mst: -------------------------------------------------------------------------------- 1 | { 2 | "event": "Branch {{name}}", 3 | "properties": { 4 | "$insert_id": "{{hexIdFunction}}", 5 | "ip":"{{user_data_ip}}", 6 | "time":{{timestampMillisFunction}}, 7 | "distinct_id":"{{userIdFunction}}", 8 | "os":"{{user_data_os}}", 9 | "~tags":"{{joinedTagsFunction}}", 10 | "+via_features":"{{joinedFeaturesFunction}}", 11 | {{> lastAttributedTouchData}} 12 | } 13 | } -------------------------------------------------------------------------------- /src/templates/mixpanel/partials/lastAttributedTouchData.partial: -------------------------------------------------------------------------------- 1 | "tilde_id" : "{{last_attributed_touch_data_tilde_id}}", 2 | "campaign_id" : "{{last_attributed_touch_data_tilde_campaign_id}}", 3 | "campaign_name": "{{last_attributed_touch_data_tilde_campaign}}", 4 | "channel": "{{last_attributed_touch_data_tilde_channel}}", 5 | "feature": "{{last_attributed_touch_data_tilde_feature}}", 6 | "tags": "{{joinedTagsFunction}}", 7 | "stage" : "{{last_attributed_touch_data_tilde_stage}}", 8 | "partner_name" : "{{last_attributed_touch_data_tilde_advertising_partner_name}}", 9 | "secondary_publisher" : "{{last_attributed_touch_data_tilde_secondary_publisher}}", 10 | "creative_name" : "{{last_attributed_touch_data_tilde_creative_name}}", 11 | "creative_id" : "{{last_attributed_touch_data_tilde_creative_id}}", 12 | "ad_set_name" : "{{last_attributed_touch_data_tilde_ad_set_name}}", 13 | "ad_set_id" : "{{last_attributed_touch_data_tilde_ad_set_id}}", 14 | "ad_name" : "{{last_attributed_touch_data_tilde_ad_name}}", 15 | "ad_id" : "{{last_attributed_touch_data_tilde_ad_id}}", 16 | "branch_ad_format" : "{{last_attributed_touch_data_tilde_branch_ad_format}}", 17 | "technology_partner" : "{{last_attributed_touch_data_tilde_technology_partner}}", 18 | "banner_dimensions" : "{{last_attributed_touch_data_tilde_banner_dimensions}}", 19 | "placement" : "{{last_attributed_touch_data_tilde_placement}}", 20 | "keyword_id" : "{{last_attributed_touch_data_tilde_keyword_id}}", 21 | "agency" : "{{last_attributed_touch_data_tilde_agency}}", 22 | "optimization_model" : "{{last_attributed_touch_data_tilde_optimization_model}}", 23 | "secondary_ad_format" : "{{last_attributed_touch_data_tilde_secondary_ad_format}}", 24 | "journey_name" : "{{last_attributed_touch_data_tilde_journey_name}}", 25 | "journey_id" : "{{last_attributed_touch_data_tilde_journey_id}}", 26 | "view_name" : "{{last_attributed_touch_data_tilde_view_name}}", 27 | "view_id" : "{{last_attributed_touch_data_tilde_view_id}}", 28 | "ad_partner" : "{{last_attributed_touch_data_dollar_3p}}" 29 | {{flattenedTouchDataFunction}} -------------------------------------------------------------------------------- /src/templates/segment/SEGMENT.mst: -------------------------------------------------------------------------------- 1 | { 2 | "anonymousId": "{{#custom_data.$segment_anonymous_id}}{{custom_data.$segment_anonymous_id}}{{/custom_data.$segment_anonymous_id}}{{^custom_data.$segment_anonymous_id}}anonymous{{/custom_data.$segment_anonymous_id}}", 3 | "event": "branch_{{name}}", 4 | "timestamp": {{timestamp}}, 5 | "context": { 6 | "app": { 7 | "name": "{{app_name}}" 8 | }, 9 | {{> campaign}}, 10 | {{> device}}, 11 | "ip": "{{user_data_ip}}", 12 | "user_agent": "{{user_data_user_agent}}", 13 | "os":"{{#lowerCasedFunction}}{{user_data_os}}{{/lowerCasedFunction}} {{user_data_os_version}}", 14 | {{> advertisingIds}}, 15 | {{> properties}} 16 | } 17 | } -------------------------------------------------------------------------------- /src/templates/segment/partials/advertisingIds.partial: -------------------------------------------------------------------------------- 1 | "idfv": "{{user_data_idfv}}", 2 | "idfa":"{{user_data_idfa}}", 3 | "android_id": "{{user_data_android_id}}", 4 | "aaid":"{{user_data_aaid}}" -------------------------------------------------------------------------------- /src/templates/segment/partials/campaign.partial: -------------------------------------------------------------------------------- 1 | "campaign": { 2 | "name": "{{last_attributed_touch_data_tilde_campaign}}", 3 | "source": "{{last_attributed_touch_data_tilde_channel}}", 4 | "medium": "{{last_attributed_touch_data_tilde_feature}}", 5 | "content": "{{last_attributed_touch_data_tilde_tags)}}", 6 | "tags": "{{joinedTagsFunction}}" 7 | } -------------------------------------------------------------------------------- /src/templates/segment/partials/device.partial: -------------------------------------------------------------------------------- 1 | "device": { 2 | "id": "{{user_data_idfv}}{{user_data_android_id}}", 3 | "advertisingId": "{{user_data_idfa}}{{user_data_aaid}}" 4 | } -------------------------------------------------------------------------------- /src/templates/segment/partials/lastAttributedTouchData.partial: -------------------------------------------------------------------------------- 1 | "tilde_id" : "{{last_attributed_touch_data_tilde_id}}", 2 | "campaign_id" : "{{last_attributed_touch_data_tilde_campaign_id}}", 3 | "stage" : "{{last_attributed_touch_data_tilde_stage}}", 4 | "partner_name" : "{{last_attributed_touch_data_tilde_advertising_partner_name}}", 5 | "secondary_publisher" : "{{last_attributed_touch_data_tilde_secondary_publisher}}", 6 | "creative_name" : "{{last_attributed_touch_data_tilde_creative_name}}", 7 | "creative_id" : "{{last_attributed_touch_data_tilde_creative_id}}", 8 | "ad_set_name" : "{{last_attributed_touch_data_tilde_ad_set_name}}", 9 | "ad_set_id" : "{{last_attributed_touch_data_tilde_ad_set_id}}", 10 | "ad_name" : "{{last_attributed_touch_data_tilde_ad_name}}", 11 | "ad_id" : "{{last_attributed_touch_data_tilde_ad_id}}", 12 | "branch_ad_format" : "{{last_attributed_touch_data_tilde_branch_ad_format}}", 13 | "technology_partner" : "{{last_attributed_touch_data_tilde_technology_partner}}", 14 | "banner_dimensions" : "{{last_attributed_touch_data_tilde_banner_dimensions}}", 15 | "placement" : "{{last_attributed_touch_data_tilde_placement}}", 16 | "keyword_id" : "{{last_attributed_touch_data_tilde_keyword_id}}", 17 | "agency" : "{{last_attributed_touch_data_tilde_agency}}", 18 | "optimization_model" : "{{last_attributed_touch_data_tilde_optimization_model}}", 19 | "secondary_ad_format" : "{{last_attributed_touch_data_tilde_secondary_ad_format}}", 20 | "journey_name" : "{{last_attributed_touch_data_tilde_journey_name}}", 21 | "journey_id" : "{{last_attributed_touch_data_tilde_journey_id}}", 22 | "view_name" : "{{last_attributed_touch_data_tilde_view_name}}", 23 | "view_id" : "{{last_attributed_touch_data_tilde_view_id}}", 24 | "dollar_3p" : "{{last_attributed_touch_data_dollar_3p}}" 25 | {{flattenedTouchDataFunction}} -------------------------------------------------------------------------------- /src/templates/segment/partials/properties.partial: -------------------------------------------------------------------------------- 1 | "properties": { 2 | "event_type":"{{last_attributed_touch_type}}", 3 | "os":"{{user_data_os}}", 4 | "platform":"{{user_data_platform}}", 5 | "ip":"{{user_data_ip}}", 6 | "timestamp":{{timestamp}}, 7 | "user_agent ":"{{user_data_user_agent}}", 8 | {{> lastAttributedTouchData}} 9 | } -------------------------------------------------------------------------------- /src/transformers/AmplitudeTransformer.ts: -------------------------------------------------------------------------------- 1 | import { Transformer, initMustache } from "./Transformer" 2 | import BranchEvent, { enableFunctions } from "../model/BranchEvent" 3 | import * as Mustache from 'mustache' 4 | import AmplitudeEvent from "../model/AmplitudeEvent" 5 | import { ExportService } from '../model/Models' 6 | 7 | export class AmplitudeTransformer implements Transformer { 8 | template: string 9 | partials = {} 10 | constructor(template: string, partials: {}) { 11 | this.template = template 12 | this.partials = partials 13 | initMustache(this.template) 14 | } 15 | transform = (event: BranchEvent): AmplitudeEvent | undefined => { 16 | enableFunctions(event, ExportService.Amplitude) 17 | const { template } = this 18 | const rendered = Mustache.render(template, event, this.loadPartial) 19 | if (!rendered || rendered.length === 0) { 20 | throw new Error(`Mustache template render failed for event: ${JSON.stringify(event)}`) 21 | } 22 | try { 23 | return this.parse(rendered) 24 | } catch (error) { 25 | throw new Error(`Parsing transformed event failed: ${rendered}`) 26 | } 27 | } 28 | loadPartial = (name: string) => { 29 | try { 30 | const partial = this.partials[name] 31 | if (!partial) { 32 | throw new Error(`Undefined, the partial: ${name} does not exist`) 33 | } 34 | return partial 35 | } catch (error) { 36 | console.error(`Unable to load partial: ${name} due to: ${error}`) 37 | throw error 38 | } 39 | } 40 | parse = (event: string): AmplitudeEvent => { 41 | return JSON.parse(event, (key, value) => { 42 | if (key === 'timestamp' && !!value) { 43 | return new Date(value) 44 | } 45 | return value 46 | }) 47 | } 48 | } -------------------------------------------------------------------------------- /src/transformers/MixpanelTransformer.ts: -------------------------------------------------------------------------------- 1 | import { Transformer, initMustache } from "./Transformer" 2 | import BranchEvent, { enableFunctions } from "../model/BranchEvent" 3 | import * as Mustache from 'mustache' 4 | import { ExportService } from '../model/Models' 5 | import MixpanelEvent from '../model/MixpanelEvent' 6 | 7 | export class MixpanelTransformer implements Transformer { 8 | template: string 9 | partials = {} 10 | constructor(template: string, partials: {}) { 11 | this.template = template 12 | this.partials = partials 13 | initMustache(this.template) 14 | } 15 | transform = (event: BranchEvent): MixpanelEvent | undefined => { 16 | enableFunctions(event, ExportService.Mixpanel) 17 | const { template } = this 18 | const rendered = Mustache.render(template, event, this.loadPartial) 19 | if (!rendered || rendered.length === 0) { 20 | throw new Error(`Mustache template render failed for event: ${JSON.stringify(event)}`) 21 | } 22 | try { 23 | return this.parse(rendered) 24 | } catch (error) { 25 | throw new Error(`Parsing transformed event failed: ${rendered}`) 26 | } 27 | } 28 | loadPartial = (name: string) => { 29 | try { 30 | const partial = this.partials[name] 31 | if (!partial) { 32 | throw new Error(`Undefined, the partial: ${name} does not exist`) 33 | } 34 | return partial 35 | } catch (error) { 36 | console.error(`Unable to load partial: ${name} due to: ${error}`) 37 | throw error 38 | } 39 | } 40 | parse = (event: string): MixpanelEvent => { 41 | return JSON.parse(event, (key, value) => { 42 | if (key === 'timestamp' && !!value) { 43 | return new Date(value) 44 | } 45 | return value 46 | }) 47 | } 48 | } -------------------------------------------------------------------------------- /src/transformers/SegmentTransformer.ts: -------------------------------------------------------------------------------- 1 | import { Transformer, initMustache } from "./Transformer"; 2 | import SegmentEvent from "../model/SegmentEvent"; 3 | import BranchEvent, { enableFunctions } from "../model/BranchEvent"; 4 | import * as Mustache from 'mustache' 5 | import { ExportService } from '../model/Models'; 6 | 7 | export class SegmentTransformer implements Transformer { 8 | template: string 9 | partials = {} 10 | constructor(template: string, partials: {}) { 11 | this.template = template 12 | this.partials = partials 13 | initMustache(this.template) 14 | } 15 | transform = (event: BranchEvent): SegmentEvent | undefined => { 16 | enableFunctions(event, ExportService.Segment) 17 | const { template } = this 18 | const rendered = Mustache.render(template, event, this.loadPartial) 19 | if (!rendered || rendered.length === 0) { 20 | throw new Error(`Mustache template render failed for event: ${JSON.stringify(event)}`) 21 | } 22 | try { 23 | return this.parse(rendered) 24 | } catch (error) { 25 | throw new Error(`Parsing transformed event failed: ${rendered}`) 26 | } 27 | } 28 | loadPartial = (name: string) => { 29 | try { 30 | const partial = this.partials[name] 31 | if (!partial) { 32 | throw new Error(`Undefined, the partial: ${name} does not exist`) 33 | } 34 | return partial 35 | } catch (error) { 36 | console.error(`Unable to load partial: ${name} due to: ${error}`) 37 | throw error 38 | } 39 | } 40 | parse = (event: string): SegmentEvent => { 41 | return JSON.parse(event, (key, value) => { 42 | if (key === 'timestamp' && !!value) { 43 | return new Date(value) 44 | } 45 | return value 46 | }) 47 | } 48 | } -------------------------------------------------------------------------------- /src/transformers/Transformer.ts: -------------------------------------------------------------------------------- 1 | import BranchEvent from "../model/BranchEvent" 2 | import * as Mustache from 'mustache' 3 | 4 | export interface Transformer { 5 | transform(event: BranchEvent): T 6 | } 7 | 8 | export function initMustache(template: any) { 9 | Mustache.parse(template) 10 | //@ts-ignore 11 | Mustache.escape = function (text: string): string { return text } 12 | } -------------------------------------------------------------------------------- /src/utils/ArrayUtils.ts: -------------------------------------------------------------------------------- 1 | export function chunk(array:Array, size: number): Array> { 2 | var chunks = [] 3 | for (var i = 0; i < array.length; i += size) { 4 | chunks.push(array.slice(i, Math.min(i + size, array.length))) 5 | } 6 | return chunks 7 | } -------------------------------------------------------------------------------- /src/utils/Compression.ts: -------------------------------------------------------------------------------- 1 | import * as zlib from 'zlib' 2 | 3 | export async function compressString(value?: string): Promise { 4 | if (!value) { 5 | return undefined 6 | } 7 | return new Promise((resolve, _reject) => { 8 | zlib.deflate(value, (error, data) => { 9 | if(!!error) { 10 | resolve(undefined) 11 | return 12 | } 13 | resolve(data) 14 | }) 15 | }) 16 | } 17 | 18 | export async function decompressString(buffer: Buffer): Promise { 19 | if (!buffer) { 20 | return undefined 21 | } 22 | return new Promise((resolve, _reject) => { 23 | zlib.inflate(buffer, (error, result) => { 24 | if (!!error) { 25 | resolve(undefined) 26 | return 27 | } 28 | resolve(result.toString()) 29 | }) 30 | }) 31 | } 32 | -------------------------------------------------------------------------------- /src/utils/Config.ts: -------------------------------------------------------------------------------- 1 | import { ExportService, EventTopic } from "../model/Models" 2 | import * as AWS from 'aws-sdk' 3 | 4 | export const s3 = new AWS.S3({ 5 | region: process.env.REGION 6 | }) 7 | 8 | export const lambda = new AWS.Lambda({ 9 | region: process.env.REGION 10 | }); 11 | 12 | export const secretsManager = new AWS.SecretsManager({ 13 | region: process.env.REGION 14 | }) 15 | 16 | export const templatesBucket = process.env.TEMPLATES_BUCKET 17 | export const filesTable = process.env.FILES_TABLE 18 | export const batchUploadTableName = process.env.BATCH_UPLOAD_TABLE 19 | 20 | export function configuredServices(): Array { 21 | const services = process.env.EXPORT_SERVICES.split(',') 22 | .map(s => s.trim()) 23 | .map(s => { 24 | switch (s) { 25 | case 'Amplitude': 26 | return ExportService.Amplitude 27 | case 'Mixpanel': 28 | return ExportService.Mixpanel 29 | case 'Segment': 30 | return ExportService.Segment 31 | } 32 | }) 33 | .filter(s => !!s) 34 | if (services.length === 0) { 35 | return [ExportService.None] 36 | } 37 | return services 38 | } 39 | 40 | export function includeOrganic(): Boolean { 41 | const value = process.env.INCLUDE_ORGANIC 42 | return (value === 'true') 43 | } 44 | 45 | export const excludedTopics = (config: string): Array => { 46 | if (!config) { 47 | return [] 48 | } 49 | return config.split(',') 50 | .map(s => s.trim().toLowerCase()) 51 | .map(s => { 52 | switch (s) { 53 | case 'click': 54 | return EventTopic.Click 55 | case 'view': 56 | return EventTopic.View 57 | case 'commerce': 58 | return EventTopic.Commerce 59 | case 'content': 60 | return EventTopic.Content 61 | case 'install': 62 | return EventTopic.Install 63 | case 'open': 64 | return EventTopic.Open 65 | case 'pageview': 66 | return EventTopic.PageView 67 | case 'reinstall': 68 | return EventTopic.Reinstall 69 | case 'smssent': 70 | return EventTopic.SMSSent 71 | case 'userlifecycleevent': 72 | return EventTopic.UserLifecycleEvent 73 | case 'websessionstart': 74 | return EventTopic.WebSessionStart 75 | case 'webtoappautoredirect': 76 | return EventTopic.WebToAppAutoRedirect 77 | } 78 | }) 79 | .filter(t => !!t) 80 | } 81 | -------------------------------------------------------------------------------- /src/utils/Secrets.ts: -------------------------------------------------------------------------------- 1 | import { secretsManager } from '../utils/Config' 2 | 3 | export enum Secret { 4 | SegmentWriteKey = "segmentKey", 5 | AmplitudeApiKey = "amplitudeKey", 6 | MixpanelToken = "mixpanelToken", 7 | MixpanelAPIKey = "mixpanelAPIKey" 8 | } 9 | 10 | export async function getSecret(secret: Secret): Promise { 11 | const stage = process.env.STAGE 12 | const appName = process.env.APP_NAME 13 | const region = process.env.REGION 14 | const secretName = `${appName}-${stage}-${region}-${secret}` 15 | try { 16 | const secretValue = await secretsManager.getSecretValue({ 17 | SecretId: secretName 18 | }).promise() 19 | if (!!secretValue.SecretString) { 20 | return secretValue.SecretString 21 | } 22 | throw Error(`Secret not defined, make sure to run 'npm run install' to ensure your environment is setup correctly.`) 23 | } catch(error) { 24 | console.error(`Unable to retreive secret: ${secretName} due to error: ${JSON.stringify(error)}`) 25 | throw error 26 | } 27 | } -------------------------------------------------------------------------------- /src/utils/StringUtils.ts: -------------------------------------------------------------------------------- 1 | 2 | export function dateInFilename(filename: string): string { 3 | const matches = filename.match('[0-9]{4}[-|\/]{1}[0-9]{2}[-|\/]{1}[0-9]{2}') 4 | if (!matches || matches.length === 0) { 5 | return "Unknown" 6 | } 7 | return matches[0] 8 | } 9 | 10 | export function hasData(...values: string[]): boolean { 11 | return values.filter(v => !!v && v.length > 0).length > 0 12 | } -------------------------------------------------------------------------------- /src/utils/s3.ts: -------------------------------------------------------------------------------- 1 | import { s3 } from './Config' 2 | import * as pathUtil from 'path' 3 | 4 | export async function getFile(bucket: string, filename: string): Promise { 5 | console.debug(`Reading file: ${bucket}/${filename}`) 6 | const object = await s3.getObject({ 7 | Bucket: bucket, 8 | Key: filename 9 | }).promise() 10 | return object.Body.toString() 11 | } 12 | 13 | export function getStream(bucket: string, filename: string): NodeJS.ReadableStream { 14 | console.debug(`Reading stream: ${bucket}/${filename}`) 15 | return s3.getObject({ 16 | Bucket: bucket, 17 | Key: filename 18 | }).createReadStream() 19 | } 20 | 21 | export async function loadTemplates(bucket: string, path?: string): Promise<{}> { 22 | console.debug(`Reading bucket: ${bucket}`) 23 | const objects = await s3.listObjects({ 24 | Bucket: bucket, 25 | Prefix: path || '' 26 | }, (error, _data)=> { 27 | if (!!error) { 28 | throw new Error(`Unable to listObjects due to: ${error}`) 29 | } 30 | }).promise() 31 | const keys = objects.Contents.map(object => object.Key) 32 | let partials = {} 33 | const templates = await Promise.all(keys.map(key => loadTemplate(bucket, key))) 34 | templates.map(template => { 35 | partials[template.name] = template.contents 36 | }) 37 | return partials 38 | } 39 | 40 | async function loadTemplate(bucket: string, key: string): Promise