├── .gitignore ├── config.env.sample ├── gulpfile.js ├── index.js ├── package.json └── test ├── record-kinesis-events.js └── video-events-test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | dist/ 3 | dist.zip 4 | config.env.production 5 | test/events.log 6 | -------------------------------------------------------------------------------- /config.env.sample: -------------------------------------------------------------------------------- 1 | FIREBASE_URL=https://adventr-dev.firebaseio.com/ 2 | -------------------------------------------------------------------------------- /gulpfile.js: -------------------------------------------------------------------------------- 1 | var gulp = require('gulp'); 2 | var gutil = require('gulp-util'); 3 | var del = require('del'); 4 | var rename = require('gulp-rename'); 5 | var install = require('gulp-install'); 6 | var zip = require('gulp-zip'); 7 | var AWS = require('aws-sdk'); 8 | var fs = require('fs'); 9 | var runSequence = require('run-sequence'); 10 | 11 | // First we need to clean out the dist folder and remove the compiled zip file. 12 | gulp.task('clean', function(cb) { 13 | del('./dist', 14 | del('./archive.zip', cb) 15 | ); 16 | }); 17 | 18 | // The js task could be replaced with gulp-coffee as desired. 19 | gulp.task('js', function() { 20 | gulp.src('index.js') 21 | .pipe(gulp.dest('dist/')) 22 | }); 23 | 24 | // Here we want to install npm packages to dist, ignoring devDependencies. 25 | gulp.task('npm', function() { 26 | gulp.src('./package.json') 27 | .pipe(gulp.dest('./dist/')) 28 | .pipe(install({production: true})); 29 | }); 30 | 31 | // Next copy over environment variables managed outside of source control. 32 | gulp.task('env', function() { 33 | gulp.src('./config.env.production') 34 | .pipe(rename('.env')) 35 | .pipe(gulp.dest('./dist')) 36 | }); 37 | 38 | // Now the dist directory is ready to go. Zip it. 39 | gulp.task('zip', function() { 40 | gulp.src(['dist/**/*', '!dist/package.json', 'dist/.*']) 41 | .pipe(zip('dist.zip')) 42 | .pipe(gulp.dest('./')); 43 | }); 44 | 45 | // Per the gulp guidelines, we do not need a plugin for something that can be 46 | // done easily with an existing node module. #CodeOverConfig 47 | // 48 | // Note: This presumes that AWS.config already has credentials. This will be 49 | // the case if you have installed and configured the AWS CLI. 50 | // 51 | // See http://aws.amazon.com/sdk-for-node-js/ 52 | gulp.task('upload', function() { 53 | 54 | // TODO: This should probably pull from package.json 55 | AWS.config.region = 'us-east-1'; 56 | var lambda = new AWS.Lambda(); 57 | var functionName = 'video-events'; 58 | 59 | lambda.getFunction({FunctionName: functionName}, function(err, data) { 60 | if (err) { 61 | if (err.statusCode === 404) { 62 | var warning = 'Unable to find lambda function ' + deploy_function + '. ' 63 | warning += 'Verify the lambda function name and AWS region are correct.' 64 | gutil.log(warning); 65 | } else { 66 | var warning = 'AWS API request failed. ' 67 | warning += 'Check your AWS credentials and permissions.' 68 | gutil.log(warning); 69 | } 70 | } 71 | 72 | // This is a bit silly, simply because these five parameters are required. 73 | var current = data.Configuration; 74 | var params = { 75 | FunctionName: functionName, 76 | Handler: current.Handler, 77 | Mode: current.Mode, 78 | Role: current.Role, 79 | Runtime: current.Runtime 80 | }; 81 | 82 | fs.readFile('./dist.zip', function(err, data) { 83 | params['FunctionZip'] = data; 84 | lambda.uploadFunction(params, function(err, data) { 85 | if (err) { 86 | var warning = 'Package upload failed. ' 87 | warning += 'Check your iam:PassRole permissions.' 88 | gutil.log(warning); 89 | } 90 | }); 91 | }); 92 | }); 93 | }); 94 | 95 | // The key to deploying as a single command is to manage the sequence of events. 96 | gulp.task('default', function(callback) { 97 | return runSequence( 98 | ['clean'], 99 | ['js', 'npm', 'env'], 100 | ['zip'], 101 | ['upload'], 102 | callback 103 | ); 104 | }); 105 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var Firebase = require('firebase'); 2 | var async = require('async'); 3 | 4 | // Extract data from the kinesis event 5 | exports.handler = function(event, context) { 6 | 7 | // This function abstracts the expected structure of any Kinesis payload, 8 | // which is a base64-encoded string of a JSON object, passing the data to 9 | // a private function. 10 | function handlePayload(record, callback) { 11 | encodedPayload = record.kinesis.data; 12 | rawPayload = new Buffer(encodedPayload, 'base64').toString('utf-8'); 13 | handleData(JSON.parse(rawPayload), callback); 14 | } 15 | 16 | // The Kinesis event may contain multiple records in a specific order. 17 | // Since our handlers are asynchronous, we handle each payload in series, 18 | // calling the parent handler's callback (context.done) upon completion. 19 | async.eachSeries(event.Records, handlePayload, context.done) 20 | }; 21 | 22 | // This is just an intermediate function. The projectId is buried in an 23 | // analyticsId of the format clientId-projectId-version. So the string split 24 | // extracts the projectId. 25 | function handleData(data, callback) { 26 | var projectRef = getProjectRef(); 27 | var analyticsId = JSON.parse(data.properties).analyticsId; 28 | var projectId = analyticsId.split('-')[1] 29 | incrementProjectEvent(projectRef, projectId, data.event, callback); 30 | }; 31 | 32 | 33 | // Lambda does not have access to environment variables, so we hardcode the 34 | // production db URL into the codebase (this is non-sensitive) but override 35 | // with local environment variables for development and test. 36 | function getProjectRef() { 37 | var prodFirebaseUrl = 'https://luminous-heat-2841.firebaseio.com/'; 38 | var firebaseUrl = process.env.FIREBASE_URL || prodFirebaseUrl; 39 | return new Firebase(firebaseUrl + 'project/'); 40 | }; 41 | 42 | // Incrementing a value is an atomic operation, meaning that you need to know 43 | // the current value in order to add to it, and multiple set operations cannot 44 | // be executed at the same time. Firebase provides for this using transactions. 45 | // 46 | // As long as the callback is executed on completion of the transaction, you 47 | // are in good shape. 48 | function incrementProjectEvent(projectRef, projectId, eventName, callback) { 49 | projectRef 50 | .child(projectId) 51 | .child('events/' + eventName) 52 | .transaction(function(currentData) { 53 | if (currentData === null) { 54 | return 1; 55 | } else { 56 | return currentData + 1; 57 | } 58 | }, function(error, committed, snapshot) { 59 | if (error) { 60 | console.log('Transaction failed abnormally!', error); 61 | } 62 | callback(); 63 | }); 64 | }; 65 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "adventr-lambda-video", 3 | "version": "0.0.0", 4 | "description": "Initial lambda functions for Adventr", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha test/*test.js", 8 | "start": "mocha test/*test.js -w -R min" 9 | }, 10 | "author": "Adam Neary (http://adamrneary.com/)", 11 | "license": "MIT", 12 | "dependencies": { 13 | "async": "^0.9.0", 14 | "dotenv": "^0.4.0", 15 | "firebase": "^2.0.6" 16 | }, 17 | "devDependencies": { 18 | "aws-sdk": "^2.1.4", 19 | "del": "^1.1.0", 20 | "guid": "0.0.12", 21 | "gulp": "^3.8.10", 22 | "gulp-install": "^0.2.0", 23 | "gulp-rename": "^1.2.0", 24 | "gulp-util": "^3.0.2", 25 | "gulp-zip": "^2.0.2", 26 | "kinesis": "^1.0.2", 27 | "mocha": "^2.1.0", 28 | "run-sequence": "^1.0.2", 29 | "stream": "0.0.2" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /test/record-kinesis-events.js: -------------------------------------------------------------------------------- 1 | var kinesis = require('kinesis'); 2 | Transform = require('stream').Transform; 3 | fs = require('fs'); 4 | 5 | // Data is retrieved as Record objects, so we transform into Buffers. 6 | var bufferify = new Transform({objectMode: true}) 7 | bufferify._transform = function(record, encoding, cb) { 8 | cb(null, record.Data) 9 | } 10 | readStream = kinesis.stream({name: 'video-events'}) 11 | .pipe(bufferify) 12 | 13 | // We can we pipe to stdout and to a newline-separated flat file. 14 | readStream.pipe(process.stdout); 15 | writeStream = fs.createWriteStream('test/events.log'); 16 | readStream.on('data', function(chunk) { 17 | data = new Buffer(chunk, 'base64').toString('utf-8') + "\r\n"; 18 | writeStream.write(data); 19 | }); 20 | -------------------------------------------------------------------------------- /test/video-events-test.js: -------------------------------------------------------------------------------- 1 | var dotenv = require('dotenv'); 2 | var Firebase = require('firebase'); 3 | var assert = require("assert"); 4 | var VideoEvents = require('../index'); 5 | 6 | dotenv.load(); 7 | 8 | // This function is boilerplate for replicating how Amazon Kinesis events are 9 | // passed to Lambda for processing. 10 | // 11 | // payloadString - The String to package 12 | // 13 | // Returns an Object as the data would be passed to Lambda 14 | function packageEvent(payloadString) { 15 | var encodedPayload = new Buffer(payloadString).toString('base64') 16 | return { 17 | "Records": [{ 18 | "kinesis": { 19 | "partitionKey": "partitionKey-3", 20 | "kinesisSchemaVersion": "1.0", 21 | "data": encodedPayload, 22 | "sequenceNumber": "49545115243490985018280067714973144582180062593244200961" 23 | }, 24 | "eventSource": "aws:kinesis", 25 | "eventID": "shardId-000000000000:49545115243490985018280067714973144582180062593244200961", 26 | "invokeIdentityArn": "arn:aws:iam::059493405231:role/testLEBRole", 27 | "eventVersion": "1.0", 28 | "eventName": "aws:kinesis:record", 29 | "eventSourceARN": "arn:aws:kinesis:us-east-1:35667example:stream/examplestream", 30 | "awsRegion": "us-east-1" 31 | }] 32 | }; 33 | } 34 | 35 | // This object is boilerplate for the context passed to an Amazon Lambda 36 | // function. Calling done() exits the process Lambda creates. 37 | var context = { 38 | done: function(error, message) { 39 | console.log('done!'); 40 | process.exit(1); 41 | } 42 | }; 43 | 44 | // This function mimics how our upstream code (in this case, the Flask beacon) 45 | // pipes video events to Kinesis. 46 | // 47 | // campaignId - An Integer id for the campaign being viewed 48 | // eventName - A String for the name of the event being recorded 49 | // 50 | // Returns a String containing the data as passed to Kinesis 51 | function generateEvent(campaignId, eventName) { 52 | var analyticsId = '123-' + campaignId + '-456'; 53 | var properties = { 54 | ip: '79.104.4.85', 55 | mp_lib: 'as3', 56 | clip: 'konets-f8gr', 57 | referrer: 'https://play.adventr.tv/video-9-76005686', 58 | viewId: 'CENYr73adwDEd55MyrmTrmjK2QEDpUAe', 59 | http_referrer: 'https://play.adventr.tv/embed.html?source=ochre-play&load=0&poster=//d252srr1zuysk4.cloudfront.net/clients/43126710716529437798-2004/4575/thumbnails/1-i5nd_poster_0000.png&width=352&height=288&x=http://static.theochre.com/clients/43126710716529437798-2004/4575/published/43126710716529437798-2004-video-9-76005686.data', 60 | analyticsId: analyticsId, 61 | flashVersion: 'MAC 16,0,0,235', 62 | token: 'ochre-vpaid', 63 | distinct_id: '6886A9F3-680C-FD5C-41BC-CB4F4E17F6F0', 64 | Video: 'Video 9', 65 | time: '1419097235', 66 | swfUrl: 'https://d252srr1zuysk4.cloudfront.net/advntr/AdvntrPlayer.swf', 67 | duration: 405, 68 | pageUrl: 'https://d252srr1zuysk4.cloudfront.net/advntr/AdvntrPlayer.swf', 69 | ua: 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) AppleWebKit/537.78.2 (KHTML, like Gecko) Version/7.0.6 Safari/537.78.2' 70 | }; 71 | var rawPayload = { 72 | event_id: '2014-12-20:35', 73 | event_date: '2014-12-20', 74 | event: eventName, 75 | event_time: '1419097235_YX4', 76 | properties: JSON.stringify(properties) 77 | }; 78 | return JSON.stringify(rawPayload); 79 | } 80 | 81 | // Copied from index.js (for now) 82 | function getFirebaseRef() { 83 | var prodFirebaseUrl = 'https://luminous-heat-2841.firebaseio.com/'; 84 | var firebaseUrl = process.env.FIREBASE_URL || prodFirebaseUrl; 85 | return new Firebase(firebaseUrl + 'project/'); 86 | }; 87 | 88 | // Yes, random numbers are a red flag for non-deterministic tests, but in 89 | // this case we are using them as unique IDs for projects. Not too flagrant 90 | // a violation given that we are removing the child after each test. 91 | function getUniqueId() { 92 | return Math.floor(Math.random()*1000000000); 93 | } 94 | 95 | describe('VideoEvents', function(){ 96 | var projectRef = getFirebaseRef(); 97 | var testId = getUniqueId(); 98 | 99 | describe('Video View', function(){ 100 | var eventName = 'Video View'; 101 | 102 | before(function(done) { 103 | projectRef 104 | .child(testId) 105 | .child('events/' + eventName) 106 | .set(100, done); 107 | }) 108 | 109 | after(function(done) { 110 | projectRef 111 | .child(testId) 112 | .remove(done); 113 | }) 114 | 115 | it('should increment the video view event value', function(done){ 116 | 117 | var testObject = packageEvent(generateEvent(testId, eventName)); 118 | var testContext = { 119 | done: function(error, message) { 120 | projectRef 121 | .child(testId) 122 | .child('events/' + eventName) 123 | .once("value", function(snapshot) { 124 | assert.equal(101, snapshot.val()); 125 | done(); 126 | }); 127 | } 128 | }; 129 | 130 | VideoEvents.handler(testObject, testContext); 131 | 132 | }) 133 | }) 134 | }) 135 | --------------------------------------------------------------------------------