├── lib ├── util │ ├── promisify.js │ ├── promisify-lambda.js │ ├── notify-unsubscribe.js │ ├── apikey-check.js │ ├── lambda-error-handler.js │ └── send-to-slack.js ├── subscription.js ├── arn.js ├── redis │ ├── namespaces.js │ └── client.js ├── subscription │ ├── check-parameters.js │ ├── get.js │ ├── remove.js │ ├── add.js │ └── broadcast.js ├── web-push.js ├── sns.js └── bulk-publish.js ├── lambda ├── batch-unsubscribe │ ├── event.json │ ├── s-function.json │ └── batch-unsubscribe.js ├── get-subscriptions │ ├── get-subscriptions.js │ └── s-function.json ├── listener │ ├── s-function.json │ └── listener.js ├── receive-broadcast │ ├── s-function.json │ └── handler.js ├── publish │ ├── publish.js │ └── s-function.json └── subscription │ ├── subscription.js │ └── s-function.json ├── .babelrc ├── hiredis-shim.js ├── .gitignore ├── test-bootstrap ├── bootstrap.js ├── env.json └── stubs.js ├── s-project.json ├── test ├── utils.js ├── fixtures │ ├── example_web_push_subscription.js │ ├── example_batch_unsubscribe.js │ └── example_sns_event.json ├── lambda │ ├── batch-unsubscribe.js │ └── subscription.js ├── lib │ ├── subscription │ │ ├── get.js │ │ ├── remove.js │ │ ├── broadcast.js │ │ └── add.js │ ├── sns.js │ └── bulk-publish.js └── flow.js ├── package.json ├── webpack.config.js ├── README.md ├── s-resources-cf.json └── s-templates.json /lib/util/promisify.js: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /lambda/batch-unsubscribe/event.json: -------------------------------------------------------------------------------- 1 | {} -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | presets: ['es2015'] 3 | } -------------------------------------------------------------------------------- /hiredis-shim.js: -------------------------------------------------------------------------------- 1 | throw new Error("nope"); 2 | 3 | module.exports = {}; -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .DS_Store 3 | _meta 4 | scratchpad 5 | admin.env 6 | .idea 7 | -------------------------------------------------------------------------------- /lib/subscription.js: -------------------------------------------------------------------------------- 1 | import add from './subscription/add'; 2 | import remove from './subscription/remove'; 3 | import broadcast from './subscription/broadcast'; 4 | import get from './subscription/get'; 5 | 6 | export default { 7 | add: add, 8 | remove: remove, 9 | broadcast: broadcast, 10 | get: get 11 | }; -------------------------------------------------------------------------------- /lambda/get-subscriptions/get-subscriptions.js: -------------------------------------------------------------------------------- 1 | import getSubscription from '../../lib/subscription/get'; 2 | 3 | 4 | export function handler(event, context) { 5 | getSubscription(event.subscription) 6 | .then((subscribed) => { 7 | context.done(null, subscribed); 8 | }) 9 | .catch((err) => { 10 | context.fail(err); 11 | }) 12 | } -------------------------------------------------------------------------------- /test-bootstrap/bootstrap.js: -------------------------------------------------------------------------------- 1 | import {withRedis} from '../lib/redis/client'; 2 | import should from 'should'; 3 | import env from './env.json'; 4 | 5 | 6 | global.should = should; 7 | 8 | Object.assign(process.env, env); 9 | 10 | afterEach(function() { 11 | return withRedis((redisClient) => redisClient.flushdb()); 12 | }); 13 | 14 | process.env.tests_bootstrapped = true; 15 | 16 | 17 | console.info = () => {} -------------------------------------------------------------------------------- /lambda/listener/s-function.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "listener", 3 | "runtime": "nodejs4.3", 4 | "description": "Lambda to pick up topic publishes from SNS and re-broadcast", 5 | "customName": false, 6 | "customRole": "${exec_role}", 7 | "handler": "listener.default", 8 | "timeout": 300, 9 | "memorySize": 1024, 10 | "authorizer": {}, 11 | "custom": { 12 | "excludePatterns": [] 13 | }, 14 | "endpoints": [], 15 | "events": [], 16 | "environment": "$${environment}", 17 | "vpc": "$${vpc}" 18 | } -------------------------------------------------------------------------------- /s-project.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pushy", 3 | "custom": { 4 | "serverless-mocha-plugin": { 5 | "bootstrap": "test-bootstrap/babel.js" 6 | }, 7 | "serverless-offline": { 8 | "babelOptions": { 9 | "presets": ["es2015-node4"] 10 | } 11 | }, 12 | "webpack": { 13 | "configPath": "./webpack.config.js" 14 | } 15 | }, 16 | "plugins": [ 17 | "serverless-webpack-plugin", 18 | "serverless-offline", 19 | "serverless-plugin-autoprune", 20 | "serverless-plugin-sns" 21 | ] 22 | } -------------------------------------------------------------------------------- /lambda/receive-broadcast/s-function.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "receive-broadcast", 3 | "runtime": "nodejs4.3", 4 | "description": "Serverless Lambda function for project: pushy", 5 | "customName": false, 6 | "customRole": false, 7 | "handler": "handler.handler", 8 | "timeout": 300, 9 | "memorySize": 1024, 10 | "authorizer": {}, 11 | "custom": { 12 | "excludePatterns": [] 13 | }, 14 | "sns": { 15 | "topic": "${stage}__batch_broadcast" 16 | }, 17 | "events": [], 18 | "environment": "$${environment}", 19 | "vpc": "$${vpc}" 20 | } -------------------------------------------------------------------------------- /test-bootstrap/env.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWS_REGION": "us-east-1", 3 | "AWS_DEFAULT_REGION": "us-east-1", 4 | "AWS_LAMBDA_LOG_GROUP_NAME": "/aws/lambda/blah", 5 | "AWS_LAMBDA_FUNCTION_MEMORY_SIZE": "1024", 6 | "AWS_LAMBDA_FUNCTION_VERSION": "11", 7 | "SERVERLESS_PROJECT": "pushy", 8 | "SERVERLESS_STAGE": "test_stage", 9 | "SERVERLESS_REGION": "us-east-1", 10 | "SERVERLESS_DATA_MODEL_STAGE": "test_stage", 11 | "IAM_ROLE": "arn:aws:iam::0000001:role/pushy-test_stage-r-IamRoleLambda-RANDOMCHARS", 12 | "GCM_API_KEY": "testkey" 13 | } -------------------------------------------------------------------------------- /lambda/publish/publish.js: -------------------------------------------------------------------------------- 1 | import checkAPIKey from '../../lib/util/apikey-check'; 2 | import LambdaErrorHandler from '../../lib/util/lambda-error-handler'; 3 | import SNS from '../../lib/sns'; 4 | 5 | export default (event, context, cb) => { 6 | return checkAPIKey("admin", event) 7 | .then(() => { 8 | return SNS.publish(event.topic, event.message); 9 | }) 10 | .then(() => { 11 | cb(null, { 12 | success: true, 13 | message: event.message 14 | }) 15 | }) 16 | .catch(LambdaErrorHandler(cb)) 17 | } -------------------------------------------------------------------------------- /lib/util/promisify-lambda.js: -------------------------------------------------------------------------------- 1 | 2 | export default function(func) { 3 | return function(event, context, cb) { 4 | context.callbackWaitsForEmptyEventLoop = false; 5 | 6 | Promise.resolve(func(event, context)) 7 | .then((data) => { 8 | cb(null, data); 9 | }) 10 | .catch((error) => { 11 | cb(JSON.stringify({ 12 | statusCode: error.statusCode || 500, 13 | response: JSON.stringify(error.toString() || "An error occurred") 14 | })); 15 | }) 16 | } 17 | } -------------------------------------------------------------------------------- /test/utils.js: -------------------------------------------------------------------------------- 1 | 2 | export function createDummySubscribers(redisClient, {topic, number, scoreGenerator, subscriptionGenerator}) { 3 | 4 | if (!scoreGenerator) { 5 | scoreGenerator = (i) => i * 3; 6 | } 7 | 8 | if (!subscriptionGenerator) { 9 | subscriptionGenerator = (i) => `doesn't-matter-${i}`; 10 | } 11 | 12 | let redisArgs = [topic]; 13 | 14 | for (let i = 0; i < number; i++) { 15 | redisArgs.push(scoreGenerator(i), subscriptionGenerator(i)); 16 | } 17 | 18 | return redisClient.zadd(redisArgs); 19 | } -------------------------------------------------------------------------------- /lib/util/notify-unsubscribe.js: -------------------------------------------------------------------------------- 1 | import SNS from '../../lib/sns'; 2 | 3 | export default function(failures, topic) { 4 | let notRegistered = failures.reduce((coll, f) => { 5 | if (f.error.message == 'NotRegistered') { 6 | coll.push(f.subscription); 7 | } 8 | 9 | return coll; 10 | }, []); 11 | 12 | console.log('Sending unsubscribe notification: ', process.env.BATCH_UNSUBSCRIBE_TOPIC, JSON.stringify({topic: topic, subscriptions: notRegistered})); 13 | return SNS.publish(process.env.BATCH_UNSUBSCRIBE_TOPIC, {topic: topic, subscriptions: notRegistered}); 14 | } -------------------------------------------------------------------------------- /test/fixtures/example_web_push_subscription.js: -------------------------------------------------------------------------------- 1 | 2 | export const BROWSER_SUBSCRIPTION_OBJECT = { 3 | "endpoint": "https://android.googleapis.com/gcm/send/f1LsxkKphfQ:APA91bFUx7ja4BK4JVrNgVjpg1cs9lGSGI6IMNL4mQ3Xe6mDGxvt_C_gItKYJI9CAx5i_Ss6cmDxdWZoLyhS2RJhkcv7LeE6hkiOsK6oBzbyifvKCdUYU7ADIRBiYNxIVpLIYeZ8kq_A", 4 | "keys": { 5 | "p256dh":"BLc4xRzKlKORKWlbdgFaBrrPK3ydWAHo4M0gs0i1oEKgPpWC5cW8OCzVrOQRv-1npXRWk8udnW3oYhIO4475rds=", 6 | "auth":"5I2Bu2oKdyy9CwL8QVF0NQ==" 7 | } 8 | } 9 | 10 | export const EXAMPLE_REQUEST = { 11 | type: "web", 12 | data: BROWSER_SUBSCRIPTION_OBJECT 13 | } -------------------------------------------------------------------------------- /test/fixtures/example_batch_unsubscribe.js: -------------------------------------------------------------------------------- 1 | export const BATCH_UNSUBSCRIBE_SNS = { 2 | topic: 'my-topic', 3 | subscriptions: [ 4 | { 5 | "endpoint": "https://android.googleapis.com/gcm/send/f1LsxkKphfQ:APA91bFUx7ja4BK4JVrNgVjpg1cs9lGSGI6IMNL4mQ3Xe6mDGxvt_C_gItKYJI9CAx5i_Ss6cmDxdWZoLyhS2RJhkcv7LeE6hkiOsK6oBzbyifvKCdUYU7ADIRBiYNxIVpLIYeZ8kq_A", 6 | "keys": { 7 | "p256dh":"BLc4xRzKlKORKWlbdgFaBrrPK3ydWAHo4M0gs0i1oEKgPpWC5cW8OCzVrOQRv-1npXRWk8udnW3oYhIO4475rds=", 8 | "auth":"5I2Bu2oKdyy9CwL8QVF0NQ==" 9 | } 10 | } 11 | ] 12 | }; -------------------------------------------------------------------------------- /lib/arn.js: -------------------------------------------------------------------------------- 1 | // Only way I could easily find to extract the current account ID without 2 | // hard-coding it in a config 3 | 4 | export const AWS_ACCOUNT_ID = process.env.IAM_ROLE.split(':')[4]; 5 | 6 | export function createArn(service, region, accountId, name) { 7 | return ['arn','aws', service, region, accountId, name].join(':'); 8 | } 9 | 10 | export function createSNSTopic(topicName) { 11 | return createArn('sns', process.env.SERVERLESS_REGION, AWS_ACCOUNT_ID, process.env.SERVERLESS_STAGE + "__" + topicName); 12 | } 13 | 14 | export function getTopicNameFromArn(arn) { 15 | return arn.split(":")[5].split('__')[1]; 16 | } -------------------------------------------------------------------------------- /lib/util/apikey-check.js: -------------------------------------------------------------------------------- 1 | 2 | export default function(keyName, event) { 3 | let keyNameUpper = keyName.toUpperCase(); 4 | return Promise.resolve() 5 | .then(() => { 6 | let keyInConfig = process.env[`API_KEY__${keyNameUpper}`]; 7 | if (event.apiKey !== keyInConfig) { 8 | console.error(`Attempt to access function with key ${event.apiKey}, expected ${keyInConfig} (${keyNameUpper})`) 9 | let err = new Error("You do not have permission to do this."); 10 | err.name = "Forbiddena"; 11 | err.code = 403; 12 | throw err; 13 | } 14 | }) 15 | 16 | } -------------------------------------------------------------------------------- /lambda/batch-unsubscribe/s-function.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "batch-unsubscribe", 3 | "runtime": "nodejs4.3", 4 | "description": "Serverless Lambda function for project: pushy", 5 | "customName": false, 6 | "customRole": false, 7 | "handler": "batch-unsubscribe.default", 8 | "timeout": 60, 9 | "memorySize": 1024, 10 | "authorizer": {}, 11 | "custom": { 12 | "excludePatterns": [] 13 | }, 14 | "endpoints": [], 15 | "events": [ 16 | { 17 | "name": "sns", 18 | "type": "sns", 19 | "config": { 20 | "topicName": "${stage}__${batch_unsubscribe_topic}" 21 | } 22 | } 23 | ], 24 | "environment": "$${environment}", 25 | "vpc": "$${vpc}" 26 | } -------------------------------------------------------------------------------- /lib/redis/namespaces.js: -------------------------------------------------------------------------------- 1 | 2 | export default { 3 | topic: (topic) => `${process.env.SERVERLESS_STAGE}:topic:subscriptions::${topic}`, 4 | snsSubscription: (topic) => `${process.env.SERVERLESS_STAGE}:sns:subscription:arn::${topic}`, 5 | performanceLogging: (messageId) => `${process.env.SERVERLESS_STAGE}:broadcast:timing::${messageId}`, 6 | allTopicList: () => `${process.env.SERVERLESS_STAGE}:topic:list`, 7 | failedSendList: (topic, messageId) => `${process.env.SERVERLESS_STAGE}:broadcast:failed::${topic}:${messageId}`, 8 | successfulCount: (topic, messageId) => `${process.env.SERVERLESS_STAGE}:broadcast:succeeded::${topic}:${messageId}`, 9 | extract: (fullStr) => fullStr.split('::')[1] 10 | } 11 | -------------------------------------------------------------------------------- /lambda/subscription/subscription.js: -------------------------------------------------------------------------------- 1 | import Subscription from '../../lib/subscription'; 2 | import lambdaErrorHandler from '../../lib/util/lambda-error-handler'; 3 | import PromisifyLambda from '../../lib/util/promisify-lambda'; 4 | 5 | export default (event, context, cb) => { 6 | return Promise.resolve() 7 | .then(() => { 8 | 9 | if (event.action === 'add') { 10 | return Subscription.add(event.topic, event.subscription); 11 | } else { 12 | return Subscription.remove(event.topic, event.subscription); 13 | } 14 | 15 | }) 16 | .then(() => { 17 | cb(null, { 18 | success: true 19 | }); 20 | }) 21 | .catch(lambdaErrorHandler(cb)) 22 | }; -------------------------------------------------------------------------------- /test-bootstrap/stubs.js: -------------------------------------------------------------------------------- 1 | import mockery from 'mockery'; 2 | import fakeredis from 'fakeredis'; 3 | import should from 'should'; 4 | /* 5 | We mock out the redis module so that we can just 6 | use a local in-memory module instead of requiring 7 | redis in the dev stack. 8 | */ 9 | 10 | mockery.enable({ useCleanCache: true }); 11 | mockery.warnOnUnregistered(false); 12 | mockery.registerMock('redis', { 13 | RedisClient: fakeredis.RedisClient, 14 | Multi: fakeredis.Multi, 15 | createClient: function() { 16 | 17 | // by default fakeredis simulates network latency, we don't really 18 | // need/want that. 19 | 20 | return fakeredis.createClient(1000, "dummyhost.local", {fast : true}) 21 | } 22 | }); 23 | 24 | console.info = () => {} -------------------------------------------------------------------------------- /lambda/listener/listener.js: -------------------------------------------------------------------------------- 1 | import Subscription from '../../lib/subscription'; 2 | import { getTopicNameFromArn } from '../../lib/arn'; 3 | 4 | export default (event, context, cb) => { 5 | Promise.resolve() 6 | .then(() => { 7 | // if (!event.Sns.Message['web-push']) { 8 | // console.warn("Ignoring message as it has no web-push key", event.Sns.Message); 9 | // return; 10 | // } 11 | let sns = event.Records[0].Sns; 12 | let topicName = getTopicNameFromArn(sns.TopicArn); 13 | console.info("Sending broadcast to " + topicName); 14 | return Subscription.broadcast(topicName, sns); 15 | }) 16 | .then(() => { 17 | cb(null, true); 18 | }) 19 | .catch((err) => { 20 | cb(err); 21 | }) 22 | } -------------------------------------------------------------------------------- /test/fixtures/example_sns_event.json: -------------------------------------------------------------------------------- 1 | { 2 | "EventSource": "aws:sns", 3 | "EventVersion": "1.0", 4 | "EventSubscriptionArn": "arn:aws:sns:us-east-1:123123123123:test:e0b04b92-c198-444e-afa7-4aab44edeef3", 5 | "Sns": { 6 | "Type": "Notification", 7 | "MessageId": "f01bb2c6-853a-5b0d-9a2c-fd575b850276", 8 | "TopicArn": "arn:aws:sns:us-east-1:123123123123:test", 9 | "Subject": null, 10 | "Message": "{\"test\":\"test\"}", 11 | "Timestamp": "2016-04-20T19:46:07.774Z", 12 | "SignatureVersion": "1", 13 | "Signature": "sig", 14 | "SigningCertUrl": "https://sns.us-east-1.amazonaws.com/cert.pem", 15 | "UnsubscribeUrl": "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe", 16 | "MessageAttributes": {} 17 | } 18 | } -------------------------------------------------------------------------------- /lib/util/lambda-error-handler.js: -------------------------------------------------------------------------------- 1 | import {forceClose as forceCloseRedis} from '../redis/client'; 2 | require('source-map-support').install(); 3 | 4 | // Generic handler for our lambdas that attempts to parse out 5 | // errors in a way our error template can parse. It'll use the 6 | // statusCode in the HTTP response, and output the response 7 | // as the JSON body. 8 | 9 | export default function(cb) { 10 | return (error) => { 11 | 12 | // Make sure we have killed our Redis connection, otherwise 13 | // the Lambda event loop won't stop 14 | 15 | forceCloseRedis(); 16 | 17 | cb(JSON.stringify({ 18 | statusCode: error.statusCode || 500, 19 | response: JSON.stringify(error.toString() || "An error occurred") 20 | })); 21 | } 22 | } -------------------------------------------------------------------------------- /lib/util/send-to-slack.js: -------------------------------------------------------------------------------- 1 | import fetch from 'node-fetch'; 2 | 3 | export default function(obj) { 4 | if (!process.env.SLACK_WEBHOOK) { 5 | return console.info("no webhook") 6 | } 7 | 8 | let toSend = Object.assign({ 9 | username: "pushy-debug-" + process.env.SERVERLESS_STAGE.toUpperCase(), 10 | icon_url: process.env.SERVERLESS_STAGE === "staging" ? "http://www.stg.gdnmobilelab.com/icon.png" : "http://www.gdnmobilelab.com/icon.png" 11 | }, obj) 12 | console.log("webhook?", process.env.SLACK_WEBHOOK) 13 | return fetch(process.env.SLACK_WEBHOOK, { 14 | method: "POST", 15 | headers: { 16 | "Content-Type": "application/json" 17 | }, 18 | body: JSON.stringify(toSend) 19 | }) 20 | .catch((err) => { 21 | console.error(err); 22 | }) 23 | .then((res) => res.text()) 24 | } -------------------------------------------------------------------------------- /lambda/batch-unsubscribe/batch-unsubscribe.js: -------------------------------------------------------------------------------- 1 | import PromiseTools from 'promise-tools'; 2 | import Subscription from '../../lib/subscription'; 3 | import sendToSlack from '../../lib/util/send-to-slack'; 4 | 5 | export default (event, context, cb) => { 6 | let sns = event.Records[0].Sns; 7 | let parsedMessage = JSON.parse(sns.Message); 8 | console.log('received batch unsubscribe', parsedMessage); 9 | 10 | PromiseTools.map(parsedMessage.subscriptions, (subscription) => { 11 | return Subscription.remove(parsedMessage.topic, { 12 | type: 'web', 13 | data: subscription 14 | }); 15 | }).then((removed) => { 16 | 17 | let slackMsg = { 18 | text: `Removed ${removed.length} NotRegistered subscribers` 19 | }; 20 | 21 | sendToSlack(slackMsg).then(() => context.done(null)); 22 | }).catch((err) => { 23 | context.done(err); 24 | }) 25 | } 26 | -------------------------------------------------------------------------------- /test/lambda/batch-unsubscribe.js: -------------------------------------------------------------------------------- 1 | import handler from '../../lambda/batch-unsubscribe/batch-unsubscribe.js'; 2 | import PromisifyLambda from 'promisify-aws-lambda'; 3 | import {EXAMPLE_REQUEST} from '../fixtures/example_batch_unsubscribe'; 4 | import sinon from 'sinon'; 5 | 6 | describe('Batch Unsubscribe endpoints', function() { 7 | 8 | // stub out the SNS subscription functions 9 | 10 | before(() => { 11 | sinon.stub(Subscription, 'remove').returns(Promise.resolve()); 12 | 13 | // dummy IAM role 14 | process.env.IAM_ROLE = 'arn:aws:iam::000000001:role/pushy-testtest-r-IamRoleLambda-RANDOMCHARS'; 15 | }); 16 | 17 | after(() => { 18 | Subscription.remove.restore(); 19 | }); 20 | 21 | 22 | it('Calls the add subscription method', function() { 23 | 24 | return PromisifyLambda(handler, EXAMPLE_REQUEST) 25 | .then((response) => { 26 | response.success.should.equal(true); 27 | }); 28 | 29 | }); 30 | }); 31 | -------------------------------------------------------------------------------- /lambda/receive-broadcast/handler.js: -------------------------------------------------------------------------------- 1 | import BulkPublish from '../../lib/bulk-publish'; 2 | require('source-map-support').install(); 3 | import notifyUnsubscribe from '../../lib/util/notify-unsubscribe'; 4 | 5 | export function handler(event, context, cb) { 6 | 7 | let sns = event.Records[0].Sns; 8 | let parsedMessage = JSON.parse(sns.Message); 9 | console.log('received message', parsedMessage); 10 | BulkPublish.send(parsedMessage.topicName, parsedMessage.range, parsedMessage.originalMessage, parsedMessage.broadcastIndex) 11 | .then((responses) => { 12 | //No needless lambda 13 | if (responses.failure.length) { 14 | notifyUnsubscribe(responses.failure, parsedMessage.topicName).then(() => { 15 | context.done(null); 16 | }).catch((err) => { 17 | context.done(err); 18 | }); 19 | } else { 20 | context.done(); 21 | } 22 | }) 23 | .catch((err) => { 24 | context.done(err); 25 | }) 26 | 27 | }; 28 | -------------------------------------------------------------------------------- /lib/subscription/check-parameters.js: -------------------------------------------------------------------------------- 1 | import url from 'url'; 2 | 3 | export default function(topic, subscription) { 4 | 5 | if (typeof topic !== 'string' || topic.length === 0) { 6 | let err = new Error("You must provide a topic for the subscription"); 7 | err.name = 'TopicRequired'; 8 | throw err; 9 | } 10 | 11 | if (subscription.type === 'web') { 12 | let parsedUrl = url.parse(subscription.data.endpoint); 13 | if (parsedUrl.protocol !== 'https:' || parsedUrl.host === '') { 14 | let err = new Error("Endpoint must be a valid, HTTPS URL."); 15 | err.name = 'InvalidEndpoint'; 16 | err.statusCode = 400; 17 | throw err; 18 | } 19 | 20 | if (!subscription.data.keys.p256dh || !subscription.data.keys.auth) { 21 | let err = new Error("Must provide both p256dh and auth keys (Chrome only, for now)"); 22 | err.name = 'InvalidEndpoint'; 23 | err.statusCode = 400; 24 | throw err; 25 | } 26 | } else { 27 | let err = new Error("Only support web notifications for now."); 28 | err.name = 'InvalidEndpoint'; 29 | err.statusCode = 400; 30 | throw err; 31 | } 32 | 33 | } -------------------------------------------------------------------------------- /lambda/get-subscriptions/s-function.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "get-subscriptions", 3 | "runtime": "nodejs4.3", 4 | "description": "Serverless Lambda function for project: pushy", 5 | "customName": false, 6 | "customRole": false, 7 | "handler": "get-subscriptions.handler", 8 | "timeout": 6, 9 | "memorySize": 128, 10 | "authorizer": {}, 11 | "custom": { 12 | "excludePatterns": [] 13 | }, 14 | "endpoints": [ 15 | { 16 | "path": "get-subscriptions", 17 | "method": "POST", 18 | "type": "AWS", 19 | "authorizationType": "none", 20 | "authorizerFunction": false, 21 | "apiKeyRequired": true, 22 | "requestParameters": {}, 23 | "requestTemplates": { 24 | "application/json": { 25 | "subscription": "$input.json('$')", 26 | "apiKey": "$input.params('x-api-key')" 27 | } 28 | }, 29 | "responses": "$${allResponseTemplates}" 30 | }, 31 | { 32 | "path": "get-subscriptions", 33 | "method": "OPTIONS", 34 | "type": "MOCK", 35 | "requestTemplates": "$${apiCorsRequestTemplate}", 36 | "responses": "$${apiCorsOptionsResponse}" 37 | } 38 | ], 39 | "events": [], 40 | "environment": "$${environment}", 41 | "vpc": "$${vpc}" 42 | } -------------------------------------------------------------------------------- /lambda/publish/s-function.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "publish", 3 | "runtime": "nodejs4.3", 4 | "description": "Serverless Lambda function for project: pushy", 5 | "customName": false, 6 | "customRole": "${exec_role}", 7 | "handler": "publish.default", 8 | "timeout": 300, 9 | "memorySize": 128, 10 | "authorizer": {}, 11 | "custom": { 12 | "excludePatterns": [] 13 | }, 14 | "endpoints": [ 15 | { 16 | "path": "topics/{topic}", 17 | "method": "POST", 18 | "type": "AWS", 19 | "authorizationType": "none", 20 | "authorizerFunction": false, 21 | "apiKeyRequired": true, 22 | "requestParameters": {}, 23 | "requestTemplates": { 24 | "application/json": { 25 | "topic": "$input.params('topic')", 26 | "message": "$input.json('$')", 27 | "apiKey": "$input.params('x-api-key')" 28 | } 29 | }, 30 | "responses": "$${limitDomainResponseTemplates}" 31 | }, 32 | { 33 | "path": "topics/{topic}", 34 | "method": "OPTIONS", 35 | "type": "MOCK", 36 | "requestTemplates": "$${apiCorsRequestTemplate}", 37 | "responses": "$${apiCorsLimitToAdminOptionsResponse}" 38 | } 39 | ], 40 | "events": [], 41 | "environment": "$${environment}", 42 | "vpc": "$${vpc}" 43 | } -------------------------------------------------------------------------------- /lib/web-push.js: -------------------------------------------------------------------------------- 1 | import webPush from 'web-push'; 2 | 3 | webPush.setGCMAPIKey(process.env.GCM_API_KEY); 4 | 5 | const push = { 6 | sendNotification: function(endpoint, options) { 7 | 8 | return webPush.sendNotification(endpoint, options) 9 | .then((resp) => { 10 | let json = JSON.parse(resp); 11 | 12 | if (json.results[0].error) { 13 | if (json.results[0].error === 'MismatchSenderId' && 14 | process.env.BACKUP_GCM_API_KEY && 15 | options.overrideGCMAPIKey !== process.env.BACKUP_GCM_API_KEY) { 16 | 17 | // stupid hack to make up for me accidentally using staging creds in 18 | // production. Can't change them in people's service workers now! 19 | 20 | let withOverride = Object.assign({}, options, { 21 | overrideGCMAPIKey: process.env.BACKUP_GCM_API_KEY 22 | }); 23 | 24 | return push.sendNotification(endpoint, withOverride); 25 | } else { 26 | throw new Error(json.results[0].error) 27 | } 28 | } 29 | return resp; 30 | }) 31 | } 32 | }; 33 | 34 | export default push; -------------------------------------------------------------------------------- /lib/subscription/get.js: -------------------------------------------------------------------------------- 1 | import {withRedis} from '../redis/client'; 2 | import namespaces from '../redis/namespaces'; 3 | 4 | export default function(subscription) { 5 | 6 | // The string we will have actually stored in Redis. 7 | let subscriptionStored = JSON.stringify(subscription.data); 8 | 9 | return withRedis((redisClient) => { 10 | if (subscription.type !== 'web') { 11 | let err = new Error("Only web notifications currently supported"); 12 | err.name = 'InvalidProtocol'; 13 | throw err; 14 | } 15 | 16 | return redisClient.smembers(namespaces.allTopicList()) 17 | .then((topics) => { 18 | 19 | // Take each of our topics and check if our subscription object 20 | // is in them. 21 | 22 | let multi = redisClient.multi(); 23 | for (let topic of topics) { 24 | multi.zrank(topic, subscriptionStored); 25 | } 26 | return multi.exec() 27 | .then((subscriptions) => { 28 | 29 | let subscribedTopics = topics.filter((t, idx) => { 30 | // if an entry does not appear in a sorted set zrank returns null 31 | return subscriptions[idx] !== null 32 | }) 33 | 34 | return subscribedTopics.map(namespaces.extract); 35 | }) 36 | }) 37 | }); 38 | 39 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pushkin", 3 | "version": "0.0.1", 4 | "description": "A Serverless Project and its Serverless Plugin dependencies.", 5 | "author": "me", 6 | "license": "MIT", 7 | "private": false, 8 | "repository": { 9 | "type": "git", 10 | "url": "git://github.com/" 11 | }, 12 | "scripts": { 13 | "test": "mocha --compilers js:babel-register --require test-bootstrap/stubs.js test-bootstrap/bootstrap.js --recursive test", 14 | "test-watch": "npm run test -- --watch", 15 | "local": "sls offline start" 16 | }, 17 | "dependencies": { 18 | "aws-sdk": "^2.3.5", 19 | "babel-loader": "^6.2.4", 20 | "babel-preset-es2015": "^6.6.0", 21 | "chai": "^3.5.0", 22 | "fakeaws": "0.0.3", 23 | "fsmonitor": "^0.2.4", 24 | "json-loader": "^0.5.4", 25 | "mock-aws-sinon": "^1.0.3", 26 | "node-fetch": "^1.5.3", 27 | "promise-redis": "0.0.5", 28 | "promise-tools": "^1.0.2", 29 | "serverless-offline": "^2.2.10", 30 | "serverless-plugin-autoprune": "^0.5.3", 31 | "serverless-plugin-sns": "0.0.10", 32 | "serverless-webpack-plugin": "^0.4.1", 33 | "source-map-support": "^0.4.0", 34 | "web-push": "^2.0.2" 35 | }, 36 | "devDependencies": { 37 | "babel-register": "^6.7.2", 38 | "fakeredis": "^1.0.3", 39 | "mockery": "^1.6.2", 40 | "nock": "^8.0.0", 41 | "sns-simulator": "^1.0.0", 42 | "webpack": "^1.12.15", 43 | "should": "^8.3.1", 44 | "sinon": "^1.17.3" 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | var webpack = require('webpack'); 2 | var path = require('path'); 3 | 4 | module.exports = { 5 | target: "node", 6 | externals: [ 7 | 'aws-sdk' // this is provided by Lambda itself 8 | ], 9 | // Make webpack exit with error code when it encounters an error. Why is this not default?! 10 | bail: true, 11 | module: { 12 | loaders: [ 13 | { 14 | test: /\.js$/, 15 | loader: 'babel', 16 | exclude: /node_modules/ 17 | }, 18 | { 19 | test: /\.json$/, 20 | loader: 'json' 21 | } 22 | ] 23 | }, 24 | resolve: { 25 | extensions: ['', '.json', '.js', '.jsx'], 26 | alias: { 27 | // This is really stupid but node_redis tries to require hiredis, which we don't 28 | // want to provide. Adding this stops the build failing. 29 | 'node_modules/redis-commands/commands': 'node_modules/redis-commands/commands.json', 30 | 'hiredis': path.join(__dirname, 'hiredis-shim.js') 31 | } 32 | }, 33 | devtool: 'source-map', 34 | plugins: [ 35 | new webpack.optimize.DedupePlugin(), 36 | new webpack.optimize.OccurenceOrderPlugin(), 37 | new webpack.optimize.UglifyJsPlugin({ 38 | compress: { 39 | unused: true, 40 | dead_code: true, 41 | warnings: false, 42 | drop_debugger: true 43 | } 44 | }) 45 | ] 46 | } -------------------------------------------------------------------------------- /lib/subscription/remove.js: -------------------------------------------------------------------------------- 1 | import checkParameters from './check-parameters'; 2 | import {withRedis} from '../redis/client'; 3 | import namespaces from '../redis/namespaces'; 4 | import SNS from '../sns'; 5 | 6 | export default function(topic, subscription) { 7 | checkParameters(topic, subscription); 8 | 9 | let namespacedTopic = namespaces.topic(topic); 10 | 11 | return withRedis((redisClient) => { 12 | return redisClient.zrem(namespacedTopic, JSON.stringify(subscription.data)) 13 | .then((numberRemoved) => { 14 | 15 | if (numberRemoved === 0) { 16 | 17 | console.info("User was not subscribed - doing nothing"); 18 | // User actually wasn't subscribed in the first place, so 19 | // no need to do our SNS check. 20 | 21 | return false; 22 | } 23 | 24 | console.info("Removed user subscription to", topic); 25 | 26 | return redisClient.zcard(namespacedTopic) 27 | .then((numberOfSubscribers) => { 28 | 29 | if (numberOfSubscribers !== 0) { 30 | return; 31 | } 32 | 33 | console.info("No subscribers left for this topic - removing SNS subscription"); 34 | 35 | return SNS.removeTopicSubscription(topic) 36 | .then(() => { 37 | return redisClient.srem(namespaces.allTopicList(), namespacedTopic); 38 | }) 39 | }) 40 | .then(() => { 41 | return true; 42 | }) 43 | }) 44 | }) 45 | } -------------------------------------------------------------------------------- /test/lib/subscription/get.js: -------------------------------------------------------------------------------- 1 | import namespaces from '../../../lib/redis/namespaces'; 2 | import SNS from '../../../lib/sns'; 3 | import Subscription from '../../../lib/subscription'; 4 | import sinon from 'sinon'; 5 | import {EXAMPLE_REQUEST, BROWSER_SUBSCRIPTION_OBJECT} from '../../fixtures/example_web_push_subscription'; 6 | import mockAWSSinon from 'mock-aws-sinon'; 7 | 8 | describe("Subscription/get", function() { 9 | 10 | before(() => { 11 | mockAWSSinon('SNS', 'subscribe', function(args, cb) { 12 | cb(null, {SubscriptionArn: 'blah-blah'}) 13 | }) 14 | 15 | mockAWSSinon('lambda', 'addPermission', function(args, cb) { 16 | cb(null, {}) 17 | }) 18 | }) 19 | 20 | after(() => { 21 | mockAWSSinon('SNS', 'subscribe').restore(); 22 | mockAWSSinon('lambda', 'addPermission').restore(); 23 | }) 24 | 25 | it("should return all subscribed topics", function() { 26 | 27 | let cloneExample = JSON.parse(JSON.stringify(EXAMPLE_REQUEST)); 28 | cloneExample.endpoint = "https://blahblah.local"; 29 | return Subscription.add('test-topic', EXAMPLE_REQUEST) 30 | .then(() => { 31 | // add a different topic to ensure that we are only returning 32 | // the relevant ones 33 | Subscription.add('test-topic-two', cloneExample) 34 | }) 35 | .then((res) => { 36 | return Subscription.get(EXAMPLE_REQUEST) 37 | }) 38 | .then((res) => { 39 | res.length.should.equal(1); 40 | res[0].should.equal('test-topic'); 41 | }) 42 | }) 43 | 44 | }) -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pushkin 2 | 3 | A Node and AWS Lambda/SNS powered system for sending 4 | [web notifications](https://developer.mozilla.org/en-US/docs/Web/API/notification) 5 | with [payloads](https://developers.google.com/web/updates/2016/03/web-push-encryption?hl=en). 6 | 7 | ## Why? 8 | 9 | Amazon SNS is able to send push notifications to both iOS and Android devices, 10 | but not web clients. Sending these notifications is not simple as the payload 11 | requires individual encryption for each client. 12 | 13 | ## What does it do? 14 | 15 | Pushkin listens to SNS topics, receiving push notifications as they are sent 16 | to iOS and Android devices. It then multiplies that message by the number 17 | of web clients subscribed, batches them up (currently into batches of 200) and 18 | fires Lambda instances for each batch, allowing us to send numerous notifications 19 | simultaneously. 20 | 21 | It uses Redis to store subscription data and the [web-push](https://github.com/marco-c/web-push) 22 | library to encrypt messages being sent. 23 | 24 | ## Requirements 25 | 26 | - Node 6.0 (5+ may be fine, not tested) 27 | - Redis (or AWS Elasticache) 28 | 29 | ## Installation 30 | 31 | Clone the repo, then run 32 | 33 | npm install 34 | 35 | to install all dependencies. It uses the [Serverless](https://github.com/serverless/serverless) 36 | framework to simplify deploying lambdas. More installation instructions are needed 37 | but code can be deployed by running: 38 | 39 | sls dash deploy 40 | 41 | Tests can be run with: 42 | 43 | npm run test 44 | 45 | ## Status 46 | 47 | This started off as an experiment and ballooned outwards, so the code is quite 48 | messy and the documentation is currently sparse. Sorry. I intend to fix both of 49 | those issues when I have time to do so. 50 | -------------------------------------------------------------------------------- /test/lambda/subscription.js: -------------------------------------------------------------------------------- 1 | 2 | import handler from '../../lambda/subscription/subscription.js'; 3 | import PromisifyLambda from 'promisify-aws-lambda'; 4 | import Subscription from '../../lib/subscription'; 5 | import {EXAMPLE_REQUEST} from '../fixtures/example_web_push_subscription'; 6 | import sinon from 'sinon'; 7 | 8 | describe('Subscription endpoints', function() { 9 | 10 | // stub out the SNS subscription functions 11 | 12 | before(() => { 13 | sinon.stub(Subscription, 'add').returns(Promise.resolve()); 14 | sinon.stub(Subscription, 'remove').returns(Promise.resolve()); 15 | 16 | // dummy IAM role 17 | process.env.IAM_ROLE = 'arn:aws:iam::000000001:role/pushy-testtest-r-IamRoleLambda-RANDOMCHARS'; 18 | }) 19 | 20 | after(() => { 21 | Subscription.add.restore(); 22 | Subscription.remove.restore(); 23 | }) 24 | 25 | 26 | it('Calls the add subscription method', function() { 27 | 28 | return PromisifyLambda(handler, { 29 | action: 'add', 30 | topic: 'test-topic', 31 | subscription: EXAMPLE_REQUEST 32 | }) 33 | .then((response) => { 34 | Subscription.add.calledWith('test-topic', EXAMPLE_REQUEST).should.equal(true); 35 | response.success.should.equal(true); 36 | }); 37 | 38 | }); 39 | 40 | it('Calls the remove subscription method', function() { 41 | 42 | return PromisifyLambda(handler, { 43 | action: 'remove', 44 | topic: 'test-topic', 45 | subscription: EXAMPLE_REQUEST 46 | }) 47 | .then((response) => { 48 | Subscription.remove.calledWith('test-topic', EXAMPLE_REQUEST).should.equal(true); 49 | response.success.should.equal(true); 50 | }); 51 | 52 | }); 53 | }); 54 | -------------------------------------------------------------------------------- /lambda/subscription/s-function.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "subscription", 3 | "runtime": "nodejs4.3", 4 | "description": "Add and remove individual user subscriptions to topics", 5 | "customName": false, 6 | "customRole": "${exec_role}", 7 | "handler": "subscription.default", 8 | "timeout": 6, 9 | "memorySize": 128, 10 | "authorizer": {}, 11 | "custom": { 12 | "excludePatterns": [] 13 | }, 14 | "endpoints": [ 15 | { 16 | "path": "topics/{topic}/subscriptions", 17 | "method": "POST", 18 | "type": "AWS", 19 | "authorizationType": "none", 20 | "authorizerFunction": false, 21 | "apiKeyRequired": true, 22 | "requestParameters": {}, 23 | "requestTemplates": { 24 | "application/json": { 25 | "action": "add", 26 | "topic": "$input.params('topic')", 27 | "subscription": "$input.json('$')", 28 | "apiKey": "$input.params('x-api-key')" 29 | } 30 | }, 31 | "responses": "$${allResponseTemplates}" 32 | }, 33 | { 34 | "path": "topics/{topic}/subscriptions", 35 | "method": "DELETE", 36 | "type": "AWS", 37 | "authorizationType": "none", 38 | "authorizerFunction": false, 39 | "apiKeyRequired": true, 40 | "requestParameters": {}, 41 | "requestTemplates": { 42 | "application/json": { 43 | "action": "remove", 44 | "topic": "$input.params('topic')", 45 | "subscription": "$input.json('$')", 46 | "apiKey": "$input.params('x-api-key')" 47 | } 48 | }, 49 | "responses": "$${allResponseTemplates}" 50 | }, 51 | { 52 | "path": "topics/{topic}/subscriptions", 53 | "method": "OPTIONS", 54 | "type": "MOCK", 55 | "requestTemplates": "$${apiCorsRequestTemplate}", 56 | "responses": "$${apiCorsOptionsResponse}" 57 | } 58 | ], 59 | "events": [], 60 | "environment": "$${environment}", 61 | "vpc": "$${vpc}" 62 | } -------------------------------------------------------------------------------- /s-resources-cf.json: -------------------------------------------------------------------------------- 1 | { 2 | "AWSTemplateFormatVersion": "2010-09-09", 3 | "Description": "The AWS CloudFormation template for this Serverless application's resources outside of Lambdas and Api Gateway", 4 | "Resources": { 5 | "IamRoleLambda": { 6 | "Type": "AWS::IAM::Role", 7 | "Properties": { 8 | "AssumeRolePolicyDocument": { 9 | "Version": "2012-10-17", 10 | "Statement": [ 11 | { 12 | "Effect": "Allow", 13 | "Principal": { 14 | "Service": [ 15 | "lambda.amazonaws.com" 16 | ] 17 | }, 18 | "Action": [ 19 | "sts:AssumeRole" 20 | ] 21 | } 22 | ] 23 | }, 24 | "Path": "/" 25 | } 26 | }, 27 | "IamPolicyLambda": { 28 | "Type": "AWS::IAM::Policy", 29 | "Properties": { 30 | "PolicyName": "${stage}-${project}-lambda", 31 | "PolicyDocument": { 32 | "Version": "2012-10-17", 33 | "Statement": [ 34 | { 35 | "Effect": "Allow", 36 | "Action": [ 37 | "logs:CreateLogGroup", 38 | "logs:CreateLogStream", 39 | "logs:PutLogEvents", 40 | "ec2:CreateNetworkInterface", 41 | "ec2:DescribeNetworkInterfaces", 42 | "ec2:DeleteNetworkInterface", 43 | "sns:Subscribe", 44 | "sns:Publish", 45 | "sns:ListSubscriptionsByTopic", 46 | "sns:CreateTopic", 47 | "sns:DeleteTopic", 48 | "sns:UnSubscribe" 49 | ], 50 | "Resource": "*" 51 | } 52 | ] 53 | }, 54 | "Roles": [ 55 | { 56 | "Ref": "IamRoleLambda" 57 | } 58 | ] 59 | } 60 | } 61 | }, 62 | "Outputs": { 63 | "IamRoleArnLambda": { 64 | "Description": "ARN of the lambda IAM role", 65 | "Value": { 66 | "Fn::GetAtt": [ 67 | "IamRoleLambda", 68 | "Arn" 69 | ] 70 | } 71 | } 72 | } 73 | } -------------------------------------------------------------------------------- /lib/redis/client.js: -------------------------------------------------------------------------------- 1 | import promiseRedis from 'promise-redis'; 2 | 3 | const redis = promiseRedis(); // uses native Promise implementation 4 | 5 | /* 6 | * By default AWS Lambdas won't finish until the event loop is closed 7 | * which means we can't keep things like Redis connections open 8 | * persistently. Instead, we use this promise chain to open and close 9 | * connections as required. It allows use to re-use an open connection 10 | * and only close it when every promise chain is complete. 11 | */ 12 | 13 | let currentActiveClient = null; 14 | let numberOfChainsAttachedToClient = 0; 15 | 16 | // Check if we have any promise chains still using this connection 17 | // if not, we can safely close out the connection 18 | 19 | const closeCurrentConnectionIfNecessary = function() { 20 | numberOfChainsAttachedToClient = numberOfChainsAttachedToClient - 1; 21 | 22 | if (numberOfChainsAttachedToClient === 0) { 23 | let deactivatingClient = currentActiveClient; 24 | currentActiveClient = null; 25 | return deactivatingClient.quit() 26 | } 27 | 28 | return Promise.resolve() 29 | } 30 | 31 | // Primarily exporting this for use in tests 32 | 33 | export function createClient() { 34 | return redis.createClient(process.env.REDIS_HOST); 35 | } 36 | 37 | export function forceClose() { 38 | if (currentActiveClient) { 39 | return currentActiveClient.quit() 40 | 41 | } 42 | return Promise.resolve(); 43 | } 44 | 45 | export function withRedis(promiseChain) { 46 | 47 | if (!currentActiveClient) { 48 | currentActiveClient = createClient(); 49 | numberOfChainsAttachedToClient = 1; 50 | } else { 51 | numberOfChainsAttachedToClient = numberOfChainsAttachedToClient + 1; 52 | } 53 | 54 | // Pass the client to our promise chain 55 | let chain = promiseChain(currentActiveClient) 56 | 57 | // Then make sure we both catch success and failures of the promise 58 | // and check if we're good to close our connection or not 59 | 60 | return chain.then( 61 | function(result) { 62 | return closeCurrentConnectionIfNecessary() 63 | .then(() => { 64 | return result 65 | }); 66 | }, 67 | function(error) { 68 | return closeCurrentConnectionIfNecessary() 69 | .then(() => { 70 | throw error 71 | }); 72 | } 73 | ); 74 | } -------------------------------------------------------------------------------- /test/lib/sns.js: -------------------------------------------------------------------------------- 1 | import SNS from '../../lib/sns'; 2 | import mockAWSSinon from 'mock-aws-sinon'; 3 | import namespaces from '../../lib/redis/namespaces'; 4 | import {createClient as createRedisClient} from '../../lib/redis/client'; 5 | import should from 'should'; 6 | import {createArn, createSNSTopic, AWS_ACCOUNT_ID} from '../../lib/arn'; 7 | import sinon from 'sinon'; 8 | 9 | const FAKE_ARN = 'arn:aws:sns:us-east-1:234234234:staging__test:asdfasdfasdfasfd'; 10 | const redisClient = createRedisClient(); 11 | 12 | describe("SNS", function() { 13 | 14 | before(() => { 15 | mockAWSSinon('SNS','subscribe').returns({ 16 | SubscriptionArn: FAKE_ARN 17 | }); 18 | 19 | mockAWSSinon('SNS','unsubscribe').returns(true); 20 | 21 | mockAWSSinon('lambda','addPermission', function(data, cb) { 22 | cb(null, {}) 23 | }) 24 | }) 25 | 26 | after(() => { 27 | mockAWSSinon('SNS','subscribe').restore(); 28 | mockAWSSinon('SNS','unsubscribe').restore(); 29 | mockAWSSinon('lambda', 'addPermission').restore(); 30 | mockAWSSinon.restore(); 31 | }) 32 | 33 | 34 | it("Adds a topic subscription and stores in Redis", () => { 35 | return SNS.addTopicSubscription('test') 36 | .then(() => { 37 | return redisClient.get(namespaces.snsSubscription('test')) 38 | }) 39 | .then((arn) => { 40 | arn.should.equal(FAKE_ARN); 41 | mockAWSSinon('SNS','subscribe').calledWith(sinon.match({ 42 | TopicArn: createSNSTopic('test'), 43 | Protocol: 'lambda', 44 | Endpoint: createArn('lambda', process.env.SERVERLESS_REGION, AWS_ACCOUNT_ID, ['function', process.env.SERVERLESS_PROJECT + '-listener', process.env.SERVERLESS_STAGE].join(':')) 45 | })).should.equal(true); 46 | }) 47 | }) 48 | 49 | it("Removes a topic subscription from Redis", () => { 50 | return SNS.addTopicSubscription('test') 51 | .then(() => { 52 | return SNS.removeTopicSubscription('test') 53 | }) 54 | .then(() => { 55 | return redisClient.get(namespaces.snsSubscription('test')) 56 | }) 57 | .then((returnArn) => { 58 | should.equal(returnArn, null); 59 | mockAWSSinon('SNS','unsubscribe').calledWith(sinon.match({ 60 | SubscriptionArn: FAKE_ARN 61 | })).should.equal(true); 62 | }) 63 | }) 64 | }) -------------------------------------------------------------------------------- /lib/subscription/add.js: -------------------------------------------------------------------------------- 1 | import checkParameters from './check-parameters'; 2 | import {withRedis} from '../redis/client'; 3 | import namespaces from '../redis/namespaces'; 4 | import SNS from '../sns'; 5 | import webPush from '../web-push'; 6 | import remove from './remove' 7 | 8 | export default function(topic, subscription) { 9 | checkParameters(topic, subscription); 10 | let namespacedTopic = namespaces.topic(topic); 11 | return withRedis((redisClient) => { 12 | 13 | return redisClient.zcard(namespacedTopic) 14 | .then((numberOfSubscribers) => { 15 | 16 | if (numberOfSubscribers === 0) { 17 | 18 | // If there are no existing subscribers then we need to set up 19 | // an SNS subscription internally, so that we receive push events 20 | // along with native apps. 21 | 22 | return SNS.addTopicSubscription(topic); 23 | } 24 | }) 25 | .then(() => { 26 | 27 | // Now that we've guaranteed we have SNS subscriptions, we can push 28 | // our new endpoint into Redis. We're using a sorted set to allow us to 29 | // later get a range of subscribers rather than all at once. Using the 30 | // current timestamp as the score, so that when that batching process 31 | // occurs we don't end up out of order. 32 | return redisClient.multi() 33 | .zadd(namespacedTopic, Date.now(), JSON.stringify(subscription.data)) 34 | .sadd(namespaces.allTopicList(),namespacedTopic) 35 | .exec(); 36 | }) 37 | }) 38 | .then(([zaddResponse, saddResponse]) => { 39 | // if zaddResponse is 0 then the user was already subscribed. 40 | if (zaddResponse === 1) { 41 | console.info("Added new subscriber to:", namespacedTopic); 42 | } else { 43 | console.info("Already subscribed user requested to be added again. Ignoring."); 44 | } 45 | return zaddResponse === 1; 46 | }) 47 | .then((newSubscriber) => { 48 | if (!subscription.confirmationNotification) { 49 | return newSubscriber; 50 | }; 51 | console.log("Sending confirmation notification...") 52 | return webPush.sendNotification(subscription.data.endpoint, { 53 | payload: JSON.stringify(subscription.confirmationNotification), 54 | userPublicKey: subscription.data.keys.p256dh, 55 | userAuth: subscription.data.keys.auth 56 | }) 57 | .catch((err) => { 58 | // If an error occurred we need to unsub 59 | console.log("Error sending confirmation...", err.toString(), Object.keys(err)) 60 | return remove(topic, subscription) 61 | .then(() => { 62 | throw err; 63 | }) 64 | }) 65 | }) 66 | } -------------------------------------------------------------------------------- /test/lib/subscription/remove.js: -------------------------------------------------------------------------------- 1 | import {createClient as createRedisClient} from '../../../lib/redis/client'; 2 | import namespaces from '../../../lib/redis/namespaces'; 3 | import SNS from '../../../lib/sns'; 4 | import Subscription from '../../../lib/subscription'; 5 | import sinon from 'sinon'; 6 | import {EXAMPLE_REQUEST, BROWSER_SUBSCRIPTION_OBJECT} from '../../fixtures/example_web_push_subscription'; 7 | import should from 'should'; 8 | 9 | const redisClient = createRedisClient(); 10 | 11 | describe('Subscription/remove', function() { 12 | 13 | // stub out the SNS subscription functions 14 | 15 | beforeEach(() => { 16 | sinon.stub(SNS, 'removeTopicSubscription').returns(Promise.resolve()); 17 | }) 18 | 19 | afterEach(() => { 20 | SNS.removeTopicSubscription.restore(); 21 | }) 22 | 23 | 24 | it('Removes users from a topic subscription', function() { 25 | // First add the entry we want to test removing. 26 | return redisClient.zadd([namespaces.topic('test-topic'), 1, JSON.stringify(BROWSER_SUBSCRIPTION_OBJECT)]) 27 | .then(() => { 28 | return Subscription.remove('test-topic', EXAMPLE_REQUEST); 29 | }) 30 | .then((response) => { 31 | response.should.equal(true); 32 | let multi = redisClient.multi(); 33 | multi.zscore(namespaces.topic('test-topic'), JSON.stringify(EXAMPLE_REQUEST)); 34 | multi.sismember(namespaces.allTopicList(), namespaces.topic('test-topic')); 35 | return multi.exec(); 36 | }) 37 | .then(([zscoreResponse, sismemberResponse]) => { 38 | should.equal(null, zscoreResponse); 39 | sismemberResponse.should.equal(0); 40 | }) 41 | }); 42 | 43 | it("Calls SNS unsubscribe when topic is empty", function() { 44 | // First add the entry we want to test removing. 45 | return redisClient.zadd([namespaces.topic('test-topic'), 1, JSON.stringify(BROWSER_SUBSCRIPTION_OBJECT)]) 46 | .then(() => { 47 | return Subscription.remove('test-topic', EXAMPLE_REQUEST); 48 | }) 49 | .then((response) => { 50 | SNS.removeTopicSubscription.calledWith('test-topic').should.equal(true); 51 | }) 52 | }); 53 | 54 | it("Does not call SNS unsubscribe when topic is non-empty", function() { 55 | 56 | let newExample = JSON.parse(JSON.stringify(EXAMPLE_REQUEST)) 57 | newExample.data.endpoint = 'https://blah.blah'; 58 | 59 | // First add the entry we want to test removing. 60 | return redisClient.zadd(namespaces.topic('test-topic'), 1, JSON.stringify(BROWSER_SUBSCRIPTION_OBJECT), 2, JSON.stringify(newExample.data)) 61 | .then((numAdded) => { 62 | return Subscription.remove('test-topic', newExample); 63 | }) 64 | .then((response) => { 65 | SNS.removeTopicSubscription.calledOnce.should.equal(false); 66 | }) 67 | }) 68 | 69 | }); 70 | -------------------------------------------------------------------------------- /test/flow.js: -------------------------------------------------------------------------------- 1 | import SNSSimulator from 'sns-simulator'; 2 | import {handler as subscriptionLambda} from '../lambda/subscription/subscription'; 3 | import {handler as listenerLambda} from '../lambda/listener/listener'; 4 | import {handler as receiveBroadcast} from '../lambda/receive-broadcast/handler'; 5 | import promisifyAwsLambda from 'promisify-aws-lambda'; 6 | import AWS from 'aws-sdk'; 7 | import Promise from 'bluebird'; 8 | import {createSNSTopic, createArn} from '../lib/arn'; 9 | import crypto from 'crypto'; 10 | import urlBase64 from 'urlsafe-base64'; 11 | import nock from 'nock'; 12 | 13 | const LISTENER_FUNC_PATH = createArn('lambda', process.env.AWS_REGION, '0000001', ['function', process.env.SERVERLESS_PROJECT + '-listener', process.env.SERVERLESS_STAGE].join(':')); 14 | //const BROADCAST_TOPIC_ARN = createSNSTopic('batch_broadcast'); 15 | 16 | var userCurve = crypto.createECDH('prime256v1'); 17 | var userPublicKey = urlBase64.encode(userCurve.generateKeys()); 18 | var userAuth = urlBase64.encode(crypto.randomBytes(16)); 19 | 20 | describe("SNS/Lambda workflow", function() { 21 | 22 | let sns = null; 23 | 24 | before(() => { 25 | SNSSimulator.setup(); 26 | SNSSimulator.registerLambda(LISTENER_FUNC_PATH, listenerLambda); 27 | SNSSimulator.registerLambda('BROADCAST_ARN', receiveBroadcast); 28 | 29 | sns = Promise.promisifyAll(new AWS.SNS()); 30 | 31 | return sns.createTopicAsync({ 32 | Name: process.env.SERVERLESS_STAGE + '__test-topic' 33 | }) 34 | .then(() => { 35 | return sns.createTopicAsync({ 36 | Name: process.env.SERVERLESS_STAGE + '__batch_broadcast' 37 | }) 38 | }) 39 | .then((resp) => { 40 | // serverless handles this for us in production 41 | return sns.subscribeAsync({ 42 | TopicArn: resp.TopicArn, 43 | Protocol: 'lambda', 44 | Endpoint: 'BROADCAST_ARN' 45 | }) 46 | }) 47 | }) 48 | 49 | after(() => { 50 | SNSSimulator.reset(); 51 | SNSSimulator.restore(); 52 | //nock.restore(); 53 | nock.cleanAll(); 54 | }) 55 | 56 | it("Should receive a published message after subscribing", () => { 57 | return true; 58 | let nockInstance = nock('https://subscription.local') 59 | .post(/endpoint\-/) 60 | .reply(201, { 61 | ok: true 62 | }); 63 | 64 | 65 | return promisifyAwsLambda(subscriptionLambda, { 66 | action: 'add', 67 | topic: 'test-topic', 68 | subscription: { 69 | type: 'web', 70 | data: { 71 | endpoint: `https://subscription.local/endpoint-1`, 72 | keys: { 73 | p256dh: userPublicKey, 74 | auth: userAuth 75 | } 76 | } 77 | 78 | } 79 | }) 80 | .then(() => { 81 | return sns.publishAsync({ 82 | TopicArn: createSNSTopic('test-topic'), 83 | Message: { 84 | "web-push": { 85 | "test": "hello" 86 | } 87 | } 88 | }) 89 | }) 90 | .then(() => { 91 | nockInstance.isDone().should.equal(true) 92 | }) 93 | }) 94 | }) -------------------------------------------------------------------------------- /test/lib/subscription/broadcast.js: -------------------------------------------------------------------------------- 1 | import {createClient as createRedisClient} from '../../../lib/redis/client'; 2 | import SNS from '../../../lib/sns'; 3 | import Subscription from '../../../lib/subscription'; 4 | import sinon from 'sinon'; 5 | import {EXAMPLE_REQUEST} from '../../fixtures/example_web_push_subscription'; 6 | import should from 'should'; 7 | import Promise from 'bluebird'; 8 | import namespaces from '../../../lib/redis/namespaces'; 9 | import {createDummySubscribers} from '../../utils'; 10 | 11 | const redisClient = createRedisClient(); 12 | 13 | describe('Subscription/broadcast', function() { 14 | 15 | beforeEach(() => { 16 | sinon.stub(SNS, 'publish').returns(true); 17 | }) 18 | 19 | afterEach(() => { 20 | SNS.publish.restore(); 21 | }) 22 | 23 | 24 | it("Only calls SNS publish once if subscribers is < batch number", () => { 25 | let topic = 'dummy-topic' 26 | let namespacedTopic = namespaces.topic(topic); 27 | 28 | return createDummySubscribers(redisClient, { 29 | topic: namespacedTopic, 30 | number: 50 31 | }) 32 | .then(() => { 33 | return Subscription.broadcast(topic, {test: 'test', MessageId: 'test-id'}) 34 | }) 35 | .then(() => { 36 | SNS.publish.calledOnce.should.equal(true); 37 | SNS.publish.calledWith('batch_broadcast', sinon.match({ 38 | range: ['-inf', '+inf'], 39 | broadcastIndex: 0, 40 | originalMessage: {test: 'test', MessageId: 'test-id'} 41 | })).should.equal(true) 42 | }) 43 | }) 44 | 45 | it("Calls SNS publish the correct number of times for batching", () => { 46 | let topic = 'dummy-topic' 47 | let namespacedTopic = namespaces.topic(topic); 48 | 49 | return createDummySubscribers(redisClient, { 50 | topic: namespacedTopic, 51 | number: 450 52 | }) 53 | .then(() => { 54 | return Subscription.broadcast(topic, {test: 'test', MessageId: 'test-id'}) 55 | }) 56 | .then(() => { 57 | SNS.publish.callCount.should.equal(3); 58 | 59 | SNS.publish.calledWith('batch_broadcast', sinon.match({ 60 | range: ['-inf', '600'], 61 | broadcastIndex: 0, 62 | originalMessage: {test: 'test', MessageId: 'test-id'} 63 | })).should.equal(true); 64 | 65 | SNS.publish.calledWith('batch_broadcast', sinon.match({ 66 | range: ['(600', '1200'], 67 | broadcastIndex: 1, 68 | originalMessage: {test: 'test', MessageId: 'test-id'} 69 | })).should.equal(true) 70 | 71 | SNS.publish.calledWith('batch_broadcast', sinon.match({ 72 | range: ['(1200', '+inf'], 73 | broadcastIndex: 2, 74 | originalMessage: {test: 'test', MessageId: 'test-id'} 75 | })).should.equal(true) 76 | 77 | return redisClient.hgetall(namespaces.performanceLogging('test-id')) 78 | }) 79 | .then((redisResponse) => { 80 | parseInt(redisResponse.start_time).should.be.greaterThan(0); // is Date.now(), so, variable 81 | redisResponse.end_time_0.should.equal('-1'); 82 | redisResponse.end_time_1.should.equal('-1'); 83 | redisResponse.end_time_2.should.equal('-1'); 84 | }) 85 | }) 86 | }); -------------------------------------------------------------------------------- /s-templates.json: -------------------------------------------------------------------------------- 1 | { 2 | "errorResponseTemplate": { 3 | "application/json": "#set ($errorMessageObj = $util.parseJson($input.path('$.errorMessage')))\n$errorMessageObj.response" 4 | }, 5 | "allResponseTemplates": { 6 | ".*\"statusCode\":400.*": { 7 | "statusCode": "400", 8 | "responseParameters": "$${corsResponseParameters}", 9 | "responseTemplates": "$${errorResponseTemplate}" 10 | }, 11 | ".*\"statusCode\":403.*": { 12 | "statusCode": "403", 13 | "responseParameters": "$${corsResponseParameters}", 14 | "responseTemplates": "$${errorResponseTemplate}" 15 | }, 16 | ".*\"statusCode\":500.*": { 17 | "statusCode": "500", 18 | "responseParameters": "$${corsResponseParameters}", 19 | "responseTemplates": "$${errorResponseTemplate}" 20 | }, 21 | ".*\"statusCode\":404.*": { 22 | "statusCode": "404", 23 | "responseParameters": "$${corsResponseParameters}", 24 | "responseTemplates": "$${errorResponseTemplate}" 25 | }, 26 | "default": { 27 | "responseParameters": "$${corsResponseParameters}", 28 | "statusCode": "200" 29 | } 30 | }, 31 | "limitDomainResponseTemplates": { 32 | ".*\"statusCode\":400.*": { 33 | "statusCode": "400", 34 | "responseParameters": "$${corsLimitDomainResponseParameters}", 35 | "responseTemplates": "$${errorResponseTemplate}" 36 | }, 37 | ".*\"statusCode\":403.*": { 38 | "statusCode": "403", 39 | "responseParameters": "$${corsLimitDomainResponseParameters}", 40 | "responseTemplates": "$${errorResponseTemplate}" 41 | }, 42 | ".*\"statusCode\":500.*": { 43 | "statusCode": "500", 44 | "responseParameters": "$${corsLimitDomainResponseParameters}", 45 | "responseTemplates": "$${errorResponseTemplate}" 46 | }, 47 | ".*\"statusCode\":404.*": { 48 | "statusCode": "404", 49 | "responseParameters": "$${corsLimitDomainResponseParameters}", 50 | "responseTemplates": "$${errorResponseTemplate}" 51 | }, 52 | "default": { 53 | "responseParameters": "$${corsLimitDomainResponseParameters}", 54 | "statusCode": "200" 55 | } 56 | }, 57 | "corsResponseParameters": { 58 | "method.response.header.Access-Control-Allow-Headers": "'Content-Type,X-Amz-Date,Authorization,X-Api-Key'", 59 | "method.response.header.Access-Control-Allow-Methods": "'GET,OPTIONS,HEAD,DELETE,PATCH,POST,PUT'", 60 | "method.response.header.Access-Control-Allow-Origin": "'*'", 61 | "method.response.header.Access-Control-Max-Age": "'3600'" 62 | }, 63 | "corsLimitDomainResponseParameters": { 64 | "method.response.header.Access-Control-Allow-Headers": "'Content-Type,X-Amz-Date,Authorization,X-Api-Key'", 65 | "method.response.header.Access-Control-Allow-Methods": "'GET,OPTIONS,HEAD,DELETE,PATCH,POST,PUT'", 66 | "method.response.header.Access-Control-Allow-Origin": "'${admin_domain}'", 67 | "method.response.header.Access-Control-Max-Age": "'3600'" 68 | }, 69 | "environment": { 70 | "SERVERLESS_PROJECT": "${project}", 71 | "SERVERLESS_STAGE": "${stage}", 72 | "SERVERLESS_REGION": "${region}", 73 | "IAM_ROLE":"${iamRoleArnLambda}", 74 | "REDIS_HOST": "${redis_host}", 75 | "GCM_API_KEY": "${gcm_api_key}", 76 | "BACKUP_GCM_API_KEY": "${backup_gcm_api_key}", 77 | "API_KEY__ADMIN": "${admin_api_key}", 78 | "SLACK_WEBHOOK": "${slack_webhook}", 79 | "BATCH_UNSUBSCRIBE_TOPIC": "${batch_unsubscribe_topic}" 80 | }, 81 | "vpc": "${vpc}", 82 | "apiCorsOptionsResponse": { 83 | "default": { 84 | "statusCode": "200", 85 | "responseParameters": "$${corsResponseParameters}", 86 | "responseModels": {}, 87 | "responseTemplates": { 88 | "application/json": "" 89 | } 90 | } 91 | }, 92 | "apiCorsLimitToAdminOptionsResponse": { 93 | "default": { 94 | "statusCode": "200", 95 | "responseParameters": "$${corsLimitDomainResponseParameters}", 96 | "responseModels": {}, 97 | "responseTemplates": { 98 | "application/json": "" 99 | } 100 | } 101 | }, 102 | "apiCorsRequestTemplate": { 103 | "application/json": { 104 | "statusCode": 200 105 | } 106 | } 107 | } -------------------------------------------------------------------------------- /lib/subscription/broadcast.js: -------------------------------------------------------------------------------- 1 | import {withRedis} from '../redis/client'; 2 | import namespaces from '../redis/namespaces'; 3 | import { createSNSTopic, getTopicNameFromArn } from '../arn'; 4 | import SNS from '../sns'; 5 | import PromiseTools from 'promise-tools'; 6 | import sendToSlack from '../util/send-to-slack'; 7 | 8 | const pushesPerBatch = 20 // Kind of just made this up. Be interesting to test what works best. 9 | 10 | export default function(topicName, sns) { 11 | let namespacedTopic = namespaces.topic(topicName); 12 | 13 | if (!sns.MessageId) { 14 | throw new Error("Must be provided with a unique MessageId to track execution time.") 15 | } 16 | 17 | let startTime = Date.now(); 18 | 19 | // This is where Redis sorted sets become very useful. We can batch our workload not 20 | // by index but by score - which doesn't change if someone unsubscribes in the middle 21 | // of the run. 22 | 23 | return withRedis((redisClient) => { 24 | let redisCountStart = Date.now(); 25 | return redisClient.zcount(namespacedTopic, "-inf", "+inf") 26 | .then((numberOfSubscribers) => { 27 | 28 | console.info("Fetched Redis count in:", Date.now() - redisCountStart); 29 | 30 | // Grab every batch interval, given the number of subscribers we have. 31 | 32 | let batchStartIndexes = []; 33 | 34 | for (let i = pushesPerBatch; i < numberOfSubscribers; i = i + pushesPerBatch) { 35 | batchStartIndexes.push(i); 36 | } 37 | 38 | return batchStartIndexes; 39 | }) 40 | .then((batchStartIndexes) => { 41 | 42 | return PromiseTools.map(batchStartIndexes, (index) => { 43 | return redisClient.zrange([namespacedTopic, index, index, 'WITHSCORES']) 44 | .then((arr) => { 45 | // this command returns both the value and the score. We only want the score. 46 | return arr[1]; 47 | }) 48 | }, batchStartIndexes.length) // Run them all at once 49 | }) 50 | .then((scores) => { 51 | 52 | // We don't hardcode the scores we're searching for at the start and end because there's 53 | // no point. Plus, using +inf in the end means we'll grab anyone that has signed up in 54 | // the time since we ran the range query. 55 | 56 | scores.unshift('-inf'); 57 | scores.push('+inf'); 58 | 59 | let ranges = []; 60 | 61 | for (let i = 0; i < scores.length - 1; i++) { 62 | 63 | let goFrom = scores[i]; 64 | let goTo = scores[i+1]; 65 | 66 | if (i > 0) { 67 | 68 | // The zrangebyscore command lets you select exclusively or inclusively. 69 | // Since we don't want to overlap and run our interval indexes twice, we 70 | // add a '(' in the second instance to make it exclusive. 71 | // 72 | // http://redis.io/commands/zrangebyscore 73 | 74 | goFrom = '(' + goFrom 75 | } 76 | 77 | ranges.push([goFrom, goTo]) 78 | } 79 | 80 | // We create a redis hash to let us track the performance of this broadcast - each receiver 81 | // logs its end time, then when all are filled we can see the overall start to end time. 82 | 83 | let multiSetArgs = [namespaces.performanceLogging(sns.MessageId), 'start_time', Date.now()]; 84 | ranges.forEach((range, index) => { 85 | 86 | // We pre-fill these hash keys with -1 so that we can detect later whether all 87 | // the receivers have completed or not. 88 | 89 | multiSetArgs.push(`end_time_${index}`, -1); 90 | }) 91 | return redisClient.hmset(multiSetArgs) 92 | .then(() => { 93 | let startBatch = Date.now(); 94 | return PromiseTools.map(ranges, (range, i) => { 95 | return SNS.publish('batch_broadcast', { 96 | range: range, 97 | broadcastIndex: i, 98 | topicName: topicName, 99 | originalMessage: sns 100 | }); 101 | }, ranges.length) 102 | .then((responses) => { 103 | let timeTaken = Date.now() - startBatch; 104 | console.info(`Sent ${responses.length} batches for processing.`) 105 | return sendToSlack({ 106 | text: `Sent ${responses.length} batches for processing in ${timeTaken}ms.` 107 | }) 108 | }) 109 | }); 110 | }) 111 | 112 | }) 113 | 114 | 115 | } -------------------------------------------------------------------------------- /test/lib/subscription/add.js: -------------------------------------------------------------------------------- 1 | import {createClient as createRedisClient} from '../../../lib/redis/client'; 2 | import namespaces from '../../../lib/redis/namespaces'; 3 | import SNS from '../../../lib/sns'; 4 | import Subscription from '../../../lib/subscription'; 5 | import sinon from 'sinon'; 6 | import {EXAMPLE_REQUEST} from '../../fixtures/example_web_push_subscription'; 7 | import should from 'should'; 8 | import webPush from '../../../lib/web-push'; 9 | 10 | const redisClient = createRedisClient(); 11 | 12 | describe('Subscription/add', function() { 13 | 14 | // stub out the SNS subscription functions 15 | 16 | beforeEach(() => { 17 | sinon.stub(SNS, 'addTopicSubscription').returns(Promise.resolve(true)); 18 | sinon.stub(SNS, 'removeTopicSubscription').returns(Promise.resolve(true)); 19 | }) 20 | 21 | afterEach(() => { 22 | SNS.addTopicSubscription.restore(); 23 | SNS.removeTopicSubscription.restore(); 24 | }) 25 | 26 | 27 | it('Adds users to topic subscription', function() { 28 | return Subscription.add('test-topic', EXAMPLE_REQUEST) 29 | .then((response) => { 30 | response.should.equal(true); 31 | 32 | let multi = redisClient.multi(); 33 | multi.zscore(namespaces.topic('test-topic'), JSON.stringify(EXAMPLE_REQUEST.data)); 34 | multi.sismember(namespaces.allTopicList(), namespaces.topic('test-topic')) 35 | return multi.exec(); 36 | 37 | }) 38 | .then(([zscoreResponse, sismemberResponse]) => { 39 | should(zscoreResponse).not.be.null(); 40 | sismemberResponse.should.equal(1); 41 | // Sanity check so we know it does return differently if not exist 42 | return redisClient.zscore('topic:subscriptions::test-topic', "SHOULD NOT EXIST") 43 | }) 44 | .then((redisResponse) => { 45 | should.equal(null, redisResponse); 46 | }) 47 | 48 | }); 49 | 50 | it("Calls SNS subscription create when subscribing to new topic", function() { 51 | return Subscription.add('test-topic', EXAMPLE_REQUEST) 52 | .then((response) => { 53 | SNS.addTopicSubscription.calledWith('test-topic').should.equal(true); 54 | }); 55 | }); 56 | 57 | it("Does not call SNS subscription create on existing topic", function() { 58 | 59 | let newExample = JSON.parse(JSON.stringify(EXAMPLE_REQUEST)) 60 | newExample.data.endpoint = 'https://blah.blah'; 61 | 62 | return redisClient.zadd([namespaces.topic('test-topic'), 1, JSON.stringify(EXAMPLE_REQUEST)]) 63 | .then(function() { 64 | return Subscription.add('test-topic', newExample) 65 | }) 66 | .then((response) => { 67 | SNS.addTopicSubscription.calledOnce.should.equal(false); 68 | }); 69 | }) 70 | 71 | describe("with confirmation notifications", function() { 72 | 73 | afterEach(() => { 74 | webPush.sendNotification.restore() 75 | }) 76 | 77 | let withNotification = Object.assign({ 78 | confirmationNotification: [ 79 | { 80 | "test": "yes" 81 | } 82 | ] 83 | }, EXAMPLE_REQUEST); 84 | 85 | 86 | 87 | it("Send a confirmation notification when specified", function() { 88 | sinon.stub(webPush, "sendNotification") 89 | .returns(Promise.resolve(JSON.stringify({ 90 | "multicast_id": 5818568061551720000, 91 | "success": 1, 92 | "failure": 0, 93 | "canonical_ids": 0, 94 | "results": [ 95 | { 96 | "message_id": "0:1463159425177924%b5e11c9ef9fd7ecd" 97 | } 98 | ] 99 | }))); 100 | 101 | return Subscription.add('test-topic', withNotification) 102 | .then((response) => { 103 | response.should.equal(true); 104 | webPush.sendNotification.calledOnce.should.equal(true); 105 | return redisClient.zscore(namespaces.topic('test-topic'), JSON.stringify(withNotification.data)); 106 | }) 107 | .then((redisResponse) => { 108 | should(redisResponse).not.be.null(); 109 | }) 110 | }); 111 | 112 | it("Remove subscription when confirmation notification fails", function() { 113 | sinon.stub(webPush, "sendNotification") 114 | .returns(Promise.resolve(JSON.stringify({ 115 | "multicast_id": 5818568061551720000, 116 | "success": 0, 117 | "failure": 1, 118 | "canonical_ids": 0, 119 | "results": [ 120 | { 121 | "error": "it failed" 122 | } 123 | ] 124 | }))); 125 | 126 | 127 | return Subscription.add('test-topic', withNotification) 128 | .catch((err) => { 129 | err.message.should.equal("it failed") 130 | return redisClient.zscore(namespaces.topic('test-topic'), JSON.stringify(withNotification.data)); 131 | }) 132 | .then((redisResponse) => { 133 | should(redisResponse).be.null(); 134 | }) 135 | }); 136 | 137 | }) 138 | 139 | }) -------------------------------------------------------------------------------- /test/lib/bulk-publish.js: -------------------------------------------------------------------------------- 1 | import {createClient as createRedisClient} from '../../lib/redis/client'; 2 | import namespaces from '../../lib/redis/namespaces'; 3 | import {createDummySubscribers} from '../utils'; 4 | import crypto from 'crypto'; 5 | import urlBase64 from 'urlsafe-base64'; 6 | import BulkPublish from '../../lib/bulk-publish'; 7 | import nock from 'nock'; 8 | import should from 'should'; 9 | import webPush from 'web-push'; 10 | import sinon from 'sinon'; 11 | 12 | const userCurve = crypto.createECDH('prime256v1'); 13 | const userPublicKey = urlBase64.encode(userCurve.generateKeys()); 14 | const userAuth = urlBase64.encode(crypto.randomBytes(16)); 15 | const redisClient = createRedisClient(); 16 | 17 | const addEndpointToNock = function(nockInstance, endpoint) { 18 | return nockInstance 19 | .post(endpoint) 20 | .reply(201, { 21 | "multicast_id": 5818568061551720000, 22 | "success": 1, 23 | "failure": 0, 24 | "canonical_ids": 0, 25 | "results": [ 26 | { 27 | "message_id": "0:1463159425177924%b5e11c9ef9fd7ecd" 28 | } 29 | ] 30 | }); 31 | } 32 | 33 | describe.only("Bulk publisher", function() { 34 | 35 | 36 | beforeEach(() => { 37 | 38 | }) 39 | 40 | afterEach(() => { 41 | nock.cleanAll(); 42 | }) 43 | 44 | it("Should publish to all endpoints", function() { 45 | 46 | let nockInstance = nock('https://subscription.local'); 47 | 48 | [0,1,2,3,4].forEach((i) => { 49 | addEndpointToNock(nockInstance, "/endpoint-" + i); 50 | }) 51 | 52 | let topic = 'dummy-topic' 53 | let namespacedTopic = namespaces.topic(topic); 54 | let hmsetArgs = [ 55 | namespaces.performanceLogging('test-message'), 56 | 'start_time', 57 | Date.now(), 58 | 'end_time_0', 59 | -1, 60 | 'end_time_1', 61 | -1 62 | ]; 63 | 64 | return redisClient.hmset(hmsetArgs) 65 | .then(() => { 66 | return createDummySubscribers(redisClient, { 67 | topic: namespacedTopic, 68 | number: 10, 69 | scoreGenerator: (i) => i, 70 | subscriptionGenerator: (i) => { 71 | return JSON.stringify({ 72 | endpoint: `https://subscription.local/endpoint-${i}`, 73 | keys: { 74 | p256dh: userPublicKey, 75 | auth: userAuth 76 | } 77 | }) 78 | } 79 | }) 80 | }) 81 | .then(() => { 82 | return BulkPublish.send(topic, [0,'(5'], {"Message": "hello", "MessageId": "test-message"}, 0); 83 | }) 84 | .then((sendResult) => { 85 | sendResult.failure.length.should.equal(0); 86 | nockInstance.isDone().should.equal(true); 87 | return redisClient.hget(namespaces.performanceLogging('test-message'), 'end_time_0') 88 | }) 89 | .then((redisResponse) => { 90 | parseInt(redisResponse,10).should.greaterThan(-1); 91 | 92 | // reset nock for the second round 93 | nockInstance = nock('https://subscription.local') 94 | 95 | let idxes = [5,6,7,8,9]; 96 | idxes.forEach((i) => { 97 | addEndpointToNock(nockInstance, "/endpoint-" + i); 98 | }) 99 | 100 | return BulkPublish.send(topic, [5,'(10'], {"Message": "hello", "MessageId": "test-message"}, 1); 101 | }) 102 | .then((sendResult) => { 103 | sendResult.failure.length.should.equal(0); 104 | return redisClient.hgetall(namespaces.performanceLogging(topic)); 105 | }) 106 | .then((redisResponse) => { 107 | nockInstance.done(); 108 | should(redisResponse).be.null(); 109 | }) 110 | }) 111 | 112 | it.only("should allow a single request to fail while other succeed", () => { 113 | let nockInstance = nock('https://subscription.local'); 114 | 115 | [0,2,3,4].forEach((i) => { 116 | addEndpointToNock(nockInstance, "/endpoint-" + i); 117 | }); 118 | 119 | let topic = 'dummy-topic' 120 | let namespacedTopic = namespaces.topic(topic); 121 | 122 | return createDummySubscribers(redisClient, { 123 | topic: namespacedTopic, 124 | number: 5, 125 | scoreGenerator: (i) => i * 1000, 126 | subscriptionGenerator: (i) => { 127 | return JSON.stringify({ 128 | endpoint: `https://subscription.local/endpoint-${i}`, 129 | keys: { 130 | p256dh: userPublicKey, 131 | auth: userAuth 132 | } 133 | }) 134 | } 135 | }) 136 | .then(() => { 137 | return BulkPublish.send(topic, ['-inf','+inf'], {"Message": "hello", "MessageId": "test-message"}, 0); 138 | }) 139 | .then((sendResults) => { 140 | 141 | sendResults.failure.length.should.equal(1); 142 | sendResults.success.length.should.equal(4); 143 | return redisClient.lrange(namespaces.failedSendList(topic, "test-message"), 0, -1) 144 | }) 145 | .then((redisResponse) => { 146 | redisResponse.length.should.equal(1); 147 | JSON.parse(redisResponse[0]).id.should.equal('1000'); 148 | }) 149 | }) 150 | }) -------------------------------------------------------------------------------- /lib/sns.js: -------------------------------------------------------------------------------- 1 | import AWS from 'aws-sdk'; 2 | import {createArn, AWS_ACCOUNT_ID, createSNSTopic} from './arn'; 3 | import {withRedis} from './redis/client'; 4 | import namespaces from './redis/namespaces'; 5 | 6 | const callbackPromise = (func) => { 7 | return new Promise((fulfill, reject) => { 8 | let cb = (err, response) => { 9 | if (err) { 10 | return reject(err); 11 | } 12 | fulfill(response); 13 | } 14 | func(cb); 15 | }) 16 | 17 | } 18 | 19 | const sns = new AWS.SNS({ 20 | region: process.env.SERVERLESS_REGION 21 | }); 22 | 23 | const lambda = new AWS.Lambda({ 24 | region: process.env.SERVERLESS_REGION 25 | }); 26 | 27 | // The path to our other Lambda function that listens to publish events. We have 28 | // to manually construct this from the environment variables we have available. 29 | 30 | const LISTENER_FUNC_PATH = ['function', process.env.SERVERLESS_PROJECT + '-listener', process.env.SERVERLESS_STAGE].join(':'); 31 | 32 | export default { 33 | addTopicSubscription(topic) { 34 | 35 | let functionArn = createArn('lambda', process.env.SERVERLESS_REGION, AWS_ACCOUNT_ID, LISTENER_FUNC_PATH); 36 | let topicArn = createSNSTopic(topic); 37 | return callbackPromise((cb) => { 38 | sns.subscribe({ 39 | TopicArn: topicArn, 40 | Protocol: 'lambda', 41 | Endpoint: functionArn 42 | }, cb); 43 | }) 44 | .catch((err) => { 45 | console.log('TOPIC DOES NOT EXIST', topicArn) 46 | if (err.name === 'NotFound') { 47 | err.statusCode = 404; 48 | 49 | } 50 | throw err; 51 | }) 52 | .then(({SubscriptionArn}) => { 53 | return withRedis((redisClient) => redisClient.set(namespaces.snsSubscription(topic), SubscriptionArn)); 54 | }) 55 | .then(() => { 56 | return new Promise((fulfill, reject) => { 57 | lambda.getPolicy({ 58 | FunctionName: functionArn, 59 | Qualifier: process.env.SERVERLESS_STAGE 60 | }, (err, resp) => { 61 | console.log('wtf err', err) 62 | 63 | 64 | if (err) { 65 | if (err.code === 'ResourceNotFoundException') { 66 | return fulfill(false); 67 | } else { 68 | return reject(err); 69 | } 70 | 71 | } 72 | 73 | let parsedPolicy = JSON.parse(resp.Policy); 74 | 75 | let statementsForSNS = parsedPolicy.Statement.filter((statement) => statement.Principal.Service === "sns.amazonaws.com"); 76 | 77 | 78 | statementsForSNS.forEach((st) => { 79 | try { 80 | if (!st.Condition) { 81 | // is wildcard 82 | return fulfill(true); 83 | } 84 | if (st.Condition.ArnLike["AWS:SourceArn"] === topicArn) { 85 | return fulfill(true); 86 | } 87 | } catch (err) { 88 | console.error(err); 89 | } 90 | }) 91 | 92 | return fulfill(false); 93 | }) 94 | 95 | }) 96 | .then((statementExists) => { 97 | if (statementExists) { 98 | return true 99 | } 100 | // We also need to give the SNS topic permission to execute this lambda. 101 | return new Promise((fulfill, reject) => { 102 | lambda.addPermission({ 103 | FunctionName: functionArn, 104 | StatementId: Date.now().toString(), 105 | Action: 'lambda:InvokeFunction', 106 | Principal: 'sns.amazonaws.com', 107 | Qualifier: process.env.SERVERLESS_STAGE 108 | }, (err, resp) => { 109 | if (err) { 110 | return reject(err); 111 | } 112 | fulfill(resp); 113 | }) 114 | }) 115 | }) 116 | 117 | 118 | }) 119 | .then((resp) => { 120 | console.info("SNS subscription created to topic: " + topic); 121 | return true; 122 | }) 123 | }, 124 | removeTopicSubscription(topic) { 125 | return withRedis((redisClient) => { 126 | return redisClient.get(namespaces.snsSubscription(topic)) 127 | .then((arn) => { 128 | if (arn === null) { 129 | throw new Error("Subscription ARN does not exist"); 130 | } 131 | return callbackPromise((cb) => sns.unsubscribe({ 132 | SubscriptionArn: arn 133 | },cb)) 134 | }) 135 | .then(() => { 136 | return redisClient.del(namespaces.snsSubscription(topic)) 137 | }) 138 | .then((response) => { 139 | console.info("SNS subscription removed for topic: " + topic); 140 | }) 141 | }) 142 | 143 | }, 144 | publish(topic, message) { 145 | return callbackPromise((cb) => sns.publish({ 146 | TopicArn: createSNSTopic(topic), 147 | Message: JSON.stringify(message), 148 | //MessageStructure: typeof message === 'string' ? null : 'json' 149 | }, cb)); 150 | } 151 | } -------------------------------------------------------------------------------- /lib/bulk-publish.js: -------------------------------------------------------------------------------- 1 | import {withRedis} from './redis/client'; 2 | import namespaces from './redis/namespaces'; 3 | import webPush from './web-push'; 4 | import PromiseTools from 'promise-tools'; 5 | import sendToSlack from './util/send-to-slack'; 6 | import removeSubscription from './subscription/remove'; 7 | 8 | export default { 9 | send: function(topic, [rangeStart, rangeEnd], message, broadcastIndex) { 10 | let namespacedTopic = namespaces.topic(topic); 11 | if (!message.MessageId) { 12 | throw new Error("Message needs ID") 13 | } 14 | 15 | let {ttl, payload} = JSON.parse(message.Message); 16 | 17 | if (!ttl || !payload) { 18 | console.error("Messages must now be sent as an object with ttl and payload keys."); 19 | return false; 20 | } 21 | 22 | return withRedis((redisClient) => { 23 | return redisClient.zrangebyscore(namespacedTopic, rangeStart, rangeEnd, 'WITHSCORES') 24 | .then((results) => { 25 | let resultObjs = []; 26 | 27 | for (let i = 0; i < results.length; i = i + 2) { 28 | resultObjs.push({ 29 | clientEndpoint: JSON.parse(results[i]), 30 | score: results[i+1] 31 | }) 32 | } 33 | 34 | let sendResults = { 35 | success: [], 36 | failure: [] 37 | }; 38 | 39 | return PromiseTools.map(resultObjs, ({clientEndpoint, score}) => { 40 | return webPush.sendNotification(clientEndpoint.endpoint, { 41 | payload: JSON.stringify(payload), 42 | TTL: ttl, 43 | userPublicKey: clientEndpoint.keys.p256dh, 44 | userAuth: clientEndpoint.keys.auth 45 | }) 46 | .then(() => { 47 | sendResults.success.push({id: score}); 48 | return true; 49 | }) 50 | .catch((err) => { 51 | console.error(err); 52 | 53 | sendResults.failure.push({ 54 | id: score, 55 | subscription: { 56 | endpoint: clientEndpoint.endpoint, 57 | keys: clientEndpoint.keys 58 | }, 59 | error: err 60 | }) 61 | return false; 62 | }) 63 | }, 50) 64 | .then(() => { 65 | return sendResults; 66 | }) 67 | }) 68 | .then(({success, failure}) => { 69 | let total = success.length + failure.length; 70 | console.info(`Successfully sent ${success.length} of ${total} requests for message ${message.MessageId}.`); 71 | 72 | let promiseArray = [ 73 | redisClient.incrby(namespaces.successfulCount(topic, message.MessageId), success.length) 74 | ]; 75 | 76 | if (failure.length !== 0) { 77 | let failedToSave = failure.map((f) => JSON.stringify({ 78 | id: f.id, 79 | error: f.error.toString() 80 | })); 81 | 82 | promiseArray.push(redisClient.lpush([namespaces.failedSendList(topic,message.MessageId)].concat(failedToSave))); 83 | } 84 | 85 | 86 | 87 | return Promise.all(promiseArray) 88 | .then(() => { 89 | return {success, failure}; 90 | }) 91 | }) 92 | .then((responses) => { 93 | return redisClient.hgetall(namespaces.performanceLogging(message.MessageId)) 94 | .then((performanceData) => { 95 | 96 | // Because we've spread our publishes across lambdas, we need to store 97 | // performance data back somewhere central, then examine it when we know we're 98 | // the last labmda to run. 99 | 100 | let latestTime = Date.now(); 101 | let startTime = null; 102 | 103 | for (let key in performanceData) { 104 | 105 | if (key === 'start_time') { 106 | startTime = parseInt(performanceData[key],10); 107 | } 108 | 109 | if (key === `end_time_${broadcastIndex}`) { 110 | // is the entry for this lambda, so ignore for now. 111 | continue; 112 | } 113 | 114 | if (performanceData[key] === '-1') { 115 | 116 | // If one hasn't been filled out yet then it means this 117 | // is not the last one to run, so we'll just log our own 118 | // data and continue; 119 | 120 | return redisClient.hset(namespaces.performanceLogging(message.MessageId), `end_time_${broadcastIndex}`, Date.now()) 121 | .then(() => { 122 | // try { 123 | // sendToSlack({ 124 | // text: "Send one batch in " + (Date.now() - startTime) + "ms" 125 | // }); 126 | // } catch (err) { 127 | // console.error("slack err", err) 128 | // } 129 | 130 | }) 131 | } 132 | 133 | 134 | latestTime = Math.max(latestTime, parseInt(performanceData[key])); 135 | } 136 | 137 | for (let key in performanceData) { 138 | let keySplit = key.split("end_time_"); 139 | if (keySplit.length === 1) { 140 | continue; 141 | } 142 | } 143 | 144 | 145 | // If we've gotten to here it means that all the others are filled out 146 | let timeTaken = latestTime - startTime; 147 | console.info(`BROADCAST: Sent all requests in ${timeTaken}ms`, latestTime, startTime); 148 | return redisClient.del(namespaces.performanceLogging(topic)) 149 | .then(() => { 150 | return redisClient.zcount(namespacedTopic, '-inf', '+inf') 151 | .then((numberOfSubscribers) => { 152 | 153 | 154 | return Promise.all([ 155 | redisClient.lrange(namespaces.failedSendList(topic, message.MessageId), 0, -1), 156 | redisClient.get(namespaces.successfulCount(topic, message.MessageId)) 157 | ]) 158 | .then(([failures, successCount]) => { 159 | 160 | let slackMsg = { 161 | text: `Sent ${successCount} messages to ${numberOfSubscribers} subscribers \`${topic}\` in ${timeTaken}ms` 162 | }; 163 | 164 | if (failures.length === 0) { 165 | return slackMsg; 166 | } 167 | 168 | let errors = failures.map((f) => JSON.parse(f).error); 169 | slackMsg.attachments = [ 170 | { 171 | fallback: "errors", 172 | text: errors.join('\n') 173 | } 174 | 175 | ] 176 | 177 | 178 | slackMsg.text += `\n\n${failures.length} failed messages. Errors encountered:`; 179 | return slackMsg; 180 | }) 181 | }) 182 | .then((completeSlackMsg) => { 183 | return sendToSlack(completeSlackMsg); 184 | }) 185 | 186 | 187 | 188 | }) 189 | }) 190 | .then(() => { 191 | return responses; 192 | }) 193 | }) 194 | }) 195 | } 196 | } --------------------------------------------------------------------------------