├── .gitignore ├── bin ├── handler.js └── index.js ├── server ├── api │ ├── poll │ │ ├── index.js │ │ └── router.js │ ├── send │ │ ├── index.js │ │ └── router.js │ └── index.js ├── queue │ └── index.js ├── parsers │ ├── json.js │ ├── index.js │ ├── html.js │ └── jsonapi.js ├── resources │ ├── index.js │ ├── text.js │ ├── json.js │ └── url.js ├── engines │ ├── jade.js │ ├── combyne.js │ ├── mustache.js │ ├── handlebars.js │ └── index.js ├── configure.js ├── email │ ├── index.js │ ├── process-queue.js │ ├── bounce-list.js │ ├── normalize-payload.js │ ├── send.js │ └── handle-undeliverables.js ├── handler │ ├── actions │ │ ├── get-payload.js │ │ ├── fetch-resource.js │ │ ├── fetch-resources.js │ │ ├── process-data.js │ │ └── queue-email.js │ └── index.js └── index.js ├── .travis.yml ├── test ├── fixtures │ ├── secrets.json │ ├── html-and-url-poll-post.js │ └── typical-poll-post.js ├── mocks │ ├── request.js │ ├── aws-sdk.js │ ├── kue.js │ └── crontab.js ├── tests │ ├── email.js │ ├── handler.js │ └── api.js └── setup.js ├── package.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | -------------------------------------------------------------------------------- /bin/handler.js: -------------------------------------------------------------------------------- 1 | require('../server/handler')(); 2 | -------------------------------------------------------------------------------- /server/api/poll/index.js: -------------------------------------------------------------------------------- 1 | exports.router = require('./router'); 2 | -------------------------------------------------------------------------------- /server/api/send/index.js: -------------------------------------------------------------------------------- 1 | exports.router = require('./router'); 2 | -------------------------------------------------------------------------------- /server/queue/index.js: -------------------------------------------------------------------------------- 1 | module.exports = require('kue').createQueue(); 2 | -------------------------------------------------------------------------------- /server/parsers/json.js: -------------------------------------------------------------------------------- 1 | module.exports = function(value) { 2 | return JSON.parse(value); 3 | }; 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | node_js: 4 | - 0.12 5 | 6 | branches: 7 | only: 8 | - master 9 | -------------------------------------------------------------------------------- /server/resources/index.js: -------------------------------------------------------------------------------- 1 | exports.json = require('./json'); 2 | exports.url = require('./url'); 3 | exports.text = require('./text'); 4 | -------------------------------------------------------------------------------- /server/parsers/index.js: -------------------------------------------------------------------------------- 1 | exports.json = require('./json'); 2 | exports.jsonapi = require('./jsonapi'); 3 | exports.html = require('./html'); 4 | -------------------------------------------------------------------------------- /test/fixtures/secrets.json: -------------------------------------------------------------------------------- 1 | { 2 | "accessKeyId": "test_key", 3 | "secretAccessKey": "test_secret", 4 | "region": "us-east-1" 5 | } 6 | -------------------------------------------------------------------------------- /server/parsers/html.js: -------------------------------------------------------------------------------- 1 | const cheerio = require('cheerio'); 2 | 3 | module.exports = function(value) { 4 | return cheerio.load(value); 5 | }; 6 | -------------------------------------------------------------------------------- /server/engines/jade.js: -------------------------------------------------------------------------------- 1 | var jade = require('jade'); 2 | 3 | exports.render = function(template, data) { 4 | return jade.render(template, data); 5 | }; 6 | -------------------------------------------------------------------------------- /server/resources/text.js: -------------------------------------------------------------------------------- 1 | const Bluebird = require('bluebird'); 2 | 3 | module.exports = function(value) { 4 | return Bluebird.resolve(String(value)); 5 | }; 6 | -------------------------------------------------------------------------------- /server/engines/combyne.js: -------------------------------------------------------------------------------- 1 | const combyne = require('combyne'); 2 | 3 | exports.render = function(template, data) { 4 | return combyne(template).render(data); 5 | }; 6 | -------------------------------------------------------------------------------- /server/engines/mustache.js: -------------------------------------------------------------------------------- 1 | const mustache = require('mustache'); 2 | 3 | exports.render = function(template, data) { 4 | return mustache.render(template, data); 5 | }; 6 | -------------------------------------------------------------------------------- /server/engines/handlebars.js: -------------------------------------------------------------------------------- 1 | const handlebars = require('handlebars'); 2 | 3 | exports.render = function(template, data) { 4 | return Handlebars.compile(template)(data); 5 | }; 6 | -------------------------------------------------------------------------------- /server/engines/index.js: -------------------------------------------------------------------------------- 1 | exports.handlebars = require('./handlebars'); 2 | exports.jade = require('./jade'); 3 | exports.mustache = require('./mustache'); 4 | exports.combyne = require('./combyne'); 5 | -------------------------------------------------------------------------------- /server/configure.js: -------------------------------------------------------------------------------- 1 | module.exports = function(app) { 2 | var env = process.env; 3 | 4 | if (env.NODE_ENV === 'production') { 5 | env.PORT = './socket'; 6 | } 7 | 8 | return app; 9 | }; 10 | -------------------------------------------------------------------------------- /test/mocks/request.js: -------------------------------------------------------------------------------- 1 | module.exports = function(url, callback) { 2 | switch (url) { 3 | case 'http://tbranyen.com/': { 4 | setTimeout(function() { 5 | callback(null, null, 'Tim'); 6 | }, 0); 7 | } 8 | } 9 | }; 10 | -------------------------------------------------------------------------------- /server/email/index.js: -------------------------------------------------------------------------------- 1 | exports.send = require('./send'); 2 | exports.normalizePayload = require('./normalize-payload'); 3 | exports.processQueue = require('./process-queue'); 4 | exports.handleUndeliverables = require('./handle-undeliverables'); 5 | exports.bounceList = require('./bounce-list'); 6 | -------------------------------------------------------------------------------- /server/resources/json.js: -------------------------------------------------------------------------------- 1 | const Bluebird = require('bluebird'); 2 | 3 | module.exports = function(value) { 4 | return new Bluebird(function(resolve, reject) { 5 | try { 6 | resolve(JSON.parse(value)); 7 | } 8 | catch(ex) { 9 | reject(ex); 10 | } 11 | }); 12 | }; 13 | -------------------------------------------------------------------------------- /test/tests/email.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const email = require('../../server/email'); 3 | 4 | describe('Email', function() { 5 | it('exports the public interface', function() { 6 | assert.ok(email.send); 7 | assert.ok(email.processQueue); 8 | assert.ok(email.normalizePayload); 9 | }); 10 | }); 11 | -------------------------------------------------------------------------------- /server/api/index.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const poll = require('./poll'); 3 | const send = require('./send'); 4 | 5 | var api = express.Router(); 6 | 7 | // Attach API components. 8 | api.use('/poll', poll.router); 9 | api.use('/send', send.router); 10 | //api.use('/jobs', jobs.router); // TODO 11 | 12 | module.exports = api; 13 | -------------------------------------------------------------------------------- /server/resources/url.js: -------------------------------------------------------------------------------- 1 | const Bluebird = require('bluebird'); 2 | const request = require('request'); 3 | 4 | module.exports = function(value) { 5 | return new Bluebird(function(resolve, reject) { 6 | request(value, function(err, resp, body) { 7 | if (err) { return reject(err); } 8 | else { resolve(body); } 9 | }); 10 | }); 11 | }; 12 | -------------------------------------------------------------------------------- /bin/index.js: -------------------------------------------------------------------------------- 1 | const app = require('../server'); 2 | 3 | var port = process.env.PORT || 8000; 4 | var host = process.env.HOST || '0.0.0.0'; 5 | 6 | if (!process.env.AWS_SES_SECRETS) { 7 | console.error('>> Missing AWS_SES_SECRETS environment variable\n'); 8 | } 9 | 10 | app.listen(port, host, function() { 11 | console.log('http://localhost:' + this.address().port); 12 | }); 13 | -------------------------------------------------------------------------------- /server/email/process-queue.js: -------------------------------------------------------------------------------- 1 | const queue = require('../queue'); 2 | const send = require('./send'); 3 | 4 | module.exports = function() { 5 | // Set a maximum of 20 concurrent jobs to process at a time, since we are 6 | // only firing off to Amazon, this is unlikely to cause a problem. 7 | queue.process('email', 20, function(job, done) { 8 | send(job.data, done); 9 | }); 10 | }; 11 | -------------------------------------------------------------------------------- /test/setup.js: -------------------------------------------------------------------------------- 1 | var path = require('path'); 2 | 3 | // Configure test environment. 4 | process.env.AWS_SES_SECRETS = path.join(__dirname, 'fixtures/secrets.json'); 5 | 6 | // Install mocks. 7 | [ 8 | 'request', 9 | 'aws-sdk', 10 | 'kue', 11 | 'crontab' 12 | ].forEach(function(name) { 13 | require.cache[require.resolve(name)] = { 14 | exports: require('./mocks/' + name) 15 | }; 16 | }); 17 | -------------------------------------------------------------------------------- /server/email/bounce-list.js: -------------------------------------------------------------------------------- 1 | exports.list = []; 2 | 3 | exports.load = function(list) { 4 | exports.list = list; 5 | }; 6 | 7 | exports.add = function(email) { 8 | exports.list.push(email); 9 | 10 | // TODO Trigger some kind of event. 11 | }; 12 | 13 | exports.lookup = function(email) { 14 | var found = exports.list.indexOf(email) > -1; 15 | 16 | // Return not found to work better with `Array#filter`. 17 | return !found; 18 | }; 19 | -------------------------------------------------------------------------------- /server/handler/actions/get-payload.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Attaches the JSON parsed payload to the state object. 3 | * 4 | * @param state 5 | * @return state 6 | */ 7 | module.exports = function getPayload(state) { 8 | var payload = state.argv[0]; 9 | 10 | state.payload = JSON.parse(payload); 11 | 12 | // Inject the `SECRETS` location into this environment. 13 | process.env.AWS_SES_SECRETS = state.payload.env.AWS_SES_SECRETS; 14 | 15 | return state; 16 | }; 17 | -------------------------------------------------------------------------------- /server/index.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const configure = require('./configure'); 3 | const api = require('./api'); 4 | const email = require('./email'); 5 | 6 | var app = configure(express()).use(api); 7 | 8 | // Expose bounce list. 9 | app.bounceList = email.bounceList; 10 | 11 | // Process sending failures. 12 | email.handleUndeliverables(); 13 | 14 | // This processes the queue as necessary. 15 | email.processQueue(); 16 | 17 | module.exports = app; 18 | -------------------------------------------------------------------------------- /server/handler/actions/fetch-resource.js: -------------------------------------------------------------------------------- 1 | const resources = require('../../resources'); 2 | 3 | /** 4 | * Using the type as a guide, fetch the specified resource. 5 | * 6 | * @param resource 7 | * @return {String} contents 8 | */ 9 | module.exports = function fetchResource(resource) { 10 | var resourceType = resources[resource.type]; 11 | 12 | if (!resourceType) { 13 | throw new Error('Invalid resource type: ' + resource.type); 14 | } 15 | 16 | return resources[resource.type](resource.value); 17 | }; 18 | -------------------------------------------------------------------------------- /test/fixtures/html-and-url-poll-post.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment'); 2 | 3 | module.exports = { 4 | id: 'test-uuid', 5 | to: ['tim@tabdeveloper.com'], 6 | from: 'tim@bocoup.com', 7 | template: { 8 | type: 'text', 9 | engine: 'combyne', 10 | value: 'Hello {{ name }}' 11 | }, 12 | data: { 13 | type: 'url', 14 | parser: 'html', 15 | value: 'http://tbranyen.com/' 16 | }, 17 | schedule: Number(moment().add(1, 'seconds')), 18 | handler: function($) { 19 | return { 20 | name: $('title').text() 21 | }; 22 | }.toString() 23 | }; 24 | -------------------------------------------------------------------------------- /test/tests/handler.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const exec = require('child_process').exec; 3 | 4 | describe('Handler', function() { 5 | it('can be initialized from the command line', function(done) { 6 | exec('node bin/handler', function(err, output) { 7 | done(); 8 | }); 9 | }); 10 | 11 | it('will error if incorrect arguments are passed', function(done) { 12 | exec('node bin/handler', function(err, output) { 13 | assert.equal(err.message, 'Command failed: /bin/sh -c node bin/handler\nInsufficient arguments to handler see --help\n'); 14 | done(); 15 | }); 16 | }); 17 | }); 18 | -------------------------------------------------------------------------------- /server/email/normalize-payload.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Normalizes a simplified payload into an object that is suitable for Amazon 3 | * SES. 4 | * 5 | * @param {Object} payload 6 | * @return {Object} 7 | */ 8 | module.exports = function normalizePayload(payload) { 9 | return { 10 | Source: Array.isArray(payload.from) ? payload.from[0] : payload.from, 11 | Destination: { 12 | ToAddresses: payload.to, 13 | CcAddresses: payload.cc || [], 14 | BccAddresses: payload.bcc || [] 15 | }, 16 | Message: { 17 | Body: { 18 | Html: { 19 | Data: String(payload.body) 20 | } 21 | }, 22 | Subject: { 23 | Data: String(payload.subject) 24 | } 25 | } 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /server/handler/actions/fetch-resources.js: -------------------------------------------------------------------------------- 1 | const Bluebird = require('bluebird'); 2 | const fetchResource = require('./fetch-resource'); 3 | 4 | /** 5 | * Fetch and save the template and data resources to the state object. 6 | * 7 | * @param state 8 | * @return state 9 | */ 10 | module.exports = function fetchResources(state) { 11 | // If the `body` property is already attached, skip this step. 12 | if (state.payload.body) { 13 | return Promise.resolve(state); 14 | } 15 | 16 | var getTemplate = fetchResource(state.payload.template); 17 | var getData = fetchResource(state.payload.data); 18 | 19 | return Bluebird.all([getTemplate, getData]).then(function(resources) { 20 | state.template = resources[0]; 21 | state.data = resources[1]; 22 | 23 | return state; 24 | }); 25 | }; 26 | -------------------------------------------------------------------------------- /test/mocks/aws-sdk.js: -------------------------------------------------------------------------------- 1 | var env = process.env; 2 | 3 | exports.env = { 4 | AWS_ACCESS_KEY_ID: env.AWS_ACCESS_KEY_ID, 5 | AWS_SECRET_ACCESS_KEY: env.AWS_SECRET_ACCESS_KEY 6 | }; 7 | 8 | exports.config = {}; 9 | exports.config.loadFromPath = function(path) { 10 | require(path); 11 | }; 12 | 13 | exports.SES = function() {}; 14 | exports.SES.prototype = { 15 | sendEmail: function(config, cb) { 16 | this.config = config; 17 | this.cb = cb; 18 | 19 | this.cb(null, { 20 | status: "success", 21 | 22 | data: { 23 | ResponseMetadata: { 24 | RequestId: "454fcee7-f293-11e4-8ce7-3f483707c2e3" 25 | }, 26 | 27 | MessageId: "0000014d20639464-dd5a5eb4-5205-48fe-96c3-efc6fd5e883a-000000" 28 | } 29 | }); 30 | } 31 | }; 32 | 33 | exports.SQS = function() {}; 34 | -------------------------------------------------------------------------------- /server/email/send.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const AWS = require('aws-sdk'); 3 | const bounceList = require('./bounce-list'); 4 | 5 | module.exports = function send(payload, done) { 6 | AWS.config.loadFromPath(process.env.AWS_SES_SECRETS); 7 | 8 | var dest = payload.Destination; 9 | 10 | // Look for blocked emails (bounces/complaints). 11 | dest.ToAddresses = dest.ToAddresses.filter(bounceList.lookup); 12 | dest.CcAddresses = dest.CcAddresses.filter(bounceList.lookup); 13 | dest.BccAddresses = dest.BccAddresses.filter(bounceList.lookup); 14 | 15 | // Must have atleast one recipient. 16 | if (dest.ToAddresses.length) { 17 | new AWS.SES({ apiVersion: '2010-12-01' }).sendEmail(payload, done); 18 | } 19 | else { 20 | done({ message: 'No addresses to send to, are they all blocked?' }); 21 | } 22 | }; 23 | -------------------------------------------------------------------------------- /server/api/send/router.js: -------------------------------------------------------------------------------- 1 | const express = require('express'); 2 | const bodyParser = require('body-parser'); 3 | const email = require('../../email'); 4 | 5 | var router = express.Router(); 6 | 7 | router.post('/', [bodyParser.json()], function(req, res) { 8 | var validPayload = true; 9 | var payload = req.body; 10 | 11 | // Ensure the payload contains valid required arguments. 12 | if (!payload.to || !payload.from || !payload.subject || !payload.body) { 13 | validPayload = false; 14 | } 15 | 16 | if (!validPayload) { 17 | return res.status(400).json({ 18 | message: 'Invalid POST body' 19 | }); 20 | } 21 | 22 | email.send(email.normalizePayload(payload), function(err, resp) { 23 | if (err) { 24 | return res.status(500).json({ 25 | error: err 26 | }); 27 | } 28 | 29 | res.json({ data: resp }); 30 | }); 31 | }); 32 | 33 | module.exports = router; 34 | -------------------------------------------------------------------------------- /server/handler/actions/process-data.js: -------------------------------------------------------------------------------- 1 | const _ = require('lodash'); 2 | const Bluebird = require('bluebird'); 3 | const parsers = require('../../parsers'); 4 | 5 | module.exports = function processData(state) { 6 | return new Bluebird(function(resolve, reject) { 7 | var handler = state.payload.handler; 8 | 9 | if (handler) { 10 | handler = new Buffer(handler, 'base64').toString('utf-8'); 11 | } 12 | else { 13 | handler = 'function(data) { return data; }'; 14 | } 15 | 16 | var fn = new Function('return ' + handler)(); 17 | var parsed = parsers[state.payload.data.parser](state.data); 18 | 19 | // Get the extracted response. 20 | var extracted = fn(parsed); 21 | 22 | // If the handler returns an Array we need to break out the 23 | if (Array.isArray(extracted)) { 24 | state.payload._extracted = extracted; 25 | } 26 | else { 27 | _.extend(state.payload, extracted); 28 | } 29 | 30 | resolve(state); 31 | }); 32 | }; 33 | -------------------------------------------------------------------------------- /test/mocks/kue.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | queue: { 3 | _created: { 4 | save: function() {}, 5 | 6 | priority: function(priority) { 7 | this.priority = priority; 8 | return this; 9 | }, 10 | 11 | ttl: function(ttl) { 12 | this.ttl = ttl; 13 | return this; 14 | }, 15 | 16 | attempts: function(attempts) { 17 | this.attempts = attempts; 18 | return this; 19 | }, 20 | 21 | backoff: function(backoff) { 22 | this.backoff = backoff; 23 | return this; 24 | } 25 | }, 26 | 27 | process: function(name, concurrency, fn) { 28 | this.name = name; 29 | this.concurrency = concurrency; 30 | this.fn = fn; 31 | }, 32 | 33 | create: function(name, payload) { 34 | this.name = name; 35 | this.payload = payload; 36 | 37 | return this._created; 38 | } 39 | }, 40 | 41 | createQueue: function() { 42 | return this.queue; 43 | } 44 | }; 45 | -------------------------------------------------------------------------------- /test/fixtures/typical-poll-post.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | template: { 3 | engine: 'handlebars', 4 | type: 'string', 5 | value: 'On {{requestDay}}, {{requestor}} submitted a time off ' + 6 | 'request for {{firstDay}} to {{lastDay}}. Please respond to ' + 7 | 'this request!' 8 | }, 9 | 10 | data: { 11 | type: 'url', 12 | value: '/leave-requests/pending-review', 13 | parser: 'jsonapi' 14 | }, 15 | 16 | schedule: '@daily', 17 | 18 | handler: function(resp) { 19 | return resp.map(function(entry) { 20 | return { 21 | requestor: entry.employee.first + ' ' + entry.employee.last, 22 | requestDay: entry.request_day, 23 | firstDay: entry.first_day, 24 | lastDay: entry.last_day, 25 | 26 | to: [entry.employee.supporter.email], 27 | from: entry.employee.email, 28 | cc: 'hr@bocoup.com', 29 | subject: 'Upcoming Time off Request (' + entry.first_day + 30 | ' - ' + entry.last_day + ')' 31 | }; 32 | }); 33 | }.toString() 34 | }; 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "pigeonpost", 3 | "version": "0.0.0", 4 | "description": "Amazon SES E-Mail scheduling and delivery API", 5 | "main": "server", 6 | "bin": "bin", 7 | "scripts": { 8 | "test": "mocha test/setup test/tests", 9 | "start": "node bin" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "https://github.com/tbranyen/pigeonpost.git" 14 | }, 15 | "author": "Tim Branyen via Bocoup", 16 | "license": "MIT", 17 | "bugs": { 18 | "url": "https://github.com/tbranyen/pigeonpost/issues" 19 | }, 20 | "homepage": "https://github.com/tbranyen/pigeonpost", 21 | "dependencies": { 22 | "aws-sdk": "^2.1.34", 23 | "bluebird": "^2.9.30", 24 | "body-parser": "^1.13.1", 25 | "cheerio": "^0.19.0", 26 | "combyne": "^0.8.0", 27 | "crontab": "^1.0.2", 28 | "express": "^4.12.4", 29 | "handlebars": "^3.0.3", 30 | "jade": "^1.11.0", 31 | "kue": "^0.9.3", 32 | "lodash": "^3.9.3", 33 | "moment": "^2.10.3", 34 | "mustache": "^2.1.2", 35 | "node-uuid": "^1.4.3" 36 | }, 37 | "devDependencies": { 38 | "mocha": "^2.2.5", 39 | "moment": "^2.10.3", 40 | "request": "^2.58.0", 41 | "supertest": "^1.0.1" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /server/handler/actions/queue-email.js: -------------------------------------------------------------------------------- 1 | const Bluebird = require('bluebird'); 2 | const _ = require('lodash'); 3 | const queue = require('../../queue'); 4 | const email = require('../../email'); 5 | const engines = require('../../engines'); 6 | 7 | function createQueue(payload) { 8 | queue.create('email', payload) 9 | .priority('normal') 10 | .backoff(true) 11 | .ttl(1000) 12 | .attempts(3) 13 | .save(); 14 | } 15 | 16 | module.exports = function queueEmail(state) { 17 | var engine = engines[state.payload.template.engine]; 18 | var payloads = []; 19 | 20 | // Working with an Array of payloads. 21 | if (state.payload._extended) { 22 | payloads = state.payload._extended.map(function(payload, i) { 23 | var merged = _.extend({}, state.payload, payload); 24 | 25 | // Set the specific `to` (not all the recipients). 26 | merged.to = [merged.to[i]]; 27 | 28 | return merged; 29 | }); 30 | } 31 | // A single payload. 32 | else { 33 | payloads.push(state.payload); 34 | } 35 | 36 | // Render the template and attach the payload body. 37 | payloads.forEach(function(payload) { 38 | payload.body = engine.render(state.template, payload); 39 | }); 40 | 41 | // Normalize and queue all payloads. 42 | payloads.map(email.normalizePayload).forEach(createQueue); 43 | 44 | return Bluebird.resolve(state); 45 | }; 46 | -------------------------------------------------------------------------------- /server/handler/index.js: -------------------------------------------------------------------------------- 1 | const Bluebird = require('bluebird'); 2 | const getPayload = require('./actions/get-payload'); 3 | const fetchResources = require('./actions/fetch-resources'); 4 | const processData = require('./actions/process-data'); 5 | const queueEmail = require('./actions/queue-email'); 6 | const crontab = require('crontab'); 7 | 8 | /** 9 | * Exports a Promise chain that processes all the handler actions. 10 | * 11 | * @param argv 12 | * @return {Promise} chain 13 | */ 14 | module.exports = function(argv) { 15 | argv = (argv || process.argv).slice(2); 16 | 17 | if (!argv.length) { 18 | console.error('Insufficient arguments to handler see --help'); 19 | return process.exit(1); 20 | } 21 | 22 | // Pass state through the Promise chain. 23 | return Bluebird.resolve({ argv: argv }) 24 | .then(getPayload) 25 | .then(fetchResources) 26 | .then(processData) 27 | .then(queueEmail) 28 | .then(function(state) { 29 | if (state.payload.expires) { 30 | return new Bluebird(function(resolve, reject) { 31 | crontab.load(function(err, crontab) { 32 | crontab.remove({ comment: state.payload.id }); 33 | crontab.save(function() { 34 | resolve(state); 35 | }); 36 | }); 37 | }); 38 | } 39 | 40 | return state; 41 | }) 42 | .catch(function(ex) { 43 | require('fs').writeFileSync('/home/tim/Desktop/error_log', ex.stack); 44 | }); 45 | }; 46 | -------------------------------------------------------------------------------- /server/parsers/jsonapi.js: -------------------------------------------------------------------------------- 1 | // this is a naive json-api parser. the emailing service should support a 2 | // variety of "parsers" as dependencies. parsers can be used to do any pre 3 | // processing of data before they hit the context function. 4 | const _ = require('lodash'); 5 | 6 | function link (included, entry) { 7 | var links = entry.links; 8 | if (links) { 9 | Object.keys(links).forEach(function (relationName) { 10 | var link = links[relationName].linkage; 11 | entry[relationName] = included[link.type][link.id]; 12 | }); 13 | } 14 | return entry; 15 | } 16 | 17 | function parse(input) { 18 | input = JSON.parse(input); 19 | 20 | // key all included by type 21 | var included = _.reduce(input.included, function (result, entry) { 22 | var type = result[entry.type]; 23 | if (!type) { 24 | result[entry.type] = type = {}; 25 | } 26 | type[entry.id] = entry; 27 | return result; 28 | }, {}); 29 | 30 | // build a linker from the included records 31 | var linker = link.bind(null, included); 32 | 33 | // interlink included records 34 | var embedded = _.reduce(included, function (result, entries, type) { 35 | result[type] = _.reduce(entries, function (typeResult, entry) { 36 | typeResult[entry.id] = linker(entry); 37 | return typeResult; 38 | }, {}); 39 | return result; 40 | }, {}); 41 | 42 | // return primary data with references to included data 43 | return input.data.map(linker); 44 | }; 45 | 46 | module.exports = parse; 47 | -------------------------------------------------------------------------------- /test/mocks/crontab.js: -------------------------------------------------------------------------------- 1 | const moment = require('moment'); 2 | const exec = require('child_process').exec; 3 | const handler = require('../../server/handler'); 4 | 5 | module.exports = { 6 | result: { 7 | comment: function() { 8 | return this.comment; 9 | } 10 | }, 11 | 12 | load: function(callback) { 13 | var result = this.result; 14 | 15 | callback(null, { 16 | create: function(command, when, uuid) { 17 | result._command = command; 18 | result._when = when; 19 | result._comment = uuid; 20 | 21 | command = command.split(' ').slice(2).join(' ').slice(1, -1); 22 | command = new Array(2).concat(command); 23 | 24 | if (typeof when !== 'object' && when.indexOf('@') === -1) { 25 | setTimeout(function() { 26 | handler(command).then(function(state) { 27 | result._state = state; 28 | }).catch(function(ex) { 29 | console.log(ex.stack); 30 | }); 31 | }, -1 * moment().diff(new Date(when))); 32 | } 33 | else { 34 | setTimeout(function() { 35 | handler(command).then(function(state) { 36 | result._state = state; 37 | }); 38 | }, 10); 39 | } 40 | 41 | return result; 42 | }, 43 | 44 | find: function(obj) { 45 | return obj ? this.result : [this.result]; 46 | }, 47 | 48 | remove: function(jobs) {}, 49 | save: function(fn) { fn(); } 50 | }); 51 | } 52 | }; 53 | -------------------------------------------------------------------------------- /server/email/handle-undeliverables.js: -------------------------------------------------------------------------------- 1 | const AWS = require('aws-sdk'); 2 | const config = require(process.env.AWS_SES_SECRETS); 3 | const bounceList = require('./bounce-list'); 4 | 5 | AWS.config.loadFromPath(process.env.AWS_SES_SECRETS); 6 | 7 | var sqs = new AWS.SQS(); 8 | 9 | /** 10 | * Handles Amazon SES failures. 11 | * 12 | * @return 13 | */ 14 | function receiveMessage() { 15 | if (!config.sqsUrl) { 16 | console.warn('>> Missing SQS Resource URL, cannot monitor SES failures'); 17 | } 18 | 19 | // Receive 1 or many messages from Amazons's queue system. This is a polling 20 | // operation. 21 | sqs.receiveMessage({ 22 | QueueUrl: config.sqsUrl, 23 | MaxNumberOfMessages: 10, 24 | WaitTimeSeconds: 20 25 | }, function (err, data) { 26 | if (data && data.Messages && data.Messages.length) { 27 | data.Messages.forEach(function(message) { 28 | if (!message.Body) { return; } 29 | 30 | var MessageBody = JSON.parse(JSON.parse(message.Body).Message); 31 | 32 | // Is a complain'd email. 33 | if (MessageBody.notificationType === 'Complaint') { 34 | MessageBody.complaint.complainedRecipients.forEach(function(recipient) { 35 | bounceList.add(recipient.emailAddress); 36 | }); 37 | } 38 | // Is a Bounce'd email. 39 | else if (MessageBody.notificationType === 'Bounce') { 40 | MessageBody.bounce.bouncedRecipients.forEach(function(recipient) { 41 | bounceList.add(recipient.emailAddress); 42 | }); 43 | } 44 | 45 | // If we don't remove the message, it will remain in the queue. 46 | sqs.deleteMessage({ 47 | QueueUrl: config.sqsUrl, 48 | ReceiptHandle: message.ReceiptHandle 49 | }, function nop() {}); 50 | }); 51 | 52 | // If there were messages and no errors, retry immediately. 53 | receiveMessage(); 54 | } 55 | // If there were no messages and no failures, wait 5 seconds before retrying. 56 | else if (!err) { 57 | setTimeout(receiveMessage, 5000); 58 | } 59 | // If there was a failure, wait a minute before retrying. 60 | else { 61 | setTimeout(receiveMessage, 60000); 62 | } 63 | }); 64 | }; 65 | 66 | module.exports = receiveMessage; 67 | -------------------------------------------------------------------------------- /server/api/poll/router.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const path = require('path'); 3 | const express = require('express'); 4 | const bodyParser = require('body-parser'); 5 | const crontab = require('crontab'); 6 | const uuid = require('node-uuid'); 7 | const Bluebird = require('bluebird'); 8 | 9 | var handlerPath = 'node ' + path.join(__dirname, '../../../bin/handler'); 10 | var router = express.Router(); 11 | 12 | router.use(bodyParser.json()); 13 | 14 | function createJob(crontab, body) { 15 | var schedule = body.schedule; 16 | 17 | // Save the secrets location. 18 | body.env = { AWS_SES_SECRETS: process.env.AWS_SES_SECRETS }; 19 | 20 | if (typeof schedule === 'number') { 21 | schedule = new Date(schedule); 22 | 23 | // Setting a date directly will cause it to be a one time only event. 24 | body.expires = true; 25 | } 26 | 27 | // Convert body.handler to base64 to avoid quoting issues on CLI. 28 | if (body.handler) { 29 | body.handler = new Buffer(body.handler).toString('base64'); 30 | } 31 | 32 | return crontab.create([ 33 | handlerPath, 34 | '\'' + JSON.stringify(body) + '\'' 35 | ].join(' '), schedule, body.id || uuid()); 36 | } 37 | 38 | function formatJob(job) { 39 | return job ? { id: job.comment() } : null; 40 | } 41 | 42 | function sendResponse(res, job) { 43 | res.json({ data: formatJob(job) }); 44 | } 45 | 46 | function sendError(res, ex) { 47 | res.status(500).json({ error: ex }); 48 | } 49 | 50 | function validateBody(body) { 51 | assert(body, 'Body payload is defined'); 52 | assert(body.template, 'Template is missing'); 53 | assert(body.data, 'Data is missing'); 54 | assert(body.schedule, 'Schedule is missing'); 55 | } 56 | 57 | var getCronTab = new Bluebird(function(resolve, reject) { 58 | crontab.load(function(err, crontab) { 59 | if (err) { reject(err); } 60 | else { resolve(crontab); } 61 | }); 62 | }).catch(function(ex) { 63 | console.error(ex); 64 | process.exit(1); 65 | }); 66 | 67 | router.get('/', function(req, res) { 68 | getCronTab.then(function(crontab) { 69 | res.json({ data: crontab.find().map(formatJob) }); 70 | }).catch(sendError.bind(null, res)); 71 | }); 72 | 73 | router.get('/:id', function(req, res) { 74 | getCronTab.then(function(crontab) { 75 | var job = crontab.find({ comment: req.params.id }); 76 | sendResponse(res, job); 77 | }).catch(sendError.bind(null, res)); 78 | }); 79 | 80 | router.post('/', function(req, res) { 81 | getCronTab.then(function(crontab) { 82 | validateBody(req.body); 83 | 84 | var id = req.body.id || uuid(); 85 | var prior = crontab.find({ comment: id }); 86 | 87 | assert(!prior || !prior.length, 'This job already exists'); 88 | 89 | var job = createJob(crontab, req.body); 90 | 91 | crontab.save(function() { 92 | sendResponse(res, job); 93 | }); 94 | }).catch(function(ex) { 95 | console.error(ex.stack); 96 | sendError(res); 97 | }); 98 | }); 99 | 100 | router.put('/', function(req, res) { 101 | getCronTab.then(function(crontab) { 102 | var prior = crontab.find({ comment: req.body.id }); 103 | 104 | if (prior) { 105 | var job = crontab.find({ comment: req.body.id }); 106 | return sendResponse(res, job); 107 | } 108 | 109 | var job = createJob(crontab, req.body); 110 | 111 | crontab.save(function() { 112 | sendResponse(res, job); 113 | }); 114 | }).catch(sendError.bind(null, res)); 115 | }); 116 | 117 | router.delete('/:id', function(req, res) { 118 | getCronTab.then(function(crontab) { 119 | var prior = crontab.find({ comment: req.params.id }); 120 | 121 | assert(prior, 'Job doesn\'t exist'); 122 | 123 | var job = crontab.remove({ comment: req.params.id }); 124 | 125 | crontab.save(function() { 126 | sendResponse(res, { comment: function() { return req.params.id; } }); 127 | }); 128 | }).catch(sendError.bind(null, res)); 129 | }); 130 | 131 | module.exports = router; 132 | -------------------------------------------------------------------------------- /test/tests/api.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const path = require('path'); 3 | const express = require('express'); 4 | const supertest = require('supertest'); 5 | const _ = require('lodash'); 6 | const crontab = require('../mocks/crontab'); 7 | const typicalPollPost = require('../fixtures/typical-poll-post'); 8 | const htmlAndUrlPollPost = require('../fixtures/html-and-url-poll-post'); 9 | 10 | var handlerPath = 'node ' + path.join(__dirname, '../../bin/handler'); 11 | var server = express(); 12 | 13 | describe('API', function() { 14 | describe('Poll', function() { 15 | before(function() { 16 | this.router = require('../../server/api/poll/router'); 17 | server.use('/poll', this.router); 18 | this.request = supertest(server); 19 | this.typicalPollPost = _.extend({}, typicalPollPost); 20 | this.htmlAndUrlPollPost = _.extend({}, htmlAndUrlPollPost); 21 | }); 22 | 23 | it('can GET all poller jobs', function(done) { 24 | this.request 25 | .get('/poll') 26 | .expect('Content-Type', /json/) 27 | .expect(200) 28 | .end(function(err, res) { 29 | done(err); 30 | }); 31 | }); 32 | 33 | it('can GET a specific poller job', function(done) { 34 | this.request 35 | .get('/poll/test-uuid') 36 | .expect('Content-Type', /json/) 37 | .expect(200) 38 | .end(function(err, res) { 39 | done(err); 40 | }); 41 | }); 42 | 43 | it('can POST to register poller', function(done) { 44 | var test = this; 45 | 46 | this.request 47 | .post('/poll') 48 | .send(test.typicalPollPost) 49 | .expect('Content-Type', /json/) 50 | .expect(200) 51 | .end(function(err, res) { 52 | var handlerAsBase64 = new Buffer(test.typicalPollPost.handler) 53 | .toString('base64'); 54 | 55 | test.typicalPollPost.handler = handlerAsBase64; 56 | test.typicalPollPost.env = { 57 | AWS_SES_SECRETS: process.env.AWS_SES_SECRETS 58 | }; 59 | 60 | var serialized = '\'' + JSON.stringify(test.typicalPollPost) + '\''; 61 | assert.equal(crontab.result._command, handlerPath + ' ' + serialized); 62 | done(err); 63 | }); 64 | }); 65 | 66 | // TODO Put example. 67 | 68 | it('can schedule with html data and url fetched data', function(done) { 69 | var test = this; 70 | 71 | this.request 72 | .post('/poll') 73 | .send(htmlAndUrlPollPost) 74 | .expect('Content-Type', /json/) 75 | .expect(200) 76 | .end(function(err, res) { 77 | var handlerAsBase64 = new Buffer(test.htmlAndUrlPollPost.handler) 78 | .toString('base64'); 79 | 80 | test.htmlAndUrlPollPost.handler = handlerAsBase64; 81 | test.htmlAndUrlPollPost.env = { 82 | AWS_SES_SECRETS: process.env.AWS_SES_SECRETS 83 | }; 84 | test.htmlAndUrlPollPost.expires = true; 85 | 86 | var serialized = '\'' + JSON.stringify(test.htmlAndUrlPollPost) + '\''; 87 | 88 | assert.equal(crontab.result._command, handlerPath + ' ' + serialized); 89 | 90 | // Test out the handler. 91 | setTimeout(function() { 92 | assert.equal(crontab.result._state.payload.body, 'Hello Tim'); 93 | done(err); 94 | }, 150); 95 | }); 96 | }); 97 | 98 | it('can DELETE a specific poller job', function() { 99 | this.request 100 | .post('/poll') 101 | .send(htmlAndUrlPollPost); 102 | 103 | this.request 104 | .delete('/poll/test-uuid') 105 | .expect('Content-Type', /json/) 106 | .expect(200) 107 | .end(function(err, res) { 108 | done(err); 109 | }); 110 | }); 111 | }); 112 | 113 | describe('Sending', function() { 114 | before(function() { 115 | this.router = require('../../server/api/send/router'); 116 | server.use('/send', this.router); 117 | this.request = supertest(server); 118 | }); 119 | 120 | it('can POST an e-mail immediately', function(done) { 121 | this.request 122 | .post('/send') 123 | .send({ 124 | from: 'test@bocoup.com', 125 | to: ['someone@email.com'], 126 | subject: 'Testing', 127 | body: 'Testing HTML' 128 | }) 129 | .expect('Content-Type', /json/) 130 | .expect(200) 131 | .end(function(err, res) { 132 | done(err); 133 | }); 134 | }); 135 | 136 | it('will reject missing required fields', function(done) { 137 | this.request 138 | .post('/send') 139 | .send({ 140 | to: '' 141 | }) 142 | .expect('Content-Type', /json/) 143 | .expect(400, done); 144 | }); 145 | 146 | it('will reject invalid POST', function(done) { 147 | this.request 148 | .post('/send') 149 | .send('{ )') 150 | .expect('Content-Type', /json/) 151 | .expect(400, done); 152 | }); 153 | }); 154 | }); 155 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pigeon Post 2 | 3 | An Amazon SES E-Mail Scheduler & Delivery API. 4 | 5 | [![Build 6 | Status](https://travis-ci.org/tbranyen/pigeonpost.svg)](https://travis-ci.org/tbranyen/pigeonpost) 7 | 8 | 9 | ## Prerequisites 10 | 11 | You will need the following dependencies properly installed on your computer. 12 | 13 | * [Git](http://git-scm.com/) 14 | * [Node.js](http://nodejs.org/) (with NPM) 15 | * [Redis](http://redis.io/) 16 | * [Crontab](http://crontab.org/) (Installed by default in OS X & Linux) 17 | 18 | Note: Due to the dependency on Crontab, this application will [not run easily on 19 | Windows](http://stackoverflow.com/questions/132971/what-is-the-windows-version-of-cron). 20 | 21 | ## Installation 22 | 23 | ``` sh 24 | npm install pigeonpost 25 | ``` 26 | 27 | Set your Amazon SES credentials in a JSON file as [outlined in their official 28 | docs](http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-configuring.html#Credentials_from_Disk): 29 | 30 | 31 | ``` json 32 | { 33 | "accessKeyId": "test_key", 34 | "secretAccessKey": "test_secret", 35 | "region": "us-east-1" 36 | } 37 | ``` 38 | 39 | ### Bounce & Complaint handling 40 | 41 | Amazon requires handling of bounce and complaint notices to mitigate potential 42 | account suspension. PigeonPost requires that you hook up SES to emit bounce 43 | and complaint notices through Amazon SNS. Once you have created a topic and 44 | marked your verified email user to forward complaints to SNS, you will need to 45 | hook up SNS to Amazon SQS. Create a queue and have it subscribe to your SNS 46 | topic. On the SQS configuration page you will find a url that points to your 47 | queue. Add this to the configuration file discussed above like so: 48 | 49 | ``` json 50 | { 51 | "accessKeyId": "test_key", 52 | "secretAccessKey": "test_secret", 53 | "region": "us-east-1", 54 | "sqsUrl": "https://sqs.us-east-1.amazonaws.com/12345/SES_UNDELIVERABLE" 55 | } 56 | ``` 57 | 58 | ## Usage 59 | 60 | By design this application is meant to be standalone on a server and act as a 61 | server daemon. If you would like to directly integrate with your pre-existing 62 | server, this application also acts as express middleware: 63 | 64 | ``` javascript 65 | app.use('/email', require('pigeonpost')); 66 | ``` 67 | 68 | #### Environment variable configuration 69 | 70 | To inform this module where the secrets file lives, set an environment 71 | variable `AWS_SES_SECRETS` to the absolute fully qualified path on disk. It 72 | must be absolute as it may be run from various locations. 73 | 74 | For example your variable may look like: 75 | 76 | ``` sh 77 | export AWS_SES_SECRETS=/mnt/secrets/ses.json 78 | ``` 79 | 80 | ## API 81 | 82 | The API should follow [JSON-API](http://jsonapi.org/) for errors and data. You 83 | must set the content type request header to JSON. 84 | 85 | *Note: jQuery and other popular request libraries may add this automatically.* 86 | 87 | ``` http 88 | Content-Type: application/json 89 | ``` 90 | 91 | ### Documentation 92 | 93 | #### Sending 94 | 95 | This is the endpoint you'll hit when you want to immediately send an email. 96 | Useful for one off emails, such as new account registration or forgot password. 97 | 98 | Description | Method | Endpoint 99 | :------------------------------ | :------- | :-------- 100 | [Send an email](#send-an-email) | `POST` | `/send` 101 | 102 | 103 | ##### Send an email: 104 | 105 | Request: 106 | 107 | Param | Type | Required 108 | :------ | :----- | :-------- 109 | to | Array | yes 110 | cc | Array | no 111 | bcc | Array | no 112 | from | String | yes 113 | subject | String | yes 114 | body | String | yes 115 | 116 | Example: 117 | 118 | ``` sh 119 | curl \ 120 | -H "Content-Type: application/json" \ 121 | -X POST \ 122 | -d \ 123 | '{ 124 | "to": ["tim@tabdeveloper.com"], 125 | "from": "tim@bocoup.com", 126 | "subject": "A test subject!", 127 | "body": "Test message" 128 | }' \ 129 | http://localhost:8000/send 130 | ``` 131 | 132 | Response: 133 | 134 | ``` json 135 | { 136 | "data": { 137 | "ResponseMetadata": { 138 | "RequestId":"2263f1cc-fccf-21e4-a378-4350dc3a1cea" 139 | }, 140 | 141 | "MessageId": "1120014e6373fd1e-2bdf239f-21bf-4c1f-b299-36eacb54dbc6-000000" 142 | } 143 | } 144 | ``` 145 | 146 | #### Pollers 147 | 148 | These are jobs that are executed on a specific schedule. They are assigned using 149 | Crontab, and allow any valid schedule expression within that format. You would 150 | use this endpoint to schedule emails to be sent every day/month/year, schedule 151 | an email to be sent 5 minutes from now, etc. 152 | 153 | 154 | Description | Method | Endpoint 155 | :------------------------------------------------------------- | :------- | :-------- 156 | [Create a new poller](#create-a-new-poller) | `POST` | `/poll` 157 | [Create or update new poller](#create-or-update-new-poller) | `PUT` | `/poll` 158 | [Get all pollers](#get-all-pollers) | `GET` | `/poll` 159 | [Get specific poller](#get-specific-poller) | `GET` | `/poll/:id` 160 | [Delete specific poller](#delete-specific-poller) | `DELETE` | `/poll/:id` 161 | 162 | ##### Create a new poller: 163 | 164 | Request: 165 | 166 | Param | Type | Required 167 | :------- | :------------- | :-------- 168 | id | String | no 169 | template | Object | yes 170 | data | Object | yes 171 | schedule | String/Number | yes 172 | handler | String | no 173 | 174 | Example: 175 | 176 | ``` sh 177 | curl \ 178 | -H "Content-Type: application/json" \ 179 | -X POST \ 180 | -d \ 181 | '{ 182 | "id": "test-uuid", 183 | "template": { 184 | "test-uuid", 185 | } 186 | }' \ 187 | http://localhost:8000/poll 188 | ``` 189 | 190 | Response: 191 | 192 | ``` json 193 | { 194 | "data": {} 195 | } 196 | ``` 197 | 198 | ##### Create or update new poller: 199 | 200 | ``` sh 201 | curl \ 202 | -H "Content-Type: application/json" \ 203 | -X PUT \ 204 | -d \ 205 | '{ 206 | "uuid": "test-uuid", 207 | }' \ 208 | http://localhost:8000/poll 209 | ``` 210 | 211 | Response: 212 | 213 | ``` json 214 | { 215 | "data": {} 216 | } 217 | ``` 218 | 219 | ##### Get all pollers: 220 | 221 | Request: 222 | 223 | Param | Type | Required 224 | :------- | :------------- | :-------- 225 | No parameters 226 | 227 | Example: 228 | 229 | ``` sh 230 | curl \ 231 | -X GET \ 232 | http://localhost:8000/poll 233 | ``` 234 | 235 | Response: 236 | 237 | ``` json 238 | { 239 | "data": [] 240 | } 241 | ``` 242 | 243 | ##### Get specific poller: 244 | 245 | Request: 246 | 247 | Param | Type | Required 248 | :------- | :------------- | :-------- 249 | id | Number | yes 250 | 251 | Example: 252 | 253 | ``` sh 254 | curl \ 255 | -X GET \ 256 | http://localhost:8000/poll/ 257 | ``` 258 | 259 | Response: 260 | 261 | ``` json 262 | { 263 | "data": {} 264 | } 265 | ``` 266 | 267 | ##### Delete specific poller: 268 | 269 | Request: 270 | 271 | Param | Type | Required 272 | :------- | :------------- | :-------- 273 | id | Number | yes 274 | 275 | Example: 276 | 277 | ``` sh 278 | curl \ 279 | -X DELETE \ 280 | http://localhost:8000/poll/ 281 | ``` 282 | 283 | Response: 284 | 285 | ``` json 286 | { 287 | "data": {} 288 | } 289 | ``` 290 | 291 | ## Running / Development 292 | 293 | ``` sh 294 | npm start 295 | ``` 296 | 297 | Visit the server at [http://localhost:8000](http://localhost:8000). 298 | 299 | ### Running Tests 300 | 301 | ``` sh 302 | npm test 303 | ``` 304 | --------------------------------------------------------------------------------